@delali/narsil 0.3.1-next.70 → 0.3.1-next.71
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-VNMTB3KC.mjs → chunk-IOR2K26O.mjs} +48 -22
- package/dist/distribution.d.ts +1 -1
- package/dist/distribution.mjs +1 -1
- package/dist/{engine-CXKfmK7I.d.ts → engine-igk0aMpQ.d.ts} +102 -60
- package/dist/index.browser.mjs +48 -22
- package/dist/index.d.ts +2 -2
- package/dist/index.mjs +1 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +1 -1
- package/dist/react.d.ts +271 -176
- package/dist/react.mjs +189 -45
- package/dist/server.d.ts +1 -1
- package/package.json +7 -7
|
@@ -8507,6 +8507,30 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
8507
8507
|
};
|
|
8508
8508
|
}
|
|
8509
8509
|
|
|
8510
|
+
// src/engine/mutations/write-visibility.ts
|
|
8511
|
+
async function awaitWriteVisibility(ctx, indexName) {
|
|
8512
|
+
await ctx.awaitRebalanceReplay(indexName);
|
|
8513
|
+
await ctx.orchestrator.awaitWrites(indexName);
|
|
8514
|
+
}
|
|
8515
|
+
|
|
8516
|
+
// src/engine/write-events.ts
|
|
8517
|
+
var WRITE_EVENT = "write";
|
|
8518
|
+
function emitWriteEvent(source, indexName) {
|
|
8519
|
+
const handlers = source.eventHandlers.get(WRITE_EVENT);
|
|
8520
|
+
if (handlers === void 0 || handlers.size === 0) return;
|
|
8521
|
+
const emit = () => {
|
|
8522
|
+
emitEngineEvent(source.eventHandlers, WRITE_EVENT, { indexName });
|
|
8523
|
+
};
|
|
8524
|
+
awaitWriteVisibility(source.mutationCtx, indexName).then(emit, emit);
|
|
8525
|
+
}
|
|
8526
|
+
async function emitWriteEventAfter(source, indexName, write) {
|
|
8527
|
+
try {
|
|
8528
|
+
return await write;
|
|
8529
|
+
} finally {
|
|
8530
|
+
emitWriteEvent(source, indexName);
|
|
8531
|
+
}
|
|
8532
|
+
}
|
|
8533
|
+
|
|
8510
8534
|
// src/engine/core.ts
|
|
8511
8535
|
function createEngineCore(config, hooks, capacityScope = "index") {
|
|
8512
8536
|
validateWorkerConfig(config?.workers, config?.lifecycle);
|
|
@@ -8646,7 +8670,11 @@ function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
8646
8670
|
}
|
|
8647
8671
|
return names;
|
|
8648
8672
|
},
|
|
8649
|
-
reloadIndex: (indexName) =>
|
|
8673
|
+
reloadIndex: (indexName) => emitWriteEventAfter(
|
|
8674
|
+
{ eventHandlers, mutationCtx },
|
|
8675
|
+
indexName,
|
|
8676
|
+
durability?.manager.reloadIndex?.(indexName) ?? Promise.resolve()
|
|
8677
|
+
),
|
|
8650
8678
|
onError(error) {
|
|
8651
8679
|
if (emitEngineEvent(eventHandlers, "invalidationError", { error }) === 0) {
|
|
8652
8680
|
console.warn("Invalidation error:", error);
|
|
@@ -9011,12 +9039,6 @@ async function rollbackUpdatedDocument(ctx, indexName, docId, restoreDoc, origin
|
|
|
9011
9039
|
}
|
|
9012
9040
|
}
|
|
9013
9041
|
|
|
9014
|
-
// src/engine/mutations/write-visibility.ts
|
|
9015
|
-
async function awaitWriteVisibility(ctx, indexName) {
|
|
9016
|
-
await ctx.awaitRebalanceReplay(indexName);
|
|
9017
|
-
await ctx.orchestrator.awaitWrites(indexName);
|
|
9018
|
-
}
|
|
9019
|
-
|
|
9020
9042
|
// src/engine/mutations/insert.ts
|
|
9021
9043
|
async function insertDocument(ctx, indexName, document, docId, options) {
|
|
9022
9044
|
ctx.guardShutdown();
|
|
@@ -10885,15 +10907,18 @@ function createNarsilFromCore(core, config) {
|
|
|
10885
10907
|
release();
|
|
10886
10908
|
}
|
|
10887
10909
|
}
|
|
10910
|
+
function withOpenIndexWrite(indexName, action) {
|
|
10911
|
+
return emitWriteEventAfter(core, indexName, withOpenIndex(indexName, action));
|
|
10912
|
+
}
|
|
10888
10913
|
const narsil = {
|
|
10889
10914
|
createIndex(name, indexConfig) {
|
|
10890
|
-
return createEngineIndex(core, config, name, indexConfig);
|
|
10915
|
+
return emitWriteEventAfter(core, name, createEngineIndex(core, config, name, indexConfig));
|
|
10891
10916
|
},
|
|
10892
10917
|
registerEmbeddingAdapter(name, adapter) {
|
|
10893
10918
|
registerEngineEmbeddingAdapter(core, name, adapter);
|
|
10894
10919
|
},
|
|
10895
10920
|
dropIndex(name) {
|
|
10896
|
-
return dropEngineIndex(core, name);
|
|
10921
|
+
return emitWriteEventAfter(core, name, dropEngineIndex(core, name));
|
|
10897
10922
|
},
|
|
10898
10923
|
open(indexName) {
|
|
10899
10924
|
guardShutdown();
|
|
@@ -10939,22 +10964,22 @@ function createNarsilFromCore(core, config) {
|
|
|
10939
10964
|
return executor.getManager(indexName)?.getPartitionStats() ?? [];
|
|
10940
10965
|
},
|
|
10941
10966
|
insert(indexName, document, docId, options) {
|
|
10942
|
-
return
|
|
10967
|
+
return withOpenIndexWrite(indexName, () => insertDocument(mutationCtx, indexName, document, docId, options));
|
|
10943
10968
|
},
|
|
10944
10969
|
insertBatch(indexName, documents, options) {
|
|
10945
|
-
return
|
|
10970
|
+
return withOpenIndexWrite(indexName, () => insertDocumentBatch(mutationCtx, indexName, documents, options));
|
|
10946
10971
|
},
|
|
10947
10972
|
remove(indexName, docId, options) {
|
|
10948
|
-
return
|
|
10973
|
+
return withOpenIndexWrite(indexName, () => removeDocument(mutationCtx, indexName, docId, options));
|
|
10949
10974
|
},
|
|
10950
10975
|
removeBatch(indexName, docIds, options) {
|
|
10951
|
-
return
|
|
10976
|
+
return withOpenIndexWrite(indexName, () => removeDocumentBatch(mutationCtx, indexName, docIds, options));
|
|
10952
10977
|
},
|
|
10953
10978
|
update(indexName, docId, document, options) {
|
|
10954
|
-
return
|
|
10979
|
+
return withOpenIndexWrite(indexName, () => updateDocument(mutationCtx, indexName, docId, document, options));
|
|
10955
10980
|
},
|
|
10956
10981
|
updateBatch(indexName, updates, options) {
|
|
10957
|
-
return
|
|
10982
|
+
return withOpenIndexWrite(indexName, () => updateDocumentBatch(mutationCtx, indexName, updates, options));
|
|
10958
10983
|
},
|
|
10959
10984
|
waitForWrites(indexName) {
|
|
10960
10985
|
guardShutdown();
|
|
@@ -11002,7 +11027,7 @@ function createNarsilFromCore(core, config) {
|
|
|
11002
11027
|
async rebuildAnalysis(indexName) {
|
|
11003
11028
|
guardShutdown();
|
|
11004
11029
|
requireIndex(indexName);
|
|
11005
|
-
await
|
|
11030
|
+
await withOpenIndexWrite(indexName, () => core.analysisRebuild.rebuild(indexName));
|
|
11006
11031
|
},
|
|
11007
11032
|
async snapshot(indexName) {
|
|
11008
11033
|
guardShutdown();
|
|
@@ -11010,11 +11035,11 @@ function createNarsilFromCore(core, config) {
|
|
|
11010
11035
|
},
|
|
11011
11036
|
async restore(indexName, data) {
|
|
11012
11037
|
guardShutdown();
|
|
11013
|
-
|
|
11038
|
+
const restored = restoreFromSnapshot(indexName, data, {
|
|
11014
11039
|
executor,
|
|
11015
11040
|
indexRegistry,
|
|
11016
11041
|
getVectorFieldPaths,
|
|
11017
|
-
dropIndex:
|
|
11042
|
+
dropIndex: (name) => dropEngineIndex(core, name),
|
|
11018
11043
|
announceIndexCreated: (name, restoredConfig) => announceIndexCreated(core, name, restoredConfig),
|
|
11019
11044
|
requireManager,
|
|
11020
11045
|
durability,
|
|
@@ -11025,6 +11050,7 @@ function createNarsilFromCore(core, config) {
|
|
|
11025
11050
|
clearAnalysisStale: core.analysisRebuild.clearStale,
|
|
11026
11051
|
indexState: core.indexState
|
|
11027
11052
|
});
|
|
11053
|
+
await emitWriteEventAfter(core, indexName, restored);
|
|
11028
11054
|
core.heapPressureNotifier.check(indexName);
|
|
11029
11055
|
},
|
|
11030
11056
|
async checkpoint(indexName) {
|
|
@@ -11037,7 +11063,7 @@ function createNarsilFromCore(core, config) {
|
|
|
11037
11063
|
async clear(indexName) {
|
|
11038
11064
|
guardShutdown();
|
|
11039
11065
|
requireIndex(indexName);
|
|
11040
|
-
await
|
|
11066
|
+
await withOpenIndexWrite(indexName, async () => {
|
|
11041
11067
|
await executor.execute({ type: "clear", indexName, requestId: indexName });
|
|
11042
11068
|
await orchestrator.replicateToWorkers({ type: "clear", indexName, requestId: `replicate-clear-${indexName}` });
|
|
11043
11069
|
core.watermarkNotifier.forget(indexName);
|
|
@@ -11046,7 +11072,7 @@ function createNarsilFromCore(core, config) {
|
|
|
11046
11072
|
async rebalance(indexName, targetPartitionCount) {
|
|
11047
11073
|
guardShutdown();
|
|
11048
11074
|
requireIndex(indexName);
|
|
11049
|
-
return
|
|
11075
|
+
return withOpenIndexWrite(
|
|
11050
11076
|
indexName,
|
|
11051
11077
|
() => executeRebalance(requireManager(indexName), indexName, targetPartitionCount, rebalanceCtx)
|
|
11052
11078
|
);
|
|
@@ -11112,14 +11138,14 @@ function createNarsilFromCore(core, config) {
|
|
|
11112
11138
|
async compactVectors(indexName, fieldName) {
|
|
11113
11139
|
guardShutdown();
|
|
11114
11140
|
requireIndex(indexName);
|
|
11115
|
-
await
|
|
11141
|
+
await withOpenIndexWrite(indexName, async () => {
|
|
11116
11142
|
compactVectors(requireManager(indexName), indexName, fieldName);
|
|
11117
11143
|
});
|
|
11118
11144
|
},
|
|
11119
11145
|
async optimizeVectors(indexName, fieldName) {
|
|
11120
11146
|
guardShutdown();
|
|
11121
11147
|
requireIndex(indexName);
|
|
11122
|
-
await
|
|
11148
|
+
await withOpenIndexWrite(indexName, () => optimizeVectors(requireManager(indexName), indexName, fieldName));
|
|
11123
11149
|
},
|
|
11124
11150
|
vectorMaintenanceStatus(indexName) {
|
|
11125
11151
|
guardShutdown();
|
package/dist/distribution.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { a as CreateIndexOptions, C as ClusterNamespace, b as ClusterNodeConfig } from './config-rBpbgTw8.js';
|
|
2
2
|
export { c as ClusterControllerConfig, d as ClusterNodeInfo, e as ClusterQueryConfig, N as NodeReadiness, R as ReplicationConfig } from './config-rBpbgTw8.js';
|
|
3
|
-
import { N as NarsilEventMap, a as Narsil } from './engine-
|
|
3
|
+
import { N as NarsilEventMap, a as Narsil } from './engine-igk0aMpQ.js';
|
|
4
4
|
import { B as BatchResult, L as ListParams, g as ListResult, j as SuggestParams, k as SuggestResult, Q as QueryParams, h as PreflightResult, f as IndexStats, P as PartitionStatsResult, M as MemoryStats, e as QueryResult } from './memory-C7UMCMH3.js';
|
|
5
5
|
import { I as IndexConfig, A as AnyDocument, a as InsertOptions } from './schema-Bgzr4g7b.js';
|
|
6
6
|
import { b as NodeCapacity, N as NodeRole } from './types-gOUqhs2F.js';
|
package/dist/distribution.mjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { MAX_MESSAGE_SIZE_BYTES } from './chunk-UNEEFTEU.mjs';
|
|
2
2
|
import { validateIndexName, indexConfigKey, indexSettingsOf, putIndexMetadata, withPartitionCount, getIndexMetadata, getClusterIndexConfig } from './chunk-3JGWULDF.mjs';
|
|
3
|
-
import { requireKnownConfig, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, applyIndexEntry, applyDeleteEntry, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, validateReplicationEntry, computeEntryChecksum, createEntryMessage, createEntryBatchMessage, createInsyncConfirmMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, createInsyncAddMessage } from './chunk-
|
|
3
|
+
import { requireKnownConfig, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, applyIndexEntry, applyDeleteEntry, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, validateReplicationEntry, computeEntryChecksum, createEntryMessage, createEntryBatchMessage, createInsyncConfirmMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, createInsyncAddMessage } from './chunk-IOR2K26O.mjs';
|
|
4
4
|
import { ReplicationMessageTypes, TransportErrorCodes, ClusterMessageTypes, QueryMessageTypes, TransportError } from './chunk-GCM3JFVA.mjs';
|
|
5
5
|
import { providedDocId, MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET } from './chunk-FVLQ63B7.mjs';
|
|
6
6
|
import { generateId, deserializePayloadV2, createPartitionIndex, FILTER_EXPRESSION_KEYS, FIELD_FILTER_OPERATORS } from './chunk-M76V3OGJ.mjs';
|
|
@@ -3,124 +3,166 @@ import { B as BatchResult, I as IndexInfo, f as IndexStats, P as PartitionStatsR
|
|
|
3
3
|
import { A as AnyDocument, a as InsertOptions, W as WriteOptions, I as IndexConfig, P as PartitionConfig } from './schema-Bgzr4g7b.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
|
-
*
|
|
6
|
+
* This map lists every event that the engine emits, with the payload of each
|
|
7
|
+
* one.
|
|
7
8
|
*
|
|
8
|
-
* {@link Narsil.on}
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
9
|
+
* {@link Narsil.on} registers a listener under a key from this map, so that the
|
|
10
|
+
* engine can pass the matching payload to that listener. Most of these events
|
|
11
|
+
* report work that the engine does in the background, away from the call that
|
|
12
|
+
* started it, so a failure that such an event reports never rejects a promise
|
|
13
|
+
* that you are holding.
|
|
12
14
|
*
|
|
13
15
|
* @public
|
|
14
16
|
*/
|
|
15
17
|
type NarsilEventMap = {
|
|
16
18
|
/**
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
19
|
+
* The engine emits this event after a call changes what a read returns from
|
|
20
|
+
* an index. The calls are the inserts, updates, and removals, singly or in a
|
|
21
|
+
* batch, together with `clear`, `restore`, `rebalance`, `rebuildAnalysis`,
|
|
22
|
+
* `compactVectors`, `optimizeVectors`, `createIndex`, and `dropIndex`. The
|
|
23
|
+
* engine emits the event after a call that throws as well, because a call can
|
|
24
|
+
* change the index before it throws. It also emits the event after it reloads
|
|
25
|
+
* an index, once a shared invalidation adapter reports that another instance
|
|
26
|
+
* saved that index. The engine emits one event per batch, however many
|
|
27
|
+
* documents the batch holds.
|
|
28
|
+
*
|
|
29
|
+
* The engine emits the event once a query can see the change, which means
|
|
30
|
+
* after every worker copy applies the change and after any rebalance in
|
|
31
|
+
* progress replays it. A query that a listener sends from its handler
|
|
32
|
+
* therefore returns results that include the change. The engine waits for the
|
|
33
|
+
* change to become visible only while a listener is registered for the event.
|
|
34
|
+
*/
|
|
35
|
+
write: {
|
|
36
|
+
/** This is the name of the index that changed. */
|
|
37
|
+
indexName: string;
|
|
38
|
+
};
|
|
39
|
+
/**
|
|
40
|
+
* A worker thread exited unexpectedly. The pool removes the thread and fails
|
|
41
|
+
* the requests that the thread was serving with `WORKER_CRASHED`. The engine
|
|
42
|
+
* then sends each of those queries again on the main thread, so the caller
|
|
43
|
+
* receives a result with no error. The remaining workers keep serving
|
|
44
|
+
* queries, because each of them holds a full worker copy of every promoted
|
|
45
|
+
* index.
|
|
46
|
+
*
|
|
47
|
+
* After a delay, the engine spawns a replacement and loads every copy onto
|
|
48
|
+
* it before it sends queries to the replacement. Once no worker is left, the
|
|
49
|
+
* main thread serves every query, because it holds every document. The first
|
|
50
|
+
* request after the delay then starts a new pool. The delay starts at one
|
|
51
|
+
* second and doubles up to a minute while replacements keep failing.
|
|
27
52
|
*/
|
|
28
53
|
workerCrash: {
|
|
29
|
-
/** This worker
|
|
54
|
+
/** This is the id of the worker that exited. */
|
|
30
55
|
workerId: number;
|
|
31
|
-
/**
|
|
56
|
+
/** These are the indexes that the worker held. */
|
|
32
57
|
indexNames: string[];
|
|
33
|
-
/** This ended the worker. */
|
|
58
|
+
/** This is the error that ended the worker. */
|
|
34
59
|
error: Error;
|
|
35
60
|
};
|
|
36
|
-
/** An index gained worker copies,
|
|
61
|
+
/** An index gained worker copies, either because it reached the copy
|
|
62
|
+
* threshold or because a request loaded its copies again after an idle
|
|
63
|
+
* spell. */
|
|
37
64
|
workerPromote: {
|
|
38
|
-
/**
|
|
65
|
+
/** This is the number of workers in the pool. */
|
|
39
66
|
workerCount: number;
|
|
40
|
-
/**
|
|
67
|
+
/** This is the reason that the index gained its copies, such as the copy
|
|
68
|
+
* threshold that it reached. */
|
|
41
69
|
reason: string;
|
|
42
70
|
};
|
|
43
|
-
/** An index could not gain worker copies
|
|
71
|
+
/** An index could not gain worker copies, so the main thread keeps serving
|
|
72
|
+
* its queries. */
|
|
44
73
|
workerPromoteFailure: {
|
|
45
|
-
/**
|
|
74
|
+
/** This is the reason that the engine tried to load the copies. */
|
|
46
75
|
reason: string;
|
|
47
|
-
/** This stopped
|
|
76
|
+
/** This is the error that stopped the load. */
|
|
48
77
|
error: Error;
|
|
49
|
-
/** This
|
|
78
|
+
/** This is true when the engine will try again at a later threshold. */
|
|
50
79
|
retryable: boolean;
|
|
51
80
|
};
|
|
52
|
-
/** An index finished spreading its documents across a new
|
|
81
|
+
/** An index finished spreading its documents across a new number of
|
|
82
|
+
* partitions. */
|
|
53
83
|
partitionRebalance: {
|
|
54
|
-
/** This index
|
|
84
|
+
/** This is the name of the index that the engine rebalanced. */
|
|
55
85
|
indexName: string;
|
|
56
|
-
/**
|
|
86
|
+
/** This is the number of partitions before the rebalance. */
|
|
57
87
|
oldCount: number;
|
|
58
|
-
/**
|
|
88
|
+
/** This is the number of partitions after the rebalance. */
|
|
59
89
|
newCount: number;
|
|
60
90
|
};
|
|
61
91
|
/**
|
|
62
92
|
* An index passed its watermark, so its partitions are close to their
|
|
63
93
|
* capacity. On a cluster node, the engine reports the document count and the
|
|
64
|
-
* capacity of the one partition that `partitionId`
|
|
65
|
-
*
|
|
94
|
+
* capacity of the one partition that `partitionId` identifies, which this
|
|
95
|
+
* node leads.
|
|
66
96
|
*/
|
|
67
97
|
partitionWatermark: {
|
|
68
|
-
/** This index crossed the mark. */
|
|
98
|
+
/** This is the name of the index that crossed the mark. */
|
|
69
99
|
indexName: string;
|
|
70
|
-
/**
|
|
100
|
+
/** This is the number of documents in the index, or in the partition on a
|
|
101
|
+
* cluster node. */
|
|
71
102
|
documentCount: number;
|
|
72
|
-
/**
|
|
103
|
+
/** This is the number of documents that the current partitions of the
|
|
104
|
+
* index can hold, or that the partition can hold on a cluster node. */
|
|
73
105
|
capacity: number;
|
|
74
|
-
/**
|
|
106
|
+
/** This is the number of partitions in the index. */
|
|
75
107
|
partitionCount: number;
|
|
76
|
-
/** On a cluster node, this partition crossed the mark
|
|
108
|
+
/** On a cluster node, this is the partition that crossed the mark, while
|
|
109
|
+
* outside a cluster the field is absent. */
|
|
77
110
|
partitionId?: number;
|
|
78
111
|
};
|
|
79
112
|
/**
|
|
80
|
-
* The process uses nine tenths of its heap, measured during a write or a
|
|
81
|
-
* load, so the next large index
|
|
82
|
-
* error.
|
|
83
|
-
*
|
|
84
|
-
*
|
|
85
|
-
*
|
|
86
|
-
* that
|
|
87
|
-
*
|
|
88
|
-
*
|
|
89
|
-
*
|
|
113
|
+
* The process uses nine tenths of its heap, as measured during a write or a
|
|
114
|
+
* load, so the next large index can end the process with an out-of-memory
|
|
115
|
+
* error.
|
|
116
|
+
*
|
|
117
|
+
* Where `--max-old-space-size` or `--max-old-space-size-percentage` sets a
|
|
118
|
+
* limit, on the command line or in `NODE_OPTIONS`, the engine measures the
|
|
119
|
+
* used bytes against that limit, which `heapLimit` then reports. Where
|
|
120
|
+
* neither flag sets a limit, the engine measures the share of the heap that
|
|
121
|
+
* V8 reports as no longer available. V8 reports a limit about 192 MB above
|
|
122
|
+
* the true ceiling, so a process on a default heap below about 2 GB can end before
|
|
123
|
+
* this event fires.
|
|
124
|
+
*
|
|
125
|
+
* The engine emits the event once per crossing, then arms it again once the
|
|
126
|
+
* free share of the heap recovers to two tenths. Raise the limit with
|
|
90
127
|
* `--max-old-space-size-percentage` or `--max-old-space-size`, or close an
|
|
91
128
|
* idle index; see {@link ProcessMemoryReport.heapLimit}.
|
|
92
129
|
*/
|
|
93
130
|
heapPressure: {
|
|
94
|
-
/**
|
|
131
|
+
/** This is the index that a write or a load was touching when the heap
|
|
132
|
+
* reached that point. */
|
|
95
133
|
indexName: string;
|
|
96
|
-
/**
|
|
134
|
+
/** This is the heap that the process uses, in bytes. */
|
|
97
135
|
heapUsed: number;
|
|
98
|
-
/**
|
|
136
|
+
/** This is the size in bytes that the heap can grow to. */
|
|
99
137
|
heapLimit: number;
|
|
100
|
-
/**
|
|
138
|
+
/** This is the estimate of the bytes that the index holds. */
|
|
101
139
|
estimatedMemoryBytes: number;
|
|
102
140
|
};
|
|
103
|
-
/**
|
|
141
|
+
/** The engine started, completed, or failed a rebuild that brings the terms
|
|
142
|
+
* of an index up to the current analysis of its language module. */
|
|
104
143
|
analysisRebuild: {
|
|
105
|
-
/** This index
|
|
144
|
+
/** This is the name of the index under rebuild. */
|
|
106
145
|
indexName: string;
|
|
107
|
-
/**
|
|
146
|
+
/** This is the stage that the rebuild has reached. */
|
|
108
147
|
status: 'started' | 'completed' | 'failed';
|
|
109
|
-
/**
|
|
148
|
+
/** This is the number of partitions that the rebuild has covered so far. */
|
|
110
149
|
partitionsRebuilt: number;
|
|
111
|
-
/**
|
|
150
|
+
/** This is the number of partitions that the rebuild covers in total. */
|
|
112
151
|
partitionCount: number;
|
|
113
|
-
/** This stopped the rebuild,
|
|
152
|
+
/** This is the error that stopped the rebuild, which only a `failed`
|
|
153
|
+
* status includes. */
|
|
114
154
|
error?: Error;
|
|
115
155
|
};
|
|
116
|
-
/**
|
|
156
|
+
/** A write to the write-ahead log or a checkpoint failed, so the writes since
|
|
157
|
+
* the last checkpoint are at risk. */
|
|
117
158
|
durabilityError: {
|
|
118
|
-
/**
|
|
159
|
+
/** This is the error that the durability layer threw. */
|
|
119
160
|
error: Error;
|
|
120
161
|
};
|
|
121
|
-
/** The invalidation channel failed, so this instance
|
|
162
|
+
/** The invalidation channel failed, so this instance can serve partitions
|
|
163
|
+
* that another instance has since changed. */
|
|
122
164
|
invalidationError: {
|
|
123
|
-
/**
|
|
165
|
+
/** This is the error that the adapter threw. */
|
|
124
166
|
error: Error;
|
|
125
167
|
};
|
|
126
168
|
};
|
package/dist/index.browser.mjs
CHANGED
|
@@ -28040,6 +28040,30 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
28040
28040
|
};
|
|
28041
28041
|
}
|
|
28042
28042
|
|
|
28043
|
+
// src/engine/mutations/write-visibility.ts
|
|
28044
|
+
async function awaitWriteVisibility(ctx, indexName) {
|
|
28045
|
+
await ctx.awaitRebalanceReplay(indexName);
|
|
28046
|
+
await ctx.orchestrator.awaitWrites(indexName);
|
|
28047
|
+
}
|
|
28048
|
+
|
|
28049
|
+
// src/engine/write-events.ts
|
|
28050
|
+
var WRITE_EVENT = "write";
|
|
28051
|
+
function emitWriteEvent(source, indexName) {
|
|
28052
|
+
const handlers = source.eventHandlers.get(WRITE_EVENT);
|
|
28053
|
+
if (handlers === void 0 || handlers.size === 0) return;
|
|
28054
|
+
const emit = () => {
|
|
28055
|
+
emitEngineEvent(source.eventHandlers, WRITE_EVENT, { indexName });
|
|
28056
|
+
};
|
|
28057
|
+
awaitWriteVisibility(source.mutationCtx, indexName).then(emit, emit);
|
|
28058
|
+
}
|
|
28059
|
+
async function emitWriteEventAfter(source, indexName, write) {
|
|
28060
|
+
try {
|
|
28061
|
+
return await write;
|
|
28062
|
+
} finally {
|
|
28063
|
+
emitWriteEvent(source, indexName);
|
|
28064
|
+
}
|
|
28065
|
+
}
|
|
28066
|
+
|
|
28043
28067
|
// src/engine/core.ts
|
|
28044
28068
|
function createEngineCore(config, hooks, capacityScope = "index") {
|
|
28045
28069
|
validateWorkerConfig(config?.workers, config?.lifecycle);
|
|
@@ -28179,7 +28203,11 @@ function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
28179
28203
|
}
|
|
28180
28204
|
return names;
|
|
28181
28205
|
},
|
|
28182
|
-
reloadIndex: (indexName) =>
|
|
28206
|
+
reloadIndex: (indexName) => emitWriteEventAfter(
|
|
28207
|
+
{ eventHandlers, mutationCtx },
|
|
28208
|
+
indexName,
|
|
28209
|
+
durability?.manager.reloadIndex?.(indexName) ?? Promise.resolve()
|
|
28210
|
+
),
|
|
28183
28211
|
onError(error) {
|
|
28184
28212
|
if (emitEngineEvent(eventHandlers, "invalidationError", { error }) === 0) {
|
|
28185
28213
|
console.warn("Invalidation error:", error);
|
|
@@ -28872,12 +28900,6 @@ async function embedChunkDocuments(entry, documents, chunkStart, chunkEnd, signa
|
|
|
28872
28900
|
}
|
|
28873
28901
|
}
|
|
28874
28902
|
|
|
28875
|
-
// src/engine/mutations/write-visibility.ts
|
|
28876
|
-
async function awaitWriteVisibility(ctx, indexName) {
|
|
28877
|
-
await ctx.awaitRebalanceReplay(indexName);
|
|
28878
|
-
await ctx.orchestrator.awaitWrites(indexName);
|
|
28879
|
-
}
|
|
28880
|
-
|
|
28881
28903
|
// src/engine/mutations/insert.ts
|
|
28882
28904
|
async function insertDocument(ctx, indexName, document2, docId, options) {
|
|
28883
28905
|
ctx.guardShutdown();
|
|
@@ -32316,15 +32338,18 @@ function createNarsilFromCore(core, config) {
|
|
|
32316
32338
|
release();
|
|
32317
32339
|
}
|
|
32318
32340
|
}
|
|
32341
|
+
function withOpenIndexWrite(indexName, action) {
|
|
32342
|
+
return emitWriteEventAfter(core, indexName, withOpenIndex(indexName, action));
|
|
32343
|
+
}
|
|
32319
32344
|
const narsil = {
|
|
32320
32345
|
createIndex(name, indexConfig) {
|
|
32321
|
-
return createEngineIndex(core, config, name, indexConfig);
|
|
32346
|
+
return emitWriteEventAfter(core, name, createEngineIndex(core, config, name, indexConfig));
|
|
32322
32347
|
},
|
|
32323
32348
|
registerEmbeddingAdapter(name, adapter) {
|
|
32324
32349
|
registerEngineEmbeddingAdapter(core, name, adapter);
|
|
32325
32350
|
},
|
|
32326
32351
|
dropIndex(name) {
|
|
32327
|
-
return dropEngineIndex(core, name);
|
|
32352
|
+
return emitWriteEventAfter(core, name, dropEngineIndex(core, name));
|
|
32328
32353
|
},
|
|
32329
32354
|
open(indexName) {
|
|
32330
32355
|
guardShutdown();
|
|
@@ -32370,22 +32395,22 @@ function createNarsilFromCore(core, config) {
|
|
|
32370
32395
|
return executor.getManager(indexName)?.getPartitionStats() ?? [];
|
|
32371
32396
|
},
|
|
32372
32397
|
insert(indexName, document2, docId, options) {
|
|
32373
|
-
return
|
|
32398
|
+
return withOpenIndexWrite(indexName, () => insertDocument(mutationCtx, indexName, document2, docId, options));
|
|
32374
32399
|
},
|
|
32375
32400
|
insertBatch(indexName, documents, options) {
|
|
32376
|
-
return
|
|
32401
|
+
return withOpenIndexWrite(indexName, () => insertDocumentBatch(mutationCtx, indexName, documents, options));
|
|
32377
32402
|
},
|
|
32378
32403
|
remove(indexName, docId, options) {
|
|
32379
|
-
return
|
|
32404
|
+
return withOpenIndexWrite(indexName, () => removeDocument(mutationCtx, indexName, docId, options));
|
|
32380
32405
|
},
|
|
32381
32406
|
removeBatch(indexName, docIds, options) {
|
|
32382
|
-
return
|
|
32407
|
+
return withOpenIndexWrite(indexName, () => removeDocumentBatch(mutationCtx, indexName, docIds, options));
|
|
32383
32408
|
},
|
|
32384
32409
|
update(indexName, docId, document2, options) {
|
|
32385
|
-
return
|
|
32410
|
+
return withOpenIndexWrite(indexName, () => updateDocument(mutationCtx, indexName, docId, document2, options));
|
|
32386
32411
|
},
|
|
32387
32412
|
updateBatch(indexName, updates, options) {
|
|
32388
|
-
return
|
|
32413
|
+
return withOpenIndexWrite(indexName, () => updateDocumentBatch(mutationCtx, indexName, updates, options));
|
|
32389
32414
|
},
|
|
32390
32415
|
waitForWrites(indexName) {
|
|
32391
32416
|
guardShutdown();
|
|
@@ -32433,7 +32458,7 @@ function createNarsilFromCore(core, config) {
|
|
|
32433
32458
|
async rebuildAnalysis(indexName) {
|
|
32434
32459
|
guardShutdown();
|
|
32435
32460
|
requireIndex(indexName);
|
|
32436
|
-
await
|
|
32461
|
+
await withOpenIndexWrite(indexName, () => core.analysisRebuild.rebuild(indexName));
|
|
32437
32462
|
},
|
|
32438
32463
|
async snapshot(indexName) {
|
|
32439
32464
|
guardShutdown();
|
|
@@ -32441,11 +32466,11 @@ function createNarsilFromCore(core, config) {
|
|
|
32441
32466
|
},
|
|
32442
32467
|
async restore(indexName, data) {
|
|
32443
32468
|
guardShutdown();
|
|
32444
|
-
|
|
32469
|
+
const restored = restoreFromSnapshot(indexName, data, {
|
|
32445
32470
|
executor,
|
|
32446
32471
|
indexRegistry,
|
|
32447
32472
|
getVectorFieldPaths,
|
|
32448
|
-
dropIndex:
|
|
32473
|
+
dropIndex: (name) => dropEngineIndex(core, name),
|
|
32449
32474
|
announceIndexCreated: (name, restoredConfig) => announceIndexCreated(core, name, restoredConfig),
|
|
32450
32475
|
requireManager,
|
|
32451
32476
|
durability,
|
|
@@ -32456,6 +32481,7 @@ function createNarsilFromCore(core, config) {
|
|
|
32456
32481
|
clearAnalysisStale: core.analysisRebuild.clearStale,
|
|
32457
32482
|
indexState: core.indexState
|
|
32458
32483
|
});
|
|
32484
|
+
await emitWriteEventAfter(core, indexName, restored);
|
|
32459
32485
|
core.heapPressureNotifier.check(indexName);
|
|
32460
32486
|
},
|
|
32461
32487
|
async checkpoint(indexName) {
|
|
@@ -32468,7 +32494,7 @@ function createNarsilFromCore(core, config) {
|
|
|
32468
32494
|
async clear(indexName) {
|
|
32469
32495
|
guardShutdown();
|
|
32470
32496
|
requireIndex(indexName);
|
|
32471
|
-
await
|
|
32497
|
+
await withOpenIndexWrite(indexName, async () => {
|
|
32472
32498
|
await executor.execute({ type: "clear", indexName, requestId: indexName });
|
|
32473
32499
|
await orchestrator.replicateToWorkers({ type: "clear", indexName, requestId: `replicate-clear-${indexName}` });
|
|
32474
32500
|
core.watermarkNotifier.forget(indexName);
|
|
@@ -32477,7 +32503,7 @@ function createNarsilFromCore(core, config) {
|
|
|
32477
32503
|
async rebalance(indexName, targetPartitionCount) {
|
|
32478
32504
|
guardShutdown();
|
|
32479
32505
|
requireIndex(indexName);
|
|
32480
|
-
return
|
|
32506
|
+
return withOpenIndexWrite(
|
|
32481
32507
|
indexName,
|
|
32482
32508
|
() => executeRebalance(requireManager(indexName), indexName, targetPartitionCount, rebalanceCtx)
|
|
32483
32509
|
);
|
|
@@ -32543,14 +32569,14 @@ function createNarsilFromCore(core, config) {
|
|
|
32543
32569
|
async compactVectors(indexName, fieldName) {
|
|
32544
32570
|
guardShutdown();
|
|
32545
32571
|
requireIndex(indexName);
|
|
32546
|
-
await
|
|
32572
|
+
await withOpenIndexWrite(indexName, async () => {
|
|
32547
32573
|
compactVectors(requireManager(indexName), indexName, fieldName);
|
|
32548
32574
|
});
|
|
32549
32575
|
},
|
|
32550
32576
|
async optimizeVectors(indexName, fieldName) {
|
|
32551
32577
|
guardShutdown();
|
|
32552
32578
|
requireIndex(indexName);
|
|
32553
|
-
await
|
|
32579
|
+
await withOpenIndexWrite(indexName, () => optimizeVectors(requireManager(indexName), indexName, fieldName));
|
|
32554
32580
|
},
|
|
32555
32581
|
vectorMaintenanceStatus(indexName) {
|
|
32556
32582
|
guardShutdown();
|
package/dist/index.d.ts
CHANGED
|
@@ -5,8 +5,8 @@ export { T as TokenizerConfig } from './language-DB8KG1z5.js';
|
|
|
5
5
|
export { A as ArrayFilter, B as BatchResult, C as ClientErrorCode, a as ClientErrorCodes, l as ComparisonFilter, D as DocumentProjection, E as ErrorCode, b as ErrorCodes, F as FacetConfig, m as FacetResult, n as FieldFilter, o as FilterExpression, G as GeoFilter, p as GeoPolygonFilter, q as GeoRadiusFilter, r as GroupConfig, s as GroupReducer, t as GroupResult, H as HighlightConfig, u as HighlightMatch, v as Hit, w as HybridConfig, I as IndexInfo, f as IndexStats, L as ListParams, g as ListResult, x as ListedDocument, M as MemoryStats, N as NarsilError, c as NarsilErrorCode, P as PartitionStatsResult, h as PreflightResult, y as PresenceFilter, z as ProcessMemoryReport, J as QueryCoverage, Q as QueryParams, e as QueryResult, K as ScoreComponents, O as SearchMode, S as ServerErrorCode, d as ServerErrorCodes, R as SortField, T as SortSpec, U as StringFilter, j as SuggestParams, k as SuggestResult, W as TermMatchPolicy, V as VectorMaintenanceResult, X as VectorQueryConfig, Y as WorkerCopyReport, i as isNarsilError } from './memory-C7UMCMH3.js';
|
|
6
6
|
import { N as NarsilConfig } from './config-Dfanyqqt.js';
|
|
7
7
|
export { A as AnalysisConfig, D as DurabilityConfig, I as IndexContext, a as IndexLifecycleConfig, b as InsertContext, M as MainCopyQueries, c as NarsilPlugin, P as PartitionContext, R as RemoveContext, S as SearchContext, d as StaleAnalysis, U as UpdateContext, W as WorkerConfig, e as WorkerContext } from './config-Dfanyqqt.js';
|
|
8
|
-
import { a as Narsil } from './engine-
|
|
9
|
-
export { D as DocumentWriteOperations, I as IndexLifecycleOperations, N as NarsilEventMap } from './engine-
|
|
8
|
+
import { a as Narsil } from './engine-igk0aMpQ.js';
|
|
9
|
+
export { D as DocumentWriteOperations, I as IndexLifecycleOperations, N as NarsilEventMap } from './engine-igk0aMpQ.js';
|
|
10
10
|
export { E as EmbeddingAdapter, I as InvalidationAdapter, a as InvalidationEvent, b as PartitionStatistics, P as PersistenceAdapter } from './adapters-79OIa4xm.js';
|
|
11
11
|
export { i as isSimdAvailable } from './simd-DW5AtJ6B.js';
|
|
12
12
|
|
package/dist/index.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { VERSION, createNarsil } from './chunk-
|
|
1
|
+
export { VERSION, createNarsil } from './chunk-IOR2K26O.mjs';
|
|
2
2
|
import './chunk-GCM3JFVA.mjs';
|
|
3
3
|
import './chunk-FVLQ63B7.mjs';
|
|
4
4
|
export { getStopWords, getTokenizer, hasStopWords, hasTokenizer, registerStopWords, registerTokenizer } from './chunk-M76V3OGJ.mjs';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { CHECKPOINT_WORKER_HEARTBEAT_MS, createDurableDirectory, writeCheckpointSegments } from '../../chunk-
|
|
1
|
+
import { CHECKPOINT_WORKER_HEARTBEAT_MS, createDurableDirectory, writeCheckpointSegments } from '../../chunk-IOR2K26O.mjs';
|
|
2
2
|
import '../../chunk-GCM3JFVA.mjs';
|
|
3
3
|
import '../../chunk-FVLQ63B7.mjs';
|
|
4
4
|
import '../../chunk-M76V3OGJ.mjs';
|