@delali/narsil 0.3.1-next.130 → 0.3.1-next.140
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/analysis-change-JN23AZQP.mjs +43 -0
- package/dist/analysis-change-entry-5JSIMEKG.mjs +1 -0
- package/dist/browser/analysis-change-57MIRJWR.mjs +499 -0
- package/dist/browser/analysis-change-entry-ICGRR3IN.mjs +1 -0
- package/dist/browser/{chunk-UDGB2MWT.mjs → chunk-35B42UFM.mjs} +4 -0
- package/dist/browser/{chunk-VBBLTD7O.mjs → chunk-3EYDURFD.mjs} +39 -27
- package/dist/browser/{chunk-AIIJDWMG.mjs → chunk-4MFH226Q.mjs} +1 -1
- package/dist/browser/{chunk-D7WQF6DV.mjs → chunk-5LYWZJE2.mjs} +2 -2
- package/dist/browser/chunk-67KSBUYR.mjs +173 -0
- package/dist/browser/{chunk-M5UAAXGU.mjs → chunk-6BSB4N6B.mjs} +2 -2
- package/dist/browser/{chunk-NSCSR6MA.mjs → chunk-CGVBGHBX.mjs} +81 -6
- package/dist/browser/chunk-E5SFTOQR.mjs +21 -0
- package/dist/browser/{chunk-VEGHFP2P.mjs → chunk-MZI3TEOW.mjs} +4 -2
- package/dist/browser/chunk-OIZS5KIX.mjs +6 -0
- package/dist/browser/chunk-QHNNULPW.mjs +265 -0
- package/dist/browser/chunk-XVMOQQ34.mjs +384 -0
- package/dist/browser/{chunk-2Y3CO4EQ.mjs → chunk-ZAIZQOTM.mjs} +256 -777
- package/dist/browser/{durability-wiring-FNZSRH4N.mjs → durability-wiring-MTIAN6VT.mjs} +145 -50
- package/dist/browser/import-Q7MXD6BS.mjs +8 -0
- package/dist/browser/{module-4HIMOQM5.mjs → module-5J3PEUEB.mjs} +7 -6
- package/dist/browser/{phrase-module-WESYVU7K.mjs → phrase-module-HGVBO24W.mjs} +1 -1
- package/dist/{restore-queue-JAMNRC6T.mjs → browser/restore-queue-DISFECE4.mjs} +8 -2
- package/dist/browser/{runtime-3X7ZJ5T3.mjs → runtime-VEUH674B.mjs} +2 -2
- package/dist/browser/{snapshot-V7VF6BN6.mjs → snapshot-YBJ5MF2M.mjs} +36 -167
- package/dist/browser/{write-YIC2MKRY.mjs → write-RTWI5MXL.mjs} +8 -6
- package/dist/{capabilities-Bl8G9taj.d.ts → capabilities-DXAmysqJ.d.ts} +11 -11
- package/dist/{chunk-NMXRXFGZ.mjs → chunk-4GCN34JV.mjs} +2 -2
- package/dist/{chunk-SOHMMNCH.mjs → chunk-6VHWXAOI.mjs} +4 -4
- package/dist/{chunk-2C3WXBQS.mjs → chunk-74XB2SXI.mjs} +1 -1
- package/dist/{chunk-QQTXT65X.mjs → chunk-B5I3IL4H.mjs} +12 -7
- package/dist/chunk-BDIDUW2B.mjs +387 -0
- package/dist/{chunk-5LX6MOIB.mjs → chunk-C3YDYX6C.mjs} +1 -1
- package/dist/{chunk-Y4BVTQ53.mjs → chunk-CCCGEPJM.mjs} +2 -2
- package/dist/{chunk-2JWNJSNE.mjs → chunk-CPCGEIOM.mjs} +11 -8
- package/dist/chunk-CZQ33DLW.mjs +78 -0
- package/dist/{chunk-4GDLFDPZ.mjs → chunk-DD2673TX.mjs} +3 -2
- package/dist/{chunk-RNSC4NFI.mjs → chunk-EEBCDP36.mjs} +3 -1
- package/dist/{chunk-HCGJKDZV.mjs → chunk-FFVLPCLH.mjs} +1 -1
- package/dist/{chunk-WVRDRLIC.mjs → chunk-FKARMUWH.mjs} +6 -6
- package/dist/{chunk-PATMDIMN.mjs → chunk-HJJWY6PT.mjs} +1 -1
- package/dist/chunk-HMDT3I63.mjs +21 -0
- package/dist/{chunk-CZ7ISLXA.mjs → chunk-HNHCNTVG.mjs} +4 -0
- package/dist/{chunk-RWESFSJI.mjs → chunk-IBATGS3J.mjs} +87 -61
- package/dist/{chunk-Y2U5VVWX.mjs → chunk-ICW45TTU.mjs} +1 -1
- package/dist/{chunk-T4OFG4IQ.mjs → chunk-IP4HJOGC.mjs} +90 -16
- package/dist/{chunk-EUCX5K74.mjs → chunk-J3EHATEZ.mjs} +3 -3
- package/dist/{chunk-X5CNUKD5.mjs → chunk-K3AQOZVM.mjs} +37 -20
- package/dist/chunk-LK3MOTXF.mjs +174 -0
- package/dist/{chunk-7OIQ2SIO.mjs → chunk-LLDFIKGP.mjs} +4 -4
- package/dist/{chunk-K77FJXKS.mjs → chunk-MTQTX4BP.mjs} +81 -8
- package/dist/{chunk-ZR4VM4A6.mjs → chunk-N37VMC62.mjs} +1 -18
- package/dist/{chunk-2WBY4Y4M.mjs → chunk-NOT5QOHU.mjs} +1 -1
- package/dist/{chunk-D5TFRB5G.mjs → chunk-NPNK57CA.mjs} +1 -1
- package/dist/{chunk-6SSVIW7Y.mjs → chunk-O4LJQI4G.mjs} +31 -5
- package/dist/{chunk-JJ6EUMXB.mjs → chunk-PNNVTKMT.mjs} +2 -2
- package/dist/{chunk-2SEL6SBS.mjs → chunk-RKAVHNRW.mjs} +2 -2
- package/dist/{chunk-XABMCBHA.mjs → chunk-SBMAWTRG.mjs} +684 -99
- package/dist/chunk-TCWQJL7O.mjs +188 -0
- package/dist/{chunk-2MKX2SCO.mjs → chunk-U4X4AWKL.mjs} +7 -7
- package/dist/{chunk-PU6SKH7O.mjs → chunk-UBXJQJII.mjs} +2 -2
- package/dist/chunk-UEV53BBO.mjs +480 -0
- package/dist/{chunk-NNO7CO3A.mjs → chunk-UIKN2OTQ.mjs} +1 -1
- package/dist/{chunk-F6AAXI7T.mjs → chunk-VSUYTPME.mjs} +3 -3
- package/dist/{chunk-TGHNXN7W.mjs → chunk-W2XDK66Z.mjs} +180 -541
- package/dist/{chunk-7X5JST5L.mjs → chunk-WNBRW4Q2.mjs} +16 -2
- package/dist/{chunk-6M6SDH3S.mjs → chunk-XMQNOD5V.mjs} +8 -8
- package/dist/{chunk-SZGRYEIC.mjs → chunk-XS2VM7UE.mjs} +3 -3
- package/dist/{chunk-JT3YZGCE.mjs → chunk-XXKWEG53.mjs} +3 -188
- package/dist/{chunk-YBWSV5DV.mjs → chunk-ZMBFWQXN.mjs} +2 -2
- package/dist/{chunk-7XY6VD7B.mjs → chunk-ZSWHPKGM.mjs} +1 -1
- package/dist/{client-Cx-TiqPE.d.ts → client-DApBeLbk.d.ts} +18 -3
- package/dist/client.d.ts +7 -8
- package/dist/client.mjs +11 -1
- package/dist/{events-DErop1LM.d.ts → config-B0H-vp4U.d.ts} +424 -2
- package/dist/{config-CzB9YvF_.d.ts → config-Brysl9NO.d.ts} +3 -3
- package/dist/{direct-executor-HQ4QVDX7.mjs → direct-executor-J5UDLPL3.mjs} +14 -14
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +24 -9
- package/dist/distribution.mjs +882 -192
- package/dist/{durability-wiring-S7OX2MAI.mjs → durability-wiring-AYOTAPNQ.mjs} +165 -70
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-B_W9sdZK.d.ts → engine-CuLMbH1-.d.ts} +64 -7
- package/dist/import-QRZAQ5RH.mjs +43 -0
- package/dist/{index-CKI1fDEl.d.ts → index-HYnqS1SE.d.ts} +2 -2
- package/dist/index.browser.mjs +4 -4
- package/dist/index.d.ts +6 -7
- package/dist/index.mjs +20 -20
- package/dist/{long-query-thread-WVSMIBYP.mjs → long-query-thread-U7K5A72G.mjs} +14 -14
- package/dist/{module-2E4XYSGT.mjs → module-L2OZO5PQ.mjs} +11 -11
- package/dist/partial-words.mjs +5 -5
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +12 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +23 -23
- package/dist/{phrase-module-JERF2IDG.mjs → phrase-module-5QHBD2VJ.mjs} +1 -1
- package/dist/react.d.ts +11 -8
- package/dist/react.mjs +4 -0
- package/dist/{rerank-DMMT4FZA.mjs → rerank-KTLC3E6N.mjs} +5 -5
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{browser/restore-queue-GEZ54CA5.mjs → restore-queue-RYSRRFS4.mjs} +8 -2
- package/dist/{runtime-7MLQ7J64.mjs → runtime-DYV75DCB.mjs} +6 -6
- package/dist/{schema-CS-clLun.d.ts → schema-BTXbHCqA.d.ts} +4 -1
- package/dist/server.d.ts +200 -11
- package/dist/server.mjs +66 -23
- package/dist/{snapshot-DOQLGGEX.mjs → snapshot-BTRZ3MZK.mjs} +59 -190
- package/dist/synonyms.mjs +6 -4
- package/dist/tasks-DjGjK70Y.d.ts +720 -0
- package/dist/{thread-DNK5XWXG.mjs → thread-USB3GOPY.mjs} +13 -13
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +5 -5
- package/dist/workers/entry.mjs +7 -7
- package/dist/{write-ZKR7SNRT.mjs → write-GHTA56OW.mjs} +25 -22
- package/openapi.json +5310 -0
- package/package.json +16 -8
- package/dist/config-hOWGBeC6.d.ts +0 -414
- package/dist/tasks-BfF-VRhm.d.ts +0 -302
|
@@ -1,18 +1,19 @@
|
|
|
1
|
-
import { requirePatternSearchFor, requirePartialWordSearchFor, createPartitionIndex, buildPartialWordTableFrom, recordValueSize, patternIndexOf, encodeFieldIndexes, encodePartialSpellingsOf, countPartialSpelling, ARRAY_POSITION_GAP, copyVectorsInto, isCompositePartition, fnv1a, partialSpellingsOfDocuments, createFrozenPartialWords, requirePatternSearch, documentValueBytes, cloneProjected, projectIncludedPaths, valueWithOwnBuffers, createSurfaceRegistry, MAX_TERM_FREQUENCY, loadPhraseCodeFor, joinPatternWorkMeter, fanOutQuery, sanitizeGlobalStats, countsWithoutScores, fanOutMatchCount, createCompositePartition, createGeoIndex, documentWithOwnBuffers, readProjectedPath, createNumericIndex, createBooleanIndex, createEnumIndex, ValueAffixes } from './chunk-
|
|
2
|
-
import { generateId
|
|
1
|
+
import { requirePatternSearchFor, requirePartialWordSearchFor, createPartitionIndex, buildPartialWordTableFrom, recordValueSize, patternIndexOf, encodeFieldIndexes, encodePartialSpellingsOf, countPartialSpelling, ARRAY_POSITION_GAP, copyVectorsInto, isCompositePartition, fnv1a, partialSpellingsOfDocuments, createFrozenPartialWords, requirePatternSearch, documentValueBytes, cloneProjected, projectIncludedPaths, valueWithOwnBuffers, createSurfaceRegistry, MAX_TERM_FREQUENCY, loadPhraseCodeFor, joinPatternWorkMeter, fanOutQuery, sanitizeGlobalStats, countsWithoutScores, fanOutMatchCount, createCompositePartition, createGeoIndex, documentWithOwnBuffers, readProjectedPath, createNumericIndex, createBooleanIndex, createEnumIndex, ValueAffixes } from './chunk-U4X4AWKL.mjs';
|
|
2
|
+
import { generateId } from './chunk-N37VMC62.mjs';
|
|
3
3
|
import { readHeapStatistics, readResidentMemory } from './chunk-ZATRDAYU.mjs';
|
|
4
4
|
import { GrowableUint32, bitsetHas, bitsetSet, createBitSet } from './chunk-JUKYYXSP.mjs';
|
|
5
|
-
import { tokenizeOptions, getNestedValue, sameFields, visitDocumentSurfaceSpellings, visitDocumentSpellings } from './chunk-
|
|
6
|
-
import {
|
|
5
|
+
import { tokenizeOptions, getNestedValue, sameFields, visitDocumentSurfaceSpellings, visitDocumentSpellings } from './chunk-J3EHATEZ.mjs';
|
|
6
|
+
import { partialFieldsOf, schemaFieldsOf, encodeDocument, validateDocument, validateDocumentStrict, extractVectorFieldsFromSchema } from './chunk-UBXJQJII.mjs';
|
|
7
7
|
import { fuzzyPrefixOf, boundedLevenshtein } from './chunk-GES7U2PM.mjs';
|
|
8
|
-
import { isWordIndexedBase } from './chunk-
|
|
9
|
-
import { resolveIndexAnalysis, synonymSearchModule } from './chunk-
|
|
10
|
-
import {
|
|
8
|
+
import { isWordIndexedBase } from './chunk-ZSWHPKGM.mjs';
|
|
9
|
+
import { resolveIndexAnalysis, synonymSearchModule } from './chunk-O4LJQI4G.mjs';
|
|
10
|
+
import { getLanguage } from './chunk-WNBRW4Q2.mjs';
|
|
11
|
+
import { visitTokens, codePointOrder, encodeStringBlob, encodeLookupKey } from './chunk-NPNK57CA.mjs';
|
|
11
12
|
import { SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, FROZEN_REMOVAL_INDEX_MAX_PROCESS_FRACTION } from './chunk-4UMZ7KQL.mjs';
|
|
12
13
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
13
|
-
import { loadedVectorCode, loadVectorCode, loadVectorCodeFor, requireLoadedVectorCode } from './chunk-
|
|
14
|
+
import { loadedVectorCode, loadVectorCode, loadVectorCodeFor, requireLoadedVectorCode } from './chunk-FFVLPCLH.mjs';
|
|
14
15
|
import { PROTOTYPE_POLLUTION_KEYS } from './chunk-L5ORED7V.mjs';
|
|
15
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
16
|
+
import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
|
|
16
17
|
import { Decoder, decode } from '@msgpack/msgpack';
|
|
17
18
|
|
|
18
19
|
// src/core/synonyms/platform.ts
|
|
@@ -112,6 +113,32 @@ function createCapacityChecks(source) {
|
|
|
112
113
|
};
|
|
113
114
|
}
|
|
114
115
|
|
|
116
|
+
// src/partitioning/manager/doc-locations.ts
|
|
117
|
+
function rebuildDocLocations(locations, partitions) {
|
|
118
|
+
locations.clear();
|
|
119
|
+
for (let i = 0; i < partitions.length; i++) {
|
|
120
|
+
for (const docId of partitions[i].docIds()) {
|
|
121
|
+
locations.set(docId, i);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
function recordSegmentDocLocations(locations, partitionId, segment) {
|
|
126
|
+
for (const internalId of segment.docStore.allInternalIds()) {
|
|
127
|
+
const docId = segment.docStore.getExternalId(internalId);
|
|
128
|
+
if (docId !== void 0) locations.set(docId, partitionId);
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
function rebuildPartitionDocLocations(locations, partitionId, partition) {
|
|
132
|
+
for (const [docId, pid] of locations) {
|
|
133
|
+
if (pid === partitionId) {
|
|
134
|
+
locations.delete(docId);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
for (const docId of partition.docIds()) {
|
|
138
|
+
locations.set(docId, partitionId);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
115
142
|
// src/partitioning/manager/segments.ts
|
|
116
143
|
function requireNewSegmentDocuments(segment, docPartitionMap) {
|
|
117
144
|
for (const internalId of segment.docStore.allInternalIds()) {
|
|
@@ -200,13 +227,15 @@ function createWriteLogRecorder() {
|
|
|
200
227
|
}
|
|
201
228
|
|
|
202
229
|
// src/partitioning/manager/index.ts
|
|
203
|
-
function createPartitionManager(indexName,
|
|
230
|
+
function createPartitionManager(indexName, initialConfig, initialLanguage, router, initialPartitionCount, vectorIndexes) {
|
|
231
|
+
let config = initialConfig;
|
|
232
|
+
let language = initialLanguage;
|
|
204
233
|
requirePatternSearchFor(config.schema);
|
|
205
234
|
requirePartialWordSearchFor(config.schema);
|
|
206
235
|
const count = initialPartitionCount ?? 1;
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
236
|
+
let analysis = resolveIndexAnalysis(config);
|
|
237
|
+
let partialWordAnalysis = resolvePartialWordAnalysis(config, language, analysis);
|
|
238
|
+
let textAnalysis = { language, options: resolvePartitionInsertOptions(config, analysis) };
|
|
210
239
|
const trackPositions = config.trackPositions ?? true;
|
|
211
240
|
const vecIndexes = vectorIndexes ?? /* @__PURE__ */ new Map();
|
|
212
241
|
let partitions = [];
|
|
@@ -240,30 +269,6 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
240
269
|
partitions[partitionId] = composite;
|
|
241
270
|
return composite;
|
|
242
271
|
}
|
|
243
|
-
function rebuildDocPartitionMap() {
|
|
244
|
-
docPartitionMap.clear();
|
|
245
|
-
for (let i = 0; i < partitions.length; i++) {
|
|
246
|
-
for (const docId of partitions[i].docIds()) {
|
|
247
|
-
docPartitionMap.set(docId, i);
|
|
248
|
-
}
|
|
249
|
-
}
|
|
250
|
-
}
|
|
251
|
-
function registerSegmentDocuments(partitionId, segment) {
|
|
252
|
-
for (const internalId of segment.docStore.allInternalIds()) {
|
|
253
|
-
const docId = segment.docStore.getExternalId(internalId);
|
|
254
|
-
if (docId !== void 0) docPartitionMap.set(docId, partitionId);
|
|
255
|
-
}
|
|
256
|
-
}
|
|
257
|
-
function rebuildDocPartitionMapForPartition(partitionId) {
|
|
258
|
-
for (const [docId, pid] of docPartitionMap) {
|
|
259
|
-
if (pid === partitionId) {
|
|
260
|
-
docPartitionMap.delete(docId);
|
|
261
|
-
}
|
|
262
|
-
}
|
|
263
|
-
for (const docId of partitions[partitionId].docIds()) {
|
|
264
|
-
docPartitionMap.set(docId, partitionId);
|
|
265
|
-
}
|
|
266
|
-
}
|
|
267
272
|
const manager = {
|
|
268
273
|
get partitionCount() {
|
|
269
274
|
return partitions.length;
|
|
@@ -303,7 +308,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
303
308
|
setPartitions(newPartitions) {
|
|
304
309
|
writeLog.partitionsReplaced();
|
|
305
310
|
partitions = [...newPartitions];
|
|
306
|
-
|
|
311
|
+
rebuildDocLocations(docPartitionMap, partitions);
|
|
307
312
|
},
|
|
308
313
|
openWriteLog() {
|
|
309
314
|
return writeLog.open();
|
|
@@ -318,6 +323,14 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
318
323
|
}
|
|
319
324
|
partitions = [...rebuilt];
|
|
320
325
|
},
|
|
326
|
+
adoptAnalysis(change) {
|
|
327
|
+
config = change.config;
|
|
328
|
+
language = change.language;
|
|
329
|
+
analysis = change.analysis;
|
|
330
|
+
partialWordAnalysis = change.partialWordAnalysis;
|
|
331
|
+
textAnalysis = { language, options: resolvePartitionInsertOptions(config, analysis) };
|
|
332
|
+
manager.synonyms = change.synonyms;
|
|
333
|
+
},
|
|
321
334
|
addPartition() {
|
|
322
335
|
const currentMaxPartitions = config.partitions?.maxPartitions;
|
|
323
336
|
if (currentMaxPartitions !== void 0 && partitions.length >= currentMaxPartitions) {
|
|
@@ -336,7 +349,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
336
349
|
validatePartitionId(partitionId);
|
|
337
350
|
writeLog.partitionsReplaced();
|
|
338
351
|
partitions.splice(partitionId, 1);
|
|
339
|
-
|
|
352
|
+
rebuildDocLocations(docPartitionMap, partitions);
|
|
340
353
|
},
|
|
341
354
|
trimPartitions(newCount) {
|
|
342
355
|
if (newCount < 1) {
|
|
@@ -358,7 +371,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
358
371
|
},
|
|
359
372
|
...createCapacityChecks({
|
|
360
373
|
indexName,
|
|
361
|
-
config,
|
|
374
|
+
config: initialConfig,
|
|
362
375
|
partitionCount: () => partitions.length,
|
|
363
376
|
documentCount: () => manager.countDocuments(),
|
|
364
377
|
getPartition: (partitionId) => manager.getPartition(partitionId)
|
|
@@ -460,7 +473,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
460
473
|
validatePartitionId(partitionId);
|
|
461
474
|
requireNewSegmentDocuments(segment, docPartitionMap);
|
|
462
475
|
asCompositePartition(partitionId).attachFrozenSegment(segment);
|
|
463
|
-
|
|
476
|
+
recordSegmentDocLocations(docPartitionMap, partitionId, segment);
|
|
464
477
|
writeLog.segmentWritten(segment);
|
|
465
478
|
},
|
|
466
479
|
swapFrozenSegments(partitionId, dropSegmentIds, replacement, removedSpellings) {
|
|
@@ -470,7 +483,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
470
483
|
replacement,
|
|
471
484
|
removedSpellings
|
|
472
485
|
);
|
|
473
|
-
|
|
486
|
+
recordSegmentDocLocations(docPartitionMap, partitionId, replacement);
|
|
474
487
|
return applied;
|
|
475
488
|
},
|
|
476
489
|
freezeLiveTail(partitionId, freeze) {
|
|
@@ -480,14 +493,14 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
480
493
|
replaceLiveTail(partitionId, segment) {
|
|
481
494
|
validatePartitionId(partitionId);
|
|
482
495
|
asCompositePartition(partitionId).replaceLiveTail(segment);
|
|
483
|
-
|
|
496
|
+
recordSegmentDocLocations(docPartitionMap, partitionId, segment);
|
|
484
497
|
},
|
|
485
498
|
deserializePartition(partitionId, data) {
|
|
486
499
|
validatePartitionId(partitionId);
|
|
487
500
|
writeLog.partitionsReplaced();
|
|
488
501
|
partitions[partitionId].deserialize(data, config.schema);
|
|
489
502
|
preparePartialWords(partitionId);
|
|
490
|
-
|
|
503
|
+
rebuildPartitionDocLocations(docPartitionMap, partitionId, partitions[partitionId]);
|
|
491
504
|
},
|
|
492
505
|
getAggregateStats() {
|
|
493
506
|
return aggregateStatsOf(partitions);
|
|
@@ -3247,6 +3260,34 @@ function heldVectorOf(held, fieldName, docId) {
|
|
|
3247
3260
|
return held.fields.get(fieldName)?.view.vectorOf(docId);
|
|
3248
3261
|
}
|
|
3249
3262
|
|
|
3263
|
+
// src/workers/direct-executor/entry.ts
|
|
3264
|
+
function indexEntryOf(manager, vectorIndexes, analysisStale, awaitedCopyLoad) {
|
|
3265
|
+
return {
|
|
3266
|
+
manager,
|
|
3267
|
+
get config() {
|
|
3268
|
+
return manager.config;
|
|
3269
|
+
},
|
|
3270
|
+
get language() {
|
|
3271
|
+
return manager.language;
|
|
3272
|
+
},
|
|
3273
|
+
searchOptions: {
|
|
3274
|
+
get stopWords() {
|
|
3275
|
+
return manager.analysis.stopWords;
|
|
3276
|
+
},
|
|
3277
|
+
get customTokenizer() {
|
|
3278
|
+
return manager.analysis.customTokenizer;
|
|
3279
|
+
},
|
|
3280
|
+
get synonyms() {
|
|
3281
|
+
return manager.synonyms;
|
|
3282
|
+
}
|
|
3283
|
+
},
|
|
3284
|
+
vectorIndexes,
|
|
3285
|
+
vectorCopies: createHeldVectorCopies(),
|
|
3286
|
+
analysisStale,
|
|
3287
|
+
awaitedCopyLoad
|
|
3288
|
+
};
|
|
3289
|
+
}
|
|
3290
|
+
|
|
3250
3291
|
// src/workers/direct-executor/index.ts
|
|
3251
3292
|
var NO_VECTOR_WORKER_COPIES = { enabled: false };
|
|
3252
3293
|
function createDirectExecutor(options) {
|
|
@@ -3298,22 +3339,7 @@ function createDirectExecutor(options) {
|
|
|
3298
3339
|
const partitionCount = config.partitions?.maxPartitions ?? 1;
|
|
3299
3340
|
const vectorIndexes = vectorIndexesFor(indexName, config, copies);
|
|
3300
3341
|
const manager = createPartitionManager(indexName, config, language, router, partitionCount, vectorIndexes);
|
|
3301
|
-
return
|
|
3302
|
-
manager,
|
|
3303
|
-
config,
|
|
3304
|
-
language,
|
|
3305
|
-
searchOptions: {
|
|
3306
|
-
stopWords: manager.analysis.stopWords,
|
|
3307
|
-
customTokenizer: manager.analysis.customTokenizer,
|
|
3308
|
-
get synonyms() {
|
|
3309
|
-
return manager.synonyms;
|
|
3310
|
-
}
|
|
3311
|
-
},
|
|
3312
|
-
vectorIndexes,
|
|
3313
|
-
vectorCopies: createHeldVectorCopies(),
|
|
3314
|
-
analysisStale,
|
|
3315
|
-
awaitedCopyLoad
|
|
3316
|
-
};
|
|
3342
|
+
return indexEntryOf(manager, vectorIndexes, analysisStale, awaitedCopyLoad);
|
|
3317
3343
|
}
|
|
3318
3344
|
function disposeEntry(entry) {
|
|
3319
3345
|
for (const vectorIndex of entry.vectorIndexes.values()) {
|
|
@@ -3534,4 +3560,4 @@ function createDirectExecutor(options) {
|
|
|
3534
3560
|
};
|
|
3535
3561
|
}
|
|
3536
3562
|
|
|
3537
|
-
export { availableSynonymModule, buildSegmentPayload, createDirectExecutor, createFrozenSegment, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeMapWithoutFields, decodeStoredDocument, freezeSegmentShared, isEmptySynonymSet, mergeFrozenSegments, mergeSegmentContents, registerSynonyms, resolvePartitionInsertOptions, segmentArraysOf, selectionKeeps, shareSegmentArrays, synonymModule };
|
|
3563
|
+
export { availableSynonymModule, buildSegmentPayload, createDirectExecutor, createFrozenSegment, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeMapWithoutFields, decodeStoredDocument, freezeSegmentShared, isEmptySynonymSet, mergeFrozenSegments, mergeSegmentContents, registerSynonyms, resolvePartialWordAnalysis, resolvePartitionInsertOptions, segmentArraysOf, selectionKeeps, shareSegmentArrays, synonymModule };
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { osqRecordBytes } from './chunk-37CD3OKS.mjs';
|
|
2
2
|
import { MAX_M, EARLIER_RELEASE_MIN_M, MIN_M } from './chunk-VJS4JFBQ.mjs';
|
|
3
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
3
|
+
import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
|
|
4
4
|
|
|
5
5
|
// src/vector/vector-index/dead-bits.ts
|
|
6
6
|
var NO_LIVE_VECTOR_AT_POSITION = -1;
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
+
import { restoredEmbedding, restoredPartitionConfig, readSharedConfigFields } from './chunk-TCWQJL7O.mjs';
|
|
1
2
|
import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR } from './chunk-P36MUFEJ.mjs';
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import { decodeSynonymRules } from './chunk-7X5JST5L.mjs';
|
|
3
|
+
import { generateId } from './chunk-N37VMC62.mjs';
|
|
4
|
+
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-EEBCDP36.mjs';
|
|
5
|
+
import { getLanguage, decodeSynonymRules } from './chunk-WNBRW4Q2.mjs';
|
|
6
6
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
7
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
7
|
+
import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
|
|
8
8
|
import { encode, decode } from '@msgpack/msgpack';
|
|
9
9
|
|
|
10
10
|
// src/narsil/write-blocks.ts
|
|
@@ -38,6 +38,9 @@ var RESHARD_CUTOVER_TIMEOUT_MS = 1e4;
|
|
|
38
38
|
var MAX_RESHARD_CUTOVER_ATTEMPTS = 10;
|
|
39
39
|
var RESHARD_POLL_INTERVAL_MS = 50;
|
|
40
40
|
var RESHARD_CONTROLLER_POLL_INTERVAL_MS = 250;
|
|
41
|
+
var ANALYSIS_CHANGE_POLL_INTERVAL_MS = 100;
|
|
42
|
+
var ANALYSIS_REBUILD_RESEND_MS = 3e4;
|
|
43
|
+
var ANALYSIS_CHANGE_RETRY_MS = 1e3;
|
|
41
44
|
var RESHARD_RECORD_CHECK_INTERVAL_MS = 1e3;
|
|
42
45
|
var RESHARD_RETRY_DELAY_MS = 200;
|
|
43
46
|
var RESHARD_FORWARD_BATCH_DOCUMENTS = 256;
|
|
@@ -173,6 +176,16 @@ async function readSynonymSet(coordinator, indexUuid, ref, reject) {
|
|
|
173
176
|
if (await digestOf(encoded) !== ref.digest) return reject("the stored synonym set does not match its digest");
|
|
174
177
|
return encoded;
|
|
175
178
|
}
|
|
179
|
+
async function copySynonymSet(coordinator, sourceUuid, targetUuid, ref, reject) {
|
|
180
|
+
const encoded = await readSynonymSet(coordinator, sourceUuid, ref, reject);
|
|
181
|
+
if (encoded === null) return false;
|
|
182
|
+
for (let chunk = 0; chunk < ref.chunkCount; chunk++) {
|
|
183
|
+
const start = chunk * SYNONYM_CHUNK_BYTES;
|
|
184
|
+
const bytes = encoded.subarray(start, Math.min(start + SYNONYM_CHUNK_BYTES, encoded.byteLength));
|
|
185
|
+
await coordinator.compareAndSet(synonymChunkKey(targetUuid, ref.generation, chunk), null, bytes);
|
|
186
|
+
}
|
|
187
|
+
return true;
|
|
188
|
+
}
|
|
176
189
|
function storedSynonymSetRules(encoded, reject) {
|
|
177
190
|
return decodeSynonymRules(encoded, (message) => reject(message));
|
|
178
191
|
}
|
|
@@ -255,6 +268,38 @@ function settingsRejection(indexName) {
|
|
|
255
268
|
);
|
|
256
269
|
};
|
|
257
270
|
}
|
|
271
|
+
function isGeneration(value) {
|
|
272
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
|
|
273
|
+
}
|
|
274
|
+
function isOptionalString(value) {
|
|
275
|
+
return value === void 0 || typeof value === "string";
|
|
276
|
+
}
|
|
277
|
+
function decodePendingAnalysis(raw, reject) {
|
|
278
|
+
if (raw === void 0) return void 0;
|
|
279
|
+
if (!isRecord2(raw) || typeof raw.changeId !== "string" || !isGeneration(raw.generation) || typeof raw.language !== "string" || !isOptionalString(raw.tokenizer) || !isOptionalString(raw.stopWords) || raw.stopWordList !== void 0 && (!Array.isArray(raw.stopWordList) || raw.stopWordList.some((word) => typeof word !== "string"))) {
|
|
280
|
+
return reject("pendingAnalysis is malformed");
|
|
281
|
+
}
|
|
282
|
+
return raw;
|
|
283
|
+
}
|
|
284
|
+
function decodeAnalysisFailure(raw, reject) {
|
|
285
|
+
if (raw === void 0) return void 0;
|
|
286
|
+
if (!isRecord2(raw) || typeof raw.changeId !== "string" || typeof raw.code !== "string" || typeof raw.message !== "string" || typeof raw.nodeId !== "string") {
|
|
287
|
+
return reject("analysisFailure is malformed");
|
|
288
|
+
}
|
|
289
|
+
return raw;
|
|
290
|
+
}
|
|
291
|
+
function decodeAnalysisMembers(raw, reject) {
|
|
292
|
+
if (raw.analysisGeneration !== void 0 && !isGeneration(raw.analysisGeneration)) {
|
|
293
|
+
return reject("analysisGeneration must be a non-negative integer");
|
|
294
|
+
}
|
|
295
|
+
const pendingAnalysis = decodePendingAnalysis(raw.pendingAnalysis, reject);
|
|
296
|
+
const analysisFailure = decodeAnalysisFailure(raw.analysisFailure, reject);
|
|
297
|
+
return {
|
|
298
|
+
...raw.analysisGeneration !== void 0 ? { analysisGeneration: raw.analysisGeneration } : {},
|
|
299
|
+
...pendingAnalysis !== void 0 ? { pendingAnalysis } : {},
|
|
300
|
+
...analysisFailure !== void 0 ? { analysisFailure } : {}
|
|
301
|
+
};
|
|
302
|
+
}
|
|
258
303
|
function decodeIndexSettings(raw, indexName) {
|
|
259
304
|
if (raw === void 0 || raw === null) {
|
|
260
305
|
return void 0;
|
|
@@ -276,6 +321,7 @@ function decodeIndexSettings(raw, indexName) {
|
|
|
276
321
|
return {
|
|
277
322
|
language: raw.language,
|
|
278
323
|
...typeof bm25Revision === "number" ? { bm25Revision } : {},
|
|
324
|
+
...decodeAnalysisMembers(raw, reject),
|
|
279
325
|
...settingsFieldsOf({
|
|
280
326
|
...readSharedConfigFields(raw, reject),
|
|
281
327
|
...partitions !== void 0 ? { partitions } : {},
|
|
@@ -291,7 +337,17 @@ function indexConfigFromSettings(schema, settings, indexName) {
|
|
|
291
337
|
if (settings === void 0) {
|
|
292
338
|
return { schema };
|
|
293
339
|
}
|
|
294
|
-
const {
|
|
340
|
+
const {
|
|
341
|
+
language,
|
|
342
|
+
embedding,
|
|
343
|
+
partitionConfig,
|
|
344
|
+
synonyms: _synonyms,
|
|
345
|
+
bm25Revision: _revision,
|
|
346
|
+
analysisGeneration: _generation,
|
|
347
|
+
pendingAnalysis: _pending,
|
|
348
|
+
analysisFailure: _failure,
|
|
349
|
+
...fields
|
|
350
|
+
} = settings;
|
|
295
351
|
return {
|
|
296
352
|
schema,
|
|
297
353
|
language,
|
|
@@ -416,16 +472,28 @@ async function putIndexMetadata(coordinator, metadata) {
|
|
|
416
472
|
return coordinator.compareAndSet(key, current, bytes);
|
|
417
473
|
}
|
|
418
474
|
async function putGenerationMetadata(coordinator, sourceName, generation) {
|
|
419
|
-
const sourceBytes = await coordinator.get(indexConfigKey(sourceName));
|
|
420
|
-
if (sourceBytes === null || sourceBytes.byteLength === 0) return false;
|
|
421
|
-
const decoded = decode(sourceBytes);
|
|
422
|
-
validateDecodedMetadata(decoded, sourceName);
|
|
423
|
-
if (!isRecord2(decoded)) return false;
|
|
424
475
|
const key = indexConfigKey(generation.indexName);
|
|
425
|
-
const
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
476
|
+
const reject = settingsRejection(sourceName);
|
|
477
|
+
let copiedSetGeneration = null;
|
|
478
|
+
for (; ; ) {
|
|
479
|
+
const sourceBytes = await coordinator.get(indexConfigKey(sourceName));
|
|
480
|
+
if (sourceBytes === null || sourceBytes.byteLength === 0) return false;
|
|
481
|
+
const decoded = decode(sourceBytes);
|
|
482
|
+
validateDecodedMetadata(decoded, sourceName);
|
|
483
|
+
if (!isRecord2(decoded)) return false;
|
|
484
|
+
const current = await coordinator.get(key);
|
|
485
|
+
if (current !== null && current.byteLength > 0) return true;
|
|
486
|
+
const held = storedSynonymSetOf(sourceBytes);
|
|
487
|
+
if (held !== null && !await copySynonymSet(coordinator, held.indexUuid, generation.indexUuid, held.ref, reject)) {
|
|
488
|
+
if (copiedSetGeneration === held.ref.generation) {
|
|
489
|
+
return reject(`a chunk of synonym set ${held.ref.generation} is missing`);
|
|
490
|
+
}
|
|
491
|
+
copiedSetGeneration = held.ref.generation;
|
|
492
|
+
continue;
|
|
493
|
+
}
|
|
494
|
+
const bytes = new Uint8Array(encode({ ...decoded, ...generation }, { ignoreUndefined: true }));
|
|
495
|
+
return coordinator.compareAndSet(key, current, bytes);
|
|
496
|
+
}
|
|
429
497
|
}
|
|
430
498
|
async function readStoredMetadata(coordinator, indexName) {
|
|
431
499
|
validateIndexName(indexName);
|
|
@@ -437,6 +505,12 @@ async function readStoredMetadata(coordinator, indexName) {
|
|
|
437
505
|
const record = validateDecodedMetadata(decoded, indexName);
|
|
438
506
|
return { bytes: raw, record, rawSettings: isRecord2(decoded) ? decoded.settings : void 0 };
|
|
439
507
|
}
|
|
508
|
+
function indexMetadataSnapshotOf(bytes, indexName) {
|
|
509
|
+
const decoded = decode(bytes);
|
|
510
|
+
const record = validateDecodedMetadata(decoded, indexName);
|
|
511
|
+
const rawSettings = isRecord2(decoded) ? decoded.settings : void 0;
|
|
512
|
+
return { bytes, record, settings: decodeIndexSettings(rawSettings, indexName) };
|
|
513
|
+
}
|
|
440
514
|
async function readIndexMetadataSnapshot(coordinator, indexName) {
|
|
441
515
|
const stored = await readStoredMetadata(coordinator, indexName);
|
|
442
516
|
if (stored === null) return null;
|
|
@@ -529,4 +603,4 @@ function storedSynonymSetOf(bytes) {
|
|
|
529
603
|
return ref === null ? null : { indexUuid: decoded.indexUuid, ref };
|
|
530
604
|
}
|
|
531
605
|
|
|
532
|
-
export { ALLOCATION_CAS_ATTEMPTS, ALLOCATION_RETRY_DELAY_MS, BOOTSTRAP_CAS_ATTEMPTS, DEFAULT_CONTROLLER_CONFIG, DEFAULT_NODE_LIFECYCLE_CONFIG, ESTIMATED_PARTITION_BYTES, HIDDEN_GENERATION_PREFIX, LEADERSHIP_IMBALANCE_THRESHOLD, MAX_RESHARD_CUTOVER_ATTEMPTS, PARTITION_STORES_TIMEOUT_MS, REBALANCE_THRESHOLD, RECOVERY_CAS_ATTEMPTS, RESHARD_CONTROLLER_POLL_INTERVAL_MS, RESHARD_CUTOVER_LAG_ENTRIES, RESHARD_CUTOVER_TIMEOUT_MS, RESHARD_FORWARD_BATCH_DOCUMENTS, RESHARD_POLL_INTERVAL_MS, RESHARD_RECORD_CAS_ATTEMPTS, RESHARD_RECORD_CHECK_INTERVAL_MS, RESHARD_RETRY_DELAY_MS, TEARDOWN_CAS_ATTEMPTS, TEARDOWN_RETRY_DELAY_MS, TEARDOWN_RETRY_ROUNDS, bindEngineWriteBlock, bindIndexReads, bindWriteBlockLookup, decodeIndexSettings, deleteSynonymSet, getClusterIndexConfig, getIndexMetadata, indexConfigFromSettings, indexConfigKey, indexReadsOf, indexSettingsOf, indexUuidOfStoredMetadata, isRecord, isRecord2, isValidInteger, metadataWithBm25, metadataWithSynonymSet, putGenerationMetadata, putIndexMetadata, putIndexReplicationFactor, readIndexMetadataSnapshot, readIndexState, storeSynonymSet, storedSynonymSetOf, validateIndexName, withPartitionCount, writeBlockLookupOf };
|
|
606
|
+
export { ALLOCATION_CAS_ATTEMPTS, ALLOCATION_RETRY_DELAY_MS, ANALYSIS_CHANGE_POLL_INTERVAL_MS, ANALYSIS_CHANGE_RETRY_MS, ANALYSIS_REBUILD_RESEND_MS, BOOTSTRAP_CAS_ATTEMPTS, DEFAULT_CONTROLLER_CONFIG, DEFAULT_NODE_LIFECYCLE_CONFIG, ESTIMATED_PARTITION_BYTES, HIDDEN_GENERATION_PREFIX, LEADERSHIP_IMBALANCE_THRESHOLD, MAX_RESHARD_CUTOVER_ATTEMPTS, PARTITION_STORES_TIMEOUT_MS, REBALANCE_THRESHOLD, RECOVERY_CAS_ATTEMPTS, RESHARD_CONTROLLER_POLL_INTERVAL_MS, RESHARD_CUTOVER_LAG_ENTRIES, RESHARD_CUTOVER_TIMEOUT_MS, RESHARD_FORWARD_BATCH_DOCUMENTS, RESHARD_POLL_INTERVAL_MS, RESHARD_RECORD_CAS_ATTEMPTS, RESHARD_RECORD_CHECK_INTERVAL_MS, RESHARD_RETRY_DELAY_MS, TEARDOWN_CAS_ATTEMPTS, TEARDOWN_RETRY_DELAY_MS, TEARDOWN_RETRY_ROUNDS, bindEngineWriteBlock, bindIndexReads, bindWriteBlockLookup, decodeIndexSettings, deleteSynonymSet, getClusterIndexConfig, getIndexMetadata, indexConfigFromSettings, indexConfigKey, indexMetadataSnapshotOf, indexReadsOf, indexSettingsOf, indexUuidOfStoredMetadata, isRecord, isRecord2, isValidInteger, metadataWithBm25, metadataWithSynonymSet, putGenerationMetadata, putIndexMetadata, putIndexReplicationFactor, readIndexMetadataSnapshot, readIndexState, storeSynonymSet, storedSynonymSetOf, validateIndexName, withPartitionCount, writeBlockLookupOf };
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { flattenSchema } from './chunk-
|
|
1
|
+
import { flattenSchema } from './chunk-UBXJQJII.mjs';
|
|
2
2
|
import { PARTIAL_WORD_MIN_CODE_POINTS, PARTIAL_WORDS_ADDED_PER_QUERY, DEFAULT_FUZZY_PREFIX_LENGTH } from './chunk-FQBFZAWG.mjs';
|
|
3
3
|
import { fuzzyTermMatches } from './chunk-GES7U2PM.mjs';
|
|
4
|
-
import { isWordIndexedBase } from './chunk-
|
|
5
|
-
import { encodeStringBlob, tokenize, resolveStopWords, transformToken, visitTokens, visitSurfaceSpellings } from './chunk-
|
|
4
|
+
import { isWordIndexedBase } from './chunk-ZSWHPKGM.mjs';
|
|
5
|
+
import { encodeStringBlob, tokenize, resolveStopWords, transformToken, visitTokens, visitSurfaceSpellings } from './chunk-NPNK57CA.mjs';
|
|
6
6
|
import { compareCodePoints, codePointCount } from './chunk-YTJCGKM4.mjs';
|
|
7
7
|
|
|
8
8
|
// src/core/partition/suggestions.ts
|
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
import { stopUsingTheNativeSearchCore, noteFallback, nativeSearchCoreIsEnabled } from './chunk-DJHHF2L5.mjs';
|
|
2
1
|
import { unpackLevels, readTrailer, stageQueryLevels, osqCodeBytes, osqStagedQueryBytes, osqRecordBytes, writeRecord } from './chunk-37CD3OKS.mjs';
|
|
2
|
+
import { stopUsingTheNativeSearchCore, noteFallback, nativeSearchCoreIsEnabled } from './chunk-DJHHF2L5.mjs';
|
|
3
3
|
import { markSimdUnavailable } from './chunk-KOKM4PQN.mjs';
|
|
4
4
|
import { MAX_WASM_PAGES, WASM_PAGE_BYTES, VECTOR_SCRATCH_SLOTS, UNPLACED_FLAG_CHUNK_ORDINALS, OSQ_STEPS, OSQ_GRID, VECTOR_BLOCK_INITIAL_SLOTS, MAX_VECTOR_ORDINALS, VECTOR_STORE_INITIAL_CAPACITY, LOCK_SPIN_ITERATIONS, LOCK_WAIT_SLICE_MS, OSQ_REFINE_ROUNDS, OSQ_REFINE_TOLERANCE, OSQ_PLACEMENT_QUERY_BITS, GRAPH_LOCK_WAIT_SLICE_MS, OSQ_QUERY_BITS, NATIVE_ATTACH_RETRY_MS, GRAPH_LOCK_WAIT_DEADLINE_MS, OSQ_LAMBDA, MAX_DOC_ID_TABLE_BYTES } from './chunk-QSK35UC2.mjs';
|
|
5
|
-
import { loadNativeCore } from './chunk-
|
|
5
|
+
import { loadNativeCore } from './chunk-74XB2SXI.mjs';
|
|
6
6
|
import { COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, PLACEMENT_ROOM_REQUESTS, MAX_LAYER_CAP, INITIAL_LIST_CAPACITY, INITIAL_UPPER_SLOTS, DEFAULT_EF_SEARCH, NEAREST_LINK_FALLBACK_HOSTS, FARTHEST_REJECTED_LINKS_KEPT } from './chunk-VJS4JFBQ.mjs';
|
|
7
7
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
8
8
|
import { threadSlotGone } from './chunk-RFBLQLLZ.mjs';
|
|
9
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
9
|
+
import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
|
|
10
10
|
|
|
11
11
|
// src/vector/shared-buffers/growable.ts
|
|
12
12
|
function sharedMemoryAvailable() {
|
|
@@ -876,6 +876,36 @@ function replaceNeighbors(adj, ord, layer, neighbors, count) {
|
|
|
876
876
|
writeNeighbors(adj, ord, 0, neighbors, count);
|
|
877
877
|
}
|
|
878
878
|
|
|
879
|
+
// src/vector/hnsw/insertion-order.ts
|
|
880
|
+
var GOLDEN_GAMMA = 2654435769;
|
|
881
|
+
var FIRST_MIX_MULTIPLIER = 569420461;
|
|
882
|
+
var SECOND_MIX_MULTIPLIER = 1935289751;
|
|
883
|
+
var TWO_TO_THE_32 = 4294967296;
|
|
884
|
+
var HALF_A_STEP = 0.5;
|
|
885
|
+
function mixedUint32(value) {
|
|
886
|
+
let mixed = value + GOLDEN_GAMMA | 0;
|
|
887
|
+
mixed ^= mixed >>> 16;
|
|
888
|
+
mixed = Math.imul(mixed, FIRST_MIX_MULTIPLIER);
|
|
889
|
+
mixed ^= mixed >>> 15;
|
|
890
|
+
mixed = Math.imul(mixed, SECOND_MIX_MULTIPLIER);
|
|
891
|
+
mixed ^= mixed >>> 15;
|
|
892
|
+
return mixed >>> 0;
|
|
893
|
+
}
|
|
894
|
+
function unitDrawOf(value) {
|
|
895
|
+
return (mixedUint32(value) + HALF_A_STEP) / TWO_TO_THE_32;
|
|
896
|
+
}
|
|
897
|
+
function shuffleInPlace(items) {
|
|
898
|
+
let stream = items.length;
|
|
899
|
+
for (let i = items.length - 1; i > 0; i--) {
|
|
900
|
+
stream = mixedUint32(stream);
|
|
901
|
+
const j = Math.floor(stream / TWO_TO_THE_32 * (i + 1));
|
|
902
|
+
const held = items[i];
|
|
903
|
+
items[i] = items[j];
|
|
904
|
+
items[j] = held;
|
|
905
|
+
}
|
|
906
|
+
return items;
|
|
907
|
+
}
|
|
908
|
+
|
|
879
909
|
// src/vector/hnsw/shared.ts
|
|
880
910
|
function entryPointOf(state) {
|
|
881
911
|
return Atomics.load(state.header, GRAPH_ENTRY_POINT);
|
|
@@ -944,10 +974,8 @@ function toScore(distance, metric) {
|
|
|
944
974
|
return 1 / (1 + distance);
|
|
945
975
|
}
|
|
946
976
|
}
|
|
947
|
-
function
|
|
948
|
-
|
|
949
|
-
if (u === 0) u = Number.MIN_VALUE;
|
|
950
|
-
return Math.min(Math.floor(-Math.log(u) * mL), MAX_LAYER_CAP);
|
|
977
|
+
function levelOfOrdinal(ord, mL) {
|
|
978
|
+
return Math.min(Math.floor(-Math.log(unitDrawOf(ord)) * mL), MAX_LAYER_CAP);
|
|
951
979
|
}
|
|
952
980
|
function entryForOrd(state, ord) {
|
|
953
981
|
return state.store.entryForOrdinal(ord);
|
|
@@ -2300,17 +2328,6 @@ function everyCandidateWasHeld() {
|
|
|
2300
2328
|
return true;
|
|
2301
2329
|
}
|
|
2302
2330
|
|
|
2303
|
-
// src/vector/hnsw/insertion-order.ts
|
|
2304
|
-
function shuffleInPlace(items) {
|
|
2305
|
-
for (let i = items.length - 1; i > 0; i--) {
|
|
2306
|
-
const j = Math.floor(Math.random() * (i + 1));
|
|
2307
|
-
const held = items[i];
|
|
2308
|
-
items[i] = items[j];
|
|
2309
|
-
items[j] = held;
|
|
2310
|
-
}
|
|
2311
|
-
return items;
|
|
2312
|
-
}
|
|
2313
|
-
|
|
2314
2331
|
// src/vector/hnsw/mutation.ts
|
|
2315
2332
|
function clearTombstone(state, ord) {
|
|
2316
2333
|
if (!reachTombstone(state, ord) || Atomics.load(state.tombstones, ord) !== 1) return;
|
|
@@ -2438,7 +2455,7 @@ function writeRecordBeforePlacement(state, ord) {
|
|
|
2438
2455
|
function placeThroughTheCore(state, ord, holdsGraphLock) {
|
|
2439
2456
|
let field = nativeFieldFor(state);
|
|
2440
2457
|
if (field === null) return false;
|
|
2441
|
-
const level2 =
|
|
2458
|
+
const level2 = levelOfOrdinal(ord, state.mL);
|
|
2442
2459
|
for (let request = 0; request < PLACEMENT_ROOM_REQUESTS && field !== null; request++) {
|
|
2443
2460
|
ensureUpperRoom(state.adjacency, level2);
|
|
2444
2461
|
const placement = nativePlace(state, field, state.buildMetric, ord, level2, holdsGraphLock);
|
|
@@ -2465,7 +2482,7 @@ function insertNode(state, ord, holdsGraphLock = false) {
|
|
|
2465
2482
|
if (!holdsGraphLock) lockGraphShared(state.locks);
|
|
2466
2483
|
try {
|
|
2467
2484
|
clearTombstone(state, ord);
|
|
2468
|
-
const level2 =
|
|
2485
|
+
const level2 = levelOfOrdinal(ord, state.mL);
|
|
2469
2486
|
createNode(state.adjacency, ord, level2);
|
|
2470
2487
|
Atomics.add(state.header, GRAPH_NODE_COUNT, 1);
|
|
2471
2488
|
if (!claimEntryIfEmpty(state, ord, level2)) linkNode(state, ord, level2);
|