@delali/narsil 0.3.1-next.161 → 0.3.1-next.163
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/{chunk-4RNSFVWI.mjs → chunk-KQXKCMGQ.mjs} +7 -5
- package/dist/browser/{core-leaf-TJAGWPZZ.mjs → core-leaf-SIWXFZLV.mjs} +66 -4
- package/dist/browser/{module-362VSIGR.mjs → module-PMQBRESM.mjs} +167 -45
- package/dist/browser/{storage-leaf-X6CNRHJS.mjs → storage-leaf-VGR2BWJJ.mjs} +35 -26
- package/dist/browser/{store-leaf-AEMTAVHX.mjs → store-leaf-3L33V35P.mjs} +3 -2
- package/dist/browser/{suggestions-leaf-3Y6MR6OF.mjs → suggestions-leaf-UBNDVKBD.mjs} +3 -1
- package/dist/{capabilities-BKDrry0z.d.ts → capabilities-DkXAkb-X.d.ts} +2 -2
- package/dist/{chunk-NC4QLCZC.mjs → chunk-3UGZBRDN.mjs} +1 -1
- package/dist/{chunk-75UNINFA.mjs → chunk-5DUTOEK5.mjs} +1 -1
- package/dist/{chunk-U6B7DUQW.mjs → chunk-5P72V66D.mjs} +4 -4
- package/dist/{chunk-TSLV5P2J.mjs → chunk-7OE73OO4.mjs} +6 -10
- package/dist/{chunk-GF4K7IUK.mjs → chunk-BKOMO7HT.mjs} +5 -5
- package/dist/{chunk-SXIRM4MX.mjs → chunk-D5GNFN6X.mjs} +26 -15
- package/dist/{chunk-SHGN3QIF.mjs → chunk-DSXFN5BF.mjs} +4 -4
- package/dist/{chunk-TINW4BSH.mjs → chunk-EEYDJB4Z.mjs} +17 -14
- package/dist/{chunk-2RIS76RJ.mjs → chunk-EI7U43MX.mjs} +1 -1
- package/dist/{chunk-LF4GB4PO.mjs → chunk-EIR3IQCL.mjs} +1 -1
- package/dist/{chunk-2JRERUNZ.mjs → chunk-ES5EOXT6.mjs} +2 -2
- package/dist/{chunk-FVZ5DB6Y.mjs → chunk-FSLQHOKZ.mjs} +1 -1
- package/dist/{chunk-W34VJXY7.mjs → chunk-GJKFXKMI.mjs} +4 -4
- package/dist/{chunk-6EZW2GQL.mjs → chunk-GQLOEACG.mjs} +4 -3
- package/dist/{chunk-H64YDF5M.mjs → chunk-K4BMDPPX.mjs} +1 -1
- package/dist/{chunk-7MWYORQC.mjs → chunk-KSOZJEYU.mjs} +1 -1
- package/dist/{chunk-BXM7XM4G.mjs → chunk-NXMGUDZA.mjs} +2 -2
- package/dist/{chunk-P5YLI5ED.mjs → chunk-Q2WQSAYH.mjs} +1 -1
- package/dist/{chunk-V3SGPXWQ.mjs → chunk-Q7CUUSJ2.mjs} +2 -2
- package/dist/{chunk-3D6RZGJH.mjs → chunk-R4F7ZTVG.mjs} +2 -2
- package/dist/{chunk-OOO54OVC.mjs → chunk-RXMR2SIQ.mjs} +3 -3
- package/dist/{chunk-HOOR4CP5.mjs → chunk-STLNPJXU.mjs} +2 -2
- package/dist/{chunk-FE3MRIBN.mjs → chunk-TOZENMM7.mjs} +5 -5
- package/dist/{chunk-PTTMGRG5.mjs → chunk-V4YIEMAI.mjs} +2 -2
- package/dist/{chunk-LAWIJRMJ.mjs → chunk-VJOS6MHF.mjs} +67 -5
- package/dist/{chunk-K67I3RF4.mjs → chunk-XRBPHCCT.mjs} +2 -2
- package/dist/{chunk-E2F6LZR3.mjs → chunk-XXWEUXE4.mjs} +1 -1
- package/dist/{react-native/chunks/chunk-ZIU6TWXG.mjs → chunk-ZBVHS5KF.mjs} +1 -1
- package/dist/{chunk-IH3G6SJO.mjs → chunk-ZGYF37MG.mjs} +1 -1
- package/dist/{client-CTp8RRiK.d.ts → client-VJmV_AjZ.d.ts} +3 -3
- package/dist/client.d.ts +6 -6
- package/dist/{config-_WT5grSL.d.ts → config-Cb3FOlE_.d.ts} +1 -1
- package/dist/{config-PCNzWjHL.d.ts → config-ZZg4cwQz.d.ts} +1 -1
- package/dist/{core-leaf-KXFGP74A.mjs → core-leaf-LOXCD7MY.mjs} +2 -2
- package/dist/{direct-executor-MKTZBR26.mjs → direct-executor-PF4UDSUW.mjs} +2 -2
- package/dist/distribution.d.ts +4 -4
- package/dist/distribution.mjs +31 -32
- package/dist/{engine-DBRVR_d-.d.ts → engine-DdOs682x.d.ts} +9 -1
- package/dist/index.browser.mjs +1 -1
- package/dist/index.d.ts +4 -4
- package/dist/index.mjs +2 -2
- package/dist/index.react-native.mjs +4 -4
- package/dist/{long-query-thread-JMGNGF6V.mjs → long-query-thread-OTVLP2OI.mjs} +12 -12
- package/dist/{module-ZJ6ARG6V.mjs → module-UIKSHY3G.mjs} +173 -51
- package/dist/{optional-reads-BR3BOVZW.mjs → optional-reads-RSLNSGNG.mjs} +1 -1
- package/dist/{optional-writes-DDFHYUOH.mjs → optional-writes-4ITCV7YT.mjs} +4 -4
- package/dist/{paged-partition-archive-G576XGVE.mjs → paged-partition-archive-VMJ2AUA6.mjs} +11 -11
- package/dist/partial-words.browser.mjs +3 -2
- package/dist/partial-words.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.mjs +9 -9
- package/dist/{phrase-module-XK3T44VU.mjs → phrase-module-IWSG2DDZ.mjs} +1 -1
- package/dist/{query-actions-OMXJEC2J.mjs → query-actions-EW72OQXY.mjs} +2 -2
- package/dist/query-suggestions/merge-worker.mjs +5 -5
- package/dist/query-suggestions.browser.mjs +1 -1
- package/dist/query-suggestions.d.ts +1 -1
- package/dist/query-suggestions.mjs +3 -3
- package/dist/react-native/chunks/{chunk-52QD7CBJ.mjs → chunk-2JMAUU26.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-72OKYHZL.mjs → chunk-3RUXKZJG.mjs} +4 -3
- package/dist/react-native/chunks/{chunk-L4R5ITVG.mjs → chunk-AUXXPSFB.mjs} +17 -15
- package/dist/{chunk-IB4BYLE3.mjs → react-native/chunks/chunk-HEELWLKL.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-IY6U6HRA.mjs → chunk-I5W7A2JF.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-AQOWRH2C.mjs → chunk-RVHCQJQM.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-XEGKQKLY.mjs → chunk-VIPNDN4B.mjs} +67 -5
- package/dist/react-native/chunks/{chunk-G3CUQ333.mjs → chunk-XBTFK3AZ.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-J2NQW353.mjs → chunk-XOCM3BFU.mjs} +1 -1
- package/dist/react-native/chunks/{core-leaf-5B4JXCVA.mjs → core-leaf-O3236QY3.mjs} +2 -2
- package/dist/react-native/chunks/{module-VUA27HZZ.mjs → module-LRR7D5ZO.mjs} +168 -46
- package/dist/react-native/chunks/{optional-reads-WDTLLHUT.mjs → optional-reads-TOWGNEDH.mjs} +1 -1
- package/dist/react-native/chunks/{optional-writes-FZV3QW34.mjs → optional-writes-7AFBIZMN.mjs} +5 -5
- package/dist/react-native/chunks/{phrase-module-KCQWVXYZ.mjs → phrase-module-3A4AKMKL.mjs} +1 -1
- package/dist/react-native/chunks/{restore-queue-EI2V2I4S.mjs → restore-queue-7FSHTGJ5.mjs} +7 -7
- package/dist/react-native/chunks/{segment-codec-leaf-6ULP7H3I.mjs → segment-codec-leaf-4QV4HO4R.mjs} +5 -5
- package/dist/react-native/chunks/{segment-merge-leaf-F4GQZNK6.mjs → segment-merge-leaf-5DUZZSXJ.mjs} +6 -6
- package/dist/react-native/chunks/{storage-leaf-OAXBU2L6.mjs → storage-leaf-PNU6HWY7.mjs} +42 -33
- package/dist/react-native/chunks/{store-leaf-GY2ZZFXB.mjs → store-leaf-ESVUGYYO.mjs} +6 -6
- package/dist/react-native/chunks/{suggestions-leaf-SQEZ4IBS.mjs → suggestions-leaf-MOVOPBGD.mjs} +7 -7
- package/dist/react-native/chunks/{worker-copies-EEHSCFJW.mjs → worker-copies-VQGF3WZX.mjs} +5 -5
- package/dist/react-native/partial-words.mjs +6 -5
- package/dist/react-native/pattern.mjs +1 -1
- package/dist/react-native/query-suggestions.mjs +4 -4
- package/dist/react-native/sync.mjs +4 -4
- package/dist/react-native/synonyms.mjs +4 -4
- package/dist/react.d.ts +7 -7
- package/dist/{rerank-Y6MWBMTT.mjs → rerank-QHJXPSAK.mjs} +2 -2
- package/dist/{restore-queue-KPYSRJ4A.mjs → restore-queue-5W5HX44H.mjs} +5 -5
- package/dist/{segment-codec-leaf-6IFBSBPN.mjs → segment-codec-leaf-7VLVXKWO.mjs} +3 -3
- package/dist/{segment-merge-leaf-VE3FUSVE.mjs → segment-merge-leaf-22XR3IGA.mjs} +3 -3
- package/dist/server.d.ts +8 -8
- package/dist/server.mjs +17 -17
- package/dist/{storage-leaf-HMIIOUYN.mjs → storage-leaf-ZZ77C3DX.mjs} +37 -35
- package/dist/{store-leaf-M7T7SUZJ.mjs → store-leaf-LJ6D2PMF.mjs} +4 -4
- package/dist/{suggestions-leaf-Q3Q5D6XV.mjs → suggestions-leaf-KTQHO3PY.mjs} +6 -6
- package/dist/sync/filesystem.d.ts +2 -2
- package/dist/sync/filesystem.mjs +6 -6
- package/dist/sync/indexeddb.d.ts +2 -2
- package/dist/sync.browser.mjs +1 -1
- package/dist/sync.d.ts +3 -3
- package/dist/sync.mjs +2 -2
- package/dist/synonyms.browser.mjs +1 -1
- package/dist/synonyms.mjs +2 -2
- package/dist/{tasks-B4tT4vmP.d.ts → tasks-CxA-J_Ty.d.ts} +2 -2
- package/dist/{thread-SNILJXX3.mjs → thread-XWDK6TAR.mjs} +11 -11
- package/dist/{types-CTg0KaMy.d.ts → types-BboB278g.d.ts} +1 -1
- package/dist/vector/search-worker.mjs +4 -4
- package/dist/{worker-copies-ZLZS2CUM.mjs → worker-copies-ODAAGQRC.mjs} +5 -5
- package/dist/workers/entry.mjs +8 -8
- package/openapi.json +2 -2
- package/package.json +7 -7
- /package/dist/{chunk-YETKPX5M.mjs → chunk-T4HMSZCX.mjs} +0 -0
|
@@ -268,6 +268,7 @@ var recordCheckpoint;
|
|
|
268
268
|
var releaseLocksHeldBy;
|
|
269
269
|
var releasePinsHeldBy;
|
|
270
270
|
var releaseReplacedVectors;
|
|
271
|
+
var requireUsableVectorOptions;
|
|
271
272
|
var requireWholeNeighbourLists;
|
|
272
273
|
var resetStoreHeader;
|
|
273
274
|
var retiredAtOf;
|
|
@@ -342,6 +343,7 @@ function bindVectorCore2Names(vectorCore) {
|
|
|
342
343
|
releaseLocksHeldBy,
|
|
343
344
|
releasePinsHeldBy,
|
|
344
345
|
releaseReplacedVectors,
|
|
346
|
+
requireUsableVectorOptions,
|
|
345
347
|
requireWholeNeighbourLists,
|
|
346
348
|
resetStoreHeader,
|
|
347
349
|
retiredAtOf,
|
|
@@ -527,18 +529,6 @@ function withFusedScore(entry, score) {
|
|
|
527
529
|
}
|
|
528
530
|
|
|
529
531
|
// src/engine/query/vector.ts
|
|
530
|
-
function oversampleOf(vectorConfig) {
|
|
531
|
-
const oversample = vectorConfig.oversample;
|
|
532
|
-
if (oversample === void 0) return void 0;
|
|
533
|
-
if (typeof oversample !== "number" || !Number.isFinite(oversample) || oversample < MIN_OVERSAMPLE) {
|
|
534
|
-
throw new NarsilError(
|
|
535
|
-
ErrorCodes.CONFIG_INVALID,
|
|
536
|
-
`vector.oversample must be a finite number of at least ${MIN_OVERSAMPLE}`,
|
|
537
|
-
{ oversample }
|
|
538
|
-
);
|
|
539
|
-
}
|
|
540
|
-
return oversample;
|
|
541
|
-
}
|
|
542
532
|
function vectorFetchDepth(limit, offset, cursorDepth) {
|
|
543
533
|
const reach = cursorDepth + offset + limit;
|
|
544
534
|
if (reach > RESULT_WINDOW) {
|
|
@@ -576,7 +566,7 @@ async function searchField(search2, k, host) {
|
|
|
576
566
|
metric: vectorConfig.metric ?? "cosine",
|
|
577
567
|
minSimilarity: vectorConfig.similarity ?? -Infinity,
|
|
578
568
|
efSearch: vectorConfig.efSearch,
|
|
579
|
-
oversample:
|
|
569
|
+
oversample: vectorConfig.oversample,
|
|
580
570
|
...rescoreWindow !== void 0 ? { rescoreWindow } : {}
|
|
581
571
|
};
|
|
582
572
|
if (filtered !== void 0) {
|
|
@@ -3321,7 +3311,7 @@ async function insertIntoGraph(state, graph, docIds, admit) {
|
|
|
3321
3311
|
const dispatch = found;
|
|
3322
3312
|
const pending = /* @__PURE__ */ new Set();
|
|
3323
3313
|
const limit = dispatch.workerCount * GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER;
|
|
3324
|
-
const fallback = { noWorkerWarned: false, shortAnswerWarned: false };
|
|
3314
|
+
const fallback = { noWorkerWarned: false, shortAnswerWarned: false, workerAnswerFailed: false };
|
|
3325
3315
|
let batchDocIds = [];
|
|
3326
3316
|
let batchOrdinals = [];
|
|
3327
3317
|
async function flush() {
|
|
@@ -3337,6 +3327,7 @@ async function insertIntoGraph(state, graph, docIds, admit) {
|
|
|
3337
3327
|
pending.add(run);
|
|
3338
3328
|
if (pending.size >= limit) await Promise.race(pending);
|
|
3339
3329
|
}
|
|
3330
|
+
state.graphsPlacedIntoSinceConnect.add(graph);
|
|
3340
3331
|
for (const docId of docIds) {
|
|
3341
3332
|
if (state.disposed) break;
|
|
3342
3333
|
if (!admit(docId)) continue;
|
|
@@ -3352,6 +3343,7 @@ async function insertIntoGraph(state, graph, docIds, admit) {
|
|
|
3352
3343
|
if (turnsAreBudgeted) await connectGraph(state, graph);
|
|
3353
3344
|
else graph.connectUnreached();
|
|
3354
3345
|
if (state.disposed) return false;
|
|
3346
|
+
if (!fallback.workerAnswerFailed) state.graphsPlacedIntoSinceConnect.delete(graph);
|
|
3355
3347
|
noteWrite(state);
|
|
3356
3348
|
return true;
|
|
3357
3349
|
}
|
|
@@ -3361,6 +3353,7 @@ async function placeChunk(state, graph, dispatch, chunkDocIds, chunkOrdinals, fa
|
|
|
3361
3353
|
outcome = await dispatch.send(chunkOrdinals);
|
|
3362
3354
|
} catch {
|
|
3363
3355
|
outcome = null;
|
|
3356
|
+
fallback.workerAnswerFailed = true;
|
|
3364
3357
|
}
|
|
3365
3358
|
if (state.disposed) return;
|
|
3366
3359
|
if (outcome === null) {
|
|
@@ -3397,6 +3390,7 @@ async function insertLocally(state, graph, docIds, admit) {
|
|
|
3397
3390
|
if (!admit(docId)) continue;
|
|
3398
3391
|
const ordinal = state.store.getOrdinal(docId);
|
|
3399
3392
|
if (ordinal === void 0) continue;
|
|
3393
|
+
state.graphsPlacedIntoSinceConnect.add(graph);
|
|
3400
3394
|
if (turnsAreBudgeted) {
|
|
3401
3395
|
graph.insertOrdinal(ordinal, true);
|
|
3402
3396
|
if (graph.holdsUnlinkedNode) await connectGraph(state, graph);
|
|
@@ -3419,6 +3413,7 @@ async function insertLocally(state, graph, docIds, admit) {
|
|
|
3419
3413
|
if (count > 0) {
|
|
3420
3414
|
if (turnsAreBudgeted) await connectGraph(state, graph);
|
|
3421
3415
|
else graph.connectUnreached();
|
|
3416
|
+
state.graphsPlacedIntoSinceConnect.delete(graph);
|
|
3422
3417
|
noteWrite(state);
|
|
3423
3418
|
}
|
|
3424
3419
|
return true;
|
|
@@ -3641,6 +3636,8 @@ async function releasePendingLocations(state) {
|
|
|
3641
3636
|
const paths = new Set(pending.map(([, location]) => location.path));
|
|
3642
3637
|
await releaseInTurn(state, () => releaseLocations(state, paths, pending));
|
|
3643
3638
|
}
|
|
3639
|
+
function chargeHeapBytes(bytes) {
|
|
3640
|
+
}
|
|
3644
3641
|
|
|
3645
3642
|
// src/runtime/background-memory.ts
|
|
3646
3643
|
var admissions = /* @__PURE__ */ new Set();
|
|
@@ -3655,15 +3652,21 @@ function noteWaitingForMemory(owner, work) {
|
|
|
3655
3652
|
if (work === null) waiting.delete(owner);
|
|
3656
3653
|
else waiting.set(owner, work);
|
|
3657
3654
|
}
|
|
3658
|
-
function chargeHeapBytes(bytes) {
|
|
3659
|
-
}
|
|
3660
3655
|
|
|
3661
3656
|
// src/vector/vector-index/memory-charge.ts
|
|
3657
|
+
function waitForMemory(state) {
|
|
3658
|
+
state.waitingForMemory = true;
|
|
3659
|
+
noteWaitingForMemory(state, { indexName: state.indexName, fieldName: state.fieldName });
|
|
3660
|
+
}
|
|
3661
|
+
function admitGrowingCopy(state, graph) {
|
|
3662
|
+
if (admitsBackgroundBytes(graph.graphBytes)) return true;
|
|
3663
|
+
waitForMemory(state);
|
|
3664
|
+
return false;
|
|
3665
|
+
}
|
|
3662
3666
|
function admitShuffledRebuild(state) {
|
|
3663
3667
|
const bytes = state.hnsw?.graphBytes ?? 0;
|
|
3664
3668
|
if (!admitsBackgroundBytes(bytes)) {
|
|
3665
|
-
state
|
|
3666
|
-
noteWaitingForMemory(state, { indexName: state.indexName, fieldName: state.fieldName });
|
|
3669
|
+
waitForMemory(state);
|
|
3667
3670
|
return false;
|
|
3668
3671
|
}
|
|
3669
3672
|
stopWaitingForMemory(state);
|
|
@@ -3687,6 +3690,62 @@ function releaseMemoryCharges(state) {
|
|
|
3687
3690
|
state.rebuildWaitTimer = null;
|
|
3688
3691
|
}
|
|
3689
3692
|
|
|
3693
|
+
// src/vector/vector-index/growing-copy.ts
|
|
3694
|
+
var lineages = /* @__PURE__ */ new WeakMap();
|
|
3695
|
+
function lineageOfGraph(graph) {
|
|
3696
|
+
const known = lineages.get(graph);
|
|
3697
|
+
if (known !== void 0) return known;
|
|
3698
|
+
const lineage = {};
|
|
3699
|
+
lineages.set(graph, lineage);
|
|
3700
|
+
return lineage;
|
|
3701
|
+
}
|
|
3702
|
+
function lineageOf(graph) {
|
|
3703
|
+
return graph === null ? null : lineageOfGraph(graph);
|
|
3704
|
+
}
|
|
3705
|
+
function moveWhatTheCopyHoldsOutOfBuffer(state, copy) {
|
|
3706
|
+
for (const docId of [...state.buffer]) {
|
|
3707
|
+
const ordinal = state.store.getOrdinal(docId);
|
|
3708
|
+
if (ordinal !== void 0 && copy.holdsNode(ordinal) && !copy.holdsTombstone(ordinal)) moveOutOfBuffer(state, docId);
|
|
3709
|
+
}
|
|
3710
|
+
}
|
|
3711
|
+
async function publishCopy(state, graph, copy) {
|
|
3712
|
+
if (state.sharedHandles.get(graph)?.searchable === true) await shareGraph(state, copy, true);
|
|
3713
|
+
if (state.disposed || state.hnsw !== graph) return false;
|
|
3714
|
+
lineages.set(copy, lineageOfGraph(graph));
|
|
3715
|
+
state.hnsw = copy;
|
|
3716
|
+
graph.retire();
|
|
3717
|
+
moveWhatTheCopyHoldsOutOfBuffer(state, copy);
|
|
3718
|
+
dropSharedGraph(state, graph);
|
|
3719
|
+
noteWrite(state);
|
|
3720
|
+
return true;
|
|
3721
|
+
}
|
|
3722
|
+
function* startingWith(first, rest) {
|
|
3723
|
+
yield first;
|
|
3724
|
+
for (let next = rest.next(); !next.done; next = rest.next()) yield next.value;
|
|
3725
|
+
}
|
|
3726
|
+
async function placeInGrowingCopy(state, graph, docIds, admit) {
|
|
3727
|
+
const pending = docIds[Symbol.iterator]();
|
|
3728
|
+
let first = pending.next();
|
|
3729
|
+
while (!first.done && !admit(first.value)) first = pending.next();
|
|
3730
|
+
if (first.done) return "nothingToPlace";
|
|
3731
|
+
if (!admitGrowingCopy(state, graph)) return "waitingForMemory";
|
|
3732
|
+
const copy = graph.copy();
|
|
3733
|
+
copy.graphBytes;
|
|
3734
|
+
state.growingCopy = copy;
|
|
3735
|
+
let published = false;
|
|
3736
|
+
try {
|
|
3737
|
+
const placed = await insertIntoGraph(state, copy, startingWith(first.value, pending), admit);
|
|
3738
|
+
published = placed && !state.disposed && state.hnsw === graph && await publishCopy(state, graph, copy);
|
|
3739
|
+
return published ? "published" : "dropped";
|
|
3740
|
+
} finally {
|
|
3741
|
+
if (state.growingCopy === copy) state.growingCopy = null;
|
|
3742
|
+
if (!published) {
|
|
3743
|
+
copy.retire();
|
|
3744
|
+
dropSharedGraph(state, copy);
|
|
3745
|
+
}
|
|
3746
|
+
}
|
|
3747
|
+
}
|
|
3748
|
+
|
|
3690
3749
|
// src/vector/vector-index/build.ts
|
|
3691
3750
|
function bufferIfTheGraphLacks(state, graph, docId) {
|
|
3692
3751
|
const ordinal = state.store.getOrdinal(docId);
|
|
@@ -3701,9 +3760,9 @@ async function bufferWhatTheGraphLacksInTurns(state, graph) {
|
|
|
3701
3760
|
await forEachInTurns(allLiveDocIds(state), visit, () => !state.disposed && state.hnsw === graph);
|
|
3702
3761
|
}
|
|
3703
3762
|
async function buildGraphFromStore(state) {
|
|
3704
|
-
const
|
|
3763
|
+
const lineage = lineageOf(state.hnsw);
|
|
3705
3764
|
const order = turnsAreBudgeted ? await shuffledLiveDocIdsInTurns(state) : shuffleInPlace([...allLiveDocIds(state)]);
|
|
3706
|
-
if (order === null || state.hnsw !==
|
|
3765
|
+
if (order === null || lineageOf(state.hnsw) !== lineage) return false;
|
|
3707
3766
|
const graph = createHNSWIndex(state.dimension, state.store, state.hnswConfig, state.osq ?? void 0);
|
|
3708
3767
|
state.freshGraph = graph;
|
|
3709
3768
|
let completed = false;
|
|
@@ -3712,16 +3771,17 @@ async function buildGraphFromStore(state) {
|
|
|
3712
3771
|
} finally {
|
|
3713
3772
|
state.freshGraph = null;
|
|
3714
3773
|
}
|
|
3715
|
-
if (!completed || state.hnsw !==
|
|
3774
|
+
if (!completed || lineageOf(state.hnsw) !== lineage) {
|
|
3716
3775
|
dropSharedGraph(state, graph);
|
|
3717
3776
|
return false;
|
|
3718
3777
|
}
|
|
3778
|
+
const replaced = state.hnsw;
|
|
3719
3779
|
adoptGraph(state, graph);
|
|
3720
3780
|
state.shuffledBuildCount = graph.size;
|
|
3721
3781
|
noteWrite(state);
|
|
3722
3782
|
if (turnsAreBudgeted) await bufferWhatTheGraphLacksInTurns(state, graph);
|
|
3723
3783
|
else bufferWhatTheGraphLacks(state, graph);
|
|
3724
|
-
if (
|
|
3784
|
+
if (replaced !== null) dropSharedGraph(state, replaced);
|
|
3725
3785
|
await releasePendingLocations(state);
|
|
3726
3786
|
return true;
|
|
3727
3787
|
}
|
|
@@ -3738,10 +3798,31 @@ async function recalibratesForRebuild(state, graph) {
|
|
|
3738
3798
|
if (state.shuffledBuildCount === null) return calibrateBeforeBuild(state);
|
|
3739
3799
|
return recalibrateWhenDrifted(state, graph.size);
|
|
3740
3800
|
}
|
|
3741
|
-
async function
|
|
3801
|
+
async function keepBuildsOut(state) {
|
|
3802
|
+
while (state.buildsKeptOut !== null) await state.buildsKeptOut.ended;
|
|
3803
|
+
if (state.disposed) return () => void 0;
|
|
3804
|
+
let end = () => void 0;
|
|
3805
|
+
const ended = new Promise((resolve) => {
|
|
3806
|
+
end = resolve;
|
|
3807
|
+
});
|
|
3808
|
+
const kept = { ended, end };
|
|
3809
|
+
state.buildsKeptOut = kept;
|
|
3810
|
+
return () => {
|
|
3811
|
+
if (state.buildsKeptOut !== kept) return;
|
|
3812
|
+
letBuildsIn(state);
|
|
3813
|
+
scheduleBuild(state);
|
|
3814
|
+
};
|
|
3815
|
+
}
|
|
3816
|
+
function letBuildsIn(state) {
|
|
3817
|
+
const kept = state.buildsKeptOut;
|
|
3818
|
+
state.buildsKeptOut = null;
|
|
3819
|
+
kept?.end();
|
|
3820
|
+
}
|
|
3821
|
+
async function buildExclusively(state, work, waitsForShuffledRebuild = true, keepsBuildsOut = false) {
|
|
3742
3822
|
for (; ; ) {
|
|
3743
3823
|
if (state.pendingBuild !== null) await state.pendingBuild;
|
|
3744
3824
|
else if (waitsForShuffledRebuild && state.shuffledRebuild !== null) await state.shuffledRebuild;
|
|
3825
|
+
else if (!keepsBuildsOut && state.buildsKeptOut !== null) await state.buildsKeptOut.ended;
|
|
3745
3826
|
else break;
|
|
3746
3827
|
}
|
|
3747
3828
|
if (state.disposed) return;
|
|
@@ -3760,7 +3841,10 @@ async function buildExclusively(state, work, waitsForShuffledRebuild = true) {
|
|
|
3760
3841
|
}
|
|
3761
3842
|
async function rebuildInShuffledOrder(state) {
|
|
3762
3843
|
await yieldBuildTurn(state);
|
|
3763
|
-
const replacing = {
|
|
3844
|
+
const replacing = {
|
|
3845
|
+
lineage: null,
|
|
3846
|
+
loadedWithoutCount: false
|
|
3847
|
+
};
|
|
3764
3848
|
await buildExclusively(
|
|
3765
3849
|
state,
|
|
3766
3850
|
async () => {
|
|
@@ -3768,12 +3852,12 @@ async function rebuildInShuffledOrder(state) {
|
|
|
3768
3852
|
if (graph === null || !shuffledRebuildDue(state)) return;
|
|
3769
3853
|
const loadedWithoutCount = state.shuffledBuildCount === null;
|
|
3770
3854
|
if (!await recalibratesForRebuild(state, graph) || state.hnsw !== graph) return;
|
|
3771
|
-
replacing.
|
|
3855
|
+
replacing.lineage = lineageOf(graph);
|
|
3772
3856
|
replacing.loadedWithoutCount = loadedWithoutCount;
|
|
3773
3857
|
},
|
|
3774
3858
|
false
|
|
3775
3859
|
);
|
|
3776
|
-
if (replacing.
|
|
3860
|
+
if (replacing.lineage === null || lineageOf(state.hnsw) !== replacing.lineage) return;
|
|
3777
3861
|
const adopted = await buildGraphFromStore(state);
|
|
3778
3862
|
if (adopted && replacing.loadedWithoutCount) state.onLoadedGraphRepaired();
|
|
3779
3863
|
}
|
|
@@ -3781,11 +3865,16 @@ function retryWhenMemoryAllows(state) {
|
|
|
3781
3865
|
if (state.rebuildWaitTimer !== null || state.disposed) return;
|
|
3782
3866
|
const timer = setTimeout(() => {
|
|
3783
3867
|
state.rebuildWaitTimer = null;
|
|
3784
|
-
|
|
3868
|
+
if (state.hnsw !== null && state.buffer.size > 0) placeWhatWaitedForMemory(state);
|
|
3869
|
+
else startShuffledRebuild(state);
|
|
3785
3870
|
}, SHUFFLED_REBUILD_MEMORY_RETRY_MS);
|
|
3786
3871
|
timer.unref?.();
|
|
3787
3872
|
state.rebuildWaitTimer = timer;
|
|
3788
3873
|
}
|
|
3874
|
+
function placeWhatWaitedForMemory(state) {
|
|
3875
|
+
if (state.building || state.buildsKeptOut !== null) retryWhenMemoryAllows(state);
|
|
3876
|
+
else triggerBuild(state);
|
|
3877
|
+
}
|
|
3789
3878
|
function startShuffledRebuild(state) {
|
|
3790
3879
|
if (state.shuffledRebuild !== null || state.disposed) return;
|
|
3791
3880
|
if (!shuffledRebuildDue(state)) {
|
|
@@ -3818,11 +3907,14 @@ async function growGraph(state, graph, bufferSnapshot) {
|
|
|
3818
3907
|
return true;
|
|
3819
3908
|
};
|
|
3820
3909
|
if (state.osq && !state.osq.isCalibrated() && !await calibrateBeforeBuild(state)) return;
|
|
3821
|
-
await
|
|
3822
|
-
|
|
3910
|
+
continueAfterGrowth(state, await placeInGrowingCopy(state, graph, bufferSnapshot, admit));
|
|
3911
|
+
}
|
|
3912
|
+
function continueAfterGrowth(state, outcome) {
|
|
3913
|
+
if (outcome === "waitingForMemory") retryWhenMemoryAllows(state);
|
|
3914
|
+
else startShuffledRebuild(state);
|
|
3823
3915
|
}
|
|
3824
3916
|
function triggerBuild(state) {
|
|
3825
|
-
if (state.building) return;
|
|
3917
|
+
if (state.building || state.buildsKeptOut !== null) return;
|
|
3826
3918
|
state.building = true;
|
|
3827
3919
|
const bufferSnapshot = turnsAreBudgeted ? null : new Set(state.buffer);
|
|
3828
3920
|
const existingGraph = state.hnsw;
|
|
@@ -3873,6 +3965,15 @@ function armBuildTimer(state, delayMs, run) {
|
|
|
3873
3965
|
timer.unref?.();
|
|
3874
3966
|
state.buildTimer = timer;
|
|
3875
3967
|
}
|
|
3968
|
+
function connectAfterUnfinishedPass(state, graph) {
|
|
3969
|
+
graph.connectUnreached();
|
|
3970
|
+
state.graphsPlacedIntoSinceConnect.delete(graph);
|
|
3971
|
+
noteWrite(state);
|
|
3972
|
+
}
|
|
3973
|
+
function connectWhereAPassStoppedEarly(state) {
|
|
3974
|
+
const graph = state.hnsw;
|
|
3975
|
+
if (graph !== null && state.graphsPlacedIntoSinceConnect.has(graph)) connectAfterUnfinishedPass(state, graph);
|
|
3976
|
+
}
|
|
3876
3977
|
function retryAfterBackoff(state) {
|
|
3877
3978
|
const delayMs = Math.min(BUILD_RETRY_MAX_DELAY_MS, Math.max(BUILD_RETRY_FIRST_DELAY_MS, state.buildRetryDelayMs * 2));
|
|
3878
3979
|
state.buildRetryDelayMs = delayMs;
|
|
@@ -3880,7 +3981,7 @@ function retryAfterBackoff(state) {
|
|
|
3880
3981
|
state.buildTimer = null;
|
|
3881
3982
|
armBuildTimer(state, delayMs, () => {
|
|
3882
3983
|
const graph = state.hnsw;
|
|
3883
|
-
if (state.building) {
|
|
3984
|
+
if (state.building || state.buildsKeptOut !== null) {
|
|
3884
3985
|
retryAfterBackoff(state);
|
|
3885
3986
|
return;
|
|
3886
3987
|
}
|
|
@@ -3889,8 +3990,7 @@ function retryAfterBackoff(state) {
|
|
|
3889
3990
|
return;
|
|
3890
3991
|
}
|
|
3891
3992
|
try {
|
|
3892
|
-
graph
|
|
3893
|
-
noteWrite(state);
|
|
3993
|
+
connectAfterUnfinishedPass(state, graph);
|
|
3894
3994
|
state.buildRetryDelayMs = 0;
|
|
3895
3995
|
scheduleWorkerCopyLoad(state);
|
|
3896
3996
|
} catch (error) {
|
|
@@ -3902,11 +4002,12 @@ function retryAfterBackoff(state) {
|
|
|
3902
4002
|
function buildDue(state) {
|
|
3903
4003
|
if (state.hnsw === null) return liveSize(state) >= state.promotionThreshold;
|
|
3904
4004
|
if (state.buffer.size === 0) return false;
|
|
4005
|
+
if (state.waitingForMemory && state.rebuildWaitTimer !== null) return false;
|
|
3905
4006
|
if (state.workerCopies.enabled && state.workerCopies.host !== void 0) return true;
|
|
3906
4007
|
return state.buffer.size >= state.promotionThreshold;
|
|
3907
4008
|
}
|
|
3908
4009
|
function scheduleBuild(state) {
|
|
3909
|
-
if (state.building || state.buildScheduled || state.disposed) return;
|
|
4010
|
+
if (state.building || state.buildScheduled || state.disposed || state.buildsKeptOut !== null) return;
|
|
3910
4011
|
if (!buildDue(state)) return;
|
|
3911
4012
|
state.buildScheduled = true;
|
|
3912
4013
|
const turn = turnsAreBudgeted ? yieldBackgroundTurn() : yieldBuildTurn(state);
|
|
@@ -3928,6 +4029,7 @@ function releaseHeldMemory(state) {
|
|
|
3928
4029
|
state.store.release();
|
|
3929
4030
|
state.hnsw = null;
|
|
3930
4031
|
state.freshGraph = null;
|
|
4032
|
+
state.growingCopy = null;
|
|
3931
4033
|
state.osq = null;
|
|
3932
4034
|
state.tombstones.clear();
|
|
3933
4035
|
state.buffer.clear();
|
|
@@ -3938,6 +4040,7 @@ function releaseHeldMemory(state) {
|
|
|
3938
4040
|
}
|
|
3939
4041
|
function disposeIndex(state) {
|
|
3940
4042
|
state.disposed = true;
|
|
4043
|
+
letBuildsIn(state);
|
|
3941
4044
|
if (state.rebuildWaitTimer !== null) clearTimeout(state.rebuildWaitTimer);
|
|
3942
4045
|
state.rebuildWaitTimer = null;
|
|
3943
4046
|
if (state.buildTimer !== null) clearTimeout(state.buildTimer);
|
|
@@ -4481,6 +4584,7 @@ function compact(state) {
|
|
|
4481
4584
|
state.compactedNodeCount += before - state.hnsw.tombstoneCount;
|
|
4482
4585
|
}
|
|
4483
4586
|
state.freshGraph?.compactTombstones(oldestPin);
|
|
4587
|
+
state.growingCopy?.compactTombstones(oldestPin);
|
|
4484
4588
|
for (const docId of state.tombstones) {
|
|
4485
4589
|
const ordinal = state.store.getOrdinal(docId);
|
|
4486
4590
|
if (ordinal !== void 0 && !state.store.mayFreeOrdinal(ordinal, oldestPin)) continue;
|
|
@@ -4497,8 +4601,7 @@ async function insertMissing(state, graph) {
|
|
|
4497
4601
|
const missingOrReplaced = (docId) => !graph.has(docId) || state.buffer.has(docId);
|
|
4498
4602
|
const heldWhenTheCallStarted = [...allLiveDocIds(state)];
|
|
4499
4603
|
const stillLiveAndUnplaced = (docId) => !state.tombstones.has(docId) && missingOrReplaced(docId);
|
|
4500
|
-
await
|
|
4501
|
-
startShuffledRebuild(state);
|
|
4604
|
+
continueAfterGrowth(state, await placeInGrowingCopy(state, graph, heldWhenTheCallStarted, stillLiveAndUnplaced));
|
|
4502
4605
|
}
|
|
4503
4606
|
async function foldIntoGraph(state) {
|
|
4504
4607
|
const rebuildNeeded = graphNeedsRebuild(state);
|
|
@@ -4541,7 +4644,7 @@ async function fillGraph(state) {
|
|
|
4541
4644
|
await insertMissing(state, graph);
|
|
4542
4645
|
}
|
|
4543
4646
|
function completeGraph(state) {
|
|
4544
|
-
return buildExclusively(state, () => fillGraph(state), false);
|
|
4647
|
+
return buildExclusively(state, () => fillGraph(state), false, true);
|
|
4545
4648
|
}
|
|
4546
4649
|
function maintenanceStatus(state) {
|
|
4547
4650
|
const storeSize = state.store.size;
|
|
@@ -4568,8 +4671,8 @@ function flatScanScratchBytes(state) {
|
|
|
4568
4671
|
}
|
|
4569
4672
|
function estimateMemoryBytes(state) {
|
|
4570
4673
|
if (state.hnsw === null) return state.store.memoryBytes() + flatScanScratchBytes(state);
|
|
4571
|
-
const
|
|
4572
|
-
return state.store.memoryBytes() + state.hnsw.graphBytes + state.hnsw.searchScratchBytes +
|
|
4674
|
+
const passGraphBytes = (state.freshGraph?.graphBytes ?? 0) + (state.growingCopy?.graphBytes ?? 0);
|
|
4675
|
+
return state.store.memoryBytes() + state.hnsw.graphBytes + state.hnsw.searchScratchBytes + passGraphBytes;
|
|
4573
4676
|
}
|
|
4574
4677
|
|
|
4575
4678
|
// src/vector/vector-index/publication.ts
|
|
@@ -5029,14 +5132,16 @@ function createVectorWrites(state) {
|
|
|
5029
5132
|
noteWrite(state);
|
|
5030
5133
|
noteSearchedDocWritten(state, docId);
|
|
5031
5134
|
state.tombstones.delete(docId);
|
|
5032
|
-
const graphHoldsPrevious = previous !== void 0 && ((state.hnsw?.holdsNode(previous) ?? false) || (state.freshGraph?.holdsNode(previous) ?? false));
|
|
5135
|
+
const graphHoldsPrevious = previous !== void 0 && ((state.hnsw?.holdsNode(previous) ?? false) || (state.freshGraph?.holdsNode(previous) ?? false) || (state.growingCopy?.holdsNode(previous) ?? false));
|
|
5033
5136
|
state.store.insert(docId, vector, partitionId, graphHoldsPrevious);
|
|
5034
5137
|
if (previous !== void 0) {
|
|
5035
5138
|
const graph = state.hnsw;
|
|
5036
5139
|
const freshGraph = state.freshGraph;
|
|
5140
|
+
const growingCopy = state.growingCopy;
|
|
5037
5141
|
afterGraphPass(state, () => {
|
|
5038
5142
|
graph?.markTombstoneOrdinal(previous);
|
|
5039
5143
|
freshGraph?.markTombstoneOrdinal(previous);
|
|
5144
|
+
growingCopy?.markTombstoneOrdinal(previous);
|
|
5040
5145
|
});
|
|
5041
5146
|
if (!graphHoldsPrevious) state.osq?.removeOrdinal(previous);
|
|
5042
5147
|
}
|
|
@@ -5057,7 +5162,11 @@ function createVectorWrites(state) {
|
|
|
5057
5162
|
if (state.buffer.retiredCount !== 0) forgetRetiredFlags();
|
|
5058
5163
|
state.pendingLocations.delete(docId);
|
|
5059
5164
|
const graph = state.hnsw;
|
|
5060
|
-
|
|
5165
|
+
const growingCopy = state.growingCopy;
|
|
5166
|
+
afterGraphPass(state, () => {
|
|
5167
|
+
graph?.markTombstoneOrdinal(ordinal);
|
|
5168
|
+
growingCopy?.markTombstoneOrdinal(ordinal);
|
|
5169
|
+
});
|
|
5061
5170
|
}
|
|
5062
5171
|
return {
|
|
5063
5172
|
insert(docId, vector, partitionId) {
|
|
@@ -5131,6 +5240,7 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
5131
5240
|
osq: codeBits === null ? null : createOsqQuantizer(dimension, codeBits, metric, store),
|
|
5132
5241
|
hnsw: null,
|
|
5133
5242
|
freshGraph: null,
|
|
5243
|
+
growingCopy: null,
|
|
5134
5244
|
...noBuildHistory(),
|
|
5135
5245
|
rebuildChargeBytes: 0,
|
|
5136
5246
|
waitingForMemory: false,
|
|
@@ -5144,6 +5254,7 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
5144
5254
|
buildRetryDelayMs: 0,
|
|
5145
5255
|
buildTurnWaiters: /* @__PURE__ */ new Map(),
|
|
5146
5256
|
pendingBuild: null,
|
|
5257
|
+
buildsKeptOut: null,
|
|
5147
5258
|
disposed: false,
|
|
5148
5259
|
revision: 0,
|
|
5149
5260
|
workerCopyPool: null,
|
|
@@ -5158,6 +5269,7 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
5158
5269
|
releaseToFilesInFlight: Promise.resolve(),
|
|
5159
5270
|
cold: null,
|
|
5160
5271
|
graphsHeldOffThread: null,
|
|
5272
|
+
graphsPlacedIntoSinceConnect: /* @__PURE__ */ new WeakSet(),
|
|
5161
5273
|
searchWatches: /* @__PURE__ */ new Set(),
|
|
5162
5274
|
writeListeners: /* @__PURE__ */ new Set()
|
|
5163
5275
|
};
|
|
@@ -5241,9 +5353,16 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
5241
5353
|
await whenLoaded();
|
|
5242
5354
|
await awaitBuilds(state);
|
|
5243
5355
|
},
|
|
5244
|
-
|
|
5356
|
+
holdGraphComplete: async () => {
|
|
5245
5357
|
await whenLoaded();
|
|
5246
|
-
await
|
|
5358
|
+
const letBuildsIn2 = await keepBuildsOut(state);
|
|
5359
|
+
try {
|
|
5360
|
+
await completeGraph(state);
|
|
5361
|
+
return letBuildsIn2;
|
|
5362
|
+
} catch (error) {
|
|
5363
|
+
letBuildsIn2();
|
|
5364
|
+
throw error;
|
|
5365
|
+
}
|
|
5247
5366
|
},
|
|
5248
5367
|
dispose: () => {
|
|
5249
5368
|
cancelColdHold(state);
|
|
@@ -5280,7 +5399,10 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
5280
5399
|
}),
|
|
5281
5400
|
planCheckpoint: withGraphFree(
|
|
5282
5401
|
state,
|
|
5283
|
-
(listedKeys, rewritesGraph, pinsRetiredVectors) =>
|
|
5402
|
+
(listedKeys, rewritesGraph, pinsRetiredVectors) => {
|
|
5403
|
+
connectWhereAPassStoppedEarly(state);
|
|
5404
|
+
return planCheckpoint(state, listedKeys, rewritesGraph, pinsRetiredVectors);
|
|
5405
|
+
}
|
|
5284
5406
|
),
|
|
5285
5407
|
recordCheckpoint: (plan, written) => recordCheckpoint(state, plan, written),
|
|
5286
5408
|
restoreCheckpoint: withGraphFree(state, (shape) => {
|
|
@@ -5318,4 +5440,4 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
5318
5440
|
};
|
|
5319
5441
|
}
|
|
5320
5442
|
|
|
5321
|
-
export { bindVectorHost, compactVectors, createSharedVectorSearcher, createVectorIndex, executeHybridSearch, executeVectorSearch, filterSubsetsMatchNothing, getVectorMaintenanceStatus, holdAdmission, linearCombination, openSharedVectorField, optimizeVectors, reciprocalRankFusion, releaseLocksHeldBy, releasePinsHeldBy, resolveHybridFusion, vectorLegWith };
|
|
5443
|
+
export { bindVectorHost, compactVectors, createSharedVectorSearcher, createVectorIndex, executeHybridSearch, executeVectorSearch, filterSubsetsMatchNothing, getVectorMaintenanceStatus, holdAdmission, linearCombination, openSharedVectorField, optimizeVectors, reciprocalRankFusion, releaseLocksHeldBy, releasePinsHeldBy, requireUsableVectorOptions, resolveHybridFusion, vectorLegWith };
|
|
@@ -385,7 +385,6 @@ var decodeVectorGraphPayload;
|
|
|
385
385
|
var decodeVectorIndexParts;
|
|
386
386
|
var defaultEfSearch;
|
|
387
387
|
var defaultQuantizationFor;
|
|
388
|
-
var defaultRescoreDepth;
|
|
389
388
|
var graphPartsOfEarlierPayload;
|
|
390
389
|
var isDead;
|
|
391
390
|
var loadNativeCore;
|
|
@@ -397,6 +396,7 @@ var osqEstimate;
|
|
|
397
396
|
var osqQuantize;
|
|
398
397
|
var osqRecordBytes;
|
|
399
398
|
var readTrailer;
|
|
399
|
+
var rescoreDepth;
|
|
400
400
|
var scalarQueryProducts;
|
|
401
401
|
var stageQueryLevels;
|
|
402
402
|
var toDistance;
|
|
@@ -424,7 +424,6 @@ function bindVectorCoreNames(vectorCore) {
|
|
|
424
424
|
decodeVectorIndexParts,
|
|
425
425
|
defaultEfSearch,
|
|
426
426
|
defaultQuantizationFor,
|
|
427
|
-
defaultRescoreDepth,
|
|
428
427
|
graphPartsOfEarlierPayload,
|
|
429
428
|
isDead,
|
|
430
429
|
loadNativeCore,
|
|
@@ -436,6 +435,7 @@ function bindVectorCoreNames(vectorCore) {
|
|
|
436
435
|
osqQuantize,
|
|
437
436
|
osqRecordBytes,
|
|
438
437
|
readTrailer,
|
|
438
|
+
rescoreDepth,
|
|
439
439
|
scalarQueryProducts,
|
|
440
440
|
stageQueryLevels,
|
|
441
441
|
toDistance,
|
|
@@ -5881,6 +5881,20 @@ function formatSegmentStart(startSeqNo) {
|
|
|
5881
5881
|
return startSeqNo.toString().padStart(16, "0");
|
|
5882
5882
|
}
|
|
5883
5883
|
|
|
5884
|
+
// src/persistence/durability/complete-graphs.ts
|
|
5885
|
+
async function whileGraphsStayComplete(vectorIndexes, capture3) {
|
|
5886
|
+
const letBuildsIn = [];
|
|
5887
|
+
try {
|
|
5888
|
+
for (const fieldPath of [...vectorIndexes.keys()].sort()) {
|
|
5889
|
+
const vectorIndex = vectorIndexes.get(fieldPath);
|
|
5890
|
+
if (vectorIndex !== void 0) letBuildsIn.push(await vectorIndex.holdGraphComplete());
|
|
5891
|
+
}
|
|
5892
|
+
return await capture3();
|
|
5893
|
+
} finally {
|
|
5894
|
+
for (const release of letBuildsIn) release();
|
|
5895
|
+
}
|
|
5896
|
+
}
|
|
5897
|
+
|
|
5884
5898
|
// src/distribution/replication/replica.ts
|
|
5885
5899
|
function applyIndexEntry(entry, manager, vectorFieldPaths, vecIndexes, schedulesGraphBuild = true) {
|
|
5886
5900
|
if (entry.document === null) {
|
|
@@ -6978,9 +6992,7 @@ function prepareSearch(data, query, k, options, accepts, excluded, shared) {
|
|
|
6978
6992
|
if (field === null) return null;
|
|
6979
6993
|
const window = options.rescoreWindow ?? k;
|
|
6980
6994
|
let depth = window;
|
|
6981
|
-
if (data.codes !== null)
|
|
6982
|
-
depth = options.oversample === void 0 ? defaultRescoreDepth(data.codes.bits, data.dimension, window) : Math.ceil(window * options.oversample);
|
|
6983
|
-
}
|
|
6995
|
+
if (data.codes !== null) depth = rescoreDepth(data.codes.bits, data.dimension, window, options.oversample);
|
|
6984
6996
|
const ef = Math.max(options.efSearch ?? defaultEfSearch(data.dimension, window), depth);
|
|
6985
6997
|
const capacity = Math.min(data.total, ef);
|
|
6986
6998
|
const allocate2 = (bytes) => shared ? createFixedBuffer(bytes) : new ArrayBuffer(bytes);
|
|
@@ -7278,9 +7290,7 @@ function* walkColdGraph(data, query, k, options, accepts, excluded, fetchesRows)
|
|
|
7278
7290
|
const window = options.rescoreWindow ?? k;
|
|
7279
7291
|
const codes = data.codes;
|
|
7280
7292
|
let depth = window;
|
|
7281
|
-
if (codes !== null)
|
|
7282
|
-
depth = options.oversample === void 0 ? defaultRescoreDepth(codes.bits, data.dimension, window) : Math.ceil(window * options.oversample);
|
|
7283
|
-
}
|
|
7293
|
+
if (codes !== null) depth = rescoreDepth(codes.bits, data.dimension, window, options.oversample);
|
|
7284
7294
|
const ef = Math.max(options.efSearch ?? defaultEfSearch(data.dimension, window), depth);
|
|
7285
7295
|
const codeRowBytes = codes === null ? null : computeCodeBlockLayout(data.dimension, codes.bits).slotStride;
|
|
7286
7296
|
const scope = {
|
|
@@ -8156,9 +8166,8 @@ function wholePartitionsWhereMostChanged(manager, rules) {
|
|
|
8156
8166
|
};
|
|
8157
8167
|
}
|
|
8158
8168
|
async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartitionsOf = noWholePartitions, priorCheckpointThatLimitsTheRecords, listedVectorFields = [], termHistoryOf, onCaptured) {
|
|
8159
|
-
|
|
8160
|
-
return
|
|
8161
|
-
[...indexState.partitions.values()],
|
|
8169
|
+
const whileNoBuildPassOrMutationRuns = (capture3) => whileGraphsStayComplete(vectorIndexes, () => whileNoMutationApplies([...indexState.partitions.values()], capture3));
|
|
8170
|
+
return whileNoBuildPassOrMutationRuns(
|
|
8162
8171
|
() => captureAppliedPrefix(manager, (cutoffs) => {
|
|
8163
8172
|
onCaptured?.();
|
|
8164
8173
|
const partitionCount = manager.partitionCount;
|
|
@@ -13974,14 +13983,15 @@ function priorPartitionsOf(earlier, partitionCount) {
|
|
|
13974
13983
|
async function writeCurrentLayout(deps, earlier, targets, fileRefs) {
|
|
13975
13984
|
const priorVectors = priorVectorsOf(earlier, fileRefs);
|
|
13976
13985
|
const plans = /* @__PURE__ */ new Map();
|
|
13977
|
-
|
|
13978
|
-
|
|
13979
|
-
|
|
13980
|
-
|
|
13981
|
-
|
|
13982
|
-
|
|
13983
|
-
|
|
13984
|
-
|
|
13986
|
+
await whileGraphsStayComplete(deps.vectorIndexes, () => {
|
|
13987
|
+
for (const [fieldPath, vectorIndex] of deps.vectorIndexes) {
|
|
13988
|
+
const listed = priorVectors.find((ref) => ref.fieldPath === fieldPath);
|
|
13989
|
+
plans.set(
|
|
13990
|
+
fieldPath,
|
|
13991
|
+
vectorIndex.planCheckpoint(listed === void 0 ? null : listed.files.map((file) => file.key), true)
|
|
13992
|
+
);
|
|
13993
|
+
}
|
|
13994
|
+
});
|
|
13985
13995
|
const written = await writeCheckpointSegments({
|
|
13986
13996
|
store: deps.store,
|
|
13987
13997
|
metadata: deps.metadata,
|
|
@@ -14625,9 +14635,11 @@ async function writeCaptured(input, store, prior, captured) {
|
|
|
14625
14635
|
}
|
|
14626
14636
|
async function writeSnapshotOnlyCheckpoint(input) {
|
|
14627
14637
|
const store = adapterCheckpointStore(input.adapter);
|
|
14628
|
-
|
|
14629
|
-
|
|
14630
|
-
|
|
14638
|
+
const { prior, captured } = await whileGraphsStayComplete(input.vectorIndexes, async () => {
|
|
14639
|
+
const prior2 = await readCurrentManifest(store, input.indexName);
|
|
14640
|
+
const captured2 = await captureAppliedPrefix(input.manager, (cutoffs) => capture(input, prior2, cutoffs));
|
|
14641
|
+
return { prior: prior2, captured: captured2 };
|
|
14642
|
+
});
|
|
14631
14643
|
try {
|
|
14632
14644
|
await writeCaptured(input, store, prior, captured);
|
|
14633
14645
|
return {
|
|
@@ -17278,10 +17290,7 @@ function capture2(input) {
|
|
|
17278
17290
|
return { targets, contents, plans };
|
|
17279
17291
|
}
|
|
17280
17292
|
async function buildArchiveFromMemory(input) {
|
|
17281
|
-
|
|
17282
|
-
for (const vectorIndex of input.vectorIndexes.values()) await vectorIndex.completeGraph();
|
|
17283
|
-
}
|
|
17284
|
-
const captured = capture2(input);
|
|
17293
|
+
const captured = input.partitionId === null ? await whileGraphsStayComplete(input.vectorIndexes, () => capture2(input)) : capture2(input);
|
|
17285
17294
|
const store = memoryCheckpointStore();
|
|
17286
17295
|
const written = await writeCheckpointSegments({
|
|
17287
17296
|
store,
|
|
@@ -401,6 +401,7 @@ var BUILD_LIST_PAGE = 1e3;
|
|
|
401
401
|
var APOSTROPHE_VARIANTS = /[\u2018\u2019\u02bc]/g;
|
|
402
402
|
var HYPHEN_VARIANTS = /[\u2010\u2011]/g;
|
|
403
403
|
var WHITE_SPACE_RUNS = /\p{White_Space}+/gu;
|
|
404
|
+
var EDGE_SPACES = /^ | $/g;
|
|
404
405
|
var ALLOWED_PHRASE = /^[\p{L}\p{M}\p{Nd} '-]+$/u;
|
|
405
406
|
var LETTER = /\p{L}/gu;
|
|
406
407
|
var DIGIT = /\p{Nd}/gu;
|
|
@@ -421,11 +422,11 @@ function codePointCount(text, stopAfter) {
|
|
|
421
422
|
}
|
|
422
423
|
function normalisePhrase(raw, maxLength) {
|
|
423
424
|
if (raw.length === 0 || raw.length > maxLength * RAW_PHRASE_UTF16_UNITS_PER_CODE_POINT) return null;
|
|
424
|
-
return raw.normalize("NFKC").toLowerCase().replace(APOSTROPHE_VARIANTS, "'").replace(HYPHEN_VARIANTS, "-").replace(WHITE_SPACE_RUNS, " ").
|
|
425
|
+
return raw.normalize("NFKC").toLowerCase().replace(APOSTROPHE_VARIANTS, "'").replace(HYPHEN_VARIANTS, "-").replace(WHITE_SPACE_RUNS, " ").replace(EDGE_SPACES, "");
|
|
425
426
|
}
|
|
426
427
|
function phraseFits(phrase, limits) {
|
|
427
428
|
if (phrase.length === 0 || phrase.length > limits.maxLength * 2) return false;
|
|
428
|
-
if (phrase
|
|
429
|
+
if (phrase.startsWith(" ") || phrase.endsWith(" ") || phrase.includes(" ")) return false;
|
|
429
430
|
if (!ALLOWED_PHRASE.test(phrase)) return false;
|
|
430
431
|
if (codePointCount(phrase, limits.maxLength) > limits.maxLength) return false;
|
|
431
432
|
if (phrase.split(" ").length > limits.maxWords) return false;
|
|
@@ -57,6 +57,7 @@ var removeRestoreLeftovers;
|
|
|
57
57
|
var requireFieldsInSchema;
|
|
58
58
|
var resolveSetting;
|
|
59
59
|
var sameBytes;
|
|
60
|
+
var unionOfBans;
|
|
60
61
|
var validateSetting;
|
|
61
62
|
var withState;
|
|
62
63
|
function bindSuggestionsHost(host, msgpack, suggestionStore) {
|
|
@@ -120,6 +121,7 @@ function bindSuggestionsHost(host, msgpack, suggestionStore) {
|
|
|
120
121
|
requireFieldsInSchema,
|
|
121
122
|
resolveSetting,
|
|
122
123
|
sameBytes,
|
|
124
|
+
unionOfBans,
|
|
123
125
|
validateSetting,
|
|
124
126
|
withState
|
|
125
127
|
} = suggestionStore);
|
|
@@ -777,7 +779,7 @@ function updateBans(runtime, indexName, raw) {
|
|
|
777
779
|
function removeQuery(runtime, indexName, raw) {
|
|
778
780
|
const phrase = normaliseBan(raw);
|
|
779
781
|
return runtime.withWriteLock(indexName, async () => {
|
|
780
|
-
await banPhrases(runtime, indexName, (current) =>
|
|
782
|
+
await banPhrases(runtime, indexName, (current) => unionOfBans(current, [phrase]));
|
|
781
783
|
return removeFromSuggestions(runtime, indexName, [phrase]);
|
|
782
784
|
});
|
|
783
785
|
}
|