@delali/narsil 0.3.1-next.160 → 0.3.1-next.162
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/{chunk-4RNSFVWI.mjs → chunk-KQXKCMGQ.mjs} +7 -5
- package/dist/browser/{core-leaf-TJAGWPZZ.mjs → core-leaf-SIWXFZLV.mjs} +66 -4
- package/dist/browser/{module-362VSIGR.mjs → module-PMQBRESM.mjs} +167 -45
- package/dist/browser/{storage-leaf-X6CNRHJS.mjs → storage-leaf-VGR2BWJJ.mjs} +35 -26
- package/dist/browser/{store-leaf-AEMTAVHX.mjs → store-leaf-3L33V35P.mjs} +3 -2
- package/dist/browser/{suggestions-leaf-3Y6MR6OF.mjs → suggestions-leaf-UBNDVKBD.mjs} +3 -1
- package/dist/{capabilities-BKDrry0z.d.ts → capabilities-DkXAkb-X.d.ts} +2 -2
- package/dist/{chunk-NC4QLCZC.mjs → chunk-3UGZBRDN.mjs} +1 -1
- package/dist/{chunk-75UNINFA.mjs → chunk-5DUTOEK5.mjs} +1 -1
- package/dist/{chunk-U6B7DUQW.mjs → chunk-5P72V66D.mjs} +4 -4
- package/dist/{chunk-TSLV5P2J.mjs → chunk-7OE73OO4.mjs} +6 -10
- package/dist/{chunk-GF4K7IUK.mjs → chunk-BKOMO7HT.mjs} +5 -5
- package/dist/{chunk-SXIRM4MX.mjs → chunk-D5GNFN6X.mjs} +26 -15
- package/dist/{chunk-SHGN3QIF.mjs → chunk-DSXFN5BF.mjs} +4 -4
- package/dist/{chunk-TINW4BSH.mjs → chunk-EEYDJB4Z.mjs} +17 -14
- package/dist/{chunk-2RIS76RJ.mjs → chunk-EI7U43MX.mjs} +1 -1
- package/dist/{chunk-LF4GB4PO.mjs → chunk-EIR3IQCL.mjs} +1 -1
- package/dist/{chunk-2JRERUNZ.mjs → chunk-ES5EOXT6.mjs} +2 -2
- package/dist/{chunk-FVZ5DB6Y.mjs → chunk-FSLQHOKZ.mjs} +1 -1
- package/dist/{chunk-W34VJXY7.mjs → chunk-GJKFXKMI.mjs} +4 -4
- package/dist/{chunk-6EZW2GQL.mjs → chunk-GQLOEACG.mjs} +4 -3
- package/dist/{chunk-H64YDF5M.mjs → chunk-K4BMDPPX.mjs} +1 -1
- package/dist/{chunk-7MWYORQC.mjs → chunk-KSOZJEYU.mjs} +1 -1
- package/dist/{chunk-BXM7XM4G.mjs → chunk-NXMGUDZA.mjs} +2 -2
- package/dist/{chunk-P5YLI5ED.mjs → chunk-Q2WQSAYH.mjs} +1 -1
- package/dist/{chunk-V3SGPXWQ.mjs → chunk-Q7CUUSJ2.mjs} +2 -2
- package/dist/{chunk-3D6RZGJH.mjs → chunk-R4F7ZTVG.mjs} +2 -2
- package/dist/{chunk-OOO54OVC.mjs → chunk-RXMR2SIQ.mjs} +3 -3
- package/dist/{chunk-HOOR4CP5.mjs → chunk-STLNPJXU.mjs} +2 -2
- package/dist/{chunk-FE3MRIBN.mjs → chunk-TOZENMM7.mjs} +5 -5
- package/dist/{chunk-PTTMGRG5.mjs → chunk-V4YIEMAI.mjs} +2 -2
- package/dist/{chunk-LAWIJRMJ.mjs → chunk-VJOS6MHF.mjs} +67 -5
- package/dist/{chunk-K67I3RF4.mjs → chunk-XRBPHCCT.mjs} +2 -2
- package/dist/{chunk-E2F6LZR3.mjs → chunk-XXWEUXE4.mjs} +1 -1
- package/dist/{react-native/chunks/chunk-ZIU6TWXG.mjs → chunk-ZBVHS5KF.mjs} +1 -1
- package/dist/{chunk-IH3G6SJO.mjs → chunk-ZGYF37MG.mjs} +1 -1
- package/dist/{client-CTp8RRiK.d.ts → client-VJmV_AjZ.d.ts} +3 -3
- package/dist/client.d.ts +6 -6
- package/dist/{config-_WT5grSL.d.ts → config-Cb3FOlE_.d.ts} +1 -1
- package/dist/{config-PCNzWjHL.d.ts → config-ZZg4cwQz.d.ts} +1 -1
- package/dist/{core-leaf-KXFGP74A.mjs → core-leaf-LOXCD7MY.mjs} +2 -2
- package/dist/{direct-executor-MKTZBR26.mjs → direct-executor-PF4UDSUW.mjs} +2 -2
- package/dist/distribution.d.ts +4 -4
- package/dist/distribution.mjs +31 -32
- package/dist/{engine-DBRVR_d-.d.ts → engine-DdOs682x.d.ts} +9 -1
- package/dist/index.browser.mjs +1 -1
- package/dist/index.d.ts +4 -4
- package/dist/index.mjs +2 -2
- package/dist/index.react-native.mjs +4 -4
- package/dist/{long-query-thread-JMGNGF6V.mjs → long-query-thread-OTVLP2OI.mjs} +12 -12
- package/dist/{module-ZJ6ARG6V.mjs → module-UIKSHY3G.mjs} +173 -51
- package/dist/{optional-reads-BR3BOVZW.mjs → optional-reads-RSLNSGNG.mjs} +1 -1
- package/dist/{optional-writes-DDFHYUOH.mjs → optional-writes-4ITCV7YT.mjs} +4 -4
- package/dist/{paged-partition-archive-G576XGVE.mjs → paged-partition-archive-VMJ2AUA6.mjs} +11 -11
- package/dist/partial-words.browser.mjs +3 -2
- package/dist/partial-words.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.mjs +9 -9
- package/dist/{phrase-module-XK3T44VU.mjs → phrase-module-IWSG2DDZ.mjs} +1 -1
- package/dist/{query-actions-OMXJEC2J.mjs → query-actions-EW72OQXY.mjs} +2 -2
- package/dist/query-suggestions/merge-worker.mjs +5 -5
- package/dist/query-suggestions.browser.mjs +1 -1
- package/dist/query-suggestions.d.ts +1 -1
- package/dist/query-suggestions.mjs +3 -3
- package/dist/react-native/chunks/{chunk-52QD7CBJ.mjs → chunk-2JMAUU26.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-72OKYHZL.mjs → chunk-3RUXKZJG.mjs} +4 -3
- package/dist/react-native/chunks/{chunk-L4R5ITVG.mjs → chunk-AUXXPSFB.mjs} +17 -15
- package/dist/{chunk-IB4BYLE3.mjs → react-native/chunks/chunk-HEELWLKL.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-IY6U6HRA.mjs → chunk-I5W7A2JF.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-AQOWRH2C.mjs → chunk-RVHCQJQM.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-XEGKQKLY.mjs → chunk-VIPNDN4B.mjs} +67 -5
- package/dist/react-native/chunks/{chunk-G3CUQ333.mjs → chunk-XBTFK3AZ.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-J2NQW353.mjs → chunk-XOCM3BFU.mjs} +1 -1
- package/dist/react-native/chunks/{core-leaf-5B4JXCVA.mjs → core-leaf-O3236QY3.mjs} +2 -2
- package/dist/react-native/chunks/{module-VUA27HZZ.mjs → module-LRR7D5ZO.mjs} +168 -46
- package/dist/react-native/chunks/{optional-reads-WDTLLHUT.mjs → optional-reads-TOWGNEDH.mjs} +1 -1
- package/dist/react-native/chunks/{optional-writes-FZV3QW34.mjs → optional-writes-7AFBIZMN.mjs} +5 -5
- package/dist/react-native/chunks/{phrase-module-KCQWVXYZ.mjs → phrase-module-3A4AKMKL.mjs} +1 -1
- package/dist/react-native/chunks/{restore-queue-EI2V2I4S.mjs → restore-queue-7FSHTGJ5.mjs} +7 -7
- package/dist/react-native/chunks/{segment-codec-leaf-6ULP7H3I.mjs → segment-codec-leaf-4QV4HO4R.mjs} +5 -5
- package/dist/react-native/chunks/{segment-merge-leaf-F4GQZNK6.mjs → segment-merge-leaf-5DUZZSXJ.mjs} +6 -6
- package/dist/react-native/chunks/{storage-leaf-OAXBU2L6.mjs → storage-leaf-PNU6HWY7.mjs} +42 -33
- package/dist/react-native/chunks/{store-leaf-GY2ZZFXB.mjs → store-leaf-ESVUGYYO.mjs} +6 -6
- package/dist/react-native/chunks/{suggestions-leaf-SQEZ4IBS.mjs → suggestions-leaf-MOVOPBGD.mjs} +7 -7
- package/dist/react-native/chunks/{worker-copies-EEHSCFJW.mjs → worker-copies-VQGF3WZX.mjs} +5 -5
- package/dist/react-native/partial-words.mjs +6 -5
- package/dist/react-native/pattern.mjs +1 -1
- package/dist/react-native/query-suggestions.mjs +4 -4
- package/dist/react-native/sync.mjs +4 -4
- package/dist/react-native/synonyms.mjs +4 -4
- package/dist/react.d.ts +7 -7
- package/dist/{rerank-Y6MWBMTT.mjs → rerank-QHJXPSAK.mjs} +2 -2
- package/dist/{restore-queue-KPYSRJ4A.mjs → restore-queue-5W5HX44H.mjs} +5 -5
- package/dist/{segment-codec-leaf-6IFBSBPN.mjs → segment-codec-leaf-7VLVXKWO.mjs} +3 -3
- package/dist/{segment-merge-leaf-VE3FUSVE.mjs → segment-merge-leaf-22XR3IGA.mjs} +3 -3
- package/dist/server.d.ts +8 -8
- package/dist/server.mjs +17 -17
- package/dist/{storage-leaf-HMIIOUYN.mjs → storage-leaf-ZZ77C3DX.mjs} +37 -35
- package/dist/{store-leaf-M7T7SUZJ.mjs → store-leaf-LJ6D2PMF.mjs} +4 -4
- package/dist/{suggestions-leaf-Q3Q5D6XV.mjs → suggestions-leaf-KTQHO3PY.mjs} +6 -6
- package/dist/sync/filesystem.d.ts +2 -2
- package/dist/sync/filesystem.mjs +6 -6
- package/dist/sync/indexeddb.d.ts +2 -2
- package/dist/sync.browser.mjs +1 -1
- package/dist/sync.d.ts +3 -3
- package/dist/sync.mjs +2 -2
- package/dist/synonyms.browser.mjs +1 -1
- package/dist/synonyms.mjs +2 -2
- package/dist/{tasks-B4tT4vmP.d.ts → tasks-CxA-J_Ty.d.ts} +2 -2
- package/dist/{thread-SNILJXX3.mjs → thread-XWDK6TAR.mjs} +11 -11
- package/dist/{types-CTg0KaMy.d.ts → types-BboB278g.d.ts} +1 -1
- package/dist/vector/search-worker.mjs +4 -4
- package/dist/{worker-copies-ZLZS2CUM.mjs → worker-copies-ODAAGQRC.mjs} +5 -5
- package/dist/workers/entry.mjs +8 -8
- package/openapi.json +2 -2
- package/package.json +7 -7
- /package/dist/{chunk-YETKPX5M.mjs → chunk-T4HMSZCX.mjs} +0 -0
|
@@ -1,14 +1,14 @@
|
|
|
1
|
-
import { resolveHybridFusion, reciprocalRankFusion, linearCombination, noteWaitingForMemory, admitsBackgroundBytes } from './chunk-
|
|
2
|
-
export { linearCombination, reciprocalRankFusion, resolveHybridFusion } from './chunk-
|
|
3
|
-
import { createVectorStore } from './chunk-
|
|
4
|
-
export { openSharedVectorField } from './chunk-
|
|
5
|
-
export { bindVectorHost } from './chunk-
|
|
6
|
-
import { VECTOR_WORKER_COPIES_ALLOWED, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, defaultQuantizationFor, osqBitsOf, DEFAULT_VECTOR_METRIC, noBuildHistory, createOsqQuantizer, recordCheckpoint, assignStorePartitions, liveSize, STORE_ORDINAL_LAYOUT_GENERATION, writeSequenceOf, writeSequenceWords, storePinsOf, admissionAt, handOverBuildTurns, filterForOptions, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, yieldBuildTurn, holdAdmission, fieldHandlesOf, ordinalFilterForSubsets, ordinalFilterValues, nativeScoresOf, toScore, ordinalFilterHas, threadedNativeCore, threadsHoldCurrentLayout, WORKER_COPY_MIN_VECTORS,
|
|
7
|
-
export { holdAdmission, releaseLocksHeldBy, releasePinsHeldBy } from './chunk-
|
|
1
|
+
import { resolveHybridFusion, reciprocalRankFusion, linearCombination, noteWaitingForMemory, admitsBackgroundBytes } from './chunk-5DUTOEK5.mjs';
|
|
2
|
+
export { linearCombination, reciprocalRankFusion, resolveHybridFusion } from './chunk-5DUTOEK5.mjs';
|
|
3
|
+
import { createVectorStore } from './chunk-Q2WQSAYH.mjs';
|
|
4
|
+
export { openSharedVectorField } from './chunk-Q2WQSAYH.mjs';
|
|
5
|
+
export { bindVectorHost } from './chunk-T4HMSZCX.mjs';
|
|
6
|
+
import { VECTOR_WORKER_COPIES_ALLOWED, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, defaultQuantizationFor, osqBitsOf, DEFAULT_VECTOR_METRIC, noBuildHistory, createOsqQuantizer, recordCheckpoint, assignStorePartitions, liveSize, STORE_ORDINAL_LAYOUT_GENERATION, writeSequenceOf, writeSequenceWords, storePinsOf, admissionAt, handOverBuildTurns, filterForOptions, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, yieldBuildTurn, holdAdmission, fieldHandlesOf, ordinalFilterForSubsets, ordinalFilterValues, nativeScoresOf, toScore, ordinalFilterHas, threadedNativeCore, threadsHoldCurrentLayout, WORKER_COPY_MIN_VECTORS, SHUFFLED_REBUILD_MEMORY_RETRY_MS, emptyFieldBeforeRestore, nativeStoreScratchBytes, createOrdinalFilter, addToOrdinalFilter, magnitude, toDistance, SEARCH_ATTEMPTS_BEFORE_HOLDING_WRITES, recalibrateFromStore, NO_LIVE_VECTOR_AT_POSITION, signatureOf, bytesToVectors, isDead, BUILD_RETRY_MAX_DELAY_MS, BUILD_RETRY_FIRST_DELAY_MS, allLiveDocIds, sharedMemoryAvailable, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, releaseReplacedVectors, createHNSWIndex, adoptGraph, topLayerInParts, shuffleInPlace, requireWholeNeighbourLists, BUILD_STEPS_BETWEEN_CLOCK_CHECKS, createCentroidSums, liveOrdinals, RECALIBRATION_CHUNK_VECTORS, shuffleStep, SEARCH_REFILL_RESERVE, centroidOfCentroidSums, meanOfCentroidSums, countOfCentroidSums, squaredLengthsOfCentroidSums, BUILD_CHUNK_SIZE, nativeStoreFor, nativeMetricCode, withNativeThreadSlot, SEARCH_SCAN_CHUNK_VECTORS, releasePinsHeldBy, releaseLocksHeldBy, DRIFT_CHECK_GROWTH_FACTOR, adoptLoadedCalibration, planCheckpoint, deserialize, RECALIBRATION_DRIFT_RATIO, DISK_ADOPTION_YIELD_INTERVAL } from './chunk-VJOS6MHF.mjs';
|
|
7
|
+
export { holdAdmission, releaseLocksHeldBy, releasePinsHeldBy, requireUsableVectorOptions } from './chunk-VJOS6MHF.mjs';
|
|
8
8
|
import { chargeHeapBytes, workerResourceLimits } from './chunk-AB27FWZM.mjs';
|
|
9
9
|
import { requirePinnedState, signalOfSearch, bindSearchSignal, createDocIdMap, resolveWorkerCount, resolveWorkerEntry, detectRuntime, spawnNodeWorker } from './chunk-OUWZKNO3.mjs';
|
|
10
|
-
import { MAX_RENUMBERED_SEARCH_ATTEMPTS, DISK_STORAGE_BLOCK_BYTES, RESULT_WINDOW, turnsAreBudgeted, yieldBackgroundTurn, UNPLACED_FLAG_CHUNK_ORDINALS, holdWritesAndBackgroundWork, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK,
|
|
11
|
-
import { noteFallback, NATIVE_OK } from './chunk-
|
|
10
|
+
import { MAX_RENUMBERED_SEARCH_ATTEMPTS, DISK_STORAGE_BLOCK_BYTES, RESULT_WINDOW, turnsAreBudgeted, yieldBackgroundTurn, UNPLACED_FLAG_CHUNK_ORDINALS, holdWritesAndBackgroundWork, backgroundTurnIsSpent, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, NATIVE_THREAD_SCAN_CHUNK_ORDINALS, turnIsSpent, yieldTurn, noteThreadSlotLive, noteThreadSlotGone, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_INSERT_TIMEOUT_MS } from './chunk-3B775F3C.mjs';
|
|
11
|
+
import { noteFallback, NATIVE_OK } from './chunk-XXWEUXE4.mjs';
|
|
12
12
|
import './chunk-ZXX5GZEX.mjs';
|
|
13
13
|
import './chunk-4PY4JYMY.mjs';
|
|
14
14
|
import { NarsilError, ErrorCodes, isNarsilError } from './chunk-Q3ZZH5Z7.mjs';
|
|
@@ -43,18 +43,6 @@ async function searchCurrentSubsets(fieldName, first, collectAgain, search2) {
|
|
|
43
43
|
}
|
|
44
44
|
|
|
45
45
|
// src/engine/query/vector.ts
|
|
46
|
-
function oversampleOf(vectorConfig) {
|
|
47
|
-
const oversample = vectorConfig.oversample;
|
|
48
|
-
if (oversample === void 0) return void 0;
|
|
49
|
-
if (typeof oversample !== "number" || !Number.isFinite(oversample) || oversample < MIN_OVERSAMPLE) {
|
|
50
|
-
throw new NarsilError(
|
|
51
|
-
ErrorCodes.CONFIG_INVALID,
|
|
52
|
-
`vector.oversample must be a finite number of at least ${MIN_OVERSAMPLE}`,
|
|
53
|
-
{ oversample }
|
|
54
|
-
);
|
|
55
|
-
}
|
|
56
|
-
return oversample;
|
|
57
|
-
}
|
|
58
46
|
function vectorFetchDepth(limit, offset, cursorDepth) {
|
|
59
47
|
const reach = cursorDepth + offset + limit;
|
|
60
48
|
if (reach > RESULT_WINDOW) {
|
|
@@ -92,7 +80,7 @@ async function searchField(search2, k, host) {
|
|
|
92
80
|
metric: vectorConfig.metric ?? "cosine",
|
|
93
81
|
minSimilarity: vectorConfig.similarity ?? -Infinity,
|
|
94
82
|
efSearch: vectorConfig.efSearch,
|
|
95
|
-
oversample:
|
|
83
|
+
oversample: vectorConfig.oversample,
|
|
96
84
|
...rescoreWindow !== void 0 ? { rescoreWindow } : {}
|
|
97
85
|
};
|
|
98
86
|
if (filtered !== void 0) {
|
|
@@ -1352,7 +1340,7 @@ async function insertIntoGraph(state, graph, docIds, admit) {
|
|
|
1352
1340
|
const dispatch = found;
|
|
1353
1341
|
const pending = /* @__PURE__ */ new Set();
|
|
1354
1342
|
const limit = dispatch.workerCount * GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER;
|
|
1355
|
-
const fallback = { noWorkerWarned: false, shortAnswerWarned: false };
|
|
1343
|
+
const fallback = { noWorkerWarned: false, shortAnswerWarned: false, workerAnswerFailed: false };
|
|
1356
1344
|
let batchDocIds = [];
|
|
1357
1345
|
let batchOrdinals = [];
|
|
1358
1346
|
async function flush() {
|
|
@@ -1368,6 +1356,7 @@ async function insertIntoGraph(state, graph, docIds, admit) {
|
|
|
1368
1356
|
pending.add(run);
|
|
1369
1357
|
if (pending.size >= limit) await Promise.race(pending);
|
|
1370
1358
|
}
|
|
1359
|
+
state.graphsPlacedIntoSinceConnect.add(graph);
|
|
1371
1360
|
for (const docId of docIds) {
|
|
1372
1361
|
if (state.disposed) break;
|
|
1373
1362
|
if (!admit(docId)) continue;
|
|
@@ -1383,6 +1372,7 @@ async function insertIntoGraph(state, graph, docIds, admit) {
|
|
|
1383
1372
|
if (turnsAreBudgeted) await connectGraph(state, graph);
|
|
1384
1373
|
else graph.connectUnreached();
|
|
1385
1374
|
if (state.disposed) return false;
|
|
1375
|
+
if (!fallback.workerAnswerFailed) state.graphsPlacedIntoSinceConnect.delete(graph);
|
|
1386
1376
|
noteWrite(state);
|
|
1387
1377
|
return true;
|
|
1388
1378
|
}
|
|
@@ -1392,6 +1382,7 @@ async function placeChunk(state, graph, dispatch, chunkDocIds, chunkOrdinals, fa
|
|
|
1392
1382
|
outcome = await dispatch.send(chunkOrdinals);
|
|
1393
1383
|
} catch {
|
|
1394
1384
|
outcome = null;
|
|
1385
|
+
fallback.workerAnswerFailed = true;
|
|
1395
1386
|
}
|
|
1396
1387
|
if (state.disposed) return;
|
|
1397
1388
|
if (outcome === null) {
|
|
@@ -1428,6 +1419,7 @@ async function insertLocally(state, graph, docIds, admit) {
|
|
|
1428
1419
|
if (!admit(docId)) continue;
|
|
1429
1420
|
const ordinal = state.store.getOrdinal(docId);
|
|
1430
1421
|
if (ordinal === void 0) continue;
|
|
1422
|
+
state.graphsPlacedIntoSinceConnect.add(graph);
|
|
1431
1423
|
if (turnsAreBudgeted) {
|
|
1432
1424
|
graph.insertOrdinal(ordinal, true);
|
|
1433
1425
|
if (graph.holdsUnlinkedNode) await connectGraph(state, graph);
|
|
@@ -1450,6 +1442,7 @@ async function insertLocally(state, graph, docIds, admit) {
|
|
|
1450
1442
|
if (count > 0) {
|
|
1451
1443
|
if (turnsAreBudgeted) await connectGraph(state, graph);
|
|
1452
1444
|
else graph.connectUnreached();
|
|
1445
|
+
state.graphsPlacedIntoSinceConnect.delete(graph);
|
|
1453
1446
|
noteWrite(state);
|
|
1454
1447
|
}
|
|
1455
1448
|
return true;
|
|
@@ -1674,11 +1667,19 @@ async function releasePendingLocations(state) {
|
|
|
1674
1667
|
}
|
|
1675
1668
|
|
|
1676
1669
|
// src/vector/vector-index/memory-charge.ts
|
|
1670
|
+
function waitForMemory(state) {
|
|
1671
|
+
state.waitingForMemory = true;
|
|
1672
|
+
noteWaitingForMemory(state, { indexName: state.indexName, fieldName: state.fieldName });
|
|
1673
|
+
}
|
|
1674
|
+
function admitGrowingCopy(state, graph) {
|
|
1675
|
+
if (admitsBackgroundBytes(graph.graphBytes)) return true;
|
|
1676
|
+
waitForMemory(state);
|
|
1677
|
+
return false;
|
|
1678
|
+
}
|
|
1677
1679
|
function admitShuffledRebuild(state) {
|
|
1678
1680
|
const bytes = state.hnsw?.graphBytes ?? 0;
|
|
1679
1681
|
if (!admitsBackgroundBytes(bytes)) {
|
|
1680
|
-
state
|
|
1681
|
-
noteWaitingForMemory(state, { indexName: state.indexName, fieldName: state.fieldName });
|
|
1682
|
+
waitForMemory(state);
|
|
1682
1683
|
return false;
|
|
1683
1684
|
}
|
|
1684
1685
|
stopWaitingForMemory(state);
|
|
@@ -1702,6 +1703,64 @@ function releaseMemoryCharges(state) {
|
|
|
1702
1703
|
state.rebuildWaitTimer = null;
|
|
1703
1704
|
}
|
|
1704
1705
|
|
|
1706
|
+
// src/vector/vector-index/growing-copy.ts
|
|
1707
|
+
var lineages = /* @__PURE__ */ new WeakMap();
|
|
1708
|
+
function lineageOfGraph(graph) {
|
|
1709
|
+
const known = lineages.get(graph);
|
|
1710
|
+
if (known !== void 0) return known;
|
|
1711
|
+
const lineage = {};
|
|
1712
|
+
lineages.set(graph, lineage);
|
|
1713
|
+
return lineage;
|
|
1714
|
+
}
|
|
1715
|
+
function lineageOf(graph) {
|
|
1716
|
+
return graph === null ? null : lineageOfGraph(graph);
|
|
1717
|
+
}
|
|
1718
|
+
function moveWhatTheCopyHoldsOutOfBuffer(state, copy) {
|
|
1719
|
+
for (const docId of [...state.buffer]) {
|
|
1720
|
+
const ordinal = state.store.getOrdinal(docId);
|
|
1721
|
+
if (ordinal !== void 0 && copy.holdsNode(ordinal) && !copy.holdsTombstone(ordinal)) moveOutOfBuffer(state, docId);
|
|
1722
|
+
}
|
|
1723
|
+
}
|
|
1724
|
+
async function publishCopy(state, graph, copy) {
|
|
1725
|
+
if (state.sharedHandles.get(graph)?.searchable === true) await shareGraph(state, copy, true);
|
|
1726
|
+
if (state.disposed || state.hnsw !== graph) return false;
|
|
1727
|
+
lineages.set(copy, lineageOfGraph(graph));
|
|
1728
|
+
state.hnsw = copy;
|
|
1729
|
+
graph.retire();
|
|
1730
|
+
moveWhatTheCopyHoldsOutOfBuffer(state, copy);
|
|
1731
|
+
dropSharedGraph(state, graph);
|
|
1732
|
+
noteWrite(state);
|
|
1733
|
+
return true;
|
|
1734
|
+
}
|
|
1735
|
+
function* startingWith(first, rest) {
|
|
1736
|
+
yield first;
|
|
1737
|
+
for (let next = rest.next(); !next.done; next = rest.next()) yield next.value;
|
|
1738
|
+
}
|
|
1739
|
+
async function placeInGrowingCopy(state, graph, docIds, admit) {
|
|
1740
|
+
const pending = docIds[Symbol.iterator]();
|
|
1741
|
+
let first = pending.next();
|
|
1742
|
+
while (!first.done && !admit(first.value)) first = pending.next();
|
|
1743
|
+
if (first.done) return "nothingToPlace";
|
|
1744
|
+
if (!admitGrowingCopy(state, graph)) return "waitingForMemory";
|
|
1745
|
+
const copy = graph.copy();
|
|
1746
|
+
const chargedBytes = copy.graphBytes;
|
|
1747
|
+
chargeHeapBytes(chargedBytes);
|
|
1748
|
+
state.growingCopy = copy;
|
|
1749
|
+
let published = false;
|
|
1750
|
+
try {
|
|
1751
|
+
const placed = await insertIntoGraph(state, copy, startingWith(first.value, pending), admit);
|
|
1752
|
+
published = placed && !state.disposed && state.hnsw === graph && await publishCopy(state, graph, copy);
|
|
1753
|
+
return published ? "published" : "dropped";
|
|
1754
|
+
} finally {
|
|
1755
|
+
if (state.growingCopy === copy) state.growingCopy = null;
|
|
1756
|
+
chargeHeapBytes(-chargedBytes);
|
|
1757
|
+
if (!published) {
|
|
1758
|
+
copy.retire();
|
|
1759
|
+
dropSharedGraph(state, copy);
|
|
1760
|
+
}
|
|
1761
|
+
}
|
|
1762
|
+
}
|
|
1763
|
+
|
|
1705
1764
|
// src/vector/vector-index/build.ts
|
|
1706
1765
|
function bufferIfTheGraphLacks(state, graph, docId) {
|
|
1707
1766
|
const ordinal = state.store.getOrdinal(docId);
|
|
@@ -1716,9 +1775,9 @@ async function bufferWhatTheGraphLacksInTurns(state, graph) {
|
|
|
1716
1775
|
await forEachInTurns(allLiveDocIds(state), visit, () => !state.disposed && state.hnsw === graph);
|
|
1717
1776
|
}
|
|
1718
1777
|
async function buildGraphFromStore(state) {
|
|
1719
|
-
const
|
|
1778
|
+
const lineage = lineageOf(state.hnsw);
|
|
1720
1779
|
const order = turnsAreBudgeted ? await shuffledLiveDocIdsInTurns(state) : shuffleInPlace([...allLiveDocIds(state)]);
|
|
1721
|
-
if (order === null || state.hnsw !==
|
|
1780
|
+
if (order === null || lineageOf(state.hnsw) !== lineage) return false;
|
|
1722
1781
|
const graph = createHNSWIndex(state.dimension, state.store, state.hnswConfig, state.osq ?? void 0);
|
|
1723
1782
|
state.freshGraph = graph;
|
|
1724
1783
|
let completed = false;
|
|
@@ -1727,16 +1786,17 @@ async function buildGraphFromStore(state) {
|
|
|
1727
1786
|
} finally {
|
|
1728
1787
|
state.freshGraph = null;
|
|
1729
1788
|
}
|
|
1730
|
-
if (!completed || state.hnsw !==
|
|
1789
|
+
if (!completed || lineageOf(state.hnsw) !== lineage) {
|
|
1731
1790
|
dropSharedGraph(state, graph);
|
|
1732
1791
|
return false;
|
|
1733
1792
|
}
|
|
1793
|
+
const replaced = state.hnsw;
|
|
1734
1794
|
adoptGraph(state, graph);
|
|
1735
1795
|
state.shuffledBuildCount = graph.size;
|
|
1736
1796
|
noteWrite(state);
|
|
1737
1797
|
if (turnsAreBudgeted) await bufferWhatTheGraphLacksInTurns(state, graph);
|
|
1738
1798
|
else bufferWhatTheGraphLacks(state, graph);
|
|
1739
|
-
if (
|
|
1799
|
+
if (replaced !== null) dropSharedGraph(state, replaced);
|
|
1740
1800
|
await releasePendingLocations(state);
|
|
1741
1801
|
return true;
|
|
1742
1802
|
}
|
|
@@ -1753,10 +1813,31 @@ async function recalibratesForRebuild(state, graph) {
|
|
|
1753
1813
|
if (state.shuffledBuildCount === null) return calibrateBeforeBuild(state);
|
|
1754
1814
|
return recalibrateWhenDrifted(state, graph.size);
|
|
1755
1815
|
}
|
|
1756
|
-
async function
|
|
1816
|
+
async function keepBuildsOut(state) {
|
|
1817
|
+
while (state.buildsKeptOut !== null) await state.buildsKeptOut.ended;
|
|
1818
|
+
if (state.disposed) return () => void 0;
|
|
1819
|
+
let end = () => void 0;
|
|
1820
|
+
const ended = new Promise((resolve) => {
|
|
1821
|
+
end = resolve;
|
|
1822
|
+
});
|
|
1823
|
+
const kept = { ended, end };
|
|
1824
|
+
state.buildsKeptOut = kept;
|
|
1825
|
+
return () => {
|
|
1826
|
+
if (state.buildsKeptOut !== kept) return;
|
|
1827
|
+
letBuildsIn(state);
|
|
1828
|
+
scheduleBuild(state);
|
|
1829
|
+
};
|
|
1830
|
+
}
|
|
1831
|
+
function letBuildsIn(state) {
|
|
1832
|
+
const kept = state.buildsKeptOut;
|
|
1833
|
+
state.buildsKeptOut = null;
|
|
1834
|
+
kept?.end();
|
|
1835
|
+
}
|
|
1836
|
+
async function buildExclusively(state, work, waitsForShuffledRebuild = true, keepsBuildsOut = false) {
|
|
1757
1837
|
for (; ; ) {
|
|
1758
1838
|
if (state.pendingBuild !== null) await state.pendingBuild;
|
|
1759
1839
|
else if (waitsForShuffledRebuild && state.shuffledRebuild !== null) await state.shuffledRebuild;
|
|
1840
|
+
else if (!keepsBuildsOut && state.buildsKeptOut !== null) await state.buildsKeptOut.ended;
|
|
1760
1841
|
else break;
|
|
1761
1842
|
}
|
|
1762
1843
|
if (state.disposed) return;
|
|
@@ -1775,7 +1856,10 @@ async function buildExclusively(state, work, waitsForShuffledRebuild = true) {
|
|
|
1775
1856
|
}
|
|
1776
1857
|
async function rebuildInShuffledOrder(state) {
|
|
1777
1858
|
await yieldBuildTurn(state);
|
|
1778
|
-
const replacing = {
|
|
1859
|
+
const replacing = {
|
|
1860
|
+
lineage: null,
|
|
1861
|
+
loadedWithoutCount: false
|
|
1862
|
+
};
|
|
1779
1863
|
await buildExclusively(
|
|
1780
1864
|
state,
|
|
1781
1865
|
async () => {
|
|
@@ -1783,12 +1867,12 @@ async function rebuildInShuffledOrder(state) {
|
|
|
1783
1867
|
if (graph === null || !shuffledRebuildDue(state)) return;
|
|
1784
1868
|
const loadedWithoutCount = state.shuffledBuildCount === null;
|
|
1785
1869
|
if (!await recalibratesForRebuild(state, graph) || state.hnsw !== graph) return;
|
|
1786
|
-
replacing.
|
|
1870
|
+
replacing.lineage = lineageOf(graph);
|
|
1787
1871
|
replacing.loadedWithoutCount = loadedWithoutCount;
|
|
1788
1872
|
},
|
|
1789
1873
|
false
|
|
1790
1874
|
);
|
|
1791
|
-
if (replacing.
|
|
1875
|
+
if (replacing.lineage === null || lineageOf(state.hnsw) !== replacing.lineage) return;
|
|
1792
1876
|
const adopted = await buildGraphFromStore(state);
|
|
1793
1877
|
if (adopted && replacing.loadedWithoutCount) state.onLoadedGraphRepaired();
|
|
1794
1878
|
}
|
|
@@ -1796,11 +1880,16 @@ function retryWhenMemoryAllows(state) {
|
|
|
1796
1880
|
if (state.rebuildWaitTimer !== null || state.disposed) return;
|
|
1797
1881
|
const timer = setTimeout(() => {
|
|
1798
1882
|
state.rebuildWaitTimer = null;
|
|
1799
|
-
|
|
1883
|
+
if (state.hnsw !== null && state.buffer.size > 0) placeWhatWaitedForMemory(state);
|
|
1884
|
+
else startShuffledRebuild(state);
|
|
1800
1885
|
}, SHUFFLED_REBUILD_MEMORY_RETRY_MS);
|
|
1801
1886
|
timer.unref?.();
|
|
1802
1887
|
state.rebuildWaitTimer = timer;
|
|
1803
1888
|
}
|
|
1889
|
+
function placeWhatWaitedForMemory(state) {
|
|
1890
|
+
if (state.building || state.buildsKeptOut !== null) retryWhenMemoryAllows(state);
|
|
1891
|
+
else triggerBuild(state);
|
|
1892
|
+
}
|
|
1804
1893
|
function startShuffledRebuild(state) {
|
|
1805
1894
|
if (state.shuffledRebuild !== null || state.disposed) return;
|
|
1806
1895
|
if (!shuffledRebuildDue(state)) {
|
|
@@ -1833,11 +1922,14 @@ async function growGraph(state, graph, bufferSnapshot) {
|
|
|
1833
1922
|
return true;
|
|
1834
1923
|
};
|
|
1835
1924
|
if (state.osq && !state.osq.isCalibrated() && !await calibrateBeforeBuild(state)) return;
|
|
1836
|
-
await
|
|
1837
|
-
|
|
1925
|
+
continueAfterGrowth(state, await placeInGrowingCopy(state, graph, bufferSnapshot, admit));
|
|
1926
|
+
}
|
|
1927
|
+
function continueAfterGrowth(state, outcome) {
|
|
1928
|
+
if (outcome === "waitingForMemory") retryWhenMemoryAllows(state);
|
|
1929
|
+
else startShuffledRebuild(state);
|
|
1838
1930
|
}
|
|
1839
1931
|
function triggerBuild(state) {
|
|
1840
|
-
if (state.building) return;
|
|
1932
|
+
if (state.building || state.buildsKeptOut !== null) return;
|
|
1841
1933
|
state.building = true;
|
|
1842
1934
|
const bufferSnapshot = turnsAreBudgeted ? null : new Set(state.buffer);
|
|
1843
1935
|
const existingGraph = state.hnsw;
|
|
@@ -1888,6 +1980,15 @@ function armBuildTimer(state, delayMs, run) {
|
|
|
1888
1980
|
timer.unref?.();
|
|
1889
1981
|
state.buildTimer = timer;
|
|
1890
1982
|
}
|
|
1983
|
+
function connectAfterUnfinishedPass(state, graph) {
|
|
1984
|
+
graph.connectUnreached();
|
|
1985
|
+
state.graphsPlacedIntoSinceConnect.delete(graph);
|
|
1986
|
+
noteWrite(state);
|
|
1987
|
+
}
|
|
1988
|
+
function connectWhereAPassStoppedEarly(state) {
|
|
1989
|
+
const graph = state.hnsw;
|
|
1990
|
+
if (graph !== null && state.graphsPlacedIntoSinceConnect.has(graph)) connectAfterUnfinishedPass(state, graph);
|
|
1991
|
+
}
|
|
1891
1992
|
function retryAfterBackoff(state) {
|
|
1892
1993
|
const delayMs = Math.min(BUILD_RETRY_MAX_DELAY_MS, Math.max(BUILD_RETRY_FIRST_DELAY_MS, state.buildRetryDelayMs * 2));
|
|
1893
1994
|
state.buildRetryDelayMs = delayMs;
|
|
@@ -1895,7 +1996,7 @@ function retryAfterBackoff(state) {
|
|
|
1895
1996
|
state.buildTimer = null;
|
|
1896
1997
|
armBuildTimer(state, delayMs, () => {
|
|
1897
1998
|
const graph = state.hnsw;
|
|
1898
|
-
if (state.building) {
|
|
1999
|
+
if (state.building || state.buildsKeptOut !== null) {
|
|
1899
2000
|
retryAfterBackoff(state);
|
|
1900
2001
|
return;
|
|
1901
2002
|
}
|
|
@@ -1904,8 +2005,7 @@ function retryAfterBackoff(state) {
|
|
|
1904
2005
|
return;
|
|
1905
2006
|
}
|
|
1906
2007
|
try {
|
|
1907
|
-
graph
|
|
1908
|
-
noteWrite(state);
|
|
2008
|
+
connectAfterUnfinishedPass(state, graph);
|
|
1909
2009
|
state.buildRetryDelayMs = 0;
|
|
1910
2010
|
scheduleWorkerCopyLoad(state);
|
|
1911
2011
|
} catch (error) {
|
|
@@ -1917,11 +2017,12 @@ function retryAfterBackoff(state) {
|
|
|
1917
2017
|
function buildDue(state) {
|
|
1918
2018
|
if (state.hnsw === null) return liveSize(state) >= state.promotionThreshold;
|
|
1919
2019
|
if (state.buffer.size === 0) return false;
|
|
2020
|
+
if (state.waitingForMemory && state.rebuildWaitTimer !== null) return false;
|
|
1920
2021
|
if (state.workerCopies.enabled && state.workerCopies.host !== void 0) return true;
|
|
1921
2022
|
return state.buffer.size >= state.promotionThreshold;
|
|
1922
2023
|
}
|
|
1923
2024
|
function scheduleBuild(state) {
|
|
1924
|
-
if (state.building || state.buildScheduled || state.disposed) return;
|
|
2025
|
+
if (state.building || state.buildScheduled || state.disposed || state.buildsKeptOut !== null) return;
|
|
1925
2026
|
if (!buildDue(state)) return;
|
|
1926
2027
|
state.buildScheduled = true;
|
|
1927
2028
|
const turn = turnsAreBudgeted ? yieldBackgroundTurn() : yieldBuildTurn(state);
|
|
@@ -1943,6 +2044,7 @@ function releaseHeldMemory(state) {
|
|
|
1943
2044
|
state.store.release();
|
|
1944
2045
|
state.hnsw = null;
|
|
1945
2046
|
state.freshGraph = null;
|
|
2047
|
+
state.growingCopy = null;
|
|
1946
2048
|
state.osq = null;
|
|
1947
2049
|
state.tombstones.clear();
|
|
1948
2050
|
state.buffer.clear();
|
|
@@ -1953,6 +2055,7 @@ function releaseHeldMemory(state) {
|
|
|
1953
2055
|
}
|
|
1954
2056
|
function disposeIndex(state) {
|
|
1955
2057
|
state.disposed = true;
|
|
2058
|
+
letBuildsIn(state);
|
|
1956
2059
|
if (state.rebuildWaitTimer !== null) clearTimeout(state.rebuildWaitTimer);
|
|
1957
2060
|
state.rebuildWaitTimer = null;
|
|
1958
2061
|
if (state.buildTimer !== null) clearTimeout(state.buildTimer);
|
|
@@ -2496,6 +2599,7 @@ function compact(state) {
|
|
|
2496
2599
|
state.compactedNodeCount += before - state.hnsw.tombstoneCount;
|
|
2497
2600
|
}
|
|
2498
2601
|
state.freshGraph?.compactTombstones(oldestPin);
|
|
2602
|
+
state.growingCopy?.compactTombstones(oldestPin);
|
|
2499
2603
|
for (const docId of state.tombstones) {
|
|
2500
2604
|
const ordinal = state.store.getOrdinal(docId);
|
|
2501
2605
|
if (ordinal !== void 0 && !state.store.mayFreeOrdinal(ordinal, oldestPin)) continue;
|
|
@@ -2512,8 +2616,7 @@ async function insertMissing(state, graph) {
|
|
|
2512
2616
|
const missingOrReplaced = (docId) => !graph.has(docId) || state.buffer.has(docId);
|
|
2513
2617
|
const heldWhenTheCallStarted = [...allLiveDocIds(state)];
|
|
2514
2618
|
const stillLiveAndUnplaced = (docId) => !state.tombstones.has(docId) && missingOrReplaced(docId);
|
|
2515
|
-
await
|
|
2516
|
-
startShuffledRebuild(state);
|
|
2619
|
+
continueAfterGrowth(state, await placeInGrowingCopy(state, graph, heldWhenTheCallStarted, stillLiveAndUnplaced));
|
|
2517
2620
|
}
|
|
2518
2621
|
async function foldIntoGraph(state) {
|
|
2519
2622
|
const rebuildNeeded = graphNeedsRebuild(state);
|
|
@@ -2556,7 +2659,7 @@ async function fillGraph(state) {
|
|
|
2556
2659
|
await insertMissing(state, graph);
|
|
2557
2660
|
}
|
|
2558
2661
|
function completeGraph(state) {
|
|
2559
|
-
return buildExclusively(state, () => fillGraph(state), false);
|
|
2662
|
+
return buildExclusively(state, () => fillGraph(state), false, true);
|
|
2560
2663
|
}
|
|
2561
2664
|
function maintenanceStatus(state) {
|
|
2562
2665
|
const storeSize = state.store.size;
|
|
@@ -2583,8 +2686,8 @@ function flatScanScratchBytes(state) {
|
|
|
2583
2686
|
}
|
|
2584
2687
|
function estimateMemoryBytes(state) {
|
|
2585
2688
|
if (state.hnsw === null) return state.store.memoryBytes() + flatScanScratchBytes(state);
|
|
2586
|
-
const
|
|
2587
|
-
return state.store.memoryBytes() + state.hnsw.graphBytes + state.hnsw.searchScratchBytes +
|
|
2689
|
+
const passGraphBytes = (state.freshGraph?.graphBytes ?? 0) + (state.growingCopy?.graphBytes ?? 0);
|
|
2690
|
+
return state.store.memoryBytes() + state.hnsw.graphBytes + state.hnsw.searchScratchBytes + passGraphBytes;
|
|
2588
2691
|
}
|
|
2589
2692
|
|
|
2590
2693
|
// src/vector/vector-index/publication.ts
|
|
@@ -3044,14 +3147,16 @@ function createVectorWrites(state) {
|
|
|
3044
3147
|
noteWrite(state);
|
|
3045
3148
|
noteSearchedDocWritten(state, docId);
|
|
3046
3149
|
state.tombstones.delete(docId);
|
|
3047
|
-
const graphHoldsPrevious = previous !== void 0 && ((state.hnsw?.holdsNode(previous) ?? false) || (state.freshGraph?.holdsNode(previous) ?? false));
|
|
3150
|
+
const graphHoldsPrevious = previous !== void 0 && ((state.hnsw?.holdsNode(previous) ?? false) || (state.freshGraph?.holdsNode(previous) ?? false) || (state.growingCopy?.holdsNode(previous) ?? false));
|
|
3048
3151
|
state.store.insert(docId, vector, partitionId, graphHoldsPrevious);
|
|
3049
3152
|
if (previous !== void 0) {
|
|
3050
3153
|
const graph = state.hnsw;
|
|
3051
3154
|
const freshGraph = state.freshGraph;
|
|
3155
|
+
const growingCopy = state.growingCopy;
|
|
3052
3156
|
afterGraphPass(state, () => {
|
|
3053
3157
|
graph?.markTombstoneOrdinal(previous);
|
|
3054
3158
|
freshGraph?.markTombstoneOrdinal(previous);
|
|
3159
|
+
growingCopy?.markTombstoneOrdinal(previous);
|
|
3055
3160
|
});
|
|
3056
3161
|
if (!graphHoldsPrevious) state.osq?.removeOrdinal(previous);
|
|
3057
3162
|
}
|
|
@@ -3072,7 +3177,11 @@ function createVectorWrites(state) {
|
|
|
3072
3177
|
if (state.buffer.retiredCount !== 0) forgetRetiredFlags();
|
|
3073
3178
|
state.pendingLocations.delete(docId);
|
|
3074
3179
|
const graph = state.hnsw;
|
|
3075
|
-
|
|
3180
|
+
const growingCopy = state.growingCopy;
|
|
3181
|
+
afterGraphPass(state, () => {
|
|
3182
|
+
graph?.markTombstoneOrdinal(ordinal);
|
|
3183
|
+
growingCopy?.markTombstoneOrdinal(ordinal);
|
|
3184
|
+
});
|
|
3076
3185
|
}
|
|
3077
3186
|
return {
|
|
3078
3187
|
insert(docId, vector, partitionId) {
|
|
@@ -3145,6 +3254,7 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
3145
3254
|
osq: codeBits === null ? null : createOsqQuantizer(dimension, codeBits, metric, store),
|
|
3146
3255
|
hnsw: null,
|
|
3147
3256
|
freshGraph: null,
|
|
3257
|
+
growingCopy: null,
|
|
3148
3258
|
...noBuildHistory(),
|
|
3149
3259
|
rebuildChargeBytes: 0,
|
|
3150
3260
|
waitingForMemory: false,
|
|
@@ -3158,6 +3268,7 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
3158
3268
|
buildRetryDelayMs: 0,
|
|
3159
3269
|
buildTurnWaiters: /* @__PURE__ */ new Map(),
|
|
3160
3270
|
pendingBuild: null,
|
|
3271
|
+
buildsKeptOut: null,
|
|
3161
3272
|
disposed: false,
|
|
3162
3273
|
revision: 0,
|
|
3163
3274
|
workerCopyPool: null,
|
|
@@ -3172,6 +3283,7 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
3172
3283
|
releaseToFilesInFlight: Promise.resolve(),
|
|
3173
3284
|
cold: null,
|
|
3174
3285
|
graphsHeldOffThread: null,
|
|
3286
|
+
graphsPlacedIntoSinceConnect: /* @__PURE__ */ new WeakSet(),
|
|
3175
3287
|
searchWatches: /* @__PURE__ */ new Set(),
|
|
3176
3288
|
writeListeners: /* @__PURE__ */ new Set()
|
|
3177
3289
|
};
|
|
@@ -3255,9 +3367,16 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
3255
3367
|
await whenLoaded();
|
|
3256
3368
|
await awaitBuilds(state);
|
|
3257
3369
|
},
|
|
3258
|
-
|
|
3370
|
+
holdGraphComplete: async () => {
|
|
3259
3371
|
await whenLoaded();
|
|
3260
|
-
await
|
|
3372
|
+
const letBuildsIn2 = await keepBuildsOut(state);
|
|
3373
|
+
try {
|
|
3374
|
+
await completeGraph(state);
|
|
3375
|
+
return letBuildsIn2;
|
|
3376
|
+
} catch (error) {
|
|
3377
|
+
letBuildsIn2();
|
|
3378
|
+
throw error;
|
|
3379
|
+
}
|
|
3261
3380
|
},
|
|
3262
3381
|
dispose: () => {
|
|
3263
3382
|
cancelColdHold(state);
|
|
@@ -3294,7 +3413,10 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
3294
3413
|
}),
|
|
3295
3414
|
planCheckpoint: withGraphFree(
|
|
3296
3415
|
state,
|
|
3297
|
-
(listedKeys, rewritesGraph, pinsRetiredVectors) =>
|
|
3416
|
+
(listedKeys, rewritesGraph, pinsRetiredVectors) => {
|
|
3417
|
+
connectWhereAPassStoppedEarly(state);
|
|
3418
|
+
return planCheckpoint(state, listedKeys, rewritesGraph, pinsRetiredVectors);
|
|
3419
|
+
}
|
|
3298
3420
|
),
|
|
3299
3421
|
recordCheckpoint: (plan, written) => recordCheckpoint(state, plan, written),
|
|
3300
3422
|
restoreCheckpoint: withGraphFree(state, (shape) => {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { nativeSearchCoreIsEnabled, loadNativeCore, noteFallback } from './chunk-
|
|
1
|
+
import { nativeSearchCoreIsEnabled, loadNativeCore, noteFallback } from './chunk-XXWEUXE4.mjs';
|
|
2
2
|
import './chunk-Q3ZZH5Z7.mjs';
|
|
3
3
|
|
|
4
4
|
// src/engine/optional-reads-host-bindings.ts
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
export { applyStoredIndexBm25, updateIndexBm25 } from './chunk-
|
|
2
|
-
import { readHeapStatistics, loadSchemaCodeFor, loadStorageCode, lookupOfStoredSynonyms, reconstructSchemaFromMetadata, getLanguage, validatePatternValueLimit, requireSurfaceFormsForPartialFields, prepareStoredSynonyms, getVectorFieldPaths, arrayPositionsAreStale, validatePartitionConfig, validateEmbeddingConfig } from './chunk-
|
|
3
|
-
export { bindOptionalWritesHost, insertDocument, insertDocumentBatch, removeDocument, removeDocumentBatch, rollbackInsertedDocument, rollbackRemovedDocument, rollbackUpdatedDocument, updateDocument, updateDocumentBatch } from './chunk-
|
|
1
|
+
export { applyStoredIndexBm25, updateIndexBm25 } from './chunk-3UGZBRDN.mjs';
|
|
2
|
+
import { readHeapStatistics, loadSchemaCodeFor, loadStorageCode, lookupOfStoredSynonyms, reconstructSchemaFromMetadata, getLanguage, validatePatternValueLimit, requireSurfaceFormsForPartialFields, prepareStoredSynonyms, getVectorFieldPaths, arrayPositionsAreStale, validatePartitionConfig, validateEmbeddingConfig } from './chunk-EEYDJB4Z.mjs';
|
|
3
|
+
export { bindOptionalWritesHost, insertDocument, insertDocumentBatch, removeDocument, removeDocumentBatch, rollbackInsertedDocument, rollbackRemovedDocument, rollbackUpdatedDocument, updateDocument, updateDocumentBatch } from './chunk-EEYDJB4Z.mjs';
|
|
4
4
|
import './chunk-KLVMK3R5.mjs';
|
|
5
5
|
import './chunk-AB27FWZM.mjs';
|
|
6
6
|
import './chunk-GXCDWLQI.mjs';
|
|
@@ -11,7 +11,7 @@ import { validateSchema, validateVectorStorage, validateTextStorage } from './ch
|
|
|
11
11
|
import './chunk-U2BKYEC7.mjs';
|
|
12
12
|
import './chunk-OUWZKNO3.mjs';
|
|
13
13
|
import './chunk-3B775F3C.mjs';
|
|
14
|
-
import './chunk-
|
|
14
|
+
import './chunk-XXWEUXE4.mjs';
|
|
15
15
|
import './chunk-ZXX5GZEX.mjs';
|
|
16
16
|
import './chunk-4PY4JYMY.mjs';
|
|
17
17
|
import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
@@ -1,19 +1,19 @@
|
|
|
1
|
-
import { resolvePartitionId } from './chunk-
|
|
2
|
-
import { readVectorFile, VECTOR_FILE_ENVELOPE_VERSION } from './chunk-
|
|
3
|
-
import './chunk-
|
|
1
|
+
import { resolvePartitionId } from './chunk-R4F7ZTVG.mjs';
|
|
2
|
+
import { readVectorFile, VECTOR_FILE_ENVELOPE_VERSION } from './chunk-7OE73OO4.mjs';
|
|
3
|
+
import './chunk-V4YIEMAI.mjs';
|
|
4
4
|
import './chunk-4CAMH7FO.mjs';
|
|
5
|
-
import { encodeSegmentManifest, loadFailed, directoryCheckpointStore, decodeSections, SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES } from './chunk-
|
|
5
|
+
import { encodeSegmentManifest, loadFailed, directoryCheckpointStore, decodeSections, SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES } from './chunk-5P72V66D.mjs';
|
|
6
6
|
import { PARTITION_COPY_VECTOR_RUN_BYTES } from './chunk-JDDXV6UF.mjs';
|
|
7
|
-
import { segmentsPrefix, vectorFileKey } from './chunk-
|
|
7
|
+
import { segmentsPrefix, vectorFileKey } from './chunk-KSOZJEYU.mjs';
|
|
8
8
|
import './chunk-77AUVDQ5.mjs';
|
|
9
9
|
import './chunk-LNGE7SLY.mjs';
|
|
10
10
|
import './chunk-QBVTYY4J.mjs';
|
|
11
|
-
import './chunk-
|
|
12
|
-
import './chunk-
|
|
13
|
-
import { vectorsPerFile, isDead, VECTOR_FILE_PAYLOAD_VERSION } from './chunk-
|
|
11
|
+
import './chunk-ZBVHS5KF.mjs';
|
|
12
|
+
import './chunk-EI7U43MX.mjs';
|
|
13
|
+
import { vectorsPerFile, isDead, VECTOR_FILE_PAYLOAD_VERSION } from './chunk-VJOS6MHF.mjs';
|
|
14
14
|
import './chunk-V6EUBAIT.mjs';
|
|
15
|
-
import { HEADER_SIZE, crc32Init, crc32Final, engineVersionParts, writeHeader, RESERVED_SIZE, crc32Update } from './chunk-
|
|
16
|
-
import { generateId, isCompositePartition } from './chunk-
|
|
15
|
+
import { HEADER_SIZE, crc32Init, crc32Final, engineVersionParts, writeHeader, RESERVED_SIZE, crc32Update } from './chunk-GQLOEACG.mjs';
|
|
16
|
+
import { generateId, isCompositePartition } from './chunk-EEYDJB4Z.mjs';
|
|
17
17
|
import './chunk-KLVMK3R5.mjs';
|
|
18
18
|
import './chunk-AB27FWZM.mjs';
|
|
19
19
|
import { decode } from './chunk-GXCDWLQI.mjs';
|
|
@@ -24,7 +24,7 @@ import { parseVectorDimension } from './chunk-R4CE4IVU.mjs';
|
|
|
24
24
|
import './chunk-U2BKYEC7.mjs';
|
|
25
25
|
import './chunk-OUWZKNO3.mjs';
|
|
26
26
|
import './chunk-3B775F3C.mjs';
|
|
27
|
-
import './chunk-
|
|
27
|
+
import './chunk-XXWEUXE4.mjs';
|
|
28
28
|
import './chunk-ZXX5GZEX.mjs';
|
|
29
29
|
import './chunk-4PY4JYMY.mjs';
|
|
30
30
|
import './chunk-Q3ZZH5Z7.mjs';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { spellingCountsOf, encodePartialSpellings, countSpelling, SurfaceTable, SURFACE_TABLE_START_CAPACITY, StringPlan, SegmentRefusal, reserveStrings, grownCapacity, PARTIAL_WORD_RUN_CODE_POINTS, encodeStringBlob, sameFields, countDocumentSpellings, tokenize, PARTIAL_WORD_MIN_CODE_POINTS, PARTIAL_WORDS_ADDED_PER_QUERY, GROWING_PARTIAL_ENTRY_BYTES, GROWING_PARTIAL_RUN_BYTES, GROWING_PARTIAL_RUN_MEMBER_BYTES, GROWING_PARTIAL_PENDING_ENTRY_BYTES, RUN_TABLE_START_CAPACITY, TERM_REGION_GROWTH_FACTOR, RUN_MEMBER_START_CAPACITY, SLICE_CAPACITIES, TERM_SLOT_LOAD_DENOMINATOR, TERM_SLOT_LOAD_NUMERATOR, PoolPlan, requireRoom, releaseRange, requireSegment, termText, partitionDocumentFrequency, visitDocumentSpellings, visitDocumentSurfaceSpellings, DEFAULT_FUZZY_PREFIX_LENGTH, resolveStopWords, transformToken, expansionSource, liveFrequency, codePointCountOf, GROWING_PARTIAL_PENDING_ENTRIES_PER_ENTRY, FNV_OFFSET_BASIS, typedRegion, regionsOverlap, mergeText, FNV_PRIME } from './browser/chunk-
|
|
1
|
+
import { spellingCountsOf, encodePartialSpellings, countSpelling, SurfaceTable, SURFACE_TABLE_START_CAPACITY, StringPlan, SegmentRefusal, reserveStrings, grownCapacity, PARTIAL_WORD_RUN_CODE_POINTS, encodeStringBlob, sameFields, countDocumentSpellings, tokenize, PARTIAL_WORD_MIN_CODE_POINTS, PARTIAL_WORDS_ADDED_PER_QUERY, GROWING_PARTIAL_ENTRY_BYTES, GROWING_PARTIAL_RUN_BYTES, GROWING_PARTIAL_RUN_MEMBER_BYTES, GROWING_PARTIAL_PENDING_ENTRY_BYTES, RUN_TABLE_START_CAPACITY, TERM_REGION_GROWTH_FACTOR, RUN_MEMBER_START_CAPACITY, SLICE_CAPACITIES, TERM_SLOT_LOAD_DENOMINATOR, TERM_SLOT_LOAD_NUMERATOR, PoolPlan, requireRoom, releaseRange, requireSegment, termText, partitionDocumentFrequency, visitDocumentSpellings, visitDocumentSurfaceSpellings, DEFAULT_FUZZY_PREFIX_LENGTH, resolveStopWords, transformToken, expansionSource, liveFrequency, codePointCountOf, GROWING_PARTIAL_PENDING_ENTRIES_PER_ENTRY, FNV_OFFSET_BASIS, typedRegion, regionsOverlap, mergeText, FNV_PRIME } from './browser/chunk-KQXKCMGQ.mjs';
|
|
2
2
|
import { fuzzyTermMatches, storedText } from './browser/chunk-MMHIIL5B.mjs';
|
|
3
3
|
import './browser/chunk-YJHGOJAC.mjs';
|
|
4
4
|
import { compareCodePoints, codePointCount, NarsilError, ErrorCodes } from './browser/chunk-MZHQ65VI.mjs';
|
|
@@ -376,7 +376,8 @@ var SpellingTable = class extends SurfaceTable {
|
|
|
376
376
|
for (const _point of word) points++;
|
|
377
377
|
if (points <= 3) continue;
|
|
378
378
|
spellings.add(word);
|
|
379
|
-
plan.
|
|
379
|
+
const entry = plan.prepare(word);
|
|
380
|
+
plan.points[entry] = points;
|
|
380
381
|
}
|
|
381
382
|
reserveStrings(this, spellings);
|
|
382
383
|
const members = collectRuns(plan);
|
package/dist/partial-words.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { partialWordSearchModule } from './chunk-
|
|
1
|
+
import { partialWordSearchModule } from './chunk-EEYDJB4Z.mjs';
|
|
2
2
|
import './chunk-KLVMK3R5.mjs';
|
|
3
3
|
import './chunk-AB27FWZM.mjs';
|
|
4
4
|
import './chunk-GXCDWLQI.mjs';
|
|
@@ -9,7 +9,7 @@ import './chunk-R4CE4IVU.mjs';
|
|
|
9
9
|
import './chunk-U2BKYEC7.mjs';
|
|
10
10
|
import './chunk-OUWZKNO3.mjs';
|
|
11
11
|
import './chunk-3B775F3C.mjs';
|
|
12
|
-
import './chunk-
|
|
12
|
+
import './chunk-XXWEUXE4.mjs';
|
|
13
13
|
import './chunk-ZXX5GZEX.mjs';
|
|
14
14
|
import './chunk-4PY4JYMY.mjs';
|
|
15
15
|
import './chunk-Q3ZZH5Z7.mjs';
|
|
@@ -1,20 +1,20 @@
|
|
|
1
1
|
import { createDurableDirectory } from '../../chunk-TW543F33.mjs';
|
|
2
|
-
import { writeCheckpointSegments } from '../../chunk-
|
|
3
|
-
import { directoryCheckpointStore } from '../../chunk-
|
|
2
|
+
import { writeCheckpointSegments } from '../../chunk-GJKFXKMI.mjs';
|
|
3
|
+
import { directoryCheckpointStore } from '../../chunk-5P72V66D.mjs';
|
|
4
4
|
import '../../chunk-5FLRXF73.mjs';
|
|
5
5
|
import '../../chunk-JDDXV6UF.mjs';
|
|
6
|
-
import '../../chunk-
|
|
6
|
+
import '../../chunk-KSOZJEYU.mjs';
|
|
7
7
|
import '../../chunk-HLPRHOSO.mjs';
|
|
8
8
|
import { CHECKPOINT_WORKER_HEARTBEAT_MS } from '../../chunk-QDZJM7NI.mjs';
|
|
9
9
|
import '../../chunk-77AUVDQ5.mjs';
|
|
10
10
|
import '../../chunk-LNGE7SLY.mjs';
|
|
11
11
|
import '../../chunk-QBVTYY4J.mjs';
|
|
12
|
-
import '../../chunk-
|
|
13
|
-
import '../../chunk-
|
|
14
|
-
import '../../chunk-
|
|
12
|
+
import '../../chunk-ZBVHS5KF.mjs';
|
|
13
|
+
import '../../chunk-EI7U43MX.mjs';
|
|
14
|
+
import '../../chunk-VJOS6MHF.mjs';
|
|
15
15
|
import '../../chunk-V6EUBAIT.mjs';
|
|
16
|
-
import '../../chunk-
|
|
17
|
-
import { importOptionalReads, loadGeoCodeFor } from '../../chunk-
|
|
16
|
+
import '../../chunk-GQLOEACG.mjs';
|
|
17
|
+
import { importOptionalReads, loadGeoCodeFor } from '../../chunk-EEYDJB4Z.mjs';
|
|
18
18
|
import '../../chunk-KLVMK3R5.mjs';
|
|
19
19
|
import '../../chunk-AB27FWZM.mjs';
|
|
20
20
|
import '../../chunk-GXCDWLQI.mjs';
|
|
@@ -25,7 +25,7 @@ import { loadMessagePack } from '../../chunk-R4CE4IVU.mjs';
|
|
|
25
25
|
import '../../chunk-U2BKYEC7.mjs';
|
|
26
26
|
import '../../chunk-OUWZKNO3.mjs';
|
|
27
27
|
import '../../chunk-3B775F3C.mjs';
|
|
28
|
-
import '../../chunk-
|
|
28
|
+
import '../../chunk-XXWEUXE4.mjs';
|
|
29
29
|
import '../../chunk-ZXX5GZEX.mjs';
|
|
30
30
|
import '../../chunk-4PY4JYMY.mjs';
|
|
31
31
|
import '../../chunk-Q3ZZH5Z7.mjs';
|