@delali/narsil 0.3.1-next.131 → 0.3.1-next.141
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/analysis-change-HQUT2Z3I.mjs +43 -0
- package/dist/analysis-change-entry-TZTG4AZB.mjs +1 -0
- package/dist/browser/analysis-change-JPV775LR.mjs +900 -0
- package/dist/browser/analysis-change-entry-7HR4LWL3.mjs +1 -0
- package/dist/browser/{chunk-UDGB2MWT.mjs → chunk-35B42UFM.mjs} +4 -0
- package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
- package/dist/browser/{chunk-2Y3CO4EQ.mjs → chunk-5DB2VDGI.mjs} +1149 -1853
- package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
- package/dist/browser/chunk-CCOAOO73.mjs +86 -0
- package/dist/browser/{chunk-AIIJDWMG.mjs → chunk-D33A5VMU.mjs} +16 -16
- package/dist/browser/{chunk-VBBLTD7O.mjs → chunk-HKX2G6WH.mjs} +1520 -2159
- package/dist/browser/{chunk-NSCSR6MA.mjs → chunk-KTOK275N.mjs} +606 -830
- package/dist/browser/{chunk-D7WQF6DV.mjs → chunk-LVY2LI35.mjs} +2 -2
- package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
- package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
- package/dist/browser/{chunk-VEGHFP2P.mjs → chunk-MZI3TEOW.mjs} +4 -2
- package/dist/browser/chunk-OIZS5KIX.mjs +6 -0
- package/dist/browser/chunk-R4WUDZQV.mjs +176 -0
- package/dist/browser/{chunk-M5UAAXGU.mjs → chunk-TCLEOZ4P.mjs} +3 -3
- package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
- package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
- package/dist/browser/chunk-XNWPP5NS.mjs +21 -0
- package/dist/browser/chunk-ZEV26AQ6.mjs +270 -0
- package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
- package/dist/browser/{durability-wiring-FNZSRH4N.mjs → durability-wiring-SNOWIT6Z.mjs} +370 -95
- package/dist/browser/import-TLB2A5LJ.mjs +9 -0
- package/dist/browser/module-BUFNZMEU.mjs +12 -0
- package/dist/browser/{module-4HIMOQM5.mjs → module-BVP4TKZU.mjs} +397 -109
- package/dist/browser/{phrase-module-WESYVU7K.mjs → phrase-module-HGVBO24W.mjs} +1 -1
- package/dist/{restore-queue-JAMNRC6T.mjs → browser/restore-queue-ONYNYUDE.mjs} +8 -2
- package/dist/browser/{runtime-3X7ZJ5T3.mjs → runtime-VEUH674B.mjs} +2 -2
- package/dist/browser/{snapshot-V7VF6BN6.mjs → snapshot-QXBKHJAG.mjs} +135 -183
- package/dist/browser/{write-YIC2MKRY.mjs → write-WHFBDWQS.mjs} +64 -90
- package/dist/{capabilities-B6WxrWxt.d.ts → capabilities-BCdUwK_L.d.ts} +10 -3
- package/dist/chunk-46NZOPHS.mjs +21 -0
- package/dist/{chunk-JT3YZGCE.mjs → chunk-5WYVUPGD.mjs} +3 -188
- package/dist/chunk-6SNBQESJ.mjs +214 -0
- package/dist/{chunk-2C3WXBQS.mjs → chunk-74XB2SXI.mjs} +1 -1
- package/dist/chunk-7AUASRE7.mjs +653 -0
- package/dist/{chunk-NMXRXFGZ.mjs → chunk-AST3SHFA.mjs} +27 -25
- package/dist/chunk-BOJZGUS6.mjs +177 -0
- package/dist/{chunk-T4OFG4IQ.mjs → chunk-BRUUBXCB.mjs} +92 -17
- package/dist/{chunk-5LX6MOIB.mjs → chunk-C3YDYX6C.mjs} +1 -1
- package/dist/chunk-CZQ33DLW.mjs +78 -0
- package/dist/{chunk-4GDLFDPZ.mjs → chunk-DD2673TX.mjs} +3 -2
- package/dist/{chunk-EUCX5K74.mjs → chunk-DQNKVOJ2.mjs} +65 -65
- package/dist/{chunk-RNSC4NFI.mjs → chunk-EEBCDP36.mjs} +3 -1
- package/dist/chunk-FOCV4CBZ.mjs +87 -0
- package/dist/{chunk-HCGJKDZV.mjs → chunk-GLZHNSGB.mjs} +1 -1
- package/dist/chunk-GWOS5DYC.mjs +2637 -0
- package/dist/{chunk-PATMDIMN.mjs → chunk-HJJWY6PT.mjs} +1 -1
- package/dist/{chunk-CZ7ISLXA.mjs → chunk-HNHCNTVG.mjs} +4 -0
- package/dist/{chunk-Y2U5VVWX.mjs → chunk-ICW45TTU.mjs} +1 -1
- package/dist/{chunk-TGHNXN7W.mjs → chunk-IO5F7ZBE.mjs} +261 -948
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/{chunk-7OIQ2SIO.mjs → chunk-KEIYVZUX.mjs} +5 -5
- package/dist/{chunk-PU6SKH7O.mjs → chunk-KGOXHLLZ.mjs} +7 -3
- package/dist/{chunk-WVRDRLIC.mjs → chunk-KZL72VTC.mjs} +70 -18
- package/dist/chunk-L76W3IZJ.mjs +193 -0
- package/dist/{chunk-2JWNJSNE.mjs → chunk-LA4WNDHL.mjs} +88 -801
- package/dist/{chunk-ZR4VM4A6.mjs → chunk-N37VMC62.mjs} +1 -18
- package/dist/{chunk-JJ6EUMXB.mjs → chunk-NJJCPFXN.mjs} +2 -2
- package/dist/chunk-NN33CZRE.mjs +86 -0
- package/dist/{chunk-2WBY4Y4M.mjs → chunk-NOT5QOHU.mjs} +1 -1
- package/dist/{chunk-D5TFRB5G.mjs → chunk-NPNK57CA.mjs} +1 -1
- package/dist/chunk-NRUXBYPH.mjs +121 -0
- package/dist/{chunk-6SSVIW7Y.mjs → chunk-O4LJQI4G.mjs} +31 -5
- package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
- package/dist/{chunk-F6AAXI7T.mjs → chunk-RHKRTYGO.mjs} +13 -4
- package/dist/{chunk-2SEL6SBS.mjs → chunk-RKAVHNRW.mjs} +2 -2
- package/dist/{chunk-RCMNL7RI.mjs → chunk-SBMAWTRG.mjs} +41 -4
- package/dist/chunk-SRJBFG2T.mjs +881 -0
- package/dist/{chunk-X5CNUKD5.mjs → chunk-TNBIK2LQ.mjs} +1663 -1652
- package/dist/{chunk-NNO7CO3A.mjs → chunk-UIKN2OTQ.mjs} +1 -1
- package/dist/chunk-UUZCQQG2.mjs +209 -0
- package/dist/{chunk-6M6SDH3S.mjs → chunk-VEFN6X6Q.mjs} +8 -8
- package/dist/chunk-VJEGOELZ.mjs +177 -0
- package/dist/chunk-W2COJ5AH.mjs +148 -0
- package/dist/{chunk-RWESFSJI.mjs → chunk-WJGO5PGV.mjs} +317 -174
- package/dist/{chunk-7X5JST5L.mjs → chunk-WNBRW4Q2.mjs} +16 -2
- package/dist/{chunk-SZGRYEIC.mjs → chunk-XS2VM7UE.mjs} +3 -3
- package/dist/{chunk-Y4BVTQ53.mjs → chunk-XZ3SRNDJ.mjs} +25 -64
- package/dist/{chunk-K77FJXKS.mjs → chunk-YHUTF6EA.mjs} +474 -93
- package/dist/{chunk-OGLV4I3L.mjs → chunk-YLNSJQLX.mjs} +12 -7
- package/dist/{chunk-SOHMMNCH.mjs → chunk-Z4HFMYNX.mjs} +32 -27
- package/dist/{chunk-5W7AEVTH.mjs → chunk-ZMBFWQXN.mjs} +2 -2
- package/dist/{chunk-2MKX2SCO.mjs → chunk-ZPWS2SKJ.mjs} +297 -274
- package/dist/{chunk-7XY6VD7B.mjs → chunk-ZSWHPKGM.mjs} +1 -1
- package/dist/{client-gNAfaHcE.d.ts → client-B12YFALP.d.ts} +18 -3
- package/dist/client.d.ts +7 -7
- package/dist/client.mjs +11 -1
- package/dist/cold-module-4X7WJCFM.mjs +783 -0
- package/dist/{config-D1cDv3sE.d.ts → config-BhZxIeb1.d.ts} +10 -1
- package/dist/{config-CmYJO34X.d.ts → config-CIG5yDyg.d.ts} +2 -2
- package/dist/{direct-executor-HQ4QVDX7.mjs → direct-executor-7XOQ7VQP.mjs} +14 -14
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +24 -8
- package/dist/distribution.mjs +982 -211
- package/dist/{durability-wiring-S7OX2MAI.mjs → durability-wiring-324D2DDD.mjs} +362 -94
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-m_Ddr1bz.d.ts → engine-CtzbHOmG.d.ts} +64 -7
- package/dist/import-OOBUW3RR.mjs +44 -0
- package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
- package/dist/index.browser.mjs +5 -5
- package/dist/index.d.ts +6 -6
- package/dist/index.mjs +22 -22
- package/dist/{long-query-thread-BSL25PD6.mjs → long-query-thread-4SW7RWDE.mjs} +15 -15
- package/dist/{module-2E4XYSGT.mjs → module-DXSJ5LT2.mjs} +378 -87
- package/dist/module-IYPLWHGC.mjs +48 -0
- package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
- package/dist/partial-words.mjs +5 -5
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +16 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +28 -26
- package/dist/{phrase-module-JERF2IDG.mjs → phrase-module-5QHBD2VJ.mjs} +1 -1
- package/dist/react.d.ts +9 -9
- package/dist/react.mjs +4 -0
- package/dist/{rerank-DMMT4FZA.mjs → rerank-KTLC3E6N.mjs} +5 -5
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{browser/restore-queue-GEZ54CA5.mjs → restore-queue-FEYAC4SB.mjs} +8 -2
- package/dist/{runtime-7MLQ7J64.mjs → runtime-DYV75DCB.mjs} +6 -6
- package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
- package/dist/server.d.ts +12 -10
- package/dist/server.mjs +54 -21
- package/dist/{snapshot-DOQLGGEX.mjs → snapshot-CGM65VCD.mjs} +101 -197
- package/dist/synonyms.mjs +6 -4
- package/dist/{tasks-Co8HfRD7.d.ts → tasks-CjRJRGTb.d.ts} +26 -8
- package/dist/{thread-SKO7CN3R.mjs → thread-KGNAIKPD.mjs} +14 -14
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +6 -5
- package/dist/workers/entry.mjs +8 -8
- package/dist/{write-ZKR7SNRT.mjs → write-2DZPEQL2.mjs} +82 -107
- package/openapi.json +114 -6
- package/package.json +12 -8
|
@@ -1,21 +1,22 @@
|
|
|
1
1
|
import { noteWaitingForMemory, admitsBackgroundBytes } from './chunk-PELHVTO7.mjs';
|
|
2
|
-
import { VECTOR_WORKER_COPIES_ALLOWED, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, noBuildHistory, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, handOverBuildTurns,
|
|
2
|
+
import { VECTOR_WORKER_COPIES_ALLOWED, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, noBuildHistory, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, handOverBuildTurns, filterForOptions, OSQ4_MIN_DIMENSION, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, yieldBuildTurn, allLiveDocIds, threadsHoldCurrentLayout, WORKER_COPY_MIN_VECTORS, SHUFFLED_REBUILD_MEMORY_RETRY_MS, recalibrateFromStore, signatureOf, adoptGraph, BUILD_RETRY_MAX_DELAY_MS, BUILD_RETRY_FIRST_DELAY_MS, fieldHandlesOf, liveOrdinals, RECALIBRATION_CHUNK_VECTORS, BUILD_CHUNK_SIZE, DRIFT_CHECK_GROWTH_FACTOR, DISK_ADOPTION_YIELD_INTERVAL, adoptLoadedCalibration, RECALIBRATION_DRIFT_RATIO } from './chunk-Z4HFMYNX.mjs';
|
|
3
3
|
import { chargeHeapBytes } from './chunk-IWVEG5CX.mjs';
|
|
4
|
-
import { isDead, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors } from './chunk-
|
|
5
|
-
import { createVectorStore } from './chunk-
|
|
6
|
-
export { openSharedVectorField } from './chunk-
|
|
7
|
-
import
|
|
8
|
-
|
|
9
|
-
|
|
4
|
+
import { isDead, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors } from './chunk-ICW45TTU.mjs';
|
|
5
|
+
import { createVectorStore } from './chunk-XZ3SRNDJ.mjs';
|
|
6
|
+
export { openSharedVectorField } from './chunk-XZ3SRNDJ.mjs';
|
|
7
|
+
import './chunk-FOCV4CBZ.mjs';
|
|
8
|
+
import { osqBitsOf, createOsqQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, ordinalFilterValues, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, magnitude, nativeStoreScratchBytes, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, createHNSWIndex, topLayerInParts, sharedMemoryAvailable, requireWholeNeighbourLists, shuffleInPlace, createCentroidSums, centroidOfCentroidSums, meanOfCentroidSums, countOfCentroidSums, squaredLengthsOfCentroidSums, releaseLocksHeldBy } from './chunk-TNBIK2LQ.mjs';
|
|
9
|
+
export { releaseLocksHeldBy } from './chunk-TNBIK2LQ.mjs';
|
|
10
10
|
import './chunk-37CD3OKS.mjs';
|
|
11
|
+
import { noteFallback } from './chunk-DJHHF2L5.mjs';
|
|
11
12
|
import { spawnNodeWorker } from './chunk-W55X2BYN.mjs';
|
|
12
|
-
import { resolveWorkerCount, resolveWorkerEntry, workerResourceLimits } from './chunk-
|
|
13
|
+
import { resolveWorkerCount, resolveWorkerEntry, workerResourceLimits } from './chunk-UIKN2OTQ.mjs';
|
|
13
14
|
import { detectRuntime } from './chunk-AV4IJKD2.mjs';
|
|
15
|
+
import './chunk-EEBCDP36.mjs';
|
|
14
16
|
import './chunk-EBIHRL7B.mjs';
|
|
15
|
-
import './chunk-RNSC4NFI.mjs';
|
|
16
17
|
import './chunk-KOKM4PQN.mjs';
|
|
17
18
|
import { MAX_RENUMBERED_SEARCH_ATTEMPTS, DISK_STORAGE_BLOCK_BYTES, UNPLACED_FLAG_CHUNK_ORDINALS, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_INSERT_TIMEOUT_MS } from './chunk-QSK35UC2.mjs';
|
|
18
|
-
import './chunk-
|
|
19
|
+
import './chunk-74XB2SXI.mjs';
|
|
19
20
|
import './chunk-U476IXKQ.mjs';
|
|
20
21
|
import './chunk-FQBFZAWG.mjs';
|
|
21
22
|
import './chunk-VJS4JFBQ.mjs';
|
|
@@ -24,7 +25,7 @@ import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
|
24
25
|
import './chunk-RIIUZJQ3.mjs';
|
|
25
26
|
import './chunk-22YPZVMG.mjs';
|
|
26
27
|
import { noteThreadSlotLive, noteThreadSlotGone } from './chunk-RFBLQLLZ.mjs';
|
|
27
|
-
import { NarsilError, ErrorCodes, isNarsilError } from './chunk-
|
|
28
|
+
import { NarsilError, ErrorCodes, isNarsilError } from './chunk-HNHCNTVG.mjs';
|
|
28
29
|
|
|
29
30
|
// src/core/heap.ts
|
|
30
31
|
function createMaxHeap(compare) {
|
|
@@ -1607,19 +1608,25 @@ function beginCheckpointRestore(state, shape) {
|
|
|
1607
1608
|
const dead = isDead(file.dead, i);
|
|
1608
1609
|
if (dead && !graphHoldsNodeFor(firstNumber + i)) continue;
|
|
1609
1610
|
const docId = payload.docIds[i];
|
|
1610
|
-
const vector = vectors.subarray(i * dimension, (i + 1) * dimension);
|
|
1611
1611
|
const offset = onDisk ? location.vectorsOffset + i * dimension * 4 : 0;
|
|
1612
|
+
const vectorHere = () => file.byParts === void 0 ? vectors.subarray(i * dimension, (i + 1) * dimension) : file.byParts.vectorAt(i);
|
|
1612
1613
|
if (dead) {
|
|
1613
|
-
ordinals[i] = state.store.insertReplaced(
|
|
1614
|
+
ordinals[i] = state.store.insertReplaced(
|
|
1615
|
+
docId,
|
|
1616
|
+
vectorHere(),
|
|
1617
|
+
fileIndex === null ? void 0 : { fileIndex, offset }
|
|
1618
|
+
);
|
|
1614
1619
|
tombstoned.add(ordinals[i]);
|
|
1615
1620
|
continue;
|
|
1616
1621
|
}
|
|
1617
1622
|
const replaced = state.store.getOrdinal(docId);
|
|
1618
1623
|
if (replaced !== void 0) state.osq?.removeOrdinal(replaced);
|
|
1619
1624
|
if (fileIndex !== null) {
|
|
1620
|
-
|
|
1625
|
+
const known = file.byParts?.magnitudes[i];
|
|
1626
|
+
ordinals[i] = state.store.insertCold(docId, known ?? magnitude(vectorHere()), { fileIndex, offset });
|
|
1621
1627
|
continue;
|
|
1622
1628
|
}
|
|
1629
|
+
const vector = vectorHere();
|
|
1623
1630
|
ordinals[i] = state.store.insert(docId, vector);
|
|
1624
1631
|
if (onDisk) state.pendingLocations.set(docId, { path: location.path, offset });
|
|
1625
1632
|
}
|
|
@@ -1661,6 +1668,223 @@ function beginCheckpointRestore(state, shape) {
|
|
|
1661
1668
|
return { addFile, finish };
|
|
1662
1669
|
}
|
|
1663
1670
|
|
|
1671
|
+
// src/vector/vector-index/cold.ts
|
|
1672
|
+
function beginColdHold(state, field, loaded, searchAnswered) {
|
|
1673
|
+
const hold = {
|
|
1674
|
+
field,
|
|
1675
|
+
loaded,
|
|
1676
|
+
searchAnswered,
|
|
1677
|
+
written: /* @__PURE__ */ new Map(),
|
|
1678
|
+
removedNumbers: /* @__PURE__ */ new Set(),
|
|
1679
|
+
removedIds: /* @__PURE__ */ new Set(),
|
|
1680
|
+
writes: [],
|
|
1681
|
+
live: field.liveCount
|
|
1682
|
+
};
|
|
1683
|
+
state.cold = hold;
|
|
1684
|
+
return hold;
|
|
1685
|
+
}
|
|
1686
|
+
function endColdHold(state) {
|
|
1687
|
+
const hold = state.cold;
|
|
1688
|
+
if (hold === null) return;
|
|
1689
|
+
state.cold = null;
|
|
1690
|
+
for (const write of hold.writes) write();
|
|
1691
|
+
hold.field.close();
|
|
1692
|
+
}
|
|
1693
|
+
function cancelColdHold(state) {
|
|
1694
|
+
const hold = state.cold;
|
|
1695
|
+
if (hold === null) return;
|
|
1696
|
+
state.cold = null;
|
|
1697
|
+
hold.field.close();
|
|
1698
|
+
}
|
|
1699
|
+
function openColdField(state, field, load) {
|
|
1700
|
+
cancelColdHold(state);
|
|
1701
|
+
let finished = () => void 0;
|
|
1702
|
+
let failed = () => void 0;
|
|
1703
|
+
const loaded = new Promise((resolve, reject) => {
|
|
1704
|
+
finished = resolve;
|
|
1705
|
+
failed = reject;
|
|
1706
|
+
});
|
|
1707
|
+
let searchAnswered = () => void 0;
|
|
1708
|
+
const firstSearchAnswered = new Promise((resolve) => {
|
|
1709
|
+
searchAnswered = resolve;
|
|
1710
|
+
});
|
|
1711
|
+
const hold = beginColdHold(state, field, loaded, searchAnswered);
|
|
1712
|
+
void load(
|
|
1713
|
+
(shape) => beginCheckpointRestore(state, shape),
|
|
1714
|
+
() => state.cold === hold,
|
|
1715
|
+
firstSearchAnswered
|
|
1716
|
+
).then(
|
|
1717
|
+
() => {
|
|
1718
|
+
if (state.cold === hold) endColdHold(state);
|
|
1719
|
+
finished();
|
|
1720
|
+
},
|
|
1721
|
+
(error) => failed(error)
|
|
1722
|
+
);
|
|
1723
|
+
loaded.catch(() => void 0);
|
|
1724
|
+
}
|
|
1725
|
+
function retireOlderCopy(hold, docId) {
|
|
1726
|
+
if (hold.written.delete(docId)) return true;
|
|
1727
|
+
if (hold.removedIds.has(docId)) return false;
|
|
1728
|
+
const number = hold.field.numberOf(docId);
|
|
1729
|
+
if (number === void 0) return false;
|
|
1730
|
+
hold.removedIds.add(docId);
|
|
1731
|
+
hold.removedNumbers.add(number);
|
|
1732
|
+
return true;
|
|
1733
|
+
}
|
|
1734
|
+
function coldInsert(hold, docId, vector, partitionId) {
|
|
1735
|
+
const replaced = retireOlderCopy(hold, docId);
|
|
1736
|
+
if (!replaced) hold.live += 1;
|
|
1737
|
+
hold.written.set(docId, { vector: vector.slice(), partitionId });
|
|
1738
|
+
return replaced;
|
|
1739
|
+
}
|
|
1740
|
+
function coldRemove(hold, docId) {
|
|
1741
|
+
const removed = retireOlderCopy(hold, docId);
|
|
1742
|
+
if (removed) hold.live -= 1;
|
|
1743
|
+
return removed;
|
|
1744
|
+
}
|
|
1745
|
+
function coldHas(hold, docId) {
|
|
1746
|
+
if (hold.written.has(docId)) return true;
|
|
1747
|
+
if (hold.removedIds.has(docId)) return false;
|
|
1748
|
+
return hold.field.numberOf(docId) !== void 0;
|
|
1749
|
+
}
|
|
1750
|
+
function coldVectorOf(hold, docId) {
|
|
1751
|
+
const written = hold.written.get(docId);
|
|
1752
|
+
if (written !== void 0) return written.vector.slice();
|
|
1753
|
+
if (hold.removedIds.has(docId)) return null;
|
|
1754
|
+
const number = hold.field.numberOf(docId);
|
|
1755
|
+
return number === void 0 ? null : hold.field.vectorOfNumber(number);
|
|
1756
|
+
}
|
|
1757
|
+
function isAdmitted(subsets, docId) {
|
|
1758
|
+
for (const subset of subsets) {
|
|
1759
|
+
const { docStore } = subset.owner;
|
|
1760
|
+
if (docStore.numberingGeneration() !== subset.numberingGeneration) continue;
|
|
1761
|
+
const internalId = docStore.getInternalId(docId);
|
|
1762
|
+
if (internalId === void 0) continue;
|
|
1763
|
+
const word = subset.bits[internalId >>> 5];
|
|
1764
|
+
if (word !== void 0 && (word & 1 << (internalId & 31)) !== 0 && docStore.resolver().isLive(internalId)) {
|
|
1765
|
+
return true;
|
|
1766
|
+
}
|
|
1767
|
+
}
|
|
1768
|
+
return false;
|
|
1769
|
+
}
|
|
1770
|
+
function admissionOf(options, hold) {
|
|
1771
|
+
const { filterSubsets, filterPartitions } = options;
|
|
1772
|
+
if (filterSubsets !== void 0) return (docId) => isAdmitted(filterSubsets, docId);
|
|
1773
|
+
if (filterPartitions !== void 0) {
|
|
1774
|
+
return (docId) => {
|
|
1775
|
+
const partition = hold.field.partitionOf(docId);
|
|
1776
|
+
return partition !== void 0 && filterPartitions.has(partition);
|
|
1777
|
+
};
|
|
1778
|
+
}
|
|
1779
|
+
return null;
|
|
1780
|
+
}
|
|
1781
|
+
function bitCount(word) {
|
|
1782
|
+
let value = word - (word >>> 1 & 1431655765);
|
|
1783
|
+
value = (value & 858993459) + (value >>> 2 & 858993459);
|
|
1784
|
+
return (value + (value >>> 4) & 252645135) * 16843009 >>> 24;
|
|
1785
|
+
}
|
|
1786
|
+
function admittedBySubsets(subsets) {
|
|
1787
|
+
let admitted = 0;
|
|
1788
|
+
for (const subset of subsets) {
|
|
1789
|
+
for (let at = 0; at < subset.bits.length; at++) admitted += bitCount(subset.bits[at]);
|
|
1790
|
+
}
|
|
1791
|
+
return admitted;
|
|
1792
|
+
}
|
|
1793
|
+
function* idsOfSubsets(subsets) {
|
|
1794
|
+
for (const subset of subsets) {
|
|
1795
|
+
const { bits } = subset;
|
|
1796
|
+
const { docStore } = subset.owner;
|
|
1797
|
+
if (docStore.numberingGeneration() !== subset.numberingGeneration) continue;
|
|
1798
|
+
for (let wordIndex = 0; wordIndex < bits.length; wordIndex++) {
|
|
1799
|
+
let word = bits[wordIndex];
|
|
1800
|
+
while (word !== 0) {
|
|
1801
|
+
const internalId = (wordIndex << 5) + (Math.clz32(word & -word) ^ 31);
|
|
1802
|
+
word &= word - 1;
|
|
1803
|
+
if (!docStore.resolver().isLive(internalId)) continue;
|
|
1804
|
+
const docId = docStore.getExternalId(internalId);
|
|
1805
|
+
if (docId !== void 0) yield docId;
|
|
1806
|
+
}
|
|
1807
|
+
}
|
|
1808
|
+
}
|
|
1809
|
+
}
|
|
1810
|
+
function scoreOf(query, qMag, vector, metric) {
|
|
1811
|
+
const vectorMagnitude = metric === "cosine" ? magnitude(vector) : 0;
|
|
1812
|
+
return toScore(toDistance(query, vector, qMag, vectorMagnitude, metric), metric);
|
|
1813
|
+
}
|
|
1814
|
+
function scanWritten(hold, query, k, options, admits) {
|
|
1815
|
+
if (hold.written.size === 0) return [];
|
|
1816
|
+
const qMag = magnitude(query);
|
|
1817
|
+
const heap = createBoundedMaxHeap(highScoreFirst, Math.max(k, 1));
|
|
1818
|
+
for (const [docId, { vector }] of hold.written) {
|
|
1819
|
+
if (admits !== null && !admits(docId)) continue;
|
|
1820
|
+
const score = scoreOf(query, qMag, vector, options.metric);
|
|
1821
|
+
if (score >= options.minSimilarity) heap.push({ docId, score });
|
|
1822
|
+
}
|
|
1823
|
+
return heap.toSortedArray().reverse();
|
|
1824
|
+
}
|
|
1825
|
+
function scanSubsets(hold, query, k, options, subsets) {
|
|
1826
|
+
const qMag = magnitude(query);
|
|
1827
|
+
const heap = createBoundedMaxHeap(highScoreFirst, Math.max(k, 1));
|
|
1828
|
+
let matched = 0;
|
|
1829
|
+
for (const docId of idsOfSubsets(subsets)) {
|
|
1830
|
+
const written = hold.written.get(docId);
|
|
1831
|
+
let score;
|
|
1832
|
+
if (written !== void 0) {
|
|
1833
|
+
score = scoreOf(query, qMag, written.vector, options.metric);
|
|
1834
|
+
} else {
|
|
1835
|
+
if (hold.removedIds.has(docId)) continue;
|
|
1836
|
+
const number = hold.field.numberOf(docId);
|
|
1837
|
+
if (number === void 0) continue;
|
|
1838
|
+
score = scoreOf(query, qMag, hold.field.reusedVectorOfNumber(number), options.metric);
|
|
1839
|
+
}
|
|
1840
|
+
if (score < options.minSimilarity) continue;
|
|
1841
|
+
matched += 1;
|
|
1842
|
+
heap.push({ docId, score });
|
|
1843
|
+
}
|
|
1844
|
+
return { results: heap.toSortedArray().reverse(), matched };
|
|
1845
|
+
}
|
|
1846
|
+
function outcomeWithoutGraphWalk(state, hold, query, k, options) {
|
|
1847
|
+
if (hold.live === 0) return { results: [], matched: 0, matchedExact: true };
|
|
1848
|
+
const subsets = options.filterSubsets;
|
|
1849
|
+
let admitted = hold.live;
|
|
1850
|
+
if (subsets !== void 0) {
|
|
1851
|
+
admitted = admittedBySubsets(subsets);
|
|
1852
|
+
if (admitted === 0) return { results: [], matched: 0, matchedExact: true };
|
|
1853
|
+
if (admitted / hold.live < state.filterThreshold) {
|
|
1854
|
+
const { results, matched } = scanSubsets(hold, query, k, options, subsets);
|
|
1855
|
+
return { results: results.slice(0, Math.max(k, 0)), matched, matchedExact: true };
|
|
1856
|
+
}
|
|
1857
|
+
}
|
|
1858
|
+
if (k <= 0) {
|
|
1859
|
+
return options.minSimilarity === Number.NEGATIVE_INFINITY ? { results: [], matched: admitted, matchedExact: true } : { results: [], matched: 0, matchedExact: false };
|
|
1860
|
+
}
|
|
1861
|
+
return { admits: admissionOf(options, hold), admitted };
|
|
1862
|
+
}
|
|
1863
|
+
function outcomeOfGraphWalk(hold, walk, fromGraph, query, k, options) {
|
|
1864
|
+
const results = mergeResults(fromGraph, scanWritten(hold, query, k, options, walk.admits), k);
|
|
1865
|
+
return options.minSimilarity === Number.NEGATIVE_INFINITY ? { results, matched: walk.admitted, matchedExact: true } : { results, matched: results.length, matchedExact: false };
|
|
1866
|
+
}
|
|
1867
|
+
function coldSearch(state, hold, query, k, options) {
|
|
1868
|
+
const walk = outcomeWithoutGraphWalk(state, hold, query, k, options);
|
|
1869
|
+
if ("results" in walk) {
|
|
1870
|
+
hold.searchAnswered();
|
|
1871
|
+
return walk;
|
|
1872
|
+
}
|
|
1873
|
+
const fromGraph = hold.field.search(query, k, options, walk.admits, hold.removedNumbers);
|
|
1874
|
+
hold.searchAnswered();
|
|
1875
|
+
return outcomeOfGraphWalk(hold, walk, fromGraph, query, k, options);
|
|
1876
|
+
}
|
|
1877
|
+
async function coldSearchConcurrently(state, hold, query, k, options) {
|
|
1878
|
+
const walk = outcomeWithoutGraphWalk(state, hold, query, k, options);
|
|
1879
|
+
if ("results" in walk) {
|
|
1880
|
+
hold.searchAnswered();
|
|
1881
|
+
return walk;
|
|
1882
|
+
}
|
|
1883
|
+
const fromGraph = await hold.field.searchConcurrently(query, k, options, walk.admits, hold.removedNumbers);
|
|
1884
|
+
hold.searchAnswered();
|
|
1885
|
+
return outcomeOfGraphWalk(hold, walk, fromGraph, query, k, options);
|
|
1886
|
+
}
|
|
1887
|
+
|
|
1664
1888
|
// src/vector/vector-index/flagged-documents.ts
|
|
1665
1889
|
var FlaggedDocuments = class extends Set {
|
|
1666
1890
|
constructor(store, layout) {
|
|
@@ -1812,7 +2036,7 @@ function completeGraph(state) {
|
|
|
1812
2036
|
}
|
|
1813
2037
|
function maintenanceStatus(state) {
|
|
1814
2038
|
const storeSize = state.store.size;
|
|
1815
|
-
const graphCount = state.hnsw ? 1 : 0;
|
|
2039
|
+
const graphCount = state.hnsw || state.cold ? 1 : 0;
|
|
1816
2040
|
const graphNodes = state.hnsw === null ? storeSize : state.hnsw.size + state.hnsw.tombstoneCount;
|
|
1817
2041
|
const tombstoned = state.tombstones.size + state.store.replacedVectorCount;
|
|
1818
2042
|
const tombstoneRatio = graphNodes > 0 ? tombstoned / graphNodes : 0;
|
|
@@ -1975,6 +2199,91 @@ function withBufferedVectors(state, query, k, options, filter, graphResults, buf
|
|
|
1975
2199
|
return mergeResults(graphResults, scan.results, k);
|
|
1976
2200
|
}
|
|
1977
2201
|
|
|
2202
|
+
// src/vector/vector-index/writes.ts
|
|
2203
|
+
function createVectorWrites(state) {
|
|
2204
|
+
function validateDimension(vector) {
|
|
2205
|
+
if (vector.length !== state.dimension) {
|
|
2206
|
+
throw new NarsilError(
|
|
2207
|
+
ErrorCodes.VECTOR_DIMENSION_MISMATCH,
|
|
2208
|
+
`Vector dimension mismatch: expected ${state.dimension}, got ${vector.length}`,
|
|
2209
|
+
{ expected: state.dimension, received: vector.length }
|
|
2210
|
+
);
|
|
2211
|
+
}
|
|
2212
|
+
}
|
|
2213
|
+
function heldVectorOrUndefinedWhereItsFileFailsToRead(ordinal) {
|
|
2214
|
+
try {
|
|
2215
|
+
return state.store.entryForOrdinal(ordinal)?.vector;
|
|
2216
|
+
} catch {
|
|
2217
|
+
return void 0;
|
|
2218
|
+
}
|
|
2219
|
+
}
|
|
2220
|
+
function holdsThisVectorAlready(docId, ordinal, vector) {
|
|
2221
|
+
if (state.tombstones.has(docId)) return false;
|
|
2222
|
+
const held = heldVectorOrUndefinedWhereItsFileFailsToRead(ordinal);
|
|
2223
|
+
if (held === void 0) return false;
|
|
2224
|
+
for (let i = 0; i < vector.length; i++) if (held[i] !== vector[i]) return false;
|
|
2225
|
+
return true;
|
|
2226
|
+
}
|
|
2227
|
+
function applyInsert(docId, vector, partitionId) {
|
|
2228
|
+
handOverBuildTurns(state);
|
|
2229
|
+
const previous = state.store.getOrdinal(docId);
|
|
2230
|
+
if (previous !== void 0 && holdsThisVectorAlready(docId, previous, vector)) {
|
|
2231
|
+
if (partitionId !== void 0 && state.store.partitionOfOrdinal(previous) !== partitionId) {
|
|
2232
|
+
noteWrite(state);
|
|
2233
|
+
state.store.setPartition(docId, partitionId);
|
|
2234
|
+
}
|
|
2235
|
+
return;
|
|
2236
|
+
}
|
|
2237
|
+
noteWrite(state);
|
|
2238
|
+
state.tombstones.delete(docId);
|
|
2239
|
+
const graphHoldsPrevious = previous !== void 0 && ((state.hnsw?.holdsNode(previous) ?? false) || (state.freshGraph?.holdsNode(previous) ?? false));
|
|
2240
|
+
state.store.insert(docId, vector, partitionId, graphHoldsPrevious);
|
|
2241
|
+
if (previous !== void 0) {
|
|
2242
|
+
state.hnsw?.markTombstoneOrdinal(previous);
|
|
2243
|
+
state.freshGraph?.markTombstoneOrdinal(previous);
|
|
2244
|
+
if (!graphHoldsPrevious) state.osq?.removeOrdinal(previous);
|
|
2245
|
+
}
|
|
2246
|
+
state.buffer.add(docId);
|
|
2247
|
+
state.pendingLocations.delete(docId);
|
|
2248
|
+
if (!threadsHoldCurrentLayout(state)) scheduleWorkerCopyLoad(state);
|
|
2249
|
+
}
|
|
2250
|
+
function applyRemove(docId) {
|
|
2251
|
+
handOverBuildTurns(state);
|
|
2252
|
+
if (!state.store.has(docId)) return;
|
|
2253
|
+
noteWrite(state);
|
|
2254
|
+
state.tombstones.add(docId);
|
|
2255
|
+
state.buffer.delete(docId);
|
|
2256
|
+
state.pendingLocations.delete(docId);
|
|
2257
|
+
if (state.hnsw) {
|
|
2258
|
+
state.hnsw.markTombstone(docId);
|
|
2259
|
+
}
|
|
2260
|
+
}
|
|
2261
|
+
return {
|
|
2262
|
+
insert(docId, vector, partitionId) {
|
|
2263
|
+
validateDimension(vector);
|
|
2264
|
+
const hold = state.cold;
|
|
2265
|
+
if (hold === null) {
|
|
2266
|
+
applyInsert(docId, vector, partitionId);
|
|
2267
|
+
return;
|
|
2268
|
+
}
|
|
2269
|
+
noteWrite(state);
|
|
2270
|
+
coldInsert(hold, docId, vector, partitionId);
|
|
2271
|
+
const held = vector.slice();
|
|
2272
|
+
hold.writes.push(() => applyInsert(docId, held, partitionId));
|
|
2273
|
+
},
|
|
2274
|
+
remove(docId) {
|
|
2275
|
+
const hold = state.cold;
|
|
2276
|
+
if (hold === null) {
|
|
2277
|
+
applyRemove(docId);
|
|
2278
|
+
return;
|
|
2279
|
+
}
|
|
2280
|
+
if (!coldRemove(hold, docId)) return;
|
|
2281
|
+
noteWrite(state);
|
|
2282
|
+
hold.writes.push(() => applyRemove(docId));
|
|
2283
|
+
}
|
|
2284
|
+
};
|
|
2285
|
+
}
|
|
2286
|
+
|
|
1978
2287
|
// src/vector/vector-index/index.ts
|
|
1979
2288
|
function defaultQuantizationFor(dimension) {
|
|
1980
2289
|
if (dimension >= OSQ4_MIN_DIMENSION) return "osq4";
|
|
@@ -2045,76 +2354,26 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
2045
2354
|
sharedHandles: /* @__PURE__ */ new Map(),
|
|
2046
2355
|
sharedLayoutRevision: 0,
|
|
2047
2356
|
sharing: Promise.resolve(),
|
|
2048
|
-
releaseToFilesInFlight: Promise.resolve()
|
|
2357
|
+
releaseToFilesInFlight: Promise.resolve(),
|
|
2358
|
+
cold: null
|
|
2049
2359
|
};
|
|
2050
|
-
|
|
2051
|
-
if (vector.length !== state.dimension) {
|
|
2052
|
-
throw new NarsilError(
|
|
2053
|
-
ErrorCodes.VECTOR_DIMENSION_MISMATCH,
|
|
2054
|
-
`Vector dimension mismatch: expected ${state.dimension}, got ${vector.length}`,
|
|
2055
|
-
{ expected: state.dimension, received: vector.length }
|
|
2056
|
-
);
|
|
2057
|
-
}
|
|
2058
|
-
}
|
|
2059
|
-
function heldVectorOrUndefinedWhereItsFileFailsToRead(ordinal) {
|
|
2060
|
-
try {
|
|
2061
|
-
return state.store.entryForOrdinal(ordinal)?.vector;
|
|
2062
|
-
} catch {
|
|
2063
|
-
return void 0;
|
|
2064
|
-
}
|
|
2065
|
-
}
|
|
2066
|
-
function holdsThisVectorAlready(docId, ordinal, vector) {
|
|
2067
|
-
if (state.tombstones.has(docId)) return false;
|
|
2068
|
-
const held = heldVectorOrUndefinedWhereItsFileFailsToRead(ordinal);
|
|
2069
|
-
if (held === void 0) return false;
|
|
2070
|
-
for (let i = 0; i < vector.length; i++) if (held[i] !== vector[i]) return false;
|
|
2071
|
-
return true;
|
|
2072
|
-
}
|
|
2073
|
-
function insert(docId, vector, partitionId) {
|
|
2074
|
-
validateDimension(vector);
|
|
2075
|
-
handOverBuildTurns(state);
|
|
2076
|
-
const previous = state.store.getOrdinal(docId);
|
|
2077
|
-
if (previous !== void 0 && holdsThisVectorAlready(docId, previous, vector)) {
|
|
2078
|
-
if (partitionId !== void 0 && state.store.partitionOfOrdinal(previous) !== partitionId) {
|
|
2079
|
-
noteWrite(state);
|
|
2080
|
-
state.store.setPartition(docId, partitionId);
|
|
2081
|
-
}
|
|
2082
|
-
return;
|
|
2083
|
-
}
|
|
2084
|
-
noteWrite(state);
|
|
2085
|
-
state.tombstones.delete(docId);
|
|
2086
|
-
const graphHoldsPrevious = previous !== void 0 && ((state.hnsw?.holdsNode(previous) ?? false) || (state.freshGraph?.holdsNode(previous) ?? false));
|
|
2087
|
-
state.store.insert(docId, vector, partitionId, graphHoldsPrevious);
|
|
2088
|
-
if (previous !== void 0) {
|
|
2089
|
-
state.hnsw?.markTombstoneOrdinal(previous);
|
|
2090
|
-
state.freshGraph?.markTombstoneOrdinal(previous);
|
|
2091
|
-
if (!graphHoldsPrevious) state.osq?.removeOrdinal(previous);
|
|
2092
|
-
}
|
|
2093
|
-
state.buffer.add(docId);
|
|
2094
|
-
state.pendingLocations.delete(docId);
|
|
2095
|
-
if (!threadsHoldCurrentLayout(state)) scheduleWorkerCopyLoad(state);
|
|
2096
|
-
}
|
|
2097
|
-
function remove(docId) {
|
|
2098
|
-
handOverBuildTurns(state);
|
|
2099
|
-
if (!state.store.has(docId)) return;
|
|
2100
|
-
noteWrite(state);
|
|
2101
|
-
state.tombstones.add(docId);
|
|
2102
|
-
state.buffer.delete(docId);
|
|
2103
|
-
state.pendingLocations.delete(docId);
|
|
2104
|
-
if (state.hnsw) {
|
|
2105
|
-
state.hnsw.markTombstone(docId);
|
|
2106
|
-
}
|
|
2107
|
-
}
|
|
2360
|
+
const { insert, remove } = createVectorWrites(state);
|
|
2108
2361
|
function getVector(docId) {
|
|
2362
|
+
if (state.cold !== null) return coldVectorOf(state.cold, docId);
|
|
2109
2363
|
if (state.tombstones.has(docId)) return null;
|
|
2110
2364
|
const entry = state.store.get(docId);
|
|
2111
2365
|
if (!entry) return null;
|
|
2112
2366
|
return new Float32Array(entry.vector);
|
|
2113
2367
|
}
|
|
2114
2368
|
function has(docId) {
|
|
2369
|
+
if (state.cold !== null) return coldHas(state.cold, docId);
|
|
2115
2370
|
return state.store.has(docId) && !state.tombstones.has(docId);
|
|
2116
2371
|
}
|
|
2372
|
+
async function whenLoaded() {
|
|
2373
|
+
while (state.cold !== null) await state.cold.loaded;
|
|
2374
|
+
}
|
|
2117
2375
|
async function searchParallel(query, k, options) {
|
|
2376
|
+
if (state.cold !== null) return coldSearchConcurrently(state, state.cold, query, k, options);
|
|
2118
2377
|
handOverBuildTurns(state);
|
|
2119
2378
|
const confined = options.filterSubsets !== void 0 || options.filterPartitions !== void 0;
|
|
2120
2379
|
let filter = filterForOptions(state, options);
|
|
@@ -2161,44 +2420,73 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
2161
2420
|
get storage() {
|
|
2162
2421
|
return state.storage;
|
|
2163
2422
|
},
|
|
2423
|
+
get metric() {
|
|
2424
|
+
return state.metric;
|
|
2425
|
+
},
|
|
2164
2426
|
insert,
|
|
2165
2427
|
remove,
|
|
2166
|
-
partitionsKnown: () => state.store.partitionsKnown,
|
|
2167
|
-
assignPartitions: (resolve) =>
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
|
|
2171
|
-
|
|
2172
|
-
|
|
2428
|
+
partitionsKnown: () => state.cold !== null || state.store.partitionsKnown,
|
|
2429
|
+
assignPartitions: (resolve) => {
|
|
2430
|
+
if (state.cold === null) assignStorePartitions(state, resolve);
|
|
2431
|
+
},
|
|
2432
|
+
scheduleBuild: () => {
|
|
2433
|
+
if (state.cold === null) scheduleBuild(state);
|
|
2434
|
+
},
|
|
2435
|
+
awaitPendingBuild: async () => {
|
|
2436
|
+
await whenLoaded();
|
|
2437
|
+
await awaitBuilds(state);
|
|
2438
|
+
},
|
|
2439
|
+
completeGraph: async () => {
|
|
2440
|
+
await whenLoaded();
|
|
2441
|
+
await completeGraph(state);
|
|
2442
|
+
},
|
|
2443
|
+
dispose: () => {
|
|
2444
|
+
cancelColdHold(state);
|
|
2445
|
+
disposeIndex(state);
|
|
2446
|
+
},
|
|
2447
|
+
search: (query, k, options) => state.cold === null ? search(state, query, k, options) : coldSearch(state, state.cold, query, k, options),
|
|
2173
2448
|
searchParallel,
|
|
2174
|
-
refreshWorkerCopies: () =>
|
|
2449
|
+
refreshWorkerCopies: () => {
|
|
2450
|
+
if (state.cold === null) refreshWorkerCopies(state);
|
|
2451
|
+
},
|
|
2175
2452
|
getVector,
|
|
2176
2453
|
has,
|
|
2177
2454
|
compact: () => {
|
|
2455
|
+
if (state.cold !== null) return;
|
|
2178
2456
|
noteWrite(state);
|
|
2179
2457
|
compact(state);
|
|
2180
2458
|
},
|
|
2181
2459
|
optimize: async () => {
|
|
2460
|
+
await whenLoaded();
|
|
2182
2461
|
noteWrite(state);
|
|
2183
2462
|
await optimize(state);
|
|
2184
2463
|
},
|
|
2185
|
-
maintenanceStatus: () => maintenanceStatus(state),
|
|
2186
|
-
estimateMemoryBytes: () => estimateMemoryBytes(state),
|
|
2464
|
+
maintenanceStatus: () => ({ ...maintenanceStatus(state), loadingFromFiles: state.cold !== null }),
|
|
2465
|
+
estimateMemoryBytes: () => estimateMemoryBytes(state) + (state.cold?.field.residentBytes ?? 0),
|
|
2187
2466
|
growthBytesFor: (vectors) => state.store.growthBytesFor(vectors),
|
|
2188
2467
|
deserialize: (parts) => {
|
|
2468
|
+
cancelColdHold(state);
|
|
2189
2469
|
invalidateWorkerCopies(state);
|
|
2190
2470
|
deserialize(state, parts);
|
|
2191
2471
|
},
|
|
2192
2472
|
planCheckpoint: (listedKeys, rewritesGraph) => planCheckpoint(state, listedKeys, rewritesGraph),
|
|
2193
2473
|
recordCheckpoint: (plan, written) => recordCheckpoint(state, plan, written),
|
|
2194
2474
|
restoreCheckpoint: (shape) => {
|
|
2475
|
+
cancelColdHold(state);
|
|
2195
2476
|
invalidateWorkerCopies(state);
|
|
2196
2477
|
return beginCheckpointRestore(state, shape);
|
|
2197
2478
|
},
|
|
2198
|
-
|
|
2199
|
-
|
|
2479
|
+
openColdField: (field, load) => openColdField(state, field, load),
|
|
2480
|
+
startShuffledRebuildWhenDue: () => {
|
|
2481
|
+
if (state.cold === null) startShuffledRebuild(state);
|
|
2482
|
+
},
|
|
2483
|
+
vectorFilesInUse: () => {
|
|
2484
|
+
const stored = state.store.dimension === 0 ? [] : state.store.handles.vectorFiles.filter((path) => path !== "");
|
|
2485
|
+
return state.cold === null ? stored : [...stored, ...state.cold.field.filePaths()];
|
|
2486
|
+
},
|
|
2200
2487
|
adoptDiskLayout: (layout) => adoptDiskLayout(state, layout),
|
|
2201
2488
|
releaseVectorFiles: async () => {
|
|
2489
|
+
await whenLoaded();
|
|
2202
2490
|
await withdrawWorkerCopies(state);
|
|
2203
2491
|
state.store.closeFiles();
|
|
2204
2492
|
},
|
|
@@ -2208,7 +2496,10 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
2208
2496
|
state.pendingLocations.set(docId, { ...location, path: rewrite(location.path) });
|
|
2209
2497
|
}
|
|
2210
2498
|
},
|
|
2211
|
-
holdWorkerCopies: () =>
|
|
2499
|
+
holdWorkerCopies: async () => {
|
|
2500
|
+
await whenLoaded();
|
|
2501
|
+
await holdWorkerCopies(state);
|
|
2502
|
+
},
|
|
2212
2503
|
resumeWorkerCopies: () => {
|
|
2213
2504
|
state.workerCopiesHeld = false;
|
|
2214
2505
|
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
export { actionPartitionsOf, adoptSegmentsOnWorker, bytesThatSegmentsHoldInMemory, checkPagedSegmentFully, closeUnusedSegments, countsAsLatestHolder, createPagedFrozenSegment, isPagedSegmentFile, keysThatSegmentsStillRead, mergePagedWindow, openFileVectorReads, openPagedSegmentScan, openPagedSnapshot, openRewriteSpill, pagedListingOf, pagedPartitionLines, pagedTombstonesFor, prepareDiskAdoption, startReadingOpeningSectionsAhead, swapPreparedPartitions, writePagedSegment } from './chunk-GWOS5DYC.mjs';
|
|
2
|
+
import './chunk-NN33CZRE.mjs';
|
|
3
|
+
import './chunk-BOJZGUS6.mjs';
|
|
4
|
+
import './chunk-W2COJ5AH.mjs';
|
|
5
|
+
import './chunk-FOCV4CBZ.mjs';
|
|
6
|
+
import './chunk-7AUASRE7.mjs';
|
|
7
|
+
import './chunk-UUZCQQG2.mjs';
|
|
8
|
+
import './chunk-KEIYVZUX.mjs';
|
|
9
|
+
import './chunk-IO5F7ZBE.mjs';
|
|
10
|
+
import './chunk-VEFN6X6Q.mjs';
|
|
11
|
+
import './chunk-5WYVUPGD.mjs';
|
|
12
|
+
import './chunk-HJSN7UYG.mjs';
|
|
13
|
+
import './chunk-UIKN2OTQ.mjs';
|
|
14
|
+
import './chunk-AV4IJKD2.mjs';
|
|
15
|
+
import './chunk-EEBCDP36.mjs';
|
|
16
|
+
import './chunk-WJGO5PGV.mjs';
|
|
17
|
+
import './chunk-ZPWS2SKJ.mjs';
|
|
18
|
+
import './chunk-N37VMC62.mjs';
|
|
19
|
+
import './chunk-ZATRDAYU.mjs';
|
|
20
|
+
import './chunk-EBIHRL7B.mjs';
|
|
21
|
+
import './chunk-KOKM4PQN.mjs';
|
|
22
|
+
import './chunk-QSK35UC2.mjs';
|
|
23
|
+
import './chunk-74XB2SXI.mjs';
|
|
24
|
+
import './chunk-RKAVHNRW.mjs';
|
|
25
|
+
import './chunk-WQBCBKY3.mjs';
|
|
26
|
+
import './chunk-JUKYYXSP.mjs';
|
|
27
|
+
import './chunk-DQNKVOJ2.mjs';
|
|
28
|
+
import './chunk-KGOXHLLZ.mjs';
|
|
29
|
+
import './chunk-U476IXKQ.mjs';
|
|
30
|
+
import './chunk-FQBFZAWG.mjs';
|
|
31
|
+
import './chunk-VJS4JFBQ.mjs';
|
|
32
|
+
import './chunk-GES7U2PM.mjs';
|
|
33
|
+
import './chunk-ZSWHPKGM.mjs';
|
|
34
|
+
import './chunk-O4LJQI4G.mjs';
|
|
35
|
+
import './chunk-X5CV2E2B.mjs';
|
|
36
|
+
import './chunk-WNBRW4Q2.mjs';
|
|
37
|
+
import './chunk-TGP5M5R7.mjs';
|
|
38
|
+
import './chunk-U2BKYEC7.mjs';
|
|
39
|
+
import './chunk-NPNK57CA.mjs';
|
|
40
|
+
import './chunk-4UMZ7KQL.mjs';
|
|
41
|
+
import './chunk-R6A773WK.mjs';
|
|
42
|
+
import './chunk-YTJCGKM4.mjs';
|
|
43
|
+
import './chunk-QHPYS427.mjs';
|
|
44
|
+
import './chunk-GLZHNSGB.mjs';
|
|
45
|
+
import './chunk-22YPZVMG.mjs';
|
|
46
|
+
import './chunk-RFBLQLLZ.mjs';
|
|
47
|
+
import './chunk-L5ORED7V.mjs';
|
|
48
|
+
import './chunk-HNHCNTVG.mjs';
|