@delali/narsil 0.3.1-next.163 → 0.3.1-next.169
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -8
- package/dist/browser/{chunk-MZHQ65VI.mjs → chunk-2Q45HI5H.mjs} +14 -10
- package/dist/browser/{chunk-MMHIIL5B.mjs → chunk-4ZICWS4V.mjs} +1 -1
- package/dist/browser/{chunk-KQXKCMGQ.mjs → chunk-JHFBQKUF.mjs} +119 -90
- package/dist/browser/{core-leaf-SIWXFZLV.mjs → core-leaf-WYFANMKC.mjs} +61 -13
- package/dist/browser/custom-token-results-R5BIX7SL.mjs +59 -0
- package/dist/browser/{embed-RF2MPGAO.mjs → embed-24IPR4LP.mjs} +2 -2
- package/dist/browser/{module-PMQBRESM.mjs → module-XVLHJ7T7.mjs} +358 -62
- package/dist/browser/{optional-writes-WMZX4OEC.mjs → optional-writes-TB34FYUW.mjs} +118 -66
- package/dist/browser/{phrase-module-22TKMXWI.mjs → phrase-module-6UGFVW5J.mjs} +1 -0
- package/dist/browser/{resolve-vector-text-KI65IHMD.mjs → resolve-vector-text-EJ6DHFF5.mjs} +8 -0
- package/dist/browser/{storage-leaf-VGR2BWJJ.mjs → storage-leaf-QFXIWDFK.mjs} +208 -87
- package/dist/browser/{worker-copies-WQCPP3TS.mjs → worker-copies-ICHLYYXM.mjs} +1 -0
- package/dist/{capabilities-DkXAkb-X.d.ts → capabilities-BQIdsaWy.d.ts} +2 -2
- package/dist/{chunk-BKOMO7HT.mjs → chunk-26E5MO3Y.mjs} +5 -5
- package/dist/{chunk-EEYDJB4Z.mjs → chunk-2C3BXZGA.mjs} +213 -149
- package/dist/{chunk-NXMGUDZA.mjs → chunk-2KWTQ2P7.mjs} +2 -2
- package/dist/{chunk-TW543F33.mjs → chunk-34IOOYXA.mjs} +1 -1
- package/dist/{chunk-D5GNFN6X.mjs → chunk-7FC2ZILY.mjs} +98 -40
- package/dist/{chunk-XRBPHCCT.mjs → chunk-A4XYDUVN.mjs} +5 -5
- package/dist/{chunk-R4F7ZTVG.mjs → chunk-BIIROQ5Q.mjs} +2 -2
- package/dist/{chunk-Q2WQSAYH.mjs → chunk-CQX5B6L2.mjs} +54 -11
- package/dist/{chunk-XXWEUXE4.mjs → chunk-D6FYUTFN.mjs} +1 -1
- package/dist/{chunk-OUWZKNO3.mjs → chunk-DG6WIEB3.mjs} +1 -1
- package/dist/{chunk-GJKFXKMI.mjs → chunk-F3FDHXCJ.mjs} +12 -9
- package/dist/{chunk-3UGZBRDN.mjs → chunk-GO6TRSJ6.mjs} +1 -1
- package/dist/{react-native/chunks/chunk-OK3WUA6N.mjs → chunk-I2WJCRIH.mjs} +1 -1
- package/dist/{chunk-Q7CUUSJ2.mjs → chunk-ISJ35523.mjs} +2 -2
- package/dist/{chunk-GQLOEACG.mjs → chunk-JCWC75NH.mjs} +2 -2
- package/dist/{chunk-ZGYF37MG.mjs → chunk-JDIATDPY.mjs} +1 -1
- package/dist/{chunk-3B775F3C.mjs → chunk-JP77OWMI.mjs} +3 -1
- package/dist/{chunk-KSOZJEYU.mjs → chunk-KHJPMZE2.mjs} +1 -1
- package/dist/{chunk-ZQ6QY42L.mjs → chunk-KL6W73BZ.mjs} +8 -0
- package/dist/{chunk-ES5EOXT6.mjs → chunk-LOQHA6QK.mjs} +3 -3
- package/dist/{chunk-AB27FWZM.mjs → chunk-LPRRFCVU.mjs} +2 -2
- package/dist/{chunk-EIR3IQCL.mjs → chunk-QFJMRB7D.mjs} +2 -1
- package/dist/{chunk-S3BSDBF7.mjs → chunk-QJGDB3WR.mjs} +1 -1
- package/dist/{chunk-EI7U43MX.mjs → chunk-R7PLSVK3.mjs} +2 -2
- package/dist/{chunk-5DUTOEK5.mjs → chunk-TBOZHYCF.mjs} +2 -2
- package/dist/{chunk-VJOS6MHF.mjs → chunk-TNEU32EN.mjs} +64 -16
- package/dist/{chunk-RXMR2SIQ.mjs → chunk-UISV46WX.mjs} +4 -4
- package/dist/{chunk-4CAMH7FO.mjs → chunk-X4UMDSB4.mjs} +2 -1
- package/dist/{chunk-R4CE4IVU.mjs → chunk-X5NHWOXT.mjs} +15 -11
- package/dist/{react-native/chunks/chunk-HEELWLKL.mjs → chunk-XFOL77WH.mjs} +1 -1
- package/dist/{chunk-TOZENMM7.mjs → chunk-XITVDRRH.mjs} +5 -5
- package/dist/{chunk-FSLQHOKZ.mjs → chunk-XNFBYAXI.mjs} +1 -1
- package/dist/{chunk-V4YIEMAI.mjs → chunk-XQQKETAF.mjs} +4 -4
- package/dist/{chunk-DSXFN5BF.mjs → chunk-XTSFRWAS.mjs} +8 -8
- package/dist/{chunk-K4BMDPPX.mjs → chunk-Y27E2MGC.mjs} +2 -2
- package/dist/{chunk-JV74RKCY.mjs → chunk-YM2HI5N5.mjs} +1 -1
- package/dist/{chunk-7OE73OO4.mjs → chunk-YPV2L7M5.mjs} +56 -6
- package/dist/{chunk-STLNPJXU.mjs → chunk-YXZDTKQS.mjs} +2 -2
- package/dist/{chunk-5P72V66D.mjs → chunk-ZB5X5WAK.mjs} +24 -20
- package/dist/{client-VJmV_AjZ.d.ts → client-CeUxvIvd.d.ts} +4 -4
- package/dist/client.d.ts +8 -8
- package/dist/{config-ZZg4cwQz.d.ts → config-CIpLXXxm.d.ts} +2 -2
- package/dist/{config-Cb3FOlE_.d.ts → config-DehU-VGL.d.ts} +2 -2
- package/dist/{core-leaf-LOXCD7MY.mjs → core-leaf-VIMO377O.mjs} +3 -3
- package/dist/custom-token-results-NXHBVY7U.mjs +59 -0
- package/dist/{direct-executor-PF4UDSUW.mjs → direct-executor-2WQSGBN3.mjs} +7 -7
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution/transport/tcp.mjs +1 -1
- package/dist/distribution.d.ts +7 -7
- package/dist/distribution.mjs +39 -39
- package/dist/{embed-4H7RTBVL.mjs → embed-UNHDFGDU.mjs} +2 -2
- package/dist/{engine-DdOs682x.d.ts → engine-P-4xhUe4.d.ts} +6 -2
- package/dist/{index-Ck9w5WHz.d.ts → index-CBUl4z3E.d.ts} +1 -1
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +6 -6
- package/dist/index.mjs +7 -7
- package/dist/index.react-native.mjs +6 -6
- package/dist/{long-query-thread-OTVLP2OI.mjs → long-query-thread-AAYTLYZK.mjs} +19 -19
- package/dist/{module-UIKSHY3G.mjs → module-5SXNUCD5.mjs} +315 -64
- package/dist/{msgpack-leaf-5WFRP6JM.mjs → msgpack-leaf-7KBYOVTF.mjs} +2 -2
- package/dist/{optional-reads-RSLNSGNG.mjs → optional-reads-ENKMVQ4Q.mjs} +1 -1
- package/dist/{optional-writes-4ITCV7YT.mjs → optional-writes-N2DFWWZ3.mjs} +39 -26
- package/dist/{paged-partition-archive-VMJ2AUA6.mjs → paged-partition-archive-X7L27E5Z.mjs} +19 -19
- package/dist/partial-words.browser.mjs +3 -3
- package/dist/partial-words.mjs +7 -7
- package/dist/pattern.browser.mjs +2 -2
- package/dist/pattern.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +1 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +17 -17
- package/dist/{phrase-module-IWSG2DDZ.mjs → phrase-module-ZY3TLTQA.mjs} +2 -1
- package/dist/{query-actions-EW72OQXY.mjs → query-actions-2JOLPELC.mjs} +7 -7
- package/dist/query-suggestions/merge-worker.mjs +10 -10
- package/dist/query-suggestions.browser.mjs +3 -3
- package/dist/query-suggestions.d.ts +2 -2
- package/dist/query-suggestions.mjs +8 -8
- package/dist/{chunk-ZBVHS5KF.mjs → react-native/chunks/chunk-57R74JWC.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-AUXXPSFB.mjs → chunk-F4MIGBBP.mjs} +214 -150
- package/dist/react-native/chunks/{chunk-3RUXKZJG.mjs → chunk-F6BJYCF2.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-RVHCQJQM.mjs → chunk-GGBATERA.mjs} +3 -3
- package/dist/react-native/chunks/{chunk-I5W7A2JF.mjs → chunk-GVWLACAT.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-XOCM3BFU.mjs → chunk-H344ZXRH.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-VIPNDN4B.mjs → chunk-I7DTUOK2.mjs} +65 -15
- package/dist/react-native/chunks/{chunk-2JMAUU26.mjs → chunk-J2BQTJJE.mjs} +1 -1
- package/dist/{chunk-QBVTYY4J.mjs → react-native/chunks/chunk-OZ3XEM6D.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-4EXKQ2IB.mjs → chunk-SK3DPAI4.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-XBTFK3AZ.mjs → chunk-WFQ6YQZA.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-FTQ7JNMU.mjs → chunk-ZDDJVR7X.mjs} +15 -11
- package/dist/react-native/chunks/{core-leaf-O3236QY3.mjs → core-leaf-UDUMVC3M.mjs} +2 -2
- package/dist/react-native/chunks/custom-token-results-3OZEMRE7.mjs +59 -0
- package/dist/react-native/chunks/{embed-3WQ2CCR5.mjs → embed-U7CIKEY4.mjs} +2 -2
- package/dist/react-native/chunks/{module-LRR7D5ZO.mjs → module-7BJ3VOWQ.mjs} +360 -66
- package/dist/react-native/chunks/{msgpack-leaf-TR66ZVAX.mjs → msgpack-leaf-GTKLF6GJ.mjs} +2 -2
- package/dist/react-native/chunks/{optional-reads-TOWGNEDH.mjs → optional-reads-74HYI4DO.mjs} +1 -1
- package/dist/react-native/chunks/{optional-writes-7AFBIZMN.mjs → optional-writes-TXTJ6HUZ.mjs} +37 -24
- package/dist/react-native/chunks/{phrase-module-3A4AKMKL.mjs → phrase-module-UC4YGEBB.mjs} +2 -1
- package/dist/react-native/chunks/{resolve-vector-text-LXRYXF7M.mjs → resolve-vector-text-FTPD5EY6.mjs} +8 -0
- package/dist/react-native/chunks/{restore-queue-7FSHTGJ5.mjs → restore-queue-SZLWIL3B.mjs} +9 -9
- package/dist/react-native/chunks/{segment-codec-leaf-4QV4HO4R.mjs → segment-codec-leaf-WMUZ5P4F.mjs} +7 -7
- package/dist/react-native/chunks/{segment-merge-leaf-5DUZZSXJ.mjs → segment-merge-leaf-Q2OEJIQ3.mjs} +8 -8
- package/dist/react-native/chunks/{storage-leaf-PNU6HWY7.mjs → storage-leaf-OXYV6O6P.mjs} +213 -100
- package/dist/react-native/chunks/{store-leaf-ESVUGYYO.mjs → store-leaf-ZMDXYK4E.mjs} +8 -8
- package/dist/react-native/chunks/{suggestions-leaf-MOVOPBGD.mjs → suggestions-leaf-FIMGQZ2T.mjs} +8 -8
- package/dist/react-native/chunks/{worker-copies-VQGF3WZX.mjs → worker-copies-72TGFYQB.mjs} +8 -7
- package/dist/react-native/partial-words.mjs +6 -6
- package/dist/react-native/pattern.mjs +3 -3
- package/dist/react-native/query-suggestions.mjs +6 -6
- package/dist/react-native/sync.mjs +6 -6
- package/dist/react-native/synonyms.mjs +6 -6
- package/dist/react.d.ts +9 -9
- package/dist/{rerank-QHJXPSAK.mjs → rerank-E2HFKKGJ.mjs} +7 -7
- package/dist/{resolve-vector-text-4PDRTPNQ.mjs → resolve-vector-text-6WPFFIX5.mjs} +1 -1
- package/dist/{restore-queue-5W5HX44H.mjs → restore-queue-PUVBN6L3.mjs} +10 -10
- package/dist/{schema-C_7yRJLt.d.ts → schema-CzG9OUw5.d.ts} +5 -1
- package/dist/{segment-codec-leaf-7VLVXKWO.mjs → segment-codec-leaf-24Z3MMQI.mjs} +8 -8
- package/dist/{segment-merge-leaf-22XR3IGA.mjs → segment-merge-leaf-FQQDU45E.mjs} +8 -8
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +26 -26
- package/dist/{storage-leaf-ZZ77C3DX.mjs → storage-leaf-KIA7XO6Y.mjs} +79 -81
- package/dist/{store-leaf-LJ6D2PMF.mjs → store-leaf-ESVLL4JO.mjs} +9 -9
- package/dist/{suggestions-leaf-KTQHO3PY.mjs → suggestions-leaf-FXF4AHAY.mjs} +11 -11
- package/dist/sync/filesystem.d.ts +3 -3
- package/dist/sync/filesystem.mjs +15 -15
- package/dist/sync/indexeddb.browser.mjs +1 -1
- package/dist/sync/indexeddb.d.ts +3 -3
- package/dist/sync/indexeddb.mjs +2 -2
- package/dist/sync.browser.mjs +3 -3
- package/dist/sync.d.ts +4 -4
- package/dist/sync.mjs +7 -7
- package/dist/synonyms.browser.mjs +3 -3
- package/dist/synonyms.mjs +7 -7
- package/dist/{tasks-CxA-J_Ty.d.ts → tasks-CXtQ3Zpp.d.ts} +4 -4
- package/dist/thread-OWXVP2ZA.mjs +30 -0
- package/dist/{types-BboB278g.d.ts → types-CDN70Y_C.d.ts} +2 -2
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +5 -5
- package/dist/{worker-copies-ODAAGQRC.mjs → worker-copies-XXX7PF4J.mjs} +10 -10
- package/dist/workers/entry.mjs +13 -13
- package/package.json +9 -8
- package/dist/thread-XWDK6TAR.mjs +0 -30
- /package/dist/{chunk-77AUVDQ5.mjs → chunk-CV3Q2EKO.mjs} +0 -0
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
// src/analysis/constants.ts
|
|
2
|
+
var MAX_TOKEN_POSITION = 4294967295;
|
|
3
|
+
|
|
4
|
+
// src/analysis/custom-token-host.ts
|
|
5
|
+
var ErrorCodes;
|
|
6
|
+
var NarsilError;
|
|
7
|
+
var withUnpairedSurrogatesReplaced;
|
|
8
|
+
function bindCustomTokenHost(host) {
|
|
9
|
+
({ ErrorCodes, NarsilError, withUnpairedSurrogatesReplaced } = host);
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
// src/analysis/custom-token-results.ts
|
|
13
|
+
function refuseTokenizerResult(source, message, details) {
|
|
14
|
+
const code = source === "document" ? ErrorCodes.DOC_VALIDATION_FAILED : ErrorCodes.CONFIG_INVALID;
|
|
15
|
+
throw new NarsilError(code, `The custom tokenizer ${message}`, details);
|
|
16
|
+
}
|
|
17
|
+
function kindOf(value) {
|
|
18
|
+
if (value === null) return "null";
|
|
19
|
+
return Array.isArray(value) ? "array" : typeof value;
|
|
20
|
+
}
|
|
21
|
+
function checkedTokenizerEntry(entry, at, source) {
|
|
22
|
+
const { token, position } = entry !== null && typeof entry === "object" ? entry : {};
|
|
23
|
+
if (typeof token !== "string") {
|
|
24
|
+
refuseTokenizerResult(
|
|
25
|
+
source,
|
|
26
|
+
`returned a token of type ${kindOf(token)} at entry ${at}, where a token is a string`,
|
|
27
|
+
{ entry: at, tokenType: kindOf(token) }
|
|
28
|
+
);
|
|
29
|
+
}
|
|
30
|
+
if (typeof position !== "number" || !Number.isInteger(position) || position < 0 || position > MAX_TOKEN_POSITION) {
|
|
31
|
+
const shown = typeof position === "number" ? String(position) : kindOf(position);
|
|
32
|
+
refuseTokenizerResult(
|
|
33
|
+
source,
|
|
34
|
+
`returned the position ${shown} at entry ${at}, where a position is a whole number from 0 to ${MAX_TOKEN_POSITION}`,
|
|
35
|
+
{ entry: at, position: shown }
|
|
36
|
+
);
|
|
37
|
+
}
|
|
38
|
+
return { token: withUnpairedSurrogatesReplaced(token), position };
|
|
39
|
+
}
|
|
40
|
+
function holdsPositionsOutOfOrder(entries) {
|
|
41
|
+
for (let index = 1; index < entries.length; index++) {
|
|
42
|
+
if (entries[index].position < entries[index - 1].position) return true;
|
|
43
|
+
}
|
|
44
|
+
return false;
|
|
45
|
+
}
|
|
46
|
+
function customTokensOf(tokenizer, text, source) {
|
|
47
|
+
const entries = tokenizer.tokenize(text);
|
|
48
|
+
if (!Array.isArray(entries)) {
|
|
49
|
+
refuseTokenizerResult(source, `returned ${kindOf(entries)}, where it returns a list of tokens`, {
|
|
50
|
+
received: kindOf(entries)
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
const checked = new Array(entries.length);
|
|
54
|
+
for (let at = 0; at < checked.length; at++) checked[at] = checkedTokenizerEntry(entries[at], at, source);
|
|
55
|
+
if (!holdsPositionsOutOfOrder(checked)) return checked;
|
|
56
|
+
return checked.sort((first, second) => first.position - second.position);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export { bindCustomTokenHost, customTokensOf };
|
|
@@ -34,10 +34,10 @@ function validateVector(vector, targetField, expectedDimensions) {
|
|
|
34
34
|
);
|
|
35
35
|
}
|
|
36
36
|
for (let i = 0; i < vector.length; i++) {
|
|
37
|
-
if (Number.
|
|
37
|
+
if (!Number.isFinite(vector[i])) {
|
|
38
38
|
throw new NarsilError(
|
|
39
39
|
ErrorCodes.EMBEDDING_FAILED,
|
|
40
|
-
`Adapter returned vector with
|
|
40
|
+
`Adapter returned vector with ${vector[i]} at index ${i} for field "${targetField}"`,
|
|
41
41
|
{ field: targetField, index: i }
|
|
42
42
|
);
|
|
43
43
|
}
|
|
@@ -268,6 +268,7 @@ var recordCheckpoint;
|
|
|
268
268
|
var releaseLocksHeldBy;
|
|
269
269
|
var releasePinsHeldBy;
|
|
270
270
|
var releaseReplacedVectors;
|
|
271
|
+
var requireBytesWithinLimit;
|
|
271
272
|
var requireUsableVectorOptions;
|
|
272
273
|
var requireWholeNeighbourLists;
|
|
273
274
|
var resetStoreHeader;
|
|
@@ -343,6 +344,7 @@ function bindVectorCore2Names(vectorCore) {
|
|
|
343
344
|
releaseLocksHeldBy,
|
|
344
345
|
releasePinsHeldBy,
|
|
345
346
|
releaseReplacedVectors,
|
|
347
|
+
requireBytesWithinLimit,
|
|
346
348
|
requireUsableVectorOptions,
|
|
347
349
|
requireWholeNeighbourLists,
|
|
348
350
|
resetStoreHeader,
|
|
@@ -716,6 +718,27 @@ function getVectorMaintenanceStatus(manager) {
|
|
|
716
718
|
return results;
|
|
717
719
|
}
|
|
718
720
|
|
|
721
|
+
// src/vector/query-values.ts
|
|
722
|
+
function isFloat32Representable(value) {
|
|
723
|
+
return typeof value === "number" && Number.isFinite(Math.fround(value));
|
|
724
|
+
}
|
|
725
|
+
function requireUsableQueryVector(value) {
|
|
726
|
+
if (value === void 0) return;
|
|
727
|
+
if (!Array.isArray(value) && !(value instanceof Float32Array)) {
|
|
728
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Vector query 'value' must be an array of numbers`, {
|
|
729
|
+
received: typeof value
|
|
730
|
+
});
|
|
731
|
+
}
|
|
732
|
+
for (let index = 0; index < value.length; index++) {
|
|
733
|
+
if (isFloat32Representable(value[index])) continue;
|
|
734
|
+
throw new NarsilError(
|
|
735
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
736
|
+
`Vector query 'value' element at index ${index} is not a number that a 32-bit float holds as a finite value`,
|
|
737
|
+
{ index, value: value[index] }
|
|
738
|
+
);
|
|
739
|
+
}
|
|
740
|
+
}
|
|
741
|
+
|
|
719
742
|
// src/core/heap.ts
|
|
720
743
|
function createMaxHeap(compare) {
|
|
721
744
|
return createHeap((a, b) => compare(b, a));
|
|
@@ -2003,15 +2026,26 @@ function ensureFloatSlot(store, ordinal) {
|
|
|
2003
2026
|
const block = handles.blocks[blockIndex];
|
|
2004
2027
|
if (block !== null) ensureBlockSlots(block, ordinal % handles.layout.capacity + 1);
|
|
2005
2028
|
}
|
|
2006
|
-
|
|
2029
|
+
var LONGEST_UTF8_BYTES_OF_A_UTF16_UNIT = 3;
|
|
2030
|
+
var encoder2 = new TextEncoder();
|
|
2031
|
+
function requireOrdinalWithinLimit(ordinal) {
|
|
2007
2032
|
const maxOrdinals = VECTOR_MEMORY_LIMITS.maxOrdinals;
|
|
2008
|
-
if (ordinal
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2033
|
+
if (ordinal < maxOrdinals) return;
|
|
2034
|
+
throw new NarsilError(
|
|
2035
|
+
ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
|
|
2036
|
+
`A vector field holds at most ${maxOrdinals} vectors, deleted ones included, until it is rebuilt`,
|
|
2037
|
+
{ ordinal, maxOrdinals }
|
|
2038
|
+
);
|
|
2039
|
+
}
|
|
2040
|
+
function requireRoomForSlot(store, docId) {
|
|
2041
|
+
const { header, docIdBytes } = store.handles;
|
|
2042
|
+
requireOrdinalWithinLimit(Atomics.load(header, STORE_SLOTS));
|
|
2043
|
+
const heldBytes = Atomics.load(header, STORE_DOC_ID_BYTES);
|
|
2044
|
+
if (heldBytes + docId.length * LONGEST_UTF8_BYTES_OF_A_UTF16_UNIT <= docIdBytes.maxByteLength) return;
|
|
2045
|
+
requireBytesWithinLimit(heldBytes + encoder2.encode(docId).length, docIdBytes.maxByteLength);
|
|
2046
|
+
}
|
|
2047
|
+
function ensureOrdinal(store, ordinal, cold) {
|
|
2048
|
+
requireOrdinalWithinLimit(ordinal);
|
|
2015
2049
|
const { handles } = store;
|
|
2016
2050
|
growBufferTo(handles.magnitudes, (ordinal + 1) * 8);
|
|
2017
2051
|
growBufferTo(handles.present, ordinal + 1);
|
|
@@ -2182,6 +2216,32 @@ function retireInWrite(store, ordinal, sequence) {
|
|
|
2182
2216
|
function endWrite(store, sequence) {
|
|
2183
2217
|
publishSequence(store.sequences, sequence);
|
|
2184
2218
|
}
|
|
2219
|
+
var NOT_RETIRED = 0n;
|
|
2220
|
+
function clearRetirement(store, ordinal) {
|
|
2221
|
+
const writes = store.sequences;
|
|
2222
|
+
coverOrdinal(writes, ordinal);
|
|
2223
|
+
Atomics.store(writes.retired, ordinal, NOT_RETIRED);
|
|
2224
|
+
}
|
|
2225
|
+
function reclaimOrdinal(state, docId, ordinal, partitionId, keepsReplacedVector) {
|
|
2226
|
+
const store = state.open;
|
|
2227
|
+
if (store === null) return false;
|
|
2228
|
+
const current = state.docToOrd.get(docId);
|
|
2229
|
+
if (current === ordinal) {
|
|
2230
|
+
clearRetirement(store, ordinal);
|
|
2231
|
+
return true;
|
|
2232
|
+
}
|
|
2233
|
+
if (state.ordToDoc[ordinal] !== void 0) return false;
|
|
2234
|
+
if (!state.replacedOrdinals.delete(ordinal) && !state.awaitingFree.delete(ordinal)) return false;
|
|
2235
|
+
const sequence = beginWrite(state, store);
|
|
2236
|
+
clearRetirement(store, ordinal);
|
|
2237
|
+
recordPartition(state, store, ordinal, partitionId);
|
|
2238
|
+
state.ordToDoc[ordinal] = docId;
|
|
2239
|
+
state.docToOrd.set(docId, ordinal);
|
|
2240
|
+
if (current !== void 0) retireInWrite(store, current, sequence);
|
|
2241
|
+
endWrite(store, sequence);
|
|
2242
|
+
settleReplacement(state, store, current, keepsReplacedVector);
|
|
2243
|
+
return true;
|
|
2244
|
+
}
|
|
2185
2245
|
function retireStoredOrdinal(state, ordinal) {
|
|
2186
2246
|
const store = state.open;
|
|
2187
2247
|
if (store === null) return false;
|
|
@@ -2435,6 +2495,12 @@ function createVectorStore(options) {
|
|
|
2435
2495
|
settleReplacement(state, store, previous, keepsReplacedVector);
|
|
2436
2496
|
return ordinal;
|
|
2437
2497
|
},
|
|
2498
|
+
requireRoomFor(docId) {
|
|
2499
|
+
if (state.open !== null) requireRoomForSlot(state.open, docId);
|
|
2500
|
+
},
|
|
2501
|
+
reclaim(docId, ordinal, partitionId, keepsReplacedVector = false) {
|
|
2502
|
+
return reclaimOrdinal(state, docId, ordinal, partitionId, keepsReplacedVector);
|
|
2503
|
+
},
|
|
2438
2504
|
retire(ordinal) {
|
|
2439
2505
|
retireStoredOrdinal(state, ordinal);
|
|
2440
2506
|
},
|
|
@@ -4232,7 +4298,10 @@ function beginColdHold(state, field, loaded, loadWanted) {
|
|
|
4232
4298
|
written: /* @__PURE__ */ new Map(),
|
|
4233
4299
|
removedNumbers: /* @__PURE__ */ new Set(),
|
|
4234
4300
|
removedIds: /* @__PURE__ */ new Set(),
|
|
4301
|
+
removals: /* @__PURE__ */ new Map(),
|
|
4302
|
+
replacements: /* @__PURE__ */ new Map(),
|
|
4235
4303
|
writes: [],
|
|
4304
|
+
reserved: { slots: 0, docIdBytes: 0 },
|
|
4236
4305
|
live: field.liveCount
|
|
4237
4306
|
};
|
|
4238
4307
|
state.cold = hold;
|
|
@@ -4242,7 +4311,12 @@ function endColdHold(state) {
|
|
|
4242
4311
|
const hold = state.cold;
|
|
4243
4312
|
if (hold === null) return;
|
|
4244
4313
|
state.cold = null;
|
|
4245
|
-
|
|
4314
|
+
try {
|
|
4315
|
+
for (const write of hold.writes) write();
|
|
4316
|
+
} catch (error) {
|
|
4317
|
+
state.cold = hold;
|
|
4318
|
+
throw error;
|
|
4319
|
+
}
|
|
4246
4320
|
hold.field.close();
|
|
4247
4321
|
}
|
|
4248
4322
|
async function untilColdLoaded(state) {
|
|
@@ -4274,35 +4348,11 @@ function openColdField(state, field, load) {
|
|
|
4274
4348
|
(shape) => beginCheckpointRestore(state, shape),
|
|
4275
4349
|
() => state.cold === hold,
|
|
4276
4350
|
loadingWanted
|
|
4277
|
-
).then(
|
|
4278
|
-
()
|
|
4279
|
-
|
|
4280
|
-
finished();
|
|
4281
|
-
},
|
|
4282
|
-
(error) => failed(error)
|
|
4283
|
-
);
|
|
4351
|
+
).then(() => {
|
|
4352
|
+
if (state.cold === hold) endColdHold(state);
|
|
4353
|
+
}).then(finished, failed);
|
|
4284
4354
|
loaded.catch(() => void 0);
|
|
4285
4355
|
}
|
|
4286
|
-
function retireOlderCopy(hold, docId) {
|
|
4287
|
-
if (hold.written.delete(docId)) return true;
|
|
4288
|
-
if (hold.removedIds.has(docId)) return false;
|
|
4289
|
-
const number = hold.field.numberOf(docId);
|
|
4290
|
-
if (number === void 0) return false;
|
|
4291
|
-
hold.removedIds.add(docId);
|
|
4292
|
-
hold.removedNumbers.add(number);
|
|
4293
|
-
return true;
|
|
4294
|
-
}
|
|
4295
|
-
function coldInsert(hold, docId, vector, partitionId) {
|
|
4296
|
-
const replaced = retireOlderCopy(hold, docId);
|
|
4297
|
-
if (!replaced) hold.live += 1;
|
|
4298
|
-
hold.written.set(docId, { vector: vector.slice(), partitionId });
|
|
4299
|
-
return replaced;
|
|
4300
|
-
}
|
|
4301
|
-
function coldRemove(hold, docId) {
|
|
4302
|
-
const removed = retireOlderCopy(hold, docId);
|
|
4303
|
-
if (removed) hold.live -= 1;
|
|
4304
|
-
return removed;
|
|
4305
|
-
}
|
|
4306
4356
|
function coldHas(hold, docId) {
|
|
4307
4357
|
if (hold.written.has(docId)) return true;
|
|
4308
4358
|
if (hold.removedIds.has(docId)) return false;
|
|
@@ -4446,6 +4496,131 @@ async function coldSearchConcurrently(state, hold, query, k, options) {
|
|
|
4446
4496
|
return outcomeOfGraphWalk(hold, walk, fromGraph, query, k, options);
|
|
4447
4497
|
}
|
|
4448
4498
|
|
|
4499
|
+
// src/vector/vector-index/cold-writes.ts
|
|
4500
|
+
var encoder3 = new TextEncoder();
|
|
4501
|
+
var queuedWriteUndone = () => void 0;
|
|
4502
|
+
function sameVector(held, vector) {
|
|
4503
|
+
for (let at = 0; at < vector.length; at++) if (held[at] !== vector[at]) return false;
|
|
4504
|
+
return true;
|
|
4505
|
+
}
|
|
4506
|
+
function claimsOfField(hold, wanted) {
|
|
4507
|
+
const atMost = hold.field.claimsAtMost;
|
|
4508
|
+
const fits = atMost.slots + wanted.slots <= VECTOR_MEMORY_LIMITS.maxOrdinals && atMost.docIdBytes + wanted.docIdBytes <= VECTOR_MEMORY_LIMITS.maxDocIdTableBytes;
|
|
4509
|
+
return fits ? atMost : hold.field.claims();
|
|
4510
|
+
}
|
|
4511
|
+
function requireRoomInHold(hold, docId) {
|
|
4512
|
+
const { reserved } = hold;
|
|
4513
|
+
const wanted = { slots: reserved.slots + 1, docIdBytes: reserved.docIdBytes + encoder3.encode(docId).length };
|
|
4514
|
+
const loaded = claimsOfField(hold, wanted);
|
|
4515
|
+
requireOrdinalWithinLimit(loaded.slots + reserved.slots);
|
|
4516
|
+
requireBytesWithinLimit(loaded.docIdBytes + wanted.docIdBytes, VECTOR_MEMORY_LIMITS.maxDocIdTableBytes);
|
|
4517
|
+
return wanted;
|
|
4518
|
+
}
|
|
4519
|
+
function reserveReplayedInsert(hold, docId) {
|
|
4520
|
+
const wanted = requireRoomInHold(hold, docId);
|
|
4521
|
+
const added = { slots: wanted.slots - hold.reserved.slots, docIdBytes: wanted.docIdBytes - hold.reserved.docIdBytes };
|
|
4522
|
+
hold.reserved.slots = wanted.slots;
|
|
4523
|
+
hold.reserved.docIdBytes = wanted.docIdBytes;
|
|
4524
|
+
return added;
|
|
4525
|
+
}
|
|
4526
|
+
function retireCopy(hold, docId) {
|
|
4527
|
+
const written = hold.written.get(docId);
|
|
4528
|
+
if (written !== void 0) {
|
|
4529
|
+
hold.written.delete(docId);
|
|
4530
|
+
return written;
|
|
4531
|
+
}
|
|
4532
|
+
if (hold.removedIds.has(docId)) return void 0;
|
|
4533
|
+
const number = hold.field.numberOf(docId);
|
|
4534
|
+
if (number === void 0) return void 0;
|
|
4535
|
+
hold.removedIds.add(docId);
|
|
4536
|
+
hold.removedNumbers.add(number);
|
|
4537
|
+
return number;
|
|
4538
|
+
}
|
|
4539
|
+
function vectorOfCopy(hold, copy) {
|
|
4540
|
+
return typeof copy === "number" ? hold.field.reusedVectorOfNumber(copy) : copy.vector;
|
|
4541
|
+
}
|
|
4542
|
+
function putBackCopy(hold, docId, copy) {
|
|
4543
|
+
if (typeof copy === "number") {
|
|
4544
|
+
hold.removedIds.delete(docId);
|
|
4545
|
+
hold.removedNumbers.delete(copy);
|
|
4546
|
+
} else {
|
|
4547
|
+
hold.written.set(docId, copy);
|
|
4548
|
+
}
|
|
4549
|
+
}
|
|
4550
|
+
function restoreRemovedCopy(hold, docId, vector) {
|
|
4551
|
+
const removal = hold.removals.get(docId);
|
|
4552
|
+
if (removal === void 0) return false;
|
|
4553
|
+
const { copy, queuedAt } = removal;
|
|
4554
|
+
if (!sameVector(vectorOfCopy(hold, copy), vector)) return false;
|
|
4555
|
+
hold.removals.delete(docId);
|
|
4556
|
+
hold.writes[queuedAt] = queuedWriteUndone;
|
|
4557
|
+
putBackCopy(hold, docId, copy);
|
|
4558
|
+
hold.live += 1;
|
|
4559
|
+
return true;
|
|
4560
|
+
}
|
|
4561
|
+
function keepReplacement(hold, docId, replacement) {
|
|
4562
|
+
const kept = hold.replacements.get(docId);
|
|
4563
|
+
if (kept === void 0) hold.replacements.set(docId, [replacement]);
|
|
4564
|
+
else kept.push(replacement);
|
|
4565
|
+
}
|
|
4566
|
+
function fieldHoldsVector(hold, docId, vector) {
|
|
4567
|
+
if (hold.removedIds.has(docId)) return false;
|
|
4568
|
+
const number = hold.field.numberOf(docId);
|
|
4569
|
+
return number !== void 0 && sameVector(hold.field.reusedVectorOfNumber(number), vector);
|
|
4570
|
+
}
|
|
4571
|
+
function coldInsert(hold, docId, vector, partitionId, keepsReplacedCopy, replay) {
|
|
4572
|
+
const restored = !hold.written.has(docId) && restoreRemovedCopy(hold, docId, vector);
|
|
4573
|
+
const written = hold.written.get(docId);
|
|
4574
|
+
if (written !== void 0 && sameVector(written.vector, vector)) {
|
|
4575
|
+
if (partitionId === void 0 || partitionId === written.partitionId) return restored;
|
|
4576
|
+
hold.written.set(docId, { vector: written.vector, partitionId });
|
|
4577
|
+
hold.writes.push(replay);
|
|
4578
|
+
return true;
|
|
4579
|
+
}
|
|
4580
|
+
if (written === void 0 && (restored || fieldHoldsVector(hold, docId, vector))) {
|
|
4581
|
+
if (partitionId === void 0) return restored;
|
|
4582
|
+
hold.writes.push(replay);
|
|
4583
|
+
return true;
|
|
4584
|
+
}
|
|
4585
|
+
const reserved = reserveReplayedInsert(hold, docId);
|
|
4586
|
+
const removal = hold.removals.get(docId);
|
|
4587
|
+
hold.removals.delete(docId);
|
|
4588
|
+
const retired = retireCopy(hold, docId);
|
|
4589
|
+
if (retired === void 0) hold.live += 1;
|
|
4590
|
+
const inserted = { vector, partitionId };
|
|
4591
|
+
hold.written.set(docId, inserted);
|
|
4592
|
+
const copy = removal === void 0 ? retired : removal.copy;
|
|
4593
|
+
if (keepsReplacedCopy && copy !== void 0) {
|
|
4594
|
+
const queuedAt = removal === void 0 ? [hold.writes.length] : [removal.queuedAt, hold.writes.length];
|
|
4595
|
+
keepReplacement(hold, docId, { copy, inserted, queuedAt, reserved });
|
|
4596
|
+
}
|
|
4597
|
+
hold.writes.push(replay);
|
|
4598
|
+
return true;
|
|
4599
|
+
}
|
|
4600
|
+
function coldRemove(hold, docId, replay) {
|
|
4601
|
+
const copy = retireCopy(hold, docId);
|
|
4602
|
+
if (copy === void 0) return false;
|
|
4603
|
+
hold.live -= 1;
|
|
4604
|
+
hold.removals.set(docId, { queuedAt: hold.writes.length, copy });
|
|
4605
|
+
hold.writes.push(replay);
|
|
4606
|
+
return true;
|
|
4607
|
+
}
|
|
4608
|
+
function coldRestore(hold, docId, vector) {
|
|
4609
|
+
if (!hold.written.has(docId)) return restoreRemovedCopy(hold, docId, vector);
|
|
4610
|
+
const replacements = hold.replacements.get(docId);
|
|
4611
|
+
const last = replacements?.[replacements.length - 1];
|
|
4612
|
+
if (replacements === void 0 || last === void 0 || hold.written.get(docId) !== last.inserted) return false;
|
|
4613
|
+
if (!sameVector(vectorOfCopy(hold, last.copy), vector)) return false;
|
|
4614
|
+
replacements.pop();
|
|
4615
|
+
if (replacements.length === 0) hold.replacements.delete(docId);
|
|
4616
|
+
for (const at of last.queuedAt) hold.writes[at] = queuedWriteUndone;
|
|
4617
|
+
hold.written.delete(docId);
|
|
4618
|
+
putBackCopy(hold, docId, last.copy);
|
|
4619
|
+
hold.reserved.slots -= last.reserved.slots;
|
|
4620
|
+
hold.reserved.docIdBytes -= last.reserved.docIdBytes;
|
|
4621
|
+
return true;
|
|
4622
|
+
}
|
|
4623
|
+
|
|
4449
4624
|
// src/vector/vector-index/flagged-documents.ts
|
|
4450
4625
|
var FlaggedDocuments = class {
|
|
4451
4626
|
constructor(store, layout) {
|
|
@@ -4530,8 +4705,10 @@ var FlaggedDocuments = class {
|
|
|
4530
4705
|
const ordinal = this.store.getOrdinal(docId);
|
|
4531
4706
|
const previous = this.ordinals.get(docId);
|
|
4532
4707
|
if (ordinal === void 0 || ordinal === previous) return;
|
|
4533
|
-
if (this.
|
|
4534
|
-
|
|
4708
|
+
if (!this.retired.delete(ordinal)) {
|
|
4709
|
+
if (this.chunks !== null) Atomics.add(this.chunks, Math.floor(ordinal / UNPLACED_FLAG_CHUNK_ORDINALS), 1);
|
|
4710
|
+
Atomics.store(this.flags, ordinal, 1);
|
|
4711
|
+
}
|
|
4535
4712
|
this.ordinals.set(docId, ordinal);
|
|
4536
4713
|
if (previous === void 0) return;
|
|
4537
4714
|
if (this.layout.keepsRetiredFlags) this.retired.add(previous);
|
|
@@ -4697,6 +4874,47 @@ function vectorPublicationSource(state) {
|
|
|
4697
4874
|
};
|
|
4698
4875
|
}
|
|
4699
4876
|
|
|
4877
|
+
// src/vector/vector-index/rollback-holds.ts
|
|
4878
|
+
function releaseHold(state, docId, hold) {
|
|
4879
|
+
hold.writes -= 1;
|
|
4880
|
+
if (hold.writes > 0) return;
|
|
4881
|
+
if (state.rollbackHolds.get(docId) === hold) state.rollbackHolds.delete(docId);
|
|
4882
|
+
for (const { view, pin } of hold.pins) storePinsOf(view).release(pin);
|
|
4883
|
+
hold.pins.length = 0;
|
|
4884
|
+
hold.ordinals.length = 0;
|
|
4885
|
+
state.cold?.replacements.delete(docId);
|
|
4886
|
+
}
|
|
4887
|
+
function holdForRollback(state, docId) {
|
|
4888
|
+
let hold = state.rollbackHolds.get(docId);
|
|
4889
|
+
if (hold === void 0) {
|
|
4890
|
+
hold = { writes: 0, ordinals: [], pins: [] };
|
|
4891
|
+
state.rollbackHolds.set(docId, hold);
|
|
4892
|
+
}
|
|
4893
|
+
hold.writes += 1;
|
|
4894
|
+
const held = hold;
|
|
4895
|
+
let released = false;
|
|
4896
|
+
return () => {
|
|
4897
|
+
if (released) return;
|
|
4898
|
+
released = true;
|
|
4899
|
+
releaseHold(state, docId, held);
|
|
4900
|
+
};
|
|
4901
|
+
}
|
|
4902
|
+
function keepRetiredOrdinal(state, docId, ordinal) {
|
|
4903
|
+
const hold = state.rollbackHolds.get(docId);
|
|
4904
|
+
if (hold === void 0 || hold.ordinals[hold.ordinals.length - 1] === ordinal) return;
|
|
4905
|
+
const view = state.store.view;
|
|
4906
|
+
hold.pins.push({ view, pin: storePinsOf(view).take() });
|
|
4907
|
+
hold.ordinals.push(ordinal);
|
|
4908
|
+
}
|
|
4909
|
+
function forgetKeptOrdinal(state, docId, ordinal) {
|
|
4910
|
+
const ordinals = state.rollbackHolds.get(docId)?.ordinals;
|
|
4911
|
+
const at = ordinals?.lastIndexOf(ordinal) ?? -1;
|
|
4912
|
+
if (ordinals !== void 0 && at !== -1) ordinals.splice(at, 1);
|
|
4913
|
+
}
|
|
4914
|
+
function keptOrdinalsOf(state, docId) {
|
|
4915
|
+
return state.rollbackHolds.get(docId)?.ordinals ?? [];
|
|
4916
|
+
}
|
|
4917
|
+
|
|
4700
4918
|
// src/vector/vector-index/search.ts
|
|
4701
4919
|
function* bufferCandidates(state, buffered, filter) {
|
|
4702
4920
|
for (const docId of buffered) {
|
|
@@ -5110,9 +5328,7 @@ function createVectorWrites(state) {
|
|
|
5110
5328
|
function holdsThisVectorAlready(docId, ordinal, vector) {
|
|
5111
5329
|
if (state.tombstones.has(docId)) return false;
|
|
5112
5330
|
const held = heldVectorOrUndefinedWhereItsFileFailsToRead(ordinal);
|
|
5113
|
-
|
|
5114
|
-
for (let i = 0; i < vector.length; i++) if (held[i] !== vector[i]) return false;
|
|
5115
|
-
return true;
|
|
5331
|
+
return held !== void 0 && sameVector(held, vector);
|
|
5116
5332
|
}
|
|
5117
5333
|
function forgetRetiredFlags() {
|
|
5118
5334
|
const oldestPin = state.store.oldestPin();
|
|
@@ -5129,27 +5345,69 @@ function createVectorWrites(state) {
|
|
|
5129
5345
|
}
|
|
5130
5346
|
return;
|
|
5131
5347
|
}
|
|
5348
|
+
state.store.requireRoomFor(docId);
|
|
5132
5349
|
noteWrite(state);
|
|
5133
5350
|
noteSearchedDocWritten(state, docId);
|
|
5134
5351
|
state.tombstones.delete(docId);
|
|
5135
|
-
const graphHoldsPrevious = previous !== void 0 && (
|
|
5352
|
+
const graphHoldsPrevious = previous !== void 0 && graphsHoldNode(previous);
|
|
5353
|
+
if (previous !== void 0) keepRetiredOrdinal(state, docId, previous);
|
|
5136
5354
|
state.store.insert(docId, vector, partitionId, graphHoldsPrevious);
|
|
5137
|
-
if (previous !== void 0)
|
|
5138
|
-
|
|
5139
|
-
|
|
5140
|
-
|
|
5141
|
-
|
|
5142
|
-
|
|
5143
|
-
|
|
5144
|
-
|
|
5145
|
-
|
|
5146
|
-
|
|
5147
|
-
|
|
5355
|
+
if (previous !== void 0) retireFromGraphs(previous, graphHoldsPrevious);
|
|
5356
|
+
finishPlacingInBuffer(docId);
|
|
5357
|
+
}
|
|
5358
|
+
function graphsHoldNode(ordinal) {
|
|
5359
|
+
return (state.hnsw?.holdsNode(ordinal) ?? false) || (state.freshGraph?.holdsNode(ordinal) ?? false) || (state.growingCopy?.holdsNode(ordinal) ?? false);
|
|
5360
|
+
}
|
|
5361
|
+
function retireFromGraphs(previous, graphHoldsPrevious) {
|
|
5362
|
+
const graph = state.hnsw;
|
|
5363
|
+
const freshGraph = state.freshGraph;
|
|
5364
|
+
const growingCopy = state.growingCopy;
|
|
5365
|
+
afterGraphPass(state, () => {
|
|
5366
|
+
graph?.markTombstoneOrdinal(previous);
|
|
5367
|
+
freshGraph?.markTombstoneOrdinal(previous);
|
|
5368
|
+
growingCopy?.markTombstoneOrdinal(previous);
|
|
5369
|
+
});
|
|
5370
|
+
if (!graphHoldsPrevious) state.osq?.removeOrdinal(previous);
|
|
5371
|
+
}
|
|
5372
|
+
function finishPlacingInBuffer(docId) {
|
|
5148
5373
|
state.buffer.add(docId);
|
|
5149
5374
|
if (state.buffer.retiredCount !== 0) forgetRetiredFlags();
|
|
5150
5375
|
state.pendingLocations.delete(docId);
|
|
5151
5376
|
if (!threadsHoldCurrentLayout(state)) scheduleWorkerCopyLoad(state);
|
|
5152
5377
|
}
|
|
5378
|
+
function keptOrdinalHolding(docId, vector) {
|
|
5379
|
+
const kept = keptOrdinalsOf(state, docId);
|
|
5380
|
+
for (let at = kept.length - 1; at >= 0; at--) {
|
|
5381
|
+
const held = heldVectorOrUndefinedWhereItsFileFailsToRead(kept[at]);
|
|
5382
|
+
if (held !== void 0 && sameVector(held, vector)) return kept[at];
|
|
5383
|
+
}
|
|
5384
|
+
return void 0;
|
|
5385
|
+
}
|
|
5386
|
+
function applyRestore(docId, vector, partitionId) {
|
|
5387
|
+
const ordinal = keptOrdinalHolding(docId, vector);
|
|
5388
|
+
if (ordinal === void 0) return false;
|
|
5389
|
+
handOverBuildTurns(state);
|
|
5390
|
+
const current = state.store.getOrdinal(docId);
|
|
5391
|
+
if (current === ordinal && !state.tombstones.has(docId)) return true;
|
|
5392
|
+
const previous = current === ordinal ? void 0 : current;
|
|
5393
|
+
const graphHoldsPrevious = previous !== void 0 && graphsHoldNode(previous);
|
|
5394
|
+
if (!state.store.reclaim(docId, ordinal, partitionId, graphHoldsPrevious)) return false;
|
|
5395
|
+
forgetKeptOrdinal(state, docId, ordinal);
|
|
5396
|
+
noteWrite(state);
|
|
5397
|
+
noteSearchedDocWritten(state, docId);
|
|
5398
|
+
state.tombstones.delete(docId);
|
|
5399
|
+
if (previous !== void 0) retireFromGraphs(previous, graphHoldsPrevious);
|
|
5400
|
+
const graph = state.hnsw;
|
|
5401
|
+
const freshGraph = state.freshGraph;
|
|
5402
|
+
const growingCopy = state.growingCopy;
|
|
5403
|
+
afterGraphPass(state, () => {
|
|
5404
|
+
graph?.reviveOrdinal(ordinal);
|
|
5405
|
+
freshGraph?.reviveOrdinal(ordinal);
|
|
5406
|
+
growingCopy?.reviveOrdinal(ordinal);
|
|
5407
|
+
});
|
|
5408
|
+
finishPlacingInBuffer(docId);
|
|
5409
|
+
return true;
|
|
5410
|
+
}
|
|
5153
5411
|
function applyRemove(docId) {
|
|
5154
5412
|
handOverBuildTurns(state);
|
|
5155
5413
|
const ordinal = state.store.getOrdinal(docId);
|
|
@@ -5157,6 +5415,7 @@ function createVectorWrites(state) {
|
|
|
5157
5415
|
noteWrite(state);
|
|
5158
5416
|
noteSearchedDocWritten(state, docId);
|
|
5159
5417
|
state.tombstones.add(docId);
|
|
5418
|
+
keepRetiredOrdinal(state, docId, ordinal);
|
|
5160
5419
|
state.store.retire(ordinal);
|
|
5161
5420
|
state.buffer.retire(docId);
|
|
5162
5421
|
if (state.buffer.retiredCount !== 0) forgetRetiredFlags();
|
|
@@ -5177,10 +5436,32 @@ function createVectorWrites(state) {
|
|
|
5177
5436
|
applyInsert(docId, vector, partitionId);
|
|
5178
5437
|
return;
|
|
5179
5438
|
}
|
|
5180
|
-
noteWrite(state);
|
|
5181
|
-
coldInsert(hold, docId, vector, partitionId);
|
|
5182
5439
|
const held = vector.slice();
|
|
5183
|
-
|
|
5440
|
+
const keepsReplacedCopy = state.rollbackHolds.has(docId);
|
|
5441
|
+
if (coldInsert(hold, docId, held, partitionId, keepsReplacedCopy, () => applyInsert(docId, held, partitionId))) {
|
|
5442
|
+
noteWrite(state);
|
|
5443
|
+
}
|
|
5444
|
+
},
|
|
5445
|
+
restore(docId, vector, partitionId) {
|
|
5446
|
+
validateDimension(vector);
|
|
5447
|
+
for (const listener of state.writeListeners) listener(docId);
|
|
5448
|
+
const hold = state.cold;
|
|
5449
|
+
if (hold === null) {
|
|
5450
|
+
if (!applyRestore(docId, vector, partitionId)) applyInsert(docId, vector, partitionId);
|
|
5451
|
+
return;
|
|
5452
|
+
}
|
|
5453
|
+
if (coldRestore(hold, docId, vector)) {
|
|
5454
|
+
noteWrite(state);
|
|
5455
|
+
return;
|
|
5456
|
+
}
|
|
5457
|
+
const held = vector.slice();
|
|
5458
|
+
if (coldInsert(hold, docId, held, partitionId, false, () => applyInsert(docId, held, partitionId))) {
|
|
5459
|
+
noteWrite(state);
|
|
5460
|
+
}
|
|
5461
|
+
},
|
|
5462
|
+
requireRoomForNewVector(docId) {
|
|
5463
|
+
if (state.cold === null) state.store.requireRoomFor(docId);
|
|
5464
|
+
else requireRoomInHold(state.cold, docId);
|
|
5184
5465
|
},
|
|
5185
5466
|
remove(docId) {
|
|
5186
5467
|
for (const listener of state.writeListeners) listener(docId);
|
|
@@ -5189,9 +5470,7 @@ function createVectorWrites(state) {
|
|
|
5189
5470
|
applyRemove(docId);
|
|
5190
5471
|
return;
|
|
5191
5472
|
}
|
|
5192
|
-
if (
|
|
5193
|
-
noteWrite(state);
|
|
5194
|
-
hold.writes.push(() => applyRemove(docId));
|
|
5473
|
+
if (coldRemove(hold, docId, () => applyRemove(docId))) noteWrite(state);
|
|
5195
5474
|
}
|
|
5196
5475
|
};
|
|
5197
5476
|
}
|
|
@@ -5271,9 +5550,10 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
5271
5550
|
graphsHeldOffThread: null,
|
|
5272
5551
|
graphsPlacedIntoSinceConnect: /* @__PURE__ */ new WeakSet(),
|
|
5273
5552
|
searchWatches: /* @__PURE__ */ new Set(),
|
|
5274
|
-
writeListeners: /* @__PURE__ */ new Set()
|
|
5553
|
+
writeListeners: /* @__PURE__ */ new Set(),
|
|
5554
|
+
rollbackHolds: /* @__PURE__ */ new Map()
|
|
5275
5555
|
};
|
|
5276
|
-
const { insert, remove } = createVectorWrites(state);
|
|
5556
|
+
const { insert, remove, requireRoomForNewVector, restore: restoreVector } = createVectorWrites(state);
|
|
5277
5557
|
function getVector(docId) {
|
|
5278
5558
|
if (state.cold !== null) return coldVectorOf(state.cold, docId);
|
|
5279
5559
|
if (state.tombstones.has(docId)) return null;
|
|
@@ -5281,6 +5561,19 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
5281
5561
|
if (!entry) return null;
|
|
5282
5562
|
return new Float32Array(entry.vector);
|
|
5283
5563
|
}
|
|
5564
|
+
function requireRoomFor(docId, vector) {
|
|
5565
|
+
try {
|
|
5566
|
+
requireRoomForNewVector(docId);
|
|
5567
|
+
} catch (error) {
|
|
5568
|
+
const held = getVector(docId);
|
|
5569
|
+
if (held === null || held.length !== vector.length || !sameVector(held, vector)) throw error;
|
|
5570
|
+
}
|
|
5571
|
+
}
|
|
5572
|
+
function restore(docId, vector, partitionId) {
|
|
5573
|
+
const held = getVector(docId);
|
|
5574
|
+
if (held !== null && held.length === vector.length && sameVector(held, vector)) return;
|
|
5575
|
+
restoreVector(docId, vector, partitionId);
|
|
5576
|
+
}
|
|
5284
5577
|
function has(docId) {
|
|
5285
5578
|
if (state.cold !== null) return coldHas(state.cold, docId);
|
|
5286
5579
|
return state.store.has(docId) && !state.tombstones.has(docId);
|
|
@@ -5341,6 +5634,9 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
5341
5634
|
return state.metric;
|
|
5342
5635
|
},
|
|
5343
5636
|
insert,
|
|
5637
|
+
requireRoomFor,
|
|
5638
|
+
restore,
|
|
5639
|
+
holdForRollback: (docId) => holdForRollback(state, docId),
|
|
5344
5640
|
remove,
|
|
5345
5641
|
partitionsKnown: () => state.cold !== null || state.store.partitionsKnown,
|
|
5346
5642
|
assignPartitions: (resolve) => {
|
|
@@ -5440,4 +5736,4 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
5440
5736
|
};
|
|
5441
5737
|
}
|
|
5442
5738
|
|
|
5443
|
-
export { bindVectorHost, compactVectors, createSharedVectorSearcher, createVectorIndex, executeHybridSearch, executeVectorSearch, filterSubsetsMatchNothing, getVectorMaintenanceStatus, holdAdmission, linearCombination, openSharedVectorField, optimizeVectors, reciprocalRankFusion, releaseLocksHeldBy, releasePinsHeldBy, requireUsableVectorOptions, resolveHybridFusion, vectorLegWith };
|
|
5739
|
+
export { bindVectorHost, compactVectors, createSharedVectorSearcher, createVectorIndex, executeHybridSearch, executeVectorSearch, filterSubsetsMatchNothing, getVectorMaintenanceStatus, holdAdmission, linearCombination, openSharedVectorField, optimizeVectors, reciprocalRankFusion, releaseLocksHeldBy, releasePinsHeldBy, requireUsableQueryVector, requireUsableVectorOptions, resolveHybridFusion, vectorLegWith };
|