@delali/narsil 0.3.1-next.91 → 0.3.1-next.95
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/browser/{chunk-66KYAOIK.mjs → chunk-3U636ADD.mjs} +2 -2
- package/dist/browser/{chunk-NOUZ6RH3.mjs → chunk-D3CC2NMZ.mjs} +238 -114
- package/dist/browser/{chunk-3ZRMZIH4.mjs → chunk-FYFQJKFW.mjs} +208 -78
- package/dist/browser/{chunk-FHFFWLEK.mjs → chunk-HARBSUWP.mjs} +3 -4
- package/dist/browser/{chunk-B6XE7KXY.mjs → chunk-LULHQXCK.mjs} +4 -4
- package/dist/browser/{chunk-NQW2YXEX.mjs → chunk-WKPJ4E73.mjs} +1 -1
- package/dist/browser/{durability-wiring-F4OGQUSU.mjs → durability-wiring-3ZRKSPJD.mjs} +498 -71
- package/dist/browser/{module-E25UDADM.mjs → module-IEOQI5GI.mjs} +102 -54
- package/dist/browser/release-76MXRPGV.mjs +33 -0
- package/dist/browser/replace-RURXTGVG.mjs +57 -0
- package/dist/browser/restore-queue-RBULSOVX.mjs +35 -0
- package/dist/browser/{snapshot-5EJ7I7QY.mjs → snapshot-Y3NGUJHX.mjs} +362 -103
- package/dist/browser/{write-K5V4XQ72.mjs → write-ZUBDNLCH.mjs} +4 -4
- package/dist/{capabilities-CvExBThq.d.ts → capabilities-BvxvFzVA.d.ts} +2 -2
- package/dist/{chunk-NVL4ZR3K.mjs → chunk-2GZGBOQI.mjs} +2 -2
- package/dist/{chunk-XLJ2S2BH.mjs → chunk-3LYY62M6.mjs} +40 -21
- package/dist/{chunk-CVQWQVFX.mjs → chunk-4ZJWRETC.mjs} +2 -2
- package/dist/{chunk-T4DB5YFB.mjs → chunk-54HYC5FT.mjs} +6 -6
- package/dist/{chunk-ZIJ63TBT.mjs → chunk-7KC56574.mjs} +1 -1
- package/dist/{chunk-67Z2V6TC.mjs → chunk-7QONURHW.mjs} +3 -3
- package/dist/{chunk-BVOJIYRV.mjs → chunk-BB7MASRQ.mjs} +1 -1
- package/dist/{chunk-6VZ3Q73N.mjs → chunk-CHOGQSHK.mjs} +44 -25
- package/dist/{chunk-QZN3MIJ3.mjs → chunk-ES33L3HO.mjs} +2 -2
- package/dist/{chunk-YCJLVNWR.mjs → chunk-F52QGNZ2.mjs} +1 -1
- package/dist/{chunk-43ORORN6.mjs → chunk-GJZHU4FC.mjs} +3 -3
- package/dist/{chunk-R7OSDAFK.mjs → chunk-HBVLF3HA.mjs} +1 -1
- package/dist/{chunk-ULRPNYAG.mjs → chunk-ICL2AELD.mjs} +120 -62
- package/dist/{chunk-3RDS4UHC.mjs → chunk-J7SDAOTP.mjs} +4 -4
- package/dist/{chunk-A4G7JSYO.mjs → chunk-JFUE6F2Y.mjs} +98 -49
- package/dist/chunk-KOKM4PQN.mjs +52 -0
- package/dist/{chunk-7OQHSDK6.mjs → chunk-L26LEDHO.mjs} +7 -7
- package/dist/{chunk-EONIHG3V.mjs → chunk-MASE2LTJ.mjs} +7 -7
- package/dist/{chunk-MXS5HWIH.mjs → chunk-MWT5WCJ5.mjs} +89 -17
- package/dist/{chunk-3TTYK5A7.mjs → chunk-Q4UTU5ZE.mjs} +9 -9
- package/dist/{chunk-T6CNO72J.mjs → chunk-QM3VPX4Z.mjs} +2 -2
- package/dist/chunk-QY7GC2MW.mjs +33 -0
- package/dist/{chunk-BJ2YKRNX.mjs → chunk-R36MIGBU.mjs} +86 -54
- package/dist/{chunk-RVM7AODE.mjs → chunk-RDJKYKMQ.mjs} +2 -2
- package/dist/{chunk-UXQE2ROB.mjs → chunk-RUIXODYU.mjs} +3 -2
- package/dist/{chunk-5ER245FI.mjs → chunk-SBCEHIPL.mjs} +3 -3
- package/dist/{chunk-TPJU7NDJ.mjs → chunk-SYWGORQB.mjs} +1 -0
- package/dist/{chunk-UKSUENHK.mjs → chunk-T2QN2FA4.mjs} +40 -26
- package/dist/{chunk-CO4MRKST.mjs → chunk-WLOF6HSP.mjs} +2 -2
- package/dist/{chunk-MKWUXWQH.mjs → chunk-ZHS44YKF.mjs} +1 -1
- package/dist/{client-Cc42b42j.d.ts → client-DKwterMb.d.ts} +2 -2
- package/dist/client.d.ts +6 -6
- package/dist/client.mjs +1 -0
- package/dist/{config-DyRowvVP.d.ts → config-6YBtxekG.d.ts} +1 -1
- package/dist/{config-BYne-yCI.d.ts → config-DPphZfuM.d.ts} +1 -1
- package/dist/direct-executor-5KJGVQHM.mjs +22 -0
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +5 -5
- package/dist/distribution.mjs +26 -26
- package/dist/{durability-wiring-6AN4QTT3.mjs → durability-wiring-FRPV6JLZ.mjs} +518 -91
- package/dist/embeddings/openai.mjs +1 -1
- package/dist/{engine-96_kLtSY.d.ts → engine-BGt9lI1V.d.ts} +11 -11
- package/dist/index.browser.mjs +2 -2
- package/dist/index.d.ts +5 -5
- package/dist/index.mjs +18 -18
- package/dist/{memory-dd3GESdc.d.ts → memory-COdoGthn.d.ts} +96 -76
- package/dist/{module-WIDYIJLY.mjs → module-IUTNXWMV.mjs} +72 -38
- package/dist/partial-words.mjs +5 -5
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.mjs +21 -21
- package/dist/react.d.ts +6 -6
- package/dist/react.mjs +1 -0
- package/dist/release-BH3DKW3T.mjs +33 -0
- package/dist/replace-CEO5KJJ3.mjs +57 -0
- package/dist/restore-queue-I5SI6AUQ.mjs +35 -0
- package/dist/server.d.ts +8 -8
- package/dist/server.mjs +10 -10
- package/dist/snapshot-YC6TS4OP.mjs +738 -0
- package/dist/synonyms.mjs +3 -3
- package/dist/{tasks-DA1Ff0G4.d.ts → tasks-BJmamiC4.d.ts} +1 -1
- package/dist/{thread-VTDLLYXC.mjs → thread-U7XYJWGG.mjs} +15 -12
- package/dist/vector/search-worker.mjs +6 -6
- package/dist/workers/entry.mjs +5 -5
- package/dist/{write-OJR3H4BI.mjs → write-M2OALSA4.mjs} +20 -20
- package/package.json +7 -7
- package/dist/chunk-Y2F66IQJ.mjs +0 -74
- package/dist/chunk-ZJ4CSAB2.mjs +0 -13
- package/dist/direct-executor-CHXVCBN7.mjs +0 -22
- package/dist/snapshot-JFZVOGZV.mjs +0 -479
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { readTrailer, unpackLevels, stageQueryLevels, osqCodeBytes, osqStagedQueryBytes, osqRecordBytes, writeRecord } from './chunk-37CD3OKS.mjs';
|
|
2
2
|
import { stopUsingTheNativeSearchCore, noteFallback, nativeSearchCoreIsEnabled } from './chunk-DJHHF2L5.mjs';
|
|
3
|
-
import { markSimdUnavailable
|
|
4
|
-
import { OSQ_STEPS, OSQ_GRID, OSQ_QUERY_BITS,
|
|
5
|
-
import { loadNativeCore } from './chunk-
|
|
6
|
-
import { COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, PLACEMENT_ROOM_REQUESTS, MAX_LAYER_CAP, INITIAL_LIST_CAPACITY,
|
|
3
|
+
import { markSimdUnavailable } from './chunk-KOKM4PQN.mjs';
|
|
4
|
+
import { MAX_WASM_PAGES, WASM_PAGE_BYTES, VECTOR_SCRATCH_SLOTS, UNPLACED_FLAG_CHUNK_ORDINALS, VECTOR_BLOCK_INITIAL_SLOTS, MAX_VECTOR_ORDINALS, VECTOR_STORE_INITIAL_CAPACITY, LOCK_SPIN_ITERATIONS, LOCK_WAIT_SLICE_MS, OSQ_STEPS, OSQ_GRID, OSQ_QUERY_BITS, NATIVE_ATTACH_RETRY_MS, GRAPH_LOCK_WAIT_SLICE_MS, OSQ_REFINE_ROUNDS, OSQ_REFINE_TOLERANCE, OSQ_LAMBDA, MAX_DOC_ID_TABLE_BYTES } from './chunk-QY7GC2MW.mjs';
|
|
5
|
+
import { loadNativeCore } from './chunk-ES33L3HO.mjs';
|
|
6
|
+
import { COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, PLACEMENT_ROOM_REQUESTS, MAX_LAYER_CAP, INITIAL_LIST_CAPACITY, INITIAL_UPPER_SLOTS, DEFAULT_EF_SEARCH } from './chunk-KEHK2O6P.mjs';
|
|
7
7
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
8
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
8
|
+
import { NarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
|
|
9
9
|
|
|
10
10
|
// src/vector/shared-buffers/growable.ts
|
|
11
11
|
function sharedMemoryAvailable() {
|
|
@@ -297,6 +297,42 @@ function releaseLocksHeldBy(handles, threadSlot) {
|
|
|
297
297
|
}
|
|
298
298
|
}
|
|
299
299
|
|
|
300
|
+
// src/vector/ordinal-filter.ts
|
|
301
|
+
function createOrdinalFilter(slots) {
|
|
302
|
+
return { bits: new Uint8Array(slots + 7 >>> 3), count: 0 };
|
|
303
|
+
}
|
|
304
|
+
function addToOrdinalFilter(filter, ordinal) {
|
|
305
|
+
const index = ordinal >>> 3;
|
|
306
|
+
if (index >= filter.bits.length) return;
|
|
307
|
+
const bit = 1 << (ordinal & 7);
|
|
308
|
+
if ((filter.bits[index] & bit) !== 0) return;
|
|
309
|
+
filter.bits[index] |= bit;
|
|
310
|
+
filter.count += 1;
|
|
311
|
+
}
|
|
312
|
+
function ordinalFilterHas(filter, ordinal) {
|
|
313
|
+
const index = ordinal >>> 3;
|
|
314
|
+
if (index >= filter.bits.length) return false;
|
|
315
|
+
return (filter.bits[index] & 1 << (ordinal & 7)) !== 0;
|
|
316
|
+
}
|
|
317
|
+
function removeFromOrdinalFilter(filter, ordinal) {
|
|
318
|
+
const index = ordinal >>> 3;
|
|
319
|
+
if (index >= filter.bits.length) return;
|
|
320
|
+
const bit = 1 << (ordinal & 7);
|
|
321
|
+
if ((filter.bits[index] & bit) === 0) return;
|
|
322
|
+
filter.bits[index] &= ~bit;
|
|
323
|
+
filter.count -= 1;
|
|
324
|
+
}
|
|
325
|
+
function* ordinalFilterValues(filter) {
|
|
326
|
+
const bits = filter.bits;
|
|
327
|
+
for (let index = 0; index < bits.length; index++) {
|
|
328
|
+
const byte = bits[index];
|
|
329
|
+
if (byte === 0) continue;
|
|
330
|
+
for (let offset = 0; offset < 8; offset++) {
|
|
331
|
+
if ((byte & 1 << offset) !== 0) yield index << 3 | offset;
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
|
|
300
336
|
// src/vector/simd-wasm-binary.ts
|
|
301
337
|
var SIMD_DISTANCE_WASM_BASE64 = "AGFzbQEAAAABIgVgA39/fwF9YAJ/fwF9YAN/f38Bf2ABewF/YAR/f39/AX8DDw4AAQAAAgIDAgIEAgQCAgUDAQABB6QBCgZtZW1vcnkCAAtkb3RfcHJvZHVjdAAACW1hZ25pdHVkZQABGnNxdWFyZWRfZXVjbGlkZWFuX2Rpc3RhbmNlAAISZXVjbGlkZWFuX2Rpc3RhbmNlAAMGZG90X3U4AAQJc3FkaXN0X3U4AAUMb3NxX2RvdF9iaXRzAAkRb3NxX2RvdF9iaXRzX3BhaXIACxNvc3FfZG90X25pYmJsZXNfNHg0AA0K2RQOygIEAX8EewF9BH8gAkFwcSEJIAJBfHEhCv0MAAAAAAAAAAAAAAAAAAAAACEEQQAhAwJAA0AgAyAJTw0BIAAgA0ECdGohCyABIANBAnRqIQwgBCAL/QAEACAM/QAEAP3mAf3kASEEIAUgC/0ABBAgDP0ABBD95gH95AEhBSAGIAv9AAQgIAz9AAQg/eYB/eQBIQYgByAL/QAEMCAM/QAEMP3mAf3kASEHIANBEGohAwwACwsCQANAIAMgCk8NASAEIAAgA0ECdGr9AAQAIAEgA0ECdGr9AAQA/eYB/eQBIQQgA0EEaiEDDAALCyAEIAX95AEgBiAH/eQB/eQBIQQgBP0fACAE/R8BkiAE/R8CIAT9HwOSkiEIAkADQCADIAJPDQEgCCAAIANBAnRqKgIAIAEgA0ECdGoqAgCUkiEIIANBAWohAwwACwsgCAupAQQBfwJ7AX0BfyABQXxxIQb9DAAAAAAAAAAAAAAAAAAAAAAhA0EAIQICQANAIAIgBk8NASAAIAJBAnRq/QAEACEEIAMgBCAE/eYB/eQBIQMgAkEEaiECDAALCyAD/R8AIAP9HwGSIAP9HwIgA/0fA5KSIQUCQANAIAIgAU8NASAFIAAgAkECdGoqAgAgACACQQJ0aioCAJSSIQUgAkEBaiECDAALCyAFkQvAAQUBfwJ7AX0BfwF9IAJBfHEhB/0MAAAAAAAAAAAAAAAAAAAAACEEQQAhAwJAA0AgAyAHTw0BIAAgA0ECdGr9AAQAIAEgA0ECdGr9AAQA/eUBIQUgBCAFIAX95gH95AEhBCADQQRqIQMMAAsLIAT9HwAgBP0fAZIgBP0fAiAE/R8DkpIhBgJAA0AgAyACTw0BIAAgA0ECdGoqAgAgASADQQJ0aioCAJMhCCAGIAggCJSSIQYgA0EBaiEDDAALCyAGCwsAIAAgASACEAKRC7QBAwF/A3sCfyACQXBxIQhBAP0RIQRBACEDAkADQCADIAhPDQEgACADav0ABAAhBSABIANq/QAEACEGIAQgBf2JASAG/YkB/boB/a4BIQQgBCAF/YoBIAb9igH9ugH9rgEhBCADQRBqIQMMAAsLIAT9GwAgBP0bAWogBP0bAiAE/RsDamohBwJAA0AgAyACTw0BIAcgACADai0AACABIANqLQAAbGohByADQQFqIQMMAAsLIAcLzQEDAX8FewN/IAJBcHEhC0EA/REhBEEAIQMCQANAIAMgC08NASAAIANq/QAEACEFIAEgA2r9AAQAIQYgBf2JASAG/YkB/ZEBIQcgBf2KASAG/YoB/ZEBIQggBCAHIAf9ugH9rgEhBCAEIAggCP26Af2uASEEIANBEGohAwwACwsgBP0bACAE/RsBaiAE/RsCIAT9GwNqaiEJAkADQCADIAJPDQEgACADai0AACABIANqLQAAayEKIAkgCiAKbGohCSADQQFqIQMMAAsLIAkLIQEBeyAA/X8hASAB/RsAIAH9GwFqIAH9GwIgAf0bA2pqC68CAwh/A3sBfyABIAJqIQggCCACaiEJIAkgAmohCiACQXBxIQUCQANAIAMgBU8NASADQYAQaiEGIAYgBUsEQCAFIQYLQQD9ECENA0AgACADav0ABAAhCyALIAogA2r9AAQA/U79YiEMIAwgDP1uIAsgCSADav0ABAD9Tv1i/W4hDCAMIAz9biALIAggA2r9AAQA/U79Yv1uIQwgDCAM/W4gCyABIANq/QAEAP1O/WL9biEMIA0gDP19/Y4BIQ0gA0EQaiEDIAMgBkkNAAsgByANEAZqIQcMAAsLAkADQCADIAJPDQEgACADai0AACEOQQAhBAJAA0AgBEEETw0BIAcgDiABIAIgBGxqIANqLQAAcWkgBHRqIQcgBEEBaiEEDAALCyADQQFqIQMMAAsLIAcLhwMDCH8IewF/IAEgAmohCCAIIAJqIQkgCSACaiEKQQP9DyERIAJBcHEhBQJAA0AgAyAFTw0BIANBgBBqIQYgBiAFSwRAIAUhBgtBAP0QIRIDQCAAIANq/QAEACELIAsgEf1OIQwgC0EC/W0gEf1OIQ0gC0EE/W0gEf1OIQ4gC0EG/W0hDyABIANq/QAEACEQIBIgDCAQ/Z4BIAwgEP2fAf2OAf2OASESIAggA2r9AAQAIRAgEiANIBD9ngEgDSAQ/Z8B/Y4B/Y4BIRIgCSADav0ABAAhECASIA4gEP2eASAOIBD9nwH9jgH9jgEhEiAKIANq/QAEACEQIBIgDyAQ/Z4BIA8gEP2fAf2OAf2OASESIANBEGohAyADIAZJDQALIAcgEhAGaiEHDAALCwJAA0AgAyACTw0BIAAgA2otAAAhE0EAIQQCQANAIARBBE8NASAHIBMgBEEBdHZBA3EgASACIARsaiADai0AAGxqIQcgBEEBaiEEDAALCyADQQFqIQMMAAsLIAcLGwAgA0ECRgR/IAAgASACEAgFIAAgASACEAcLC54BAgR/AXsgAkFwcSEEAkADQCADIARPDQEgA0GAEGohBSAFIARLBEAgBCEFC0EA/RAhBwNAIAcgACADav0ABAAgASADav0ABAD9Tv1i/X39jgEhByADQRBqIQMgAyAFSQ0ACyAGIAcQBmohBgwACwsCQANAIAMgAk8NASAGIAAgA2otAAAgASADai0AAHFpaiEGIANBAWohAwwACwsgBgsbACADQQJGBH8gACABIAIQDAUgACABIAIQCgsL2gIDBH8IewR/QdUAIRFB1QAhEkHVAP0PIQdBqgH9DyEIIAJBcHEhBAJAA0AgAyAETw0BIANB4ANqIQUgBSAESwRAIAQhBQtBAP0PIQ5BAP0PIQxBAP0PIQ0DQCAAIANq/QAEACEJIAEgA2r9AAQAIQogCSAK/U4hCyAOIAv9Yv1uIQ4gDCALIAj9Tv1i/W4hDCANIAkgCkEB/W0gB/1OIApBAf1rIAj9Tv1Q/U79Yv1uIQ0gA0EQaiEDIAMgBUkNAAsgBiAO/X0QBiAM/X0QBkEDbCAN/X0QBkEBdGpqaiEGDAALCwJAA0AgAyACTw0BIAAgA2otAAAhDyABIANqLQAAIRAgBiAPIBFxIBBxaSAPIBFxIBBBAXYgEnFxaUEBdGogD0EBdiAScSAQIBFxcWlBAXQgD0EBdiAScSAQQQF2IBJxcWlBAnRqamohBiADQQFqIQMMAAsLIAYL/AEDA38IewJ/IAJBcHEhBEEP/Q8hDEEA/REhDUEAIQMCQANAIAMgBE8NASAAIANq/QAEACEGIAEgA2r9AAQAIQcgBiAM/U4hCCAGQQT9bSEJIAcgDP1OIQogB0EE/W0hCyANIAggCv2eASAJIAv9ngH9jgH9fyAIIAr9nwEgCSAL/Z8B/Y4B/X/9rgH9rgEhDSADQRBqIQMMAAsLIA39GwAgDf0bAWogDf0bAiAN/RsDamohBQJAA0AgAyACTw0BIAAgA2otAAAhDyABIANqLQAAIQ4gBSAPQQ9xIA5BD3FsIA9BBHYgDkEEdmxqaiEFIANBAWohAwwACwsgBQs=";
|
|
302
338
|
var SIMD_DISTANCE_SHARED_MEMORY_WASM_BASE64 = "AGFzbQEAAAABIgVgA39/fwF9YAJ/fwF9YAN/f38Bf2ABewF/YAR/f39/AX8CEgEDZW52Bm1lbW9yeQIDAYCABAMPDgABAAACAgMCAgQCBAICB5sBCQtkb3RfcHJvZHVjdAAACW1hZ25pdHVkZQABGnNxdWFyZWRfZXVjbGlkZWFuX2Rpc3RhbmNlAAISZXVjbGlkZWFuX2Rpc3RhbmNlAAMGZG90X3U4AAQJc3FkaXN0X3U4AAUMb3NxX2RvdF9iaXRzAAkRb3NxX2RvdF9iaXRzX3BhaXIACxNvc3FfZG90X25pYmJsZXNfNHg0AA0K2RQOygIEAX8EewF9BH8gAkFwcSEJIAJBfHEhCv0MAAAAAAAAAAAAAAAAAAAAACEEQQAhAwJAA0AgAyAJTw0BIAAgA0ECdGohCyABIANBAnRqIQwgBCAL/QAEACAM/QAEAP3mAf3kASEEIAUgC/0ABBAgDP0ABBD95gH95AEhBSAGIAv9AAQgIAz9AAQg/eYB/eQBIQYgByAL/QAEMCAM/QAEMP3mAf3kASEHIANBEGohAwwACwsCQANAIAMgCk8NASAEIAAgA0ECdGr9AAQAIAEgA0ECdGr9AAQA/eYB/eQBIQQgA0EEaiEDDAALCyAEIAX95AEgBiAH/eQB/eQBIQQgBP0fACAE/R8BkiAE/R8CIAT9HwOSkiEIAkADQCADIAJPDQEgCCAAIANBAnRqKgIAIAEgA0ECdGoqAgCUkiEIIANBAWohAwwACwsgCAupAQQBfwJ7AX0BfyABQXxxIQb9DAAAAAAAAAAAAAAAAAAAAAAhA0EAIQICQANAIAIgBk8NASAAIAJBAnRq/QAEACEEIAMgBCAE/eYB/eQBIQMgAkEEaiECDAALCyAD/R8AIAP9HwGSIAP9HwIgA/0fA5KSIQUCQANAIAIgAU8NASAFIAAgAkECdGoqAgAgACACQQJ0aioCAJSSIQUgAkEBaiECDAALCyAFkQvAAQUBfwJ7AX0BfwF9IAJBfHEhB/0MAAAAAAAAAAAAAAAAAAAAACEEQQAhAwJAA0AgAyAHTw0BIAAgA0ECdGr9AAQAIAEgA0ECdGr9AAQA/eUBIQUgBCAFIAX95gH95AEhBCADQQRqIQMMAAsLIAT9HwAgBP0fAZIgBP0fAiAE/R8DkpIhBgJAA0AgAyACTw0BIAAgA0ECdGoqAgAgASADQQJ0aioCAJMhCCAGIAggCJSSIQYgA0EBaiEDDAALCyAGCwsAIAAgASACEAKRC7QBAwF/A3sCfyACQXBxIQhBAP0RIQRBACEDAkADQCADIAhPDQEgACADav0ABAAhBSABIANq/QAEACEGIAQgBf2JASAG/YkB/boB/a4BIQQgBCAF/YoBIAb9igH9ugH9rgEhBCADQRBqIQMMAAsLIAT9GwAgBP0bAWogBP0bAiAE/RsDamohBwJAA0AgAyACTw0BIAcgACADai0AACABIANqLQAAbGohByADQQFqIQMMAAsLIAcLzQEDAX8FewN/IAJBcHEhC0EA/REhBEEAIQMCQANAIAMgC08NASAAIANq/QAEACEFIAEgA2r9AAQAIQYgBf2JASAG/YkB/ZEBIQcgBf2KASAG/YoB/ZEBIQggBCAHIAf9ugH9rgEhBCAEIAggCP26Af2uASEEIANBEGohAwwACwsgBP0bACAE/RsBaiAE/RsCIAT9GwNqaiEJAkADQCADIAJPDQEgACADai0AACABIANqLQAAayEKIAkgCiAKbGohCSADQQFqIQMMAAsLIAkLIQEBeyAA/X8hASAB/RsAIAH9GwFqIAH9GwIgAf0bA2pqC68CAwh/A3sBfyABIAJqIQggCCACaiEJIAkgAmohCiACQXBxIQUCQANAIAMgBU8NASADQYAQaiEGIAYgBUsEQCAFIQYLQQD9ECENA0AgACADav0ABAAhCyALIAogA2r9AAQA/U79YiEMIAwgDP1uIAsgCSADav0ABAD9Tv1i/W4hDCAMIAz9biALIAggA2r9AAQA/U79Yv1uIQwgDCAM/W4gCyABIANq/QAEAP1O/WL9biEMIA0gDP19/Y4BIQ0gA0EQaiEDIAMgBkkNAAsgByANEAZqIQcMAAsLAkADQCADIAJPDQEgACADai0AACEOQQAhBAJAA0AgBEEETw0BIAcgDiABIAIgBGxqIANqLQAAcWkgBHRqIQcgBEEBaiEEDAALCyADQQFqIQMMAAsLIAcLhwMDCH8IewF/IAEgAmohCCAIIAJqIQkgCSACaiEKQQP9DyERIAJBcHEhBQJAA0AgAyAFTw0BIANBgBBqIQYgBiAFSwRAIAUhBgtBAP0QIRIDQCAAIANq/QAEACELIAsgEf1OIQwgC0EC/W0gEf1OIQ0gC0EE/W0gEf1OIQ4gC0EG/W0hDyABIANq/QAEACEQIBIgDCAQ/Z4BIAwgEP2fAf2OAf2OASESIAggA2r9AAQAIRAgEiANIBD9ngEgDSAQ/Z8B/Y4B/Y4BIRIgCSADav0ABAAhECASIA4gEP2eASAOIBD9nwH9jgH9jgEhEiAKIANq/QAEACEQIBIgDyAQ/Z4BIA8gEP2fAf2OAf2OASESIANBEGohAyADIAZJDQALIAcgEhAGaiEHDAALCwJAA0AgAyACTw0BIAAgA2otAAAhE0EAIQQCQANAIARBBE8NASAHIBMgBEEBdHZBA3EgASACIARsaiADai0AAGxqIQcgBEEBaiEEDAALCyADQQFqIQMMAAsLIAcLGwAgA0ECRgR/IAAgASACEAgFIAAgASACEAcLC54BAgR/AXsgAkFwcSEEAkADQCADIARPDQEgA0GAEGohBSAFIARLBEAgBCEFC0EA/RAhBwNAIAcgACADav0ABAAgASADav0ABAD9Tv1i/X39jgEhByADQRBqIQMgAyAFSQ0ACyAGIAcQBmohBgwACwsCQANAIAMgAk8NASAGIAAgA2otAAAgASADai0AAHFpaiEGIANBAWohAwwACwsgBgsbACADQQJGBH8gACABIAIQDAUgACABIAIQCgsL2gIDBH8IewR/QdUAIRFB1QAhEkHVAP0PIQdBqgH9DyEIIAJBcHEhBAJAA0AgAyAETw0BIANB4ANqIQUgBSAESwRAIAQhBQtBAP0PIQ5BAP0PIQxBAP0PIQ0DQCAAIANq/QAEACEJIAEgA2r9AAQAIQogCSAK/U4hCyAOIAv9Yv1uIQ4gDCALIAj9Tv1i/W4hDCANIAkgCkEB/W0gB/1OIApBAf1rIAj9Tv1Q/U79Yv1uIQ0gA0EQaiEDIAMgBUkNAAsgBiAO/X0QBiAM/X0QBkEDbCAN/X0QBkEBdGpqaiEGDAALCwJAA0AgAyACTw0BIAAgA2otAAAhDyABIANqLQAAIRAgBiAPIBFxIBBxaSAPIBFxIBBBAXYgEnFxaUEBdGogD0EBdiAScSAQIBFxcWlBAXQgD0EBdiAScSAQQQF2IBJxcWlBAnRqamohBiADQQFqIQMMAAsLIAYL/AEDA38IewJ/IAJBcHEhBEEP/Q8hDEEA/REhDUEAIQMCQANAIAMgBE8NASAAIANq/QAEACEGIAEgA2r9AAQAIQcgBiAM/U4hCCAGQQT9bSEJIAcgDP1OIQogB0EE/W0hCyANIAggCv2eASAJIAv9ngH9jgH9fyAIIAr9nwEgCSAL/Z8B/Y4B/X/9rgH9rgEhDSADQRBqIQMMAAsLIA39GwAgDf0bAWogDf0bAiAN/RsDamohBQJAA0AgAyACTw0BIAAgA2otAAAhDyABIANqLQAAIQ4gBSAPQQ9xIA5BD3FsIA9BBHYgDkEEdmxqaiEFIANBAWohAwwACwsgBQs=";
|
|
@@ -1099,8 +1135,18 @@ var STORE_CALIBRATION_GENERATION = 7;
|
|
|
1099
1135
|
var STORE_RELEASED_VECTORS_TO_DISK = 8;
|
|
1100
1136
|
var STORE_UNPLACED_COUNT = 9;
|
|
1101
1137
|
var STORE_REMOVED_COUNT = 10;
|
|
1138
|
+
var STORE_ORDINAL_LAYOUT_GENERATION = 11;
|
|
1102
1139
|
var STORE_HEADER_WORDS = 16;
|
|
1103
1140
|
var IN_MEMORY = -1;
|
|
1141
|
+
function resetStoreHeader(header) {
|
|
1142
|
+
Atomics.store(header, STORE_SLOTS, 0);
|
|
1143
|
+
Atomics.store(header, STORE_RELEASED_VECTORS_TO_DISK, 0);
|
|
1144
|
+
Atomics.store(header, STORE_LIVE_COUNT, 0);
|
|
1145
|
+
Atomics.store(header, STORE_CODE_COUNT, 0);
|
|
1146
|
+
Atomics.store(header, STORE_CALIBRATED, 0);
|
|
1147
|
+
Atomics.add(header, STORE_CALIBRATION_GENERATION, 1);
|
|
1148
|
+
Atomics.add(header, STORE_ORDINAL_LAYOUT_GENERATION, 1);
|
|
1149
|
+
}
|
|
1104
1150
|
function fixedInt32(words) {
|
|
1105
1151
|
return new Int32Array(createFixedBuffer(words * 4), 0, words);
|
|
1106
1152
|
}
|
|
@@ -1338,42 +1384,6 @@ function nativeQuantise(handles, metric, ordinals) {
|
|
|
1338
1384
|
return workOnRecords(handles, metric, ordinals, "quantise");
|
|
1339
1385
|
}
|
|
1340
1386
|
|
|
1341
|
-
// src/vector/ordinal-filter.ts
|
|
1342
|
-
function createOrdinalFilter(slots) {
|
|
1343
|
-
return { bits: new Uint8Array(slots + 7 >>> 3), count: 0 };
|
|
1344
|
-
}
|
|
1345
|
-
function addToOrdinalFilter(filter, ordinal) {
|
|
1346
|
-
const index = ordinal >>> 3;
|
|
1347
|
-
if (index >= filter.bits.length) return;
|
|
1348
|
-
const bit = 1 << (ordinal & 7);
|
|
1349
|
-
if ((filter.bits[index] & bit) !== 0) return;
|
|
1350
|
-
filter.bits[index] |= bit;
|
|
1351
|
-
filter.count += 1;
|
|
1352
|
-
}
|
|
1353
|
-
function ordinalFilterHas(filter, ordinal) {
|
|
1354
|
-
const index = ordinal >>> 3;
|
|
1355
|
-
if (index >= filter.bits.length) return false;
|
|
1356
|
-
return (filter.bits[index] & 1 << (ordinal & 7)) !== 0;
|
|
1357
|
-
}
|
|
1358
|
-
function removeFromOrdinalFilter(filter, ordinal) {
|
|
1359
|
-
const index = ordinal >>> 3;
|
|
1360
|
-
if (index >= filter.bits.length) return;
|
|
1361
|
-
const bit = 1 << (ordinal & 7);
|
|
1362
|
-
if ((filter.bits[index] & bit) === 0) return;
|
|
1363
|
-
filter.bits[index] &= ~bit;
|
|
1364
|
-
filter.count -= 1;
|
|
1365
|
-
}
|
|
1366
|
-
function* ordinalFilterValues(filter) {
|
|
1367
|
-
const bits = filter.bits;
|
|
1368
|
-
for (let index = 0; index < bits.length; index++) {
|
|
1369
|
-
const byte = bits[index];
|
|
1370
|
-
if (byte === 0) continue;
|
|
1371
|
-
for (let offset = 0; offset < 8; offset++) {
|
|
1372
|
-
if ((byte & 1 << offset) !== 0) yield index << 3 | offset;
|
|
1373
|
-
}
|
|
1374
|
-
}
|
|
1375
|
-
}
|
|
1376
|
-
|
|
1377
1387
|
// src/vector/native/field.ts
|
|
1378
1388
|
var WAKE_COUNT_WORDS = 1;
|
|
1379
1389
|
var WAKES_THE_CORE_KEEPS_PER_CALL = 256;
|
|
@@ -1954,7 +1964,7 @@ function readNeighborsLocked(state, ord, layer) {
|
|
|
1954
1964
|
if (nodeReadHeld(state.locks, ord, version)) return count;
|
|
1955
1965
|
}
|
|
1956
1966
|
}
|
|
1957
|
-
function searchLayer(state, qVec, qMag, ef, layer, metric, excludeTombstones, distFn, results) {
|
|
1967
|
+
function searchLayer(state, qVec, qMag, ef, layer, metric, excludeTombstones, distFn, results, accepted) {
|
|
1958
1968
|
const getDistance = distFn ?? ((ord) => queryDistanceByOrd(state, qVec, qMag, ord, metric));
|
|
1959
1969
|
const workspace = state.workspace;
|
|
1960
1970
|
const frontier = workspace.frontier;
|
|
@@ -1974,6 +1984,7 @@ function searchLayer(state, qVec, qMag, ef, layer, metric, excludeTombstones, di
|
|
|
1974
1984
|
if (dist === Number.POSITIVE_INFINITY) continue;
|
|
1975
1985
|
pushHeap(frontier, epOrd, dist);
|
|
1976
1986
|
if (excludeTombstones && isTombstoned(state, epOrd)) continue;
|
|
1987
|
+
if (accepted && !ordinalFilterHas(accepted, epOrd)) continue;
|
|
1977
1988
|
pushHeap(found, epOrd, dist);
|
|
1978
1989
|
if (found.size > ef) {
|
|
1979
1990
|
popHeap(found);
|
|
@@ -1994,6 +2005,7 @@ function searchLayer(state, qVec, qMag, ef, layer, metric, excludeTombstones, di
|
|
|
1994
2005
|
if (dist < furthestDist || found.size < ef) {
|
|
1995
2006
|
pushHeap(frontier, neighborOrd, dist);
|
|
1996
2007
|
if (excludeTombstones && isTombstoned(state, neighborOrd)) continue;
|
|
2008
|
+
if (accepted && !ordinalFilterHas(accepted, neighborOrd)) continue;
|
|
1997
2009
|
pushHeap(found, neighborOrd, dist);
|
|
1998
2010
|
if (found.size > ef) {
|
|
1999
2011
|
popHeap(found);
|
|
@@ -2355,7 +2367,7 @@ function copyCandidates(store, count, target) {
|
|
|
2355
2367
|
target.distances.set(store.distances.subarray(0, count));
|
|
2356
2368
|
target.size = count;
|
|
2357
2369
|
}
|
|
2358
|
-
function nativeTraverse(state, field, query, candidateCount, metric, candidates) {
|
|
2370
|
+
function nativeTraverse(state, field, query, candidateCount, metric, filter, candidates) {
|
|
2359
2371
|
const store = field.store;
|
|
2360
2372
|
reserveOrdinals(store, candidateCount);
|
|
2361
2373
|
let count;
|
|
@@ -2368,7 +2380,8 @@ function nativeTraverse(state, field, query, candidateCount, metric, candidates)
|
|
|
2368
2380
|
candidateCount,
|
|
2369
2381
|
state.locks.threadSlot,
|
|
2370
2382
|
store.ordinals,
|
|
2371
|
-
store.distances
|
|
2383
|
+
store.distances,
|
|
2384
|
+
filter === void 0 ? null : filter.bits
|
|
2372
2385
|
);
|
|
2373
2386
|
} catch (error) {
|
|
2374
2387
|
stopUsingTheNativeSearchCore(error);
|
|
@@ -2383,7 +2396,7 @@ function nativeRescoredHits(store, candidates, query, depth, metric, minSimilari
|
|
|
2383
2396
|
reserveOrdinals(store, nearest);
|
|
2384
2397
|
const ordinals = store.ordinals;
|
|
2385
2398
|
let kept = 0;
|
|
2386
|
-
for (let i = 0; i < nearest; i++) {
|
|
2399
|
+
for (let i = 0; i < candidates.size && kept < nearest; i++) {
|
|
2387
2400
|
const ord = candidates.ords[i];
|
|
2388
2401
|
if (filter && !ordinalFilterHas(filter, ord)) continue;
|
|
2389
2402
|
ordinals[kept++] = ord;
|
|
@@ -2400,30 +2413,74 @@ function nativeRescoredHits(store, candidates, query, depth, metric, minSimilari
|
|
|
2400
2413
|
return hits;
|
|
2401
2414
|
}
|
|
2402
2415
|
|
|
2403
|
-
// src/vector/
|
|
2404
|
-
|
|
2405
|
-
|
|
2406
|
-
|
|
2416
|
+
// src/vector/search-depth/constants.ts
|
|
2417
|
+
var NARROW_DIMENSION_LIMIT = 768;
|
|
2418
|
+
var WIDE_DIMENSION_LIMIT = 1024;
|
|
2419
|
+
var DEFAULT_EF_SEARCH_MARGIN_BY_BAND = {
|
|
2420
|
+
narrow: 50,
|
|
2421
|
+
middle: 75,
|
|
2422
|
+
wide: 0
|
|
2423
|
+
};
|
|
2424
|
+
var DEFAULT_RESCORE_DEPTH_FLOOR_PER_LIMIT_BY_BITS = {
|
|
2425
|
+
1: 3,
|
|
2426
|
+
2: 3,
|
|
2427
|
+
4: 1,
|
|
2428
|
+
8: 1
|
|
2429
|
+
};
|
|
2430
|
+
var DEFAULT_RESCORE_DEPTH_BY_BITS = {
|
|
2431
|
+
1: {
|
|
2432
|
+
narrow: { perLimit: 3.23, perSquareRootOfLimit: 55.9 },
|
|
2433
|
+
middle: { perLimit: 2.2, perSquareRootOfLimit: 27 },
|
|
2434
|
+
wide: { perLimit: 3, perSquareRootOfLimit: 0 }
|
|
2435
|
+
},
|
|
2436
|
+
2: {
|
|
2437
|
+
narrow: { perLimit: 2.6, perSquareRootOfLimit: 22.5 },
|
|
2438
|
+
middle: { perLimit: 1, perSquareRootOfLimit: 16 },
|
|
2439
|
+
wide: { perLimit: 3, perSquareRootOfLimit: 0 }
|
|
2440
|
+
},
|
|
2441
|
+
4: {
|
|
2442
|
+
narrow: { perLimit: 5, perSquareRootOfLimit: 0 },
|
|
2443
|
+
middle: { perLimit: 5, perSquareRootOfLimit: 0 },
|
|
2444
|
+
wide: { perLimit: 2, perSquareRootOfLimit: 0 }
|
|
2445
|
+
},
|
|
2446
|
+
8: {
|
|
2447
|
+
narrow: { perLimit: 2, perSquareRootOfLimit: 0 },
|
|
2448
|
+
middle: { perLimit: 2, perSquareRootOfLimit: 0 },
|
|
2449
|
+
wide: { perLimit: 2, perSquareRootOfLimit: 0 }
|
|
2450
|
+
}
|
|
2451
|
+
};
|
|
2452
|
+
|
|
2453
|
+
// src/vector/search-depth/index.ts
|
|
2454
|
+
function dimensionBand(dimension) {
|
|
2455
|
+
if (dimension < NARROW_DIMENSION_LIMIT) return "narrow";
|
|
2456
|
+
return dimension < WIDE_DIMENSION_LIMIT ? "middle" : "wide";
|
|
2457
|
+
}
|
|
2458
|
+
function defaultRescoreDepth(bits, dimension, k) {
|
|
2459
|
+
const { perLimit, perSquareRootOfLimit } = DEFAULT_RESCORE_DEPTH_BY_BITS[bits][dimensionBand(dimension)];
|
|
2460
|
+
const floor = DEFAULT_RESCORE_DEPTH_FLOOR_PER_LIMIT_BY_BITS[bits] * k;
|
|
2461
|
+
return Math.max(floor, Math.ceil(perLimit * k + perSquareRootOfLimit * Math.sqrt(k)));
|
|
2462
|
+
}
|
|
2463
|
+
function defaultEfSearch(dimension, k) {
|
|
2464
|
+
return Math.max(DEFAULT_EF_SEARCH, k + DEFAULT_EF_SEARCH_MARGIN_BY_BAND[dimensionBand(dimension)]);
|
|
2407
2465
|
}
|
|
2466
|
+
|
|
2467
|
+
// src/vector/hnsw/search.ts
|
|
2408
2468
|
function traverse(state, query, k, searchMetric, options) {
|
|
2409
|
-
const liveSize = nodeCountOf(state) - tombstoneCountOf(state);
|
|
2410
2469
|
const calibrated = activeQuantizer(state);
|
|
2411
2470
|
const quantizer = calibrated === void 0 || calibrated.size === 0 ? void 0 : calibrated;
|
|
2412
2471
|
const useQuantized = quantizer !== void 0;
|
|
2413
|
-
|
|
2414
|
-
|
|
2415
|
-
|
|
2416
|
-
if (filter && filter.count < liveSize) {
|
|
2417
|
-
const selectivity = filter.count / liveSize;
|
|
2418
|
-
ef = Math.max(ef, Math.ceil(k / Math.max(selectivity, 0.01)));
|
|
2419
|
-
ef = Math.min(ef, liveSize);
|
|
2472
|
+
let depth = k;
|
|
2473
|
+
if (quantizer !== void 0) {
|
|
2474
|
+
depth = options.oversample === void 0 ? defaultRescoreDepth(quantizer.bits, state.dimension, k) : Math.ceil(k * options.oversample);
|
|
2420
2475
|
}
|
|
2476
|
+
const ef = Math.max(options.efSearch ?? defaultEfSearch(state.dimension, k), depth);
|
|
2477
|
+
const filter = options.filter;
|
|
2421
2478
|
const workspace = state.workspace;
|
|
2422
2479
|
const candidates = workspace.traversal;
|
|
2423
2480
|
const native = nativeFieldFor(state);
|
|
2424
2481
|
if (native !== null) {
|
|
2425
2482
|
candidates.size = 0;
|
|
2426
|
-
if (nativeTraverse(state, native, query, ef, searchMetric, candidates)) {
|
|
2483
|
+
if (nativeTraverse(state, native, query, ef, searchMetric, filter, candidates)) {
|
|
2427
2484
|
return { candidates, depth: useQuantized ? depth : 0, arenaQuery: null, qMag: 0, native };
|
|
2428
2485
|
}
|
|
2429
2486
|
}
|
|
@@ -2451,7 +2508,7 @@ function traverse(state, query, k, searchMetric, options) {
|
|
|
2451
2508
|
setSingleEntryPoint(workspace, candidates.ords[0]);
|
|
2452
2509
|
}
|
|
2453
2510
|
}
|
|
2454
|
-
searchLayer(state, query, qMag, ef, 0, searchMetric, true, distFn, candidates);
|
|
2511
|
+
searchLayer(state, query, qMag, ef, 0, searchMetric, true, distFn, candidates, filter);
|
|
2455
2512
|
} finally {
|
|
2456
2513
|
unlockGraphShared(state.locks);
|
|
2457
2514
|
}
|
|
@@ -2555,10 +2612,11 @@ function collectHits(state, query, k, searchMetric, minSimilarity, options) {
|
|
|
2555
2612
|
function rescoreWithFullPrecision(state, candidates, query, qMag, arenaQuery, depth, metric, minSimilarity, filter) {
|
|
2556
2613
|
noteFallback("score");
|
|
2557
2614
|
const rescored = [];
|
|
2558
|
-
|
|
2559
|
-
for (let i = 0; i <
|
|
2615
|
+
let taken = 0;
|
|
2616
|
+
for (let i = 0; i < candidates.size && taken < depth; i++) {
|
|
2560
2617
|
const ord = candidates.ords[i];
|
|
2561
2618
|
if (filter && !ordinalFilterHas(filter, ord)) continue;
|
|
2619
|
+
taken += 1;
|
|
2562
2620
|
let fullDistance;
|
|
2563
2621
|
if (arenaQuery) {
|
|
2564
2622
|
fullDistance = state.store.distanceFromArena(arenaQuery, ord, metric);
|
|
@@ -3590,4 +3648,4 @@ function createHNSWIndex(dimension, store, config, quantizer, handles) {
|
|
|
3590
3648
|
};
|
|
3591
3649
|
}
|
|
3592
3650
|
|
|
3593
|
-
export { BLOCK_MAX_BYTES, GRAPH_RETIRED, IN_MEMORY, NOTHING_STAGED, QUERY_STAGED, STORE_BLOCK_COUNT,
|
|
3651
|
+
export { BLOCK_MAX_BYTES, GRAPH_RETIRED, IN_MEMORY, NOTHING_STAGED, QUERY_STAGED, STORE_BLOCK_COUNT, STORE_CALIBRATION_GENERATION, STORE_CODE_BLOCK_COUNT, STORE_DOC_ID_BYTES, STORE_LIVE_COUNT, STORE_ORDINAL_LAYOUT_GENERATION, STORE_RELEASED_VECTORS_TO_DISK, STORE_REMOVED_COUNT, STORE_SLOTS, STORE_UNPLACED_COUNT, addToOrdinalFilter, arenaFloat32Distance, cosineSimilarityWithMagnitudes, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createSharedVectorStoreHandles, createVectorBlock, detachNativeStore, dotProduct, ensureBlockSlots, euclideanDistance, fixedView, graphWithoutNodes, growBufferTo, insertNode, magnitude, nativeScoresOf, nativeStoreScratchBytes, nodeCountOf, openGraphState, openSharedQuantizer, openVectorBlock, ordinalFilterHas, ordinalFilterValues, osqBitsOf, releaseLocksHeldBy, removeFromOrdinalFilter, requireWholeNeighbourLists, resetStoreHeader, searchOrdinals, sharedMemoryAvailable, sharedVectorStoreBytes, slotByteOffset, toDistance, toScore, tombstoneCountOf, topLayerInParts, unplacedChunkCount };
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors } from './chunk-
|
|
2
|
-
import { getNestedValue } from './chunk-
|
|
3
|
-
import { requireValidFieldTypes } from './chunk-
|
|
1
|
+
import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors } from './chunk-JFUE6F2Y.mjs';
|
|
2
|
+
import { getNestedValue } from './chunk-SBCEHIPL.mjs';
|
|
3
|
+
import { requireValidFieldTypes } from './chunk-4ZJWRETC.mjs';
|
|
4
4
|
import { VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
5
|
-
import { ErrorCodes, NarsilError } from './chunk-
|
|
5
|
+
import { ErrorCodes, NarsilError } from './chunk-SYWGORQB.mjs';
|
|
6
6
|
import { decode } from '@msgpack/msgpack';
|
|
7
7
|
|
|
8
8
|
function validateReplicationEntry(entry, localPrimaryTerm, log) {
|
|
@@ -1,21 +1,21 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import { requirePatternSearchFor, requirePartialWordSearchFor, collectQueryTermStats, mergePartitionStats, createPartitionIndex, oversampledFacetConfig, availablePatternSearch, PATTERN_HOST, kWayMerge, mergeFacets, isCompositePartition, fnv1a, encodeSegmentState } from './chunk-
|
|
5
|
-
import { resolveVectorText, bindEngineCore, restoredConfigFields, restoredEmbedding, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, requireDocumentObject, admitInsert, asBatchInsertError, requireMatchingDocId, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-
|
|
1
|
+
import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, queryBindingOf, executeQuery, executePreflight, executeSuggest, executeListDocuments, validateDocId } from './chunk-R36MIGBU.mjs';
|
|
2
|
+
import { splitWorkerBudget, resolveWorkerCount, NATIVE_CRC32_MIN_BYTES, resolveWorkerEntry, missingWorkerEntry, MAX_FILE_BYTES, DEFAULT_REQUEST_TIMEOUT_MS, resolveRequestThreadCount, CHECKSUM_YIELD_CHUNK_BYTES, workerResourceLimits, DEFAULT_BACKPRESSURE_LIMIT, SHUTDOWN_TIMEOUT_MS, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH } from './chunk-RDJKYKMQ.mjs';
|
|
3
|
+
import { isEmptySynonymSet, synonymModule, createDirectExecutor, createPartitionRouter, readHeapStatistics, availableSynonymModule, mergeFrozenSegments, createSharedFrozenSegment, freezeSegmentShared, resolvePartitionInsertOptions, createFrozenSegment } from './chunk-3LYY62M6.mjs';
|
|
4
|
+
import { requirePatternSearchFor, requirePartialWordSearchFor, collectQueryTermStats, mergePartitionStats, createPartitionIndex, oversampledFacetConfig, availablePatternSearch, PATTERN_HOST, kWayMerge, mergeFacets, isCompositePartition, fnv1a, encodeSegmentState } from './chunk-CHOGQSHK.mjs';
|
|
5
|
+
import { resolveVectorText, bindEngineCore, restoredConfigFields, restoredEmbedding, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, requireDocumentObject, admitInsert, asBatchInsertError, requireMatchingDocId, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-WLOF6HSP.mjs';
|
|
6
6
|
import { detectRuntime } from './chunk-AV4IJKD2.mjs';
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
import { requireSearchBackendSettings } from './chunk-
|
|
10
|
-
import { DEFAULT_PATTERN_WORK_CAP, MATCHER_STATE_BUDGET_BYTES } from './chunk-
|
|
11
|
-
import { getNestedValue } from './chunk-
|
|
12
|
-
import { flattenSchema, raiseEarlierReleaseGraphSetting, messagePackUpperBoundBytes, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, extractVectorFieldsFromSchema, schemaFieldsOf, validateRequiredFields, requireDocumentWithinSizeLimit, validateDocument, validateDocumentStrict } from './chunk-
|
|
13
|
-
import { isSingleStringFieldType } from './chunk-
|
|
7
|
+
import { MAX_INDEX_CONFIGURATION_BYTES, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, HEAP_PRESSURE_WARNING_FRACTION, FOREIGN_STATS_TTL_MS, MAX_IMPORT_REFUSALS_REPORTED, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, BATCH_CHUNK_SIZE } from './chunk-Q5DC6WAB.mjs';
|
|
8
|
+
import { generateId, getLanguage } from './chunk-ZHS44YKF.mjs';
|
|
9
|
+
import { requireSearchBackendSettings } from './chunk-ES33L3HO.mjs';
|
|
10
|
+
import { DEFAULT_PATTERN_WORK_CAP, MATCHER_STATE_BUDGET_BYTES } from './chunk-QM3VPX4Z.mjs';
|
|
11
|
+
import { getNestedValue } from './chunk-SBCEHIPL.mjs';
|
|
12
|
+
import { flattenSchema, raiseEarlierReleaseGraphSetting, messagePackUpperBoundBytes, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, extractVectorFieldsFromSchema, schemaFieldsOf, validateRequiredFields, requireDocumentWithinSizeLimit, validateDocument, validateDocumentStrict } from './chunk-4ZJWRETC.mjs';
|
|
13
|
+
import { isSingleStringFieldType } from './chunk-BB7MASRQ.mjs';
|
|
14
14
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
15
15
|
import { createRequestId, isValidWorkerResponse, threadSlotOfWorker } from './chunk-R6PR4WYZ.mjs';
|
|
16
|
-
import { loadVectorCodeFor, requireLoadedVectorCode } from './chunk-
|
|
16
|
+
import { loadVectorCodeFor, requireLoadedVectorCode } from './chunk-HBVLF3HA.mjs';
|
|
17
17
|
import { MAX_DOCUMENT_NESTING_DEPTH } from './chunk-L5ORED7V.mjs';
|
|
18
|
-
import { ErrorCodes, NarsilError } from './chunk-
|
|
18
|
+
import { ErrorCodes, NarsilError } from './chunk-SYWGORQB.mjs';
|
|
19
19
|
import { encode, decode } from '@msgpack/msgpack';
|
|
20
20
|
import * as zlib from 'zlib';
|
|
21
21
|
|
|
@@ -993,6 +993,7 @@ function createLifecycleEntry(state) {
|
|
|
993
993
|
waiterCount: 0,
|
|
994
994
|
closePromise: null,
|
|
995
995
|
dropPromise: null,
|
|
996
|
+
replacePromise: null,
|
|
996
997
|
drainResolvers: /* @__PURE__ */ new Set()
|
|
997
998
|
};
|
|
998
999
|
}
|
|
@@ -1032,7 +1033,7 @@ function createIndexStateCoordinator(config, callbacks) {
|
|
|
1032
1033
|
return entry;
|
|
1033
1034
|
}
|
|
1034
1035
|
function publicState(entry) {
|
|
1035
|
-
return entry.state === "closing" || entry.state === "dropping" ? "open" : entry.state;
|
|
1036
|
+
return entry.state === "closing" || entry.state === "dropping" || entry.state === "replacing" ? "open" : entry.state;
|
|
1036
1037
|
}
|
|
1037
1038
|
async function waitForDrain(entry) {
|
|
1038
1039
|
if (entry.activeOperations === 0) return;
|
|
@@ -1072,6 +1073,11 @@ function createIndexStateCoordinator(config, callbacks) {
|
|
|
1072
1073
|
entry = entryOf(indexName);
|
|
1073
1074
|
continue;
|
|
1074
1075
|
}
|
|
1076
|
+
if (entry.state === "replacing" && entry.replacePromise !== null) {
|
|
1077
|
+
await entry.replacePromise.catch(() => void 0);
|
|
1078
|
+
entry = entryOf(indexName);
|
|
1079
|
+
continue;
|
|
1080
|
+
}
|
|
1075
1081
|
if (entry.state === "closing" && entry.closePromise !== null) {
|
|
1076
1082
|
await waitForTransition(
|
|
1077
1083
|
indexName,
|
|
@@ -1087,6 +1093,7 @@ function createIndexStateCoordinator(config, callbacks) {
|
|
|
1087
1093
|
function transitionPending(indexName, entry) {
|
|
1088
1094
|
if (entries.get(indexName) !== entry) return true;
|
|
1089
1095
|
if (entry.state === "dropping" && entry.dropPromise !== null) return true;
|
|
1096
|
+
if (entry.state === "replacing" && entry.replacePromise !== null) return true;
|
|
1090
1097
|
return entry.state === "closing" && entry.closePromise !== null;
|
|
1091
1098
|
}
|
|
1092
1099
|
async function reopen(indexName, explicit) {
|
|
@@ -1134,9 +1141,10 @@ function createIndexStateCoordinator(config, callbacks) {
|
|
|
1134
1141
|
entry = entryOf(indexName);
|
|
1135
1142
|
continue;
|
|
1136
1143
|
}
|
|
1137
|
-
|
|
1144
|
+
const settling = entry.reopenPromise ?? entry.replacePromise;
|
|
1145
|
+
if (settling !== null) {
|
|
1138
1146
|
if (automatic) return false;
|
|
1139
|
-
await
|
|
1147
|
+
await settling.catch(() => void 0);
|
|
1140
1148
|
entry = entryOf(indexName);
|
|
1141
1149
|
continue;
|
|
1142
1150
|
}
|
|
@@ -1178,13 +1186,9 @@ function createIndexStateCoordinator(config, callbacks) {
|
|
|
1178
1186
|
await entry.dropPromise;
|
|
1179
1187
|
return;
|
|
1180
1188
|
}
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
continue;
|
|
1185
|
-
}
|
|
1186
|
-
if (entry.closePromise !== null) {
|
|
1187
|
-
await entry.closePromise.catch(() => void 0);
|
|
1189
|
+
const pending = entry.replacePromise ?? entry.reopenPromise ?? entry.closePromise;
|
|
1190
|
+
if (pending !== null) {
|
|
1191
|
+
await pending.catch(() => void 0);
|
|
1188
1192
|
entry = entryOf(indexName);
|
|
1189
1193
|
continue;
|
|
1190
1194
|
}
|
|
@@ -1205,6 +1209,11 @@ function createIndexStateCoordinator(config, callbacks) {
|
|
|
1205
1209
|
entry.dropPromise = run;
|
|
1206
1210
|
await run;
|
|
1207
1211
|
}
|
|
1212
|
+
const replaceContext = {
|
|
1213
|
+
entryOf,
|
|
1214
|
+
waitForDrain,
|
|
1215
|
+
forget: (indexName) => entries.delete(indexName)
|
|
1216
|
+
};
|
|
1208
1217
|
function exceedsLimits() {
|
|
1209
1218
|
let open = 0;
|
|
1210
1219
|
let bytes = 0;
|
|
@@ -1298,6 +1307,25 @@ function createIndexStateCoordinator(config, callbacks) {
|
|
|
1298
1307
|
drop(indexName, action) {
|
|
1299
1308
|
return dropEntry(indexName, action);
|
|
1300
1309
|
},
|
|
1310
|
+
async replace(indexName, action) {
|
|
1311
|
+
const { replaceExclusively } = await import('./replace-CEO5KJJ3.mjs');
|
|
1312
|
+
await replaceExclusively(replaceContext, indexName, action);
|
|
1313
|
+
},
|
|
1314
|
+
async registerReplacing(indexName, action) {
|
|
1315
|
+
const entry = createLifecycleEntry("open");
|
|
1316
|
+
entries.set(indexName, entry);
|
|
1317
|
+
entry.state = "replacing";
|
|
1318
|
+
const run = import('./replace-CEO5KJJ3.mjs').then(
|
|
1319
|
+
({ replaceUnregistered }) => replaceUnregistered(replaceContext, indexName, entry, action),
|
|
1320
|
+
(error) => {
|
|
1321
|
+
if (entries.get(indexName) === entry) entries.delete(indexName);
|
|
1322
|
+
throw error;
|
|
1323
|
+
}
|
|
1324
|
+
);
|
|
1325
|
+
entry.replacePromise = run;
|
|
1326
|
+
await run;
|
|
1327
|
+
await enforceLimits(indexName);
|
|
1328
|
+
},
|
|
1301
1329
|
stateOf(indexName) {
|
|
1302
1330
|
return publicState(entryOf(indexName));
|
|
1303
1331
|
},
|
|
@@ -1309,7 +1337,7 @@ function createIndexStateCoordinator(config, callbacks) {
|
|
|
1309
1337
|
let closed = 0;
|
|
1310
1338
|
let reopens = 0;
|
|
1311
1339
|
for (const entry of entries.values()) {
|
|
1312
|
-
if (entry.state === "open" || entry.state === "closing") open += 1;
|
|
1340
|
+
if (entry.state === "open" || entry.state === "closing" || entry.state === "replacing") open += 1;
|
|
1313
1341
|
else closed += 1;
|
|
1314
1342
|
reopens += entry.reopenCount;
|
|
1315
1343
|
}
|
|
@@ -3718,6 +3746,7 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
3718
3746
|
reportedIneligible: /* @__PURE__ */ new Set(),
|
|
3719
3747
|
scaledOutIndexes: /* @__PURE__ */ new Set(),
|
|
3720
3748
|
desyncedIndexes: /* @__PURE__ */ new Set(),
|
|
3749
|
+
copyReleaseHolds: /* @__PURE__ */ new Map(),
|
|
3721
3750
|
copyLoadBuffers: /* @__PURE__ */ new Map(),
|
|
3722
3751
|
copyTransitions: /* @__PURE__ */ new Map(),
|
|
3723
3752
|
droppedCopies: /* @__PURE__ */ new Map(),
|
|
@@ -3752,10 +3781,21 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
3752
3781
|
for (let pending = settlingCopyWork(indexName); pending !== void 0; pending = settlingCopyWork(indexName)) {
|
|
3753
3782
|
await pending;
|
|
3754
3783
|
}
|
|
3784
|
+
const hold = state.copyReleaseHolds.get(indexName);
|
|
3785
|
+
if (hold !== void 0) {
|
|
3786
|
+
hold.reloads ||= wasScaledOut;
|
|
3787
|
+
return;
|
|
3788
|
+
}
|
|
3755
3789
|
state.desyncedIndexes.delete(indexName);
|
|
3756
3790
|
if (!wasScaledOut) return;
|
|
3757
3791
|
await scaleOutIndex(state, indexName, copyThresholdReason(state, indexName));
|
|
3758
3792
|
}
|
|
3793
|
+
const copyReleaseSteps = {
|
|
3794
|
+
awaitReplicationIdle,
|
|
3795
|
+
cancelIdleMerge,
|
|
3796
|
+
heldNowhere: (error) => error instanceof NarsilError && error.code === ErrorCodes.INDEX_NOT_FOUND,
|
|
3797
|
+
resyncIndex
|
|
3798
|
+
};
|
|
3759
3799
|
async function openIndex(indexName) {
|
|
3760
3800
|
state.droppedCopies.delete(indexName);
|
|
3761
3801
|
if (!indexReadyForCopies(state, indexName)) return;
|
|
@@ -3865,7 +3905,7 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
3865
3905
|
pool.removeIndex(indexName);
|
|
3866
3906
|
}
|
|
3867
3907
|
state.droppedCopies.delete(indexName);
|
|
3868
|
-
state.desyncedIndexes.delete(indexName);
|
|
3908
|
+
if (!state.copyReleaseHolds.has(indexName)) state.desyncedIndexes.delete(indexName);
|
|
3869
3909
|
state.lastAccessAt.delete(indexName);
|
|
3870
3910
|
state.replicationQueues.delete(indexName);
|
|
3871
3911
|
state.segmentLedger.delete(indexName);
|
|
@@ -3911,6 +3951,18 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
3911
3951
|
if (wasScaledOut) withdrawCopies(indexName);
|
|
3912
3952
|
return wasScaledOut || loading !== void 0 && loading.kind !== "drop";
|
|
3913
3953
|
},
|
|
3954
|
+
async releaseIndexCopies(indexName) {
|
|
3955
|
+
const hold = state.copyReleaseHolds.get(indexName);
|
|
3956
|
+
if (hold === void 0) state.copyReleaseHolds.set(indexName, { holders: 1, reloads: false });
|
|
3957
|
+
else hold.holders += 1;
|
|
3958
|
+
state.desyncedIndexes.add(indexName);
|
|
3959
|
+
const { releaseIndexCopies } = await import('./release-BH3DKW3T.mjs');
|
|
3960
|
+
return releaseIndexCopies(state, indexName, copyReleaseSteps);
|
|
3961
|
+
},
|
|
3962
|
+
async endCopyRelease(indexName, reloads) {
|
|
3963
|
+
const { endCopyRelease } = await import('./release-BH3DKW3T.mjs');
|
|
3964
|
+
await endCopyRelease(state, indexName, reloads, copyReleaseSteps);
|
|
3965
|
+
},
|
|
3914
3966
|
resyncIndex,
|
|
3915
3967
|
noteAccess: (indexName) => noteAccess(state, indexName),
|
|
3916
3968
|
workerCopies,
|
|
@@ -3956,7 +4008,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
3956
4008
|
if (config?.lifecycle !== void 0 && durabilityTier === null) {
|
|
3957
4009
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
|
|
3958
4010
|
}
|
|
3959
|
-
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-
|
|
4011
|
+
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-FRPV6JLZ.mjs');
|
|
3960
4012
|
const filesystemDurability = durabilityTier?.kind === "wal";
|
|
3961
4013
|
const executor = createDirectExecutor({
|
|
3962
4014
|
vectorWorkerCopies: vectorCopyPolicy,
|
|
@@ -4680,6 +4732,9 @@ async function runLifecycleHook(core, hookName, context) {
|
|
|
4680
4732
|
async function announceIndexCreated(core, name, config) {
|
|
4681
4733
|
await runLifecycleHook(core, "onIndexCreate", { indexName: name, config });
|
|
4682
4734
|
}
|
|
4735
|
+
async function announceIndexDropped(core, name, config) {
|
|
4736
|
+
await runLifecycleHook(core, "onIndexDrop", { indexName: name, config });
|
|
4737
|
+
}
|
|
4683
4738
|
async function createEngineIndex(core, config, name, requestedConfig, creation = {}) {
|
|
4684
4739
|
const { indexUuid, adapterBinding = "required", appliesCreationLimits = false } = creation;
|
|
4685
4740
|
const loadingVectorCode = loadVectorCodeFor(requestedConfig.schema);
|
|
@@ -6834,6 +6889,14 @@ function portabilityOperations(core, config, engine) {
|
|
|
6834
6889
|
release();
|
|
6835
6890
|
}
|
|
6836
6891
|
}
|
|
6892
|
+
const restoresInFlight = /* @__PURE__ */ new Map();
|
|
6893
|
+
const restoreHost = {
|
|
6894
|
+
getVectorFieldPaths,
|
|
6895
|
+
announceIndexCreated: (name, restoredConfig) => announceIndexCreated(core, name, restoredConfig),
|
|
6896
|
+
announceIndexDropped: (name, replacedConfig) => announceIndexDropped(core, name, replacedConfig),
|
|
6897
|
+
defaultEmbeddingAdapter: config?.embedding ?? null,
|
|
6898
|
+
releaseTimeoutMs: DEFAULT_REQUEST_TIMEOUT_MS
|
|
6899
|
+
};
|
|
6837
6900
|
function snapshotDepsOf(indexName) {
|
|
6838
6901
|
return {
|
|
6839
6902
|
indexName,
|
|
@@ -6842,41 +6905,27 @@ function portabilityOperations(core, config, engine) {
|
|
|
6842
6905
|
durability: core.durability
|
|
6843
6906
|
};
|
|
6844
6907
|
}
|
|
6845
|
-
async function restoreIndex(indexName, data) {
|
|
6846
|
-
const { restoreFromSnapshot } = await import('./snapshot-JFZVOGZV.mjs');
|
|
6847
|
-
await restoreFromSnapshot(indexName, data, {
|
|
6848
|
-
executor: core.executor,
|
|
6849
|
-
indexRegistry: core.indexRegistry,
|
|
6850
|
-
getVectorFieldPaths,
|
|
6851
|
-
dropIndex: (name) => dropEngineIndex(core, name),
|
|
6852
|
-
announceIndexCreated: (name, restoredConfig) => announceIndexCreated(core, name, restoredConfig),
|
|
6853
|
-
requireManager: core.requireManager,
|
|
6854
|
-
durability: core.durability,
|
|
6855
|
-
filesystemDurability: core.filesystemDurability,
|
|
6856
|
-
embeddingAdapters: core.embeddingAdapters,
|
|
6857
|
-
defaultEmbeddingAdapter: config?.embedding ?? null,
|
|
6858
|
-
markAnalysisStale: core.analysisRebuild.markStale,
|
|
6859
|
-
clearAnalysisStale: core.analysisRebuild.clearStale,
|
|
6860
|
-
indexState: core.indexState
|
|
6861
|
-
});
|
|
6862
|
-
}
|
|
6863
6908
|
return {
|
|
6864
6909
|
snapshot(indexName) {
|
|
6865
6910
|
return withOpenIndex(indexName, async () => {
|
|
6866
|
-
const { createSnapshot } = await import('./snapshot-
|
|
6911
|
+
const { createSnapshot } = await import('./snapshot-YC6TS4OP.mjs');
|
|
6867
6912
|
return createSnapshot(snapshotDepsOf(indexName));
|
|
6868
6913
|
});
|
|
6869
6914
|
},
|
|
6870
6915
|
async restore(indexName, data) {
|
|
6871
6916
|
core.guardShutdown();
|
|
6872
|
-
|
|
6917
|
+
validateIndexName(indexName);
|
|
6918
|
+
const restored = import('./restore-queue-I5SI6AUQ.mjs').then(
|
|
6919
|
+
({ queueRestore }) => queueRestore(core, restoreHost, restoresInFlight, indexName, data)
|
|
6920
|
+
);
|
|
6921
|
+
await emitWriteEventAfter(core, indexName, restored);
|
|
6873
6922
|
core.heapPressureNotifier.check(indexName);
|
|
6874
6923
|
},
|
|
6875
6924
|
exportDocuments(indexName) {
|
|
6876
6925
|
return withOpenIndex(indexName, async () => {
|
|
6877
6926
|
const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
|
|
6878
|
-
import('./snapshot-
|
|
6879
|
-
import('./write-
|
|
6927
|
+
import('./snapshot-YC6TS4OP.mjs'),
|
|
6928
|
+
import('./write-M2OALSA4.mjs')
|
|
6880
6929
|
]);
|
|
6881
6930
|
const header = indexConfigurationOf(core.requireIndex(indexName));
|
|
6882
6931
|
return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
|