@delali/narsil 0.3.1-next.131 → 0.3.1-next.140
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/analysis-change-JN23AZQP.mjs +43 -0
- package/dist/analysis-change-entry-5JSIMEKG.mjs +1 -0
- package/dist/browser/analysis-change-57MIRJWR.mjs +499 -0
- package/dist/browser/analysis-change-entry-ICGRR3IN.mjs +1 -0
- package/dist/browser/{chunk-UDGB2MWT.mjs → chunk-35B42UFM.mjs} +4 -0
- package/dist/browser/{chunk-VBBLTD7O.mjs → chunk-3EYDURFD.mjs} +39 -27
- package/dist/browser/{chunk-AIIJDWMG.mjs → chunk-4MFH226Q.mjs} +1 -1
- package/dist/browser/{chunk-D7WQF6DV.mjs → chunk-5LYWZJE2.mjs} +2 -2
- package/dist/browser/chunk-67KSBUYR.mjs +173 -0
- package/dist/browser/{chunk-M5UAAXGU.mjs → chunk-6BSB4N6B.mjs} +2 -2
- package/dist/browser/{chunk-NSCSR6MA.mjs → chunk-CGVBGHBX.mjs} +81 -6
- package/dist/browser/chunk-E5SFTOQR.mjs +21 -0
- package/dist/browser/{chunk-VEGHFP2P.mjs → chunk-MZI3TEOW.mjs} +4 -2
- package/dist/browser/chunk-OIZS5KIX.mjs +6 -0
- package/dist/browser/chunk-QHNNULPW.mjs +265 -0
- package/dist/browser/chunk-XVMOQQ34.mjs +384 -0
- package/dist/browser/{chunk-2Y3CO4EQ.mjs → chunk-ZAIZQOTM.mjs} +256 -777
- package/dist/browser/{durability-wiring-FNZSRH4N.mjs → durability-wiring-MTIAN6VT.mjs} +145 -50
- package/dist/browser/import-Q7MXD6BS.mjs +8 -0
- package/dist/browser/{module-4HIMOQM5.mjs → module-5J3PEUEB.mjs} +7 -6
- package/dist/browser/{phrase-module-WESYVU7K.mjs → phrase-module-HGVBO24W.mjs} +1 -1
- package/dist/{restore-queue-JAMNRC6T.mjs → browser/restore-queue-DISFECE4.mjs} +8 -2
- package/dist/browser/{runtime-3X7ZJ5T3.mjs → runtime-VEUH674B.mjs} +2 -2
- package/dist/browser/{snapshot-V7VF6BN6.mjs → snapshot-YBJ5MF2M.mjs} +36 -167
- package/dist/browser/{write-YIC2MKRY.mjs → write-RTWI5MXL.mjs} +8 -6
- package/dist/{capabilities-B6WxrWxt.d.ts → capabilities-DXAmysqJ.d.ts} +10 -3
- package/dist/{chunk-NMXRXFGZ.mjs → chunk-4GCN34JV.mjs} +2 -2
- package/dist/{chunk-SOHMMNCH.mjs → chunk-6VHWXAOI.mjs} +4 -4
- package/dist/{chunk-2C3WXBQS.mjs → chunk-74XB2SXI.mjs} +1 -1
- package/dist/{chunk-OGLV4I3L.mjs → chunk-B5I3IL4H.mjs} +12 -7
- package/dist/chunk-BDIDUW2B.mjs +387 -0
- package/dist/{chunk-5LX6MOIB.mjs → chunk-C3YDYX6C.mjs} +1 -1
- package/dist/{chunk-Y4BVTQ53.mjs → chunk-CCCGEPJM.mjs} +2 -2
- package/dist/{chunk-2JWNJSNE.mjs → chunk-CPCGEIOM.mjs} +11 -8
- package/dist/chunk-CZQ33DLW.mjs +78 -0
- package/dist/{chunk-4GDLFDPZ.mjs → chunk-DD2673TX.mjs} +3 -2
- package/dist/{chunk-RNSC4NFI.mjs → chunk-EEBCDP36.mjs} +3 -1
- package/dist/{chunk-HCGJKDZV.mjs → chunk-FFVLPCLH.mjs} +1 -1
- package/dist/{chunk-WVRDRLIC.mjs → chunk-FKARMUWH.mjs} +6 -6
- package/dist/{chunk-PATMDIMN.mjs → chunk-HJJWY6PT.mjs} +1 -1
- package/dist/chunk-HMDT3I63.mjs +21 -0
- package/dist/{chunk-CZ7ISLXA.mjs → chunk-HNHCNTVG.mjs} +4 -0
- package/dist/{chunk-RWESFSJI.mjs → chunk-IBATGS3J.mjs} +87 -61
- package/dist/{chunk-Y2U5VVWX.mjs → chunk-ICW45TTU.mjs} +1 -1
- package/dist/{chunk-T4OFG4IQ.mjs → chunk-IP4HJOGC.mjs} +90 -16
- package/dist/{chunk-EUCX5K74.mjs → chunk-J3EHATEZ.mjs} +3 -3
- package/dist/{chunk-X5CNUKD5.mjs → chunk-K3AQOZVM.mjs} +37 -20
- package/dist/chunk-LK3MOTXF.mjs +174 -0
- package/dist/{chunk-7OIQ2SIO.mjs → chunk-LLDFIKGP.mjs} +4 -4
- package/dist/{chunk-K77FJXKS.mjs → chunk-MTQTX4BP.mjs} +81 -8
- package/dist/{chunk-ZR4VM4A6.mjs → chunk-N37VMC62.mjs} +1 -18
- package/dist/{chunk-2WBY4Y4M.mjs → chunk-NOT5QOHU.mjs} +1 -1
- package/dist/{chunk-D5TFRB5G.mjs → chunk-NPNK57CA.mjs} +1 -1
- package/dist/{chunk-6SSVIW7Y.mjs → chunk-O4LJQI4G.mjs} +31 -5
- package/dist/{chunk-JJ6EUMXB.mjs → chunk-PNNVTKMT.mjs} +2 -2
- package/dist/{chunk-2SEL6SBS.mjs → chunk-RKAVHNRW.mjs} +2 -2
- package/dist/{chunk-RCMNL7RI.mjs → chunk-SBMAWTRG.mjs} +41 -4
- package/dist/chunk-TCWQJL7O.mjs +188 -0
- package/dist/{chunk-2MKX2SCO.mjs → chunk-U4X4AWKL.mjs} +7 -7
- package/dist/{chunk-PU6SKH7O.mjs → chunk-UBXJQJII.mjs} +2 -2
- package/dist/chunk-UEV53BBO.mjs +480 -0
- package/dist/{chunk-NNO7CO3A.mjs → chunk-UIKN2OTQ.mjs} +1 -1
- package/dist/{chunk-F6AAXI7T.mjs → chunk-VSUYTPME.mjs} +3 -3
- package/dist/{chunk-TGHNXN7W.mjs → chunk-W2XDK66Z.mjs} +180 -541
- package/dist/{chunk-7X5JST5L.mjs → chunk-WNBRW4Q2.mjs} +16 -2
- package/dist/{chunk-6M6SDH3S.mjs → chunk-XMQNOD5V.mjs} +8 -8
- package/dist/{chunk-SZGRYEIC.mjs → chunk-XS2VM7UE.mjs} +3 -3
- package/dist/{chunk-JT3YZGCE.mjs → chunk-XXKWEG53.mjs} +3 -188
- package/dist/{chunk-5W7AEVTH.mjs → chunk-ZMBFWQXN.mjs} +2 -2
- package/dist/{chunk-7XY6VD7B.mjs → chunk-ZSWHPKGM.mjs} +1 -1
- package/dist/{client-gNAfaHcE.d.ts → client-DApBeLbk.d.ts} +17 -2
- package/dist/client.d.ts +5 -5
- package/dist/client.mjs +11 -1
- package/dist/{config-D1cDv3sE.d.ts → config-B0H-vp4U.d.ts} +4 -0
- package/dist/{config-CmYJO34X.d.ts → config-Brysl9NO.d.ts} +1 -1
- package/dist/{direct-executor-HQ4QVDX7.mjs → direct-executor-J5UDLPL3.mjs} +14 -14
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +21 -5
- package/dist/distribution.mjs +882 -192
- package/dist/{durability-wiring-S7OX2MAI.mjs → durability-wiring-AYOTAPNQ.mjs} +165 -70
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-m_Ddr1bz.d.ts → engine-CuLMbH1-.d.ts} +63 -6
- package/dist/import-QRZAQ5RH.mjs +43 -0
- package/dist/index.browser.mjs +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.mjs +20 -20
- package/dist/{long-query-thread-BSL25PD6.mjs → long-query-thread-U7K5A72G.mjs} +14 -14
- package/dist/{module-2E4XYSGT.mjs → module-L2OZO5PQ.mjs} +11 -11
- package/dist/partial-words.mjs +5 -5
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +11 -0
- package/dist/persistence/durability/checkpoint-worker.mjs +23 -23
- package/dist/{phrase-module-JERF2IDG.mjs → phrase-module-5QHBD2VJ.mjs} +1 -1
- package/dist/react.d.ts +7 -7
- package/dist/react.mjs +4 -0
- package/dist/{rerank-DMMT4FZA.mjs → rerank-KTLC3E6N.mjs} +5 -5
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{browser/restore-queue-GEZ54CA5.mjs → restore-queue-RYSRRFS4.mjs} +8 -2
- package/dist/{runtime-7MLQ7J64.mjs → runtime-DYV75DCB.mjs} +6 -6
- package/dist/server.d.ts +10 -8
- package/dist/server.mjs +51 -19
- package/dist/{snapshot-DOQLGGEX.mjs → snapshot-BTRZ3MZK.mjs} +59 -190
- package/dist/synonyms.mjs +6 -4
- package/dist/{tasks-Co8HfRD7.d.ts → tasks-DjGjK70Y.d.ts} +22 -6
- package/dist/{thread-SKO7CN3R.mjs → thread-USB3GOPY.mjs} +13 -13
- package/dist/vector/search-worker.mjs +5 -5
- package/dist/workers/entry.mjs +7 -7
- package/dist/{write-ZKR7SNRT.mjs → write-GHTA56OW.mjs} +25 -22
- package/openapi.json +101 -6
- package/package.json +7 -7
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { unpackLevels, osqRecordBytes, VECTOR_FILE_MAX_VECTORS, NO_LIVE_VECTOR_AT_POSITION, readTrailer, stageQueryLevels, osqCodeBytes, osqStagedQueryBytes, bytesToVectors, VECTOR_GRAPH_PART_VECTORS, VECTOR_GRAPH_PART_PAYLOAD_VERSION, VECTOR_FILE_PAYLOAD_VERSION, deadBitsWhere, vectorsToBytes, writeRecord } from './chunk-
|
|
2
|
-
import { markSimdUnavailable, MAX_WASM_PAGES, WASM_PAGE_BYTES, VECTOR_SCRATCH_SLOTS, UNPLACED_FLAG_CHUNK_ORDINALS, COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, MAX_FILE_BYTES, VECTOR_BLOCK_INITIAL_SLOTS, MAX_VECTOR_ORDINALS, VECTOR_STORE_INITIAL_CAPACITY, PLACEMENT_ROOM_REQUESTS, LOCK_SPIN_ITERATIONS, LOCK_WAIT_SLICE_MS, MAX_LAYER_CAP, INITIAL_LIST_CAPACITY, OSQ_PLACEMENT_QUERY_BITS, GRAPH_LOCK_WAIT_SLICE_MS, OSQ_STEPS, OSQ_GRID, OSQ_QUERY_BITS, GRAPH_LOCK_WAIT_DEADLINE_MS, compareCodePoints, INITIAL_UPPER_SLOTS, OSQ_REFINE_ROUNDS, OSQ_REFINE_TOLERANCE, threadSlotGone, DEFAULT_EF_SEARCH, OSQ_LAMBDA, NEAREST_LINK_FALLBACK_HOSTS, FARTHEST_REJECTED_LINKS_KEPT, MAX_DOC_ID_TABLE_BYTES } from './chunk-
|
|
3
|
-
import { MAX_DOC_ID_LENGTH } from './chunk-
|
|
4
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
1
|
+
import { unpackLevels, osqRecordBytes, VECTOR_FILE_MAX_VECTORS, NO_LIVE_VECTOR_AT_POSITION, readTrailer, stageQueryLevels, osqCodeBytes, osqStagedQueryBytes, bytesToVectors, VECTOR_GRAPH_PART_VECTORS, VECTOR_GRAPH_PART_PAYLOAD_VERSION, VECTOR_FILE_PAYLOAD_VERSION, deadBitsWhere, vectorsToBytes, writeRecord } from './chunk-5LYWZJE2.mjs';
|
|
2
|
+
import { markSimdUnavailable, MAX_WASM_PAGES, WASM_PAGE_BYTES, VECTOR_SCRATCH_SLOTS, UNPLACED_FLAG_CHUNK_ORDINALS, COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, MAX_FILE_BYTES, VECTOR_BLOCK_INITIAL_SLOTS, MAX_VECTOR_ORDINALS, VECTOR_STORE_INITIAL_CAPACITY, PLACEMENT_ROOM_REQUESTS, LOCK_SPIN_ITERATIONS, LOCK_WAIT_SLICE_MS, MAX_LAYER_CAP, INITIAL_LIST_CAPACITY, OSQ_PLACEMENT_QUERY_BITS, GRAPH_LOCK_WAIT_SLICE_MS, OSQ_STEPS, OSQ_GRID, OSQ_QUERY_BITS, GRAPH_LOCK_WAIT_DEADLINE_MS, compareCodePoints, INITIAL_UPPER_SLOTS, OSQ_REFINE_ROUNDS, OSQ_REFINE_TOLERANCE, threadSlotGone, DEFAULT_EF_SEARCH, OSQ_LAMBDA, NEAREST_LINK_FALLBACK_HOSTS, FARTHEST_REJECTED_LINKS_KEPT, MAX_DOC_ID_TABLE_BYTES } from './chunk-4MFH226Q.mjs';
|
|
3
|
+
import { MAX_DOC_ID_LENGTH } from './chunk-MZI3TEOW.mjs';
|
|
4
|
+
import { NarsilError, ErrorCodes } from './chunk-35B42UFM.mjs';
|
|
5
5
|
|
|
6
6
|
// src/vector/shared-buffers/growable.ts
|
|
7
7
|
function sharedMemoryAvailable() {
|
|
@@ -938,6 +938,36 @@ function replaceNeighbors(adj, ord, layer, neighbors, count) {
|
|
|
938
938
|
writeNeighbors(adj, ord, 0, neighbors, count);
|
|
939
939
|
}
|
|
940
940
|
|
|
941
|
+
// src/vector/hnsw/insertion-order.ts
|
|
942
|
+
var GOLDEN_GAMMA = 2654435769;
|
|
943
|
+
var FIRST_MIX_MULTIPLIER = 569420461;
|
|
944
|
+
var SECOND_MIX_MULTIPLIER = 1935289751;
|
|
945
|
+
var TWO_TO_THE_32 = 4294967296;
|
|
946
|
+
var HALF_A_STEP = 0.5;
|
|
947
|
+
function mixedUint32(value) {
|
|
948
|
+
let mixed = value + GOLDEN_GAMMA | 0;
|
|
949
|
+
mixed ^= mixed >>> 16;
|
|
950
|
+
mixed = Math.imul(mixed, FIRST_MIX_MULTIPLIER);
|
|
951
|
+
mixed ^= mixed >>> 15;
|
|
952
|
+
mixed = Math.imul(mixed, SECOND_MIX_MULTIPLIER);
|
|
953
|
+
mixed ^= mixed >>> 15;
|
|
954
|
+
return mixed >>> 0;
|
|
955
|
+
}
|
|
956
|
+
function unitDrawOf(value) {
|
|
957
|
+
return (mixedUint32(value) + HALF_A_STEP) / TWO_TO_THE_32;
|
|
958
|
+
}
|
|
959
|
+
function shuffleInPlace(items) {
|
|
960
|
+
let stream = items.length;
|
|
961
|
+
for (let i = items.length - 1; i > 0; i--) {
|
|
962
|
+
stream = mixedUint32(stream);
|
|
963
|
+
const j = Math.floor(stream / TWO_TO_THE_32 * (i + 1));
|
|
964
|
+
const held = items[i];
|
|
965
|
+
items[i] = items[j];
|
|
966
|
+
items[j] = held;
|
|
967
|
+
}
|
|
968
|
+
return items;
|
|
969
|
+
}
|
|
970
|
+
|
|
941
971
|
// src/vector/hnsw/shared.ts
|
|
942
972
|
function entryPointOf(state) {
|
|
943
973
|
return Atomics.load(state.header, GRAPH_ENTRY_POINT);
|
|
@@ -1006,10 +1036,8 @@ function toScore(distance, metric) {
|
|
|
1006
1036
|
return 1 / (1 + distance);
|
|
1007
1037
|
}
|
|
1008
1038
|
}
|
|
1009
|
-
function
|
|
1010
|
-
|
|
1011
|
-
if (u === 0) u = Number.MIN_VALUE;
|
|
1012
|
-
return Math.min(Math.floor(-Math.log(u) * mL), MAX_LAYER_CAP);
|
|
1039
|
+
function levelOfOrdinal(ord, mL) {
|
|
1040
|
+
return Math.min(Math.floor(-Math.log(unitDrawOf(ord)) * mL), MAX_LAYER_CAP);
|
|
1013
1041
|
}
|
|
1014
1042
|
function entryForOrd(state, ord) {
|
|
1015
1043
|
return state.store.entryForOrdinal(ord);
|
|
@@ -2232,17 +2260,6 @@ function everyCandidateWasHeld() {
|
|
|
2232
2260
|
return true;
|
|
2233
2261
|
}
|
|
2234
2262
|
|
|
2235
|
-
// src/vector/hnsw/insertion-order.ts
|
|
2236
|
-
function shuffleInPlace(items) {
|
|
2237
|
-
for (let i = items.length - 1; i > 0; i--) {
|
|
2238
|
-
const j = Math.floor(Math.random() * (i + 1));
|
|
2239
|
-
const held = items[i];
|
|
2240
|
-
items[i] = items[j];
|
|
2241
|
-
items[j] = held;
|
|
2242
|
-
}
|
|
2243
|
-
return items;
|
|
2244
|
-
}
|
|
2245
|
-
|
|
2246
2263
|
// src/vector/hnsw/mutation.ts
|
|
2247
2264
|
function clearTombstone(state, ord) {
|
|
2248
2265
|
if (!reachTombstone(state, ord) || Atomics.load(state.tombstones, ord) !== 1) return;
|
|
@@ -2370,7 +2387,7 @@ function writeRecordBeforePlacement(state, ord) {
|
|
|
2370
2387
|
function placeThroughTheCore(state, ord, holdsGraphLock) {
|
|
2371
2388
|
let field = nativeFieldFor();
|
|
2372
2389
|
if (field === null) return false;
|
|
2373
|
-
const level2 =
|
|
2390
|
+
const level2 = levelOfOrdinal(ord, state.mL);
|
|
2374
2391
|
for (let request = 0; request < PLACEMENT_ROOM_REQUESTS && field !== null; request++) {
|
|
2375
2392
|
ensureUpperRoom(state.adjacency, level2);
|
|
2376
2393
|
const placement = nativePlace(state, field, state.buildMetric, ord, level2, holdsGraphLock);
|
|
@@ -2396,7 +2413,7 @@ function insertNode(state, ord, holdsGraphLock = false) {
|
|
|
2396
2413
|
if (!holdsGraphLock) lockGraphShared(state.locks);
|
|
2397
2414
|
try {
|
|
2398
2415
|
clearTombstone(state, ord);
|
|
2399
|
-
const level2 =
|
|
2416
|
+
const level2 = levelOfOrdinal(ord, state.mL);
|
|
2400
2417
|
createNode(state.adjacency, ord, level2);
|
|
2401
2418
|
Atomics.add(state.header, GRAPH_NODE_COUNT, 1);
|
|
2402
2419
|
if (!claimEntryIfEmpty(state, ord, level2)) linkNode(state, ord, level2);
|
|
@@ -4519,9 +4536,4 @@ function recordCheckpoint(state, plan, written) {
|
|
|
4519
4536
|
state.savedSignature = plan.signature;
|
|
4520
4537
|
}
|
|
4521
4538
|
|
|
4522
|
-
|
|
4523
|
-
function collectAndReturnPages() {
|
|
4524
|
-
return false;
|
|
4525
|
-
}
|
|
4526
|
-
|
|
4527
|
-
export { BLOCK_MAX_BYTES, BUILD_CHUNK_SIZE, BUILD_RETRY_FIRST_DELAY_MS, BUILD_RETRY_MAX_DELAY_MS, DEFAULT_FILTER_THRESHOLD, DEFAULT_PROMOTION_THRESHOLD, DISK_ADOPTION_YIELD_INTERVAL, DRIFT_CHECK_GROWTH_FACTOR, ESTIMATED_MS_PER_TOMBSTONE, ESTIMATED_MS_PER_VECTOR_REBUILD, GRAPH_RETIRED, IN_MEMORY, NOTHING_STAGED, OSQ4_MIN_DIMENSION, QUERY_STAGED, RECALIBRATION_CHUNK_VECTORS, RECALIBRATION_DRIFT_RATIO, SHUFFLED_REBUILD_MEMORY_RETRY_MS, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, STORE_DOC_ID_BYTES, STORE_LIVE_COUNT, STORE_ORDINAL_LAYOUT_GENERATION, STORE_RELEASED_VECTORS_TO_DISK, STORE_REMOVED_COUNT, STORE_SLOTS, STORE_UNPLACED_COUNT, VECTOR_WORKER_COPIES_ALLOWED, WORKER_COPY_MIN_VECTORS, addToOrdinalFilter, adoptGraph, adoptLoadedCalibration, allLiveDocIds, arenaFloat32Distance, assignStorePartitions, blocksGrowthBytes, centroidOfCentroidSums, collectAndReturnPages, cosineSimilarityWithMagnitudes, countOfCentroidSums, createCentroidSums, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createSharedVectorStoreHandles, createVectorBlock, deserialize, detachNativeStore, dotProduct, emptyFieldBeforeRestore, ensureBlockSlots, euclideanDistance, fieldHandlesOf, filterForOptions, fixedView, graphNeedsRebuild, growBufferTo, grownByteLength, handOverBuildTurns, insertNode, liveOrdinals, liveSize, magnitude, meanOfCentroidSums, nativeScoresOf, nativeStoreScratchBytes, noBuildHistory, nodeCountOf, noteFallback, openGraphState, openSharedQuantizer, openVectorBlock, ordinalFilterForSubsets, ordinalFilterHas, ordinalFilterValues, osqBitsOf, planCheckpoint, recalibrateFromStore, recordCheckpoint, releaseLocksHeldBy, releaseReplacedVectors, requireWholeNeighbourLists, resetStoreHeader, searchOrdinals, sharedMemoryAvailable, sharedVectorStoreBytes, shuffleInPlace, signatureOf, slotByteOffset, squaredLengthsOfCentroidSums, threadsHoldCurrentLayout, toDistance, toScore, tombstoneCountOf, topLayerInParts, unplacedChunkCount, vectorsPerFile, yieldBuildTurn };
|
|
4539
|
+
export { BLOCK_MAX_BYTES, BUILD_CHUNK_SIZE, BUILD_RETRY_FIRST_DELAY_MS, BUILD_RETRY_MAX_DELAY_MS, DEFAULT_FILTER_THRESHOLD, DEFAULT_PROMOTION_THRESHOLD, DISK_ADOPTION_YIELD_INTERVAL, DRIFT_CHECK_GROWTH_FACTOR, ESTIMATED_MS_PER_TOMBSTONE, ESTIMATED_MS_PER_VECTOR_REBUILD, GRAPH_RETIRED, IN_MEMORY, NOTHING_STAGED, OSQ4_MIN_DIMENSION, QUERY_STAGED, RECALIBRATION_CHUNK_VECTORS, RECALIBRATION_DRIFT_RATIO, SHUFFLED_REBUILD_MEMORY_RETRY_MS, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, STORE_DOC_ID_BYTES, STORE_LIVE_COUNT, STORE_ORDINAL_LAYOUT_GENERATION, STORE_RELEASED_VECTORS_TO_DISK, STORE_REMOVED_COUNT, STORE_SLOTS, STORE_UNPLACED_COUNT, VECTOR_WORKER_COPIES_ALLOWED, WORKER_COPY_MIN_VECTORS, addToOrdinalFilter, adoptGraph, adoptLoadedCalibration, allLiveDocIds, arenaFloat32Distance, assignStorePartitions, blocksGrowthBytes, centroidOfCentroidSums, cosineSimilarityWithMagnitudes, countOfCentroidSums, createCentroidSums, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createSharedVectorStoreHandles, createVectorBlock, deserialize, detachNativeStore, dotProduct, emptyFieldBeforeRestore, ensureBlockSlots, euclideanDistance, fieldHandlesOf, filterForOptions, fixedView, graphNeedsRebuild, growBufferTo, grownByteLength, handOverBuildTurns, insertNode, liveOrdinals, liveSize, magnitude, meanOfCentroidSums, nativeScoresOf, nativeStoreScratchBytes, noBuildHistory, nodeCountOf, noteFallback, openGraphState, openSharedQuantizer, openVectorBlock, ordinalFilterForSubsets, ordinalFilterHas, ordinalFilterValues, osqBitsOf, planCheckpoint, recalibrateFromStore, recordCheckpoint, releaseLocksHeldBy, releaseReplacedVectors, requireWholeNeighbourLists, resetStoreHeader, searchOrdinals, sharedMemoryAvailable, sharedVectorStoreBytes, shuffleInPlace, signatureOf, slotByteOffset, squaredLengthsOfCentroidSums, threadsHoldCurrentLayout, toDistance, toScore, tombstoneCountOf, topLayerInParts, unplacedChunkCount, vectorsPerFile, yieldBuildTurn };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { MAX_M, EARLIER_RELEASE_MIN_M, MIN_M } from './chunk-
|
|
2
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
1
|
+
import { MAX_M, EARLIER_RELEASE_MIN_M, MIN_M } from './chunk-4MFH226Q.mjs';
|
|
2
|
+
import { NarsilError, ErrorCodes } from './chunk-35B42UFM.mjs';
|
|
3
3
|
|
|
4
4
|
// src/vector/vector-index/dead-bits.ts
|
|
5
5
|
var NO_LIVE_VECTOR_AT_POSITION = -1;
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
import { collectAndReturnPages } from './chunk-OIZS5KIX.mjs';
|
|
2
|
+
import { extractVectorFieldsFromSchema, readHeapStatistics, readResidentMemory } from './chunk-ZAIZQOTM.mjs';
|
|
3
|
+
import { NarsilError, ErrorCodes } from './chunk-35B42UFM.mjs';
|
|
4
|
+
|
|
5
|
+
// src/engine/snapshot/constants.ts
|
|
6
|
+
var RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE = 0.2;
|
|
7
|
+
var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 7.86;
|
|
8
|
+
var RESTORE_WORST_HEAP_BYTES_PER_VECTOR = 264;
|
|
9
|
+
var RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION = 0.24;
|
|
10
|
+
var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 738;
|
|
11
|
+
var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 0.2;
|
|
12
|
+
var RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE = 4.06;
|
|
13
|
+
var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 14.5;
|
|
14
|
+
var RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR = 1482;
|
|
15
|
+
var RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION = 9.29;
|
|
16
|
+
var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 2141;
|
|
17
|
+
var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 24.15;
|
|
18
|
+
var RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE = 0.16;
|
|
19
|
+
var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 6.16;
|
|
20
|
+
var RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR = 204;
|
|
21
|
+
var RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION = 0.17;
|
|
22
|
+
var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 695;
|
|
23
|
+
var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 0.04;
|
|
24
|
+
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE = 2.14;
|
|
25
|
+
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 5.05;
|
|
26
|
+
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR = 1080;
|
|
27
|
+
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION = 0.72;
|
|
28
|
+
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 1744;
|
|
29
|
+
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 20.05;
|
|
30
|
+
var RESTORE_ESTIMATE_MARGIN = 1.25;
|
|
31
|
+
var MAX_SNAPSHOT_ANALYSIS_ATTEMPTS = 3;
|
|
32
|
+
|
|
33
|
+
// src/engine/snapshot/memory-budget.ts
|
|
34
|
+
var MEGABYTE_BYTES = 1048576;
|
|
35
|
+
var ARCHIVE_RATES = {
|
|
36
|
+
worst: {
|
|
37
|
+
heap: {
|
|
38
|
+
perStoredByte: RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE,
|
|
39
|
+
perVector: RESTORE_WORST_HEAP_BYTES_PER_VECTOR,
|
|
40
|
+
perVectorDimension: RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION
|
|
41
|
+
},
|
|
42
|
+
external: {
|
|
43
|
+
perStoredByte: RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE,
|
|
44
|
+
perVector: RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR,
|
|
45
|
+
perVectorDimension: RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
|
|
46
|
+
}
|
|
47
|
+
},
|
|
48
|
+
floor: {
|
|
49
|
+
heap: {
|
|
50
|
+
perStoredByte: RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE,
|
|
51
|
+
perVector: RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR,
|
|
52
|
+
perVectorDimension: RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION
|
|
53
|
+
},
|
|
54
|
+
external: {
|
|
55
|
+
perStoredByte: RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE,
|
|
56
|
+
perVector: RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR,
|
|
57
|
+
perVectorDimension: RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
};
|
|
61
|
+
var EARLIER_SNAPSHOT_RATES = {
|
|
62
|
+
worst: {
|
|
63
|
+
heap: {
|
|
64
|
+
perStoredByte: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
65
|
+
perVector: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
66
|
+
perVectorDimension: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
67
|
+
},
|
|
68
|
+
external: {
|
|
69
|
+
perStoredByte: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
70
|
+
perVector: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
71
|
+
perVectorDimension: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
72
|
+
}
|
|
73
|
+
},
|
|
74
|
+
floor: {
|
|
75
|
+
heap: {
|
|
76
|
+
perStoredByte: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
77
|
+
perVector: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
78
|
+
perVectorDimension: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
79
|
+
},
|
|
80
|
+
external: {
|
|
81
|
+
perStoredByte: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
82
|
+
perVector: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
83
|
+
perVectorDimension: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
};
|
|
87
|
+
function bytesAt(rate, storedBytes, vectors) {
|
|
88
|
+
let bytes = storedBytes * rate.perStoredByte;
|
|
89
|
+
for (const { dimension, count } of vectors) bytes += count * (rate.perVector + dimension * rate.perVectorDimension);
|
|
90
|
+
return bytes;
|
|
91
|
+
}
|
|
92
|
+
function footprintAt(rates, replacesIndex, storedBytes, vectors) {
|
|
93
|
+
const chosen = replacesIndex ? rates.worst : rates.floor;
|
|
94
|
+
const scale = replacesIndex ? RESTORE_ESTIMATE_MARGIN : 1 / RESTORE_ESTIMATE_MARGIN;
|
|
95
|
+
return {
|
|
96
|
+
heapBytes: bytesAt(chosen.heap, storedBytes, vectors) * scale,
|
|
97
|
+
externalBytes: bytesAt(chosen.external, storedBytes, vectors) * scale
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
function deadCountOf(dead) {
|
|
101
|
+
if (dead === null) return 0;
|
|
102
|
+
let count = 0;
|
|
103
|
+
for (const byte of dead) {
|
|
104
|
+
for (let bits = byte; bits !== 0; bits &= bits - 1) count += 1;
|
|
105
|
+
}
|
|
106
|
+
return count;
|
|
107
|
+
}
|
|
108
|
+
function archiveFootprint(prepared, manifest, replacesIndex) {
|
|
109
|
+
let segmentBytes = 0;
|
|
110
|
+
for (const partition of manifest.partitions) {
|
|
111
|
+
for (const segment of partition.segments) segmentBytes += segment.bytes;
|
|
112
|
+
}
|
|
113
|
+
const dimensions = extractVectorFieldsFromSchema(prepared.config.schema);
|
|
114
|
+
const vectors = [];
|
|
115
|
+
for (const field of manifest.vectors) {
|
|
116
|
+
const dimension = dimensions.get(field.fieldPath) ?? 0;
|
|
117
|
+
for (const file of field.files) vectors.push({ dimension, count: Math.max(0, file.count - deadCountOf(file.dead)) });
|
|
118
|
+
}
|
|
119
|
+
return footprintAt(ARCHIVE_RATES, replacesIndex, segmentBytes, vectors);
|
|
120
|
+
}
|
|
121
|
+
function earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts, replacesIndex) {
|
|
122
|
+
const dimensions = extractVectorFieldsFromSchema(prepared.config.schema);
|
|
123
|
+
const vectors = [...vectorCounts].map(([fieldPath, count]) => ({ dimension: dimensions.get(fieldPath) ?? 0, count }));
|
|
124
|
+
return footprintAt(EARLIER_SNAPSHOT_RATES, replacesIndex, partitionBytes, vectors);
|
|
125
|
+
}
|
|
126
|
+
function inMegabytes(bytes) {
|
|
127
|
+
return Math.ceil(bytes / MEGABYTE_BYTES);
|
|
128
|
+
}
|
|
129
|
+
function shortfallOf(heapBytes, processBytes, countInUse) {
|
|
130
|
+
const heap = readHeapStatistics();
|
|
131
|
+
if (heap !== null) {
|
|
132
|
+
const limitBytes = heap.configuredLimitBytes ?? heap.limitBytes;
|
|
133
|
+
const availableBytes = countInUse ? limitBytes - heap.usedBytes : limitBytes;
|
|
134
|
+
if (heapBytes > availableBytes) return { budget: "heap", neededBytes: heapBytes, availableBytes };
|
|
135
|
+
}
|
|
136
|
+
const resident = readResidentMemory();
|
|
137
|
+
if (resident !== null) {
|
|
138
|
+
const availableBytes = countInUse ? resident.limitBytes - resident.residentBytes : resident.limitBytes;
|
|
139
|
+
if (processBytes > availableBytes) return { budget: "process", neededBytes: processBytes, availableBytes };
|
|
140
|
+
}
|
|
141
|
+
return null;
|
|
142
|
+
}
|
|
143
|
+
function refuse(indexName, replacesIndex, shortfall) {
|
|
144
|
+
const { budget, neededBytes, availableBytes } = shortfall;
|
|
145
|
+
const where = budget === "heap" ? "of JavaScript heap" : "of process memory";
|
|
146
|
+
const outcome = replacesIndex ? "so the index keeps its earlier contents. Drop the index first to free the memory that it holds, or give the process more memory, then restore again" : "so the engine creates no index. Give the process more memory, then restore again";
|
|
147
|
+
throw new NarsilError(
|
|
148
|
+
ErrorCodes.RESTORE_MEMORY_EXCEEDED,
|
|
149
|
+
`The restore of index "${indexName}" needs about ${inMegabytes(neededBytes)} MB ${where}, and this process has ${inMegabytes(Math.max(0, availableBytes))} MB free, ${outcome}`,
|
|
150
|
+
{ indexName, budget, neededBytes, availableBytes }
|
|
151
|
+
);
|
|
152
|
+
}
|
|
153
|
+
function requireAnalysisRebuildMemory(indexName, termBytes) {
|
|
154
|
+
if (shortfallOf(termBytes, termBytes, true) === null) return;
|
|
155
|
+
const shortfall = shortfallOf(termBytes, termBytes, collectAndReturnPages());
|
|
156
|
+
if (shortfall === null) return;
|
|
157
|
+
const { budget, neededBytes, availableBytes } = shortfall;
|
|
158
|
+
const where = budget === "heap" ? "of JavaScript heap" : "of process memory";
|
|
159
|
+
throw new NarsilError(
|
|
160
|
+
ErrorCodes.ANALYSIS_REBUILD_MEMORY_EXCEEDED,
|
|
161
|
+
`The analysis change of index "${indexName}" needs about ${inMegabytes(neededBytes)} MB ${where} for a second copy of its terms, and this process has ${inMegabytes(Math.max(0, availableBytes))} MB free, so the index keeps its current settings. Give the process more memory, then change the settings again`,
|
|
162
|
+
{ indexName, budget, neededBytes, availableBytes }
|
|
163
|
+
);
|
|
164
|
+
}
|
|
165
|
+
function requireRestoreMemory(indexName, footprint, replacesIndex) {
|
|
166
|
+
const processBytes = footprint.heapBytes + footprint.externalBytes;
|
|
167
|
+
if (shortfallOf(footprint.heapBytes, processBytes, true) === null) return;
|
|
168
|
+
const collected = collectAndReturnPages();
|
|
169
|
+
const shortfall = shortfallOf(footprint.heapBytes, processBytes, collected || replacesIndex);
|
|
170
|
+
if (shortfall !== null) refuse(indexName, replacesIndex, shortfall);
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
export { MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, archiveFootprint, earlierSnapshotFootprint, requireAnalysisRebuildMemory, requireRestoreMemory };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors, getNestedValue, requireValidFieldTypes, VECTOR_PATTERN } from './chunk-
|
|
2
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
1
|
+
import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors, getNestedValue, requireValidFieldTypes, VECTOR_PATTERN } from './chunk-ZAIZQOTM.mjs';
|
|
2
|
+
import { NarsilError, ErrorCodes } from './chunk-35B42UFM.mjs';
|
|
3
3
|
import { decode } from '@msgpack/msgpack';
|
|
4
4
|
|
|
5
5
|
function applyIndexEntry(entry, manager, vectorFieldPaths, vecIndexes) {
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes, uint64Bytes } from './chunk-
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import { compareCodePoints, MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-
|
|
5
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
1
|
+
import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes, uint64Bytes } from './chunk-6BSB4N6B.mjs';
|
|
2
|
+
import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-5LYWZJE2.mjs';
|
|
3
|
+
import { VERSION, DEFAULT_BM25_B, DEFAULT_BM25_K1, rulesOfStoredSynonyms, flattenSchema, packChecksummedEnvelope, openChecksummedEnvelope, requireValidFieldTypes, isCompositePartition, crc32, reconstructSchemaFromMetadata, extractVectorFieldsFromSchema, readHeader, HEADER_SIZE, engineVersionParts, writeHeader, partialFieldsOf, getLanguage, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, VECTOR_PATTERN, isQuantizationMode, isStorageMode, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, fnv1a, selectionKeeps, decodeMapWithoutFields, prepareDocumentVectors, messagePackUpperBoundBytes, segmentArraysOf, mergeSegmentContents, resolveIndexAnalysis } from './chunk-ZAIZQOTM.mjs';
|
|
4
|
+
import { compareCodePoints, MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-4MFH226Q.mjs';
|
|
5
|
+
import { NarsilError, ErrorCodes } from './chunk-35B42UFM.mjs';
|
|
6
6
|
import { decode, encode } from '@msgpack/msgpack';
|
|
7
7
|
|
|
8
8
|
// src/engine/durability-metadata.ts
|
|
@@ -42,6 +42,12 @@ function metadataFromConfig(indexName, config, partitionCount, documentCount) {
|
|
|
42
42
|
if (config.stopWordList !== void 0) {
|
|
43
43
|
metadata.stopWordList = config.stopWordList;
|
|
44
44
|
}
|
|
45
|
+
if (config.analysisGeneration !== void 0) {
|
|
46
|
+
metadata.analysisGeneration = config.analysisGeneration;
|
|
47
|
+
}
|
|
48
|
+
if (config.pendingAnalysis !== void 0) {
|
|
49
|
+
metadata.pendingAnalysis = config.pendingAnalysis;
|
|
50
|
+
}
|
|
45
51
|
if (config.partitionLimits !== void 0) {
|
|
46
52
|
metadata.partitionLimits = config.partitionLimits;
|
|
47
53
|
}
|
|
@@ -96,6 +102,8 @@ function metadataConfigOf(entry) {
|
|
|
96
102
|
...typeof entry.config.tokenizer === "string" ? { tokenizer: entry.config.tokenizer } : {},
|
|
97
103
|
...typeof entry.config.stopWords === "string" ? { stopWords: entry.config.stopWords } : {},
|
|
98
104
|
...entry.config.stopWords instanceof Set ? { stopWordList: [...entry.config.stopWords].sort(compareCodePoints) } : {},
|
|
105
|
+
...entry.analysisGeneration > 0 ? { analysisGeneration: entry.analysisGeneration } : {},
|
|
106
|
+
...entry.pendingAnalysis !== null ? { pendingAnalysis: entry.pendingAnalysis } : {},
|
|
99
107
|
...entry.config.partitions !== void 0 ? { partitionLimits: entry.config.partitions } : {},
|
|
100
108
|
...entry.config.defaultScoring !== void 0 ? { defaultScoring: entry.config.defaultScoring } : {},
|
|
101
109
|
...entry.config.trackPositions !== void 0 ? { trackPositions: entry.config.trackPositions } : {},
|
|
@@ -106,6 +114,38 @@ function metadataConfigOf(entry) {
|
|
|
106
114
|
...entry.synonyms !== null ? { synonyms: rulesOfStoredSynonyms(entry.synonyms) } : {}
|
|
107
115
|
};
|
|
108
116
|
}
|
|
117
|
+
function durableMetadataConfigOf(entry) {
|
|
118
|
+
const terms = metadataConfigOf(entry);
|
|
119
|
+
const durable = entry.durableAnalysis;
|
|
120
|
+
if (durable === null) return terms;
|
|
121
|
+
const {
|
|
122
|
+
language,
|
|
123
|
+
tokenizer,
|
|
124
|
+
stopWords,
|
|
125
|
+
stopWordList,
|
|
126
|
+
analysisRevision: _analysisRevision,
|
|
127
|
+
arrayPositionGap: _arrayPositionGap,
|
|
128
|
+
analysisGeneration: _analysisGeneration,
|
|
129
|
+
...rest
|
|
130
|
+
} = terms;
|
|
131
|
+
return {
|
|
132
|
+
...rest,
|
|
133
|
+
language: durable.language,
|
|
134
|
+
...durable.tokenizer !== void 0 ? { tokenizer: durable.tokenizer } : {},
|
|
135
|
+
...durable.stopWords !== void 0 ? { stopWords: durable.stopWords } : {},
|
|
136
|
+
...durable.stopWordList !== void 0 ? { stopWordList: durable.stopWordList } : {},
|
|
137
|
+
...durable.analysisRevision !== null ? { analysisRevision: durable.analysisRevision } : {},
|
|
138
|
+
...durable.arrayPositionGap !== null ? { arrayPositionGap: durable.arrayPositionGap } : {},
|
|
139
|
+
...durable.generation > 0 ? { analysisGeneration: durable.generation } : {},
|
|
140
|
+
pendingAnalysis: {
|
|
141
|
+
generation: entry.analysisGeneration,
|
|
142
|
+
language,
|
|
143
|
+
...tokenizer !== void 0 ? { tokenizer } : {},
|
|
144
|
+
...stopWords !== void 0 ? { stopWords } : {},
|
|
145
|
+
...stopWordList !== void 0 ? { stopWordList } : {}
|
|
146
|
+
}
|
|
147
|
+
};
|
|
148
|
+
}
|
|
109
149
|
function indexMetadataOf(indexName, entry, partitionCount) {
|
|
110
150
|
return metadataFromConfig(indexName, metadataConfigOf(entry), partitionCount);
|
|
111
151
|
}
|
|
@@ -337,6 +377,19 @@ function metadataToWire(meta) {
|
|
|
337
377
|
if (meta.stopWordList !== void 0) {
|
|
338
378
|
wire.stop_word_list = meta.stopWordList;
|
|
339
379
|
}
|
|
380
|
+
if (meta.analysisGeneration !== void 0) {
|
|
381
|
+
wire.analysis_generation = meta.analysisGeneration;
|
|
382
|
+
}
|
|
383
|
+
if (meta.pendingAnalysis !== void 0) {
|
|
384
|
+
const pending = meta.pendingAnalysis;
|
|
385
|
+
wire.pending_analysis = {
|
|
386
|
+
generation: pending.generation,
|
|
387
|
+
language: pending.language,
|
|
388
|
+
...pending.tokenizer !== void 0 ? { tokenizer: pending.tokenizer } : {},
|
|
389
|
+
...pending.stopWords !== void 0 ? { stop_words: pending.stopWords } : {},
|
|
390
|
+
...pending.stopWordList !== void 0 ? { stop_word_list: pending.stopWordList } : {}
|
|
391
|
+
};
|
|
392
|
+
}
|
|
340
393
|
if (meta.partitionLimits !== void 0) {
|
|
341
394
|
wire.partition_limits = {
|
|
342
395
|
...meta.partitionLimits.maxDocsPerPartition !== void 0 ? { max_docs_per_partition: meta.partitionLimits.maxDocsPerPartition } : {},
|
|
@@ -432,6 +485,19 @@ function wireToMetadata(raw) {
|
|
|
432
485
|
if (Array.isArray(raw.stop_word_list) && raw.stop_word_list.every((word) => typeof word === "string")) {
|
|
433
486
|
meta.stopWordList = raw.stop_word_list;
|
|
434
487
|
}
|
|
488
|
+
if (isPositiveInteger(raw.analysis_generation)) {
|
|
489
|
+
meta.analysisGeneration = raw.analysis_generation;
|
|
490
|
+
}
|
|
491
|
+
const pending = raw.pending_analysis;
|
|
492
|
+
if (pending && isPositiveInteger(pending.generation) && typeof pending.language === "string") {
|
|
493
|
+
meta.pendingAnalysis = {
|
|
494
|
+
generation: pending.generation,
|
|
495
|
+
language: pending.language,
|
|
496
|
+
...typeof pending.tokenizer === "string" ? { tokenizer: pending.tokenizer } : {},
|
|
497
|
+
...typeof pending.stop_words === "string" ? { stopWords: pending.stop_words } : {},
|
|
498
|
+
...isStringList(pending.stop_word_list) ? { stopWordList: pending.stop_word_list } : {}
|
|
499
|
+
};
|
|
500
|
+
}
|
|
435
501
|
if (raw.partition_limits && typeof raw.partition_limits === "object") {
|
|
436
502
|
const limits = {
|
|
437
503
|
...isPositiveInteger(raw.partition_limits.max_docs_per_partition) ? { maxDocsPerPartition: raw.partition_limits.max_docs_per_partition } : {},
|
|
@@ -524,6 +590,9 @@ function wireToMetadata(raw) {
|
|
|
524
590
|
function isNonNegativeInteger(value) {
|
|
525
591
|
return typeof value === "number" && Number.isInteger(value) && value >= 0;
|
|
526
592
|
}
|
|
593
|
+
function isStringList(value) {
|
|
594
|
+
return Array.isArray(value) && value.every((word) => typeof word === "string");
|
|
595
|
+
}
|
|
527
596
|
function isPositiveInteger(value) {
|
|
528
597
|
return typeof value === "number" && Number.isInteger(value) && value >= 1;
|
|
529
598
|
}
|
|
@@ -745,6 +814,7 @@ function encodeSegmentManifest(manifest) {
|
|
|
745
814
|
version: SEGMENT_MANIFEST_VERSION,
|
|
746
815
|
schema: manifest.schema,
|
|
747
816
|
language: manifest.language,
|
|
817
|
+
...manifest.analysisGeneration ? { analysisGeneration: manifest.analysisGeneration } : {},
|
|
748
818
|
checkpoint: manifest.checkpoint.map((c) => ({
|
|
749
819
|
partitionId: c.partitionId,
|
|
750
820
|
lastSeqNo: c.lastSeqNo,
|
|
@@ -920,12 +990,16 @@ function decodeManifestPayload(raw, indexName) {
|
|
|
920
990
|
if (!isRecord(schema)) invalidManifest("is missing its schema");
|
|
921
991
|
requireValidFieldTypes(schema);
|
|
922
992
|
if (typeof raw.language !== "string") invalidManifest("is missing its language");
|
|
993
|
+
if (raw.analysisGeneration !== void 0 && !isSequenceNumber(raw.analysisGeneration)) {
|
|
994
|
+
invalidManifest("has an invalid analysis generation");
|
|
995
|
+
}
|
|
923
996
|
return {
|
|
924
997
|
kind: "current",
|
|
925
998
|
manifest: {
|
|
926
999
|
version: SEGMENT_MANIFEST_VERSION,
|
|
927
1000
|
schema,
|
|
928
1001
|
language: raw.language,
|
|
1002
|
+
...raw.analysisGeneration ? { analysisGeneration: raw.analysisGeneration } : {},
|
|
929
1003
|
checkpoint: normalizeCheckpoint(raw.checkpoint),
|
|
930
1004
|
partitions: normalizePartitions(raw.partitions, indexName),
|
|
931
1005
|
vectors: normalizeVectorFields(raw.vectors, indexName)
|
|
@@ -2644,6 +2718,7 @@ async function commitCheckpointManifest(store, metadata, written, vectors) {
|
|
|
2644
2718
|
const manifest = {
|
|
2645
2719
|
schema: metadata.schema,
|
|
2646
2720
|
language: metadata.language,
|
|
2721
|
+
...metadata.analysisGeneration ? { analysisGeneration: metadata.analysisGeneration } : {},
|
|
2647
2722
|
checkpoint: written.checkpoint,
|
|
2648
2723
|
partitions: written.partitions,
|
|
2649
2724
|
vectors
|
|
@@ -2866,4 +2941,4 @@ function deserializePayloadV2(data) {
|
|
|
2866
2941
|
};
|
|
2867
2942
|
}
|
|
2868
2943
|
|
|
2869
|
-
export { CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_IDLE_MS, CHECKPOINT_WORKER_TIMEOUT_MS, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_SEGMENT_MAX_BYTES, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_SETTLE_MS, LOG_RECORDS_PER_CHECKPOINT, MAX_SEGMENTS_PER_PARTITION, SEGMENT_HEADER_SIZE, SINGLE_NODE_PRIMARY_TERM, VECTOR_FILE_ENVELOPE_VERSION, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, adapterCheckpointStore, buildEntry, captureWholePartition, checkpointLastSeqNo, checkpointPrimaryTerm, commitCheckpointManifest, createSeqOwner, decodeManifestPayload, decodeSegmentManifest, deserializePayloadV1, deserializePayloadV2, directoryCheckpointStore, documentsOfPayloadV2, durableRecordMissing, encodeMarkerSlot, encodeSegmentManifest, frameRecord, growPartitionsTo, indexMetadataOf, invalidManifest, isId, isRecord, isSequenceNumber, isUint32, listedKeysOf, loadSegmentedCheckpoint, manifestFileRefs, manifestKey, manifestReferencedKeys, markerSlotOffset, metadataConfigOf, metadataFromConfig, nextMarkerSlot, openManifestPayload, partitionContentsOf, readCommitMarker, readCurrentManifest, readGraphParts, readListedFile, readMetadataEnvelope, readSegmentFile, readSegmentManifest, readVectorFile, readWalSegments, reclaimUnlistedKeys, removeCheckpointGarbage, snapshotBundleKey, unpackEnvelopeBytes, unpackIndexSnapshotEnvelope, writeCheckpointSegments, writeLiveVectors, writeMetadataEnvelope, writeSegmentHeader };
|
|
2944
|
+
export { CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_IDLE_MS, CHECKPOINT_WORKER_TIMEOUT_MS, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_SEGMENT_MAX_BYTES, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_SETTLE_MS, LOG_RECORDS_PER_CHECKPOINT, MAX_SEGMENTS_PER_PARTITION, SEGMENT_HEADER_SIZE, SINGLE_NODE_PRIMARY_TERM, VECTOR_FILE_ENVELOPE_VERSION, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, adapterCheckpointStore, buildEntry, captureWholePartition, checkpointLastSeqNo, checkpointPrimaryTerm, commitCheckpointManifest, createSeqOwner, decodeManifestPayload, decodeSegmentManifest, deserializePayloadV1, deserializePayloadV2, directoryCheckpointStore, documentsOfPayloadV2, durableMetadataConfigOf, durableRecordMissing, encodeMarkerSlot, encodeSegmentManifest, frameRecord, growPartitionsTo, indexMetadataOf, invalidManifest, isId, isRecord, isSequenceNumber, isUint32, listedKeysOf, loadSegmentedCheckpoint, manifestFileRefs, manifestKey, manifestReferencedKeys, markerSlotOffset, metadataConfigOf, metadataFromConfig, nextMarkerSlot, openManifestPayload, partitionContentsOf, readCommitMarker, readCurrentManifest, readGraphParts, readListedFile, readMetadataEnvelope, readSegmentFile, readSegmentManifest, readVectorFile, readWalSegments, reclaimUnlistedKeys, removeCheckpointGarbage, snapshotBundleKey, unpackEnvelopeBytes, unpackIndexSnapshotEnvelope, writeCheckpointSegments, writeLiveVectors, writeMetadataEnvelope, writeSegmentHeader };
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
// src/engine/analysis-change-entry.ts
|
|
2
|
+
function holdsChangeWork(entry) {
|
|
3
|
+
return entry !== void 0 && (entry.pendingAnalysis !== null || entry.durableAnalysis !== null);
|
|
4
|
+
}
|
|
5
|
+
async function abandonChangeOf(core, indexName) {
|
|
6
|
+
if (!holdsChangeWork(core.indexRegistry.get(indexName))) return;
|
|
7
|
+
const { abandonAnalysisChange } = await import('./analysis-change-57MIRJWR.mjs');
|
|
8
|
+
await abandonAnalysisChange(core, indexName);
|
|
9
|
+
}
|
|
10
|
+
async function resumeChangeOf(core, indexName) {
|
|
11
|
+
if ((core.indexRegistry.get(indexName)?.pendingAnalysis ?? null) === null) return;
|
|
12
|
+
if (core.mutationCtx.capacityScope === "partition") return;
|
|
13
|
+
if (core.executor.getManager(indexName) === void 0) return;
|
|
14
|
+
const { resumeAnalysisChange } = await import('./analysis-change-57MIRJWR.mjs');
|
|
15
|
+
void resumeAnalysisChange(core, indexName).catch(() => void 0);
|
|
16
|
+
}
|
|
17
|
+
async function resumeEveryChange(core) {
|
|
18
|
+
for (const indexName of [...core.indexRegistry.keys()]) await resumeChangeOf(core, indexName);
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export { abandonChangeOf, resumeChangeOf, resumeEveryChange };
|
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
import { DEFAULT_PAGE_SIZE } from './chunk-
|
|
1
|
+
import { DEFAULT_PAGE_SIZE } from './chunk-35B42UFM.mjs';
|
|
2
2
|
|
|
3
3
|
// src/engine/constants.ts
|
|
4
4
|
var MIN_DOCUMENTS_FOR_SEGMENTS = 64;
|
|
5
5
|
var BATCH_CHUNK_SIZE = 1e3;
|
|
6
|
+
var ANALYSIS_HOLD_APPLY_INTERVAL_MS = 100;
|
|
7
|
+
var ANALYSIS_CHANGE_MEMBERS = ["language", "tokenizer", "stopWords"];
|
|
6
8
|
var DEFAULT_LIMIT = DEFAULT_PAGE_SIZE;
|
|
7
9
|
var DEFAULT_OFFSET = 0;
|
|
8
10
|
var INDEX_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/;
|
|
@@ -27,4 +29,4 @@ var SUGGEST_OVERSAMPLE_FACTOR = 1.5;
|
|
|
27
29
|
var SUGGEST_OVERSAMPLE_PADDING = 10;
|
|
28
30
|
var MAX_SUGGEST_SCATTER_LIMIT = Math.ceil(MAX_SUGGEST_LIMIT * SUGGEST_OVERSAMPLE_FACTOR) + SUGGEST_OVERSAMPLE_PADDING;
|
|
29
31
|
|
|
30
|
-
export { ABSOLUTE_WINDOWS_PATH_PATTERN, BATCH_CHUNK_SIZE, DEFAULT_LIMIT, DEFAULT_OFFSET, DEFAULT_SUGGEST_LIMIT, EXPORT_CHUNK_BYTES, FOREIGN_STATS_TTL_MS, HEAP_PRESSURE_REARM_FRACTION, HEAP_PRESSURE_WARNING_FRACTION, IMPORT_BATCH_DOCUMENTS, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, MAX_EXPORT_LINE_BYTES, MAX_IMPORT_REFUSALS_REPORTED, MAX_INDEX_CONFIGURATION_BYTES, MAX_INDEX_NAME_LENGTH, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, MAX_SWEEP_INTERVAL_MS, MIN_DOCUMENTS_FOR_SEGMENTS, MIN_SWEEP_INTERVAL_MS, MODULE_URL_SCHEME_PATTERN, STATISTICS_BROADCAST_INTERVAL_MS };
|
|
32
|
+
export { ABSOLUTE_WINDOWS_PATH_PATTERN, ANALYSIS_CHANGE_MEMBERS, ANALYSIS_HOLD_APPLY_INTERVAL_MS, BATCH_CHUNK_SIZE, DEFAULT_LIMIT, DEFAULT_OFFSET, DEFAULT_SUGGEST_LIMIT, EXPORT_CHUNK_BYTES, FOREIGN_STATS_TTL_MS, HEAP_PRESSURE_REARM_FRACTION, HEAP_PRESSURE_WARNING_FRACTION, IMPORT_BATCH_DOCUMENTS, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, MAX_EXPORT_LINE_BYTES, MAX_IMPORT_REFUSALS_REPORTED, MAX_INDEX_CONFIGURATION_BYTES, MAX_INDEX_NAME_LENGTH, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, MAX_SWEEP_INTERVAL_MS, MIN_DOCUMENTS_FOR_SEGMENTS, MIN_SWEEP_INTERVAL_MS, MODULE_URL_SCHEME_PATTERN, STATISTICS_BROADCAST_INTERVAL_MS };
|