@delali/narsil 0.3.1-next.112 → 0.3.1-next.115
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/{chunk-UR5MNN53.mjs → chunk-2TMWIATH.mjs} +108 -67
- package/dist/browser/{chunk-OVY4RPDS.mjs → chunk-73U5DWOZ.mjs} +39 -4
- package/dist/browser/{chunk-O5IKTA27.mjs → chunk-FODCC3MK.mjs} +1 -1
- package/dist/browser/{chunk-YRYBTRWB.mjs → chunk-I45FAGOF.mjs} +2 -2
- package/dist/browser/{durability-wiring-CPEMDDK7.mjs → durability-wiring-CSMTKEYX.mjs} +3 -3
- package/dist/browser/{module-DYKR4INA.mjs → module-G2PPZV3B.mjs} +127 -33
- package/dist/browser/{restore-queue-2H4QDQV4.mjs → restore-queue-N7N42PBS.mjs} +1 -1
- package/dist/browser/{runtime-FL5K4JOH.mjs → runtime-BX6XRFYR.mjs} +2 -2
- package/dist/browser/{snapshot-ISZMBDCN.mjs → snapshot-EBH6RXQO.mjs} +6 -5
- package/dist/browser/{write-IABOLE4N.mjs → write-5ZKFINX3.mjs} +2 -2
- package/dist/{capabilities-ByaX8Arf.d.ts → capabilities-DOgB3pPf.d.ts} +2 -2
- package/dist/{chunk-ZXZ26WSO.mjs → chunk-22YPZVMG.mjs} +5 -1
- package/dist/{chunk-R2MRGYU5.mjs → chunk-4ZKBKPBV.mjs} +22 -2
- package/dist/{chunk-GLZI636G.mjs → chunk-5VVSHCXT.mjs} +27 -12
- package/dist/{chunk-3COL24MT.mjs → chunk-7EWQZQDT.mjs} +1 -1
- package/dist/{chunk-QI6JW4PM.mjs → chunk-BCNUJVTT.mjs} +1 -1
- package/dist/{chunk-3RRATITJ.mjs → chunk-BCSWODXP.mjs} +75 -55
- package/dist/{thread-WGBCVJII.mjs → chunk-DEXID4ZT.mjs} +301 -200
- package/dist/chunk-EBIHRL7B.mjs +21 -0
- package/dist/{chunk-YIDSHBQS.mjs → chunk-HC5NURS6.mjs} +33 -4
- package/dist/{chunk-V7G627TA.mjs → chunk-HFJXEFFV.mjs} +13 -62
- package/dist/{chunk-63HFWQEA.mjs → chunk-IE43SRL3.mjs} +276 -102
- package/dist/chunk-IWVEG5CX.mjs +67 -0
- package/dist/{chunk-2IFU7SXG.mjs → chunk-KGVC6KJ3.mjs} +4 -4
- package/dist/{chunk-7CI34JAP.mjs → chunk-KVLX65KD.mjs} +4 -4
- package/dist/{chunk-5WEBVYOA.mjs → chunk-N53HN5VA.mjs} +16 -9
- package/dist/chunk-N6J6EJW3.mjs +198 -0
- package/dist/{chunk-PT7VRWS7.mjs → chunk-NX7KU77V.mjs} +3 -2
- package/dist/{chunk-QIFOXLWF.mjs → chunk-OGFSUXYY.mjs} +2 -2
- package/dist/chunk-PELHVTO7.mjs +24 -0
- package/dist/{chunk-V7ISFBFC.mjs → chunk-Q62NDKEC.mjs} +5 -4
- package/dist/{chunk-6OXNHLNZ.mjs → chunk-RIIUZJQ3.mjs} +1 -1
- package/dist/{chunk-BVOUSX3U.mjs → chunk-UDZYIWDP.mjs} +3 -3
- package/dist/{chunk-HNAP2KTG.mjs → chunk-VFAXPDBH.mjs} +1 -0
- package/dist/{chunk-IBTKJIV7.mjs → chunk-VOKEYJHU.mjs} +1 -1
- package/dist/{chunk-C7SLA6CM.mjs → chunk-W55X2BYN.mjs} +5 -1
- package/dist/{chunk-IM76DLA7.mjs → chunk-YCFQ5754.mjs} +2 -2
- package/dist/{chunk-VLRT47GU.mjs → chunk-YCSRB3IN.mjs} +3 -3
- package/dist/{chunk-CQUMNIIG.mjs → chunk-Z4DZ4KK2.mjs} +1 -0
- package/dist/{chunk-SOADQINJ.mjs → chunk-ZATRDAYU.mjs} +31 -3
- package/dist/{chunk-2RAQKITH.mjs → chunk-ZR63OGT2.mjs} +1 -1
- package/dist/{client-BdnzDFtO.d.ts → client-BGzLNt78.d.ts} +2 -2
- package/dist/client.d.ts +6 -6
- package/dist/{config-ujZESkJz.d.ts → config-CS5ADSrx.d.ts} +18 -2
- package/dist/{config-CSgvApt0.d.ts → config-CSYMTTm1.d.ts} +1 -1
- package/dist/{direct-executor-ZSTKRA6Z.mjs → direct-executor-OUY4RPTU.mjs} +7 -6
- package/dist/distribution.d.ts +5 -5
- package/dist/distribution.mjs +79 -42
- package/dist/{durability-wiring-EQTFL3B4.mjs → durability-wiring-5V6DVHFK.mjs} +17 -16
- package/dist/{engine-C_HSy3iz.d.ts → engine-CCBBGmMR.d.ts} +1 -1
- package/dist/{events-CJsXF1F9.d.ts → events-D8GPB1-H.d.ts} +5 -0
- package/dist/index.browser.mjs +1 -1
- package/dist/index.d.ts +5 -5
- package/dist/index.mjs +10 -9
- package/dist/long-query-thread-HMQM2FWL.mjs +118 -0
- package/dist/{module-WP4VVGV2.mjs → module-3IMTXNPW.mjs} +92 -29
- package/dist/pattern.mjs +1 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +14 -13
- package/dist/react.d.ts +6 -6
- package/dist/{rerank-3POMPQNX.mjs → rerank-NYX5S7OG.mjs} +3 -3
- package/dist/{restore-queue-WZTMTJP2.mjs → restore-queue-J6PCG4ON.mjs} +1 -1
- package/dist/{runtime-C2435MPK.mjs → runtime-XNYW6AWX.mjs} +2 -2
- package/dist/server.d.ts +8 -8
- package/dist/server.mjs +110 -258
- package/dist/{snapshot-EU6NNB2P.mjs → snapshot-57WF53MK.mjs} +18 -17
- package/dist/{tasks-CI3Fvkxw.d.ts → tasks-5_LCFEzF.d.ts} +1 -1
- package/dist/thread-FDEHU5G3.mjs +31 -0
- package/dist/vector/search-worker.mjs +2 -2
- package/dist/workers/entry.mjs +19 -7
- package/dist/{write-CKRWS3P5.mjs → write-YEI7A5OR.mjs} +11 -10
- package/package.json +7 -7
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { nodeThreadId, spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
|
|
2
|
-
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, decodeSegmentManifest, readListedFile, adapterCheckpointStore, SINGLE_NODE_PRIMARY_TERM, readMetadataEnvelope, writeMetadataEnvelope, metadataFromConfig, DIRECTORY_LOCK_ATTEMPTS, createSeqOwner, readSegmentManifest, snapshotBundleKey, reclaimUnlistedKeys, manifestReferencedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, manifestFileRefs, DIRECTORY_LOCK_SETTLE_MS, readSegmentFile, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, frameRecord, isId, isUint32, isSequenceNumber, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, commitCheckpointManifest, removeCheckpointGarbage, readWalSegments, durableRecordMissing, MAX_SEGMENTS_PER_PARTITION, growPartitionsTo, writeSegmentHeader, VECTOR_FILE_ENVELOPE_VERSION, SEGMENT_HEADER_SIZE, documentsOfPayloadV2, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, metadataConfigOf, buildEntry, readCurrentManifest, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-
|
|
3
|
-
import { applyDeleteEntry, applyIndexEntry } from './chunk-
|
|
4
|
-
import { encodeDocument, fnv1a, requireValidFieldTypes, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-
|
|
2
|
+
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, decodeSegmentManifest, readListedFile, adapterCheckpointStore, SINGLE_NODE_PRIMARY_TERM, readMetadataEnvelope, writeMetadataEnvelope, metadataFromConfig, DIRECTORY_LOCK_ATTEMPTS, createSeqOwner, readSegmentManifest, snapshotBundleKey, reclaimUnlistedKeys, manifestReferencedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, manifestFileRefs, DIRECTORY_LOCK_SETTLE_MS, readSegmentFile, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, frameRecord, isId, isUint32, isSequenceNumber, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, commitCheckpointManifest, removeCheckpointGarbage, readWalSegments, durableRecordMissing, MAX_SEGMENTS_PER_PARTITION, growPartitionsTo, writeSegmentHeader, VECTOR_FILE_ENVELOPE_VERSION, SEGMENT_HEADER_SIZE, documentsOfPayloadV2, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, metadataConfigOf, buildEntry, readCurrentManifest, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-I45FAGOF.mjs';
|
|
3
|
+
import { applyDeleteEntry, applyIndexEntry } from './chunk-FODCC3MK.mjs';
|
|
4
|
+
import { encodeDocument, fnv1a, requireValidFieldTypes, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-2TMWIATH.mjs';
|
|
5
5
|
import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-P7C2S5JX.mjs';
|
|
6
6
|
import { compareCodePoints, MIN_M, MAX_FILE_BYTES, detectRuntime, resolveWorkerEntry } from './chunk-2CBQ2PCC.mjs';
|
|
7
7
|
import './chunk-JPTQZWJQ.mjs';
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
|
|
2
|
-
import { VECTOR_WORKER_COPIES_ALLOWED, openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, osqBitsOf, noBuildHistory, createOsqQuantizer, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, ordinalFilterValues, handOverBuildTurns, threadsHoldCurrentLayout, filterForOptions, openGraphState, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, fixedView, OSQ4_MIN_DIMENSION, sharedVectorStoreBytes, STORE_SLOTS, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, yieldBuildTurn, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, WORKER_COPY_MIN_VECTORS, allLiveDocIds, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, slotByteOffset, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createSharedVectorStoreHandles, BLOCK_MAX_BYTES,
|
|
3
|
-
export { releaseLocksHeldBy } from './chunk-
|
|
2
|
+
import { VECTOR_WORKER_COPIES_ALLOWED, openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, osqBitsOf, noBuildHistory, createOsqQuantizer, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, ordinalFilterValues, handOverBuildTurns, threadsHoldCurrentLayout, filterForOptions, openGraphState, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, fixedView, OSQ4_MIN_DIMENSION, sharedVectorStoreBytes, STORE_SLOTS, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, yieldBuildTurn, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, WORKER_COPY_MIN_VECTORS, allLiveDocIds, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, slotByteOffset, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createSharedVectorStoreHandles, BLOCK_MAX_BYTES, STORE_DOC_ID_BYTES, unplacedChunkCount, blocksGrowthBytes, growBufferTo, SHUFFLED_REBUILD_MEMORY_RETRY_MS, recalibrateFromStore, signatureOf, nativeStoreScratchBytes, noteFallback, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, QUERY_STAGED, grownByteLength, createVectorBlock, ensureBlockSlots, createHNSWIndex, adoptGraph, topLayerInParts, BUILD_RETRY_MAX_DELAY_MS, BUILD_RETRY_FIRST_DELAY_MS, fieldHandlesOf, sharedMemoryAvailable, requireWholeNeighbourLists, shuffleInPlace, createCentroidSums, liveOrdinals, RECALIBRATION_CHUNK_VECTORS, BUILD_CHUNK_SIZE, centroidOfCentroidSums, meanOfCentroidSums, countOfCentroidSums, squaredLengthsOfCentroidSums, releaseLocksHeldBy, DRIFT_CHECK_GROWTH_FACTOR, DISK_ADOPTION_YIELD_INTERVAL, adoptLoadedCalibration, RECALIBRATION_DRIFT_RATIO } from './chunk-73U5DWOZ.mjs';
|
|
3
|
+
export { releaseLocksHeldBy } from './chunk-73U5DWOZ.mjs';
|
|
4
4
|
import { isDead, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors } from './chunk-P7C2S5JX.mjs';
|
|
5
5
|
import { MAX_RENUMBERED_SEARCH_ATTEMPTS, DISK_STORAGE_BLOCK_BYTES, compareCodePoints, UNPLACED_FLAG_CHUNK_ORDINALS, MAX_VECTOR_ORDINALS, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, resolveWorkerCount, resolveWorkerEntry, detectRuntime, noteThreadSlotLive, noteThreadSlotGone, WORKER_YOUNG_GENERATION_MB, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_INSERT_TIMEOUT_MS } from './chunk-2CBQ2PCC.mjs';
|
|
6
6
|
import { NarsilError, ErrorCodes, isNarsilError } from './chunk-LJS6PREA.mjs';
|
|
@@ -173,12 +173,13 @@ function readStoreFlags(store) {
|
|
|
173
173
|
}
|
|
174
174
|
|
|
175
175
|
// src/vector/shared-field/searcher.ts
|
|
176
|
-
function distancesInTypeScript(store, query, metric, ordinals) {
|
|
176
|
+
function distancesInTypeScript(store, query, metric, ordinals, clock) {
|
|
177
177
|
noteFallback("score");
|
|
178
178
|
const arenaQuery = store.prepareQueryArena(query);
|
|
179
179
|
const queryMagnitude = arenaQuery ? arenaQuery.magnitude : magnitude(query);
|
|
180
180
|
const distances = new Float64Array(ordinals.length).fill(Number.POSITIVE_INFINITY);
|
|
181
181
|
for (let i = 0; i < ordinals.length; i++) {
|
|
182
|
+
if (clock !== void 0) clock.tick(query.length);
|
|
182
183
|
if (arenaQuery) {
|
|
183
184
|
distances[i] = store.distanceFromArena(arenaQuery, ordinals[i], metric);
|
|
184
185
|
continue;
|
|
@@ -188,10 +189,12 @@ function distancesInTypeScript(store, query, metric, ordinals) {
|
|
|
188
189
|
}
|
|
189
190
|
return distances;
|
|
190
191
|
}
|
|
191
|
-
function scoreCandidates(store, candidates, query, k, metric, minSimilarity) {
|
|
192
|
+
function scoreCandidates(store, candidates, query, k, metric, minSimilarity, clock) {
|
|
192
193
|
const { ordinals, docIds } = candidates;
|
|
193
194
|
if (ordinals.length === 0) return { results: [], matched: 0 };
|
|
194
|
-
|
|
195
|
+
clock?.check();
|
|
196
|
+
const distances = nativeScoresOf(store.handles, query, metric, ordinals) ?? distancesInTypeScript(store, query, metric, ordinals, clock);
|
|
197
|
+
clock?.check();
|
|
195
198
|
const heap = createBoundedMaxHeap(highScoreFirst, k);
|
|
196
199
|
let matched = 0;
|
|
197
200
|
for (let i = 0; i < ordinals.length; i++) {
|
|
@@ -262,6 +265,7 @@ function createSharedVectorSearcher(options) {
|
|
|
262
265
|
}
|
|
263
266
|
function search2(query, k, searchOptions, filter) {
|
|
264
267
|
const { metric, minSimilarity, efSearch, oversample, rescoreWindow } = searchOptions;
|
|
268
|
+
const clock = searchOptions.clock;
|
|
265
269
|
const empty = { results: [], matched: 0, matchedExact: true, resolvedEveryHit: true };
|
|
266
270
|
const liveDocuments = flags.liveDocuments();
|
|
267
271
|
if (liveDocuments <= 0) return empty;
|
|
@@ -277,23 +281,27 @@ function createSharedVectorSearcher(options) {
|
|
|
277
281
|
const graph = view.graph;
|
|
278
282
|
if (graph === null) {
|
|
279
283
|
const ordinals = filter === void 0 ? flags.unplacedOrdinals() : ordinalFilterValues(filter);
|
|
280
|
-
return scanned(
|
|
284
|
+
return scanned(
|
|
285
|
+
scoreCandidates(view.store, candidatesOf(ordinals, filter), query, k, metric, minSimilarity, clock)
|
|
286
|
+
);
|
|
281
287
|
}
|
|
282
288
|
if (filter !== void 0) {
|
|
283
289
|
const graphLiveSize = view.liveSize;
|
|
284
290
|
const selectivity = graphLiveSize > 0 ? filter.count / graphLiveSize : 1;
|
|
285
291
|
if (selectivity < view.handles.filterThreshold) {
|
|
286
292
|
const candidates = candidatesOf(ordinalFilterValues(filter), filter);
|
|
287
|
-
return scanned(scoreCandidates(view.store, candidates, query, k, metric, minSimilarity));
|
|
293
|
+
return scanned(scoreCandidates(view.store, candidates, query, k, metric, minSimilarity, clock));
|
|
288
294
|
}
|
|
289
295
|
}
|
|
296
|
+
const unplaced = flags.unplacedOrdinals();
|
|
290
297
|
let resolvedEveryHit = true;
|
|
291
298
|
const graphResults = [];
|
|
292
299
|
for (const hit of view.searchOrdinals(query, k, metric, minSimilarity, {
|
|
293
300
|
filter,
|
|
294
301
|
efSearch,
|
|
295
302
|
oversample,
|
|
296
|
-
rescoreWindow
|
|
303
|
+
rescoreWindow,
|
|
304
|
+
clock
|
|
297
305
|
})) {
|
|
298
306
|
const docId = view.docIdOf(hit.ord);
|
|
299
307
|
if (docId === void 0) {
|
|
@@ -302,9 +310,8 @@ function createSharedVectorSearcher(options) {
|
|
|
302
310
|
}
|
|
303
311
|
graphResults.push({ docId, score: hit.score });
|
|
304
312
|
}
|
|
305
|
-
const unplaced = flags.unplacedOrdinals();
|
|
306
313
|
if (unplaced.length === 0) return fetched(graphResults, resolvedEveryHit);
|
|
307
|
-
const scan = scoreCandidates(view.store, candidatesOf(unplaced, filter), query, k, metric, minSimilarity);
|
|
314
|
+
const scan = scoreCandidates(view.store, candidatesOf(unplaced, filter), query, k, metric, minSimilarity, clock);
|
|
308
315
|
return fetched(mergeResults(graphResults, scan.results, k), resolvedEveryHit);
|
|
309
316
|
}
|
|
310
317
|
return {
|
|
@@ -887,7 +894,8 @@ function openHandles(dimension, codeBits, blockBytes) {
|
|
|
887
894
|
present: new Uint8Array(handles.present),
|
|
888
895
|
partitions: new Int32Array(handles.partitions),
|
|
889
896
|
diskFile: new Int32Array(handles.diskFile),
|
|
890
|
-
diskOffset: new Uint32Array(handles.diskOffset)
|
|
897
|
+
diskOffset: new Uint32Array(handles.diskOffset),
|
|
898
|
+
longestDocIdBytes: 0
|
|
891
899
|
};
|
|
892
900
|
}
|
|
893
901
|
function relayout(store) {
|
|
@@ -898,7 +906,10 @@ function claimSlot(store, docId, cold) {
|
|
|
898
906
|
const ordinal = Atomics.load(store.handles.header, STORE_SLOTS);
|
|
899
907
|
ensureOrdinal(store, ordinal, cold);
|
|
900
908
|
store.diskFile[ordinal] = IN_MEMORY;
|
|
909
|
+
const idBytesBefore = Atomics.load(store.handles.header, STORE_DOC_ID_BYTES);
|
|
901
910
|
appendDocId(store.handles, ordinal, docId);
|
|
911
|
+
const idBytes = Atomics.load(store.handles.header, STORE_DOC_ID_BYTES) - idBytesBefore;
|
|
912
|
+
if (idBytes > store.longestDocIdBytes) store.longestDocIdBytes = idBytes;
|
|
902
913
|
return ordinal;
|
|
903
914
|
}
|
|
904
915
|
function publishOrdinal(store, ordinal) {
|
|
@@ -974,6 +985,19 @@ function ensureOrdinal(store, ordinal, cold) {
|
|
|
974
985
|
if (!cold) ensureFloatSlot(store, ordinal);
|
|
975
986
|
ensureCodeSlot(store, ordinal);
|
|
976
987
|
}
|
|
988
|
+
function bufferGrowthBytes(buffer, bytes) {
|
|
989
|
+
return grownByteLength(buffer.byteLength, bytes, buffer.maxByteLength) - buffer.byteLength;
|
|
990
|
+
}
|
|
991
|
+
function growthBytesFor(store, vectors) {
|
|
992
|
+
const { handles } = store;
|
|
993
|
+
const slots = Atomics.load(handles.header, STORE_SLOTS);
|
|
994
|
+
const last = Math.min(slots + vectors, MAX_VECTOR_ORDINALS) - 1;
|
|
995
|
+
if (last < slots) return 0;
|
|
996
|
+
const idBytes = Atomics.load(handles.header, STORE_DOC_ID_BYTES);
|
|
997
|
+
let bytes = bufferGrowthBytes(handles.magnitudes, (last + 1) * 8) + bufferGrowthBytes(handles.present, last + 1) + bufferGrowthBytes(handles.unplaced, last + 1) + bufferGrowthBytes(handles.unplacedChunks, unplacedChunkCount(last) * 4) + bufferGrowthBytes(handles.removed, last + 1) + bufferGrowthBytes(handles.partitions, (last + 1) * 4) + bufferGrowthBytes(handles.codePresent, last + 1) + bufferGrowthBytes(handles.diskFile, (last + 1) * 4) + bufferGrowthBytes(handles.diskOffset, (last + 1) * 4) + bufferGrowthBytes(handles.docIdOffsets, (last + 2) * 4) + bufferGrowthBytes(handles.docIdBytes, idBytes + (last + 1 - slots) * store.longestDocIdBytes) + blocksGrowthBytes(handles.blocks, handles.layout, slots, last);
|
|
998
|
+
if (handles.codeLayout !== null) bytes += blocksGrowthBytes(handles.codeBlocks, handles.codeLayout, slots, last);
|
|
999
|
+
return bytes;
|
|
1000
|
+
}
|
|
977
1001
|
function writeVector(store, ordinal, vector) {
|
|
978
1002
|
const block = store.view.blockOf(ordinal);
|
|
979
1003
|
const base = slotByteOffset(store.handles.layout, store.view.localOrdinal(ordinal)) / 4;
|
|
@@ -1310,6 +1334,9 @@ function createVectorStore(options) {
|
|
|
1310
1334
|
},
|
|
1311
1335
|
memoryBytes() {
|
|
1312
1336
|
return open === null ? 0 : sharedVectorStoreBytes(open.handles);
|
|
1337
|
+
},
|
|
1338
|
+
growthBytesFor(vectors) {
|
|
1339
|
+
return open === null ? 0 : growthBytesFor(open, vectors);
|
|
1313
1340
|
}
|
|
1314
1341
|
};
|
|
1315
1342
|
}
|
|
@@ -1372,9 +1399,9 @@ function releaseSlotLocks(slot) {
|
|
|
1372
1399
|
for (const waiter of [...slot.idleWaiters]) waiter();
|
|
1373
1400
|
}
|
|
1374
1401
|
function rejectPending(slot, err) {
|
|
1375
|
-
for (const [,
|
|
1376
|
-
clearTimeout(
|
|
1377
|
-
|
|
1402
|
+
for (const [, waiting2] of slot.pending) {
|
|
1403
|
+
clearTimeout(waiting2.timer);
|
|
1404
|
+
waiting2.reject(err);
|
|
1378
1405
|
}
|
|
1379
1406
|
slot.pending.clear();
|
|
1380
1407
|
}
|
|
@@ -1453,11 +1480,11 @@ async function createVectorSearchPool(requestedCount) {
|
|
|
1453
1480
|
const key = message.requestId;
|
|
1454
1481
|
if (typeof key !== "string") return;
|
|
1455
1482
|
noteAnswered(slot, key, sharedFields);
|
|
1456
|
-
const
|
|
1457
|
-
if (!
|
|
1483
|
+
const waiting2 = slot.pending.get(key);
|
|
1484
|
+
if (!waiting2) return;
|
|
1458
1485
|
slot.pending.delete(key);
|
|
1459
|
-
clearTimeout(
|
|
1460
|
-
|
|
1486
|
+
clearTimeout(waiting2.timer);
|
|
1487
|
+
waiting2.resolve(message);
|
|
1461
1488
|
});
|
|
1462
1489
|
listenForFailure(worker, (err) => {
|
|
1463
1490
|
slot.alive = false;
|
|
@@ -2166,6 +2193,51 @@ async function releasePendingLocations(state) {
|
|
|
2166
2193
|
await releaseInTurn(state, () => releaseLocations(state, paths, pending));
|
|
2167
2194
|
}
|
|
2168
2195
|
|
|
2196
|
+
// src/runtime/background-memory.ts
|
|
2197
|
+
var admissions = /* @__PURE__ */ new Set();
|
|
2198
|
+
var waiting = /* @__PURE__ */ new Map();
|
|
2199
|
+
function admitsBackgroundBytes(bytes) {
|
|
2200
|
+
for (const admission of admissions) {
|
|
2201
|
+
if (!admission(bytes)) return false;
|
|
2202
|
+
}
|
|
2203
|
+
return true;
|
|
2204
|
+
}
|
|
2205
|
+
function noteWaitingForMemory(owner, work) {
|
|
2206
|
+
if (work === null) waiting.delete(owner);
|
|
2207
|
+
else waiting.set(owner, work);
|
|
2208
|
+
}
|
|
2209
|
+
function chargeHeapBytes(bytes) {
|
|
2210
|
+
}
|
|
2211
|
+
|
|
2212
|
+
// src/vector/vector-index/memory-charge.ts
|
|
2213
|
+
function admitShuffledRebuild(state) {
|
|
2214
|
+
const bytes = state.hnsw?.graphBytes ?? 0;
|
|
2215
|
+
if (!admitsBackgroundBytes(bytes)) {
|
|
2216
|
+
state.waitingForMemory = true;
|
|
2217
|
+
noteWaitingForMemory(state, { indexName: state.indexName, fieldName: state.fieldName });
|
|
2218
|
+
return false;
|
|
2219
|
+
}
|
|
2220
|
+
stopWaitingForMemory(state);
|
|
2221
|
+
chargeHeapBytes(bytes - state.rebuildChargeBytes);
|
|
2222
|
+
state.rebuildChargeBytes = bytes;
|
|
2223
|
+
return true;
|
|
2224
|
+
}
|
|
2225
|
+
function stopWaitingForMemory(state) {
|
|
2226
|
+
if (!state.waitingForMemory) return;
|
|
2227
|
+
state.waitingForMemory = false;
|
|
2228
|
+
noteWaitingForMemory(state, null);
|
|
2229
|
+
}
|
|
2230
|
+
function releaseRebuildCharge(state) {
|
|
2231
|
+
chargeHeapBytes(-state.rebuildChargeBytes);
|
|
2232
|
+
state.rebuildChargeBytes = 0;
|
|
2233
|
+
}
|
|
2234
|
+
function releaseMemoryCharges(state) {
|
|
2235
|
+
releaseRebuildCharge(state);
|
|
2236
|
+
stopWaitingForMemory(state);
|
|
2237
|
+
if (state.rebuildWaitTimer !== null) clearTimeout(state.rebuildWaitTimer);
|
|
2238
|
+
state.rebuildWaitTimer = null;
|
|
2239
|
+
}
|
|
2240
|
+
|
|
2169
2241
|
// src/vector/vector-index/build.ts
|
|
2170
2242
|
function bufferWhatTheGraphLacks(state, graph) {
|
|
2171
2243
|
for (const docId of allLiveDocIds(state)) {
|
|
@@ -2248,8 +2320,25 @@ async function rebuildInShuffledOrder(state) {
|
|
|
2248
2320
|
const adopted = await buildGraphFromStore(state);
|
|
2249
2321
|
if (adopted && replacing.loadedWithoutCount) state.onLoadedGraphRepaired();
|
|
2250
2322
|
}
|
|
2323
|
+
function retryWhenMemoryAllows(state) {
|
|
2324
|
+
if (state.rebuildWaitTimer !== null || state.disposed) return;
|
|
2325
|
+
const timer = setTimeout(() => {
|
|
2326
|
+
state.rebuildWaitTimer = null;
|
|
2327
|
+
startShuffledRebuild(state);
|
|
2328
|
+
}, SHUFFLED_REBUILD_MEMORY_RETRY_MS);
|
|
2329
|
+
timer.unref?.();
|
|
2330
|
+
state.rebuildWaitTimer = timer;
|
|
2331
|
+
}
|
|
2251
2332
|
function startShuffledRebuild(state) {
|
|
2252
|
-
if (state.shuffledRebuild !== null || state.disposed
|
|
2333
|
+
if (state.shuffledRebuild !== null || state.disposed) return;
|
|
2334
|
+
if (!shuffledRebuildDue(state)) {
|
|
2335
|
+
stopWaitingForMemory(state);
|
|
2336
|
+
return;
|
|
2337
|
+
}
|
|
2338
|
+
if (!admitShuffledRebuild(state)) {
|
|
2339
|
+
retryWhenMemoryAllows(state);
|
|
2340
|
+
return;
|
|
2341
|
+
}
|
|
2253
2342
|
state.shuffledRebuild = rebuildInShuffledOrder(state).catch((error) => {
|
|
2254
2343
|
console.warn(
|
|
2255
2344
|
`Rebuilding the graph of "${state.indexName}/${state.fieldName}" in a shuffled order failed, so the field keeps the graph it holds:`,
|
|
@@ -2257,6 +2346,7 @@ function startShuffledRebuild(state) {
|
|
|
2257
2346
|
);
|
|
2258
2347
|
}).finally(() => {
|
|
2259
2348
|
state.shuffledRebuild = null;
|
|
2349
|
+
releaseRebuildCharge(state);
|
|
2260
2350
|
scheduleBuild(state);
|
|
2261
2351
|
scheduleWorkerCopyLoad(state);
|
|
2262
2352
|
});
|
|
@@ -2382,9 +2472,12 @@ function releaseHeldMemory(state) {
|
|
|
2382
2472
|
state.savedFiles = [];
|
|
2383
2473
|
state.savedSignature = null;
|
|
2384
2474
|
state.store.release();
|
|
2475
|
+
releaseMemoryCharges(state);
|
|
2385
2476
|
}
|
|
2386
2477
|
function disposeIndex(state) {
|
|
2387
2478
|
state.disposed = true;
|
|
2479
|
+
if (state.rebuildWaitTimer !== null) clearTimeout(state.rebuildWaitTimer);
|
|
2480
|
+
state.rebuildWaitTimer = null;
|
|
2388
2481
|
if (state.buildTimer !== null) clearTimeout(state.buildTimer);
|
|
2389
2482
|
state.buildTimer = null;
|
|
2390
2483
|
invalidateWorkerCopies(state);
|
|
@@ -2723,6 +2816,7 @@ function maintenanceStatus(state) {
|
|
|
2723
2816
|
graphCount,
|
|
2724
2817
|
bufferSize: state.buffer.size,
|
|
2725
2818
|
building: state.building || state.buildScheduled || state.buildTimer !== null || state.shuffledRebuild !== null,
|
|
2819
|
+
waitingForMemory: state.waitingForMemory,
|
|
2726
2820
|
estimatedCompactMs,
|
|
2727
2821
|
estimatedOptimizeMs
|
|
2728
2822
|
};
|
|
@@ -2737,8 +2831,8 @@ function estimateMemoryBytes(state) {
|
|
|
2737
2831
|
}
|
|
2738
2832
|
|
|
2739
2833
|
// src/vector/vector-index/search.ts
|
|
2740
|
-
function* bufferCandidates(state, filter) {
|
|
2741
|
-
for (const docId of
|
|
2834
|
+
function* bufferCandidates(state, buffered, filter) {
|
|
2835
|
+
for (const docId of buffered) {
|
|
2742
2836
|
if (state.tombstones.has(docId)) continue;
|
|
2743
2837
|
if (filter) {
|
|
2744
2838
|
const ordinal = state.store.getOrdinal(docId);
|
|
@@ -2863,18 +2957,13 @@ function searchWithFilter(state, query, k, options, filter) {
|
|
|
2863
2957
|
}
|
|
2864
2958
|
}
|
|
2865
2959
|
const hnswResults = state.hnsw.search(query, k, metric, minSimilarity, { filter, efSearch, oversample, rescoreWindow }).map((r) => ({ docId: r.docId, score: r.score }));
|
|
2866
|
-
|
|
2867
|
-
|
|
2868
|
-
withBufferedVectors(state, query, k, options, filter, hnswResults),
|
|
2869
|
-
minSimilarity,
|
|
2870
|
-
filter
|
|
2871
|
-
);
|
|
2960
|
+
const results = state.buffer.size === 0 ? hnswResults : withBufferedVectors(state, query, k, options, filter, hnswResults, state.buffer);
|
|
2961
|
+
return fetchedOutcome(state, results, minSimilarity, filter);
|
|
2872
2962
|
}
|
|
2873
|
-
function withBufferedVectors(state, query, k, options, filter, graphResults) {
|
|
2874
|
-
|
|
2875
|
-
const
|
|
2876
|
-
|
|
2877
|
-
return mergeResults(graphResults, buffered.results, k);
|
|
2963
|
+
function withBufferedVectors(state, query, k, options, filter, graphResults, buffered) {
|
|
2964
|
+
const candidates = bufferCandidates(state, buffered, filter);
|
|
2965
|
+
const scan = bruteForceSearch(state, query, k, options.metric, options.minSimilarity, candidates);
|
|
2966
|
+
return mergeResults(graphResults, scan.results, k);
|
|
2878
2967
|
}
|
|
2879
2968
|
|
|
2880
2969
|
// src/vector/vector-index/index.ts
|
|
@@ -2925,6 +3014,9 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
2925
3014
|
hnsw: null,
|
|
2926
3015
|
freshGraph: null,
|
|
2927
3016
|
...noBuildHistory(),
|
|
3017
|
+
rebuildChargeBytes: 0,
|
|
3018
|
+
waitingForMemory: false,
|
|
3019
|
+
rebuildWaitTimer: null,
|
|
2928
3020
|
onLoadedGraphRepaired,
|
|
2929
3021
|
compactedNodeCount: 0,
|
|
2930
3022
|
building: false,
|
|
@@ -3027,6 +3119,7 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
3027
3119
|
}
|
|
3028
3120
|
}
|
|
3029
3121
|
const filterRevision = state.revision;
|
|
3122
|
+
const buffered = state.buffer.size === 0 ? null : [...state.buffer];
|
|
3030
3123
|
scheduleWorkerCopyLoad(state);
|
|
3031
3124
|
const viaWorkerCopy = await searchViaWorkerCopies(state, query, k, options.metric, options.minSimilarity, {
|
|
3032
3125
|
filter,
|
|
@@ -3038,7 +3131,7 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
3038
3131
|
filter = filterForOptions(state, options);
|
|
3039
3132
|
}
|
|
3040
3133
|
if (viaWorkerCopy !== null) {
|
|
3041
|
-
const results = withBufferedVectors(state, query, k, options, filter, viaWorkerCopy);
|
|
3134
|
+
const results = buffered === null ? viaWorkerCopy : withBufferedVectors(state, query, k, options, filter, viaWorkerCopy, buffered);
|
|
3042
3135
|
return fetchedOutcome(state, results, options.minSimilarity, filter);
|
|
3043
3136
|
}
|
|
3044
3137
|
return searchWithFilter(state, query, k, options, filter);
|
|
@@ -3082,6 +3175,7 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
3082
3175
|
},
|
|
3083
3176
|
maintenanceStatus: () => maintenanceStatus(state),
|
|
3084
3177
|
estimateMemoryBytes: () => estimateMemoryBytes(state),
|
|
3178
|
+
growthBytesFor: (vectors) => state.store.growthBytesFor(vectors),
|
|
3085
3179
|
deserialize: (parts) => {
|
|
3086
3180
|
invalidateWorkerCopies(state);
|
|
3087
3181
|
deserialize(state, parts);
|
|
@@ -18,7 +18,7 @@ function restoreDepsOf(core, host) {
|
|
|
18
18
|
function queueRestore(core, host, restoresInFlight, indexName, data) {
|
|
19
19
|
const previous = restoresInFlight.get(indexName) ?? Promise.resolve();
|
|
20
20
|
const restored = previous.then(async () => {
|
|
21
|
-
const { restoreFromSnapshot } = await import('./snapshot-
|
|
21
|
+
const { restoreFromSnapshot } = await import('./snapshot-EBH6RXQO.mjs');
|
|
22
22
|
await restoreFromSnapshot(indexName, data, restoreDepsOf(core, host));
|
|
23
23
|
});
|
|
24
24
|
const settled = restored.then(
|
|
@@ -49,7 +49,7 @@ function pageByPosition(placed, organic, { start, limit, pinnedIds, binding, enc
|
|
|
49
49
|
sortKey: null,
|
|
50
50
|
sortSignature: null,
|
|
51
51
|
depth,
|
|
52
|
-
binding
|
|
52
|
+
binding: binding()
|
|
53
53
|
});
|
|
54
54
|
return { page, cursor };
|
|
55
55
|
}
|
|
@@ -229,7 +229,7 @@ function rerankTextOf(values) {
|
|
|
229
229
|
// src/rerank/pipeline.ts
|
|
230
230
|
function rerankCursorDepth(cursor, binding, host) {
|
|
231
231
|
const decoded = host.decodePageCursor(cursor);
|
|
232
|
-
host.requireMatchingCursor(decoded, cursor, null, true, binding);
|
|
232
|
+
host.requireMatchingCursor(decoded, cursor, null, true, binding());
|
|
233
233
|
return decoded.depth ?? 0;
|
|
234
234
|
}
|
|
235
235
|
function requireWithinReach(cursorDepth, offset, limit) {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { vectorsPerFile } from './chunk-
|
|
2
|
-
import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-
|
|
3
|
-
import './chunk-
|
|
4
|
-
import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, prepareStoredSynonyms, rejectSnapshot, readHeapStatistics, readResidentMemory, extractVectorFieldsFromSchema, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, insertDocumentVectors, decodeIndexConfiguration, flattenSchema } from './chunk-
|
|
1
|
+
import { collectAndReturnPages, vectorsPerFile } from './chunk-73U5DWOZ.mjs';
|
|
2
|
+
import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-I45FAGOF.mjs';
|
|
3
|
+
import './chunk-FODCC3MK.mjs';
|
|
4
|
+
import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, prepareStoredSynonyms, rejectSnapshot, readHeapStatistics, readResidentMemory, extractVectorFieldsFromSchema, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, insertDocumentVectors, decodeIndexConfiguration, flattenSchema } from './chunk-2TMWIATH.mjs';
|
|
5
5
|
import { decodeVectorIndexParts, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors } from './chunk-P7C2S5JX.mjs';
|
|
6
6
|
import { MAX_FILE_BYTES } from './chunk-2CBQ2PCC.mjs';
|
|
7
7
|
import './chunk-JPTQZWJQ.mjs';
|
|
@@ -663,7 +663,8 @@ function refuse(indexName, replacesIndex, shortfall) {
|
|
|
663
663
|
function requireRestoreMemory(indexName, footprint, replacesIndex) {
|
|
664
664
|
const processBytes = footprint.heapBytes + footprint.externalBytes;
|
|
665
665
|
if (shortfallOf(footprint.heapBytes, processBytes, true) === null) return;
|
|
666
|
-
const
|
|
666
|
+
const collected = collectAndReturnPages();
|
|
667
|
+
const shortfall = shortfallOf(footprint.heapBytes, processBytes, collected || replacesIndex);
|
|
667
668
|
if (shortfall !== null) refuse(indexName, replacesIndex, shortfall);
|
|
668
669
|
}
|
|
669
670
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-
|
|
2
|
-
import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-
|
|
1
|
+
import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-FODCC3MK.mjs';
|
|
2
|
+
import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-2TMWIATH.mjs';
|
|
3
3
|
import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-P7C2S5JX.mjs';
|
|
4
4
|
import './chunk-2CBQ2PCC.mjs';
|
|
5
5
|
import './chunk-JPTQZWJQ.mjs';
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { E as EmbeddingAdapter, R as RerankAdapter } from './adapters-BQFQ4PRz.js';
|
|
2
|
-
import { C as ClusterNamespace } from './config-
|
|
3
|
-
import { h as TaskStore } from './tasks-
|
|
2
|
+
import { C as ClusterNamespace } from './config-CS5ADSrx.js';
|
|
3
|
+
import { h as TaskStore } from './tasks-5_LCFEzF.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* Context handed to {@link OnRequestHook} for every inbound request, captured
|
|
@@ -10,5 +10,9 @@ var REFUSAL_YOUNG_COLLECTION_MIN_INTERVAL_MS = 10;
|
|
|
10
10
|
var HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE = 0.75;
|
|
11
11
|
var HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE = 0.65;
|
|
12
12
|
var PROCESS_LIMIT_REFRESH_INTERVAL_MS = 1e3;
|
|
13
|
+
var V8_YOUNG_GENERATION_BYTES_BY_MAJOR = {
|
|
14
|
+
12: 48 * 1024 * 1024,
|
|
15
|
+
13: 192 * 1024 * 1024
|
|
16
|
+
};
|
|
13
17
|
|
|
14
|
-
export { HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, IDLE_HEAP_CLEANUP_CHECK_MS, IDLE_HEAP_CLEANUP_MAX_GROWTH_BYTES, IDLE_HEAP_CLEANUP_MAX_UTILISATION, IDLE_HEAP_CLEANUP_MIN_RECLAIMABLE_BYTES, IDLE_HEAP_CLEANUP_QUIET_CHECKS, PROCESS_LIMIT_REFRESH_INTERVAL_MS, REFUSAL_COLLECTION_MIN_INTERVAL_MS, REFUSAL_COLLECTION_TIME_MULTIPLE, REFUSAL_YOUNG_COLLECTION_MIN_INTERVAL_MS };
|
|
18
|
+
export { HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, IDLE_HEAP_CLEANUP_CHECK_MS, IDLE_HEAP_CLEANUP_MAX_GROWTH_BYTES, IDLE_HEAP_CLEANUP_MAX_UTILISATION, IDLE_HEAP_CLEANUP_MIN_RECLAIMABLE_BYTES, IDLE_HEAP_CLEANUP_QUIET_CHECKS, PROCESS_LIMIT_REFRESH_INTERVAL_MS, REFUSAL_COLLECTION_MIN_INTERVAL_MS, REFUSAL_COLLECTION_TIME_MULTIPLE, REFUSAL_YOUNG_COLLECTION_MIN_INTERVAL_MS, V8_YOUNG_GENERATION_BYTES_BY_MAJOR };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, BLOCK_MAX_BYTES, sharedVectorStoreBytes, STORE_SLOTS, STORE_LIVE_COUNT, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, openGraphState, fixedView, createSharedVectorStoreHandles,
|
|
1
|
+
import { openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, BLOCK_MAX_BYTES, sharedVectorStoreBytes, STORE_SLOTS, STORE_LIVE_COUNT, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, openGraphState, fixedView, createSharedVectorStoreHandles, STORE_DOC_ID_BYTES, unplacedChunkCount, blocksGrowthBytes, growBufferTo, createOrdinalFilter, addToOrdinalFilter, slotByteOffset, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, grownByteLength, createVectorBlock, ensureBlockSlots, QUERY_STAGED, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes } from './chunk-HC5NURS6.mjs';
|
|
2
2
|
import { MAX_VECTOR_ORDINALS } from './chunk-QSK35UC2.mjs';
|
|
3
3
|
import { NarsilError, ErrorCodes } from './chunk-OBWUWZRA.mjs';
|
|
4
4
|
import { readSync, closeSync, openSync } from 'fs';
|
|
@@ -581,7 +581,8 @@ function openHandles(dimension, codeBits, blockBytes) {
|
|
|
581
581
|
present: new Uint8Array(handles.present),
|
|
582
582
|
partitions: new Int32Array(handles.partitions),
|
|
583
583
|
diskFile: new Int32Array(handles.diskFile),
|
|
584
|
-
diskOffset: new Uint32Array(handles.diskOffset)
|
|
584
|
+
diskOffset: new Uint32Array(handles.diskOffset),
|
|
585
|
+
longestDocIdBytes: 0
|
|
585
586
|
};
|
|
586
587
|
}
|
|
587
588
|
function relayout(store) {
|
|
@@ -592,7 +593,10 @@ function claimSlot(store, docId, cold) {
|
|
|
592
593
|
const ordinal = Atomics.load(store.handles.header, STORE_SLOTS);
|
|
593
594
|
ensureOrdinal(store, ordinal, cold);
|
|
594
595
|
store.diskFile[ordinal] = IN_MEMORY;
|
|
596
|
+
const idBytesBefore = Atomics.load(store.handles.header, STORE_DOC_ID_BYTES);
|
|
595
597
|
appendDocId(store.handles, ordinal, docId);
|
|
598
|
+
const idBytes = Atomics.load(store.handles.header, STORE_DOC_ID_BYTES) - idBytesBefore;
|
|
599
|
+
if (idBytes > store.longestDocIdBytes) store.longestDocIdBytes = idBytes;
|
|
596
600
|
return ordinal;
|
|
597
601
|
}
|
|
598
602
|
function publishOrdinal(store, ordinal) {
|
|
@@ -668,6 +672,19 @@ function ensureOrdinal(store, ordinal, cold) {
|
|
|
668
672
|
if (!cold) ensureFloatSlot(store, ordinal);
|
|
669
673
|
ensureCodeSlot(store, ordinal);
|
|
670
674
|
}
|
|
675
|
+
function bufferGrowthBytes(buffer, bytes) {
|
|
676
|
+
return grownByteLength(buffer.byteLength, bytes, buffer.maxByteLength) - buffer.byteLength;
|
|
677
|
+
}
|
|
678
|
+
function growthBytesFor(store, vectors) {
|
|
679
|
+
const { handles } = store;
|
|
680
|
+
const slots = Atomics.load(handles.header, STORE_SLOTS);
|
|
681
|
+
const last = Math.min(slots + vectors, MAX_VECTOR_ORDINALS) - 1;
|
|
682
|
+
if (last < slots) return 0;
|
|
683
|
+
const idBytes = Atomics.load(handles.header, STORE_DOC_ID_BYTES);
|
|
684
|
+
let bytes = bufferGrowthBytes(handles.magnitudes, (last + 1) * 8) + bufferGrowthBytes(handles.present, last + 1) + bufferGrowthBytes(handles.unplaced, last + 1) + bufferGrowthBytes(handles.unplacedChunks, unplacedChunkCount(last) * 4) + bufferGrowthBytes(handles.removed, last + 1) + bufferGrowthBytes(handles.partitions, (last + 1) * 4) + bufferGrowthBytes(handles.codePresent, last + 1) + bufferGrowthBytes(handles.diskFile, (last + 1) * 4) + bufferGrowthBytes(handles.diskOffset, (last + 1) * 4) + bufferGrowthBytes(handles.docIdOffsets, (last + 2) * 4) + bufferGrowthBytes(handles.docIdBytes, idBytes + (last + 1 - slots) * store.longestDocIdBytes) + blocksGrowthBytes(handles.blocks, handles.layout, slots, last);
|
|
685
|
+
if (handles.codeLayout !== null) bytes += blocksGrowthBytes(handles.codeBlocks, handles.codeLayout, slots, last);
|
|
686
|
+
return bytes;
|
|
687
|
+
}
|
|
671
688
|
function writeVector(store, ordinal, vector) {
|
|
672
689
|
const block = store.view.blockOf(ordinal);
|
|
673
690
|
const base = slotByteOffset(store.handles.layout, store.view.localOrdinal(ordinal)) / 4;
|
|
@@ -1004,6 +1021,9 @@ function createVectorStore(options) {
|
|
|
1004
1021
|
},
|
|
1005
1022
|
memoryBytes() {
|
|
1006
1023
|
return open === null ? 0 : sharedVectorStoreBytes(open.handles);
|
|
1024
|
+
},
|
|
1025
|
+
growthBytesFor(vectors) {
|
|
1026
|
+
return open === null ? 0 : growthBytesFor(open, vectors);
|
|
1007
1027
|
}
|
|
1008
1028
|
};
|
|
1009
1029
|
}
|