@delali/narsil 0.3.1-next.104 → 0.3.1-next.107
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/{chunk-PW7GJFPW.mjs → chunk-JJKVYLL3.mjs} +2 -1
- package/dist/browser/{chunk-ECSLQ3S5.mjs → chunk-NNQPZPTB.mjs} +4 -4
- package/dist/browser/{chunk-6E4SWUYZ.mjs → chunk-OKZGAJEF.mjs} +110 -9
- package/dist/browser/{chunk-SFXM7PLV.mjs → chunk-OL7S2KLL.mjs} +1 -1
- package/dist/browser/{chunk-CWWD2UHP.mjs → chunk-PA56PRWX.mjs} +1 -1
- package/dist/browser/{chunk-5YMT23EX.mjs → chunk-WZUSMF7A.mjs} +7 -7
- package/dist/browser/{durability-wiring-HTBWDAOX.mjs → durability-wiring-HFQULBSA.mjs} +5 -5
- package/dist/browser/{module-6FVNK7DO.mjs → module-VROKZ7IZ.mjs} +19 -15
- package/dist/browser/{restore-queue-CL43WZTZ.mjs → restore-queue-EU77FRCV.mjs} +1 -1
- package/dist/browser/{snapshot-NL4GSDUD.mjs → snapshot-G77LQBZX.mjs} +6 -6
- package/dist/browser/{write-YFOOPLAN.mjs → write-RUC64JIH.mjs} +4 -4
- package/dist/{capabilities-DGaz7cb3.d.ts → capabilities-ByKPha_q.d.ts} +1 -1
- package/dist/{chunk-MKPE3VZF.mjs → chunk-3BRFOMGX.mjs} +1 -1
- package/dist/{chunk-LPQOEU5M.mjs → chunk-5FPMDU54.mjs} +38 -9
- package/dist/{chunk-35KEGAOS.mjs → chunk-7IBX2LRA.mjs} +6 -22
- package/dist/{chunk-PQHR5FHC.mjs → chunk-BRIOLMIX.mjs} +1 -1
- package/dist/{chunk-O372FZM4.mjs → chunk-CKNOGER5.mjs} +14 -14
- package/dist/{chunk-RWXFSGKJ.mjs → chunk-DNKM67DF.mjs} +2 -2
- package/dist/{chunk-XWJGJ2SE.mjs → chunk-FEOLYA2Y.mjs} +2 -2
- package/dist/{chunk-BLDJ5I6I.mjs → chunk-G6YATIRT.mjs} +112 -9
- package/dist/chunk-HAQUJYMN.mjs +344 -0
- package/dist/{chunk-47PZE6CJ.mjs → chunk-HNAQNAFS.mjs} +2 -2
- package/dist/{chunk-VFEALJEA.mjs → chunk-IA3H54X4.mjs} +1 -1
- package/dist/{chunk-KSQS77DK.mjs → chunk-IA6I5U36.mjs} +1 -1
- package/dist/{chunk-J2KRH2RJ.mjs → chunk-IZLM2OOO.mjs} +3 -3
- package/dist/{chunk-5W2ODPPO.mjs → chunk-KOCHOOS5.mjs} +175 -7
- package/dist/{chunk-DQ5NJSBZ.mjs → chunk-M5EZVXIM.mjs} +1 -1
- package/dist/{chunk-6GFVUXUH.mjs → chunk-NRARRUKN.mjs} +2 -2
- package/dist/{chunk-OIN27OQH.mjs → chunk-OKZPYJZX.mjs} +1 -1
- package/dist/{chunk-KNRU5CJ4.mjs → chunk-QA6F5MML.mjs} +5 -5
- package/dist/{chunk-WU4P5J4G.mjs → chunk-UM6Z7AIP.mjs} +3 -3
- package/dist/{chunk-PAWVKEVS.mjs → chunk-VHPVBJOI.mjs} +1 -1
- package/dist/{chunk-HMFNFVP7.mjs → chunk-VJS4JFBQ.mjs} +2 -1
- package/dist/{chunk-RFVUDM5E.mjs → chunk-VVDQNCOG.mjs} +2 -15
- package/dist/{chunk-3IQZYNW6.mjs → chunk-VVFKFHWE.mjs} +1 -1
- package/dist/{chunk-PFQQ3LCQ.mjs → chunk-VWFFCDOL.mjs} +5 -5
- package/dist/{chunk-IGUTV3MX.mjs → chunk-WCJJCBR4.mjs} +5 -5
- package/dist/{chunk-TDUBQ546.mjs → chunk-WXVTQ6NO.mjs} +1 -1
- package/dist/chunk-ZXZ26WSO.mjs +14 -0
- package/dist/client.d.ts +3 -3
- package/dist/{config-DWxtDsJ6.d.ts → config-BZ_e3X8w.d.ts} +6 -3
- package/dist/{direct-executor-5NX2ZIAG.mjs → direct-executor-BPBD2PBK.mjs} +8 -7
- package/dist/distribution/coordinator/etcd.d.ts +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +1 -1
- package/dist/distribution.d.ts +4 -4
- package/dist/distribution.mjs +92 -169
- package/dist/{durability-wiring-LD4GWPWD.mjs → durability-wiring-AZL4EFLO.mjs} +19 -18
- package/dist/{index-Bkdt8nDw.d.ts → index-Cl5qQauj.d.ts} +6 -1
- package/dist/index.browser.mjs +2 -2
- package/dist/index.mjs +12 -11
- package/dist/{module-QFWZ5OBF.mjs → module-6EY7GW2W.mjs} +23 -19
- package/dist/partial-words.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.mjs +15 -14
- package/dist/{restore-queue-7QIKM6PY.mjs → restore-queue-DDXFDP7U.mjs} +1 -1
- package/dist/server.d.ts +4 -4
- package/dist/server.mjs +25 -12
- package/dist/{snapshot-JYTT36WI.mjs → snapshot-IDAM65JO.mjs} +21 -20
- package/dist/{thread-AHLISTRZ.mjs → thread-LMSYLCPI.mjs} +12 -10
- package/dist/vector/search-worker.d.ts +2 -0
- package/dist/vector/search-worker.mjs +4 -4
- package/dist/workers/entry.mjs +6 -5
- package/dist/{write-GPFM2EIX.mjs → write-3LIFIGOL.mjs} +14 -13
- package/package.json +7 -7
- package/dist/chunk-B7BSQZ27.mjs +0 -206
|
@@ -349,6 +349,7 @@ var FARTHEST_REJECTED_LINKS_KEPT = 1;
|
|
|
349
349
|
var COMPACTION_TOMBSTONE_RATIO = 0.1;
|
|
350
350
|
var COMPACTION_ABSOLUTE_THRESHOLD = 1e3;
|
|
351
351
|
var DEFAULT_EF_SEARCH = 50;
|
|
352
|
+
var NEAREST_LINK_FALLBACK_HOSTS = 4;
|
|
352
353
|
|
|
353
354
|
// src/workers/thread-slot.ts
|
|
354
355
|
var goneThreadSlots = /* @__PURE__ */ new Set();
|
|
@@ -490,4 +491,4 @@ function missingWorkerEntry(moduleUrl) {
|
|
|
490
491
|
);
|
|
491
492
|
}
|
|
492
493
|
|
|
493
|
-
export { CHECKSUM_YIELD_CHUNK_BYTES, COMPACTION_ABSOLUTE_THRESHOLD, COMPACTION_TOMBSTONE_RATIO, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_EF_SEARCH, DEFAULT_REQUEST_TIMEOUT_MS, DISK_STORAGE_BLOCK_BYTES, DOCUMENT_ENCODER_RETAINED_BUFFER_BYTES, EARLIER_RELEASE_MIN_M, FARTHEST_REJECTED_LINKS_KEPT, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, GRAPH_LOCK_WAIT_DEADLINE_MS, GRAPH_LOCK_WAIT_SLICE_MS, INITIAL_LIST_CAPACITY, INITIAL_UPPER_SLOTS, LOCK_SPIN_ITERATIONS, LOCK_WAIT_SLICE_MS, MAX_DOCUMENT_ENCODING_DEPTH, MAX_DOC_ID_TABLE_BYTES, MAX_FILE_BYTES, MAX_LAYER_CAP, MAX_M, MAX_RENUMBERED_SEARCH_ATTEMPTS, MAX_VECTOR_ORDINALS, MAX_WASM_PAGES, MIN_M, MIN_OVERSAMPLE, NATIVE_ATTACH_RETRY_MS, NATIVE_CRC32_MIN_BYTES, OSQ_GRID, OSQ_LAMBDA, OSQ_PLACEMENT_QUERY_BITS, OSQ_QUERY_BITS, OSQ_REFINE_ROUNDS, OSQ_REFINE_TOLERANCE, OSQ_STEPS, PLACEMENT_ROOM_REQUESTS, SHUTDOWN_TIMEOUT_MS, SORT_MODES, SORT_VALUE_MAX_CODE_POINTS, UNPLACED_FLAG_CHUNK_ORDINALS, VALID_HNSW_METRICS, VECTOR_BLOCK_INITIAL_SLOTS, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_STORE_INITIAL_CAPACITY, WASM_PAGE_BYTES, WORKER_YOUNG_GENERATION_MB, appendFoldedCodePoint, codePointCount, compareCodePoints, compareComparableKeys, compareComparableValues, compareSortValues, defaultSortMode, detectRuntime, exceedsCodePoints, holdsUnpairedSurrogate, isSimdAvailable, isSortMode, markSimdUnavailable, missingWorkerEntry, noteThreadSlotGone, noteThreadSlotLive, readSortField, resolveRequestThreadCount, resolveWorkerCount, resolveWorkerEntry, singleFoldTable, sortModeOf, splitWorkerBudget, threadSlotGone, threadSlotOfWorker, toComparableSortValue, toReducedSortValue, truncateSortString };
|
|
494
|
+
export { CHECKSUM_YIELD_CHUNK_BYTES, COMPACTION_ABSOLUTE_THRESHOLD, COMPACTION_TOMBSTONE_RATIO, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_EF_SEARCH, DEFAULT_REQUEST_TIMEOUT_MS, DISK_STORAGE_BLOCK_BYTES, DOCUMENT_ENCODER_RETAINED_BUFFER_BYTES, EARLIER_RELEASE_MIN_M, FARTHEST_REJECTED_LINKS_KEPT, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, GRAPH_LOCK_WAIT_DEADLINE_MS, GRAPH_LOCK_WAIT_SLICE_MS, INITIAL_LIST_CAPACITY, INITIAL_UPPER_SLOTS, LOCK_SPIN_ITERATIONS, LOCK_WAIT_SLICE_MS, MAX_DOCUMENT_ENCODING_DEPTH, MAX_DOC_ID_TABLE_BYTES, MAX_FILE_BYTES, MAX_LAYER_CAP, MAX_M, MAX_RENUMBERED_SEARCH_ATTEMPTS, MAX_VECTOR_ORDINALS, MAX_WASM_PAGES, MIN_M, MIN_OVERSAMPLE, NATIVE_ATTACH_RETRY_MS, NATIVE_CRC32_MIN_BYTES, NEAREST_LINK_FALLBACK_HOSTS, OSQ_GRID, OSQ_LAMBDA, OSQ_PLACEMENT_QUERY_BITS, OSQ_QUERY_BITS, OSQ_REFINE_ROUNDS, OSQ_REFINE_TOLERANCE, OSQ_STEPS, PLACEMENT_ROOM_REQUESTS, SHUTDOWN_TIMEOUT_MS, SORT_MODES, SORT_VALUE_MAX_CODE_POINTS, UNPLACED_FLAG_CHUNK_ORDINALS, VALID_HNSW_METRICS, VECTOR_BLOCK_INITIAL_SLOTS, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_STORE_INITIAL_CAPACITY, WASM_PAGE_BYTES, WORKER_YOUNG_GENERATION_MB, appendFoldedCodePoint, codePointCount, compareCodePoints, compareComparableKeys, compareComparableValues, compareSortValues, defaultSortMode, detectRuntime, exceedsCodePoints, holdsUnpairedSurrogate, isSimdAvailable, isSortMode, markSimdUnavailable, missingWorkerEntry, noteThreadSlotGone, noteThreadSlotLive, readSortField, resolveRequestThreadCount, resolveWorkerCount, resolveWorkerEntry, singleFoldTable, sortModeOf, splitWorkerBudget, threadSlotGone, threadSlotOfWorker, toComparableSortValue, toReducedSortValue, truncateSortString };
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes } from './chunk-
|
|
2
|
-
import { VERSION, rulesOfStoredSynonyms, flattenSchema, packChecksummedEnvelope, openChecksummedEnvelope, requireValidFieldTypes, isCompositePartition, crc32, reconstructSchemaFromMetadata, extractVectorFieldsFromSchema, readHeader, HEADER_SIZE, engineVersionParts, writeHeader, partialFieldsOf, getLanguage, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, VECTOR_PATTERN, isQuantizationMode, isStorageMode, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, fnv1a, selectionKeeps, decodeMapWithoutFields, prepareDocumentVectors, messagePackUpperBoundBytes, segmentArraysOf, mergeSegmentContents, resolveIndexAnalysis } from './chunk-
|
|
3
|
-
import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-
|
|
4
|
-
import { compareCodePoints, MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-
|
|
1
|
+
import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes } from './chunk-PA56PRWX.mjs';
|
|
2
|
+
import { VERSION, rulesOfStoredSynonyms, flattenSchema, packChecksummedEnvelope, openChecksummedEnvelope, requireValidFieldTypes, isCompositePartition, crc32, reconstructSchemaFromMetadata, extractVectorFieldsFromSchema, readHeader, HEADER_SIZE, engineVersionParts, writeHeader, partialFieldsOf, getLanguage, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, VECTOR_PATTERN, isQuantizationMode, isStorageMode, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, fnv1a, selectionKeeps, decodeMapWithoutFields, prepareDocumentVectors, messagePackUpperBoundBytes, segmentArraysOf, mergeSegmentContents, resolveIndexAnalysis } from './chunk-WZUSMF7A.mjs';
|
|
3
|
+
import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-OL7S2KLL.mjs';
|
|
4
|
+
import { compareCodePoints, MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-JJKVYLL3.mjs';
|
|
5
5
|
import { NarsilError, ErrorCodes } from './chunk-J3QB4LBD.mjs';
|
|
6
6
|
import { decode, encode } from '@msgpack/msgpack';
|
|
7
7
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { unpackLevels, osqRecordBytes, VECTOR_FILE_MAX_VECTORS, NO_LIVE_VECTOR_AT_POSITION, readTrailer, stageQueryLevels, osqCodeBytes, osqStagedQueryBytes, bytesToVectors, VECTOR_GRAPH_PART_VECTORS, VECTOR_GRAPH_PART_PAYLOAD_VERSION, VECTOR_FILE_PAYLOAD_VERSION, deadBitsWhere, vectorsToBytes, writeRecord } from './chunk-
|
|
2
|
-
import { markSimdUnavailable, MAX_WASM_PAGES, WASM_PAGE_BYTES, VECTOR_SCRATCH_SLOTS, UNPLACED_FLAG_CHUNK_ORDINALS, COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, MAX_FILE_BYTES, VECTOR_BLOCK_INITIAL_SLOTS, MAX_VECTOR_ORDINALS, VECTOR_STORE_INITIAL_CAPACITY, PLACEMENT_ROOM_REQUESTS, LOCK_SPIN_ITERATIONS, LOCK_WAIT_SLICE_MS, MAX_LAYER_CAP, INITIAL_LIST_CAPACITY, OSQ_PLACEMENT_QUERY_BITS, GRAPH_LOCK_WAIT_SLICE_MS, OSQ_STEPS, OSQ_GRID, OSQ_QUERY_BITS, GRAPH_LOCK_WAIT_DEADLINE_MS, compareCodePoints, INITIAL_UPPER_SLOTS, OSQ_REFINE_ROUNDS, OSQ_REFINE_TOLERANCE, threadSlotGone, DEFAULT_EF_SEARCH, OSQ_LAMBDA, FARTHEST_REJECTED_LINKS_KEPT, MAX_DOC_ID_TABLE_BYTES } from './chunk-
|
|
1
|
+
import { unpackLevels, osqRecordBytes, VECTOR_FILE_MAX_VECTORS, NO_LIVE_VECTOR_AT_POSITION, readTrailer, stageQueryLevels, osqCodeBytes, osqStagedQueryBytes, bytesToVectors, VECTOR_GRAPH_PART_VECTORS, VECTOR_GRAPH_PART_PAYLOAD_VERSION, VECTOR_FILE_PAYLOAD_VERSION, deadBitsWhere, vectorsToBytes, writeRecord } from './chunk-OL7S2KLL.mjs';
|
|
2
|
+
import { markSimdUnavailable, MAX_WASM_PAGES, WASM_PAGE_BYTES, VECTOR_SCRATCH_SLOTS, UNPLACED_FLAG_CHUNK_ORDINALS, COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, MAX_FILE_BYTES, VECTOR_BLOCK_INITIAL_SLOTS, MAX_VECTOR_ORDINALS, VECTOR_STORE_INITIAL_CAPACITY, PLACEMENT_ROOM_REQUESTS, LOCK_SPIN_ITERATIONS, LOCK_WAIT_SLICE_MS, MAX_LAYER_CAP, INITIAL_LIST_CAPACITY, OSQ_PLACEMENT_QUERY_BITS, GRAPH_LOCK_WAIT_SLICE_MS, OSQ_STEPS, OSQ_GRID, OSQ_QUERY_BITS, GRAPH_LOCK_WAIT_DEADLINE_MS, compareCodePoints, INITIAL_UPPER_SLOTS, OSQ_REFINE_ROUNDS, OSQ_REFINE_TOLERANCE, threadSlotGone, DEFAULT_EF_SEARCH, OSQ_LAMBDA, NEAREST_LINK_FALLBACK_HOSTS, FARTHEST_REJECTED_LINKS_KEPT, MAX_DOC_ID_TABLE_BYTES } from './chunk-JJKVYLL3.mjs';
|
|
3
3
|
import { NarsilError, ErrorCodes, MAX_DOC_ID_LENGTH } from './chunk-J3QB4LBD.mjs';
|
|
4
4
|
|
|
5
5
|
// src/vector/shared-buffers/growable.ts
|
|
@@ -101,6 +101,7 @@ function createSharedGraphHandles(shape) {
|
|
|
101
101
|
locks: perOrdinal(4, VECTOR_STORE_INITIAL_CAPACITY),
|
|
102
102
|
tombstones: perOrdinal(1, VECTOR_STORE_INITIAL_CAPACITY),
|
|
103
103
|
incoming: perOrdinal(4, VECTOR_STORE_INITIAL_CAPACITY),
|
|
104
|
+
nearestLinkDue: perOrdinal(1, VECTOR_STORE_INITIAL_CAPACITY),
|
|
104
105
|
heldLocks: new Int32Array(createFixedBuffer(heldWords * 4), 0, heldWords)
|
|
105
106
|
};
|
|
106
107
|
publishGraphCapacity(handles);
|
|
@@ -121,7 +122,8 @@ function publishGraphCapacity(handles) {
|
|
|
121
122
|
Math.floor(handles.upperBase.byteLength / 4),
|
|
122
123
|
Math.floor(handles.locks.byteLength / 4),
|
|
123
124
|
handles.tombstones.byteLength,
|
|
124
|
-
Math.floor(handles.incoming.byteLength / 4)
|
|
125
|
+
Math.floor(handles.incoming.byteLength / 4),
|
|
126
|
+
handles.nearestLinkDue.byteLength
|
|
125
127
|
);
|
|
126
128
|
raiseHeaderWord(handles.header, GRAPH_SLOT_CAPACITY, slotCapacity);
|
|
127
129
|
raiseHeaderWord(handles.header, GRAPH_UPPER_CAPACITY, Math.floor(handles.upper.byteLength / 4));
|
|
@@ -665,12 +667,14 @@ function openAdjacency(handles, m, mMax0) {
|
|
|
665
667
|
level0: fixedView(handles.level0, Int32Array),
|
|
666
668
|
upperBase: fixedView(handles.upperBase, Int32Array),
|
|
667
669
|
upper: fixedView(handles.upper, Int32Array),
|
|
668
|
-
incoming: fixedView(handles.incoming, Int32Array)
|
|
670
|
+
incoming: fixedView(handles.incoming, Int32Array),
|
|
671
|
+
nearestLinkDue: fixedView(handles.nearestLinkDue, Uint8Array)
|
|
669
672
|
};
|
|
670
673
|
}
|
|
671
674
|
function rebindNodes(adj) {
|
|
672
675
|
adj.nodeLevels = fixedView(adj.handles.nodeLevels, Uint8Array);
|
|
673
676
|
adj.incoming = fixedView(adj.handles.incoming, Int32Array);
|
|
677
|
+
adj.nearestLinkDue = fixedView(adj.handles.nearestLinkDue, Uint8Array);
|
|
674
678
|
adj.level0 = fixedView(adj.handles.level0, Int32Array);
|
|
675
679
|
adj.upperBase = fixedView(adj.handles.upperBase, Int32Array);
|
|
676
680
|
}
|
|
@@ -708,6 +712,7 @@ function ensureAdjacencyCapacity(adj, needed) {
|
|
|
708
712
|
growBufferToExactly(handles.locks, capacity * 4);
|
|
709
713
|
growBufferToExactly(handles.tombstones, capacity);
|
|
710
714
|
growBufferToExactly(handles.incoming, capacity * 4);
|
|
715
|
+
growBufferToExactly(handles.nearestLinkDue, capacity);
|
|
711
716
|
growBufferToExactly(handles.nodeLevels, capacity);
|
|
712
717
|
publishGraphCapacity(handles);
|
|
713
718
|
rebindNodes(adj);
|
|
@@ -747,6 +752,7 @@ function createNode(adj, ord, maxLayer) {
|
|
|
747
752
|
adj.upperBase[ord] = maxLayer >= 1 ? allocateUpperBlock(adj, maxLayer) + 1 : 0;
|
|
748
753
|
adj.level0[ord * adj.level0Stride] = 0;
|
|
749
754
|
raiseSlots(adj, ord + 1);
|
|
755
|
+
Atomics.store(adj.nearestLinkDue, ord, 1);
|
|
750
756
|
Atomics.store(adj.nodeLevels, ord, maxLayer + 1);
|
|
751
757
|
}
|
|
752
758
|
function deleteNode(adj, ord) {
|
|
@@ -755,11 +761,13 @@ function deleteNode(adj, ord) {
|
|
|
755
761
|
adj.upperBase[ord] = 0;
|
|
756
762
|
adj.level0[ord * adj.level0Stride] = 0;
|
|
757
763
|
Atomics.store(adj.incoming, ord, 0);
|
|
764
|
+
Atomics.store(adj.nearestLinkDue, ord, 0);
|
|
758
765
|
}
|
|
759
766
|
function resetAdjacency(adj) {
|
|
760
767
|
rebindNodes(adj);
|
|
761
768
|
rebindUpper(adj);
|
|
762
769
|
adj.incoming.fill(0);
|
|
770
|
+
adj.nearestLinkDue.fill(0);
|
|
763
771
|
adj.nodeLevels.fill(0);
|
|
764
772
|
adj.level0.fill(0);
|
|
765
773
|
adj.upperBase.fill(0);
|
|
@@ -776,7 +784,7 @@ function hasNode(adj, ord) {
|
|
|
776
784
|
}
|
|
777
785
|
function graphBytes(adj) {
|
|
778
786
|
const handles = adj.handles;
|
|
779
|
-
return handles.header.byteLength + handles.nodeLevels.byteLength + handles.level0.byteLength + handles.upperBase.byteLength + handles.upper.byteLength + handles.locks.byteLength + handles.tombstones.byteLength + handles.incoming.byteLength + handles.heldLocks.byteLength;
|
|
787
|
+
return handles.header.byteLength + handles.nodeLevels.byteLength + handles.level0.byteLength + handles.upperBase.byteLength + handles.upper.byteLength + handles.locks.byteLength + handles.tombstones.byteLength + handles.incoming.byteLength + handles.nearestLinkDue.byteLength + handles.heldLocks.byteLength;
|
|
780
788
|
}
|
|
781
789
|
|
|
782
790
|
// src/vector/hnsw/adjacency/incoming.ts
|
|
@@ -790,6 +798,7 @@ function noteLinked(adj, ord) {
|
|
|
790
798
|
}
|
|
791
799
|
function noteUnlinked(adj, ord) {
|
|
792
800
|
if (ord < 0 || !reachNode(adj, ord)) return 0;
|
|
801
|
+
Atomics.store(adj.nearestLinkDue, ord, 1);
|
|
793
802
|
for (; ; ) {
|
|
794
803
|
const seen = Atomics.load(adj.incoming, ord);
|
|
795
804
|
if (seen <= 0) return 0;
|
|
@@ -801,7 +810,9 @@ function releaseUnlessLast(adj, ord) {
|
|
|
801
810
|
for (; ; ) {
|
|
802
811
|
const seen = Atomics.load(adj.incoming, ord);
|
|
803
812
|
if (seen <= 1) return false;
|
|
804
|
-
if (Atomics.compareExchange(adj.incoming, ord, seen, seen - 1)
|
|
813
|
+
if (Atomics.compareExchange(adj.incoming, ord, seen, seen - 1) !== seen) continue;
|
|
814
|
+
Atomics.store(adj.nearestLinkDue, ord, 1);
|
|
815
|
+
return true;
|
|
805
816
|
}
|
|
806
817
|
}
|
|
807
818
|
function holdsNodeAt(adj, ord) {
|
|
@@ -1775,6 +1786,95 @@ function writeKeepingLastLinks(state, ord, sorted, kept, held) {
|
|
|
1775
1786
|
writeNeighbors(adjacency, ord, 0, kept.ords, kept.size);
|
|
1776
1787
|
}
|
|
1777
1788
|
|
|
1789
|
+
// src/vector/hnsw/nearest-links.ts
|
|
1790
|
+
var NOT_YET_FOUND = -2;
|
|
1791
|
+
function nearer(a, b) {
|
|
1792
|
+
return a.distance < b.distance || a.distance === b.distance && a.ord < b.ord;
|
|
1793
|
+
}
|
|
1794
|
+
function listOf(state, ord) {
|
|
1795
|
+
const adjacency = state.adjacency;
|
|
1796
|
+
const base = layerBase(adjacency, ord, 0);
|
|
1797
|
+
if (base === ABSENT) return null;
|
|
1798
|
+
return adjacency.level0.subarray(base + 1, base + 1 + adjacency.level0[base]);
|
|
1799
|
+
}
|
|
1800
|
+
function nearestInList(pass, ord) {
|
|
1801
|
+
const known = pass.nearest[ord];
|
|
1802
|
+
if (known !== NOT_YET_FOUND) return known;
|
|
1803
|
+
let best = null;
|
|
1804
|
+
for (const neighbour of listOf(pass.state, ord) ?? []) {
|
|
1805
|
+
if (neighbour === ord || !isLive(pass.state, neighbour)) continue;
|
|
1806
|
+
const scored = { ord: neighbour, distance: pass.distance(ord, neighbour) };
|
|
1807
|
+
if (best === null || nearer(scored, best)) best = scored;
|
|
1808
|
+
}
|
|
1809
|
+
const nearest = best === null ? -1 : best.ord;
|
|
1810
|
+
pass.nearest[ord] = nearest;
|
|
1811
|
+
return nearest;
|
|
1812
|
+
}
|
|
1813
|
+
function fallbackHostsFor(pass, ord, nearest) {
|
|
1814
|
+
const around = [];
|
|
1815
|
+
for (const neighbour of listOf(pass.state, nearest) ?? []) {
|
|
1816
|
+
if (neighbour === ord || !isLive(pass.state, neighbour)) continue;
|
|
1817
|
+
around.push({ ord: neighbour, distance: pass.distance(ord, neighbour) });
|
|
1818
|
+
}
|
|
1819
|
+
around.sort((a, b) => nearer(a, b) ? -1 : nearer(b, a) ? 1 : 0);
|
|
1820
|
+
return around.slice(0, NEAREST_LINK_FALLBACK_HOSTS).map((scored) => scored.ord);
|
|
1821
|
+
}
|
|
1822
|
+
function replaceableSlot(pass, host, list) {
|
|
1823
|
+
let farthest = -1;
|
|
1824
|
+
let farthestNode = null;
|
|
1825
|
+
for (let i = 0; i < list.length; i++) {
|
|
1826
|
+
const neighbour = list[i];
|
|
1827
|
+
if (incomingCount(pass.state.adjacency, neighbour) < 2) continue;
|
|
1828
|
+
if (isLive(pass.state, neighbour) && nearestInList(pass, neighbour) === host) continue;
|
|
1829
|
+
const scored = { ord: neighbour, distance: pass.distance(host, neighbour) };
|
|
1830
|
+
if (farthestNode === null || nearer(farthestNode, scored)) {
|
|
1831
|
+
farthest = i;
|
|
1832
|
+
farthestNode = scored;
|
|
1833
|
+
}
|
|
1834
|
+
}
|
|
1835
|
+
return farthest;
|
|
1836
|
+
}
|
|
1837
|
+
function linkFrom(pass, host, ord) {
|
|
1838
|
+
const adjacency = pass.state.adjacency;
|
|
1839
|
+
const base = layerBase(adjacency, host, 0);
|
|
1840
|
+
if (base === ABSENT) return false;
|
|
1841
|
+
const list = adjacency.level0.subarray(base + 1, base + 1 + adjacency.level0[base]);
|
|
1842
|
+
if (list.includes(ord)) return true;
|
|
1843
|
+
if (list.length < pass.state.Mmax0) {
|
|
1844
|
+
adjacency.level0[base + list.length + 1] = ord;
|
|
1845
|
+
adjacency.level0[base] = list.length + 1;
|
|
1846
|
+
noteLinked(adjacency, ord);
|
|
1847
|
+
pass.nearest[host] = NOT_YET_FOUND;
|
|
1848
|
+
return true;
|
|
1849
|
+
}
|
|
1850
|
+
const slot = replaceableSlot(pass, host, list);
|
|
1851
|
+
if (slot === -1) return false;
|
|
1852
|
+
const replaced = list[slot];
|
|
1853
|
+
list[slot] = ord;
|
|
1854
|
+
noteUnlinked(adjacency, replaced);
|
|
1855
|
+
noteLinked(adjacency, ord);
|
|
1856
|
+
pass.nearest[host] = NOT_YET_FOUND;
|
|
1857
|
+
return true;
|
|
1858
|
+
}
|
|
1859
|
+
function linkFromNearest(state) {
|
|
1860
|
+
const adjacency = state.adjacency;
|
|
1861
|
+
const slots = adjacencySlots(adjacency);
|
|
1862
|
+
if (slots > 0) reachNode(adjacency, slots - 1);
|
|
1863
|
+
const pass = {
|
|
1864
|
+
state,
|
|
1865
|
+
distance: nodeDistanceFunction(state, state.buildMetric),
|
|
1866
|
+
nearest: new Int32Array(slots).fill(NOT_YET_FOUND)
|
|
1867
|
+
};
|
|
1868
|
+
for (let ord = 0; ord < slots; ord++) {
|
|
1869
|
+
if (Atomics.load(adjacency.nearestLinkDue, ord) !== 1) continue;
|
|
1870
|
+
Atomics.store(adjacency.nearestLinkDue, ord, 0);
|
|
1871
|
+
if (!isLive(state, ord)) continue;
|
|
1872
|
+
const nearest = nearestInList(pass, ord);
|
|
1873
|
+
if (nearest === -1 || linkFrom(pass, nearest, ord)) continue;
|
|
1874
|
+
for (const host of fallbackHostsFor(pass, ord, nearest)) if (linkFrom(pass, host, ord)) break;
|
|
1875
|
+
}
|
|
1876
|
+
}
|
|
1877
|
+
|
|
1778
1878
|
// src/vector/hnsw/connect-unreached.ts
|
|
1779
1879
|
var NOT_REACHED_THROUGH_A_NODE = -1;
|
|
1780
1880
|
function reach(walk, node, through) {
|
|
@@ -1815,7 +1915,7 @@ function holdsLinkOutsideTheWalk(state, walk, host) {
|
|
|
1815
1915
|
for (let i = 1; i <= count; i++) if (reachedThroughAnotherNode(walk, host, adjacency.level0[base + i])) return true;
|
|
1816
1916
|
return false;
|
|
1817
1917
|
}
|
|
1818
|
-
function
|
|
1918
|
+
function listOf2(state, ord) {
|
|
1819
1919
|
const adjacency = state.adjacency;
|
|
1820
1920
|
const base = layerBase(adjacency, ord, 0);
|
|
1821
1921
|
if (base === ABSENT) return new Int32Array(0);
|
|
@@ -1834,7 +1934,7 @@ function nearestHost(state, walk, orphan, nearby, test) {
|
|
|
1834
1934
|
}
|
|
1835
1935
|
};
|
|
1836
1936
|
for (const host of nearby.list) consider(host);
|
|
1837
|
-
for (const host of
|
|
1937
|
+
for (const host of listOf2(state, orphan))
|
|
1838
1938
|
if (host >= 0 && host < walk.reached.length && nearby.holds[host] !== 1) consider(host);
|
|
1839
1939
|
if (best !== -1) return best;
|
|
1840
1940
|
for (let host = 0; host < walk.reached.length; host++) consider(host);
|
|
@@ -1915,6 +2015,7 @@ function holdsUnlinkedNode(state) {
|
|
|
1915
2015
|
function connectThroughEitherCore(state, nearby) {
|
|
1916
2016
|
const field = nativeFieldFor();
|
|
1917
2017
|
if (field !== null && nativeConnect(state, field, state.buildMetric, nearby)) return;
|
|
2018
|
+
linkFromNearest(state);
|
|
1918
2019
|
connectUnreached(state, nearby);
|
|
1919
2020
|
}
|
|
1920
2021
|
function connectAfterRemoval(state, nearby) {
|
|
@@ -1940,7 +2041,7 @@ function baseNeighboursOfRemoved(state, removed) {
|
|
|
1940
2041
|
const leaving = new Set(removed);
|
|
1941
2042
|
const nearby = [];
|
|
1942
2043
|
for (const ord of removed) {
|
|
1943
|
-
for (const neighbour of
|
|
2044
|
+
for (const neighbour of listOf2(state, ord)) if (!leaving.has(neighbour)) nearby.push(neighbour);
|
|
1944
2045
|
}
|
|
1945
2046
|
return Int32Array.from(nearby);
|
|
1946
2047
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors, getNestedValue, requireValidFieldTypes, VECTOR_PATTERN } from './chunk-
|
|
1
|
+
import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors, getNestedValue, requireValidFieldTypes, VECTOR_PATTERN } from './chunk-WZUSMF7A.mjs';
|
|
2
2
|
import { NarsilError, ErrorCodes } from './chunk-J3QB4LBD.mjs';
|
|
3
3
|
import { decode } from '@msgpack/msgpack';
|
|
4
4
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { MAX_DOCUMENT_ENCODING_DEPTH, singleFoldTable, appendFoldedCodePoint, compareCodePoints, EARLIER_RELEASE_MIN_M, MIN_M, MAX_M, DOCUMENT_ENCODER_RETAINED_BUFFER_BYTES, MAX_FILE_BYTES, detectRuntime, splitWorkerBudget, resolveWorkerCount, CHECKSUM_YIELD_CHUNK_BYTES, resolveWorkerEntry, missingWorkerEntry, DEFAULT_REQUEST_TIMEOUT_MS, toReducedSortValue, resolveRequestThreadCount, defaultSortMode, holdsUnpairedSurrogate, compareComparableValues, readSortField, exceedsCodePoints, toComparableSortValue, SORT_VALUE_MAX_CODE_POINTS, compareComparableKeys, truncateSortString, sortModeOf, isSortMode, SORT_MODES, SHUTDOWN_TIMEOUT_MS, MIN_OVERSAMPLE, MAX_RENUMBERED_SEARCH_ATTEMPTS, compareSortValues, DEFAULT_BACKPRESSURE_LIMIT, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, codePointCount, noteThreadSlotGone, threadSlotOfWorker } from './chunk-
|
|
1
|
+
import { MAX_DOCUMENT_ENCODING_DEPTH, singleFoldTable, appendFoldedCodePoint, compareCodePoints, EARLIER_RELEASE_MIN_M, MIN_M, MAX_M, DOCUMENT_ENCODER_RETAINED_BUFFER_BYTES, MAX_FILE_BYTES, detectRuntime, splitWorkerBudget, resolveWorkerCount, CHECKSUM_YIELD_CHUNK_BYTES, resolveWorkerEntry, missingWorkerEntry, DEFAULT_REQUEST_TIMEOUT_MS, toReducedSortValue, resolveRequestThreadCount, defaultSortMode, holdsUnpairedSurrogate, compareComparableValues, readSortField, exceedsCodePoints, toComparableSortValue, SORT_VALUE_MAX_CODE_POINTS, compareComparableKeys, truncateSortString, sortModeOf, isSortMode, SORT_MODES, SHUTDOWN_TIMEOUT_MS, MIN_OVERSAMPLE, MAX_RENUMBERED_SEARCH_ATTEMPTS, compareSortValues, DEFAULT_BACKPRESSURE_LIMIT, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, codePointCount, noteThreadSlotGone, threadSlotOfWorker } from './chunk-JJKVYLL3.mjs';
|
|
2
2
|
import { DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS } from './chunk-JPTQZWJQ.mjs';
|
|
3
3
|
import { ErrorCodes, NarsilError, createNarsilError, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, MAX_SUGGEST_LIMIT, MAX_INDEX_CONFIGURATION_BYTES, FOREIGN_STATS_TTL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_IMPORT_REFUSALS_REPORTED, DEFAULT_SUGGEST_LIMIT, DEFAULT_FUZZY_PREFIX_LENGTH, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, RESULT_WINDOW, DEFAULT_LIMIT, DEFAULT_OFFSET, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, MIN_DOCUMENTS_FOR_SEGMENTS, DEFAULT_PAGE_SIZE, DEFAULT_MAX_PER_GROUP, PREFIX_MAX_EXPANSIONS, HEAP_PRESSURE_REARM_FRACTION, MAX_TOLERANCE, MAX_PREFIX_LENGTH, BATCH_CHUNK_SIZE, MAX_DOC_ID_LENGTH } from './chunk-J3QB4LBD.mjs';
|
|
4
4
|
import { Encoder, Decoder, decode, encode } from '@msgpack/msgpack';
|
|
@@ -14949,7 +14949,7 @@ function requireLoadedVectorCode() {
|
|
|
14949
14949
|
function loadVectorCode() {
|
|
14950
14950
|
if (loaded !== null) return Promise.resolve(loaded);
|
|
14951
14951
|
if (loading === null) {
|
|
14952
|
-
loading = import('./module-
|
|
14952
|
+
loading = import('./module-VROKZ7IZ.mjs').then(
|
|
14953
14953
|
(code) => {
|
|
14954
14954
|
loaded = code;
|
|
14955
14955
|
return code;
|
|
@@ -19037,7 +19037,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
19037
19037
|
if (config?.lifecycle !== void 0 && durabilityTier === null) {
|
|
19038
19038
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
|
|
19039
19039
|
}
|
|
19040
|
-
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-
|
|
19040
|
+
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-HFQULBSA.mjs');
|
|
19041
19041
|
const filesystemDurability = durabilityTier?.kind === "wal";
|
|
19042
19042
|
const executor = createDirectExecutor({
|
|
19043
19043
|
vectorWorkerCopies: vectorCopyPolicy,
|
|
@@ -22470,14 +22470,14 @@ function portabilityOperations(core, config, engine) {
|
|
|
22470
22470
|
return {
|
|
22471
22471
|
snapshot(indexName) {
|
|
22472
22472
|
return withOpenIndex(indexName, async () => {
|
|
22473
|
-
const { createSnapshot } = await import('./snapshot-
|
|
22473
|
+
const { createSnapshot } = await import('./snapshot-G77LQBZX.mjs');
|
|
22474
22474
|
return createSnapshot(snapshotDepsOf(indexName));
|
|
22475
22475
|
});
|
|
22476
22476
|
},
|
|
22477
22477
|
async restore(indexName, data) {
|
|
22478
22478
|
core.guardShutdown();
|
|
22479
22479
|
validateIndexName(indexName);
|
|
22480
|
-
const restored = import('./restore-queue-
|
|
22480
|
+
const restored = import('./restore-queue-EU77FRCV.mjs').then(
|
|
22481
22481
|
({ queueRestore }) => queueRestore(core, restoreHost, restoresInFlight, indexName, data)
|
|
22482
22482
|
);
|
|
22483
22483
|
await emitWriteEventAfter(core, indexName, restored);
|
|
@@ -22486,8 +22486,8 @@ function portabilityOperations(core, config, engine) {
|
|
|
22486
22486
|
exportDocuments(indexName) {
|
|
22487
22487
|
return withOpenIndex(indexName, async () => {
|
|
22488
22488
|
const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
|
|
22489
|
-
import('./snapshot-
|
|
22490
|
-
import('./write-
|
|
22489
|
+
import('./snapshot-G77LQBZX.mjs'),
|
|
22490
|
+
import('./write-RUC64JIH.mjs')
|
|
22491
22491
|
]);
|
|
22492
22492
|
const header = indexConfigurationOf(core.requireIndex(indexName));
|
|
22493
22493
|
return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { nodeThreadId, spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
|
|
2
|
-
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, decodeSegmentManifest, readListedFile, adapterCheckpointStore, SINGLE_NODE_PRIMARY_TERM, readMetadataEnvelope, writeMetadataEnvelope, metadataFromConfig, DIRECTORY_LOCK_ATTEMPTS, createSeqOwner, readSegmentManifest, snapshotBundleKey, reclaimUnlistedKeys, manifestReferencedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, manifestFileRefs, DIRECTORY_LOCK_SETTLE_MS, readSegmentFile, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, frameRecord, isId, isUint32, isSequenceNumber, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, commitCheckpointManifest, removeCheckpointGarbage, readWalSegments, durableRecordMissing, MAX_SEGMENTS_PER_PARTITION, growPartitionsTo, writeSegmentHeader, VECTOR_FILE_ENVELOPE_VERSION, SEGMENT_HEADER_SIZE, documentsOfPayloadV2, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, metadataConfigOf, buildEntry, readCurrentManifest, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-
|
|
3
|
-
import { applyDeleteEntry, applyIndexEntry } from './chunk-
|
|
4
|
-
import { encodeDocument, fnv1a, requireValidFieldTypes, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-
|
|
5
|
-
import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-
|
|
6
|
-
import { compareCodePoints, MIN_M, MAX_FILE_BYTES, detectRuntime, resolveWorkerEntry } from './chunk-
|
|
2
|
+
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, decodeSegmentManifest, readListedFile, adapterCheckpointStore, SINGLE_NODE_PRIMARY_TERM, readMetadataEnvelope, writeMetadataEnvelope, metadataFromConfig, DIRECTORY_LOCK_ATTEMPTS, createSeqOwner, readSegmentManifest, snapshotBundleKey, reclaimUnlistedKeys, manifestReferencedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, manifestFileRefs, DIRECTORY_LOCK_SETTLE_MS, readSegmentFile, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, frameRecord, isId, isUint32, isSequenceNumber, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, commitCheckpointManifest, removeCheckpointGarbage, readWalSegments, durableRecordMissing, MAX_SEGMENTS_PER_PARTITION, growPartitionsTo, writeSegmentHeader, VECTOR_FILE_ENVELOPE_VERSION, SEGMENT_HEADER_SIZE, documentsOfPayloadV2, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, metadataConfigOf, buildEntry, readCurrentManifest, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-NNQPZPTB.mjs';
|
|
3
|
+
import { applyDeleteEntry, applyIndexEntry } from './chunk-PA56PRWX.mjs';
|
|
4
|
+
import { encodeDocument, fnv1a, requireValidFieldTypes, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-WZUSMF7A.mjs';
|
|
5
|
+
import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-OL7S2KLL.mjs';
|
|
6
|
+
import { compareCodePoints, MIN_M, MAX_FILE_BYTES, detectRuntime, resolveWorkerEntry } from './chunk-JJKVYLL3.mjs';
|
|
7
7
|
import './chunk-JPTQZWJQ.mjs';
|
|
8
8
|
import { NarsilError, ErrorCodes, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-J3QB4LBD.mjs';
|
|
9
9
|
import { decode } from '@msgpack/msgpack';
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
|
|
2
|
-
import { VECTOR_WORKER_COPIES_ALLOWED, openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, osqBitsOf, noBuildHistory, createOsqQuantizer, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, ordinalFilterValues, handOverBuildTurns, threadsHoldCurrentLayout, filterForOptions, openGraphState, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, fixedView, OSQ4_MIN_DIMENSION, sharedVectorStoreBytes, STORE_SLOTS, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, yieldBuildTurn, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, WORKER_COPY_MIN_VECTORS, allLiveDocIds, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, slotByteOffset, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createSharedVectorStoreHandles, BLOCK_MAX_BYTES, growBufferTo, unplacedChunkCount, STORE_DOC_ID_BYTES, recalibrateFromStore, signatureOf, nativeStoreScratchBytes, noteFallback, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, QUERY_STAGED, createVectorBlock, ensureBlockSlots, createHNSWIndex, adoptGraph, topLayerInParts, BUILD_RETRY_MAX_DELAY_MS, BUILD_RETRY_FIRST_DELAY_MS, fieldHandlesOf, sharedMemoryAvailable, requireWholeNeighbourLists, shuffleInPlace, createCentroidSums, liveOrdinals, RECALIBRATION_CHUNK_VECTORS, BUILD_CHUNK_SIZE, centroidOfCentroidSums, meanOfCentroidSums, countOfCentroidSums, squaredLengthsOfCentroidSums, releaseLocksHeldBy, DRIFT_CHECK_GROWTH_FACTOR, DISK_ADOPTION_YIELD_INTERVAL, adoptLoadedCalibration, RECALIBRATION_DRIFT_RATIO } from './chunk-
|
|
3
|
-
export { releaseLocksHeldBy } from './chunk-
|
|
4
|
-
import { isDead, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors } from './chunk-
|
|
5
|
-
import { MAX_RENUMBERED_SEARCH_ATTEMPTS, DISK_STORAGE_BLOCK_BYTES, compareCodePoints, UNPLACED_FLAG_CHUNK_ORDINALS, MAX_VECTOR_ORDINALS, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, resolveWorkerCount, resolveWorkerEntry, detectRuntime, noteThreadSlotLive, noteThreadSlotGone, WORKER_YOUNG_GENERATION_MB, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_INSERT_TIMEOUT_MS } from './chunk-
|
|
2
|
+
import { VECTOR_WORKER_COPIES_ALLOWED, openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, osqBitsOf, noBuildHistory, createOsqQuantizer, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, ordinalFilterValues, handOverBuildTurns, threadsHoldCurrentLayout, filterForOptions, openGraphState, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, fixedView, OSQ4_MIN_DIMENSION, sharedVectorStoreBytes, STORE_SLOTS, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, yieldBuildTurn, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, WORKER_COPY_MIN_VECTORS, allLiveDocIds, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, slotByteOffset, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createSharedVectorStoreHandles, BLOCK_MAX_BYTES, growBufferTo, unplacedChunkCount, STORE_DOC_ID_BYTES, recalibrateFromStore, signatureOf, nativeStoreScratchBytes, noteFallback, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, QUERY_STAGED, createVectorBlock, ensureBlockSlots, createHNSWIndex, adoptGraph, topLayerInParts, BUILD_RETRY_MAX_DELAY_MS, BUILD_RETRY_FIRST_DELAY_MS, fieldHandlesOf, sharedMemoryAvailable, requireWholeNeighbourLists, shuffleInPlace, createCentroidSums, liveOrdinals, RECALIBRATION_CHUNK_VECTORS, BUILD_CHUNK_SIZE, centroidOfCentroidSums, meanOfCentroidSums, countOfCentroidSums, squaredLengthsOfCentroidSums, releaseLocksHeldBy, DRIFT_CHECK_GROWTH_FACTOR, DISK_ADOPTION_YIELD_INTERVAL, adoptLoadedCalibration, RECALIBRATION_DRIFT_RATIO } from './chunk-OKZGAJEF.mjs';
|
|
3
|
+
export { releaseLocksHeldBy } from './chunk-OKZGAJEF.mjs';
|
|
4
|
+
import { isDead, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors } from './chunk-OL7S2KLL.mjs';
|
|
5
|
+
import { MAX_RENUMBERED_SEARCH_ATTEMPTS, DISK_STORAGE_BLOCK_BYTES, compareCodePoints, UNPLACED_FLAG_CHUNK_ORDINALS, MAX_VECTOR_ORDINALS, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, resolveWorkerCount, resolveWorkerEntry, detectRuntime, noteThreadSlotLive, noteThreadSlotGone, WORKER_YOUNG_GENERATION_MB, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_INSERT_TIMEOUT_MS } from './chunk-JJKVYLL3.mjs';
|
|
6
6
|
import { NarsilError, ErrorCodes, isNarsilError } from './chunk-J3QB4LBD.mjs';
|
|
7
7
|
|
|
8
8
|
// src/core/heap.ts
|
|
@@ -1659,16 +1659,20 @@ function nextHandle(state) {
|
|
|
1659
1659
|
return `${state.indexName}/${state.fieldName}#${handleCounter}`;
|
|
1660
1660
|
}
|
|
1661
1661
|
async function dropHandle(state, handle) {
|
|
1662
|
-
const host = state.workerCopies.host;
|
|
1663
|
-
if (host !== void 0) {
|
|
1664
|
-
await host.drop(state.indexName, state.fieldName, handle).catch(() => void 0);
|
|
1665
|
-
return;
|
|
1666
|
-
}
|
|
1667
1662
|
const pool = state.workerCopyPool;
|
|
1668
1663
|
if (pool !== null) {
|
|
1669
1664
|
await pool.drop(handle).catch(() => void 0);
|
|
1670
|
-
|
|
1665
|
+
return;
|
|
1671
1666
|
}
|
|
1667
|
+
const host = state.workerCopies.host;
|
|
1668
|
+
if (host !== void 0) await host.drop(state.indexName, state.fieldName, handle).catch(() => void 0);
|
|
1669
|
+
}
|
|
1670
|
+
function letGoOfUnusedPool(state) {
|
|
1671
|
+
if (state.workerCopyPool === null || state.sharedHandles.size > 0 || state.workerCopyHandle !== null) {
|
|
1672
|
+
return Promise.resolve();
|
|
1673
|
+
}
|
|
1674
|
+
state.workerCopyPool = null;
|
|
1675
|
+
return releaseVectorSearchPool().catch(() => void 0);
|
|
1672
1676
|
}
|
|
1673
1677
|
function dropSharedGraph(state, graph) {
|
|
1674
1678
|
const shared = state.sharedHandles.get(graph);
|
|
@@ -1691,9 +1695,7 @@ function invalidateWorkerCopies(state) {
|
|
|
1691
1695
|
state.workerCopyMode = null;
|
|
1692
1696
|
void dropHandle(state, handle);
|
|
1693
1697
|
}
|
|
1694
|
-
|
|
1695
|
-
state.workerCopyPool = null;
|
|
1696
|
-
}
|
|
1698
|
+
void letGoOfUnusedPool(state);
|
|
1697
1699
|
}
|
|
1698
1700
|
async function withdrawWorkerCopies(state) {
|
|
1699
1701
|
state.revision += 1;
|
|
@@ -1705,7 +1707,7 @@ async function withdrawWorkerCopies(state) {
|
|
|
1705
1707
|
state.workerCopyRevision = -1;
|
|
1706
1708
|
state.workerCopyMode = null;
|
|
1707
1709
|
await Promise.all([...handles].map((handle) => dropHandle(state, handle)));
|
|
1708
|
-
|
|
1710
|
+
await letGoOfUnusedPool(state);
|
|
1709
1711
|
}
|
|
1710
1712
|
async function holdWorkerCopies(state) {
|
|
1711
1713
|
state.workerCopiesHeld = true;
|
|
@@ -2409,12 +2411,14 @@ function bufferEveryVectorOutsideTheGraph(state) {
|
|
|
2409
2411
|
}
|
|
2410
2412
|
function insertNodesOfLaterGraph(state, graph, later, ordinals) {
|
|
2411
2413
|
for (const part of later.parts) requireWholeNeighbourLists(part);
|
|
2414
|
+
let placed = 0;
|
|
2412
2415
|
for (let number = 0; number < ordinals.length; number++) {
|
|
2413
2416
|
const ordinal = ordinals[number];
|
|
2414
2417
|
if (ordinal === NO_LIVE_VECTOR_AT_POSITION || topLayerInParts(later, number) < 0) continue;
|
|
2415
2418
|
const docId = state.store.docIdForOrdinal(ordinal);
|
|
2416
|
-
if (docId !== void 0 && !graph.has(docId)
|
|
2419
|
+
if (docId !== void 0 && !graph.has(docId) && graph.insertOrdinal(ordinal)) placed += 1;
|
|
2417
2420
|
}
|
|
2421
|
+
if (placed > 0) graph.connectUnreached();
|
|
2418
2422
|
}
|
|
2419
2423
|
function restoreGraph(state, payload, ordinalOfNumber, tombstoned, calibrationLoaded) {
|
|
2420
2424
|
const first = payload?.graphs[0]?.parts[0];
|
|
@@ -18,7 +18,7 @@ function restoreDepsOf(core, host) {
|
|
|
18
18
|
function queueRestore(core, host, restoresInFlight, indexName, data) {
|
|
19
19
|
const previous = restoresInFlight.get(indexName) ?? Promise.resolve();
|
|
20
20
|
const restored = previous.then(async () => {
|
|
21
|
-
const { restoreFromSnapshot } = await import('./snapshot-
|
|
21
|
+
const { restoreFromSnapshot } = await import('./snapshot-G77LQBZX.mjs');
|
|
22
22
|
await restoreFromSnapshot(indexName, data, restoreDepsOf(core, host));
|
|
23
23
|
});
|
|
24
24
|
const settled = restored.then(
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { vectorsPerFile } from './chunk-
|
|
2
|
-
import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-
|
|
3
|
-
import './chunk-
|
|
4
|
-
import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, prepareStoredSynonyms, rejectSnapshot, readHeapStatistics, readResidentMemory, extractVectorFieldsFromSchema, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, insertDocumentVectors, decodeIndexConfiguration, flattenSchema } from './chunk-
|
|
5
|
-
import { decodeVectorIndexParts, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors } from './chunk-
|
|
6
|
-
import { MAX_FILE_BYTES } from './chunk-
|
|
1
|
+
import { vectorsPerFile } from './chunk-OKZGAJEF.mjs';
|
|
2
|
+
import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-NNQPZPTB.mjs';
|
|
3
|
+
import './chunk-PA56PRWX.mjs';
|
|
4
|
+
import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, prepareStoredSynonyms, rejectSnapshot, readHeapStatistics, readResidentMemory, extractVectorFieldsFromSchema, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, insertDocumentVectors, decodeIndexConfiguration, flattenSchema } from './chunk-WZUSMF7A.mjs';
|
|
5
|
+
import { decodeVectorIndexParts, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors } from './chunk-OL7S2KLL.mjs';
|
|
6
|
+
import { MAX_FILE_BYTES } from './chunk-JJKVYLL3.mjs';
|
|
7
7
|
import './chunk-JPTQZWJQ.mjs';
|
|
8
8
|
import { NarsilError, ErrorCodes } from './chunk-J3QB4LBD.mjs';
|
|
9
9
|
import { decode } from '@msgpack/msgpack';
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-
|
|
2
|
-
import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-
|
|
3
|
-
import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-
|
|
4
|
-
import './chunk-
|
|
1
|
+
import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-PA56PRWX.mjs';
|
|
2
|
+
import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-WZUSMF7A.mjs';
|
|
3
|
+
import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-OL7S2KLL.mjs';
|
|
4
|
+
import './chunk-JJKVYLL3.mjs';
|
|
5
5
|
import './chunk-JPTQZWJQ.mjs';
|
|
6
6
|
import { EXPORT_CHUNK_BYTES, NarsilError, ErrorCodes } from './chunk-J3QB4LBD.mjs';
|
|
7
7
|
import { decode } from '@msgpack/msgpack';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { E as EmbeddingAdapter, R as RerankAdapter } from './adapters-BQFQ4PRz.js';
|
|
2
|
-
import { C as ClusterNamespace } from './config-
|
|
2
|
+
import { C as ClusterNamespace } from './config-BZ_e3X8w.js';
|
|
3
3
|
import { h as TaskStore } from './tasks-BhjQZ8Qf.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { requireDocumentWithinSizeLimit, validateRequiredFields } from './chunk-
|
|
1
|
+
import { requireDocumentWithinSizeLimit, validateRequiredFields } from './chunk-VVFKFHWE.mjs';
|
|
2
2
|
import { ErrorCodes, NarsilError } from './chunk-6JUOMMHQ.mjs';
|
|
3
3
|
|
|
4
4
|
// src/narsil/internals.ts
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { PROCESS_LIMIT_REFRESH_INTERVAL_MS } from './chunk-ZXZ26WSO.mjs';
|
|
2
|
+
import { totalmem, freemem } from 'os';
|
|
2
3
|
import { getHeapSpaceStatistics, getHeapStatistics } from 'v8';
|
|
3
4
|
|
|
4
|
-
// src/runtime/heap-statistics.ts
|
|
5
5
|
var MEGABYTE_BYTES = 1048576;
|
|
6
6
|
var IN_MEGABYTES = /^--max-old-space-size=(\d+)$/;
|
|
7
7
|
var AS_PERCENTAGE = /^--max-old-space-size-percentage=(\d+(?:\.\d+)?)$/;
|
|
@@ -72,9 +72,9 @@ function readOldGenerationStatistics() {
|
|
|
72
72
|
usedBytes += space.space_used_size;
|
|
73
73
|
}
|
|
74
74
|
}
|
|
75
|
-
const
|
|
76
|
-
if (!Number.isFinite(usedBytes) || !Number.isFinite(
|
|
77
|
-
return { usedBytes, limitBytes, youngUsedBytes: Number.isFinite(youngUsedBytes) ? youngUsedBytes : 0 };
|
|
75
|
+
const limitBytes2 = configuredLimitBytes() ?? heapSizeLimitBytes() - (configuredYoungGenerationBytes() ?? youngCapacityBytes);
|
|
76
|
+
if (!Number.isFinite(usedBytes) || !Number.isFinite(limitBytes2) || limitBytes2 <= 0) return null;
|
|
77
|
+
return { usedBytes, limitBytes: limitBytes2, youngUsedBytes: Number.isFinite(youngUsedBytes) ? youngUsedBytes : 0 };
|
|
78
78
|
} catch {
|
|
79
79
|
return null;
|
|
80
80
|
}
|
|
@@ -83,14 +83,43 @@ function readHeapStatistics() {
|
|
|
83
83
|
try {
|
|
84
84
|
const stats = getHeapStatistics();
|
|
85
85
|
const usedBytes = stats.used_heap_size;
|
|
86
|
-
const
|
|
87
|
-
if (!Number.isFinite(usedBytes) || !Number.isFinite(
|
|
86
|
+
const limitBytes2 = stats.heap_size_limit;
|
|
87
|
+
if (!Number.isFinite(usedBytes) || !Number.isFinite(limitBytes2) || limitBytes2 <= 0) return null;
|
|
88
88
|
const available = stats.total_available_size;
|
|
89
89
|
const availableBytes = Number.isFinite(available) && available >= 0 ? available : null;
|
|
90
|
-
return { usedBytes, limitBytes, availableBytes, configuredLimitBytes: configuredLimitBytes() };
|
|
90
|
+
return { usedBytes, limitBytes: limitBytes2, availableBytes, configuredLimitBytes: configuredLimitBytes() };
|
|
91
|
+
} catch {
|
|
92
|
+
return null;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
var limitBytes = 0;
|
|
96
|
+
var limitReadAt = Number.NEGATIVE_INFINITY;
|
|
97
|
+
function readProcessLimit(residentBytes) {
|
|
98
|
+
const hostBytes = totalmem();
|
|
99
|
+
const constrained = typeof process.constrainedMemory === "function" ? process.constrainedMemory() : 0;
|
|
100
|
+
if (Number.isFinite(constrained) && constrained > 0 && constrained < hostBytes) return constrained;
|
|
101
|
+
if (process.platform !== "linux") return hostBytes;
|
|
102
|
+
const availableBytes = freemem();
|
|
103
|
+
if (!Number.isFinite(availableBytes) || availableBytes < 0) return hostBytes;
|
|
104
|
+
return Math.min(hostBytes, residentBytes + availableBytes);
|
|
105
|
+
}
|
|
106
|
+
function processLimitBytes(residentBytes) {
|
|
107
|
+
const now = performance.now();
|
|
108
|
+
if (now - limitReadAt >= PROCESS_LIMIT_REFRESH_INTERVAL_MS) {
|
|
109
|
+
limitBytes = readProcessLimit(residentBytes);
|
|
110
|
+
limitReadAt = now;
|
|
111
|
+
}
|
|
112
|
+
return limitBytes;
|
|
113
|
+
}
|
|
114
|
+
function readResidentMemory() {
|
|
115
|
+
try {
|
|
116
|
+
const residentBytes = process.memoryUsage.rss();
|
|
117
|
+
const limitBytes2 = processLimitBytes(residentBytes);
|
|
118
|
+
if (!Number.isFinite(residentBytes) || !Number.isFinite(limitBytes2) || limitBytes2 <= 0) return null;
|
|
119
|
+
return { residentBytes, limitBytes: limitBytes2 };
|
|
91
120
|
} catch {
|
|
92
121
|
return null;
|
|
93
122
|
}
|
|
94
123
|
}
|
|
95
124
|
|
|
96
|
-
export { readHeapStatistics, readOldGenerationStatistics };
|
|
125
|
+
export { processLimitBytes, readHeapStatistics, readOldGenerationStatistics, readResidentMemory };
|
|
@@ -1,19 +1,18 @@
|
|
|
1
|
-
import { requirePatternSearchFor, requirePartialWordSearchFor, createPartitionIndex, buildPartialWordTableFrom, recordValueSize, patternIndexOf, encodeFieldIndexes, encodePartialSpellingsOf, countPartialSpelling, copyVectorsInto, isCompositePartition, fnv1a, partialSpellingsOfDocuments, createFrozenPartialWords, requirePatternSearch, documentValueBytes, cloneProjected, projectIncludedPaths, valueWithOwnBuffers, createSurfaceRegistry, MAX_TERM_FREQUENCY, joinPatternWorkMeter, fanOutQuery, sanitizeGlobalStats, countsWithoutScores, fanOutMatchCount, createCompositePartition, createGeoIndex, documentWithOwnBuffers, readProjectedPath, createNumericIndex, createBooleanIndex, createEnumIndex, ValueAffixes } from './chunk-
|
|
1
|
+
import { requirePatternSearchFor, requirePartialWordSearchFor, createPartitionIndex, buildPartialWordTableFrom, recordValueSize, patternIndexOf, encodeFieldIndexes, encodePartialSpellingsOf, countPartialSpelling, copyVectorsInto, isCompositePartition, fnv1a, partialSpellingsOfDocuments, createFrozenPartialWords, requirePatternSearch, documentValueBytes, cloneProjected, projectIncludedPaths, valueWithOwnBuffers, createSurfaceRegistry, MAX_TERM_FREQUENCY, joinPatternWorkMeter, fanOutQuery, sanitizeGlobalStats, countsWithoutScores, fanOutMatchCount, createCompositePartition, createGeoIndex, documentWithOwnBuffers, readProjectedPath, createNumericIndex, createBooleanIndex, createEnumIndex, ValueAffixes } from './chunk-FEOLYA2Y.mjs';
|
|
2
2
|
import { generateId, getLanguage } from './chunk-QALAFGF5.mjs';
|
|
3
|
-
import { readHeapStatistics } from './chunk-
|
|
3
|
+
import { readHeapStatistics, readResidentMemory } from './chunk-5FPMDU54.mjs';
|
|
4
4
|
import { GrowableUint32, bitsetHas, bitsetSet, createBitSet } from './chunk-2MNJ2HX7.mjs';
|
|
5
|
-
import { tokenizeOptions, getNestedValue, sameFields, visitDocumentSurfaceSpellings, visitDocumentSpellings } from './chunk-
|
|
6
|
-
import { schemaFieldsOf, partialFieldsOf, encodeDocument, validateDocument, validateDocumentStrict, extractVectorFieldsFromSchema } from './chunk-
|
|
5
|
+
import { tokenizeOptions, getNestedValue, sameFields, visitDocumentSurfaceSpellings, visitDocumentSpellings } from './chunk-WXVTQ6NO.mjs';
|
|
6
|
+
import { schemaFieldsOf, partialFieldsOf, encodeDocument, validateDocument, validateDocumentStrict, extractVectorFieldsFromSchema } from './chunk-VVFKFHWE.mjs';
|
|
7
7
|
import { fuzzyPrefixOf, boundedLevenshtein } from './chunk-GES7U2PM.mjs';
|
|
8
8
|
import { isWordIndexedBase } from './chunk-77SO7G55.mjs';
|
|
9
9
|
import { resolveIndexAnalysis, synonymSearchModule } from './chunk-RI5VNFAI.mjs';
|
|
10
10
|
import { visitTokens, codePointOrder, encodeStringBlob, encodeLookupKey, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, FROZEN_REMOVAL_INDEX_MAX_PROCESS_FRACTION } from './chunk-AEF2O7B4.mjs';
|
|
11
11
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
12
|
-
import { loadedVectorCode, loadVectorCode, loadVectorCodeFor, requireLoadedVectorCode } from './chunk-
|
|
12
|
+
import { loadedVectorCode, loadVectorCode, loadVectorCodeFor, requireLoadedVectorCode } from './chunk-IA6I5U36.mjs';
|
|
13
13
|
import { PROTOTYPE_POLLUTION_KEYS } from './chunk-L5ORED7V.mjs';
|
|
14
14
|
import { NarsilError, ErrorCodes } from './chunk-6JUOMMHQ.mjs';
|
|
15
15
|
import { Decoder, decode } from '@msgpack/msgpack';
|
|
16
|
-
import { totalmem } from 'os';
|
|
17
16
|
|
|
18
17
|
// src/core/synonyms/platform.ts
|
|
19
18
|
function platformSynonymSearch() {
|
|
@@ -1393,21 +1392,6 @@ function createFrozenInvertedReader(table, views, counts) {
|
|
|
1393
1392
|
}
|
|
1394
1393
|
};
|
|
1395
1394
|
}
|
|
1396
|
-
function processLimitBytes() {
|
|
1397
|
-
const hostBytes = totalmem();
|
|
1398
|
-
const constrained = typeof process.constrainedMemory === "function" ? process.constrainedMemory() : 0;
|
|
1399
|
-
return Number.isFinite(constrained) && constrained > 0 && constrained < hostBytes ? constrained : hostBytes;
|
|
1400
|
-
}
|
|
1401
|
-
function readResidentMemory() {
|
|
1402
|
-
try {
|
|
1403
|
-
const residentBytes = process.memoryUsage.rss();
|
|
1404
|
-
const limitBytes = processLimitBytes();
|
|
1405
|
-
if (!Number.isFinite(residentBytes) || !Number.isFinite(limitBytes) || limitBytes <= 0) return null;
|
|
1406
|
-
return { residentBytes, limitBytes };
|
|
1407
|
-
} catch {
|
|
1408
|
-
return null;
|
|
1409
|
-
}
|
|
1410
|
-
}
|
|
1411
1395
|
|
|
1412
1396
|
// src/core/partition/frozen/live-counts.ts
|
|
1413
1397
|
var UINT32_BYTES = 4;
|
|
@@ -3534,4 +3518,4 @@ function createDirectExecutor(options) {
|
|
|
3534
3518
|
};
|
|
3535
3519
|
}
|
|
3536
3520
|
|
|
3537
|
-
export { availableSynonymModule, buildSegmentPayload, createDirectExecutor, createFrozenSegment, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeMapWithoutFields, decodeStoredDocument, freezeSegmentShared, isEmptySynonymSet, mergeFrozenSegments, mergeSegmentContents,
|
|
3521
|
+
export { availableSynonymModule, buildSegmentPayload, createDirectExecutor, createFrozenSegment, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeMapWithoutFields, decodeStoredDocument, freezeSegmentShared, isEmptySynonymSet, mergeFrozenSegments, mergeSegmentContents, registerSynonyms, resolvePartitionInsertOptions, segmentArraysOf, selectionKeeps, shareSegmentArrays, synonymModule };
|