@delali/narsil 0.3.1-next.112 → 0.3.1-next.113
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/{chunk-YRYBTRWB.mjs → chunk-3YPLBK4C.mjs} +2 -2
- package/dist/browser/{chunk-O5IKTA27.mjs → chunk-4FVSBSXY.mjs} +1 -1
- package/dist/browser/{chunk-UR5MNN53.mjs → chunk-DC2SZO4D.mjs} +7 -6
- package/dist/browser/{chunk-OVY4RPDS.mjs → chunk-YZPOBLAS.mjs} +35 -1
- package/dist/browser/{durability-wiring-CPEMDDK7.mjs → durability-wiring-2RG22WQJ.mjs} +3 -3
- package/dist/browser/{module-DYKR4INA.mjs → module-5EWHBKHZ.mjs} +102 -11
- package/dist/browser/{restore-queue-2H4QDQV4.mjs → restore-queue-WCFUB4NS.mjs} +1 -1
- package/dist/browser/{snapshot-ISZMBDCN.mjs → snapshot-BE3F2PNC.mjs} +6 -5
- package/dist/browser/{write-IABOLE4N.mjs → write-MV7K7OCU.mjs} +2 -2
- package/dist/{capabilities-ByaX8Arf.d.ts → capabilities-DOgB3pPf.d.ts} +2 -2
- package/dist/{chunk-ZXZ26WSO.mjs → chunk-22YPZVMG.mjs} +5 -1
- package/dist/{chunk-2IFU7SXG.mjs → chunk-27ZEY73B.mjs} +3 -3
- package/dist/{chunk-QI6JW4PM.mjs → chunk-4PFOOQFQ.mjs} +1 -1
- package/dist/{chunk-YIDSHBQS.mjs → chunk-4XWIW3F3.mjs} +29 -1
- package/dist/{chunk-V7ISFBFC.mjs → chunk-5QIRLJH4.mjs} +2 -2
- package/dist/{chunk-63HFWQEA.mjs → chunk-AOLD3LPV.mjs} +273 -102
- package/dist/{chunk-PT7VRWS7.mjs → chunk-CU4OB64V.mjs} +3 -2
- package/dist/{chunk-BVOUSX3U.mjs → chunk-DZXGNLK5.mjs} +3 -3
- package/dist/chunk-EBIHRL7B.mjs +21 -0
- package/dist/{chunk-R2MRGYU5.mjs → chunk-H2GNWBLB.mjs} +22 -2
- package/dist/{chunk-V7G627TA.mjs → chunk-HFJXEFFV.mjs} +13 -62
- package/dist/chunk-IWVEG5CX.mjs +67 -0
- package/dist/{chunk-VLRT47GU.mjs → chunk-JUAFWJON.mjs} +3 -3
- package/dist/{chunk-3COL24MT.mjs → chunk-LVNBY2TS.mjs} +1 -1
- package/dist/{chunk-IM76DLA7.mjs → chunk-OABTAUCW.mjs} +2 -2
- package/dist/{chunk-IBTKJIV7.mjs → chunk-OM5G2J6X.mjs} +1 -1
- package/dist/{chunk-3RRATITJ.mjs → chunk-PABYSTKO.mjs} +11 -8
- package/dist/chunk-PELHVTO7.mjs +24 -0
- package/dist/{chunk-7CI34JAP.mjs → chunk-QGVMEMCG.mjs} +4 -4
- package/dist/{chunk-6OXNHLNZ.mjs → chunk-RIIUZJQ3.mjs} +1 -1
- package/dist/{chunk-C7SLA6CM.mjs → chunk-W55X2BYN.mjs} +5 -1
- package/dist/{chunk-SOADQINJ.mjs → chunk-ZATRDAYU.mjs} +31 -3
- package/dist/{client-BdnzDFtO.d.ts → client-BGzLNt78.d.ts} +2 -2
- package/dist/client.d.ts +6 -6
- package/dist/{config-ujZESkJz.d.ts → config-CS5ADSrx.d.ts} +18 -2
- package/dist/{config-CSgvApt0.d.ts → config-CSYMTTm1.d.ts} +1 -1
- package/dist/{direct-executor-ZSTKRA6Z.mjs → direct-executor-R4HOSVPE.mjs} +5 -4
- package/dist/distribution.d.ts +5 -5
- package/dist/distribution.mjs +72 -35
- package/dist/{durability-wiring-EQTFL3B4.mjs → durability-wiring-XZI6355I.mjs} +13 -12
- package/dist/{engine-C_HSy3iz.d.ts → engine-CCBBGmMR.d.ts} +1 -1
- package/dist/{events-CJsXF1F9.d.ts → events-D8GPB1-H.d.ts} +5 -0
- package/dist/index.browser.mjs +1 -1
- package/dist/index.d.ts +5 -5
- package/dist/index.mjs +6 -5
- package/dist/{module-WP4VVGV2.mjs → module-WVKSJL74.mjs} +67 -7
- package/dist/persistence/durability/checkpoint-worker.mjs +10 -9
- package/dist/react.d.ts +6 -6
- package/dist/{restore-queue-WZTMTJP2.mjs → restore-queue-JAZAB5IQ.mjs} +1 -1
- package/dist/server.d.ts +8 -8
- package/dist/server.mjs +20 -62
- package/dist/{snapshot-EU6NNB2P.mjs → snapshot-PDUGHQ25.mjs} +14 -13
- package/dist/{tasks-CI3Fvkxw.d.ts → tasks-5_LCFEzF.d.ts} +1 -1
- package/dist/{thread-WGBCVJII.mjs → thread-S4SUCV4Y.mjs} +8 -5
- package/dist/vector/search-worker.mjs +2 -2
- package/dist/workers/entry.mjs +6 -6
- package/dist/{write-CKRWS3P5.mjs → write-KVJ7YJJ6.mjs} +7 -6
- package/package.json +7 -7
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes, uint64Bytes } from './chunk-
|
|
2
|
-
import { VERSION, rulesOfStoredSynonyms, flattenSchema, packChecksummedEnvelope, openChecksummedEnvelope, requireValidFieldTypes, isCompositePartition, crc32, reconstructSchemaFromMetadata, extractVectorFieldsFromSchema, readHeader, HEADER_SIZE, engineVersionParts, writeHeader, partialFieldsOf, getLanguage, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, VECTOR_PATTERN, isQuantizationMode, isStorageMode, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, fnv1a, selectionKeeps, decodeMapWithoutFields, prepareDocumentVectors, messagePackUpperBoundBytes, segmentArraysOf, mergeSegmentContents, resolveIndexAnalysis } from './chunk-
|
|
1
|
+
import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes, uint64Bytes } from './chunk-4FVSBSXY.mjs';
|
|
2
|
+
import { VERSION, rulesOfStoredSynonyms, flattenSchema, packChecksummedEnvelope, openChecksummedEnvelope, requireValidFieldTypes, isCompositePartition, crc32, reconstructSchemaFromMetadata, extractVectorFieldsFromSchema, readHeader, HEADER_SIZE, engineVersionParts, writeHeader, partialFieldsOf, getLanguage, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, VECTOR_PATTERN, isQuantizationMode, isStorageMode, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, fnv1a, selectionKeeps, decodeMapWithoutFields, prepareDocumentVectors, messagePackUpperBoundBytes, segmentArraysOf, mergeSegmentContents, resolveIndexAnalysis } from './chunk-DC2SZO4D.mjs';
|
|
3
3
|
import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-P7C2S5JX.mjs';
|
|
4
4
|
import { compareCodePoints, MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-2CBQ2PCC.mjs';
|
|
5
5
|
import { NarsilError, ErrorCodes } from './chunk-LJS6PREA.mjs';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors, getNestedValue, requireValidFieldTypes, VECTOR_PATTERN } from './chunk-
|
|
1
|
+
import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors, getNestedValue, requireValidFieldTypes, VECTOR_PATTERN } from './chunk-DC2SZO4D.mjs';
|
|
2
2
|
import { NarsilError, ErrorCodes } from './chunk-LJS6PREA.mjs';
|
|
3
3
|
import { decode } from '@msgpack/msgpack';
|
|
4
4
|
|
|
@@ -15087,7 +15087,7 @@ function requireLoadedVectorCode() {
|
|
|
15087
15087
|
function loadVectorCode() {
|
|
15088
15088
|
if (loaded !== null) return Promise.resolve(loaded);
|
|
15089
15089
|
if (loading === null) {
|
|
15090
|
-
loading = import('./module-
|
|
15090
|
+
loading = import('./module-5EWHBKHZ.mjs').then(
|
|
15091
15091
|
(code) => {
|
|
15092
15092
|
loaded = code;
|
|
15093
15093
|
return code;
|
|
@@ -19175,7 +19175,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
19175
19175
|
if (config?.lifecycle !== void 0 && durabilityTier === null) {
|
|
19176
19176
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
|
|
19177
19177
|
}
|
|
19178
|
-
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-
|
|
19178
|
+
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-2RG22WQJ.mjs');
|
|
19179
19179
|
const filesystemDurability = durabilityTier?.kind === "wal";
|
|
19180
19180
|
const executor = createDirectExecutor({
|
|
19181
19181
|
vectorWorkerCopies: vectorCopyPolicy,
|
|
@@ -21919,6 +21919,7 @@ function getVectorMaintenanceStatus(manager) {
|
|
|
21919
21919
|
graphCount: status.graphCount,
|
|
21920
21920
|
bufferSize: status.bufferSize,
|
|
21921
21921
|
building: status.building,
|
|
21922
|
+
waitingForMemory: status.waitingForMemory,
|
|
21922
21923
|
estimatedCompactMs: status.estimatedCompactMs,
|
|
21923
21924
|
estimatedOptimizeMs: status.estimatedOptimizeMs
|
|
21924
21925
|
});
|
|
@@ -22608,14 +22609,14 @@ function portabilityOperations(core, config, engine) {
|
|
|
22608
22609
|
return {
|
|
22609
22610
|
snapshot(indexName) {
|
|
22610
22611
|
return withOpenIndex(indexName, async () => {
|
|
22611
|
-
const { createSnapshot } = await import('./snapshot-
|
|
22612
|
+
const { createSnapshot } = await import('./snapshot-BE3F2PNC.mjs');
|
|
22612
22613
|
return createSnapshot(snapshotDepsOf(indexName));
|
|
22613
22614
|
});
|
|
22614
22615
|
},
|
|
22615
22616
|
async restore(indexName, data) {
|
|
22616
22617
|
core.guardShutdown();
|
|
22617
22618
|
validateIndexName(indexName);
|
|
22618
|
-
const restored = import('./restore-queue-
|
|
22619
|
+
const restored = import('./restore-queue-WCFUB4NS.mjs').then(
|
|
22619
22620
|
({ queueRestore }) => queueRestore(core, restoreHost, restoresInFlight, indexName, data)
|
|
22620
22621
|
);
|
|
22621
22622
|
await emitWriteEventAfter(core, indexName, restored);
|
|
@@ -22624,8 +22625,8 @@ function portabilityOperations(core, config, engine) {
|
|
|
22624
22625
|
exportDocuments(indexName) {
|
|
22625
22626
|
return withOpenIndex(indexName, async () => {
|
|
22626
22627
|
const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
|
|
22627
|
-
import('./snapshot-
|
|
22628
|
-
import('./write-
|
|
22628
|
+
import('./snapshot-BE3F2PNC.mjs'),
|
|
22629
|
+
import('./write-MV7K7OCU.mjs')
|
|
22629
22630
|
]);
|
|
22630
22631
|
const header = indexConfigurationOf(core.requireIndex(indexName));
|
|
22631
22632
|
return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
|
|
@@ -19,6 +19,15 @@ function createFixedBuffer(bytes) {
|
|
|
19
19
|
function nextGrowthTarget(have, needed, max) {
|
|
20
20
|
return Math.min(max, Math.max(needed, have * 2));
|
|
21
21
|
}
|
|
22
|
+
function grownByteLength(have, needed, max, step = 1) {
|
|
23
|
+
let size = have;
|
|
24
|
+
while (size < needed) {
|
|
25
|
+
const next = Math.ceil(nextGrowthTarget(size, Math.min(needed, size + 1), max) / step) * step;
|
|
26
|
+
if (next <= size) return size;
|
|
27
|
+
size = next;
|
|
28
|
+
}
|
|
29
|
+
return size;
|
|
30
|
+
}
|
|
22
31
|
function growBufferTo(buffer, bytes) {
|
|
23
32
|
const held = buffer.byteLength;
|
|
24
33
|
if (held >= bytes) return;
|
|
@@ -1115,6 +1124,25 @@ function ensureBlockSlots(handle, slots) {
|
|
|
1115
1124
|
}
|
|
1116
1125
|
growMemoryTo(storage.kind === "shared-memory" ? storage.memory : storage.simd.memory, bytes, handle.layout);
|
|
1117
1126
|
}
|
|
1127
|
+
function blockGrowthBytes(handle, layout, slots) {
|
|
1128
|
+
const bytes = layout.slotsOffset + Math.min(slots, layout.capacity) * layout.slotStride;
|
|
1129
|
+
if (handle !== null && handle.storage.kind === "bytes") {
|
|
1130
|
+
const buffer = handle.storage.buffer;
|
|
1131
|
+
return grownByteLength(buffer.byteLength, bytes, buffer.maxByteLength) - buffer.byteLength;
|
|
1132
|
+
}
|
|
1133
|
+
const have = handle === null ? 0 : blockBuffer(handle).byteLength;
|
|
1134
|
+
const start = handle === null ? pagesFor(initialBlockBytes(layout)) * WASM_PAGE_BYTES : have;
|
|
1135
|
+
return grownByteLength(start, bytes, fullBlockBytes(layout), WASM_PAGE_BYTES) - have;
|
|
1136
|
+
}
|
|
1137
|
+
function blocksGrowthBytes(blocks, layout, firstOrdinal, lastOrdinal) {
|
|
1138
|
+
let bytes = 0;
|
|
1139
|
+
const lastBlock = Math.floor(lastOrdinal / layout.capacity);
|
|
1140
|
+
for (let block = Math.floor(firstOrdinal / layout.capacity); block <= lastBlock; block++) {
|
|
1141
|
+
const slots = block === lastBlock ? lastOrdinal % layout.capacity + 1 : layout.capacity;
|
|
1142
|
+
bytes += blockGrowthBytes(blocks[block] ?? null, layout, slots);
|
|
1143
|
+
}
|
|
1144
|
+
return bytes;
|
|
1145
|
+
}
|
|
1118
1146
|
function blockBuffer(handle) {
|
|
1119
1147
|
const storage = handle.storage;
|
|
1120
1148
|
if (storage.kind === "bytes") return storage.buffer;
|
|
@@ -3458,6 +3486,7 @@ var BUILD_CHUNK_SIZE = 100;
|
|
|
3458
3486
|
var BUILD_TURN_HANDOVER_AFTER_MS = 50;
|
|
3459
3487
|
var BUILD_RETRY_FIRST_DELAY_MS = 250;
|
|
3460
3488
|
var BUILD_RETRY_MAX_DELAY_MS = 3e4;
|
|
3489
|
+
var SHUFFLED_REBUILD_MEMORY_RETRY_MS = 1e3;
|
|
3461
3490
|
var REBUILD_REMOVED_RATIO = 0.2;
|
|
3462
3491
|
var DRIFT_CHECK_GROWTH_FACTOR = 2;
|
|
3463
3492
|
var RECALIBRATION_DRIFT_RATIO = 2.5;
|
|
@@ -4480,4 +4509,9 @@ function recordCheckpoint(state, plan, written) {
|
|
|
4480
4509
|
state.savedSignature = plan.signature;
|
|
4481
4510
|
}
|
|
4482
4511
|
|
|
4483
|
-
|
|
4512
|
+
// src/runtime/idle-heap-cleanup.browser.ts
|
|
4513
|
+
function collectAndReturnPages() {
|
|
4514
|
+
return false;
|
|
4515
|
+
}
|
|
4516
|
+
|
|
4517
|
+
export { BLOCK_MAX_BYTES, BUILD_CHUNK_SIZE, BUILD_RETRY_FIRST_DELAY_MS, BUILD_RETRY_MAX_DELAY_MS, DEFAULT_FILTER_THRESHOLD, DEFAULT_PROMOTION_THRESHOLD, DISK_ADOPTION_YIELD_INTERVAL, DRIFT_CHECK_GROWTH_FACTOR, ESTIMATED_MS_PER_TOMBSTONE, ESTIMATED_MS_PER_VECTOR_REBUILD, GRAPH_RETIRED, IN_MEMORY, NOTHING_STAGED, OSQ4_MIN_DIMENSION, QUERY_STAGED, RECALIBRATION_CHUNK_VECTORS, RECALIBRATION_DRIFT_RATIO, SHUFFLED_REBUILD_MEMORY_RETRY_MS, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, STORE_DOC_ID_BYTES, STORE_LIVE_COUNT, STORE_ORDINAL_LAYOUT_GENERATION, STORE_RELEASED_VECTORS_TO_DISK, STORE_REMOVED_COUNT, STORE_SLOTS, STORE_UNPLACED_COUNT, VECTOR_WORKER_COPIES_ALLOWED, WORKER_COPY_MIN_VECTORS, addToOrdinalFilter, adoptGraph, adoptLoadedCalibration, allLiveDocIds, arenaFloat32Distance, assignStorePartitions, blocksGrowthBytes, centroidOfCentroidSums, collectAndReturnPages, cosineSimilarityWithMagnitudes, countOfCentroidSums, createCentroidSums, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createSharedVectorStoreHandles, createVectorBlock, deserialize, detachNativeStore, dotProduct, emptyFieldBeforeRestore, ensureBlockSlots, euclideanDistance, fieldHandlesOf, filterForOptions, fixedView, graphNeedsRebuild, growBufferTo, grownByteLength, handOverBuildTurns, insertNode, liveOrdinals, liveSize, magnitude, meanOfCentroidSums, nativeScoresOf, nativeStoreScratchBytes, noBuildHistory, nodeCountOf, noteFallback, openGraphState, openSharedQuantizer, openVectorBlock, ordinalFilterForSubsets, ordinalFilterHas, ordinalFilterValues, osqBitsOf, planCheckpoint, recalibrateFromStore, recordCheckpoint, releaseLocksHeldBy, releaseReplacedVectors, requireWholeNeighbourLists, resetStoreHeader, searchOrdinals, sharedMemoryAvailable, sharedVectorStoreBytes, shuffleInPlace, signatureOf, slotByteOffset, squaredLengthsOfCentroidSums, threadsHoldCurrentLayout, toDistance, toScore, tombstoneCountOf, topLayerInParts, unplacedChunkCount, vectorsPerFile, yieldBuildTurn };
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { nodeThreadId, spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
|
|
2
|
-
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, decodeSegmentManifest, readListedFile, adapterCheckpointStore, SINGLE_NODE_PRIMARY_TERM, readMetadataEnvelope, writeMetadataEnvelope, metadataFromConfig, DIRECTORY_LOCK_ATTEMPTS, createSeqOwner, readSegmentManifest, snapshotBundleKey, reclaimUnlistedKeys, manifestReferencedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, manifestFileRefs, DIRECTORY_LOCK_SETTLE_MS, readSegmentFile, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, frameRecord, isId, isUint32, isSequenceNumber, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, commitCheckpointManifest, removeCheckpointGarbage, readWalSegments, durableRecordMissing, MAX_SEGMENTS_PER_PARTITION, growPartitionsTo, writeSegmentHeader, VECTOR_FILE_ENVELOPE_VERSION, SEGMENT_HEADER_SIZE, documentsOfPayloadV2, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, metadataConfigOf, buildEntry, readCurrentManifest, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-
|
|
3
|
-
import { applyDeleteEntry, applyIndexEntry } from './chunk-
|
|
4
|
-
import { encodeDocument, fnv1a, requireValidFieldTypes, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-
|
|
2
|
+
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, decodeSegmentManifest, readListedFile, adapterCheckpointStore, SINGLE_NODE_PRIMARY_TERM, readMetadataEnvelope, writeMetadataEnvelope, metadataFromConfig, DIRECTORY_LOCK_ATTEMPTS, createSeqOwner, readSegmentManifest, snapshotBundleKey, reclaimUnlistedKeys, manifestReferencedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, manifestFileRefs, DIRECTORY_LOCK_SETTLE_MS, readSegmentFile, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, frameRecord, isId, isUint32, isSequenceNumber, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, commitCheckpointManifest, removeCheckpointGarbage, readWalSegments, durableRecordMissing, MAX_SEGMENTS_PER_PARTITION, growPartitionsTo, writeSegmentHeader, VECTOR_FILE_ENVELOPE_VERSION, SEGMENT_HEADER_SIZE, documentsOfPayloadV2, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, metadataConfigOf, buildEntry, readCurrentManifest, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-3YPLBK4C.mjs';
|
|
3
|
+
import { applyDeleteEntry, applyIndexEntry } from './chunk-4FVSBSXY.mjs';
|
|
4
|
+
import { encodeDocument, fnv1a, requireValidFieldTypes, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-DC2SZO4D.mjs';
|
|
5
5
|
import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-P7C2S5JX.mjs';
|
|
6
6
|
import { compareCodePoints, MIN_M, MAX_FILE_BYTES, detectRuntime, resolveWorkerEntry } from './chunk-2CBQ2PCC.mjs';
|
|
7
7
|
import './chunk-JPTQZWJQ.mjs';
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
|
|
2
|
-
import { VECTOR_WORKER_COPIES_ALLOWED, openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, osqBitsOf, noBuildHistory, createOsqQuantizer, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, ordinalFilterValues, handOverBuildTurns, threadsHoldCurrentLayout, filterForOptions, openGraphState, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, fixedView, OSQ4_MIN_DIMENSION, sharedVectorStoreBytes, STORE_SLOTS, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, yieldBuildTurn, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, WORKER_COPY_MIN_VECTORS, allLiveDocIds, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, slotByteOffset, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createSharedVectorStoreHandles, BLOCK_MAX_BYTES,
|
|
3
|
-
export { releaseLocksHeldBy } from './chunk-
|
|
2
|
+
import { VECTOR_WORKER_COPIES_ALLOWED, openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, osqBitsOf, noBuildHistory, createOsqQuantizer, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, ordinalFilterValues, handOverBuildTurns, threadsHoldCurrentLayout, filterForOptions, openGraphState, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, fixedView, OSQ4_MIN_DIMENSION, sharedVectorStoreBytes, STORE_SLOTS, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, yieldBuildTurn, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, WORKER_COPY_MIN_VECTORS, allLiveDocIds, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, slotByteOffset, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createSharedVectorStoreHandles, BLOCK_MAX_BYTES, STORE_DOC_ID_BYTES, unplacedChunkCount, blocksGrowthBytes, growBufferTo, SHUFFLED_REBUILD_MEMORY_RETRY_MS, recalibrateFromStore, signatureOf, nativeStoreScratchBytes, noteFallback, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, QUERY_STAGED, grownByteLength, createVectorBlock, ensureBlockSlots, createHNSWIndex, adoptGraph, topLayerInParts, BUILD_RETRY_MAX_DELAY_MS, BUILD_RETRY_FIRST_DELAY_MS, fieldHandlesOf, sharedMemoryAvailable, requireWholeNeighbourLists, shuffleInPlace, createCentroidSums, liveOrdinals, RECALIBRATION_CHUNK_VECTORS, BUILD_CHUNK_SIZE, centroidOfCentroidSums, meanOfCentroidSums, countOfCentroidSums, squaredLengthsOfCentroidSums, releaseLocksHeldBy, DRIFT_CHECK_GROWTH_FACTOR, DISK_ADOPTION_YIELD_INTERVAL, adoptLoadedCalibration, RECALIBRATION_DRIFT_RATIO } from './chunk-YZPOBLAS.mjs';
|
|
3
|
+
export { releaseLocksHeldBy } from './chunk-YZPOBLAS.mjs';
|
|
4
4
|
import { isDead, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors } from './chunk-P7C2S5JX.mjs';
|
|
5
5
|
import { MAX_RENUMBERED_SEARCH_ATTEMPTS, DISK_STORAGE_BLOCK_BYTES, compareCodePoints, UNPLACED_FLAG_CHUNK_ORDINALS, MAX_VECTOR_ORDINALS, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, resolveWorkerCount, resolveWorkerEntry, detectRuntime, noteThreadSlotLive, noteThreadSlotGone, WORKER_YOUNG_GENERATION_MB, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_INSERT_TIMEOUT_MS } from './chunk-2CBQ2PCC.mjs';
|
|
6
6
|
import { NarsilError, ErrorCodes, isNarsilError } from './chunk-LJS6PREA.mjs';
|
|
@@ -887,7 +887,8 @@ function openHandles(dimension, codeBits, blockBytes) {
|
|
|
887
887
|
present: new Uint8Array(handles.present),
|
|
888
888
|
partitions: new Int32Array(handles.partitions),
|
|
889
889
|
diskFile: new Int32Array(handles.diskFile),
|
|
890
|
-
diskOffset: new Uint32Array(handles.diskOffset)
|
|
890
|
+
diskOffset: new Uint32Array(handles.diskOffset),
|
|
891
|
+
longestDocIdBytes: 0
|
|
891
892
|
};
|
|
892
893
|
}
|
|
893
894
|
function relayout(store) {
|
|
@@ -898,7 +899,10 @@ function claimSlot(store, docId, cold) {
|
|
|
898
899
|
const ordinal = Atomics.load(store.handles.header, STORE_SLOTS);
|
|
899
900
|
ensureOrdinal(store, ordinal, cold);
|
|
900
901
|
store.diskFile[ordinal] = IN_MEMORY;
|
|
902
|
+
const idBytesBefore = Atomics.load(store.handles.header, STORE_DOC_ID_BYTES);
|
|
901
903
|
appendDocId(store.handles, ordinal, docId);
|
|
904
|
+
const idBytes = Atomics.load(store.handles.header, STORE_DOC_ID_BYTES) - idBytesBefore;
|
|
905
|
+
if (idBytes > store.longestDocIdBytes) store.longestDocIdBytes = idBytes;
|
|
902
906
|
return ordinal;
|
|
903
907
|
}
|
|
904
908
|
function publishOrdinal(store, ordinal) {
|
|
@@ -974,6 +978,19 @@ function ensureOrdinal(store, ordinal, cold) {
|
|
|
974
978
|
if (!cold) ensureFloatSlot(store, ordinal);
|
|
975
979
|
ensureCodeSlot(store, ordinal);
|
|
976
980
|
}
|
|
981
|
+
function bufferGrowthBytes(buffer, bytes) {
|
|
982
|
+
return grownByteLength(buffer.byteLength, bytes, buffer.maxByteLength) - buffer.byteLength;
|
|
983
|
+
}
|
|
984
|
+
function growthBytesFor(store, vectors) {
|
|
985
|
+
const { handles } = store;
|
|
986
|
+
const slots = Atomics.load(handles.header, STORE_SLOTS);
|
|
987
|
+
const last = Math.min(slots + vectors, MAX_VECTOR_ORDINALS) - 1;
|
|
988
|
+
if (last < slots) return 0;
|
|
989
|
+
const idBytes = Atomics.load(handles.header, STORE_DOC_ID_BYTES);
|
|
990
|
+
let bytes = bufferGrowthBytes(handles.magnitudes, (last + 1) * 8) + bufferGrowthBytes(handles.present, last + 1) + bufferGrowthBytes(handles.unplaced, last + 1) + bufferGrowthBytes(handles.unplacedChunks, unplacedChunkCount(last) * 4) + bufferGrowthBytes(handles.removed, last + 1) + bufferGrowthBytes(handles.partitions, (last + 1) * 4) + bufferGrowthBytes(handles.codePresent, last + 1) + bufferGrowthBytes(handles.diskFile, (last + 1) * 4) + bufferGrowthBytes(handles.diskOffset, (last + 1) * 4) + bufferGrowthBytes(handles.docIdOffsets, (last + 2) * 4) + bufferGrowthBytes(handles.docIdBytes, idBytes + (last + 1 - slots) * store.longestDocIdBytes) + blocksGrowthBytes(handles.blocks, handles.layout, slots, last);
|
|
991
|
+
if (handles.codeLayout !== null) bytes += blocksGrowthBytes(handles.codeBlocks, handles.codeLayout, slots, last);
|
|
992
|
+
return bytes;
|
|
993
|
+
}
|
|
977
994
|
function writeVector(store, ordinal, vector) {
|
|
978
995
|
const block = store.view.blockOf(ordinal);
|
|
979
996
|
const base = slotByteOffset(store.handles.layout, store.view.localOrdinal(ordinal)) / 4;
|
|
@@ -1310,6 +1327,9 @@ function createVectorStore(options) {
|
|
|
1310
1327
|
},
|
|
1311
1328
|
memoryBytes() {
|
|
1312
1329
|
return open === null ? 0 : sharedVectorStoreBytes(open.handles);
|
|
1330
|
+
},
|
|
1331
|
+
growthBytesFor(vectors) {
|
|
1332
|
+
return open === null ? 0 : growthBytesFor(open, vectors);
|
|
1313
1333
|
}
|
|
1314
1334
|
};
|
|
1315
1335
|
}
|
|
@@ -1372,9 +1392,9 @@ function releaseSlotLocks(slot) {
|
|
|
1372
1392
|
for (const waiter of [...slot.idleWaiters]) waiter();
|
|
1373
1393
|
}
|
|
1374
1394
|
function rejectPending(slot, err) {
|
|
1375
|
-
for (const [,
|
|
1376
|
-
clearTimeout(
|
|
1377
|
-
|
|
1395
|
+
for (const [, waiting2] of slot.pending) {
|
|
1396
|
+
clearTimeout(waiting2.timer);
|
|
1397
|
+
waiting2.reject(err);
|
|
1378
1398
|
}
|
|
1379
1399
|
slot.pending.clear();
|
|
1380
1400
|
}
|
|
@@ -1453,11 +1473,11 @@ async function createVectorSearchPool(requestedCount) {
|
|
|
1453
1473
|
const key = message.requestId;
|
|
1454
1474
|
if (typeof key !== "string") return;
|
|
1455
1475
|
noteAnswered(slot, key, sharedFields);
|
|
1456
|
-
const
|
|
1457
|
-
if (!
|
|
1476
|
+
const waiting2 = slot.pending.get(key);
|
|
1477
|
+
if (!waiting2) return;
|
|
1458
1478
|
slot.pending.delete(key);
|
|
1459
|
-
clearTimeout(
|
|
1460
|
-
|
|
1479
|
+
clearTimeout(waiting2.timer);
|
|
1480
|
+
waiting2.resolve(message);
|
|
1461
1481
|
});
|
|
1462
1482
|
listenForFailure(worker, (err) => {
|
|
1463
1483
|
slot.alive = false;
|
|
@@ -2166,6 +2186,51 @@ async function releasePendingLocations(state) {
|
|
|
2166
2186
|
await releaseInTurn(state, () => releaseLocations(state, paths, pending));
|
|
2167
2187
|
}
|
|
2168
2188
|
|
|
2189
|
+
// src/runtime/background-memory.ts
|
|
2190
|
+
var admissions = /* @__PURE__ */ new Set();
|
|
2191
|
+
var waiting = /* @__PURE__ */ new Map();
|
|
2192
|
+
function admitsBackgroundBytes(bytes) {
|
|
2193
|
+
for (const admission of admissions) {
|
|
2194
|
+
if (!admission(bytes)) return false;
|
|
2195
|
+
}
|
|
2196
|
+
return true;
|
|
2197
|
+
}
|
|
2198
|
+
function noteWaitingForMemory(owner, work) {
|
|
2199
|
+
if (work === null) waiting.delete(owner);
|
|
2200
|
+
else waiting.set(owner, work);
|
|
2201
|
+
}
|
|
2202
|
+
function chargeHeapBytes(bytes) {
|
|
2203
|
+
}
|
|
2204
|
+
|
|
2205
|
+
// src/vector/vector-index/memory-charge.ts
|
|
2206
|
+
function admitShuffledRebuild(state) {
|
|
2207
|
+
const bytes = state.hnsw?.graphBytes ?? 0;
|
|
2208
|
+
if (!admitsBackgroundBytes(bytes)) {
|
|
2209
|
+
state.waitingForMemory = true;
|
|
2210
|
+
noteWaitingForMemory(state, { indexName: state.indexName, fieldName: state.fieldName });
|
|
2211
|
+
return false;
|
|
2212
|
+
}
|
|
2213
|
+
stopWaitingForMemory(state);
|
|
2214
|
+
chargeHeapBytes(bytes - state.rebuildChargeBytes);
|
|
2215
|
+
state.rebuildChargeBytes = bytes;
|
|
2216
|
+
return true;
|
|
2217
|
+
}
|
|
2218
|
+
function stopWaitingForMemory(state) {
|
|
2219
|
+
if (!state.waitingForMemory) return;
|
|
2220
|
+
state.waitingForMemory = false;
|
|
2221
|
+
noteWaitingForMemory(state, null);
|
|
2222
|
+
}
|
|
2223
|
+
function releaseRebuildCharge(state) {
|
|
2224
|
+
chargeHeapBytes(-state.rebuildChargeBytes);
|
|
2225
|
+
state.rebuildChargeBytes = 0;
|
|
2226
|
+
}
|
|
2227
|
+
function releaseMemoryCharges(state) {
|
|
2228
|
+
releaseRebuildCharge(state);
|
|
2229
|
+
stopWaitingForMemory(state);
|
|
2230
|
+
if (state.rebuildWaitTimer !== null) clearTimeout(state.rebuildWaitTimer);
|
|
2231
|
+
state.rebuildWaitTimer = null;
|
|
2232
|
+
}
|
|
2233
|
+
|
|
2169
2234
|
// src/vector/vector-index/build.ts
|
|
2170
2235
|
function bufferWhatTheGraphLacks(state, graph) {
|
|
2171
2236
|
for (const docId of allLiveDocIds(state)) {
|
|
@@ -2248,8 +2313,25 @@ async function rebuildInShuffledOrder(state) {
|
|
|
2248
2313
|
const adopted = await buildGraphFromStore(state);
|
|
2249
2314
|
if (adopted && replacing.loadedWithoutCount) state.onLoadedGraphRepaired();
|
|
2250
2315
|
}
|
|
2316
|
+
function retryWhenMemoryAllows(state) {
|
|
2317
|
+
if (state.rebuildWaitTimer !== null || state.disposed) return;
|
|
2318
|
+
const timer = setTimeout(() => {
|
|
2319
|
+
state.rebuildWaitTimer = null;
|
|
2320
|
+
startShuffledRebuild(state);
|
|
2321
|
+
}, SHUFFLED_REBUILD_MEMORY_RETRY_MS);
|
|
2322
|
+
timer.unref?.();
|
|
2323
|
+
state.rebuildWaitTimer = timer;
|
|
2324
|
+
}
|
|
2251
2325
|
function startShuffledRebuild(state) {
|
|
2252
|
-
if (state.shuffledRebuild !== null || state.disposed
|
|
2326
|
+
if (state.shuffledRebuild !== null || state.disposed) return;
|
|
2327
|
+
if (!shuffledRebuildDue(state)) {
|
|
2328
|
+
stopWaitingForMemory(state);
|
|
2329
|
+
return;
|
|
2330
|
+
}
|
|
2331
|
+
if (!admitShuffledRebuild(state)) {
|
|
2332
|
+
retryWhenMemoryAllows(state);
|
|
2333
|
+
return;
|
|
2334
|
+
}
|
|
2253
2335
|
state.shuffledRebuild = rebuildInShuffledOrder(state).catch((error) => {
|
|
2254
2336
|
console.warn(
|
|
2255
2337
|
`Rebuilding the graph of "${state.indexName}/${state.fieldName}" in a shuffled order failed, so the field keeps the graph it holds:`,
|
|
@@ -2257,6 +2339,7 @@ function startShuffledRebuild(state) {
|
|
|
2257
2339
|
);
|
|
2258
2340
|
}).finally(() => {
|
|
2259
2341
|
state.shuffledRebuild = null;
|
|
2342
|
+
releaseRebuildCharge(state);
|
|
2260
2343
|
scheduleBuild(state);
|
|
2261
2344
|
scheduleWorkerCopyLoad(state);
|
|
2262
2345
|
});
|
|
@@ -2382,9 +2465,12 @@ function releaseHeldMemory(state) {
|
|
|
2382
2465
|
state.savedFiles = [];
|
|
2383
2466
|
state.savedSignature = null;
|
|
2384
2467
|
state.store.release();
|
|
2468
|
+
releaseMemoryCharges(state);
|
|
2385
2469
|
}
|
|
2386
2470
|
function disposeIndex(state) {
|
|
2387
2471
|
state.disposed = true;
|
|
2472
|
+
if (state.rebuildWaitTimer !== null) clearTimeout(state.rebuildWaitTimer);
|
|
2473
|
+
state.rebuildWaitTimer = null;
|
|
2388
2474
|
if (state.buildTimer !== null) clearTimeout(state.buildTimer);
|
|
2389
2475
|
state.buildTimer = null;
|
|
2390
2476
|
invalidateWorkerCopies(state);
|
|
@@ -2723,6 +2809,7 @@ function maintenanceStatus(state) {
|
|
|
2723
2809
|
graphCount,
|
|
2724
2810
|
bufferSize: state.buffer.size,
|
|
2725
2811
|
building: state.building || state.buildScheduled || state.buildTimer !== null || state.shuffledRebuild !== null,
|
|
2812
|
+
waitingForMemory: state.waitingForMemory,
|
|
2726
2813
|
estimatedCompactMs,
|
|
2727
2814
|
estimatedOptimizeMs
|
|
2728
2815
|
};
|
|
@@ -2925,6 +3012,9 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
2925
3012
|
hnsw: null,
|
|
2926
3013
|
freshGraph: null,
|
|
2927
3014
|
...noBuildHistory(),
|
|
3015
|
+
rebuildChargeBytes: 0,
|
|
3016
|
+
waitingForMemory: false,
|
|
3017
|
+
rebuildWaitTimer: null,
|
|
2928
3018
|
onLoadedGraphRepaired,
|
|
2929
3019
|
compactedNodeCount: 0,
|
|
2930
3020
|
building: false,
|
|
@@ -3082,6 +3172,7 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
3082
3172
|
},
|
|
3083
3173
|
maintenanceStatus: () => maintenanceStatus(state),
|
|
3084
3174
|
estimateMemoryBytes: () => estimateMemoryBytes(state),
|
|
3175
|
+
growthBytesFor: (vectors) => state.store.growthBytesFor(vectors),
|
|
3085
3176
|
deserialize: (parts) => {
|
|
3086
3177
|
invalidateWorkerCopies(state);
|
|
3087
3178
|
deserialize(state, parts);
|
|
@@ -18,7 +18,7 @@ function restoreDepsOf(core, host) {
|
|
|
18
18
|
function queueRestore(core, host, restoresInFlight, indexName, data) {
|
|
19
19
|
const previous = restoresInFlight.get(indexName) ?? Promise.resolve();
|
|
20
20
|
const restored = previous.then(async () => {
|
|
21
|
-
const { restoreFromSnapshot } = await import('./snapshot-
|
|
21
|
+
const { restoreFromSnapshot } = await import('./snapshot-BE3F2PNC.mjs');
|
|
22
22
|
await restoreFromSnapshot(indexName, data, restoreDepsOf(core, host));
|
|
23
23
|
});
|
|
24
24
|
const settled = restored.then(
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { vectorsPerFile } from './chunk-
|
|
2
|
-
import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-
|
|
3
|
-
import './chunk-
|
|
4
|
-
import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, prepareStoredSynonyms, rejectSnapshot, readHeapStatistics, readResidentMemory, extractVectorFieldsFromSchema, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, insertDocumentVectors, decodeIndexConfiguration, flattenSchema } from './chunk-
|
|
1
|
+
import { collectAndReturnPages, vectorsPerFile } from './chunk-YZPOBLAS.mjs';
|
|
2
|
+
import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-3YPLBK4C.mjs';
|
|
3
|
+
import './chunk-4FVSBSXY.mjs';
|
|
4
|
+
import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, prepareStoredSynonyms, rejectSnapshot, readHeapStatistics, readResidentMemory, extractVectorFieldsFromSchema, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, insertDocumentVectors, decodeIndexConfiguration, flattenSchema } from './chunk-DC2SZO4D.mjs';
|
|
5
5
|
import { decodeVectorIndexParts, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors } from './chunk-P7C2S5JX.mjs';
|
|
6
6
|
import { MAX_FILE_BYTES } from './chunk-2CBQ2PCC.mjs';
|
|
7
7
|
import './chunk-JPTQZWJQ.mjs';
|
|
@@ -663,7 +663,8 @@ function refuse(indexName, replacesIndex, shortfall) {
|
|
|
663
663
|
function requireRestoreMemory(indexName, footprint, replacesIndex) {
|
|
664
664
|
const processBytes = footprint.heapBytes + footprint.externalBytes;
|
|
665
665
|
if (shortfallOf(footprint.heapBytes, processBytes, true) === null) return;
|
|
666
|
-
const
|
|
666
|
+
const collected = collectAndReturnPages();
|
|
667
|
+
const shortfall = shortfallOf(footprint.heapBytes, processBytes, collected || replacesIndex);
|
|
667
668
|
if (shortfall !== null) refuse(indexName, replacesIndex, shortfall);
|
|
668
669
|
}
|
|
669
670
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-
|
|
2
|
-
import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-
|
|
1
|
+
import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-4FVSBSXY.mjs';
|
|
2
|
+
import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-DC2SZO4D.mjs';
|
|
3
3
|
import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-P7C2S5JX.mjs';
|
|
4
4
|
import './chunk-2CBQ2PCC.mjs';
|
|
5
5
|
import './chunk-JPTQZWJQ.mjs';
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { E as EmbeddingAdapter, R as RerankAdapter } from './adapters-BQFQ4PRz.js';
|
|
2
|
-
import { C as ClusterNamespace } from './config-
|
|
3
|
-
import { h as TaskStore } from './tasks-
|
|
2
|
+
import { C as ClusterNamespace } from './config-CS5ADSrx.js';
|
|
3
|
+
import { h as TaskStore } from './tasks-5_LCFEzF.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* Context handed to {@link OnRequestHook} for every inbound request, captured
|
|
@@ -10,5 +10,9 @@ var REFUSAL_YOUNG_COLLECTION_MIN_INTERVAL_MS = 10;
|
|
|
10
10
|
var HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE = 0.75;
|
|
11
11
|
var HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE = 0.65;
|
|
12
12
|
var PROCESS_LIMIT_REFRESH_INTERVAL_MS = 1e3;
|
|
13
|
+
var V8_YOUNG_GENERATION_BYTES_BY_MAJOR = {
|
|
14
|
+
12: 48 * 1024 * 1024,
|
|
15
|
+
13: 192 * 1024 * 1024
|
|
16
|
+
};
|
|
13
17
|
|
|
14
|
-
export { HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, IDLE_HEAP_CLEANUP_CHECK_MS, IDLE_HEAP_CLEANUP_MAX_GROWTH_BYTES, IDLE_HEAP_CLEANUP_MAX_UTILISATION, IDLE_HEAP_CLEANUP_MIN_RECLAIMABLE_BYTES, IDLE_HEAP_CLEANUP_QUIET_CHECKS, PROCESS_LIMIT_REFRESH_INTERVAL_MS, REFUSAL_COLLECTION_MIN_INTERVAL_MS, REFUSAL_COLLECTION_TIME_MULTIPLE, REFUSAL_YOUNG_COLLECTION_MIN_INTERVAL_MS };
|
|
18
|
+
export { HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, IDLE_HEAP_CLEANUP_CHECK_MS, IDLE_HEAP_CLEANUP_MAX_GROWTH_BYTES, IDLE_HEAP_CLEANUP_MAX_UTILISATION, IDLE_HEAP_CLEANUP_MIN_RECLAIMABLE_BYTES, IDLE_HEAP_CLEANUP_QUIET_CHECKS, PROCESS_LIMIT_REFRESH_INTERVAL_MS, REFUSAL_COLLECTION_MIN_INTERVAL_MS, REFUSAL_COLLECTION_TIME_MULTIPLE, REFUSAL_YOUNG_COLLECTION_MIN_INTERVAL_MS, V8_YOUNG_GENERATION_BYTES_BY_MAJOR };
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes, uint64Bytes } from './chunk-
|
|
1
|
+
import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes, uint64Bytes } from './chunk-LVNBY2TS.mjs';
|
|
2
2
|
import { WIRE_BATCH_BUDGET } from './chunk-P36MUFEJ.mjs';
|
|
3
3
|
import { ReplicationMessageTypes } from './chunk-YH5OIL7B.mjs';
|
|
4
|
-
import { packChecksummedEnvelope, crc32, openChecksummedEnvelope, reconstructSchemaFromMetadata, HEADER_SIZE, prepareDocumentVectors } from './chunk-
|
|
5
|
-
import { freezeSegmentShared, createSharedFrozenSegment, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, selectionKeeps, decodeMapWithoutFields, shareSegmentArrays, segmentArraysOf, mergeSegmentContents } from './chunk-
|
|
4
|
+
import { packChecksummedEnvelope, crc32, openChecksummedEnvelope, reconstructSchemaFromMetadata, HEADER_SIZE, prepareDocumentVectors } from './chunk-PABYSTKO.mjs';
|
|
5
|
+
import { freezeSegmentShared, createSharedFrozenSegment, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, selectionKeeps, decodeMapWithoutFields, shareSegmentArrays, segmentArraysOf, mergeSegmentContents } from './chunk-5QIRLJH4.mjs';
|
|
6
6
|
import { fnv1a, isCompositePartition, survivorDocumentsOf, partitionStateOf } from './chunk-5WEBVYOA.mjs';
|
|
7
7
|
import { generateId, getLanguage } from './chunk-FUUP45IS.mjs';
|
|
8
8
|
import { extractVectorFieldsFromSchema, partialFieldsOf, messagePackUpperBoundBytes } from './chunk-DG62H7OD.mjs';
|
|
@@ -14,7 +14,7 @@ function requireLoadedVectorCode() {
|
|
|
14
14
|
function loadVectorCode() {
|
|
15
15
|
if (loaded !== null) return Promise.resolve(loaded);
|
|
16
16
|
if (loading === null) {
|
|
17
|
-
loading = import('./module-
|
|
17
|
+
loading = import('./module-WVKSJL74.mjs').then(
|
|
18
18
|
(code) => {
|
|
19
19
|
loaded = code;
|
|
20
20
|
return code;
|
|
@@ -25,6 +25,15 @@ function createFixedBuffer(bytes) {
|
|
|
25
25
|
function nextGrowthTarget(have, needed, max) {
|
|
26
26
|
return Math.min(max, Math.max(needed, have * 2));
|
|
27
27
|
}
|
|
28
|
+
function grownByteLength(have, needed, max, step = 1) {
|
|
29
|
+
let size = have;
|
|
30
|
+
while (size < needed) {
|
|
31
|
+
const next = Math.ceil(nextGrowthTarget(size, Math.min(needed, size + 1), max) / step) * step;
|
|
32
|
+
if (next <= size) return size;
|
|
33
|
+
size = next;
|
|
34
|
+
}
|
|
35
|
+
return size;
|
|
36
|
+
}
|
|
28
37
|
function growBufferTo(buffer, bytes) {
|
|
29
38
|
const held = buffer.byteLength;
|
|
30
39
|
if (held >= bytes) return;
|
|
@@ -1038,6 +1047,25 @@ function ensureBlockSlots(handle, slots) {
|
|
|
1038
1047
|
}
|
|
1039
1048
|
growMemoryTo(storage.kind === "shared-memory" ? storage.memory : storage.simd.memory, bytes, handle.layout);
|
|
1040
1049
|
}
|
|
1050
|
+
function blockGrowthBytes(handle, layout, slots) {
|
|
1051
|
+
const bytes = layout.slotsOffset + Math.min(slots, layout.capacity) * layout.slotStride;
|
|
1052
|
+
if (handle !== null && handle.storage.kind === "bytes") {
|
|
1053
|
+
const buffer = handle.storage.buffer;
|
|
1054
|
+
return grownByteLength(buffer.byteLength, bytes, buffer.maxByteLength) - buffer.byteLength;
|
|
1055
|
+
}
|
|
1056
|
+
const have = handle === null ? 0 : blockBuffer(handle).byteLength;
|
|
1057
|
+
const start = handle === null ? pagesFor(initialBlockBytes(layout)) * WASM_PAGE_BYTES : have;
|
|
1058
|
+
return grownByteLength(start, bytes, fullBlockBytes(layout), WASM_PAGE_BYTES) - have;
|
|
1059
|
+
}
|
|
1060
|
+
function blocksGrowthBytes(blocks, layout, firstOrdinal, lastOrdinal) {
|
|
1061
|
+
let bytes = 0;
|
|
1062
|
+
const lastBlock = Math.floor(lastOrdinal / layout.capacity);
|
|
1063
|
+
for (let block = Math.floor(firstOrdinal / layout.capacity); block <= lastBlock; block++) {
|
|
1064
|
+
const slots = block === lastBlock ? lastOrdinal % layout.capacity + 1 : layout.capacity;
|
|
1065
|
+
bytes += blockGrowthBytes(blocks[block] ?? null, layout, slots);
|
|
1066
|
+
}
|
|
1067
|
+
return bytes;
|
|
1068
|
+
}
|
|
1041
1069
|
function blockBuffer(handle) {
|
|
1042
1070
|
const storage = handle.storage;
|
|
1043
1071
|
if (storage.kind === "bytes") return storage.buffer;
|
|
@@ -3980,4 +4008,4 @@ function createHNSWIndex(dimension, store, config, quantizer, handles) {
|
|
|
3980
4008
|
};
|
|
3981
4009
|
}
|
|
3982
4010
|
|
|
3983
|
-
export { BLOCK_MAX_BYTES, GRAPH_RETIRED, IN_MEMORY, NOTHING_STAGED, QUERY_STAGED, STORE_BLOCK_COUNT, STORE_CALIBRATION_GENERATION, STORE_CODE_BLOCK_COUNT, STORE_DOC_ID_BYTES, STORE_LIVE_COUNT, STORE_ORDINAL_LAYOUT_GENERATION, STORE_RELEASED_VECTORS_TO_DISK, STORE_REMOVED_COUNT, STORE_SLOTS, STORE_UNPLACED_COUNT, addToOrdinalFilter, arenaFloat32Distance, centroidOfCentroidSums, cosineSimilarityWithMagnitudes, countOfCentroidSums, createCentroidSums, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createOsqScratch, createSharedVectorStoreHandles, createVectorBlock, detachNativeStore, dotProduct, ensureBlockSlots, euclideanDistance, fixedView, graphWithoutNodes, growBufferTo, insertNode, magnitude, meanOfCentroidSums, nativeScoresOf, nativeStoreScratchBytes, nodeCountOf, openGraphState, openSharedQuantizer, openVectorBlock, ordinalFilterHas, ordinalFilterValues, osqBitsOf, osqQuantize, releaseLocksHeldBy, removeFromOrdinalFilter, requireWholeNeighbourLists, resetStoreHeader, searchOrdinals, sharedMemoryAvailable, sharedVectorStoreBytes, shuffleInPlace, slotByteOffset, squaredLengthsOfCentroidSums, toDistance, toScore, tombstoneCountOf, topLayerInParts, unplacedChunkCount };
|
|
4011
|
+
export { BLOCK_MAX_BYTES, GRAPH_RETIRED, IN_MEMORY, NOTHING_STAGED, QUERY_STAGED, STORE_BLOCK_COUNT, STORE_CALIBRATION_GENERATION, STORE_CODE_BLOCK_COUNT, STORE_DOC_ID_BYTES, STORE_LIVE_COUNT, STORE_ORDINAL_LAYOUT_GENERATION, STORE_RELEASED_VECTORS_TO_DISK, STORE_REMOVED_COUNT, STORE_SLOTS, STORE_UNPLACED_COUNT, addToOrdinalFilter, arenaFloat32Distance, blocksGrowthBytes, centroidOfCentroidSums, cosineSimilarityWithMagnitudes, countOfCentroidSums, createCentroidSums, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createOsqScratch, createSharedVectorStoreHandles, createVectorBlock, detachNativeStore, dotProduct, ensureBlockSlots, euclideanDistance, fixedView, graphWithoutNodes, growBufferTo, grownByteLength, insertNode, magnitude, meanOfCentroidSums, nativeScoresOf, nativeStoreScratchBytes, nodeCountOf, openGraphState, openSharedQuantizer, openVectorBlock, ordinalFilterHas, ordinalFilterValues, osqBitsOf, osqQuantize, releaseLocksHeldBy, removeFromOrdinalFilter, requireWholeNeighbourLists, resetStoreHeader, searchOrdinals, sharedMemoryAvailable, sharedVectorStoreBytes, shuffleInPlace, slotByteOffset, squaredLengthsOfCentroidSums, toDistance, toScore, tombstoneCountOf, topLayerInParts, unplacedChunkCount };
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { requirePatternSearchFor, requirePartialWordSearchFor, createPartitionIndex, buildPartialWordTableFrom, recordValueSize, patternIndexOf, encodeFieldIndexes, encodePartialSpellingsOf, countPartialSpelling, copyVectorsInto, isCompositePartition, fnv1a, partialSpellingsOfDocuments, createFrozenPartialWords, requirePatternSearch, documentValueBytes, cloneProjected, projectIncludedPaths, valueWithOwnBuffers, createSurfaceRegistry, MAX_TERM_FREQUENCY, joinPatternWorkMeter, fanOutQuery, sanitizeGlobalStats, countsWithoutScores, fanOutMatchCount, createCompositePartition, createGeoIndex, documentWithOwnBuffers, readProjectedPath, createNumericIndex, createBooleanIndex, createEnumIndex, ValueAffixes } from './chunk-5WEBVYOA.mjs';
|
|
2
2
|
import { generateId, getLanguage } from './chunk-FUUP45IS.mjs';
|
|
3
|
-
import { readHeapStatistics, readResidentMemory } from './chunk-
|
|
3
|
+
import { readHeapStatistics, readResidentMemory } from './chunk-ZATRDAYU.mjs';
|
|
4
4
|
import { GrowableUint32, bitsetHas, bitsetSet, createBitSet } from './chunk-HNAP2KTG.mjs';
|
|
5
5
|
import { tokenizeOptions, getNestedValue, sameFields, visitDocumentSurfaceSpellings, visitDocumentSpellings } from './chunk-KCQG2PJT.mjs';
|
|
6
6
|
import { schemaFieldsOf, partialFieldsOf, encodeDocument, validateDocument, validateDocumentStrict, extractVectorFieldsFromSchema } from './chunk-DG62H7OD.mjs';
|
|
@@ -9,7 +9,7 @@ import { isWordIndexedBase } from './chunk-6ODIY6ZB.mjs';
|
|
|
9
9
|
import { resolveIndexAnalysis, synonymSearchModule } from './chunk-IR4KQ4WB.mjs';
|
|
10
10
|
import { visitTokens, codePointOrder, encodeStringBlob, encodeLookupKey, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, FROZEN_REMOVAL_INDEX_MAX_PROCESS_FRACTION } from './chunk-OFYKBKKV.mjs';
|
|
11
11
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
12
|
-
import { loadedVectorCode, loadVectorCode, loadVectorCodeFor, requireLoadedVectorCode } from './chunk-
|
|
12
|
+
import { loadedVectorCode, loadVectorCode, loadVectorCodeFor, requireLoadedVectorCode } from './chunk-4PFOOQFQ.mjs';
|
|
13
13
|
import { PROTOTYPE_POLLUTION_KEYS } from './chunk-L5ORED7V.mjs';
|
|
14
14
|
import { NarsilError, ErrorCodes } from './chunk-OBWUWZRA.mjs';
|
|
15
15
|
import { Decoder, decode } from '@msgpack/msgpack';
|