@delali/narsil 0.3.1-next.151 → 0.3.1-next.153
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{analysis-change-ZCJNYBJM.mjs → analysis-change-2OYM7ARW.mjs} +4 -4
- package/dist/{react-native/chunks/analysis-change-entry-BNRU6STO.mjs → analysis-change-entry-QD7YNDEE.mjs} +1 -1
- package/dist/browser/{analysis-change-M65AZQQG.mjs → analysis-change-2V2HZ2NY.mjs} +2 -2
- package/dist/{analysis-change-entry-YN3ELL6C.mjs → browser/analysis-change-entry-JL5U2N3Q.mjs} +1 -1
- package/dist/browser/{chunk-SRL47USS.mjs → chunk-2A2IPRC3.mjs} +1 -1
- package/dist/browser/chunk-3I27KXQD.mjs +34 -0
- package/dist/browser/{chunk-KAEVGLYN.mjs → chunk-55WYRLDZ.mjs} +15 -0
- package/dist/browser/{chunk-6OWX3OP2.mjs → chunk-7ZJTRUVW.mjs} +1 -1
- package/dist/browser/{chunk-QMNZFFD3.mjs → chunk-B3JM6Z74.mjs} +6 -2
- package/dist/browser/{chunk-BRWY3JCN.mjs → chunk-CSCKZBVJ.mjs} +4 -29
- package/dist/browser/{chunk-D6UFGBF5.mjs → chunk-GJKXW3SF.mjs} +43 -24
- package/dist/browser/{chunk-5V74FFZE.mjs → chunk-HT7J73QL.mjs} +1 -1
- package/dist/browser/{chunk-BVX2MEHA.mjs → chunk-KWYQ6KX4.mjs} +17 -17
- package/dist/browser/{chunk-2JABMB4S.mjs → chunk-PM4MUM5N.mjs} +2 -2
- package/dist/browser/{chunk-W643QV3L.mjs → chunk-QJCO3TJU.mjs} +8 -4
- package/dist/browser/{chunk-ODQP7P3H.mjs → chunk-TQ5UMFG5.mjs} +1 -1
- package/dist/browser/{chunk-4DYLCICG.mjs → chunk-VBX2TI5V.mjs} +2 -2
- package/dist/browser/{chunk-XILIT5MT.mjs → chunk-VD7JEUIR.mjs} +5 -5
- package/dist/browser/{cold-module-A5S6JFDN.mjs → cold-module-YECAK4MN.mjs} +18 -12
- package/dist/browser/{durability-wiring-W5AVV3RF.mjs → durability-wiring-LM7WFL6X.mjs} +61 -50
- package/dist/browser/{import-PTT63AGK.mjs → import-QOY6ZH6X.mjs} +3 -3
- package/dist/browser/{module-CEKA7MQH.mjs → module-KAMRAJLE.mjs} +6 -6
- package/dist/browser/{module-X5PGOE47.mjs → module-XQLEOYNM.mjs} +24 -23
- package/dist/{react-native/chunks/restore-queue-OI5EH7R3.mjs → browser/restore-queue-UQ6AAUVX.mjs} +2 -2
- package/dist/browser/{snapshot-DQ6A4PQP.mjs → snapshot-UGZVMP4H.mjs} +11 -10
- package/dist/browser/{write-MAGRAIEH.mjs → write-I5P5MW4O.mjs} +7 -7
- package/dist/{chunk-VQDZVWN3.mjs → chunk-2PNYKG2D.mjs} +16 -16
- package/dist/{chunk-U6NE6RCV.mjs → chunk-3E4RRF5J.mjs} +1 -1
- package/dist/{chunk-ZYKUD5NX.mjs → chunk-4QCSHLNM.mjs} +56 -6
- package/dist/{chunk-YKEHQUSZ.mjs → chunk-52QQWEBS.mjs} +2 -2
- package/dist/{chunk-KZ5GQE4Z.mjs → chunk-6RKSY3BX.mjs} +6 -6
- package/dist/{chunk-6C5EX2QR.mjs → chunk-7LDMWKP6.mjs} +1 -1
- package/dist/{chunk-ADBS5XRI.mjs → chunk-7TE2AJIH.mjs} +6 -6
- package/dist/{chunk-QN5WVV5K.mjs → chunk-CSHHU7PA.mjs} +5 -5
- package/dist/{chunk-ZAYWOHVG.mjs → chunk-DGRM4EZZ.mjs} +1 -1
- package/dist/{chunk-7N223GWV.mjs → chunk-DMNPYF54.mjs} +6 -2
- package/dist/{chunk-7HKWO4BJ.mjs → chunk-FDG4KXX5.mjs} +1 -1
- package/dist/{chunk-QEYF2SQ6.mjs → chunk-FNBNZCGP.mjs} +1 -1
- package/dist/{chunk-ZEKDO7YX.mjs → chunk-G5KHSM6E.mjs} +15 -0
- package/dist/{chunk-3BVGJ4JR.mjs → chunk-IJSFWGVW.mjs} +3 -28
- package/dist/{chunk-B7XIOAMK.mjs → chunk-J4BDAH2W.mjs} +1 -1
- package/dist/{chunk-CYYSZPVV.mjs → chunk-KQJVNPRM.mjs} +18 -19
- package/dist/chunk-MATMVH65.mjs +9 -0
- package/dist/{react-native/chunks/chunk-R4EEMYM6.mjs → chunk-NITCPDFL.mjs} +2 -2
- package/dist/{chunk-U2726TL5.mjs → chunk-NPXVQS33.mjs} +6 -2
- package/dist/{chunk-DOY6EG76.mjs → chunk-RBEHLIKX.mjs} +2 -2
- package/dist/{chunk-F22IOCR2.mjs → chunk-SEAJS664.mjs} +1 -1
- package/dist/{chunk-O4KUNDVF.mjs → chunk-UGVUW27S.mjs} +3 -3
- package/dist/chunk-VD7J552I.mjs +28 -0
- package/dist/{chunk-L7TIAYPX.mjs → chunk-ZHAXP7SD.mjs} +7 -7
- package/dist/{chunk-XKPNKR3V.mjs → chunk-ZS6CMXAE.mjs} +31 -10
- package/dist/{cold-module-2WWGBFJC.mjs → cold-module-PYSSBIL7.mjs} +18 -12
- package/dist/{direct-executor-QLVHMELJ.mjs → direct-executor-WYALQSMD.mjs} +2 -2
- package/dist/distribution/coordinator/etcd.d.ts +9 -0
- package/dist/distribution/coordinator/etcd.mjs +267 -106
- package/dist/distribution.mjs +23 -21
- package/dist/{durability-wiring-GNWQSZI3.mjs → durability-wiring-7VBZEJLG.mjs} +19 -45
- package/dist/{import-DX55PN4G.mjs → import-GJOMB2HC.mjs} +4 -4
- package/dist/index.browser.mjs +1 -1
- package/dist/index.mjs +3 -3
- package/dist/index.react-native.mjs +1 -1
- package/dist/{module-V5K4ZP23.mjs → module-6RLIKSZQ.mjs} +8 -8
- package/dist/{module-HW6IXUM6.mjs → module-AVQEIB7O.mjs} +23 -21
- package/dist/{paged-partition-archive-FDIIWZNR.mjs → paged-partition-archive-KGRW366Y.mjs} +11 -10
- package/dist/persistence/durability/checkpoint-worker.mjs +8 -8
- package/dist/react-native/chunks/{analysis-change-HGW3L6NY.mjs → analysis-change-DXPZRGWF.mjs} +2 -2
- package/dist/{browser/analysis-change-entry-H3YIOLXZ.mjs → react-native/chunks/analysis-change-entry-CXNRW324.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-5FV4ZEVA.mjs → chunk-7IQR3SZO.mjs} +43 -24
- package/dist/react-native/chunks/{chunk-7VWQLEFV.mjs → chunk-CLE3LZ77.mjs} +6 -2
- package/dist/react-native/chunks/{chunk-SLGIAD6B.mjs → chunk-D7LFYPYI.mjs} +4 -4
- package/dist/{chunk-KZQIVGW7.mjs → react-native/chunks/chunk-DFQJ4IJB.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-B6OD63Z3.mjs → chunk-FS7BYE23.mjs} +8 -4
- package/dist/react-native/chunks/chunk-HJBJBH26.mjs +34 -0
- package/dist/react-native/chunks/{chunk-YJSMEKRE.mjs → chunk-IDFMNRIC.mjs} +17 -17
- package/dist/react-native/chunks/{chunk-FKGG4UOF.mjs → chunk-JYL35A4A.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-GV5KFQ6M.mjs → chunk-KLZGAOLT.mjs} +4 -29
- package/dist/react-native/chunks/{chunk-HURV3ASY.mjs → chunk-KOCTCUXW.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-KUJOBRKS.mjs → chunk-UCJU6WNH.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-LQ6Q4L5B.mjs → chunk-VGUWIZFQ.mjs} +15 -0
- package/dist/react-native/chunks/{cold-module-TSFWLYWC.mjs → cold-module-KFGEWYEL.mjs} +16 -10
- package/dist/react-native/chunks/{durability-wiring-PZR6KOQK.mjs → durability-wiring-QKJB2QBT.mjs} +61 -50
- package/dist/react-native/chunks/{import-VTODLPLO.mjs → import-XFBCCENV.mjs} +2 -2
- package/dist/react-native/chunks/{module-VEK2KCKU.mjs → module-FHJ2GOBI.mjs} +4 -4
- package/dist/react-native/chunks/{module-JBBH6PZH.mjs → module-UJ5VUARS.mjs} +24 -23
- package/dist/{browser/restore-queue-DD4LWZK7.mjs → react-native/chunks/restore-queue-H5VCN453.mjs} +2 -2
- package/dist/react-native/chunks/{snapshot-AUZMEV36.mjs → snapshot-KVBD4LRC.mjs} +11 -10
- package/dist/react-native/chunks/{write-DDC2QBRJ.mjs → write-ES5A3WUA.mjs} +5 -5
- package/dist/{restore-queue-BQWWNNQI.mjs → restore-queue-AOF5OTQV.mjs} +2 -2
- package/dist/server.mjs +5 -5
- package/dist/{snapshot-EHXTOTW5.mjs → snapshot-WMX6JCLP.mjs} +18 -16
- package/dist/workers/entry.mjs +2 -2
- package/dist/{write-SDJXXQ2G.mjs → write-SKJ6HW7O.mjs} +7 -7
- package/package.json +7 -7
|
@@ -1,14 +1,14 @@
|
|
|
1
|
-
import { readWholeSectionChecked, checkEverySectionPage, readFileRange, openPagedFile, readCacheOfThisThread, createSectionReader, closePagedFile } from './chunk-
|
|
2
|
-
export { actionPartitionsOf, adoptSegmentsOnWorker, bytesThatSegmentsHoldInMemory, checkPagedSegmentFully, closeUnusedSegments, countsAsLatestHolder, createPagedFrozenSegment, isPagedSegmentFile, keysThatSegmentsStillRead, mergePagedWindow, openFileVectorReads, openPagedSegmentScan, openPagedSnapshot, openRewriteSpill, pagedListingOf, pagedPartitionLines, pagedTombstonesFor, prepareDiskAdoption, startReadingOpeningSectionsAhead, swapPreparedPartitions, writePagedSegment } from './chunk-
|
|
3
|
-
import './chunk-
|
|
4
|
-
import { COLD_LOAD_WAIT_WITHOUT_SEARCH_MS, loadFailed, decodeSections, SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES, COLD_WALK_NODES_READ_AHEAD } from './chunk-
|
|
5
|
-
import './chunk-
|
|
6
|
-
import './chunk-
|
|
7
|
-
import { magnitude, defaultRescoreDepth, defaultEfSearch, computeCodeBlockLayout, toScore, osqQuantize, createOsqScratch, scalarQueryProducts, osqEstimate, osqDistance, toDistance } from './chunk-
|
|
8
|
-
export { createHNSWIndex, osqBitsOf } from './chunk-
|
|
9
|
-
import { isDead, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors, stageQueryLevels, readTrailer } from './chunk-
|
|
1
|
+
import { readWholeSectionChecked, checkEverySectionPage, readFileRange, openPagedFile, readCacheOfThisThread, createSectionReader, closePagedFile } from './chunk-VD7JEUIR.mjs';
|
|
2
|
+
export { actionPartitionsOf, adoptSegmentsOnWorker, bytesThatSegmentsHoldInMemory, checkPagedSegmentFully, closeUnusedSegments, countsAsLatestHolder, createPagedFrozenSegment, isPagedSegmentFile, keysThatSegmentsStillRead, mergePagedWindow, openFileVectorReads, openPagedSegmentScan, openPagedSnapshot, openRewriteSpill, pagedListingOf, pagedPartitionLines, pagedTombstonesFor, prepareDiskAdoption, startReadingOpeningSectionsAhead, swapPreparedPartitions, writePagedSegment } from './chunk-VD7JEUIR.mjs';
|
|
3
|
+
import './chunk-7ZJTRUVW.mjs';
|
|
4
|
+
import { COLD_LOAD_WAIT_WITHOUT_SEARCH_MS, loadFailed, decodeSections, SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES, COLD_WALK_NODES_READ_AHEAD } from './chunk-QJCO3TJU.mjs';
|
|
5
|
+
import './chunk-B3JM6Z74.mjs';
|
|
6
|
+
import './chunk-HT7J73QL.mjs';
|
|
7
|
+
import { magnitude, defaultRescoreDepth, defaultEfSearch, computeCodeBlockLayout, toScore, osqQuantize, createOsqScratch, scalarQueryProducts, osqEstimate, osqDistance, toDistance } from './chunk-TQ5UMFG5.mjs';
|
|
8
|
+
export { createHNSWIndex, osqBitsOf } from './chunk-TQ5UMFG5.mjs';
|
|
9
|
+
import { isDead, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors, stageQueryLevels, readTrailer } from './chunk-55WYRLDZ.mjs';
|
|
10
10
|
import './chunk-LURXVCHG.mjs';
|
|
11
|
-
import { yieldToEventLoop, HEADER_SIZE, crc32 } from './chunk-
|
|
11
|
+
import { yieldToEventLoop, HEADER_SIZE, crc32, readHeader } from './chunk-KWYQ6KX4.mjs';
|
|
12
12
|
import './chunk-MPK76GVS.mjs';
|
|
13
13
|
import './chunk-JPTQZWJQ.mjs';
|
|
14
14
|
import './chunk-26A5JFT2.mjs';
|
|
@@ -307,9 +307,15 @@ var BIN16_MAX_BYTES = 65535;
|
|
|
307
307
|
var VECTOR_PAGE_SIZES = /* @__PURE__ */ new Set([SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES]);
|
|
308
308
|
var NEIGHBOUR_PAGE_SIZES = /* @__PURE__ */ new Set([RECORD_PAGE_BYTES]);
|
|
309
309
|
var LEVEL_PAGE_SIZES = /* @__PURE__ */ new Set([SEGMENT_PAGE_BYTES]);
|
|
310
|
+
function holdsRawPayload(path) {
|
|
311
|
+
const header = new Uint8Array(HEADER_SIZE);
|
|
312
|
+
readFileRange(path, 0, header);
|
|
313
|
+
return !readHeader(header).flags.compressionEnabled;
|
|
314
|
+
}
|
|
310
315
|
function readHead(path, ref) {
|
|
311
316
|
const { headOffset, headLength, headChecksum } = ref;
|
|
312
317
|
if (headOffset === void 0 || headLength === void 0 || headChecksum === void 0) return null;
|
|
318
|
+
if (!holdsRawPayload(path)) return null;
|
|
313
319
|
const head = new Uint8Array(headLength);
|
|
314
320
|
readFileRange(path, HEADER_SIZE + headOffset, head);
|
|
315
321
|
if (crc32(head) !== headChecksum) {
|
|
@@ -663,11 +669,11 @@ function openColdVectorField(input) {
|
|
|
663
669
|
var FILE_PAGE_SIZES = /* @__PURE__ */ new Set([SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES]);
|
|
664
670
|
var FLOAT32_BYTES = 4;
|
|
665
671
|
var FLOAT64_BYTES2 = 8;
|
|
666
|
-
function waitBeforeColdLoad(
|
|
672
|
+
function waitBeforeColdLoad(loadingWanted) {
|
|
667
673
|
return new Promise((resolve) => {
|
|
668
674
|
const timer = setTimeout(resolve, COLD_LOAD_WAIT_WITHOUT_SEARCH_MS);
|
|
669
675
|
if (typeof timer.unref === "function") timer.unref();
|
|
670
|
-
void
|
|
676
|
+
void loadingWanted.then(() => {
|
|
671
677
|
clearTimeout(timer);
|
|
672
678
|
resolve();
|
|
673
679
|
});
|
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
import { writeMetadataEnvelope, decodeSegmentManifest, metadataFromConfig, SINGLE_NODE_PRIMARY_TERM, readMetadataEnvelope, createSeqOwner, readSegmentManifest, reclaimUnlistedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, nextSegmentKey, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, frameRecord, isId, isUint32, isSequenceNumber, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, commitCheckpointManifest, removeCheckpointGarbage, readWalSegments, durableRecordMissing, growPartitionsTo, writeSegmentHeader, VECTOR_FILE_ENVELOPE_VERSION,
|
|
1
|
+
import { writeMetadataEnvelope, decodeSegmentManifest, metadataFromConfig, SINGLE_NODE_PRIMARY_TERM, readMetadataEnvelope, createSeqOwner, readSegmentManifest, reclaimUnlistedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, nextSegmentKey, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, frameRecord, isId, isUint32, isSequenceNumber, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, commitCheckpointManifest, removeCheckpointGarbage, readWalSegments, durableRecordMissing, growPartitionsTo, writeSegmentHeader, VECTOR_FILE_ENVELOPE_VERSION, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, documentsOfPayloadV2, SEGMENT_HEADER_SIZE, metadataConfigOf, durableMetadataConfigOf, buildEntry, readCurrentManifest, captureWholePartition } from './chunk-GJKXW3SF.mjs';
|
|
2
2
|
import './chunk-JNOAKDUM.mjs';
|
|
3
|
-
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, readListedFile, adapterCheckpointStore, DIRECTORY_LOCK_ATTEMPTS, snapshotBundleKey, manifestReferencedKeys, manifestFileRefs, DIRECTORY_LOCK_SETTLE_MS, readSegmentFile, MAX_SEGMENTS_PER_PARTITION, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-
|
|
4
|
-
import { applyDeleteEntry, applyIndexEntry } from './chunk-
|
|
3
|
+
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, readListedFile, adapterCheckpointStore, DIRECTORY_LOCK_ATTEMPTS, snapshotBundleKey, manifestReferencedKeys, manifestFileRefs, DIRECTORY_LOCK_SETTLE_MS, readSegmentFile, MAX_SEGMENTS_PER_PARTITION, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-QJCO3TJU.mjs';
|
|
4
|
+
import { applyDeleteEntry, applyIndexEntry } from './chunk-B3JM6Z74.mjs';
|
|
5
5
|
import { nodeThreadId, spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
|
|
6
|
-
import
|
|
7
|
-
import {
|
|
6
|
+
import './chunk-3I27KXQD.mjs';
|
|
7
|
+
import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-55WYRLDZ.mjs';
|
|
8
|
+
import { encodeDocument, toError, requireValidFieldTypes, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-KWYQ6KX4.mjs';
|
|
8
9
|
import { fnv1a, createRequestId } from './chunk-MPK76GVS.mjs';
|
|
9
10
|
import './chunk-JPTQZWJQ.mjs';
|
|
10
11
|
import './chunk-26A5JFT2.mjs';
|
|
@@ -1090,6 +1091,34 @@ function formatSegmentStart(startSeqNo) {
|
|
|
1090
1091
|
return startSeqNo.toString().padStart(16, "0");
|
|
1091
1092
|
}
|
|
1092
1093
|
|
|
1094
|
+
// src/persistence/durability/marker-writer.ts
|
|
1095
|
+
function markerKey(indexName, partitionId) {
|
|
1096
|
+
return `${indexName}/wal/${partitionId}/commit`;
|
|
1097
|
+
}
|
|
1098
|
+
async function createMarkerWriter(directory, indexName, partitionId) {
|
|
1099
|
+
const key = markerKey(indexName, partitionId);
|
|
1100
|
+
const handle = await directory.markerHandle(key);
|
|
1101
|
+
const existing = await handle.read();
|
|
1102
|
+
const current = readCommitMarker(existing);
|
|
1103
|
+
const createdFresh = existing.length === 0;
|
|
1104
|
+
let lastSlot = current?.slotIndex ?? null;
|
|
1105
|
+
let writeSeq = current?.state.writeSeq ?? 0;
|
|
1106
|
+
return {
|
|
1107
|
+
created: createdFresh,
|
|
1108
|
+
existingHighestDurableSeqNo: current?.state.highestDurableSeqNo ?? 0,
|
|
1109
|
+
async commit(state) {
|
|
1110
|
+
const slotIndex = nextMarkerSlot(lastSlot);
|
|
1111
|
+
writeSeq += 1;
|
|
1112
|
+
const slot = encodeMarkerSlot({ ...state, writeSeq });
|
|
1113
|
+
await handle.writeSlot(markerSlotOffset(slotIndex), slot);
|
|
1114
|
+
lastSlot = slotIndex;
|
|
1115
|
+
},
|
|
1116
|
+
async close() {
|
|
1117
|
+
await handle.close();
|
|
1118
|
+
}
|
|
1119
|
+
};
|
|
1120
|
+
}
|
|
1121
|
+
|
|
1093
1122
|
// src/persistence/durability/recovery.ts
|
|
1094
1123
|
async function removeLeftoverRewriteFiles(directory, names) {
|
|
1095
1124
|
for (const indexName of names) {
|
|
@@ -1146,11 +1175,33 @@ async function replayWal(directory, indexName, partitionId, fromSeqNoExclusive,
|
|
|
1146
1175
|
await replay(activeTail.entries);
|
|
1147
1176
|
}
|
|
1148
1177
|
await deleteOrphanSegments(directory, segments, marker.state.activeSegmentSeqNo);
|
|
1149
|
-
if (activeTail !== null
|
|
1150
|
-
await
|
|
1178
|
+
if (activeTail !== null) {
|
|
1179
|
+
await makeReplayedTailDurable(directory, indexName, partitionId, marker, activeTail);
|
|
1151
1180
|
}
|
|
1152
1181
|
return { highestSeqNo, highestPrimaryTerm };
|
|
1153
1182
|
}
|
|
1183
|
+
async function makeReplayedTailDurable(directory, indexName, partitionId, marker, tail) {
|
|
1184
|
+
const lastReplayed = tail.entries.at(-1);
|
|
1185
|
+
if (lastReplayed === void 0 && tail.cleanEnd >= tail.segmentLength) return;
|
|
1186
|
+
const handle = await directory.appendHandle(tail.key);
|
|
1187
|
+
try {
|
|
1188
|
+
if (tail.cleanEnd < await handle.size()) await handle.truncate(tail.cleanEnd);
|
|
1189
|
+
else await handle.sync();
|
|
1190
|
+
} finally {
|
|
1191
|
+
await handle.close();
|
|
1192
|
+
}
|
|
1193
|
+
if (lastReplayed === void 0) return;
|
|
1194
|
+
const writer = await createMarkerWriter(directory, indexName, partitionId);
|
|
1195
|
+
try {
|
|
1196
|
+
await writer.commit({
|
|
1197
|
+
activeSegmentSeqNo: marker.state.activeSegmentSeqNo,
|
|
1198
|
+
durableByteLength: tail.cleanEnd,
|
|
1199
|
+
highestDurableSeqNo: Math.max(marker.state.highestDurableSeqNo, lastReplayed.seqNo)
|
|
1200
|
+
});
|
|
1201
|
+
} finally {
|
|
1202
|
+
await writer.close();
|
|
1203
|
+
}
|
|
1204
|
+
}
|
|
1154
1205
|
async function deleteOrphanSegments(directory, segments, activeSegmentSeqNo) {
|
|
1155
1206
|
for (const { key, startSeqNo } of segments) {
|
|
1156
1207
|
if (startSeqNo > activeSegmentSeqNo) {
|
|
@@ -1158,18 +1209,6 @@ async function deleteOrphanSegments(directory, segments, activeSegmentSeqNo) {
|
|
|
1158
1209
|
}
|
|
1159
1210
|
}
|
|
1160
1211
|
}
|
|
1161
|
-
async function truncateSegmentTail(directory, key, durableByteLength) {
|
|
1162
|
-
const cleanLength = Math.max(durableByteLength, SEGMENT_HEADER_SIZE);
|
|
1163
|
-
const handle = await directory.appendHandle(key);
|
|
1164
|
-
try {
|
|
1165
|
-
const currentSize = await handle.size();
|
|
1166
|
-
if (cleanLength < currentSize) {
|
|
1167
|
-
await handle.truncate(cleanLength);
|
|
1168
|
-
}
|
|
1169
|
-
} finally {
|
|
1170
|
-
await handle.close();
|
|
1171
|
-
}
|
|
1172
|
-
}
|
|
1173
1212
|
function applyEntry(entry, deps) {
|
|
1174
1213
|
deps.writeRecords?.replayed(entry);
|
|
1175
1214
|
if (entry.operation === "DELETE") {
|
|
@@ -1678,7 +1717,7 @@ async function writeOneBoundedCheckpoint(input) {
|
|
|
1678
1717
|
}
|
|
1679
1718
|
async function adoptWrittenSegments(input, manager, partitions, changes) {
|
|
1680
1719
|
const { indexName, indexState, hooks } = input;
|
|
1681
|
-
const { actionPartitionsOf, closeUnusedSegments, prepareDiskAdoption, swapPreparedPartitions } = await import('./module-
|
|
1720
|
+
const { actionPartitionsOf, closeUnusedSegments, prepareDiskAdoption, swapPreparedPartitions } = await import('./module-KAMRAJLE.mjs');
|
|
1682
1721
|
const adoption = await prepareDiskAdoption(input.directory, manager, partitions);
|
|
1683
1722
|
if (adoption === null) return [];
|
|
1684
1723
|
const adopted = await whileNoMutationApplies([...indexState.partitions.values()], () => {
|
|
@@ -1733,7 +1772,7 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
|
|
|
1733
1772
|
const droppedThisCheckpoint = changes !== null && capture2.recordsLeftInLog === 0 ? new Set(await adoptWrittenSegments(input, manager, segments.partitions, changes)) : /* @__PURE__ */ new Set();
|
|
1734
1773
|
if (!input.holdsCheckpointFiles()) {
|
|
1735
1774
|
const removable = await garbageThatNoFieldStillReads(store, vectorIndexes, written.garbage);
|
|
1736
|
-
const stillRead = metadata.textStorage === "disk" ? (await import('./module-
|
|
1775
|
+
const stillRead = metadata.textStorage === "disk" ? (await import('./module-KAMRAJLE.mjs')).keysThatSegmentsStillRead(manager) : /* @__PURE__ */ new Set();
|
|
1737
1776
|
await removeCheckpointGarbage(
|
|
1738
1777
|
store,
|
|
1739
1778
|
removable.filter((key) => !droppedThisCheckpoint.has(key) && !stillRead.has(key))
|
|
@@ -2221,34 +2260,6 @@ function createGroupCommitCoordinator(sync) {
|
|
|
2221
2260
|
};
|
|
2222
2261
|
}
|
|
2223
2262
|
|
|
2224
|
-
// src/persistence/durability/marker-writer.ts
|
|
2225
|
-
function markerKey(indexName, partitionId) {
|
|
2226
|
-
return `${indexName}/wal/${partitionId}/commit`;
|
|
2227
|
-
}
|
|
2228
|
-
async function createMarkerWriter(directory, indexName, partitionId) {
|
|
2229
|
-
const key = markerKey(indexName, partitionId);
|
|
2230
|
-
const handle = await directory.markerHandle(key);
|
|
2231
|
-
const existing = await handle.read();
|
|
2232
|
-
const current = readCommitMarker(existing);
|
|
2233
|
-
const createdFresh = existing.length === 0;
|
|
2234
|
-
let lastSlot = current?.slotIndex ?? null;
|
|
2235
|
-
let writeSeq = current?.state.writeSeq ?? 0;
|
|
2236
|
-
return {
|
|
2237
|
-
created: createdFresh,
|
|
2238
|
-
existingHighestDurableSeqNo: current?.state.highestDurableSeqNo ?? 0,
|
|
2239
|
-
async commit(state) {
|
|
2240
|
-
const slotIndex = nextMarkerSlot(lastSlot);
|
|
2241
|
-
writeSeq += 1;
|
|
2242
|
-
const slot = encodeMarkerSlot({ ...state, writeSeq });
|
|
2243
|
-
await handle.writeSlot(markerSlotOffset(slotIndex), slot);
|
|
2244
|
-
lastSlot = slotIndex;
|
|
2245
|
-
},
|
|
2246
|
-
async close() {
|
|
2247
|
-
await handle.close();
|
|
2248
|
-
}
|
|
2249
|
-
};
|
|
2250
|
-
}
|
|
2251
|
-
|
|
2252
2263
|
// src/persistence/durability/wal-writer.ts
|
|
2253
2264
|
var SEGMENT_TAIL_PATTERN = /^\d{16}$/;
|
|
2254
2265
|
function segmentKey(indexName, partitionId, startSeqNo) {
|
|
@@ -2977,7 +2988,7 @@ var decoder = new TextDecoder();
|
|
|
2977
2988
|
async function countPagedPartition(store, partition) {
|
|
2978
2989
|
const { directory } = store;
|
|
2979
2990
|
if (directory === void 0 || partition.segments.length === 0) return null;
|
|
2980
|
-
const { countsAsLatestHolder, isPagedSegmentFile, openPagedSegmentScan } = await import('./module-
|
|
2991
|
+
const { countsAsLatestHolder, isPagedSegmentFile, openPagedSegmentScan } = await import('./module-KAMRAJLE.mjs');
|
|
2981
2992
|
for (const ref of partition.segments) if (!await isPagedSegmentFile(directory, ref.key)) return null;
|
|
2982
2993
|
const scans = [];
|
|
2983
2994
|
try {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
export { DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, readDocumentExport } from './chunk-
|
|
1
|
+
export { DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, readDocumentExport } from './chunk-PM4MUM5N.mjs';
|
|
2
2
|
import './chunk-NVEJJP4R.mjs';
|
|
3
|
-
import './chunk-
|
|
4
|
-
import './chunk-
|
|
3
|
+
import './chunk-HT7J73QL.mjs';
|
|
4
|
+
import './chunk-KWYQ6KX4.mjs';
|
|
5
5
|
import './chunk-MPK76GVS.mjs';
|
|
6
6
|
import './chunk-JPTQZWJQ.mjs';
|
|
7
7
|
import './chunk-26A5JFT2.mjs';
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
export { actionPartitionsOf, adoptSegmentsOnWorker, bytesThatSegmentsHoldInMemory, checkPagedSegmentFully, closeUnusedSegments, countsAsLatestHolder, createPagedFrozenSegment, isPagedSegmentFile, keysThatSegmentsStillRead, mergePagedWindow, openFileVectorReads, openPagedSegmentScan, openPagedSnapshot, openRewriteSpill, pagedListingOf, pagedPartitionLines, pagedTombstonesFor, prepareDiskAdoption, startReadingOpeningSectionsAhead, swapPreparedPartitions, writePagedSegment } from './chunk-
|
|
2
|
-
import './chunk-
|
|
3
|
-
import './chunk-
|
|
4
|
-
import './chunk-
|
|
5
|
-
import './chunk-
|
|
1
|
+
export { actionPartitionsOf, adoptSegmentsOnWorker, bytesThatSegmentsHoldInMemory, checkPagedSegmentFully, closeUnusedSegments, countsAsLatestHolder, createPagedFrozenSegment, isPagedSegmentFile, keysThatSegmentsStillRead, mergePagedWindow, openFileVectorReads, openPagedSegmentScan, openPagedSnapshot, openRewriteSpill, pagedListingOf, pagedPartitionLines, pagedTombstonesFor, prepareDiskAdoption, startReadingOpeningSectionsAhead, swapPreparedPartitions, writePagedSegment } from './chunk-VD7JEUIR.mjs';
|
|
2
|
+
import './chunk-7ZJTRUVW.mjs';
|
|
3
|
+
import './chunk-QJCO3TJU.mjs';
|
|
4
|
+
import './chunk-B3JM6Z74.mjs';
|
|
5
|
+
import './chunk-HT7J73QL.mjs';
|
|
6
6
|
import './chunk-LURXVCHG.mjs';
|
|
7
|
-
import './chunk-
|
|
7
|
+
import './chunk-KWYQ6KX4.mjs';
|
|
8
8
|
import './chunk-MPK76GVS.mjs';
|
|
9
9
|
import './chunk-JPTQZWJQ.mjs';
|
|
10
10
|
import './chunk-26A5JFT2.mjs';
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
|
|
2
|
-
import { VECTOR_WORKER_COPIES_ALLOWED,
|
|
3
|
-
import {
|
|
4
|
-
|
|
5
|
-
|
|
2
|
+
import { VECTOR_WORKER_COPIES_ALLOWED, noBuildHistory, recordCheckpoint, assignStorePartitions, liveSize, ordinalFilterForSubsets, handOverBuildTurns, filterForOptions, graphNeedsRebuild, yieldBuildTurn, threadsHoldCurrentLayout, emptyFieldBeforeRestore, recalibrateFromStore, signatureOf, allLiveDocIds, fieldHandlesOf, releaseReplacedVectors, adoptGraph, liveOrdinals, adoptLoadedCalibration, planCheckpoint, deserialize } from './chunk-CSCKZBVJ.mjs';
|
|
3
|
+
import { DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, defaultQuantizationFor, DEFAULT_VECTOR_METRIC, ESTIMATED_MS_PER_TOMBSTONE, ESTIMATED_MS_PER_VECTOR_REBUILD, WORKER_COPY_MIN_VECTORS, SHUFFLED_REBUILD_MEMORY_RETRY_MS, SEARCH_ATTEMPTS_BEFORE_HOLDING_WRITES, BUILD_RETRY_MAX_DELAY_MS, BUILD_RETRY_FIRST_DELAY_MS, BUILD_STEPS_BETWEEN_CLOCK_CHECKS, RECALIBRATION_CHUNK_VECTORS, BUILD_CHUNK_SIZE, SEARCH_REFILL_RESERVE, SEARCH_SCAN_CHUNK_VECTORS, DRIFT_CHECK_GROWTH_FACTOR, RECALIBRATION_DRIFT_RATIO, DISK_ADOPTION_YIELD_INTERVAL } from './chunk-3I27KXQD.mjs';
|
|
4
|
+
import { openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, osqBitsOf, createOsqQuantizer, ordinalFilterValues, openGraphState, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, fixedView, adoptBackgroundScheduler, VECTOR_MEMORY_LIMITS, sharedVectorStoreBytes, STORE_SLOTS, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, turnsAreBudgeted, yieldBackgroundTurn, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, threadedNativeCore, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, slotByteOffset, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createSharedVectorStoreHandles, STORE_DOC_ID_BYTES, unplacedChunkCount, blocksGrowthBytes, growBufferTo, nativeStoreScratchBytes, noteFallback, toDistance, holdWritesAndBackgroundWork, QUERY_STAGED, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, UI_THREAD_HOP_BEFORE_TURN_WITHIN_MS, grownByteLength, createVectorBlock, ensureBlockSlots, sharedMemoryAvailable, UI_THREAD_TURN_BUDGET_MS, createHNSWIndex, topLayerInParts, shuffleInPlace, requireWholeNeighbourLists, backgroundTurnIsSpent, createCentroidSums, shuffleStep, centroidOfCentroidSums, meanOfCentroidSums, countOfCentroidSums, squaredLengthsOfCentroidSums, nativeStoreFor, withNativeThreadSlot, nativeMetricCode, NATIVE_OK, releaseLocksHeldBy, UI_THREAD_BACKGROUND_SLICE_DURING_WRITES_MS } from './chunk-TQ5UMFG5.mjs';
|
|
5
|
+
export { releaseLocksHeldBy } from './chunk-TQ5UMFG5.mjs';
|
|
6
|
+
import { NO_LIVE_VECTOR_AT_POSITION, bytesToVectors, isDead } from './chunk-55WYRLDZ.mjs';
|
|
6
7
|
import { createVectorFileReader } from './chunk-LURXVCHG.mjs';
|
|
7
8
|
import './chunk-OIZS5KIX.mjs';
|
|
8
9
|
import { createDocIdMap, compareCodePoints, turnIsSpent, yieldTurn, resolveWorkerEntry, detectRuntime, noteThreadSlotLive, noteThreadSlotGone } from './chunk-IAXZAVYL.mjs';
|
|
@@ -3183,11 +3184,11 @@ function beginCheckpointRestore(state, shape) {
|
|
|
3183
3184
|
}
|
|
3184
3185
|
|
|
3185
3186
|
// src/vector/vector-index/cold.ts
|
|
3186
|
-
function beginColdHold(state, field, loaded,
|
|
3187
|
+
function beginColdHold(state, field, loaded, loadWanted) {
|
|
3187
3188
|
const hold = {
|
|
3188
3189
|
field,
|
|
3189
3190
|
loaded,
|
|
3190
|
-
|
|
3191
|
+
loadWanted,
|
|
3191
3192
|
written: /* @__PURE__ */ new Map(),
|
|
3192
3193
|
removedNumbers: /* @__PURE__ */ new Set(),
|
|
3193
3194
|
removedIds: /* @__PURE__ */ new Set(),
|
|
@@ -3204,6 +3205,12 @@ function endColdHold(state) {
|
|
|
3204
3205
|
for (const write of hold.writes) write();
|
|
3205
3206
|
hold.field.close();
|
|
3206
3207
|
}
|
|
3208
|
+
async function untilColdLoaded(state) {
|
|
3209
|
+
while (state.cold !== null) {
|
|
3210
|
+
state.cold.loadWanted();
|
|
3211
|
+
await state.cold.loaded;
|
|
3212
|
+
}
|
|
3213
|
+
}
|
|
3207
3214
|
function cancelColdHold(state) {
|
|
3208
3215
|
const hold = state.cold;
|
|
3209
3216
|
if (hold === null) return;
|
|
@@ -3218,15 +3225,15 @@ function openColdField(state, field, load) {
|
|
|
3218
3225
|
finished = resolve;
|
|
3219
3226
|
failed = reject;
|
|
3220
3227
|
});
|
|
3221
|
-
let
|
|
3222
|
-
const
|
|
3223
|
-
|
|
3228
|
+
let loadWanted = () => void 0;
|
|
3229
|
+
const loadingWanted = new Promise((resolve) => {
|
|
3230
|
+
loadWanted = resolve;
|
|
3224
3231
|
});
|
|
3225
|
-
const hold = beginColdHold(state, field, loaded,
|
|
3232
|
+
const hold = beginColdHold(state, field, loaded, loadWanted);
|
|
3226
3233
|
void load(
|
|
3227
3234
|
(shape) => beginCheckpointRestore(state, shape),
|
|
3228
3235
|
() => state.cold === hold,
|
|
3229
|
-
|
|
3236
|
+
loadingWanted
|
|
3230
3237
|
).then(
|
|
3231
3238
|
() => {
|
|
3232
3239
|
if (state.cold === hold) endColdHold(state);
|
|
@@ -3381,21 +3388,21 @@ function outcomeOfGraphWalk(hold, walk, fromGraph, query, k, options) {
|
|
|
3381
3388
|
function coldSearch(state, hold, query, k, options) {
|
|
3382
3389
|
const walk = outcomeWithoutGraphWalk(state, hold, query, k, options);
|
|
3383
3390
|
if ("results" in walk) {
|
|
3384
|
-
hold.
|
|
3391
|
+
hold.loadWanted();
|
|
3385
3392
|
return walk;
|
|
3386
3393
|
}
|
|
3387
3394
|
const fromGraph = hold.field.search(query, k, options, walk.admits, hold.removedNumbers);
|
|
3388
|
-
hold.
|
|
3395
|
+
hold.loadWanted();
|
|
3389
3396
|
return outcomeOfGraphWalk(hold, walk, fromGraph, query, k, options);
|
|
3390
3397
|
}
|
|
3391
3398
|
async function coldSearchConcurrently(state, hold, query, k, options) {
|
|
3392
3399
|
const walk = outcomeWithoutGraphWalk(state, hold, query, k, options);
|
|
3393
3400
|
if ("results" in walk) {
|
|
3394
|
-
hold.
|
|
3401
|
+
hold.loadWanted();
|
|
3395
3402
|
return walk;
|
|
3396
3403
|
}
|
|
3397
3404
|
const fromGraph = await hold.field.searchConcurrently(query, k, options, walk.admits, hold.removedNumbers);
|
|
3398
|
-
hold.
|
|
3405
|
+
hold.loadWanted();
|
|
3399
3406
|
return outcomeOfGraphWalk(hold, walk, fromGraph, query, k, options);
|
|
3400
3407
|
}
|
|
3401
3408
|
|
|
@@ -4053,10 +4060,6 @@ function createVectorWrites(state) {
|
|
|
4053
4060
|
}
|
|
4054
4061
|
|
|
4055
4062
|
// src/vector/vector-index/index.ts
|
|
4056
|
-
function defaultQuantizationFor(dimension) {
|
|
4057
|
-
if (dimension >= OSQ4_MIN_DIMENSION) return "osq4";
|
|
4058
|
-
return "osq8";
|
|
4059
|
-
}
|
|
4060
4063
|
function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_WORKER_COPIES_ALLOWED, indexName = "", storage = "memory", onLoadedGraphRepaired = () => void 0) {
|
|
4061
4064
|
if (!Number.isInteger(dimension) || dimension <= 0) {
|
|
4062
4065
|
throw new NarsilError(
|
|
@@ -4070,7 +4073,7 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
4070
4073
|
const filterThreshold = Math.max(0, Math.min(1, config?.filterThreshold ?? DEFAULT_FILTER_THRESHOLD));
|
|
4071
4074
|
const quantizationMode = config?.quantization ?? defaultQuantizationFor(dimension);
|
|
4072
4075
|
const codeBits = osqBitsOf(quantizationMode);
|
|
4073
|
-
const metric = config?.hnswConfig?.metric ??
|
|
4076
|
+
const metric = config?.hnswConfig?.metric ?? DEFAULT_VECTOR_METRIC;
|
|
4074
4077
|
const rawHnswM = config?.hnswConfig?.m;
|
|
4075
4078
|
const hnswConfig = config?.hnswConfig ? { ...config.hnswConfig, m: rawHnswM !== void 0 ? Math.max(rawHnswM, 2) : void 0 } : void 0;
|
|
4076
4079
|
const dimensionScale = dimension / 256;
|
|
@@ -4140,9 +4143,7 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
4140
4143
|
if (state.cold !== null) return coldHas(state.cold, docId);
|
|
4141
4144
|
return state.store.has(docId) && !state.tombstones.has(docId);
|
|
4142
4145
|
}
|
|
4143
|
-
|
|
4144
|
-
while (state.cold !== null) await state.cold.loaded;
|
|
4145
|
-
}
|
|
4146
|
+
const whenLoaded = () => untilColdLoaded(state);
|
|
4146
4147
|
async function searchParallel(query, k, options) {
|
|
4147
4148
|
if (state.graphsHeldOffThread !== null) await untilGraphReleased(state);
|
|
4148
4149
|
if (state.cold !== null) return coldSearchConcurrently(state, state.cold, query, k, options);
|
package/dist/{react-native/chunks/restore-queue-OI5EH7R3.mjs → browser/restore-queue-UQ6AAUVX.mjs}
RENAMED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { abandonChangeOf } from './chunk-
|
|
1
|
+
import { abandonChangeOf } from './chunk-VBX2TI5V.mjs';
|
|
2
2
|
|
|
3
3
|
// src/narsil/restore-queue.ts
|
|
4
4
|
function restoreDepsOf(core, host) {
|
|
@@ -24,7 +24,7 @@ function restoreDepsOf(core, host) {
|
|
|
24
24
|
function queueRestore(core, host, restoresInFlight, indexName, data) {
|
|
25
25
|
const previous = restoresInFlight.get(indexName) ?? Promise.resolve();
|
|
26
26
|
const restored = previous.then(async () => {
|
|
27
|
-
const { restoreFromSnapshot } = await import('./snapshot-
|
|
27
|
+
const { restoreFromSnapshot } = await import('./snapshot-UGZVMP4H.mjs');
|
|
28
28
|
await restoreFromSnapshot(indexName, data, restoreDepsOf(core, host));
|
|
29
29
|
});
|
|
30
30
|
const settled = restored.then(
|
|
@@ -1,13 +1,14 @@
|
|
|
1
|
-
import { indexMetadataOf, writeCheckpointSegments, writeLiveVectors, commitCheckpointManifest, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, unpackIndexSnapshotEnvelope, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-
|
|
1
|
+
import { indexMetadataOf, writeCheckpointSegments, writeLiveVectors, commitCheckpointManifest, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, unpackIndexSnapshotEnvelope, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-GJKXW3SF.mjs';
|
|
2
2
|
import { createByteReader, restoredConfigFields, restoredEmbedding, rejectSnapshot } from './chunk-NVEJJP4R.mjs';
|
|
3
3
|
import './chunk-JNOAKDUM.mjs';
|
|
4
|
-
import { DEFAULT_COMPACTION_THRESHOLD, manifestKey, manifestFileRefs, encodeSegmentManifest, readSegmentFile } from './chunk-
|
|
5
|
-
import './chunk-
|
|
6
|
-
import { vectorsPerFile } from './chunk-
|
|
7
|
-
import './chunk-
|
|
8
|
-
import
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
4
|
+
import { DEFAULT_COMPACTION_THRESHOLD, manifestKey, manifestFileRefs, encodeSegmentManifest, readSegmentFile } from './chunk-QJCO3TJU.mjs';
|
|
5
|
+
import './chunk-B3JM6Z74.mjs';
|
|
6
|
+
import { vectorsPerFile } from './chunk-CSCKZBVJ.mjs';
|
|
7
|
+
import './chunk-3I27KXQD.mjs';
|
|
8
|
+
import './chunk-TQ5UMFG5.mjs';
|
|
9
|
+
import { decodeVectorIndexParts, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors } from './chunk-55WYRLDZ.mjs';
|
|
10
|
+
import { MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, requireRestoreMemory, earlierSnapshotFootprint, archiveFootprint, storedBytesOfSegments } from './chunk-2A2IPRC3.mjs';
|
|
11
|
+
import { encodeIndexConfiguration, hasNrslMagic, HEADER_SIZE, readHeader, ARRAY_POSITION_GAP, raiseEarlierReleaseGraphSetting, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateTextStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, storedArrayPositionGap, prepareStoredSynonyms, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, isCompositePartition, arrayPositionsAreStale, insertDocumentVectors, decodeIndexConfiguration, flattenSchema } from './chunk-KWYQ6KX4.mjs';
|
|
11
12
|
import './chunk-MPK76GVS.mjs';
|
|
12
13
|
import './chunk-JPTQZWJQ.mjs';
|
|
13
14
|
import './chunk-26A5JFT2.mjs';
|
|
@@ -561,7 +562,7 @@ async function validateStagedCheckpoint(store, manifest) {
|
|
|
561
562
|
for (const ref of partition.segments) {
|
|
562
563
|
const { directory } = store;
|
|
563
564
|
if (directory !== void 0) {
|
|
564
|
-
const { checkPagedSegmentFully, isPagedSegmentFile } = await import('./module-
|
|
565
|
+
const { checkPagedSegmentFully, isPagedSegmentFile } = await import('./module-KAMRAJLE.mjs');
|
|
565
566
|
if (await isPagedSegmentFile(directory, ref.key)) {
|
|
566
567
|
requireSchemaOfManifest(ref.key, await checkPagedSegmentFully(directory, ref), manifest);
|
|
567
568
|
continue;
|
|
@@ -929,7 +930,7 @@ async function stageArchive(indexName, reader, staging, deps) {
|
|
|
929
930
|
if (manifest === null) missingPart("manifest");
|
|
930
931
|
const { directory } = staging;
|
|
931
932
|
if (keepsTextOnDisk(prepared, staging) && directory !== void 0) {
|
|
932
|
-
const { bytesThatSegmentsHoldInMemory } = await import('./module-
|
|
933
|
+
const { bytesThatSegmentsHoldInMemory } = await import('./module-KAMRAJLE.mjs');
|
|
933
934
|
const replacesIndex = deps.indexRegistry.has(indexName);
|
|
934
935
|
const segmentBytes = await bytesThatSegmentsHoldInMemory(directory, manifest);
|
|
935
936
|
requireRestoreMemory(indexName, archiveFootprint(prepared, manifest, replacesIndex, segmentBytes), replacesIndex);
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-
|
|
1
|
+
import { DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-PM4MUM5N.mjs';
|
|
2
2
|
import './chunk-NVEJJP4R.mjs';
|
|
3
|
-
import { encodeExactJson, encodeDocumentJson } from './chunk-
|
|
4
|
-
import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-
|
|
5
|
-
import './chunk-
|
|
6
|
-
import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-
|
|
7
|
-
import { decodeStoredDocument, openChecksummedEnvelope } from './chunk-
|
|
3
|
+
import { encodeExactJson, encodeDocumentJson } from './chunk-7ZJTRUVW.mjs';
|
|
4
|
+
import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-B3JM6Z74.mjs';
|
|
5
|
+
import './chunk-HT7J73QL.mjs';
|
|
6
|
+
import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-55WYRLDZ.mjs';
|
|
7
|
+
import { decodeStoredDocument, openChecksummedEnvelope } from './chunk-KWYQ6KX4.mjs';
|
|
8
8
|
import './chunk-MPK76GVS.mjs';
|
|
9
9
|
import './chunk-JPTQZWJQ.mjs';
|
|
10
10
|
import './chunk-26A5JFT2.mjs';
|
|
@@ -94,7 +94,7 @@ async function wholeFileVectorReads(checkpoint) {
|
|
|
94
94
|
};
|
|
95
95
|
}
|
|
96
96
|
async function* pagedDocumentLines(checkpoint) {
|
|
97
|
-
const { openFileVectorReads, pagedPartitionLines } = await import('./module-
|
|
97
|
+
const { openFileVectorReads, pagedPartitionLines } = await import('./module-KAMRAJLE.mjs');
|
|
98
98
|
const vectors = await openFileVectorReads(checkpoint, () => wholeFileVectorReads(checkpoint));
|
|
99
99
|
try {
|
|
100
100
|
for (const partition of checkpoint.manifest.partitions) {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { DEFAULT_BLOCK_WRITES_ABOVE, DEFAULT_RESUME_WRITES_BELOW, validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, executeQuery, lazyQueryBinding, executePreflight, executeSuggest, executeListDocuments, ENGINE_MEMORY_FIGURES_MAX_AGE_MS, storedBM25Params, readEngineMemoryFigures, FULL_MEMORY_COLLECTION_INTERVAL_MS, validateBatchList, validateDocId } from './chunk-6F6USYII.mjs';
|
|
2
|
-
import { resolvePartialWordAnalysis, resolvePartitionInsertOptions, isEmptySynonymSet, synonymModule, createDirectExecutor, generateId, createPartitionRouter, availableSynonymModule, mergeFrozenSegments, createSharedFrozenSegment, freezeSegmentShared, createFrozenSegment } from './chunk-
|
|
2
|
+
import { resolvePartialWordAnalysis, resolvePartitionInsertOptions, isEmptySynonymSet, synonymModule, createDirectExecutor, generateId, createPartitionRouter, availableSynonymModule, mergeFrozenSegments, createSharedFrozenSegment, freezeSegmentShared, createFrozenSegment } from './chunk-UGVUW27S.mjs';
|
|
3
3
|
import { ARRAY_POSITION_GAP, requirePatternSearchFor, requirePartialWordSearchFor, loadPhraseCodeFor, collectQueryTermStats, requireLoadedPhraseCode, readHeapStatistics, createPartitionIndex, mergePartitionStats, oversampledFacetConfig, availablePatternSearch, PATTERN_HOST, kWayMerge, mergeFacets, isCompositePartition, encodeSegmentState } from './chunk-3HV7M5OJ.mjs';
|
|
4
4
|
import { fnv1a } from './chunk-KAKQ74LS.mjs';
|
|
5
5
|
import { DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS } from './chunk-HJSN7UYG.mjs';
|
|
@@ -9,7 +9,7 @@ import { countEngineThread } from './chunk-Q2UL2TPC.mjs';
|
|
|
9
9
|
import { splitWorkerBudget, resolveWorkerCount, DEFAULT_REQUEST_TIMEOUT_MS, resolveRequestThreadCount, DEFAULT_BACKPRESSURE_LIMIT, SHUTDOWN_TIMEOUT_MS, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH } from './chunk-3MXV3VHR.mjs';
|
|
10
10
|
import { MAX_INDEX_CONFIGURATION_BYTES, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, FOREIGN_STATS_TTL_MS, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, BATCH_CHUNK_SIZE } from './chunk-EEBCDP36.mjs';
|
|
11
11
|
import { getLanguage, withOpenIndex, oneAtATime } from './chunk-2HJWQPR6.mjs';
|
|
12
|
-
import { loadVectorCodeFor, loadedVectorCode, loadVectorCode, requireLoadedVectorCode } from './chunk-
|
|
12
|
+
import { loadVectorCodeFor, loadedVectorCode, loadVectorCode, requireLoadedVectorCode } from './chunk-DGRM4EZZ.mjs';
|
|
13
13
|
import { createRequestId, isValidWorkerResponse } from './chunk-QHPYS427.mjs';
|
|
14
14
|
import { collectAndReturnPages } from './chunk-QYRNLLLI.mjs';
|
|
15
15
|
import { HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE } from './chunk-ZXNQEW43.mjs';
|
|
@@ -608,7 +608,7 @@ var TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS = 1e3;
|
|
|
608
608
|
// src/partitioning/rewrite-spill-of.ts
|
|
609
609
|
async function rewriteSpillOf(storage, manager, target) {
|
|
610
610
|
if (storage === void 0 || (target?.config ?? manager.config).textStorage !== "disk") return null;
|
|
611
|
-
const { openRewriteSpill } = await import('./module-
|
|
611
|
+
const { openRewriteSpill } = await import('./module-6RLIKSZQ.mjs');
|
|
612
612
|
return openRewriteSpill(storage, manager, target);
|
|
613
613
|
}
|
|
614
614
|
|
|
@@ -1152,7 +1152,7 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
1152
1152
|
const existing = deps.indexRegistry.get(metadata.indexName);
|
|
1153
1153
|
if (existing !== void 0) {
|
|
1154
1154
|
if ((metadata.analysisGeneration ?? 0) > existing.analysisGeneration) {
|
|
1155
|
-
const { adoptSavedAnalysis } = await import('./analysis-change-
|
|
1155
|
+
const { adoptSavedAnalysis } = await import('./analysis-change-2OYM7ARW.mjs');
|
|
1156
1156
|
adoptSavedAnalysis(deps.executor.getManager(metadata.indexName), existing, metadata);
|
|
1157
1157
|
}
|
|
1158
1158
|
if (loadData && deps.executor.getManager(metadata.indexName) === void 0) {
|
|
@@ -3999,7 +3999,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
3999
3999
|
if (config?.lifecycle !== void 0 && durabilityTier === null) {
|
|
4000
4000
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
|
|
4001
4001
|
}
|
|
4002
|
-
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-
|
|
4002
|
+
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-7VBZEJLG.mjs');
|
|
4003
4003
|
const filesystemDurability = durabilityTier?.kind === "wal";
|
|
4004
4004
|
const executor = createDirectExecutor({
|
|
4005
4005
|
vectorWorkerCopies: vectorCopyPolicy,
|
|
@@ -4184,7 +4184,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
4184
4184
|
requireManager,
|
|
4185
4185
|
resumeChange: async (indexName) => {
|
|
4186
4186
|
if ((indexRegistry.get(indexName)?.pendingAnalysis ?? null) === null) return;
|
|
4187
|
-
await (await import('./analysis-change-entry-
|
|
4187
|
+
await (await import('./analysis-change-entry-QD7YNDEE.mjs')).resumeChangeOf(engineCore, indexName);
|
|
4188
4188
|
},
|
|
4189
4189
|
onClose: hooks?.onIndexClose,
|
|
4190
4190
|
onOpen: (indexName) => {
|
|
@@ -4269,7 +4269,7 @@ async function startEngineCore(core, config) {
|
|
|
4269
4269
|
checkHeapAfterRecovery(core);
|
|
4270
4270
|
}
|
|
4271
4271
|
if (core.invalidation) await core.invalidation.start();
|
|
4272
|
-
if (core.durability) await (await import('./analysis-change-entry-
|
|
4272
|
+
if (core.durability) await (await import('./analysis-change-entry-QD7YNDEE.mjs')).resumeEveryChange(core);
|
|
4273
4273
|
if (config?.lifecycle === void 0) await core.analysisRebuild.reviewStaleIndexes();
|
|
4274
4274
|
} catch (err) {
|
|
4275
4275
|
await core.durability?.manager.shutdown().catch(() => void 0);
|
|
@@ -4856,7 +4856,7 @@ async function dropEngineIndex(core, name) {
|
|
|
4856
4856
|
const entry = core.requireIndex(name);
|
|
4857
4857
|
const changeRecorded = () => entry.pendingAnalysis !== null || entry.durableAnalysis !== null;
|
|
4858
4858
|
const abandonChange = async () => {
|
|
4859
|
-
await (await import('./analysis-change-entry-
|
|
4859
|
+
await (await import('./analysis-change-entry-QD7YNDEE.mjs')).abandonChangeOf(core, name);
|
|
4860
4860
|
};
|
|
4861
4861
|
if (changeRecorded()) await abandonChange();
|
|
4862
4862
|
await core.indexState.drop(name, async () => {
|
|
@@ -6736,14 +6736,14 @@ function portabilityOperations(core, config, engine) {
|
|
|
6736
6736
|
return {
|
|
6737
6737
|
snapshot(indexName) {
|
|
6738
6738
|
return withOpenIndex2(indexName, async () => {
|
|
6739
|
-
const { createSnapshot } = await import('./snapshot-
|
|
6739
|
+
const { createSnapshot } = await import('./snapshot-WMX6JCLP.mjs');
|
|
6740
6740
|
return createSnapshot(snapshotDepsOf(indexName));
|
|
6741
6741
|
});
|
|
6742
6742
|
},
|
|
6743
6743
|
async restore(indexName, data) {
|
|
6744
6744
|
core.guardShutdown();
|
|
6745
6745
|
validateIndexName(indexName);
|
|
6746
|
-
const restored = import('./restore-queue-
|
|
6746
|
+
const restored = import('./restore-queue-AOF5OTQV.mjs').then(
|
|
6747
6747
|
({ queueRestore }) => queueRestore(core, restoreHost, restoresInFlight, indexName, data)
|
|
6748
6748
|
);
|
|
6749
6749
|
await emitWriteEventAfter(core, indexName, restored);
|
|
@@ -6752,8 +6752,8 @@ function portabilityOperations(core, config, engine) {
|
|
|
6752
6752
|
exportDocuments(indexName) {
|
|
6753
6753
|
return withOpenIndex2(indexName, async () => {
|
|
6754
6754
|
const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
|
|
6755
|
-
import('./snapshot-
|
|
6756
|
-
import('./write-
|
|
6755
|
+
import('./snapshot-WMX6JCLP.mjs'),
|
|
6756
|
+
import('./write-SKJ6HW7O.mjs')
|
|
6757
6757
|
]);
|
|
6758
6758
|
const header = indexConfigurationOf(core.requireIndex(indexName));
|
|
6759
6759
|
return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
|
|
@@ -6761,7 +6761,7 @@ function portabilityOperations(core, config, engine) {
|
|
|
6761
6761
|
},
|
|
6762
6762
|
importDocumentExport(indexName, data) {
|
|
6763
6763
|
core.guardShutdown();
|
|
6764
|
-
const imported = import('./import-
|
|
6764
|
+
const imported = import('./import-GJOMB2HC.mjs').then(
|
|
6765
6765
|
({ readDocumentExport }) => readDocumentExport(engine(), indexName, data)
|
|
6766
6766
|
);
|
|
6767
6767
|
return emitWriteEventAfter(core, indexName, imported);
|
|
@@ -7197,7 +7197,7 @@ function createNarsilFromCore(core, config) {
|
|
|
7197
7197
|
const entry = requireIndex(indexName);
|
|
7198
7198
|
await withOpenIndexWrite(indexName, async () => {
|
|
7199
7199
|
if (entry.pendingAnalysis === null) return core.analysisRebuild.rebuild(indexName);
|
|
7200
|
-
return (await import('./analysis-change-
|
|
7200
|
+
return (await import('./analysis-change-2OYM7ARW.mjs')).resumeAnalysisChange(core, indexName);
|
|
7201
7201
|
});
|
|
7202
7202
|
},
|
|
7203
7203
|
async changeAnalysis(indexName, change) {
|
|
@@ -7205,7 +7205,7 @@ function createNarsilFromCore(core, config) {
|
|
|
7205
7205
|
requireIndex(indexName);
|
|
7206
7206
|
const changed = withOpenIndex2(
|
|
7207
7207
|
indexName,
|
|
7208
|
-
async () => (await import('./analysis-change-
|
|
7208
|
+
async () => (await import('./analysis-change-2OYM7ARW.mjs')).changeIndexAnalysis(core, indexName, change)
|
|
7209
7209
|
);
|
|
7210
7210
|
await emitWriteEventAfter(core, indexName, changed);
|
|
7211
7211
|
},
|
|
@@ -7243,7 +7243,7 @@ function createNarsilFromCore(core, config) {
|
|
|
7243
7243
|
}
|
|
7244
7244
|
rebalancesStarting.set(indexName, started);
|
|
7245
7245
|
const rebalanced = withOpenIndex2(indexName, async () => {
|
|
7246
|
-
const { executeRebalance } = await import('./analysis-change-
|
|
7246
|
+
const { executeRebalance } = await import('./analysis-change-2OYM7ARW.mjs');
|
|
7247
7247
|
const running = executeRebalance(requireManager(indexName), indexName, targetPartitionCount, rebalanceCtx);
|
|
7248
7248
|
releaseWrites();
|
|
7249
7249
|
return running;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { restoredEmbedding, restoredPartitionConfig, readSharedConfigFields } from './chunk-IRWP524D.mjs';
|
|
2
2
|
import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR } from './chunk-JDDXV6UF.mjs';
|
|
3
|
-
import { generateId } from './chunk-
|
|
3
|
+
import { generateId } from './chunk-UGVUW27S.mjs';
|
|
4
4
|
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-EEBCDP36.mjs';
|
|
5
5
|
import { getLanguage, decodeSynonymRules } from './chunk-2HJWQPR6.mjs';
|
|
6
6
|
import { NarsilError, ErrorCodes } from './chunk-MDF6PDIA.mjs';
|