@delali/narsil 0.3.1-next.151 → 0.3.1-next.153
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{analysis-change-ZCJNYBJM.mjs → analysis-change-2OYM7ARW.mjs} +4 -4
- package/dist/{react-native/chunks/analysis-change-entry-BNRU6STO.mjs → analysis-change-entry-QD7YNDEE.mjs} +1 -1
- package/dist/browser/{analysis-change-M65AZQQG.mjs → analysis-change-2V2HZ2NY.mjs} +2 -2
- package/dist/{analysis-change-entry-YN3ELL6C.mjs → browser/analysis-change-entry-JL5U2N3Q.mjs} +1 -1
- package/dist/browser/{chunk-SRL47USS.mjs → chunk-2A2IPRC3.mjs} +1 -1
- package/dist/browser/chunk-3I27KXQD.mjs +34 -0
- package/dist/browser/{chunk-KAEVGLYN.mjs → chunk-55WYRLDZ.mjs} +15 -0
- package/dist/browser/{chunk-6OWX3OP2.mjs → chunk-7ZJTRUVW.mjs} +1 -1
- package/dist/browser/{chunk-QMNZFFD3.mjs → chunk-B3JM6Z74.mjs} +6 -2
- package/dist/browser/{chunk-BRWY3JCN.mjs → chunk-CSCKZBVJ.mjs} +4 -29
- package/dist/browser/{chunk-D6UFGBF5.mjs → chunk-GJKXW3SF.mjs} +43 -24
- package/dist/browser/{chunk-5V74FFZE.mjs → chunk-HT7J73QL.mjs} +1 -1
- package/dist/browser/{chunk-BVX2MEHA.mjs → chunk-KWYQ6KX4.mjs} +17 -17
- package/dist/browser/{chunk-2JABMB4S.mjs → chunk-PM4MUM5N.mjs} +2 -2
- package/dist/browser/{chunk-W643QV3L.mjs → chunk-QJCO3TJU.mjs} +8 -4
- package/dist/browser/{chunk-ODQP7P3H.mjs → chunk-TQ5UMFG5.mjs} +1 -1
- package/dist/browser/{chunk-4DYLCICG.mjs → chunk-VBX2TI5V.mjs} +2 -2
- package/dist/browser/{chunk-XILIT5MT.mjs → chunk-VD7JEUIR.mjs} +5 -5
- package/dist/browser/{cold-module-A5S6JFDN.mjs → cold-module-YECAK4MN.mjs} +18 -12
- package/dist/browser/{durability-wiring-W5AVV3RF.mjs → durability-wiring-LM7WFL6X.mjs} +61 -50
- package/dist/browser/{import-PTT63AGK.mjs → import-QOY6ZH6X.mjs} +3 -3
- package/dist/browser/{module-CEKA7MQH.mjs → module-KAMRAJLE.mjs} +6 -6
- package/dist/browser/{module-X5PGOE47.mjs → module-XQLEOYNM.mjs} +24 -23
- package/dist/{react-native/chunks/restore-queue-OI5EH7R3.mjs → browser/restore-queue-UQ6AAUVX.mjs} +2 -2
- package/dist/browser/{snapshot-DQ6A4PQP.mjs → snapshot-UGZVMP4H.mjs} +11 -10
- package/dist/browser/{write-MAGRAIEH.mjs → write-I5P5MW4O.mjs} +7 -7
- package/dist/{chunk-VQDZVWN3.mjs → chunk-2PNYKG2D.mjs} +16 -16
- package/dist/{chunk-U6NE6RCV.mjs → chunk-3E4RRF5J.mjs} +1 -1
- package/dist/{chunk-ZYKUD5NX.mjs → chunk-4QCSHLNM.mjs} +56 -6
- package/dist/{chunk-YKEHQUSZ.mjs → chunk-52QQWEBS.mjs} +2 -2
- package/dist/{chunk-KZ5GQE4Z.mjs → chunk-6RKSY3BX.mjs} +6 -6
- package/dist/{chunk-6C5EX2QR.mjs → chunk-7LDMWKP6.mjs} +1 -1
- package/dist/{chunk-ADBS5XRI.mjs → chunk-7TE2AJIH.mjs} +6 -6
- package/dist/{chunk-QN5WVV5K.mjs → chunk-CSHHU7PA.mjs} +5 -5
- package/dist/{chunk-ZAYWOHVG.mjs → chunk-DGRM4EZZ.mjs} +1 -1
- package/dist/{chunk-7N223GWV.mjs → chunk-DMNPYF54.mjs} +6 -2
- package/dist/{chunk-7HKWO4BJ.mjs → chunk-FDG4KXX5.mjs} +1 -1
- package/dist/{chunk-QEYF2SQ6.mjs → chunk-FNBNZCGP.mjs} +1 -1
- package/dist/{chunk-ZEKDO7YX.mjs → chunk-G5KHSM6E.mjs} +15 -0
- package/dist/{chunk-3BVGJ4JR.mjs → chunk-IJSFWGVW.mjs} +3 -28
- package/dist/{chunk-B7XIOAMK.mjs → chunk-J4BDAH2W.mjs} +1 -1
- package/dist/{chunk-CYYSZPVV.mjs → chunk-KQJVNPRM.mjs} +18 -19
- package/dist/chunk-MATMVH65.mjs +9 -0
- package/dist/{react-native/chunks/chunk-R4EEMYM6.mjs → chunk-NITCPDFL.mjs} +2 -2
- package/dist/{chunk-U2726TL5.mjs → chunk-NPXVQS33.mjs} +6 -2
- package/dist/{chunk-DOY6EG76.mjs → chunk-RBEHLIKX.mjs} +2 -2
- package/dist/{chunk-F22IOCR2.mjs → chunk-SEAJS664.mjs} +1 -1
- package/dist/{chunk-O4KUNDVF.mjs → chunk-UGVUW27S.mjs} +3 -3
- package/dist/chunk-VD7J552I.mjs +28 -0
- package/dist/{chunk-L7TIAYPX.mjs → chunk-ZHAXP7SD.mjs} +7 -7
- package/dist/{chunk-XKPNKR3V.mjs → chunk-ZS6CMXAE.mjs} +31 -10
- package/dist/{cold-module-2WWGBFJC.mjs → cold-module-PYSSBIL7.mjs} +18 -12
- package/dist/{direct-executor-QLVHMELJ.mjs → direct-executor-WYALQSMD.mjs} +2 -2
- package/dist/distribution/coordinator/etcd.d.ts +9 -0
- package/dist/distribution/coordinator/etcd.mjs +267 -106
- package/dist/distribution.mjs +23 -21
- package/dist/{durability-wiring-GNWQSZI3.mjs → durability-wiring-7VBZEJLG.mjs} +19 -45
- package/dist/{import-DX55PN4G.mjs → import-GJOMB2HC.mjs} +4 -4
- package/dist/index.browser.mjs +1 -1
- package/dist/index.mjs +3 -3
- package/dist/index.react-native.mjs +1 -1
- package/dist/{module-V5K4ZP23.mjs → module-6RLIKSZQ.mjs} +8 -8
- package/dist/{module-HW6IXUM6.mjs → module-AVQEIB7O.mjs} +23 -21
- package/dist/{paged-partition-archive-FDIIWZNR.mjs → paged-partition-archive-KGRW366Y.mjs} +11 -10
- package/dist/persistence/durability/checkpoint-worker.mjs +8 -8
- package/dist/react-native/chunks/{analysis-change-HGW3L6NY.mjs → analysis-change-DXPZRGWF.mjs} +2 -2
- package/dist/{browser/analysis-change-entry-H3YIOLXZ.mjs → react-native/chunks/analysis-change-entry-CXNRW324.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-5FV4ZEVA.mjs → chunk-7IQR3SZO.mjs} +43 -24
- package/dist/react-native/chunks/{chunk-7VWQLEFV.mjs → chunk-CLE3LZ77.mjs} +6 -2
- package/dist/react-native/chunks/{chunk-SLGIAD6B.mjs → chunk-D7LFYPYI.mjs} +4 -4
- package/dist/{chunk-KZQIVGW7.mjs → react-native/chunks/chunk-DFQJ4IJB.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-B6OD63Z3.mjs → chunk-FS7BYE23.mjs} +8 -4
- package/dist/react-native/chunks/chunk-HJBJBH26.mjs +34 -0
- package/dist/react-native/chunks/{chunk-YJSMEKRE.mjs → chunk-IDFMNRIC.mjs} +17 -17
- package/dist/react-native/chunks/{chunk-FKGG4UOF.mjs → chunk-JYL35A4A.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-GV5KFQ6M.mjs → chunk-KLZGAOLT.mjs} +4 -29
- package/dist/react-native/chunks/{chunk-HURV3ASY.mjs → chunk-KOCTCUXW.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-KUJOBRKS.mjs → chunk-UCJU6WNH.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-LQ6Q4L5B.mjs → chunk-VGUWIZFQ.mjs} +15 -0
- package/dist/react-native/chunks/{cold-module-TSFWLYWC.mjs → cold-module-KFGEWYEL.mjs} +16 -10
- package/dist/react-native/chunks/{durability-wiring-PZR6KOQK.mjs → durability-wiring-QKJB2QBT.mjs} +61 -50
- package/dist/react-native/chunks/{import-VTODLPLO.mjs → import-XFBCCENV.mjs} +2 -2
- package/dist/react-native/chunks/{module-VEK2KCKU.mjs → module-FHJ2GOBI.mjs} +4 -4
- package/dist/react-native/chunks/{module-JBBH6PZH.mjs → module-UJ5VUARS.mjs} +24 -23
- package/dist/{browser/restore-queue-DD4LWZK7.mjs → react-native/chunks/restore-queue-H5VCN453.mjs} +2 -2
- package/dist/react-native/chunks/{snapshot-AUZMEV36.mjs → snapshot-KVBD4LRC.mjs} +11 -10
- package/dist/react-native/chunks/{write-DDC2QBRJ.mjs → write-ES5A3WUA.mjs} +5 -5
- package/dist/{restore-queue-BQWWNNQI.mjs → restore-queue-AOF5OTQV.mjs} +2 -2
- package/dist/server.mjs +5 -5
- package/dist/{snapshot-EHXTOTW5.mjs → snapshot-WMX6JCLP.mjs} +18 -16
- package/dist/workers/entry.mjs +2 -2
- package/dist/{write-SDJXXQ2G.mjs → write-SKJ6HW7O.mjs} +7 -7
- package/package.json +7 -7
|
@@ -1,8 +1,36 @@
|
|
|
1
|
-
import { readMetadataEnvelope } from './chunk-
|
|
2
|
-
import { nextSegmentKey, readWalSegments, durableRecordMissing, checkpointLastSeqNo, SEGMENT_HEADER_SIZE } from './chunk-
|
|
3
|
-
import { applyDeleteEntry, applyIndexEntry } from './chunk-
|
|
1
|
+
import { readMetadataEnvelope } from './chunk-ZS6CMXAE.mjs';
|
|
2
|
+
import { readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, nextSegmentKey, readWalSegments, durableRecordMissing, checkpointLastSeqNo, SEGMENT_HEADER_SIZE } from './chunk-KQJVNPRM.mjs';
|
|
3
|
+
import { applyDeleteEntry, applyIndexEntry } from './chunk-NPXVQS33.mjs';
|
|
4
4
|
import { turnIsSpent, yieldTurn } from './chunk-LQG5G2LY.mjs';
|
|
5
5
|
|
|
6
|
+
// src/persistence/durability/marker-writer.ts
|
|
7
|
+
function markerKey(indexName, partitionId) {
|
|
8
|
+
return `${indexName}/wal/${partitionId}/commit`;
|
|
9
|
+
}
|
|
10
|
+
async function createMarkerWriter(directory, indexName, partitionId) {
|
|
11
|
+
const key = markerKey(indexName, partitionId);
|
|
12
|
+
const handle = await directory.markerHandle(key);
|
|
13
|
+
const existing = await handle.read();
|
|
14
|
+
const current = readCommitMarker(existing);
|
|
15
|
+
const createdFresh = existing.length === 0;
|
|
16
|
+
let lastSlot = current?.slotIndex ?? null;
|
|
17
|
+
let writeSeq = current?.state.writeSeq ?? 0;
|
|
18
|
+
return {
|
|
19
|
+
created: createdFresh,
|
|
20
|
+
existingHighestDurableSeqNo: current?.state.highestDurableSeqNo ?? 0,
|
|
21
|
+
async commit(state) {
|
|
22
|
+
const slotIndex = nextMarkerSlot(lastSlot);
|
|
23
|
+
writeSeq += 1;
|
|
24
|
+
const slot = encodeMarkerSlot({ ...state, writeSeq });
|
|
25
|
+
await handle.writeSlot(markerSlotOffset(slotIndex), slot);
|
|
26
|
+
lastSlot = slotIndex;
|
|
27
|
+
},
|
|
28
|
+
async close() {
|
|
29
|
+
await handle.close();
|
|
30
|
+
}
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
|
|
6
34
|
// src/persistence/durability/recovery.ts
|
|
7
35
|
async function removeLeftoverRewriteFiles(directory, names) {
|
|
8
36
|
for (const indexName of names) {
|
|
@@ -59,11 +87,33 @@ async function replayWal(directory, indexName, partitionId, fromSeqNoExclusive,
|
|
|
59
87
|
await replay(activeTail.entries);
|
|
60
88
|
}
|
|
61
89
|
await deleteOrphanSegments(directory, segments, marker.state.activeSegmentSeqNo);
|
|
62
|
-
if (activeTail !== null
|
|
63
|
-
await
|
|
90
|
+
if (activeTail !== null) {
|
|
91
|
+
await makeReplayedTailDurable(directory, indexName, partitionId, marker, activeTail);
|
|
64
92
|
}
|
|
65
93
|
return { highestSeqNo, highestPrimaryTerm };
|
|
66
94
|
}
|
|
95
|
+
async function makeReplayedTailDurable(directory, indexName, partitionId, marker, tail) {
|
|
96
|
+
const lastReplayed = tail.entries.at(-1);
|
|
97
|
+
if (lastReplayed === void 0 && tail.cleanEnd >= tail.segmentLength) return;
|
|
98
|
+
const handle = await directory.appendHandle(tail.key);
|
|
99
|
+
try {
|
|
100
|
+
if (tail.cleanEnd < await handle.size()) await handle.truncate(tail.cleanEnd);
|
|
101
|
+
else await handle.sync();
|
|
102
|
+
} finally {
|
|
103
|
+
await handle.close();
|
|
104
|
+
}
|
|
105
|
+
if (lastReplayed === void 0) return;
|
|
106
|
+
const writer = await createMarkerWriter(directory, indexName, partitionId);
|
|
107
|
+
try {
|
|
108
|
+
await writer.commit({
|
|
109
|
+
activeSegmentSeqNo: marker.state.activeSegmentSeqNo,
|
|
110
|
+
durableByteLength: tail.cleanEnd,
|
|
111
|
+
highestDurableSeqNo: Math.max(marker.state.highestDurableSeqNo, lastReplayed.seqNo)
|
|
112
|
+
});
|
|
113
|
+
} finally {
|
|
114
|
+
await writer.close();
|
|
115
|
+
}
|
|
116
|
+
}
|
|
67
117
|
async function replaySyncedWal(directory, indexName, partitionId, fromSeqNoExclusive, deps) {
|
|
68
118
|
await directory.remove(nextSegmentKey(indexName, partitionId));
|
|
69
119
|
const read = await readWalSegments(directory, indexName, partitionId);
|
|
@@ -153,4 +203,4 @@ async function drainIndexStateForUnload(indexState, pendingMetadataWrite, closeW
|
|
|
153
203
|
}
|
|
154
204
|
}
|
|
155
205
|
|
|
156
|
-
export { closePartitionWriter, drainIndexStateForUnload, listPersistedIndexes, loadMetadata, removeLeftoverRewriteFiles, replaySyncedWal, replayWal, snapshotCheckpointFor };
|
|
206
|
+
export { closePartitionWriter, createMarkerWriter, drainIndexStateForUnload, listPersistedIndexes, loadMetadata, removeLeftoverRewriteFiles, replaySyncedWal, replayWal, snapshotCheckpointFor };
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { isValidInteger, isRecord } from './chunk-
|
|
1
|
+
import { isValidInteger, isRecord } from './chunk-3E4RRF5J.mjs';
|
|
2
2
|
import { MAX_REPLICATION_FACTOR, MAX_PARTITION_COUNT } from './chunk-JDDXV6UF.mjs';
|
|
3
|
-
import { openChecksummedEnvelope, packChecksummedEnvelope } from './chunk-
|
|
3
|
+
import { openChecksummedEnvelope, packChecksummedEnvelope } from './chunk-2PNYKG2D.mjs';
|
|
4
4
|
import { joinChunks } from './chunk-3TVDTACC.mjs';
|
|
5
5
|
import { MAX_FILE_BYTES } from './chunk-U476IXKQ.mjs';
|
|
6
6
|
import { NarsilError, ErrorCodes } from './chunk-MDF6PDIA.mjs';
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead } from './chunk-
|
|
2
|
-
import { readListedFile, requireListedChecksum, requireListedSize } from './chunk-
|
|
3
|
-
import { HEADER_SIZE, openChecksummedEnvelope } from './chunk-
|
|
1
|
+
import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead } from './chunk-G5KHSM6E.mjs';
|
|
2
|
+
import { readListedFile, requireListedChecksum, requireListedSize } from './chunk-7LDMWKP6.mjs';
|
|
3
|
+
import { HEADER_SIZE, openChecksummedEnvelope } from './chunk-2PNYKG2D.mjs';
|
|
4
4
|
import { NarsilError, ErrorCodes } from './chunk-MDF6PDIA.mjs';
|
|
5
5
|
import { decode } from '@msgpack/msgpack';
|
|
6
6
|
|
|
@@ -64,7 +64,7 @@ function holdsEveryHead(ref) {
|
|
|
64
64
|
async function loadFieldOnDemand(store, ref, vectorIndex, resolvePartition) {
|
|
65
65
|
const { pathOf } = store;
|
|
66
66
|
if (pathOf === void 0 || vectorIndex.storage !== "disk" || !holdsEveryHead(ref)) return false;
|
|
67
|
-
const { loadFilesByParts, openColdVectorField, osqBitsOf, waitBeforeColdLoad } = await import('./cold-module-
|
|
67
|
+
const { loadFilesByParts, openColdVectorField, osqBitsOf, waitBeforeColdLoad } = await import('./cold-module-PYSSBIL7.mjs');
|
|
68
68
|
await Promise.all([
|
|
69
69
|
...ref.files.map((file) => requireListedSize(store, file.key, { bytes: file.bytes, what: "vector file" })),
|
|
70
70
|
...ref.graphParts.map((part) => requireListedSize(store, part.key, { bytes: part.bytes, what: "graph part" }))
|
|
@@ -85,8 +85,8 @@ async function loadFieldOnDemand(store, ref, vectorIndex, resolvePartition) {
|
|
|
85
85
|
resolvePartition
|
|
86
86
|
});
|
|
87
87
|
if (field === null) return false;
|
|
88
|
-
vectorIndex.openColdField(field, async (beginRestore, stillWanted,
|
|
89
|
-
await waitBeforeColdLoad(
|
|
88
|
+
vectorIndex.openColdField(field, async (beginRestore, stillWanted, loadingWanted) => {
|
|
89
|
+
await waitBeforeColdLoad(loadingWanted);
|
|
90
90
|
if (!stillWanted()) return;
|
|
91
91
|
const graph = await readGraphParts(store, ref);
|
|
92
92
|
if (!stillWanted()) return;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { packChecksummedEnvelope } from './chunk-
|
|
1
|
+
import { packChecksummedEnvelope } from './chunk-2PNYKG2D.mjs';
|
|
2
2
|
import { fnv1a } from './chunk-KAKQ74LS.mjs';
|
|
3
3
|
import { ErrorCodes, NarsilError } from './chunk-MDF6PDIA.mjs';
|
|
4
4
|
import { encode } from '@msgpack/msgpack';
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { encodeDocumentJson } from './chunk-3SBJVQLL.mjs';
|
|
2
|
-
import { loadFailed, sectionNamed, PAGED_SEGMENT_ENVELOPE_VERSION, encodePagedSegmentHead, SECTION_ALIGNMENT_BYTES, SEGMENT_PAGE_BYTES, PAYLOAD_PREFIX_BYTES, decodePagedSegmentHead, BLOCK_SUMMARY_BYTES, SMALLEST_BLOCK_POSTINGS, DOCUMENT_BLOCK_BYTES, decodeSections, DOCUMENT_BLOCK_MIN_RAW_BYTES, ID_SAMPLE_INTERVAL, POSITION_WIDTH_BYTES, POSTING_WIDTH_BYTES, RECORD_PAGE_BYTES, MAX_DOCUMENT_BLOCK_RAW_BYTES } from './chunk-
|
|
2
|
+
import { loadFailed, sectionNamed, PAGED_SEGMENT_ENVELOPE_VERSION, encodePagedSegmentHead, SECTION_ALIGNMENT_BYTES, SEGMENT_PAGE_BYTES, PAYLOAD_PREFIX_BYTES, decodePagedSegmentHead, BLOCK_SUMMARY_BYTES, SMALLEST_BLOCK_POSTINGS, DOCUMENT_BLOCK_BYTES, decodeSections, DOCUMENT_BLOCK_MIN_RAW_BYTES, ID_SAMPLE_INTERVAL, POSITION_WIDTH_BYTES, POSTING_WIDTH_BYTES, RECORD_PAGE_BYTES, MAX_DOCUMENT_BLOCK_RAW_BYTES } from './chunk-DMNPYF54.mjs';
|
|
3
3
|
import { createVectorFileReader } from './chunk-FOCV4CBZ.mjs';
|
|
4
|
-
import { writeSegmentsOf, MessagePackChunks, writeSurfaceForms, writeFieldLengths, writeValueColumns, writeBooleanColumns, writeEnumColumns, writeGeopointColumns, writePatternColumns, writePartialWords, FILE_ASSEMBLY_CHUNK_BYTES, REWRITE_SPILL_BYTES, LIST_VIEWS_HELD_PER_SEGMENT, TOMBSTONES_BETWEEN_YIELDS, readSegmentFile, READS_AHEAD_AT_ONCE, WRITER_YIELD_INTERVAL_TOKENS, LARGEST_CACHED_ENTRY_SHARE, MAX_THREAD_FILE_NUMBER, REMOVAL_BUCKET_WRITE_BUFFER_BYTES, SECTION_WRITE_BUFFER_BYTES, READ_AHEAD_CHUNK_BYTES, LARGEST_REMOVAL_BUCKET_COUNT, REMOVAL_BUCKET_TARGET_BYTES, THREAD_READ_CACHE_BYTES, MERGE_READ_CACHE_BYTES } from './chunk-
|
|
5
|
-
import { requireOffsets, requireUtf8Entries, requireEntriesAscend, decodeFieldLengths, decodePartialWords, decodePatternColumns, decodeGeopointColumns, decodeEnumColumns, decodeBooleanColumns, decodeValueColumns, setNestedValue, littleEndianBytes, uint32sFrom, uint64Bytes, decodeSurfaceForms } from './chunk-
|
|
6
|
-
import { HEADER_SIZE, requireWithinFileLimit, crc32Init, crc32Update, crc32Final, engineVersionParts, writeHeader, crc32, readHeader } from './chunk-
|
|
7
|
-
import { generateId, wrapFrozenTokenTable, createEncodedDocumentSource, assemblePagedFrozenSegment, decodeStoredDocument, mergeFieldLengths, mergeSurfaceForms, encodeSurfaceTable, mergePartialWords, mergeFieldIndexes, resolvePartitionInsertOptions, decodeSurfaceTable, createSharedFrozenSegment, shareSegmentArrays, selectionKeeps, fieldIndexRemapOf, freezeSegmentShared, createFrozenSegment, holdsADiscardedField, endOfValue } from './chunk-
|
|
4
|
+
import { writeSegmentsOf, MessagePackChunks, writeSurfaceForms, writeFieldLengths, writeValueColumns, writeBooleanColumns, writeEnumColumns, writeGeopointColumns, writePatternColumns, writePartialWords, FILE_ASSEMBLY_CHUNK_BYTES, REWRITE_SPILL_BYTES, LIST_VIEWS_HELD_PER_SEGMENT, TOMBSTONES_BETWEEN_YIELDS, readSegmentFile, READS_AHEAD_AT_ONCE, WRITER_YIELD_INTERVAL_TOKENS, LARGEST_CACHED_ENTRY_SHARE, MAX_THREAD_FILE_NUMBER, REMOVAL_BUCKET_WRITE_BUFFER_BYTES, SECTION_WRITE_BUFFER_BYTES, READ_AHEAD_CHUNK_BYTES, LARGEST_REMOVAL_BUCKET_COUNT, REMOVAL_BUCKET_TARGET_BYTES, THREAD_READ_CACHE_BYTES, MERGE_READ_CACHE_BYTES } from './chunk-CSHHU7PA.mjs';
|
|
5
|
+
import { requireOffsets, requireUtf8Entries, requireEntriesAscend, decodeFieldLengths, decodePartialWords, decodePatternColumns, decodeGeopointColumns, decodeEnumColumns, decodeBooleanColumns, decodeValueColumns, setNestedValue, littleEndianBytes, uint32sFrom, uint64Bytes, decodeSurfaceForms } from './chunk-NPXVQS33.mjs';
|
|
6
|
+
import { HEADER_SIZE, requireWithinFileLimit, crc32Init, crc32Update, crc32Final, engineVersionParts, writeHeader, crc32, readHeader } from './chunk-2PNYKG2D.mjs';
|
|
7
|
+
import { generateId, wrapFrozenTokenTable, createEncodedDocumentSource, assemblePagedFrozenSegment, decodeStoredDocument, mergeFieldLengths, mergeSurfaceForms, encodeSurfaceTable, mergePartialWords, mergeFieldIndexes, resolvePartitionInsertOptions, decodeSurfaceTable, createSharedFrozenSegment, shareSegmentArrays, selectionKeeps, fieldIndexRemapOf, freezeSegmentShared, createFrozenSegment, holdsADiscardedField, endOfValue } from './chunk-UGVUW27S.mjs';
|
|
8
8
|
import { isCompositePartition, buildPartialWordTableFrom, createPartitionIndex, createCompositePartition, mergeDocFrequencies, scanPostingBlocks } from './chunk-3HV7M5OJ.mjs';
|
|
9
9
|
import { flattenSchema, parseVectorDimension, encodeDocument } from './chunk-SOUZMPME.mjs';
|
|
10
10
|
import { VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
@@ -1307,7 +1307,7 @@ async function coldFieldOf(checkpoint, field) {
|
|
|
1307
1307
|
const dimension = fieldType === void 0 ? null : parseVectorDimension(fieldType);
|
|
1308
1308
|
if (dimension === null) return null;
|
|
1309
1309
|
const withPath = async (ref) => ({ ref, path: await directory.pathOf(ref.key) });
|
|
1310
|
-
const { openColdVectorField } = await import('./cold-module-
|
|
1310
|
+
const { openColdVectorField } = await import('./cold-module-PYSSBIL7.mjs');
|
|
1311
1311
|
return openColdVectorField({
|
|
1312
1312
|
fieldPath: field.fieldPath,
|
|
1313
1313
|
dimension,
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { littleEndianBytes, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, KEYWORD_SEGMENT_PAYLOAD_VERSION, uint64Bytes } from './chunk-
|
|
2
|
-
import { readListedFile, requireListedChecksum, segmentKey, MAX_SEGMENTS_PER_PARTITION } from './chunk-
|
|
3
|
-
import { openChecksummedEnvelope, HEADER_SIZE, packChecksummedEnvelope } from './chunk-
|
|
4
|
-
import { selectionKeeps, segmentArraysOf, mergeSegmentContents } from './chunk-
|
|
1
|
+
import { littleEndianBytes, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, KEYWORD_SEGMENT_PAYLOAD_VERSION, uint64Bytes } from './chunk-NPXVQS33.mjs';
|
|
2
|
+
import { readListedFile, requireListedChecksum, segmentKey, MAX_SEGMENTS_PER_PARTITION } from './chunk-7LDMWKP6.mjs';
|
|
3
|
+
import { openChecksummedEnvelope, HEADER_SIZE, packChecksummedEnvelope } from './chunk-2PNYKG2D.mjs';
|
|
4
|
+
import { selectionKeeps, segmentArraysOf, mergeSegmentContents } from './chunk-UGVUW27S.mjs';
|
|
5
5
|
import { messagePackUpperBoundBytes } from './chunk-SOUZMPME.mjs';
|
|
6
6
|
import { MAX_FILE_BYTES } from './chunk-U476IXKQ.mjs';
|
|
7
7
|
import { NarsilError, ErrorCodes } from './chunk-MDF6PDIA.mjs';
|
|
@@ -566,7 +566,7 @@ async function writePagedGroup(state, group) {
|
|
|
566
566
|
const tombstones = state.tombstonesWritten ? [] : [...request.tombstones];
|
|
567
567
|
const id = state.nextSegmentId;
|
|
568
568
|
const key = segmentKey(request.indexName, request.partitionId, id);
|
|
569
|
-
const { writePagedSegment } = await import('./module-
|
|
569
|
+
const { writePagedSegment } = await import('./module-6RLIKSZQ.mjs');
|
|
570
570
|
const written = await writePagedSegment({
|
|
571
571
|
directory,
|
|
572
572
|
key,
|
|
@@ -14,7 +14,7 @@ function requireLoadedVectorCode() {
|
|
|
14
14
|
function loadVectorCode() {
|
|
15
15
|
if (loaded !== null) return Promise.resolve(loaded);
|
|
16
16
|
if (loading === null) {
|
|
17
|
-
loading = import('./module-
|
|
17
|
+
loading = import('./module-AVQEIB7O.mjs').then(
|
|
18
18
|
(code) => {
|
|
19
19
|
loaded = code;
|
|
20
20
|
return code;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { crc32 } from './chunk-
|
|
1
|
+
import { crc32 } from './chunk-2PNYKG2D.mjs';
|
|
2
2
|
import { NarsilError, ErrorCodes } from './chunk-MDF6PDIA.mjs';
|
|
3
3
|
import { decode, encode } from '@msgpack/msgpack';
|
|
4
4
|
|
|
@@ -82,7 +82,11 @@ function decodePagedSegmentHead(head, payloadLength, what) {
|
|
|
82
82
|
if (raw === null || typeof raw !== "object") loadFailed(`The head of ${what} is not a map`);
|
|
83
83
|
const entry = raw;
|
|
84
84
|
if (entry.v !== PAGED_SEGMENT_HEAD_VERSION) {
|
|
85
|
-
|
|
85
|
+
throw new NarsilError(
|
|
86
|
+
ErrorCodes.ENVELOPE_VERSION_MISMATCH,
|
|
87
|
+
`The head of ${what} was written in layout ${String(entry.v)}, and this build reads layout ${PAGED_SEGMENT_HEAD_VERSION} alone`,
|
|
88
|
+
{ version: entry.v }
|
|
89
|
+
);
|
|
86
90
|
}
|
|
87
91
|
if (!isStringRecord(entry.schema)) loadFailed(`The head of ${what} holds no schema`);
|
|
88
92
|
if (!isStringList(entry.tombstones)) loadFailed(`The head of ${what} holds no tombstone list`);
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { nodeThreadId } from './chunk-BEZH7VEH.mjs';
|
|
2
2
|
import { getFs, getPath } from './chunk-HLPRHOSO.mjs';
|
|
3
|
-
import { DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_SETTLE_MS, DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS } from './chunk-
|
|
3
|
+
import { DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_SETTLE_MS, DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS } from './chunk-CSHHU7PA.mjs';
|
|
4
4
|
import { NarsilError, ErrorCodes } from './chunk-MDF6PDIA.mjs';
|
|
5
5
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
6
6
|
import { openSync, close, readFileSync } from 'fs';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { sanitizeSurfaceForms, patternIndexesFromWire, dropStoredVectorValues } from './chunk-
|
|
1
|
+
import { sanitizeSurfaceForms, patternIndexesFromWire, dropStoredVectorValues } from './chunk-ZS6CMXAE.mjs';
|
|
2
2
|
import { requireValidFieldTypes } from './chunk-SOUZMPME.mjs';
|
|
3
3
|
import { VALID_HNSW_METRICS } from './chunk-U476IXKQ.mjs';
|
|
4
4
|
import { decode } from '@msgpack/msgpack';
|
|
@@ -241,6 +241,20 @@ function decodeVectorGraphPartPayload(raw) {
|
|
|
241
241
|
function sameGraphHeader(a, b) {
|
|
242
242
|
return a.entryPoint === b.entryPoint && a.maxLayer === b.maxLayer && a.m === b.m && a.efConstruction === b.efConstruction && a.metric === b.metric && a.shuffledBuildCount === b.shuffledBuildCount && a.calibratedMeanLength === b.calibratedMeanLength;
|
|
243
243
|
}
|
|
244
|
+
function requireOneGraphPerVector(part, p) {
|
|
245
|
+
if (part.graphs.length < 2) return;
|
|
246
|
+
const held = new Uint8Array(VECTOR_GRAPH_PART_VECTORS);
|
|
247
|
+
for (const graph of part.graphs) {
|
|
248
|
+
for (let at = 0; at < graph.levels.length; at++) {
|
|
249
|
+
if (graph.levels[at] === 0) continue;
|
|
250
|
+
if (held[at] === 1) {
|
|
251
|
+
const vector = p * VECTOR_GRAPH_PART_VECTORS + at;
|
|
252
|
+
invalid2("vector graph part", `part ${p} gives vector ${vector} a node in two graphs`, { part: p, vector });
|
|
253
|
+
}
|
|
254
|
+
held[at] = 1;
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
}
|
|
244
258
|
function assembleGraphParts(parts) {
|
|
245
259
|
const name = "vector graph part";
|
|
246
260
|
const first = parts[0];
|
|
@@ -261,6 +275,7 @@ function assembleGraphParts(parts) {
|
|
|
261
275
|
if (!fits) invalid2(name, `part ${p} gives graph ${g} ${levels} bytes of levels`, { part: p, levels });
|
|
262
276
|
partLists[g].push(graph);
|
|
263
277
|
}
|
|
278
|
+
requireOneGraphPerVector(part, p);
|
|
264
279
|
}
|
|
265
280
|
const graphs = partLists.map((list) => ({ partSize: VECTOR_GRAPH_PART_VECTORS, parts: list }));
|
|
266
281
|
return { fieldName: first.fieldName, graphs };
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { REBUILD_REMOVED_RATIO, BUILD_TURN_HANDOVER_AFTER_MS, DRIFT_CHECK_GROWTH_FACTOR, VECTOR_FILE_REPLACED_ABOVE_DEAD_SHARE, VECTOR_FILE_FIXED_OVERHEAD_BYTES, VECTOR_FILE_CENTROID_BYTES_PER_COMPONENT, VECTOR_FILE_STRING_HEADER_BYTES } from './chunk-VD7J552I.mjs';
|
|
2
|
+
import { VECTOR_FILE_MAX_VECTORS, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors, VECTOR_GRAPH_PART_VECTORS, VECTOR_GRAPH_PART_PAYLOAD_VERSION, VECTOR_FILE_PAYLOAD_VERSION, deadBitsWhere, vectorsToBytes } from './chunk-G5KHSM6E.mjs';
|
|
2
3
|
import { createOrdinalFilter, addToOrdinalFilter, osqBitsOf, STORE_ORDINAL_LAYOUT_GENERATION, removeFromOrdinalFilter, createHNSWIndex, STORE_CALIBRATION_GENERATION, graphWithoutNodes, createOsqScratch, osqQuantize } from './chunk-AWNHDBKV.mjs';
|
|
3
4
|
import { osqRecordBytes, writeRecord } from './chunk-37CD3OKS.mjs';
|
|
4
5
|
import { MAX_DOC_ID_LENGTH } from './chunk-EEBCDP36.mjs';
|
|
@@ -6,32 +7,6 @@ import { turnsAreBudgeted, backgroundTurnIsSpent, yieldBackgroundTurn } from './
|
|
|
6
7
|
import { MAX_FILE_BYTES } from './chunk-U476IXKQ.mjs';
|
|
7
8
|
import { NarsilError, ErrorCodes } from './chunk-MDF6PDIA.mjs';
|
|
8
9
|
|
|
9
|
-
// src/vector/vector-index/constants.ts
|
|
10
|
-
var DEFAULT_PROMOTION_THRESHOLD = 1024;
|
|
11
|
-
var DEFAULT_FILTER_THRESHOLD = 0.03;
|
|
12
|
-
var ESTIMATED_MS_PER_TOMBSTONE = 0.05;
|
|
13
|
-
var ESTIMATED_MS_PER_VECTOR_REBUILD = 0.15;
|
|
14
|
-
var WORKER_COPY_MIN_VECTORS = 1024;
|
|
15
|
-
var BUILD_CHUNK_SIZE = 100;
|
|
16
|
-
var BUILD_STEPS_BETWEEN_CLOCK_CHECKS = 256;
|
|
17
|
-
var SEARCH_SCAN_CHUNK_VECTORS = 4096;
|
|
18
|
-
var SEARCH_REFILL_RESERVE = 64;
|
|
19
|
-
var SEARCH_ATTEMPTS_BEFORE_HOLDING_WRITES = 1;
|
|
20
|
-
var BUILD_TURN_HANDOVER_AFTER_MS = 50;
|
|
21
|
-
var BUILD_RETRY_FIRST_DELAY_MS = 250;
|
|
22
|
-
var BUILD_RETRY_MAX_DELAY_MS = 3e4;
|
|
23
|
-
var SHUFFLED_REBUILD_MEMORY_RETRY_MS = 1e3;
|
|
24
|
-
var REBUILD_REMOVED_RATIO = 0.2;
|
|
25
|
-
var DRIFT_CHECK_GROWTH_FACTOR = 2;
|
|
26
|
-
var RECALIBRATION_DRIFT_RATIO = 2.5;
|
|
27
|
-
var RECALIBRATION_CHUNK_VECTORS = 512;
|
|
28
|
-
var VECTOR_FILE_REPLACED_ABOVE_DEAD_SHARE = 0.2;
|
|
29
|
-
var VECTOR_FILE_FIXED_OVERHEAD_BYTES = 65536;
|
|
30
|
-
var VECTOR_FILE_STRING_HEADER_BYTES = 5;
|
|
31
|
-
var VECTOR_FILE_CENTROID_BYTES_PER_COMPONENT = 9;
|
|
32
|
-
var OSQ4_MIN_DIMENSION = 384;
|
|
33
|
-
var DISK_ADOPTION_YIELD_INTERVAL = 4096;
|
|
34
|
-
|
|
35
10
|
// src/vector/filter-ordinals.ts
|
|
36
11
|
var UNRESOLVED_ORDINAL = -1;
|
|
37
12
|
var NO_VECTOR_MARK = -2;
|
|
@@ -677,4 +652,4 @@ function recordCheckpoint(state, plan, written) {
|
|
|
677
652
|
state.savedSignature = plan.signature;
|
|
678
653
|
}
|
|
679
654
|
|
|
680
|
-
export {
|
|
655
|
+
export { VECTOR_WORKER_COPIES_ALLOWED, adoptGraph, adoptLoadedCalibration, allLiveDocIds, assignStorePartitions, deserialize, emptyFieldBeforeRestore, fieldHandlesOf, filterForOptions, graphNeedsRebuild, handOverBuildTurns, liveOrdinals, liveSize, noBuildHistory, ordinalFilterForSubsets, planCheckpoint, recalibrateFromStore, recordCheckpoint, releaseReplacedVectors, signatureOf, threadsHoldCurrentLayout, vectorsPerFile, yieldBuildTurn };
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { WIRE_BATCH_BUDGET } from './chunk-JDDXV6UF.mjs';
|
|
2
|
-
import { DEFAULT_COMPACTION_THRESHOLD, writeSegmentsOf, readSegmentFile, MERGED_SEGMENT_GROWTH_ALLOWANCE, MERGED_SEGMENT_TARGET_BYTES, COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT, SEGMENT_FILE_FRAMING_SLACK_BYTES } from './chunk-
|
|
3
|
-
import { restoreVectorFields } from './chunk-
|
|
4
|
-
import { encodeSegmentManifest, SEGMENT_MANIFEST_VERSION, manifestKey, manifestReferencedKeys, segmentsPrefix, snapshotBundleKey, MAX_SEGMENTS_PER_PARTITION } from './chunk-
|
|
2
|
+
import { DEFAULT_COMPACTION_THRESHOLD, writeSegmentsOf, readSegmentFile, MERGED_SEGMENT_GROWTH_ALLOWANCE, MERGED_SEGMENT_TARGET_BYTES, COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT, SEGMENT_FILE_FRAMING_SLACK_BYTES } from './chunk-CSHHU7PA.mjs';
|
|
3
|
+
import { restoreVectorFields } from './chunk-NPXVQS33.mjs';
|
|
4
|
+
import { encodeSegmentManifest, SEGMENT_MANIFEST_VERSION, manifestKey, manifestReferencedKeys, segmentsPrefix, snapshotBundleKey, MAX_SEGMENTS_PER_PARTITION } from './chunk-7LDMWKP6.mjs';
|
|
5
5
|
import { ReplicationMessageTypes } from './chunk-5QMHMELB.mjs';
|
|
6
|
-
import { crc32, reconstructSchemaFromMetadata, prepareDocumentVectors } from './chunk-
|
|
7
|
-
import { generateId, freezeSegmentShared, createSharedFrozenSegment, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, decodeMapWithoutFields, shareSegmentArrays } from './chunk-
|
|
6
|
+
import { crc32, reconstructSchemaFromMetadata, prepareDocumentVectors } from './chunk-2PNYKG2D.mjs';
|
|
7
|
+
import { generateId, freezeSegmentShared, createSharedFrozenSegment, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, decodeMapWithoutFields, shareSegmentArrays } from './chunk-UGVUW27S.mjs';
|
|
8
8
|
import { isCompositePartition, survivorDocumentsOf, partitionStateOf } from './chunk-3HV7M5OJ.mjs';
|
|
9
9
|
import { getLanguage, resolveIndexAnalysis } from './chunk-2HJWQPR6.mjs';
|
|
10
10
|
import { extractVectorFieldsFromSchema, partialFieldsOf } from './chunk-SOUZMPME.mjs';
|
|
@@ -478,18 +478,19 @@ function frameRecord(entry) {
|
|
|
478
478
|
}
|
|
479
479
|
function checkSegmentHeader(data) {
|
|
480
480
|
if (data.length < SEGMENT_HEADER_SIZE) {
|
|
481
|
-
return { ok: false, reason: "segment shorter than the 8-byte header" };
|
|
481
|
+
return { ok: false, reason: "segment shorter than the 8-byte header", code: ErrorCodes.PERSISTENCE_WAL_CORRUPT };
|
|
482
482
|
}
|
|
483
483
|
for (let i = 0; i < WAL_MAGIC.length; i += 1) {
|
|
484
484
|
if (data[i] !== WAL_MAGIC[i]) {
|
|
485
|
-
return { ok: false, reason: "missing NRSW magic" };
|
|
485
|
+
return { ok: false, reason: "missing NRSW magic", code: ErrorCodes.PERSISTENCE_WAL_CORRUPT };
|
|
486
486
|
}
|
|
487
487
|
}
|
|
488
488
|
const formatVersion = data[4];
|
|
489
489
|
if (formatVersion > WAL_FORMAT_VERSION) {
|
|
490
490
|
return {
|
|
491
491
|
ok: false,
|
|
492
|
-
reason: `WAL format version ${formatVersion} is newer than supported version ${WAL_FORMAT_VERSION}
|
|
492
|
+
reason: `WAL format version ${formatVersion} is newer than supported version ${WAL_FORMAT_VERSION}, so a newer release wrote this segment`,
|
|
493
|
+
code: ErrorCodes.ENVELOPE_VERSION_MISMATCH
|
|
493
494
|
};
|
|
494
495
|
}
|
|
495
496
|
return { ok: true };
|
|
@@ -508,7 +509,9 @@ function corrupt(message, details) {
|
|
|
508
509
|
function readDurableRegion(data, durableByteLength, maxRecordBytes = MAX_WAL_RECORD_BYTES) {
|
|
509
510
|
const header = checkSegmentHeader(data);
|
|
510
511
|
if (!header.ok) {
|
|
511
|
-
|
|
512
|
+
throw new NarsilError(header.code, `WAL segment header invalid in durable region: ${header.reason}`, {
|
|
513
|
+
reason: header.reason
|
|
514
|
+
});
|
|
512
515
|
}
|
|
513
516
|
if (durableByteLength < SEGMENT_HEADER_SIZE || durableByteLength > data.length) {
|
|
514
517
|
corrupt("Commit marker durable byte length is outside the segment", {
|
|
@@ -656,14 +659,10 @@ async function* readSegmentsInOrder(directory, wal) {
|
|
|
656
659
|
if (startSeqNo < marker.state.activeSegmentSeqNo) {
|
|
657
660
|
const header = checkSegmentHeader(bytes);
|
|
658
661
|
if (!header.ok) {
|
|
659
|
-
throw new NarsilError(
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
key,
|
|
664
|
-
reason: header.reason
|
|
665
|
-
}
|
|
666
|
-
);
|
|
662
|
+
throw new NarsilError(header.code, `Sealed WAL segment header invalid: ${header.reason}`, {
|
|
663
|
+
key,
|
|
664
|
+
reason: header.reason
|
|
665
|
+
});
|
|
667
666
|
}
|
|
668
667
|
const durableEntries2 = readDurableRegion(bytes, bytes.length);
|
|
669
668
|
for (const entry of durableEntries2) highestRead = Math.max(highestRead, entry.seqNo);
|
|
@@ -877,7 +876,7 @@ async function compactPartitionSegments(input) {
|
|
|
877
876
|
if (paged) {
|
|
878
877
|
const { directory } = input.store;
|
|
879
878
|
if (directory === void 0) return { segments, nextSegmentId: input.nextSegmentId };
|
|
880
|
-
const { mergePagedWindow } = await import('./module-
|
|
879
|
+
const { mergePagedWindow } = await import('./module-6RLIKSZQ.mjs');
|
|
881
880
|
return replacing(
|
|
882
881
|
await mergePagedWindow({
|
|
883
882
|
store: input.store,
|
|
@@ -965,7 +964,7 @@ function pagedSettingsOf(context) {
|
|
|
965
964
|
async function tombstonesOfDelta(context, older, selections, deleted) {
|
|
966
965
|
const { directory } = context.input.store;
|
|
967
966
|
if (context.input.metadata.textStorage !== "disk" || directory === void 0) return deleted;
|
|
968
|
-
const { pagedTombstonesFor } = await import('./module-
|
|
967
|
+
const { pagedTombstonesFor } = await import('./module-6RLIKSZQ.mjs');
|
|
969
968
|
return pagedTombstonesFor(
|
|
970
969
|
directory,
|
|
971
970
|
older,
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { OSQ4_MIN_DIMENSION } from './chunk-VD7J552I.mjs';
|
|
2
|
+
|
|
3
|
+
// src/vector/vector-index/default-quantization.ts
|
|
4
|
+
function defaultQuantizationFor(dimension) {
|
|
5
|
+
if (dimension >= OSQ4_MIN_DIMENSION) return "osq4";
|
|
6
|
+
return "osq8";
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
export { defaultQuantizationFor };
|
|
@@ -4,14 +4,14 @@ function holdsChangeWork(entry) {
|
|
|
4
4
|
}
|
|
5
5
|
async function abandonChangeOf(core, indexName) {
|
|
6
6
|
if (!holdsChangeWork(core.indexRegistry.get(indexName))) return;
|
|
7
|
-
const { abandonAnalysisChange } = await import('./analysis-change-
|
|
7
|
+
const { abandonAnalysisChange } = await import('./analysis-change-2OYM7ARW.mjs');
|
|
8
8
|
await abandonAnalysisChange(core, indexName);
|
|
9
9
|
}
|
|
10
10
|
async function resumeChangeOf(core, indexName) {
|
|
11
11
|
if ((core.indexRegistry.get(indexName)?.pendingAnalysis ?? null) === null) return;
|
|
12
12
|
if (core.mutationCtx.capacityScope === "partition") return;
|
|
13
13
|
if (core.executor.getManager(indexName) === void 0) return;
|
|
14
|
-
const { resumeAnalysisChange } = await import('./analysis-change-
|
|
14
|
+
const { resumeAnalysisChange } = await import('./analysis-change-2OYM7ARW.mjs');
|
|
15
15
|
void resumeAnalysisChange(core, indexName).catch(() => void 0);
|
|
16
16
|
}
|
|
17
17
|
async function resumeEveryChange(core) {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors } from './chunk-
|
|
1
|
+
import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors } from './chunk-2PNYKG2D.mjs';
|
|
2
2
|
import { getNestedValue, requireValidFieldTypes } from './chunk-SOUZMPME.mjs';
|
|
3
3
|
import { VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
4
4
|
import { ErrorCodes, NarsilError } from './chunk-MDF6PDIA.mjs';
|
|
@@ -617,7 +617,11 @@ function decodeKeywordSegment(payload, shared) {
|
|
|
617
617
|
}
|
|
618
618
|
const raw = requireRecord(decoded, "payload");
|
|
619
619
|
if (raw.v !== KEYWORD_SEGMENT_PAYLOAD_VERSION) {
|
|
620
|
-
|
|
620
|
+
throw new NarsilError(
|
|
621
|
+
ErrorCodes.ENVELOPE_VERSION_MISMATCH,
|
|
622
|
+
`This keyword segment payload was written in layout ${String(raw.v)}, and this build reads layout ${KEYWORD_SEGMENT_PAYLOAD_VERSION} alone`,
|
|
623
|
+
{ version: raw.v }
|
|
624
|
+
);
|
|
621
625
|
}
|
|
622
626
|
const schema = decodeSchema(raw.schema);
|
|
623
627
|
const tombstones = requireStrings(raw.tombstones, "tombstones");
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { requireAnalysisRebuildMemory } from './chunk-VP6XRMWR.mjs';
|
|
2
|
-
import { requireConfigurationWithinSizeLimit, DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, rewriteSpillOf, emitEngineEvent, reconstructSchemaFromMetadata, prepareStoredSynonyms, rulesOfStoredSynonyms, prepareConfiguredSynonyms, REBALANCE_CHUNK_SIZE, yieldToEventLoop, prepareDocumentVectors, removeDocumentVectors, insertDocumentVectors, vectorsEqual, rebuildTextIndexes, textRebuildTargetOf, extractVectorFromDoc, deleteNestedValue } from './chunk-
|
|
3
|
-
import { resolvePartialWordAnalysis } from './chunk-
|
|
2
|
+
import { requireConfigurationWithinSizeLimit, DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, rewriteSpillOf, emitEngineEvent, reconstructSchemaFromMetadata, prepareStoredSynonyms, rulesOfStoredSynonyms, prepareConfiguredSynonyms, REBALANCE_CHUNK_SIZE, yieldToEventLoop, prepareDocumentVectors, removeDocumentVectors, insertDocumentVectors, vectorsEqual, rebuildTextIndexes, textRebuildTargetOf, extractVectorFromDoc, deleteNestedValue } from './chunk-2PNYKG2D.mjs';
|
|
3
|
+
import { resolvePartialWordAnalysis } from './chunk-UGVUW27S.mjs';
|
|
4
4
|
import { createPartitionIndex, ARRAY_POSITION_GAP } from './chunk-3HV7M5OJ.mjs';
|
|
5
5
|
import { ANALYSIS_CHANGE_MEMBERS, ANALYSIS_HOLD_APPLY_INTERVAL_MS } from './chunk-EEBCDP36.mjs';
|
|
6
6
|
import { getLanguage, configUnderAnalysis, resolveIndexAnalysis, targetOfPending } from './chunk-2HJWQPR6.mjs';
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { untagDocumentLine } from './chunk-NYSONUB7.mjs';
|
|
2
2
|
import { chunksOf } from './chunk-CZQ33DLW.mjs';
|
|
3
3
|
import { restoredConfigFields, restoredEmbedding } from './chunk-IRWP524D.mjs';
|
|
4
|
-
import { requireConfigurationShape } from './chunk-
|
|
4
|
+
import { requireConfigurationShape } from './chunk-2PNYKG2D.mjs';
|
|
5
5
|
import { MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, MAX_IMPORT_REFUSALS_REPORTED } from './chunk-EEBCDP36.mjs';
|
|
6
6
|
import { turnsAreBudgeted, turnIsSpent, yieldTurn } from './chunk-LQG5G2LY.mjs';
|
|
7
7
|
import { flattenSchema } from './chunk-SOUZMPME.mjs';
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { requirePatternSearchFor, requirePartialWordSearchFor, createPartitionIndex, cloneProjected, projectIncludedPaths, valueWithOwnBuffers, buildPartialWordTableFrom, createSurfaceRegistry, encodePartialSpellingsOf, recordValueSize, patternIndexOf, encodeFieldIndexes, countPartialSpelling, ARRAY_POSITION_GAP, readHeapStatistics, documentWithOwnBuffers, readProjectedPath, copyVectorsInto, isCompositePartition, partialSpellingsOfDocuments, createFrozenPartialWords, requirePatternSearch, documentValueBytes, MAX_TERM_FREQUENCY, loadPhraseCodeFor, joinPatternWorkMeter, fanOutQuery, sanitizeGlobalStats, countsWithoutScores, fanOutMatchCount, createGeoIndex, createNumericIndex, createBooleanIndex, createEnumIndex, ValueAffixes, readResidentMemory, createCompositePartition } from './chunk-3HV7M5OJ.mjs';
|
|
2
2
|
import { fnv1a } from './chunk-KAKQ74LS.mjs';
|
|
3
3
|
import { resolveIndexAnalysis, getLanguage, synonymSearchModule } from './chunk-2HJWQPR6.mjs';
|
|
4
|
-
import { loadedVectorCode, loadVectorCode, loadVectorCodeFor, requireLoadedVectorCode } from './chunk-
|
|
4
|
+
import { loadedVectorCode, loadVectorCode, loadVectorCodeFor, requireLoadedVectorCode } from './chunk-DGRM4EZZ.mjs';
|
|
5
5
|
import { createDocIdMap } from './chunk-LQG5G2LY.mjs';
|
|
6
6
|
import { GrowableUint32, bitsetHas, bitsetSet, createBitSet } from './chunk-JUKYYXSP.mjs';
|
|
7
7
|
import { partialFieldsOf, getNestedValue, schemaFieldsOf, tokenizeOptions, sameFields, encodeDocument, validateDocument, validateDocumentStrict, extractVectorFieldsFromSchema, visitDocumentSurfaceSpellings, visitDocumentSpellings } from './chunk-SOUZMPME.mjs';
|
|
@@ -3234,7 +3234,7 @@ async function attachSegments(entry, action) {
|
|
|
3234
3234
|
growPartitionsTo(entry.manager, segment.partitionId);
|
|
3235
3235
|
const surfaceForms = segment.survivingSurfaceForms;
|
|
3236
3236
|
const spellings = surfaceForms === void 0 ? void 0 : NO_REMOVED_SPELLINGS;
|
|
3237
|
-
const frozen = isPagedSnapshot(segment.snapshot) ? (await import('./module-
|
|
3237
|
+
const frozen = isPagedSnapshot(segment.snapshot) ? (await import('./module-6RLIKSZQ.mjs')).createPagedFrozenSegment(segment.snapshot) : createSharedFrozenSegment(segment.snapshot, surfaceForms);
|
|
3238
3238
|
for (const docId of segment.tombstonedDocIds ?? []) frozen.tombstoneDocument(docId, spellings);
|
|
3239
3239
|
entry.manager.attachFrozenSegment(segment.partitionId, frozen);
|
|
3240
3240
|
}
|
|
@@ -3289,7 +3289,7 @@ function runSegmentAction(entry, action) {
|
|
|
3289
3289
|
case "attachSegments":
|
|
3290
3290
|
return attachSegments(entry, action);
|
|
3291
3291
|
case "adoptDiskSegments":
|
|
3292
|
-
return import('./module-
|
|
3292
|
+
return import('./module-6RLIKSZQ.mjs').then((code) => code.adoptSegmentsOnWorker(entry.manager, action));
|
|
3293
3293
|
case "compactSegments": {
|
|
3294
3294
|
const partition = requireComposite(entry, action.indexName, action.partitionId, "compact");
|
|
3295
3295
|
return mergeFrozenSegments(
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
// src/vector/vector-index/constants.ts
|
|
2
|
+
var DEFAULT_PROMOTION_THRESHOLD = 1024;
|
|
3
|
+
var DEFAULT_FILTER_THRESHOLD = 0.03;
|
|
4
|
+
var ESTIMATED_MS_PER_TOMBSTONE = 0.05;
|
|
5
|
+
var ESTIMATED_MS_PER_VECTOR_REBUILD = 0.15;
|
|
6
|
+
var WORKER_COPY_MIN_VECTORS = 1024;
|
|
7
|
+
var BUILD_CHUNK_SIZE = 100;
|
|
8
|
+
var BUILD_STEPS_BETWEEN_CLOCK_CHECKS = 256;
|
|
9
|
+
var SEARCH_SCAN_CHUNK_VECTORS = 4096;
|
|
10
|
+
var SEARCH_REFILL_RESERVE = 64;
|
|
11
|
+
var SEARCH_ATTEMPTS_BEFORE_HOLDING_WRITES = 1;
|
|
12
|
+
var BUILD_TURN_HANDOVER_AFTER_MS = 50;
|
|
13
|
+
var BUILD_RETRY_FIRST_DELAY_MS = 250;
|
|
14
|
+
var BUILD_RETRY_MAX_DELAY_MS = 3e4;
|
|
15
|
+
var SHUFFLED_REBUILD_MEMORY_RETRY_MS = 1e3;
|
|
16
|
+
var REBUILD_REMOVED_RATIO = 0.2;
|
|
17
|
+
var DRIFT_CHECK_GROWTH_FACTOR = 2;
|
|
18
|
+
var RECALIBRATION_DRIFT_RATIO = 2.5;
|
|
19
|
+
var RECALIBRATION_CHUNK_VECTORS = 512;
|
|
20
|
+
var VECTOR_FILE_REPLACED_ABOVE_DEAD_SHARE = 0.2;
|
|
21
|
+
var VECTOR_FILE_FIXED_OVERHEAD_BYTES = 65536;
|
|
22
|
+
var VECTOR_FILE_STRING_HEADER_BYTES = 5;
|
|
23
|
+
var VECTOR_FILE_CENTROID_BYTES_PER_COMPONENT = 9;
|
|
24
|
+
var OSQ4_MIN_DIMENSION = 384;
|
|
25
|
+
var DEFAULT_VECTOR_METRIC = "cosine";
|
|
26
|
+
var DISK_ADOPTION_YIELD_INTERVAL = 4096;
|
|
27
|
+
|
|
28
|
+
export { BUILD_CHUNK_SIZE, BUILD_RETRY_FIRST_DELAY_MS, BUILD_RETRY_MAX_DELAY_MS, BUILD_STEPS_BETWEEN_CLOCK_CHECKS, BUILD_TURN_HANDOVER_AFTER_MS, DEFAULT_FILTER_THRESHOLD, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_VECTOR_METRIC, DISK_ADOPTION_YIELD_INTERVAL, DRIFT_CHECK_GROWTH_FACTOR, ESTIMATED_MS_PER_TOMBSTONE, ESTIMATED_MS_PER_VECTOR_REBUILD, OSQ4_MIN_DIMENSION, REBUILD_REMOVED_RATIO, RECALIBRATION_CHUNK_VECTORS, RECALIBRATION_DRIFT_RATIO, SEARCH_ATTEMPTS_BEFORE_HOLDING_WRITES, SEARCH_REFILL_RESERVE, SEARCH_SCAN_CHUNK_VECTORS, SHUFFLED_REBUILD_MEMORY_RETRY_MS, VECTOR_FILE_CENTROID_BYTES_PER_COMPONENT, VECTOR_FILE_FIXED_OVERHEAD_BYTES, VECTOR_FILE_REPLACED_ABOVE_DEAD_SHARE, VECTOR_FILE_STRING_HEADER_BYTES, WORKER_COPY_MIN_VECTORS };
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { writeLiveVectors, decodeSegmentManifest, openManifestPayload, listedKeysOf, decodeManifestPayload, SINGLE_NODE_PRIMARY_TERM } from './chunk-
|
|
2
|
-
import { vectorsPerFile } from './chunk-
|
|
3
|
-
import { vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION } from './chunk-
|
|
4
|
-
import { writeCheckpointSegments, commitCheckpointManifest, partitionContentsOf } from './chunk-
|
|
5
|
-
import { DEFAULT_COMPACTION_THRESHOLD } from './chunk-
|
|
6
|
-
import { manifestKey, manifestFileRefs } from './chunk-
|
|
7
|
-
import { HEADER_SIZE } from './chunk-
|
|
1
|
+
import { writeLiveVectors, decodeSegmentManifest, openManifestPayload, listedKeysOf, decodeManifestPayload, SINGLE_NODE_PRIMARY_TERM } from './chunk-ZS6CMXAE.mjs';
|
|
2
|
+
import { vectorsPerFile } from './chunk-IJSFWGVW.mjs';
|
|
3
|
+
import { vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION } from './chunk-G5KHSM6E.mjs';
|
|
4
|
+
import { writeCheckpointSegments, commitCheckpointManifest, partitionContentsOf } from './chunk-KQJVNPRM.mjs';
|
|
5
|
+
import { DEFAULT_COMPACTION_THRESHOLD } from './chunk-CSHHU7PA.mjs';
|
|
6
|
+
import { manifestKey, manifestFileRefs } from './chunk-7LDMWKP6.mjs';
|
|
7
|
+
import { HEADER_SIZE } from './chunk-2PNYKG2D.mjs';
|
|
8
8
|
import { MAX_FILE_BYTES } from './chunk-U476IXKQ.mjs';
|
|
9
9
|
import { NarsilError, ErrorCodes } from './chunk-MDF6PDIA.mjs';
|
|
10
10
|
|