@delali/narsil 0.3.1-next.109 → 0.3.1-next.112
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/browser/{chunk-RYWURYTW.mjs → chunk-2CBQ2PCC.mjs} +1 -1
- package/dist/browser/{chunk-EQ7BQ6VZ.mjs → chunk-LJS6PREA.mjs} +1 -0
- package/dist/browser/{chunk-3RXEGUGO.mjs → chunk-O5IKTA27.mjs} +41 -4
- package/dist/browser/{chunk-Z2K5AGZL.mjs → chunk-OVY4RPDS.mjs} +3 -3
- package/dist/browser/{chunk-S7KNCVPU.mjs → chunk-P7C2S5JX.mjs} +2 -2
- package/dist/browser/{chunk-SCBBEQWD.mjs → chunk-UR5MNN53.mjs} +189 -51
- package/dist/browser/{chunk-XHRHOH2P.mjs → chunk-YRYBTRWB.mjs} +139 -28
- package/dist/browser/{durability-wiring-JJGE7MED.mjs → durability-wiring-CPEMDDK7.mjs} +48 -20
- package/dist/browser/{module-PCKQADMQ.mjs → module-DYKR4INA.mjs} +5 -5
- package/dist/browser/{restore-queue-2STSWU2E.mjs → restore-queue-2H4QDQV4.mjs} +1 -1
- package/dist/browser/{runtime-ARKW5NZX.mjs → runtime-FL5K4JOH.mjs} +1 -1
- package/dist/browser/{snapshot-3AZDQKNA.mjs → snapshot-ISZMBDCN.mjs} +27 -14
- package/dist/browser/{write-XXLMPT3Y.mjs → write-IABOLE4N.mjs} +5 -5
- package/dist/{capabilities-D4MUmjfe.d.ts → capabilities-ByaX8Arf.d.ts} +2 -2
- package/dist/{chunk-CYXLPFBG.mjs → chunk-2IFU7SXG.mjs} +123 -35
- package/dist/{chunk-FN75BCXC.mjs → chunk-2RAQKITH.mjs} +1 -1
- package/dist/{chunk-VIXXQEX6.mjs → chunk-3COL24MT.mjs} +43 -6
- package/dist/{chunk-ICZW4J3I.mjs → chunk-3RRATITJ.mjs} +21 -21
- package/dist/{chunk-SAZB7ZHT.mjs → chunk-542TD4BL.mjs} +1 -1
- package/dist/{chunk-C57FPCTL.mjs → chunk-5WEBVYOA.mjs} +182 -44
- package/dist/{chunk-YP23IAV5.mjs → chunk-63HFWQEA.mjs} +4 -3
- package/dist/{chunk-FV5CP3E2.mjs → chunk-6ODIY6ZB.mjs} +1 -1
- package/dist/{chunk-RQ7LOBLD.mjs → chunk-6PN3U6XW.mjs} +1 -1
- package/dist/{chunk-SYHS56F7.mjs → chunk-7CI34JAP.mjs} +26 -13
- package/dist/{chunk-IBJWO5PF.mjs → chunk-7ZZQNOIY.mjs} +1 -1
- package/dist/{chunk-H5465KUZ.mjs → chunk-BVOUSX3U.mjs} +4 -3
- package/dist/{chunk-AWZVJCCW.mjs → chunk-DG62H7OD.mjs} +2 -2
- package/dist/{chunk-LWQTFKRS.mjs → chunk-FUUP45IS.mjs} +1 -1
- package/dist/{chunk-HL2GAS5N.mjs → chunk-GLZI636G.mjs} +7 -7
- package/dist/{chunk-C55OFKU6.mjs → chunk-HNAP2KTG.mjs} +2 -2
- package/dist/{chunk-A2HWRBQE.mjs → chunk-IBTKJIV7.mjs} +2 -2
- package/dist/{chunk-PA5WZWLC.mjs → chunk-IM76DLA7.mjs} +2 -2
- package/dist/{chunk-5LP7OU6B.mjs → chunk-IR4KQ4WB.mjs} +2 -2
- package/dist/{chunk-FABI2JA7.mjs → chunk-J6YVB2XM.mjs} +1 -1
- package/dist/{chunk-62GK7UUS.mjs → chunk-KCQG2PJT.mjs} +3 -3
- package/dist/{chunk-U6FT7LKG.mjs → chunk-LOBYK2FG.mjs} +3 -3
- package/dist/{chunk-XY6Z7T5W.mjs → chunk-N57XXGPG.mjs} +2 -2
- package/dist/{chunk-S2CKO6PP.mjs → chunk-OBWUWZRA.mjs} +1 -0
- package/dist/{chunk-CKRP7GCS.mjs → chunk-OCB7TMUQ.mjs} +1 -1
- package/dist/{chunk-NYWUBTMH.mjs → chunk-OFYKBKKV.mjs} +1 -1
- package/dist/{chunk-ZBZRBM2Z.mjs → chunk-PT7VRWS7.mjs} +3 -3
- package/dist/{chunk-DUWYU4D5.mjs → chunk-PWIWSUCX.mjs} +2 -2
- package/dist/{chunk-5W7ADHK7.mjs → chunk-QI6JW4PM.mjs} +1 -1
- package/dist/{chunk-SD2EYVU4.mjs → chunk-QIFOXLWF.mjs} +2 -2
- package/dist/{chunk-N75MAOM7.mjs → chunk-R2MRGYU5.mjs} +2 -2
- package/dist/{chunk-5DPTHLMX.mjs → chunk-V7G627TA.mjs} +1 -1
- package/dist/{chunk-4GD43FSW.mjs → chunk-V7ISFBFC.mjs} +10 -10
- package/dist/{chunk-MVWWAHTR.mjs → chunk-VK45J47R.mjs} +1 -1
- package/dist/{chunk-RSSLOFRP.mjs → chunk-VLRT47GU.mjs} +39 -8
- package/dist/{chunk-RPIE4QCL.mjs → chunk-YH5OIL7B.mjs} +2 -0
- package/dist/{chunk-JKOBHB4A.mjs → chunk-YIDSHBQS.mjs} +2 -2
- package/dist/{client-Ba9yP-Rr.d.ts → client-BdnzDFtO.d.ts} +2 -2
- package/dist/client.d.ts +6 -6
- package/dist/client.mjs +1 -0
- package/dist/{config-C-xXz8bC.d.ts → config-CSgvApt0.d.ts} +1 -1
- package/dist/{config-Dc_8-byz.d.ts → config-ujZESkJz.d.ts} +1 -1
- package/dist/{direct-executor-QH2M6AK2.mjs → direct-executor-ZSTKRA6Z.mjs} +11 -11
- package/dist/distribution/coordinator/etcd.mjs +94 -5
- package/dist/distribution/transport/grpc.mjs +24 -8
- package/dist/distribution/transport/in-memory.mjs +4 -4
- package/dist/distribution/transport/tcp.mjs +6 -6
- package/dist/distribution.d.ts +5 -5
- package/dist/distribution.mjs +1232 -377
- package/dist/{durability-wiring-4HWX7OVP.mjs → durability-wiring-EQTFL3B4.mjs} +65 -38
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-CMZ5XSLy.d.ts → engine-C_HSy3iz.d.ts} +1 -1
- package/dist/{events-dnvgprIl.d.ts → events-CJsXF1F9.d.ts} +1 -0
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +5 -5
- package/dist/index.mjs +16 -16
- package/dist/{module-IYILOTU3.mjs → module-WP4VVGV2.mjs} +9 -9
- package/dist/partial-words.mjs +5 -5
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +12 -0
- package/dist/persistence/durability/checkpoint-worker.mjs +25 -23
- package/dist/react.d.ts +6 -6
- package/dist/react.mjs +1 -0
- package/dist/{rerank-EHJVXEDP.mjs → rerank-3POMPQNX.mjs} +4 -4
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{restore-queue-BONSQWCI.mjs → restore-queue-WZTMTJP2.mjs} +1 -1
- package/dist/{runtime-LEX2F7OW.mjs → runtime-C2435MPK.mjs} +5 -5
- package/dist/server.d.ts +8 -8
- package/dist/server.mjs +12 -12
- package/dist/{snapshot-DWDLN5YW.mjs → snapshot-EU6NNB2P.mjs} +25 -25
- package/dist/synonyms.mjs +3 -3
- package/dist/{tasks-B4B2cMK5.d.ts → tasks-CI3Fvkxw.d.ts} +1 -1
- package/dist/{thread-I32PUURR.mjs → thread-WGBCVJII.mjs} +11 -11
- package/dist/vector/search-worker.mjs +4 -4
- package/dist/workers/entry.mjs +5 -5
- package/dist/{write-AOFSHACR.mjs → write-CKRWS3P5.mjs} +18 -18
- package/package.json +7 -7
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes } from './chunk-
|
|
2
|
-
import { VERSION, rulesOfStoredSynonyms, flattenSchema, packChecksummedEnvelope, openChecksummedEnvelope, requireValidFieldTypes, isCompositePartition, crc32, reconstructSchemaFromMetadata, extractVectorFieldsFromSchema, readHeader, HEADER_SIZE, engineVersionParts, writeHeader, partialFieldsOf, getLanguage, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, VECTOR_PATTERN, isQuantizationMode, isStorageMode, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, fnv1a, selectionKeeps, decodeMapWithoutFields, prepareDocumentVectors, messagePackUpperBoundBytes, segmentArraysOf, mergeSegmentContents, resolveIndexAnalysis } from './chunk-
|
|
3
|
-
import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-
|
|
4
|
-
import { compareCodePoints, MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-
|
|
5
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
1
|
+
import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes, uint64Bytes } from './chunk-O5IKTA27.mjs';
|
|
2
|
+
import { VERSION, rulesOfStoredSynonyms, flattenSchema, packChecksummedEnvelope, openChecksummedEnvelope, requireValidFieldTypes, isCompositePartition, crc32, reconstructSchemaFromMetadata, extractVectorFieldsFromSchema, readHeader, HEADER_SIZE, engineVersionParts, writeHeader, partialFieldsOf, getLanguage, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, VECTOR_PATTERN, isQuantizationMode, isStorageMode, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, fnv1a, selectionKeeps, decodeMapWithoutFields, prepareDocumentVectors, messagePackUpperBoundBytes, segmentArraysOf, mergeSegmentContents, resolveIndexAnalysis } from './chunk-UR5MNN53.mjs';
|
|
3
|
+
import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-P7C2S5JX.mjs';
|
|
4
|
+
import { compareCodePoints, MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-2CBQ2PCC.mjs';
|
|
5
|
+
import { NarsilError, ErrorCodes } from './chunk-LJS6PREA.mjs';
|
|
6
6
|
import { decode, encode } from '@msgpack/msgpack';
|
|
7
7
|
|
|
8
8
|
// src/engine/durability-metadata.ts
|
|
@@ -733,7 +733,10 @@ function encodeSegmentManifest(manifest) {
|
|
|
733
733
|
checkpoint: manifest.checkpoint.map((c) => ({
|
|
734
734
|
partitionId: c.partitionId,
|
|
735
735
|
lastSeqNo: c.lastSeqNo,
|
|
736
|
-
primaryTerm: c.primaryTerm
|
|
736
|
+
primaryTerm: c.primaryTerm,
|
|
737
|
+
...c.termHistory === void 0 ? {} : {
|
|
738
|
+
termHistory: c.termHistory.map((start) => ({ primaryTerm: start.primaryTerm, firstSeqNo: start.firstSeqNo }))
|
|
739
|
+
}
|
|
737
740
|
})),
|
|
738
741
|
partitions: manifest.partitions.map((p) => ({
|
|
739
742
|
partitionId: p.partitionId,
|
|
@@ -922,7 +925,33 @@ function normalizeCheckpoint(raw) {
|
|
|
922
925
|
invalidManifest("has an invalid partition checkpoint", { entry });
|
|
923
926
|
}
|
|
924
927
|
seen.add(entry.partitionId);
|
|
925
|
-
|
|
928
|
+
const checkpoint = {
|
|
929
|
+
partitionId: entry.partitionId,
|
|
930
|
+
lastSeqNo: entry.lastSeqNo,
|
|
931
|
+
primaryTerm: entry.primaryTerm
|
|
932
|
+
};
|
|
933
|
+
if (entry.termHistory !== void 0) {
|
|
934
|
+
checkpoint.termHistory = normalizeTermHistory(entry.termHistory, entry.partitionId, entry.lastSeqNo);
|
|
935
|
+
}
|
|
936
|
+
return checkpoint;
|
|
937
|
+
});
|
|
938
|
+
}
|
|
939
|
+
function normalizeTermHistory(raw, partitionId, lastSeqNo) {
|
|
940
|
+
if (!Array.isArray(raw)) invalidManifest("has a term history that is not a list", { partitionId });
|
|
941
|
+
let previous = null;
|
|
942
|
+
return raw.map((start) => {
|
|
943
|
+
if (!isRecord(start) || !isSequenceNumber(start.primaryTerm) || !isSequenceNumber(start.firstSeqNo)) {
|
|
944
|
+
invalidManifest("has an invalid term history entry", { partitionId });
|
|
945
|
+
}
|
|
946
|
+
const read = { primaryTerm: start.primaryTerm, firstSeqNo: start.firstSeqNo };
|
|
947
|
+
if (read.firstSeqNo > lastSeqNo) {
|
|
948
|
+
invalidManifest(`has a term starting at seqNo ${read.firstSeqNo}, above lastSeqNo ${lastSeqNo}`, { partitionId });
|
|
949
|
+
}
|
|
950
|
+
if (previous !== null && (read.primaryTerm <= previous.primaryTerm || read.firstSeqNo <= previous.firstSeqNo)) {
|
|
951
|
+
invalidManifest("has a term history that fails to ascend strictly", { partitionId });
|
|
952
|
+
}
|
|
953
|
+
previous = read;
|
|
954
|
+
return read;
|
|
926
955
|
});
|
|
927
956
|
}
|
|
928
957
|
function normalizePartitions(raw, indexName) {
|
|
@@ -997,30 +1026,40 @@ function frozenSegmentsOf(partition, shared) {
|
|
|
997
1026
|
if (tail !== null) segments.push(tail);
|
|
998
1027
|
return segments;
|
|
999
1028
|
}
|
|
1000
|
-
function partitionContentsOf(manager, partitionId) {
|
|
1029
|
+
function partitionContentsOf(manager, partitionId, captureRecords) {
|
|
1001
1030
|
const partition = manager.getPartition(partitionId);
|
|
1002
|
-
const
|
|
1003
|
-
|
|
1031
|
+
const segments = frozenSegmentsOf(partition, false);
|
|
1032
|
+
const selections = segments.map((segment) => selectionOfMask(segment, keepMaskOf(segment)));
|
|
1033
|
+
const contents = { partitionId, docCount: partition.count(), selections };
|
|
1034
|
+
if (captureRecords !== void 0) contents.writeRecords = captureRecords(segments);
|
|
1035
|
+
return contents;
|
|
1004
1036
|
}
|
|
1005
|
-
function captureWholePartition(manager, partitionId) {
|
|
1037
|
+
function captureWholePartition(manager, partitionId, captureRecords) {
|
|
1006
1038
|
if (typeof SharedArrayBuffer !== "function") return null;
|
|
1007
1039
|
const partition = manager.getPartition(partitionId);
|
|
1008
1040
|
if (!isCompositePartition(partition) || partition.frozenSegmentCount() === 0) return null;
|
|
1041
|
+
const frozen = frozenSegmentsOf(partition, true);
|
|
1042
|
+
const records = captureRecords?.(frozen);
|
|
1009
1043
|
const segments = [];
|
|
1010
|
-
for (const segment of
|
|
1044
|
+
for (const segment of frozen) {
|
|
1011
1045
|
if (segment.sharedSnapshot === null) return null;
|
|
1012
|
-
|
|
1046
|
+
const captured = { snapshot: segment.sharedSnapshot, keep: keepMaskOf(segment) };
|
|
1047
|
+
const held = records?.get(segment);
|
|
1048
|
+
if (held !== void 0) captured.writeRecords = held;
|
|
1049
|
+
segments.push(captured);
|
|
1013
1050
|
}
|
|
1014
1051
|
return { partitionId, docCount: partition.count(), segments };
|
|
1015
1052
|
}
|
|
1016
1053
|
function contentsOfCaptured(captured) {
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
};
|
|
1054
|
+
const writeRecords = /* @__PURE__ */ new Map();
|
|
1055
|
+
const selections = captured.segments.map(({ snapshot, keep, writeRecords: held }) => {
|
|
1056
|
+
const segment = createSharedFrozenSegment(snapshot);
|
|
1057
|
+
if (held !== void 0) writeRecords.set(segment, held);
|
|
1058
|
+
return selectionOfMask(segment, keep);
|
|
1059
|
+
});
|
|
1060
|
+
const contents = { partitionId: captured.partitionId, docCount: captured.docCount, selections };
|
|
1061
|
+
if (writeRecords.size > 0) contents.writeRecords = writeRecords;
|
|
1062
|
+
return contents;
|
|
1024
1063
|
}
|
|
1025
1064
|
|
|
1026
1065
|
// src/persistence/durability/segment/store.ts
|
|
@@ -1383,6 +1422,15 @@ function optionalIndexUuid(value, payloadName) {
|
|
|
1383
1422
|
}
|
|
1384
1423
|
return value;
|
|
1385
1424
|
}
|
|
1425
|
+
function optionalPreviousPrimaryTerm(value, payloadName) {
|
|
1426
|
+
if (value === void 0 || value === null) {
|
|
1427
|
+
return void 0;
|
|
1428
|
+
}
|
|
1429
|
+
if (typeof value !== "number" || !Number.isInteger(value) || value < 0) {
|
|
1430
|
+
throw new Error(`Invalid ${payloadName}: "previousPrimaryTerm" must be a non-negative integer or absent`);
|
|
1431
|
+
}
|
|
1432
|
+
return value;
|
|
1433
|
+
}
|
|
1386
1434
|
function validateEntryPayload(decoded) {
|
|
1387
1435
|
if (!isRecord2(decoded) || !isRecord2(decoded.entry)) {
|
|
1388
1436
|
throw new Error('Invalid EntryPayload: missing or invalid "entry" field');
|
|
@@ -1413,6 +1461,7 @@ function validateEntryPayload(decoded) {
|
|
|
1413
1461
|
throw new Error('Invalid EntryPayload: "entry.checksum" must be a number');
|
|
1414
1462
|
}
|
|
1415
1463
|
optionalIndexUuid(decoded.indexUuid, "EntryPayload");
|
|
1464
|
+
optionalPreviousPrimaryTerm(decoded.previousPrimaryTerm, "EntryPayload");
|
|
1416
1465
|
return decoded;
|
|
1417
1466
|
}
|
|
1418
1467
|
function computeEntryChecksum(entry) {
|
|
@@ -1699,6 +1748,17 @@ async function* walEntriesInRange(directory, indexName, partitionId, fromSeqNoEx
|
|
|
1699
1748
|
throw durableRecordMissing(indexName, partitionId, highestRead, wal.marker);
|
|
1700
1749
|
}
|
|
1701
1750
|
}
|
|
1751
|
+
function writeRecordsOf(documents, written) {
|
|
1752
|
+
const seqNos = new Float64Array(documents.length);
|
|
1753
|
+
const primaryTerms = new Float64Array(documents.length);
|
|
1754
|
+
for (let at = 0; at < documents.length; at++) {
|
|
1755
|
+
const entry = written.get(documents[at].docId);
|
|
1756
|
+
if (entry === void 0) continue;
|
|
1757
|
+
seqNos[at] = entry.seqNo;
|
|
1758
|
+
primaryTerms[at] = entry.primaryTerm;
|
|
1759
|
+
}
|
|
1760
|
+
return { seqNos, primaryTerms };
|
|
1761
|
+
}
|
|
1702
1762
|
function everyVectorFieldIsTopLevel(vectorFieldPaths) {
|
|
1703
1763
|
for (const fieldPath of vectorFieldPaths) {
|
|
1704
1764
|
if (fieldPath.includes(".")) return false;
|
|
@@ -1716,15 +1776,18 @@ async function buildSegmentFromEntries(input) {
|
|
|
1716
1776
|
const leavesVectorsUndecoded = everyVectorFieldIsTopLevel(input.vectorFieldPaths);
|
|
1717
1777
|
const live = /* @__PURE__ */ new Map();
|
|
1718
1778
|
const deleted = /* @__PURE__ */ new Set();
|
|
1779
|
+
const written = /* @__PURE__ */ new Map();
|
|
1719
1780
|
for await (const entry of input.entries) {
|
|
1720
1781
|
if (entry.operation === "DELETE") {
|
|
1721
1782
|
deleted.add(entry.documentId);
|
|
1722
1783
|
live.delete(entry.documentId);
|
|
1784
|
+
written.delete(entry.documentId);
|
|
1723
1785
|
continue;
|
|
1724
1786
|
}
|
|
1725
1787
|
if (entry.document === null) continue;
|
|
1726
1788
|
live.delete(entry.documentId);
|
|
1727
1789
|
live.set(entry.documentId, documentWithoutVectors(entry.document, input.vectorFieldPaths, leavesVectorsUndecoded));
|
|
1790
|
+
if (input.keepsWriteRecords) written.set(entry.documentId, entry);
|
|
1728
1791
|
}
|
|
1729
1792
|
const tombstones = [];
|
|
1730
1793
|
for (const docId of deleted) {
|
|
@@ -1747,7 +1810,9 @@ async function buildSegmentFromEntries(input) {
|
|
|
1747
1810
|
payload,
|
|
1748
1811
|
documents.map((entry) => entry.document)
|
|
1749
1812
|
);
|
|
1750
|
-
|
|
1813
|
+
const built = { selections: [{ segment }], tombstones, docCount: live.size };
|
|
1814
|
+
if (input.keepsWriteRecords) built.writeRecords = /* @__PURE__ */ new Map([[segment, writeRecordsOf(documents, written)]]);
|
|
1815
|
+
return built;
|
|
1751
1816
|
}
|
|
1752
1817
|
var FIXMAP = 128;
|
|
1753
1818
|
var MAP16 = 222;
|
|
@@ -1950,7 +2015,8 @@ function encodeKeywordSegment(contents) {
|
|
|
1950
2015
|
const { arrays } = contents;
|
|
1951
2016
|
const postings = postingsInTokenOrder(arrays);
|
|
1952
2017
|
const out = new MessagePackChunks();
|
|
1953
|
-
|
|
2018
|
+
const { writeRecords } = contents;
|
|
2019
|
+
out.map(writeRecords === void 0 ? TOP_LEVEL_ENTRIES : TOP_LEVEL_ENTRIES + 2);
|
|
1954
2020
|
out.key("v").value(KEYWORD_SEGMENT_PAYLOAD_VERSION);
|
|
1955
2021
|
out.key("schema").value(contents.schema);
|
|
1956
2022
|
out.key("tombstones").value(contents.tombstones);
|
|
@@ -1988,6 +2054,10 @@ function encodeKeywordSegment(contents) {
|
|
|
1988
2054
|
writeSurfaceForms(out, arrays);
|
|
1989
2055
|
out.key("partialWords");
|
|
1990
2056
|
writePartialWords(out, arrays);
|
|
2057
|
+
if (writeRecords !== void 0) {
|
|
2058
|
+
out.key("seqNos").binary(uint64Bytes(writeRecords.seqNos));
|
|
2059
|
+
out.key("primaryTerms").binary(uint64Bytes(writeRecords.primaryTerms));
|
|
2060
|
+
}
|
|
1991
2061
|
return out;
|
|
1992
2062
|
}
|
|
1993
2063
|
|
|
@@ -2199,6 +2269,28 @@ function noSpellingsOf(analysis) {
|
|
|
2199
2269
|
if (analysis === null || analysis.fields.length === 0) return null;
|
|
2200
2270
|
return { fields: [...analysis.fields], tokens: [], spellings: [], counts: [] };
|
|
2201
2271
|
}
|
|
2272
|
+
function writeRecordsOfGroup(group, records, count) {
|
|
2273
|
+
const seqNos = new Float64Array(count);
|
|
2274
|
+
const primaryTerms = new Float64Array(count);
|
|
2275
|
+
let at = 0;
|
|
2276
|
+
for (const selection of group?.selections ?? []) {
|
|
2277
|
+
const source = records.get(selection.segment);
|
|
2278
|
+
for (let ordinal = 0; ordinal < selection.segment.arrays.documentCount; ordinal++) {
|
|
2279
|
+
if (!selectionKeeps(selection, ordinal)) continue;
|
|
2280
|
+
if (source !== void 0) {
|
|
2281
|
+
seqNos[at] = source.seqNos[ordinal];
|
|
2282
|
+
primaryTerms[at] = source.primaryTerms[ordinal];
|
|
2283
|
+
}
|
|
2284
|
+
at += 1;
|
|
2285
|
+
}
|
|
2286
|
+
}
|
|
2287
|
+
return { seqNos, primaryTerms };
|
|
2288
|
+
}
|
|
2289
|
+
function withWriteRecords(contents, group, request) {
|
|
2290
|
+
if (request.writeRecords === void 0) return contents;
|
|
2291
|
+
const writeRecords = writeRecordsOfGroup(group, request.writeRecords, contents.arrays.documentCount);
|
|
2292
|
+
return { ...contents, writeRecords };
|
|
2293
|
+
}
|
|
2202
2294
|
function contentsOfGroup(group, request, tombstones) {
|
|
2203
2295
|
const { schema, partialWordAnalysis, textAnalysis } = request;
|
|
2204
2296
|
if (group === null) {
|
|
@@ -2227,7 +2319,7 @@ function requireRoomForAnotherSegment(state) {
|
|
|
2227
2319
|
async function writeGroup(state, group) {
|
|
2228
2320
|
const { request } = state;
|
|
2229
2321
|
const tombstones = state.tombstonesWritten ? [] : [...request.tombstones];
|
|
2230
|
-
const contents = contentsOfGroup(group, request, tombstones);
|
|
2322
|
+
const contents = withWriteRecords(contentsOfGroup(group, request, tombstones), group, request);
|
|
2231
2323
|
const encoded = encodeKeywordSegment(contents);
|
|
2232
2324
|
if (HEADER_SIZE + encoded.byteLength > MAX_FILE_BYTES && group !== null && group.documentCount > 1) {
|
|
2233
2325
|
const [first, second] = halvesOf(group);
|
|
@@ -2314,6 +2406,15 @@ function applyTimeOrder(frozen, contents) {
|
|
|
2314
2406
|
}
|
|
2315
2407
|
}
|
|
2316
2408
|
}
|
|
2409
|
+
function writeRecordsOf2(frozen, contents) {
|
|
2410
|
+
if (contents.every((segment) => segment.writeRecords === void 0)) return void 0;
|
|
2411
|
+
const records = /* @__PURE__ */ new Map();
|
|
2412
|
+
for (let at = 0; at < frozen.length; at++) {
|
|
2413
|
+
const held = contents[at].writeRecords;
|
|
2414
|
+
if (held !== void 0) records.set(frozen[at], held);
|
|
2415
|
+
}
|
|
2416
|
+
return records;
|
|
2417
|
+
}
|
|
2317
2418
|
function retainedTombstones(frozen, contents) {
|
|
2318
2419
|
const retained = /* @__PURE__ */ new Set();
|
|
2319
2420
|
for (const segment of contents) {
|
|
@@ -2337,6 +2438,7 @@ async function compactPartitionSegments(input) {
|
|
|
2337
2438
|
for (const ref of windowRefs) contents.push(await readSegmentFile(input.store, ref));
|
|
2338
2439
|
const frozen = contents.map(frozenOf);
|
|
2339
2440
|
applyTimeOrder(frozen, contents);
|
|
2441
|
+
const writeRecords = writeRecordsOf2(frozen, contents);
|
|
2340
2442
|
const written = await writeSegmentsOf({
|
|
2341
2443
|
store: input.store,
|
|
2342
2444
|
indexName: input.indexName,
|
|
@@ -2347,7 +2449,8 @@ async function compactPartitionSegments(input) {
|
|
|
2347
2449
|
textAnalysis: input.textAnalysis,
|
|
2348
2450
|
selections: frozen.map((segment) => ({ segment })),
|
|
2349
2451
|
tombstones: retainedTombstones(frozen, contents),
|
|
2350
|
-
mergesIntoOneFile: true
|
|
2452
|
+
mergesIntoOneFile: true,
|
|
2453
|
+
...writeRecords === void 0 ? {} : { writeRecords }
|
|
2351
2454
|
});
|
|
2352
2455
|
return {
|
|
2353
2456
|
segments: [
|
|
@@ -2413,7 +2516,8 @@ async function writeWholePartition(context, partitionId, priorPartition, whole)
|
|
|
2413
2516
|
partialWordAnalysis: context.partialWordAnalysis,
|
|
2414
2517
|
textAnalysis: context.textAnalysis,
|
|
2415
2518
|
selections: whole.selections,
|
|
2416
|
-
tombstones: []
|
|
2519
|
+
tombstones: [],
|
|
2520
|
+
...whole.writeRecords === void 0 ? {} : { writeRecords: whole.writeRecords }
|
|
2417
2521
|
});
|
|
2418
2522
|
return { partitionId, nextSegmentId: written.nextSegmentId, segments: written.segments };
|
|
2419
2523
|
}
|
|
@@ -2423,7 +2527,8 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
|
|
|
2423
2527
|
config: context.config,
|
|
2424
2528
|
language: getLanguage(context.config.language ?? "english"),
|
|
2425
2529
|
vectorFieldPaths: context.vectorFieldPaths,
|
|
2426
|
-
entries
|
|
2530
|
+
entries,
|
|
2531
|
+
keepsWriteRecords: context.input.keepsWriteRecords === true
|
|
2427
2532
|
});
|
|
2428
2533
|
let segments = priorPartition ? [...priorPartition.segments] : [];
|
|
2429
2534
|
let nextSegmentId = priorPartition?.nextSegmentId ?? 0;
|
|
@@ -2437,7 +2542,8 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
|
|
|
2437
2542
|
partialWordAnalysis: context.partialWordAnalysis,
|
|
2438
2543
|
textAnalysis: context.textAnalysis,
|
|
2439
2544
|
selections: built.selections,
|
|
2440
|
-
tombstones: built.tombstones
|
|
2545
|
+
tombstones: built.tombstones,
|
|
2546
|
+
...built.writeRecords === void 0 ? {} : { writeRecords: built.writeRecords }
|
|
2441
2547
|
});
|
|
2442
2548
|
segments.push(...written.segments);
|
|
2443
2549
|
nextSegmentId = written.nextSegmentId;
|
|
@@ -2580,6 +2686,7 @@ async function loadPartitionSegments(store, partition, manager, options) {
|
|
|
2580
2686
|
if (options.decodesEveryDocument) decodeEveryStoredDocument(contents.arrays.documentTable);
|
|
2581
2687
|
const segment = createSharedFrozenSegment(shareSegmentArrays(contents.arrays, manager.partialWordAnalysis));
|
|
2582
2688
|
removeWhatTheSegmentReplaces(manager, segment, contents);
|
|
2689
|
+
options.writeRecords?.loadSegment(manager.indexName, partition.partitionId, segment, contents);
|
|
2583
2690
|
if (contents.arrays.documentCount > 0) manager.attachFrozenSegment(partition.partitionId, segment);
|
|
2584
2691
|
await yieldToEventLoop();
|
|
2585
2692
|
}
|
|
@@ -2626,7 +2733,11 @@ async function loadSegmentedCheckpoint(store, indexName, manifest, deps, decodes
|
|
|
2626
2733
|
{ indexName, partitionId: partition.partitionId, partitionCount: deps.manager.partitionCount }
|
|
2627
2734
|
);
|
|
2628
2735
|
}
|
|
2629
|
-
await loadPartitionSegments(store, partition, deps.manager, {
|
|
2736
|
+
await loadPartitionSegments(store, partition, deps.manager, {
|
|
2737
|
+
schema: manifest.schema,
|
|
2738
|
+
decodesEveryDocument,
|
|
2739
|
+
writeRecords: deps.writeRecords ?? null
|
|
2740
|
+
});
|
|
2630
2741
|
}
|
|
2631
2742
|
for (const vector of manifest.vectors) {
|
|
2632
2743
|
const vectorIndex = deps.vectorIndexes.get(vector.fieldPath);
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { nodeThreadId, spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
|
|
2
|
-
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, decodeSegmentManifest, readListedFile, adapterCheckpointStore, SINGLE_NODE_PRIMARY_TERM, readMetadataEnvelope, writeMetadataEnvelope, metadataFromConfig, DIRECTORY_LOCK_ATTEMPTS, createSeqOwner, readSegmentManifest, snapshotBundleKey, reclaimUnlistedKeys, manifestReferencedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, manifestFileRefs, DIRECTORY_LOCK_SETTLE_MS, readSegmentFile, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, frameRecord, isId, isUint32, isSequenceNumber, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, commitCheckpointManifest, removeCheckpointGarbage, readWalSegments, durableRecordMissing, MAX_SEGMENTS_PER_PARTITION, growPartitionsTo, writeSegmentHeader, VECTOR_FILE_ENVELOPE_VERSION, SEGMENT_HEADER_SIZE, documentsOfPayloadV2, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, metadataConfigOf, buildEntry, readCurrentManifest, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-
|
|
3
|
-
import { applyDeleteEntry, applyIndexEntry } from './chunk-
|
|
4
|
-
import { encodeDocument, fnv1a, requireValidFieldTypes, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-
|
|
5
|
-
import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-
|
|
6
|
-
import { compareCodePoints, MIN_M, MAX_FILE_BYTES, detectRuntime, resolveWorkerEntry } from './chunk-
|
|
2
|
+
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, decodeSegmentManifest, readListedFile, adapterCheckpointStore, SINGLE_NODE_PRIMARY_TERM, readMetadataEnvelope, writeMetadataEnvelope, metadataFromConfig, DIRECTORY_LOCK_ATTEMPTS, createSeqOwner, readSegmentManifest, snapshotBundleKey, reclaimUnlistedKeys, manifestReferencedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, manifestFileRefs, DIRECTORY_LOCK_SETTLE_MS, readSegmentFile, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, frameRecord, isId, isUint32, isSequenceNumber, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, commitCheckpointManifest, removeCheckpointGarbage, readWalSegments, durableRecordMissing, MAX_SEGMENTS_PER_PARTITION, growPartitionsTo, writeSegmentHeader, VECTOR_FILE_ENVELOPE_VERSION, SEGMENT_HEADER_SIZE, documentsOfPayloadV2, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, metadataConfigOf, buildEntry, readCurrentManifest, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-YRYBTRWB.mjs';
|
|
3
|
+
import { applyDeleteEntry, applyIndexEntry } from './chunk-O5IKTA27.mjs';
|
|
4
|
+
import { encodeDocument, fnv1a, requireValidFieldTypes, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-UR5MNN53.mjs';
|
|
5
|
+
import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-P7C2S5JX.mjs';
|
|
6
|
+
import { compareCodePoints, MIN_M, MAX_FILE_BYTES, detectRuntime, resolveWorkerEntry } from './chunk-2CBQ2PCC.mjs';
|
|
7
7
|
import './chunk-JPTQZWJQ.mjs';
|
|
8
|
-
import { NarsilError, ErrorCodes, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-
|
|
8
|
+
import { NarsilError, ErrorCodes, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-LJS6PREA.mjs';
|
|
9
9
|
import { decode } from '@msgpack/msgpack';
|
|
10
10
|
|
|
11
11
|
// src/persistence/durability/node-fs.browser.ts
|
|
@@ -1108,6 +1108,7 @@ async function truncateSegmentTail(directory, key, durableByteLength) {
|
|
|
1108
1108
|
}
|
|
1109
1109
|
}
|
|
1110
1110
|
function applyEntry(entry, deps) {
|
|
1111
|
+
deps.writeRecords?.replayed(entry);
|
|
1111
1112
|
if (entry.operation === "DELETE") {
|
|
1112
1113
|
applyDeleteEntry(entry, deps.manager, deps.vectorIndexes);
|
|
1113
1114
|
return;
|
|
@@ -1166,7 +1167,7 @@ function noWholePartitions() {
|
|
|
1166
1167
|
return { contents: [], captured: [] };
|
|
1167
1168
|
}
|
|
1168
1169
|
function wholePartitionsWhereMostChanged(manager, rules) {
|
|
1169
|
-
const { priorCheckpoint, priorPartitionIds, everyPartition, aWorkerCanSerialise } = rules;
|
|
1170
|
+
const { priorCheckpoint, priorPartitionIds, everyPartition, aWorkerCanSerialise, captureRecords } = rules;
|
|
1170
1171
|
return (targets) => {
|
|
1171
1172
|
const whole = { contents: [], captured: [] };
|
|
1172
1173
|
const changed = targets.filter(
|
|
@@ -1180,17 +1181,18 @@ function wholePartitionsWhereMostChanged(manager, rules) {
|
|
|
1180
1181
|
const logMatchesMemory = targets.length === 1 && priorPartitionIds.every((partitionId) => partitionId === 0);
|
|
1181
1182
|
if (!everyPartition && !(mostOfEachChanged && logMatchesMemory)) return whole;
|
|
1182
1183
|
for (const { partitionId } of changed) {
|
|
1183
|
-
const
|
|
1184
|
+
const capture2 = captureRecords?.(partitionId);
|
|
1185
|
+
const captured = aWorkerCanSerialise ? captureWholePartition(manager, partitionId, capture2) : null;
|
|
1184
1186
|
if (captured !== null) {
|
|
1185
1187
|
whole.captured.push(captured);
|
|
1186
1188
|
} else if (everyPartition || !aWorkerCanSerialise) {
|
|
1187
|
-
whole.contents.push(partitionContentsOf(manager, partitionId));
|
|
1189
|
+
whole.contents.push(partitionContentsOf(manager, partitionId, capture2));
|
|
1188
1190
|
}
|
|
1189
1191
|
}
|
|
1190
1192
|
return whole;
|
|
1191
1193
|
};
|
|
1192
1194
|
}
|
|
1193
|
-
async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartitionsOf = noWholePartitions, priorCheckpointThatLimitsTheRecords, listedVectorFields = []) {
|
|
1195
|
+
async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartitionsOf = noWholePartitions, priorCheckpointThatLimitsTheRecords, listedVectorFields = [], termHistoryOf) {
|
|
1194
1196
|
for (const vectorIndex of vectorIndexes.values()) await vectorIndex.completeGraph();
|
|
1195
1197
|
return whileNoMutationApplies([...indexState.partitions.values()], () => {
|
|
1196
1198
|
const documentCount = manager.countDocuments();
|
|
@@ -1205,11 +1207,13 @@ async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartit
|
|
|
1205
1207
|
snapshotCheckpointFor(priorCheckpointThatLimitsTheRecords, partitionId) + LOG_RECORDS_PER_CHECKPOINT
|
|
1206
1208
|
);
|
|
1207
1209
|
recordsLeftInLog += appliedSeqNo - lastSeqNo;
|
|
1208
|
-
|
|
1210
|
+
const target = {
|
|
1209
1211
|
partitionId,
|
|
1210
1212
|
lastSeqNo,
|
|
1211
1213
|
primaryTerm: partition?.seqOwner.primaryTerm ?? SINGLE_NODE_PRIMARY_TERM
|
|
1212
|
-
}
|
|
1214
|
+
};
|
|
1215
|
+
if (termHistoryOf !== void 0) target.termHistory = termHistoryOf(partitionId, lastSeqNo);
|
|
1216
|
+
targets.push(target);
|
|
1213
1217
|
}
|
|
1214
1218
|
const vectorPlans = /* @__PURE__ */ new Map();
|
|
1215
1219
|
for (const [fieldPath, vectorIndex] of vectorIndexes) {
|
|
@@ -1415,13 +1419,15 @@ function workerCanWriteEverySegment(input) {
|
|
|
1415
1419
|
}
|
|
1416
1420
|
async function writeIndexCheckpointSegments(input) {
|
|
1417
1421
|
const { directory, metadata, targets, compactionThreshold, prior, wholePartitions, capturedPartitions } = input;
|
|
1422
|
+
const { keepsWriteRecords } = input;
|
|
1418
1423
|
const offloaded = workerCanWriteEverySegment(input) ? await runCheckpointOnWorker({
|
|
1419
1424
|
root: directory.root,
|
|
1420
1425
|
metadata,
|
|
1421
1426
|
targets,
|
|
1422
1427
|
compactionThreshold,
|
|
1423
1428
|
prior,
|
|
1424
|
-
capturedPartitions: [...capturedPartitions]
|
|
1429
|
+
capturedPartitions: [...capturedPartitions],
|
|
1430
|
+
keepsWriteRecords
|
|
1425
1431
|
}) : null;
|
|
1426
1432
|
if (offloaded !== null) {
|
|
1427
1433
|
return offloaded;
|
|
@@ -1434,7 +1440,8 @@ async function writeIndexCheckpointSegments(input) {
|
|
|
1434
1440
|
compactionThreshold,
|
|
1435
1441
|
prior,
|
|
1436
1442
|
wholePartitions,
|
|
1437
|
-
capturedPartitions
|
|
1443
|
+
capturedPartitions,
|
|
1444
|
+
keepsWriteRecords
|
|
1438
1445
|
});
|
|
1439
1446
|
}
|
|
1440
1447
|
|
|
@@ -1539,6 +1546,8 @@ async function writeOneBoundedCheckpoint(input) {
|
|
|
1539
1546
|
}
|
|
1540
1547
|
const vectorIndexes = input.hooks.getVectorIndexes(input.indexName);
|
|
1541
1548
|
const store = directoryCheckpointStore(input.directory);
|
|
1549
|
+
const writeRecords = input.hooks.writeRecords ?? null;
|
|
1550
|
+
const { indexName } = input;
|
|
1542
1551
|
const priorManifest = await readCurrentManifest(store, input.indexName);
|
|
1543
1552
|
const capture2 = await captureCheckpoint(
|
|
1544
1553
|
input.indexState,
|
|
@@ -1548,10 +1557,14 @@ async function writeOneBoundedCheckpoint(input) {
|
|
|
1548
1557
|
priorCheckpoint: priorManifest?.checkpoint ?? [],
|
|
1549
1558
|
priorPartitionIds: (priorManifest?.partitions ?? []).map((partition) => partition.partitionId),
|
|
1550
1559
|
everyPartition: input.fromMemory,
|
|
1551
|
-
aWorkerCanSerialise: input.canOffload && checkpointWorkerIsUsable()
|
|
1560
|
+
aWorkerCanSerialise: input.canOffload && checkpointWorkerIsUsable(),
|
|
1561
|
+
...writeRecords === null ? {} : {
|
|
1562
|
+
captureRecords: (partitionId) => (segments2) => writeRecords.captureSegments(indexName, partitionId, segments2)
|
|
1563
|
+
}
|
|
1552
1564
|
}),
|
|
1553
1565
|
input.fromMemory ? void 0 : priorManifest?.checkpoint ?? [],
|
|
1554
|
-
priorManifest?.vectors ?? []
|
|
1566
|
+
priorManifest?.vectors ?? [],
|
|
1567
|
+
writeRecords === null ? void 0 : (partitionId, lastSeqNo) => writeRecords.termHistoryOf(indexName, partitionId, lastSeqNo)
|
|
1555
1568
|
);
|
|
1556
1569
|
const { targets, documentCount } = capture2;
|
|
1557
1570
|
await makeEveryAppliedMutationDurable(input.indexState, input.failPartition);
|
|
@@ -1566,7 +1579,8 @@ async function writeOneBoundedCheckpoint(input) {
|
|
|
1566
1579
|
canOffload: input.canOffload,
|
|
1567
1580
|
prior: priorManifest,
|
|
1568
1581
|
wholePartitions: capture2.wholePartitions.contents,
|
|
1569
|
-
capturedPartitions: capture2.wholePartitions.captured
|
|
1582
|
+
capturedPartitions: capture2.wholePartitions.captured,
|
|
1583
|
+
keepsWriteRecords: writeRecords !== null
|
|
1570
1584
|
})
|
|
1571
1585
|
);
|
|
1572
1586
|
await makeEveryAppliedMutationDurable(input.indexState, input.failPartition);
|
|
@@ -2980,7 +2994,8 @@ async function recoverCheckpoint(deps) {
|
|
|
2980
2994
|
}
|
|
2981
2995
|
const checkpoint = await loadSegmentedCheckpoint(deps.store, deps.indexName, decoded.manifest, {
|
|
2982
2996
|
manager: deps.manager,
|
|
2983
|
-
vectorIndexes: deps.vectorIndexes
|
|
2997
|
+
vectorIndexes: deps.vectorIndexes,
|
|
2998
|
+
writeRecords: deps.writeRecords ?? null
|
|
2984
2999
|
});
|
|
2985
3000
|
return { checkpoint: await deps.replay(checkpoint), upgradedDocumentCount: null };
|
|
2986
3001
|
}
|
|
@@ -3020,15 +3035,23 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
|
|
|
3020
3035
|
if (manager === void 0 || current === void 0) {
|
|
3021
3036
|
return;
|
|
3022
3037
|
}
|
|
3038
|
+
const writeRecords = hooks.writeRecords ?? null;
|
|
3039
|
+
writeRecords?.dropIndex(indexName);
|
|
3023
3040
|
const replayDeps = {
|
|
3024
3041
|
manager,
|
|
3025
3042
|
vectorFieldPaths: hooks.getVectorFieldPaths(indexName),
|
|
3026
|
-
vectorIndexes: hooks.getVectorIndexes(indexName)
|
|
3043
|
+
vectorIndexes: hooks.getVectorIndexes(indexName),
|
|
3044
|
+
writeRecords
|
|
3027
3045
|
};
|
|
3028
3046
|
const replay = async (checkpoint) => {
|
|
3029
3047
|
const replayed = [];
|
|
3030
3048
|
for (let partitionId = 0; partitionId < manager.partitionCount; partitionId += 1) {
|
|
3031
3049
|
const fromSeqNo = snapshotCheckpointFor(checkpoint, partitionId);
|
|
3050
|
+
writeRecords?.loadTermHistory(
|
|
3051
|
+
indexName,
|
|
3052
|
+
partitionId,
|
|
3053
|
+
checkpoint.find((entry) => entry.partitionId === partitionId)
|
|
3054
|
+
);
|
|
3032
3055
|
const { highestSeqNo, highestPrimaryTerm } = await replayWal(
|
|
3033
3056
|
directory,
|
|
3034
3057
|
indexName,
|
|
@@ -3055,7 +3078,8 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
|
|
|
3055
3078
|
compactionThreshold: deps.compactionThreshold,
|
|
3056
3079
|
keepsUnlistedFiles: deps.holdsCheckpointFiles(indexName),
|
|
3057
3080
|
replay,
|
|
3058
|
-
onLayoutUpgrade: hooks.onLayoutUpgrade
|
|
3081
|
+
onLayoutUpgrade: hooks.onLayoutUpgrade,
|
|
3082
|
+
writeRecords
|
|
3059
3083
|
});
|
|
3060
3084
|
for (const { partitionId, lastSeqNo, primaryTerm } of recovered.checkpoint) {
|
|
3061
3085
|
deps.registerPartition(indexName, partitionId, lastSeqNo, primaryTerm);
|
|
@@ -3281,6 +3305,9 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
3281
3305
|
commitDeferral = defersCommit ?? null;
|
|
3282
3306
|
},
|
|
3283
3307
|
isolatePartitionFailures: partitionFailures.isolate,
|
|
3308
|
+
keepWriteRecordsWith(keeper) {
|
|
3309
|
+
hooks.writeRecords = keeper;
|
|
3310
|
+
},
|
|
3284
3311
|
heldCheckpoints: () => heldCheckpoints.report(indexes),
|
|
3285
3312
|
async commitPartition(indexName, partitionId) {
|
|
3286
3313
|
const partition = indexes.get(indexName)?.partitions.get(partitionId);
|
|
@@ -3324,6 +3351,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
3324
3351
|
}
|
|
3325
3352
|
},
|
|
3326
3353
|
async removeIndex(indexName) {
|
|
3354
|
+
hooks.writeRecords?.dropIndex(indexName);
|
|
3327
3355
|
const indexState = indexes.get(indexName);
|
|
3328
3356
|
if (indexState !== void 0) {
|
|
3329
3357
|
await drainIndexStateForUnload(indexState, metadataWrites.get(indexName), closeWriterReportingFailure);
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
|
|
2
|
-
import { VECTOR_WORKER_COPIES_ALLOWED, openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, osqBitsOf, noBuildHistory, createOsqQuantizer, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, ordinalFilterValues, handOverBuildTurns, threadsHoldCurrentLayout, filterForOptions, openGraphState, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, fixedView, OSQ4_MIN_DIMENSION, sharedVectorStoreBytes, STORE_SLOTS, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, yieldBuildTurn, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, WORKER_COPY_MIN_VECTORS, allLiveDocIds, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, slotByteOffset, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createSharedVectorStoreHandles, BLOCK_MAX_BYTES, growBufferTo, unplacedChunkCount, STORE_DOC_ID_BYTES, recalibrateFromStore, signatureOf, nativeStoreScratchBytes, noteFallback, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, QUERY_STAGED, createVectorBlock, ensureBlockSlots, createHNSWIndex, adoptGraph, topLayerInParts, BUILD_RETRY_MAX_DELAY_MS, BUILD_RETRY_FIRST_DELAY_MS, fieldHandlesOf, sharedMemoryAvailable, requireWholeNeighbourLists, shuffleInPlace, createCentroidSums, liveOrdinals, RECALIBRATION_CHUNK_VECTORS, BUILD_CHUNK_SIZE, centroidOfCentroidSums, meanOfCentroidSums, countOfCentroidSums, squaredLengthsOfCentroidSums, releaseLocksHeldBy, DRIFT_CHECK_GROWTH_FACTOR, DISK_ADOPTION_YIELD_INTERVAL, adoptLoadedCalibration, RECALIBRATION_DRIFT_RATIO } from './chunk-
|
|
3
|
-
export { releaseLocksHeldBy } from './chunk-
|
|
4
|
-
import { isDead, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors } from './chunk-
|
|
5
|
-
import { MAX_RENUMBERED_SEARCH_ATTEMPTS, DISK_STORAGE_BLOCK_BYTES, compareCodePoints, UNPLACED_FLAG_CHUNK_ORDINALS, MAX_VECTOR_ORDINALS, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, resolveWorkerCount, resolveWorkerEntry, detectRuntime, noteThreadSlotLive, noteThreadSlotGone, WORKER_YOUNG_GENERATION_MB, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_INSERT_TIMEOUT_MS } from './chunk-
|
|
6
|
-
import { NarsilError, ErrorCodes, isNarsilError } from './chunk-
|
|
2
|
+
import { VECTOR_WORKER_COPIES_ALLOWED, openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, osqBitsOf, noBuildHistory, createOsqQuantizer, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, ordinalFilterValues, handOverBuildTurns, threadsHoldCurrentLayout, filterForOptions, openGraphState, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, fixedView, OSQ4_MIN_DIMENSION, sharedVectorStoreBytes, STORE_SLOTS, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, yieldBuildTurn, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, WORKER_COPY_MIN_VECTORS, allLiveDocIds, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, slotByteOffset, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createSharedVectorStoreHandles, BLOCK_MAX_BYTES, growBufferTo, unplacedChunkCount, STORE_DOC_ID_BYTES, recalibrateFromStore, signatureOf, nativeStoreScratchBytes, noteFallback, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, QUERY_STAGED, createVectorBlock, ensureBlockSlots, createHNSWIndex, adoptGraph, topLayerInParts, BUILD_RETRY_MAX_DELAY_MS, BUILD_RETRY_FIRST_DELAY_MS, fieldHandlesOf, sharedMemoryAvailable, requireWholeNeighbourLists, shuffleInPlace, createCentroidSums, liveOrdinals, RECALIBRATION_CHUNK_VECTORS, BUILD_CHUNK_SIZE, centroidOfCentroidSums, meanOfCentroidSums, countOfCentroidSums, squaredLengthsOfCentroidSums, releaseLocksHeldBy, DRIFT_CHECK_GROWTH_FACTOR, DISK_ADOPTION_YIELD_INTERVAL, adoptLoadedCalibration, RECALIBRATION_DRIFT_RATIO } from './chunk-OVY4RPDS.mjs';
|
|
3
|
+
export { releaseLocksHeldBy } from './chunk-OVY4RPDS.mjs';
|
|
4
|
+
import { isDead, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors } from './chunk-P7C2S5JX.mjs';
|
|
5
|
+
import { MAX_RENUMBERED_SEARCH_ATTEMPTS, DISK_STORAGE_BLOCK_BYTES, compareCodePoints, UNPLACED_FLAG_CHUNK_ORDINALS, MAX_VECTOR_ORDINALS, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, resolveWorkerCount, resolveWorkerEntry, detectRuntime, noteThreadSlotLive, noteThreadSlotGone, WORKER_YOUNG_GENERATION_MB, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_INSERT_TIMEOUT_MS } from './chunk-2CBQ2PCC.mjs';
|
|
6
|
+
import { NarsilError, ErrorCodes, isNarsilError } from './chunk-LJS6PREA.mjs';
|
|
7
7
|
|
|
8
8
|
// src/core/heap.ts
|
|
9
9
|
function createMaxHeap(compare) {
|
|
@@ -18,7 +18,7 @@ function restoreDepsOf(core, host) {
|
|
|
18
18
|
function queueRestore(core, host, restoresInFlight, indexName, data) {
|
|
19
19
|
const previous = restoresInFlight.get(indexName) ?? Promise.resolve();
|
|
20
20
|
const restored = previous.then(async () => {
|
|
21
|
-
const { restoreFromSnapshot } = await import('./snapshot-
|
|
21
|
+
const { restoreFromSnapshot } = await import('./snapshot-ISZMBDCN.mjs');
|
|
22
22
|
await restoreFromSnapshot(indexName, data, restoreDepsOf(core, host));
|
|
23
23
|
});
|
|
24
24
|
const settled = restored.then(
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { MIN_RERANK_TIMEOUT_MS, MAX_RERANK_TIMEOUT_MS, MIN_RERANK_WINDOW, MAX_RERANK_WINDOW, DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS, DEFAULT_RERANK_WINDOW } from './chunk-JPTQZWJQ.mjs';
|
|
2
|
-
import { NarsilError, ErrorCodes, MIN_HYBRID_DEPTH, MAX_HYBRID_DEPTH, HYBRID_RESCORE_WINDOW, DEFAULT_LIMIT, DEFAULT_OFFSET, RESULT_WINDOW } from './chunk-
|
|
2
|
+
import { NarsilError, ErrorCodes, MIN_HYBRID_DEPTH, MAX_HYBRID_DEPTH, HYBRID_RESCORE_WINDOW, DEFAULT_LIMIT, DEFAULT_OFFSET, RESULT_WINDOW } from './chunk-LJS6PREA.mjs';
|
|
3
3
|
|
|
4
4
|
// src/search/hybrid-depth.ts
|
|
5
5
|
function fixedHybridDepthOf(hybrid) {
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import { vectorsPerFile } from './chunk-
|
|
2
|
-
import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-
|
|
3
|
-
import './chunk-
|
|
4
|
-
import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, prepareStoredSynonyms, rejectSnapshot, readHeapStatistics, readResidentMemory, extractVectorFieldsFromSchema, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, insertDocumentVectors, decodeIndexConfiguration, flattenSchema } from './chunk-
|
|
5
|
-
import { decodeVectorIndexParts, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors } from './chunk-
|
|
6
|
-
import { MAX_FILE_BYTES } from './chunk-
|
|
1
|
+
import { vectorsPerFile } from './chunk-OVY4RPDS.mjs';
|
|
2
|
+
import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-YRYBTRWB.mjs';
|
|
3
|
+
import './chunk-O5IKTA27.mjs';
|
|
4
|
+
import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, prepareStoredSynonyms, rejectSnapshot, readHeapStatistics, readResidentMemory, extractVectorFieldsFromSchema, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, insertDocumentVectors, decodeIndexConfiguration, flattenSchema } from './chunk-UR5MNN53.mjs';
|
|
5
|
+
import { decodeVectorIndexParts, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors } from './chunk-P7C2S5JX.mjs';
|
|
6
|
+
import { MAX_FILE_BYTES } from './chunk-2CBQ2PCC.mjs';
|
|
7
7
|
import './chunk-JPTQZWJQ.mjs';
|
|
8
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
8
|
+
import { NarsilError, ErrorCodes } from './chunk-LJS6PREA.mjs';
|
|
9
9
|
import { decode } from '@msgpack/msgpack';
|
|
10
10
|
|
|
11
11
|
// src/persistence/archive/memory-store.ts
|
|
@@ -79,12 +79,24 @@ function partitionVectorPlan(vectorIndex, docIds) {
|
|
|
79
79
|
}
|
|
80
80
|
function capture(input) {
|
|
81
81
|
const partitionIds = input.partitionId === null ? Array.from({ length: input.manager.partitionCount }, (_, partitionId) => partitionId) : [input.partitionId];
|
|
82
|
-
const
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
82
|
+
const { indexName } = input;
|
|
83
|
+
const writeRecords = input.writeRecords ?? null;
|
|
84
|
+
const targets = partitionIds.map((partitionId) => {
|
|
85
|
+
const target = {
|
|
86
|
+
partitionId,
|
|
87
|
+
lastSeqNo: input.appliedSeqNo(partitionId),
|
|
88
|
+
primaryTerm: SINGLE_NODE_PRIMARY_TERM
|
|
89
|
+
};
|
|
90
|
+
if (writeRecords !== null) target.termHistory = writeRecords.termHistoryOf(indexName, partitionId, target.lastSeqNo);
|
|
91
|
+
return target;
|
|
92
|
+
});
|
|
93
|
+
const contents = partitionIds.map(
|
|
94
|
+
(partitionId) => partitionContentsOf(
|
|
95
|
+
input.manager,
|
|
96
|
+
partitionId,
|
|
97
|
+
writeRecords === null ? void 0 : (segments) => writeRecords.captureSegments(indexName, partitionId, segments)
|
|
98
|
+
)
|
|
99
|
+
);
|
|
88
100
|
const plans = /* @__PURE__ */ new Map();
|
|
89
101
|
const partitionDocIds = input.partitionId === null ? null : [...input.manager.getPartition(input.partitionId).docIds()];
|
|
90
102
|
for (const [fieldPath, vectorIndex] of input.vectorIndexes) {
|
|
@@ -107,7 +119,8 @@ async function buildArchiveFromMemory(input) {
|
|
|
107
119
|
targets: captured.targets,
|
|
108
120
|
compactionThreshold: DEFAULT_COMPACTION_THRESHOLD,
|
|
109
121
|
prior: null,
|
|
110
|
-
wholePartitions: captured.contents
|
|
122
|
+
wholePartitions: captured.contents,
|
|
123
|
+
keepsWriteRecords: input.writeRecords != null
|
|
111
124
|
});
|
|
112
125
|
const vectors = await writeLiveVectors({ store, indexName: input.indexName, plans: captured.plans, priorVectors: [] });
|
|
113
126
|
const refs = vectors.refs;
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-
|
|
2
|
-
import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-
|
|
3
|
-
import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-
|
|
4
|
-
import './chunk-
|
|
1
|
+
import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-O5IKTA27.mjs';
|
|
2
|
+
import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-UR5MNN53.mjs';
|
|
3
|
+
import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-P7C2S5JX.mjs';
|
|
4
|
+
import './chunk-2CBQ2PCC.mjs';
|
|
5
5
|
import './chunk-JPTQZWJQ.mjs';
|
|
6
|
-
import { EXPORT_CHUNK_BYTES, NarsilError, ErrorCodes } from './chunk-
|
|
6
|
+
import { EXPORT_CHUNK_BYTES, NarsilError, ErrorCodes } from './chunk-LJS6PREA.mjs';
|
|
7
7
|
import { decode } from '@msgpack/msgpack';
|
|
8
8
|
|
|
9
9
|
// src/engine/export/json-line.ts
|