@delali/narsil 0.3.0 → 0.3.1-next.102
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -8
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/{adapters-79OIa4xm.d.ts → adapters-BQFQ4PRz.d.ts} +27 -1
- package/dist/browser/chunk-BDOSMEEP.mjs +493 -0
- package/dist/browser/chunk-EJDUFWSO.mjs +602 -0
- package/dist/browser/chunk-HOZW3PKP.mjs +9 -0
- package/dist/browser/chunk-JOVZ6HMT.mjs +4358 -0
- package/dist/browser/chunk-JPTQZWJQ.mjs +10 -0
- package/dist/browser/chunk-KPJMDPHG.mjs +24558 -0
- package/dist/browser/chunk-PGIM3HIM.mjs +2743 -0
- package/dist/browser/chunk-RHG4O77A.mjs +206 -0
- package/dist/browser/chunk-RIJSLY2I.mjs +360 -0
- package/dist/browser/durability-wiring-SRUYT3VV.mjs +3875 -0
- package/dist/browser/module-CBB2ETYP.mjs +3111 -0
- package/dist/browser/release-76MXRPGV.mjs +33 -0
- package/dist/browser/replace-RURXTGVG.mjs +57 -0
- package/dist/browser/restore-queue-EEZQO6RF.mjs +35 -0
- package/dist/browser/runtime-7ZAHCHWR.mjs +557 -0
- package/dist/browser/snapshot-OAAA5UJ7.mjs +1210 -0
- package/dist/browser/write-7D6VIK6O.mjs +208 -0
- package/dist/{capabilities-DSsOGz6F.d.ts → capabilities-ByQRMOx-.d.ts} +68 -7
- package/dist/chunk-2RIRGOUL.mjs +113 -0
- package/dist/chunk-37CD3OKS.mjs +87 -0
- package/dist/chunk-44W3CBNG.mjs +7 -0
- package/dist/chunk-46D6JQIG.mjs +426 -0
- package/dist/{chunk-VMRR3DQU.mjs → chunk-5JNA5SKS.mjs} +546 -35
- package/dist/chunk-747OHOUA.mjs +11 -0
- package/dist/chunk-75XIEF6H.mjs +126 -0
- package/dist/chunk-7X5JST5L.mjs +114 -0
- package/dist/chunk-BOFBLJFZ.mjs +209 -0
- package/dist/{chunk-USKFU7J5.mjs → chunk-BWJVNZPR.mjs} +8 -1
- package/dist/chunk-C7SLA6CM.mjs +19 -0
- package/dist/{chunk-POWOTUKJ.mjs → chunk-CQUMNIIG.mjs} +3 -6
- package/dist/chunk-CVXLLEQ3.mjs +1011 -0
- package/dist/chunk-DJHHF2L5.mjs +17 -0
- package/dist/chunk-E74YLAIG.mjs +24 -0
- package/dist/{chunk-UXNWQZST.mjs → chunk-EMRMGYRE.mjs} +236 -18
- package/dist/chunk-EWZLIYHV.mjs +57 -0
- package/dist/chunk-GBEEG5AN.mjs +35 -0
- package/dist/chunk-GES7U2PM.mjs +68 -0
- package/dist/chunk-HJSN7UYG.mjs +10 -0
- package/dist/chunk-HLPRHOSO.mjs +17 -0
- package/dist/chunk-HMFNFVP7.mjs +14 -0
- package/dist/chunk-J5QP52HA.mjs +1751 -0
- package/dist/{chunk-YNVIUGF5.mjs → chunk-JZVARCRZ.mjs} +3 -28
- package/dist/chunk-KOKM4PQN.mjs +52 -0
- package/dist/{chunk-B2TFROFX.mjs → chunk-KWTW6VTG.mjs} +465 -250
- package/dist/chunk-KZ3NX2P7.mjs +21 -0
- package/dist/chunk-L5ORED7V.mjs +25 -0
- package/dist/{chunk-THYRHHO2.mjs → chunk-LGAMKHA2.mjs} +2009 -2351
- package/dist/chunk-LKOB5COA.mjs +117 -0
- package/dist/chunk-LPQOEU5M.mjs +96 -0
- package/dist/{chunk-3LHGNRYE.mjs → chunk-LXWDQLTW.mjs} +7 -2
- package/dist/{chunk-L6TNPLAR.mjs → chunk-OEYI3KRB.mjs} +270 -108
- package/dist/chunk-OM362BRO.mjs +948 -0
- package/dist/chunk-OZDFA4WC.mjs +67 -0
- package/dist/chunk-P36MUFEJ.mjs +10 -0
- package/dist/chunk-PC2G76VW.mjs +2540 -0
- package/dist/{chunk-GCM3JFVA.mjs → chunk-PNEYIHA4.mjs} +11 -2
- package/dist/chunk-PNFKUCG3.mjs +1130 -0
- package/dist/chunk-Q6XWVUZX.mjs +44 -0
- package/dist/chunk-R27EMH5M.mjs +617 -0
- package/dist/{chunk-YMZCMH5M.mjs → chunk-R6A773WK.mjs} +19 -24
- package/dist/chunk-RF7PLG43.mjs +106 -0
- package/dist/chunk-RFBLQLLZ.mjs +16 -0
- package/dist/{chunk-Y3L5MDRV.mjs → chunk-RFVUDM5E.mjs} +17 -2
- package/dist/chunk-S7FL6UAH.mjs +521 -0
- package/dist/chunk-S7GNDRPS.mjs +651 -0
- package/dist/chunk-SP3NM3ZG.mjs +3537 -0
- package/dist/chunk-U476IXKQ.mjs +10 -0
- package/dist/chunk-UA3AUGNV.mjs +462 -0
- package/dist/chunk-UTG5YNXV.mjs +507 -0
- package/dist/chunk-VI4R6W25.mjs +8484 -0
- package/dist/chunk-WQBCBKY3.mjs +8 -0
- package/dist/{chunk-24RXHMEL.mjs → chunk-WVCOCKBU.mjs} +3794 -7570
- package/dist/chunk-X5CV2E2B.mjs +1 -0
- package/dist/chunk-XADZPPZI.mjs +614 -0
- package/dist/chunk-YEEQHT4V.mjs +212 -0
- package/dist/chunk-YTJCGKM4.mjs +56 -0
- package/dist/chunk-YYQIODMF.mjs +275 -0
- package/dist/chunk-ZI34TVOA.mjs +144 -0
- package/dist/{chunk-OOSIDHA3.mjs → chunk-ZKWYWW4Y.mjs} +1 -1
- package/dist/chunk-ZM7KU4YW.mjs +35 -0
- package/dist/{client-BQK6Tf6A.d.ts → client-BOStAkc5.d.ts} +160 -37
- package/dist/client.d.ts +11 -10
- package/dist/client.mjs +209 -45
- package/dist/{config-DmXULPa-.d.ts → config-Cjvzbs_j.d.ts} +51 -3
- package/dist/config-H6SGzbPH.d.ts +421 -0
- package/dist/direct-executor-ZD64PPEZ.mjs +24 -0
- package/dist/distribution/coordinator/etcd.d.ts +79 -18
- package/dist/distribution/coordinator/etcd.mjs +553 -109
- package/dist/distribution/coordinator/in-memory.d.ts +4 -3
- package/dist/distribution/coordinator/in-memory.mjs +22 -3
- package/dist/distribution/transport/grpc.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +79 -14
- package/dist/distribution/transport/in-memory.d.ts +2 -2
- package/dist/distribution/transport/in-memory.mjs +12 -7
- package/dist/distribution/transport/tcp.d.ts +2 -2
- package/dist/distribution/transport/tcp.mjs +50 -28
- package/dist/distribution.d.ts +144 -23
- package/dist/distribution.mjs +14479 -8173
- package/dist/durability-wiring-FEUBR5BT.mjs +3330 -0
- package/dist/embeddings/openai.d.ts +2 -2
- package/dist/embeddings/openai.mjs +22 -133
- package/dist/{engine-CIOyFozK.d.ts → engine-rC0vjjY-.d.ts} +165 -141
- package/dist/{memory-yFaVn2xZ.d.ts → events-DqOIg5xa.d.ts} +645 -136
- package/dist/{types-DAK7mnUZ.d.ts → index-Bkdt8nDw.d.ts} +182 -45
- package/dist/index.browser.mjs +4 -32180
- package/dist/index.d.ts +91 -20
- package/dist/index.mjs +33 -16
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/filesystem.mjs +2 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/module-3S65MVCM.mjs +2151 -0
- package/dist/partial-words-DTIHO8G4.d.ts +19 -0
- package/dist/partial-words.d.ts +22 -0
- package/dist/partial-words.mjs +17 -0
- package/dist/pattern-Cledccbl.d.ts +18 -0
- package/dist/pattern.d.ts +22 -0
- package/dist/pattern.mjs +12 -0
- package/dist/persistence/durability/checkpoint-worker.d.ts +119 -72
- package/dist/persistence/durability/checkpoint-worker.mjs +51 -18
- package/dist/react.d.ts +279 -183
- package/dist/react.mjs +201 -48
- package/dist/release-BH3DKW3T.mjs +33 -0
- package/dist/replace-CEO5KJJ3.mjs +57 -0
- package/dist/rerank-K7C4WZFG.mjs +60 -0
- package/dist/rerankers/http.d.ts +50 -0
- package/dist/rerankers/http.mjs +104 -0
- package/dist/restore-queue-KQHA7ZV2.mjs +35 -0
- package/dist/runtime-6Y4ZYXOG.mjs +177 -0
- package/dist/{schema-d5Afu03A.d.ts → schema-4W_7tk1f.d.ts} +81 -11
- package/dist/server.d.ts +13 -12
- package/dist/server.mjs +611 -80
- package/dist/snapshot-QGZPODM5.mjs +799 -0
- package/dist/synonyms-CpOAHPa-.d.ts +52 -0
- package/dist/synonyms.d.ts +22 -0
- package/dist/synonyms.mjs +12 -0
- package/dist/{tasks-CH2Dod8w.d.ts → tasks-CwPTz-tP.d.ts} +10 -3
- package/dist/thread-N3IUGUS3.mjs +697 -0
- package/dist/{types-BQDlIvfS.d.ts → types-2qoitTXg.d.ts} +1 -1
- package/dist/{types-D5OwhtsQ.d.ts → types-Cz5okLml.d.ts} +2 -2
- package/dist/vector/search-worker.d.ts +46 -4
- package/dist/vector/search-worker.mjs +21 -11
- package/dist/workers/entry.mjs +9 -6
- package/dist/write-U5RI7MAX.mjs +241 -0
- package/package.json +55 -9
- package/dist/chunk-3L4U6EJQ.mjs +0 -49
- package/dist/chunk-IFBKLBGB.mjs +0 -7
- package/dist/chunk-ORBLD3EL.mjs +0 -6
- package/dist/chunk-U7KYW4BA.mjs +0 -135
- package/dist/chunk-ZJ4CSAB2.mjs +0 -13
- package/dist/chunk-ZNZBXMP5.mjs +0 -2117
- package/dist/chunk-ZZT3EAUC.mjs +0 -11168
- package/dist/config-BhldZgaK.d.ts +0 -194
- package/dist/direct-executor-ALAVE7FC.mjs +0 -12
- package/dist/serialization/crc32-worker.d.ts +0 -20
- package/dist/serialization/crc32-worker.mjs +0 -62
- package/dist/simd-DW5AtJ6B.d.ts +0 -41
- package/dist/thread-DYHEQRVY.mjs +0 -402
|
@@ -0,0 +1,1751 @@
|
|
|
1
|
+
import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes } from './chunk-XADZPPZI.mjs';
|
|
2
|
+
import { WIRE_BATCH_BUDGET } from './chunk-P36MUFEJ.mjs';
|
|
3
|
+
import { ReplicationMessageTypes } from './chunk-PNEYIHA4.mjs';
|
|
4
|
+
import { packChecksummedEnvelope, crc32, openChecksummedEnvelope, reconstructSchemaFromMetadata, HEADER_SIZE, prepareDocumentVectors } from './chunk-WVCOCKBU.mjs';
|
|
5
|
+
import { freezeSegmentShared, createSharedFrozenSegment, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, selectionKeeps, decodeMapWithoutFields, shareSegmentArrays, segmentArraysOf, mergeSegmentContents } from './chunk-SP3NM3ZG.mjs';
|
|
6
|
+
import { fnv1a, isCompositePartition, survivorDocumentsOf, partitionStateOf } from './chunk-VI4R6W25.mjs';
|
|
7
|
+
import { generateId, getLanguage } from './chunk-ZM7KU4YW.mjs';
|
|
8
|
+
import { extractVectorFieldsFromSchema, partialFieldsOf, messagePackUpperBoundBytes } from './chunk-OEYI3KRB.mjs';
|
|
9
|
+
import { MAX_FILE_BYTES } from './chunk-U476IXKQ.mjs';
|
|
10
|
+
import { resolveIndexAnalysis } from './chunk-46D6JQIG.mjs';
|
|
11
|
+
import { NarsilError, ErrorCodes } from './chunk-BWJVNZPR.mjs';
|
|
12
|
+
import { encode, decode } from '@msgpack/msgpack';
|
|
13
|
+
|
|
14
|
+
// src/persistence/durability/constants.ts
|
|
15
|
+
var DEFAULT_CHECKPOINT_INTERVAL_MS = 3e5;
|
|
16
|
+
var DEFAULT_CHECKPOINT_MUTATION_THRESHOLD = 1e5;
|
|
17
|
+
var DEFAULT_ASYNC_FLUSH_INTERVAL_MS = 1e3;
|
|
18
|
+
var DIRECTORY_IDENTITY_CHECK_INTERVAL_MS = 1e3;
|
|
19
|
+
var DIRECTORY_LOCK_ATTEMPTS = 3;
|
|
20
|
+
var DIRECTORY_LOCK_SETTLE_MS = 50;
|
|
21
|
+
var DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS = 1e3;
|
|
22
|
+
var DEFAULT_SEGMENT_MAX_BYTES = 67108864;
|
|
23
|
+
var DEFAULT_COMPACTION_THRESHOLD = 12;
|
|
24
|
+
var COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT = 2e5;
|
|
25
|
+
var SEGMENT_FILE_FRAMING_SLACK_BYTES = 65536;
|
|
26
|
+
var SEGMENT_TOKEN_TABLE_SHARE_OF_FILE = 0.25;
|
|
27
|
+
var WHOLE_PARTITION_MIN_CHANGED_SHARE = 0.5;
|
|
28
|
+
var WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT = 1e5;
|
|
29
|
+
var LOG_RECORDS_PER_CHECKPOINT = 1e5;
|
|
30
|
+
var WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES = 2147483648;
|
|
31
|
+
var CHECKPOINT_WORKER_TIMEOUT_MS = 6e5;
|
|
32
|
+
var CHECKPOINT_WORKER_HEARTBEAT_MS = 5e3;
|
|
33
|
+
var CHECKPOINT_WORKER_IDLE_MS = 5e3;
|
|
34
|
+
var CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS = 50;
|
|
35
|
+
|
|
36
|
+
// src/persistence/durability/segment/layout.ts
|
|
37
|
+
var SEGMENT_ID_WIDTH = 16;
|
|
38
|
+
function manifestKey(indexName) {
|
|
39
|
+
return `${indexName}/manifest`;
|
|
40
|
+
}
|
|
41
|
+
function snapshotBundleKey(indexName) {
|
|
42
|
+
return `${indexName}/snapshot`;
|
|
43
|
+
}
|
|
44
|
+
function segmentPrefix(indexName, partitionId) {
|
|
45
|
+
return `${indexName}/segments/${partitionId}/`;
|
|
46
|
+
}
|
|
47
|
+
function segmentKey(indexName, partitionId, segmentId) {
|
|
48
|
+
return `${segmentPrefix(indexName, partitionId)}s${formatSegmentId(segmentId)}`;
|
|
49
|
+
}
|
|
50
|
+
function segmentsPrefix(indexName) {
|
|
51
|
+
return `${indexName}/segments/`;
|
|
52
|
+
}
|
|
53
|
+
function vectorFileKey(indexName, fieldPath, fileId) {
|
|
54
|
+
return `${segmentsPrefix(indexName)}vec-${encodeFieldPath(fieldPath)}-f${formatSegmentId(fileId)}`;
|
|
55
|
+
}
|
|
56
|
+
function requireGeneration(generation) {
|
|
57
|
+
if (Number.isSafeInteger(generation) && generation > 0) return;
|
|
58
|
+
throw new NarsilError(
|
|
59
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
60
|
+
`Vector graph generation ${generation} must be a positive integer`,
|
|
61
|
+
{ generation }
|
|
62
|
+
);
|
|
63
|
+
}
|
|
64
|
+
function graphPartKey(indexName, fieldPath, generation, part) {
|
|
65
|
+
requireGeneration(generation);
|
|
66
|
+
return `${segmentsPrefix(indexName)}vec-${encodeFieldPath(fieldPath)}-graph-g${generation}-p${formatSegmentId(part)}`;
|
|
67
|
+
}
|
|
68
|
+
function formatSegmentId(segmentId) {
|
|
69
|
+
if (!Number.isInteger(segmentId) || segmentId < 0) {
|
|
70
|
+
throw new NarsilError(
|
|
71
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
72
|
+
`Segment id ${segmentId} must be a non-negative integer`,
|
|
73
|
+
{ segmentId }
|
|
74
|
+
);
|
|
75
|
+
}
|
|
76
|
+
return segmentId.toString().padStart(SEGMENT_ID_WIDTH, "0");
|
|
77
|
+
}
|
|
78
|
+
var FIELD_PATH_PATTERN = /^[A-Za-z0-9_.]+$/;
|
|
79
|
+
function encodeFieldPath(fieldPath) {
|
|
80
|
+
if (!FIELD_PATH_PATTERN.test(fieldPath)) {
|
|
81
|
+
throw new NarsilError(
|
|
82
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
83
|
+
`Vector field path "${fieldPath}" contains characters that cannot be encoded in a segment key`,
|
|
84
|
+
{ fieldPath }
|
|
85
|
+
);
|
|
86
|
+
}
|
|
87
|
+
const fingerprint = fnv1a(fieldPath).toString(36);
|
|
88
|
+
return `${fieldPath.replace(/\./g, "_")}-${fingerprint}`;
|
|
89
|
+
}
|
|
90
|
+
var SEGMENT_MANIFEST_VERSION = 7;
|
|
91
|
+
var EARLIER_SEGMENT_MANIFEST_VERSION = 6;
|
|
92
|
+
var MANIFEST_ENVELOPE_VERSION = 2;
|
|
93
|
+
var MAX_SEGMENTS_PER_PARTITION = 65536;
|
|
94
|
+
var MAX_SEGMENT_ID = 281474976710655;
|
|
95
|
+
function fileRefOnTheWire(ref) {
|
|
96
|
+
return { key: ref.key, bytes: ref.bytes, checksum: ref.checksum };
|
|
97
|
+
}
|
|
98
|
+
function encodeSegmentManifest(manifest) {
|
|
99
|
+
const payload = encode({
|
|
100
|
+
version: SEGMENT_MANIFEST_VERSION,
|
|
101
|
+
schema: manifest.schema,
|
|
102
|
+
language: manifest.language,
|
|
103
|
+
checkpoint: manifest.checkpoint.map((c) => ({
|
|
104
|
+
partitionId: c.partitionId,
|
|
105
|
+
lastSeqNo: c.lastSeqNo,
|
|
106
|
+
primaryTerm: c.primaryTerm
|
|
107
|
+
})),
|
|
108
|
+
partitions: manifest.partitions.map((p) => ({
|
|
109
|
+
partitionId: p.partitionId,
|
|
110
|
+
nextSegmentId: p.nextSegmentId,
|
|
111
|
+
segments: p.segments.map((s) => ({
|
|
112
|
+
id: s.id,
|
|
113
|
+
key: s.key,
|
|
114
|
+
docCount: s.docCount,
|
|
115
|
+
tombstoneCount: s.tombstoneCount,
|
|
116
|
+
bytes: s.bytes,
|
|
117
|
+
checksum: s.checksum
|
|
118
|
+
}))
|
|
119
|
+
})),
|
|
120
|
+
vectors: manifest.vectors.map((v) => ({
|
|
121
|
+
fieldPath: v.fieldPath,
|
|
122
|
+
nextFileId: v.nextFileId,
|
|
123
|
+
files: v.files.map((f) => ({ id: f.id, ...fileRefOnTheWire(f), count: f.count, dead: f.dead })),
|
|
124
|
+
graphGeneration: v.graphGeneration,
|
|
125
|
+
graphParts: v.graphParts.map(fileRefOnTheWire)
|
|
126
|
+
}))
|
|
127
|
+
});
|
|
128
|
+
return packChecksummedEnvelope([payload], MANIFEST_ENVELOPE_VERSION, "segment manifest");
|
|
129
|
+
}
|
|
130
|
+
function manifestReferencedKeys(manifest) {
|
|
131
|
+
const keys = /* @__PURE__ */ new Set();
|
|
132
|
+
for (const partition of manifest.partitions) {
|
|
133
|
+
for (const segment of partition.segments) {
|
|
134
|
+
keys.add(segment.key);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
for (const vector of manifest.vectors) {
|
|
138
|
+
for (const file of vector.files) keys.add(file.key);
|
|
139
|
+
for (const part of vector.graphParts) keys.add(part.key);
|
|
140
|
+
}
|
|
141
|
+
return keys;
|
|
142
|
+
}
|
|
143
|
+
function manifestFileRefs(manifest) {
|
|
144
|
+
const refs = [];
|
|
145
|
+
for (const partition of manifest.partitions) refs.push(...partition.segments);
|
|
146
|
+
for (const vector of manifest.vectors) refs.push(...vector.files, ...vector.graphParts);
|
|
147
|
+
return refs;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
// src/persistence/durability/segment/store.ts
|
|
151
|
+
function directoryCheckpointStore(directory) {
|
|
152
|
+
const store = {
|
|
153
|
+
read: (key) => directory.read(key),
|
|
154
|
+
write: (key, chunks) => directory.atomicWrite(key, chunks),
|
|
155
|
+
remove: (key) => directory.remove(key),
|
|
156
|
+
list: (prefix) => directory.list(prefix),
|
|
157
|
+
pathOf: (key) => directory.pathOf(key)
|
|
158
|
+
};
|
|
159
|
+
const byteLength = directory.byteLength;
|
|
160
|
+
if (byteLength !== void 0) store.byteLength = (key) => byteLength.call(directory, key);
|
|
161
|
+
return store;
|
|
162
|
+
}
|
|
163
|
+
function joined(chunks) {
|
|
164
|
+
if (chunks.length === 1) return chunks[0];
|
|
165
|
+
let total = 0;
|
|
166
|
+
for (const chunk of chunks) total += chunk.byteLength;
|
|
167
|
+
const bytes = new Uint8Array(total);
|
|
168
|
+
let offset = 0;
|
|
169
|
+
for (const chunk of chunks) {
|
|
170
|
+
bytes.set(chunk, offset);
|
|
171
|
+
offset += chunk.byteLength;
|
|
172
|
+
}
|
|
173
|
+
return bytes;
|
|
174
|
+
}
|
|
175
|
+
function adapterFailure(code, verb, key, error) {
|
|
176
|
+
if (error instanceof NarsilError) return error;
|
|
177
|
+
const cause = error instanceof Error ? error.message : String(error);
|
|
178
|
+
return new NarsilError(code, `The persistence adapter failed to ${verb} "${key}": ${cause}`, { key, cause });
|
|
179
|
+
}
|
|
180
|
+
async function guarded(run, code, verb, key) {
|
|
181
|
+
try {
|
|
182
|
+
return await run();
|
|
183
|
+
} catch (error) {
|
|
184
|
+
throw adapterFailure(code, verb, key, error);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
function adapterCheckpointStore(adapter) {
|
|
188
|
+
return {
|
|
189
|
+
read: (key) => guarded(() => adapter.load(key), ErrorCodes.PERSISTENCE_LOAD_FAILED, "load", key),
|
|
190
|
+
write: (key, chunks) => guarded(() => adapter.save(key, joined(chunks)), ErrorCodes.PERSISTENCE_SAVE_FAILED, "save", key),
|
|
191
|
+
remove: (key) => guarded(() => adapter.delete(key), ErrorCodes.PERSISTENCE_DELETE_FAILED, "delete", key),
|
|
192
|
+
list: (prefix) => guarded(() => adapter.list(prefix), ErrorCodes.PERSISTENCE_LOAD_FAILED, "list", prefix)
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
async function readListedFile(store, key, expected) {
|
|
196
|
+
const length = store.byteLength === void 0 ? void 0 : await store.byteLength(key);
|
|
197
|
+
if (length === null) missingFile(expected.what, key);
|
|
198
|
+
if (length !== void 0) requireListedLength(expected.what, key, length, expected.bytes);
|
|
199
|
+
const bytes = await store.read(key);
|
|
200
|
+
if (bytes === null) missingFile(expected.what, key);
|
|
201
|
+
requireListedLength(expected.what, key, bytes.byteLength, expected.bytes);
|
|
202
|
+
return bytes;
|
|
203
|
+
}
|
|
204
|
+
function missingFile(what, key) {
|
|
205
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The ${what} "${key}" that the manifest lists is missing`, {
|
|
206
|
+
key
|
|
207
|
+
});
|
|
208
|
+
}
|
|
209
|
+
function requireListedLength(what, key, found, listed) {
|
|
210
|
+
if (found === listed) return;
|
|
211
|
+
throw new NarsilError(
|
|
212
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
213
|
+
`The ${what} "${key}" holds ${found} bytes where the manifest lists ${listed}`,
|
|
214
|
+
{ key, bytes: found, listed }
|
|
215
|
+
);
|
|
216
|
+
}
|
|
217
|
+
function requireListedChecksum(what, key, found, listed) {
|
|
218
|
+
if (found === listed) return;
|
|
219
|
+
throw new NarsilError(
|
|
220
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
221
|
+
`The ${what} "${key}" carries checksum ${found} where the manifest lists ${listed}`,
|
|
222
|
+
{ key, checksum: found, listed }
|
|
223
|
+
);
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
// src/persistence/durability/segment/captured-partition.ts
|
|
227
|
+
function keepMaskOf(segment) {
|
|
228
|
+
const keep = new Uint8Array(segment.arrays.documentCount);
|
|
229
|
+
for (let ordinal = 0; ordinal < keep.length; ordinal++) keep[ordinal] = segment.isTombstoned(ordinal) ? 0 : 1;
|
|
230
|
+
return keep;
|
|
231
|
+
}
|
|
232
|
+
function selectionOfMask(segment, keep) {
|
|
233
|
+
return { segment, keep: (ordinal) => keep[ordinal] === 1 };
|
|
234
|
+
}
|
|
235
|
+
function frozenLiveTail(live, shared) {
|
|
236
|
+
if (live.count() === 0) return null;
|
|
237
|
+
const payload = live.encodeSegment();
|
|
238
|
+
const documents = survivorDocumentsOf(partitionStateOf(live), payload);
|
|
239
|
+
if (shared) {
|
|
240
|
+
const snapshot = freezeSegmentShared(payload, documents);
|
|
241
|
+
if (snapshot !== null) return createSharedFrozenSegment(snapshot);
|
|
242
|
+
}
|
|
243
|
+
return createFrozenSegment(payload, documents);
|
|
244
|
+
}
|
|
245
|
+
function frozenSegmentsOf(partition, shared) {
|
|
246
|
+
const segments = [];
|
|
247
|
+
const live = isCompositePartition(partition) ? partition.live : partition;
|
|
248
|
+
if (isCompositePartition(partition)) {
|
|
249
|
+
const ids = partition.frozenSegmentSizes().map((size) => size.segmentId);
|
|
250
|
+
segments.push(...partition.frozenSegmentsById(ids));
|
|
251
|
+
}
|
|
252
|
+
const tail = frozenLiveTail(live, shared);
|
|
253
|
+
if (tail !== null) segments.push(tail);
|
|
254
|
+
return segments;
|
|
255
|
+
}
|
|
256
|
+
function partitionContentsOf(manager, partitionId) {
|
|
257
|
+
const partition = manager.getPartition(partitionId);
|
|
258
|
+
const selections = frozenSegmentsOf(partition, false).map((segment) => selectionOfMask(segment, keepMaskOf(segment)));
|
|
259
|
+
return { partitionId, docCount: partition.count(), selections };
|
|
260
|
+
}
|
|
261
|
+
function captureWholePartition(manager, partitionId) {
|
|
262
|
+
if (typeof SharedArrayBuffer !== "function") return null;
|
|
263
|
+
const partition = manager.getPartition(partitionId);
|
|
264
|
+
if (!isCompositePartition(partition) || partition.frozenSegmentCount() === 0) return null;
|
|
265
|
+
const segments = [];
|
|
266
|
+
for (const segment of frozenSegmentsOf(partition, true)) {
|
|
267
|
+
if (segment.sharedSnapshot === null) return null;
|
|
268
|
+
segments.push({ snapshot: segment.sharedSnapshot, keep: keepMaskOf(segment) });
|
|
269
|
+
}
|
|
270
|
+
return { partitionId, docCount: partition.count(), segments };
|
|
271
|
+
}
|
|
272
|
+
function contentsOfCaptured(captured) {
|
|
273
|
+
return {
|
|
274
|
+
partitionId: captured.partitionId,
|
|
275
|
+
docCount: captured.docCount,
|
|
276
|
+
selections: captured.segments.map(
|
|
277
|
+
({ snapshot, keep }) => selectionOfMask(createSharedFrozenSegment(snapshot), keep)
|
|
278
|
+
)
|
|
279
|
+
};
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
// src/persistence/durability/partition-checkpoint.ts
|
|
283
|
+
function checkpointLastSeqNo(checkpoint, partitionId) {
|
|
284
|
+
for (const c of checkpoint) {
|
|
285
|
+
if (c.partitionId === partitionId) {
|
|
286
|
+
return c.lastSeqNo;
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
return 0;
|
|
290
|
+
}
|
|
291
|
+
function checkpointPrimaryTerm(checkpoint, partitionId) {
|
|
292
|
+
for (const c of checkpoint) {
|
|
293
|
+
if (c.partitionId === partitionId) {
|
|
294
|
+
return c.primaryTerm;
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
return 0;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
// src/persistence/durability/commit-marker.ts
|
|
301
|
+
var COMMIT_MARKER_SLOT_SIZE = 36;
|
|
302
|
+
var COMMIT_MARKER_SLOT_COUNT = 2;
|
|
303
|
+
var CRC_OFFSET = 32;
|
|
304
|
+
function encodeMarkerSlot(state) {
|
|
305
|
+
const slot = new Uint8Array(COMMIT_MARKER_SLOT_SIZE);
|
|
306
|
+
const view = new DataView(slot.buffer);
|
|
307
|
+
view.setBigUint64(0, BigInt(state.writeSeq), false);
|
|
308
|
+
view.setBigUint64(8, BigInt(state.activeSegmentSeqNo), false);
|
|
309
|
+
view.setBigUint64(16, BigInt(state.durableByteLength), false);
|
|
310
|
+
view.setBigUint64(24, BigInt(state.highestDurableSeqNo), false);
|
|
311
|
+
view.setUint32(CRC_OFFSET, crc32(slot.subarray(0, CRC_OFFSET)), false);
|
|
312
|
+
return slot;
|
|
313
|
+
}
|
|
314
|
+
function decodeSlot(data, slotIndex) {
|
|
315
|
+
const base = slotIndex * COMMIT_MARKER_SLOT_SIZE;
|
|
316
|
+
if (base + COMMIT_MARKER_SLOT_SIZE > data.length) {
|
|
317
|
+
return null;
|
|
318
|
+
}
|
|
319
|
+
const slot = data.subarray(base, base + COMMIT_MARKER_SLOT_SIZE);
|
|
320
|
+
const view = new DataView(slot.buffer, slot.byteOffset, slot.byteLength);
|
|
321
|
+
const storedCrc = view.getUint32(CRC_OFFSET, false);
|
|
322
|
+
if (crc32(slot.subarray(0, CRC_OFFSET)) !== storedCrc) {
|
|
323
|
+
return null;
|
|
324
|
+
}
|
|
325
|
+
const writeSeq = view.getBigUint64(0, false);
|
|
326
|
+
const activeSegmentSeqNo = view.getBigUint64(8, false);
|
|
327
|
+
const durableByteLength = view.getBigUint64(16, false);
|
|
328
|
+
const highestDurableSeqNo = view.getBigUint64(24, false);
|
|
329
|
+
if (writeSeq > BigInt(Number.MAX_SAFE_INTEGER) || activeSegmentSeqNo > BigInt(Number.MAX_SAFE_INTEGER) || durableByteLength > BigInt(Number.MAX_SAFE_INTEGER) || highestDurableSeqNo > BigInt(Number.MAX_SAFE_INTEGER)) {
|
|
330
|
+
return null;
|
|
331
|
+
}
|
|
332
|
+
return {
|
|
333
|
+
writeSeq: Number(writeSeq),
|
|
334
|
+
activeSegmentSeqNo: Number(activeSegmentSeqNo),
|
|
335
|
+
durableByteLength: Number(durableByteLength),
|
|
336
|
+
highestDurableSeqNo: Number(highestDurableSeqNo)
|
|
337
|
+
};
|
|
338
|
+
}
|
|
339
|
+
function readCommitMarker(data) {
|
|
340
|
+
let best = null;
|
|
341
|
+
for (let slotIndex = 0; slotIndex < COMMIT_MARKER_SLOT_COUNT; slotIndex += 1) {
|
|
342
|
+
const state = decodeSlot(data, slotIndex);
|
|
343
|
+
if (state === null) {
|
|
344
|
+
continue;
|
|
345
|
+
}
|
|
346
|
+
if (best === null || state.writeSeq > best.state.writeSeq) {
|
|
347
|
+
best = { state, slotIndex };
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
return best;
|
|
351
|
+
}
|
|
352
|
+
function nextMarkerSlot(lastWrittenSlot) {
|
|
353
|
+
if (lastWrittenSlot === null) {
|
|
354
|
+
return 0;
|
|
355
|
+
}
|
|
356
|
+
return (lastWrittenSlot + 1) % COMMIT_MARKER_SLOT_COUNT;
|
|
357
|
+
}
|
|
358
|
+
function markerSlotOffset(slotIndex) {
|
|
359
|
+
return slotIndex * COMMIT_MARKER_SLOT_SIZE;
|
|
360
|
+
}
|
|
361
|
+
function createEntryMessage(entry, sourceId, indexUuid) {
|
|
362
|
+
const payload = indexUuid === void 0 ? { entry } : { entry, indexUuid };
|
|
363
|
+
return {
|
|
364
|
+
type: ReplicationMessageTypes.ENTRY,
|
|
365
|
+
sourceId,
|
|
366
|
+
requestId: generateId(),
|
|
367
|
+
payload: encode(payload)
|
|
368
|
+
};
|
|
369
|
+
}
|
|
370
|
+
function createEntryBatchMessage(entries, sourceId, indexUuid) {
|
|
371
|
+
const payload = indexUuid === void 0 ? { entries } : { entries, indexUuid };
|
|
372
|
+
return {
|
|
373
|
+
type: ReplicationMessageTypes.ENTRY_BATCH,
|
|
374
|
+
sourceId,
|
|
375
|
+
requestId: generateId(),
|
|
376
|
+
payload: encode(payload)
|
|
377
|
+
};
|
|
378
|
+
}
|
|
379
|
+
function createAckMessage(seqNo, partitionId, indexName, sourceId, requestId) {
|
|
380
|
+
const payload = { seqNo, partitionId, indexName };
|
|
381
|
+
return {
|
|
382
|
+
type: ReplicationMessageTypes.ACK,
|
|
383
|
+
sourceId,
|
|
384
|
+
requestId,
|
|
385
|
+
payload: encode(payload)
|
|
386
|
+
};
|
|
387
|
+
}
|
|
388
|
+
function createForwardMessage(payload, sourceId) {
|
|
389
|
+
return {
|
|
390
|
+
type: ReplicationMessageTypes.FORWARD,
|
|
391
|
+
sourceId,
|
|
392
|
+
requestId: generateId(),
|
|
393
|
+
payload: encode(payload)
|
|
394
|
+
};
|
|
395
|
+
}
|
|
396
|
+
var MAX_FORWARD_BATCH_OPERATIONS = WIRE_BATCH_BUDGET.maxCount;
|
|
397
|
+
function createForwardBatchMessage(payload, sourceId) {
|
|
398
|
+
return {
|
|
399
|
+
type: ReplicationMessageTypes.FORWARD_BATCH,
|
|
400
|
+
sourceId,
|
|
401
|
+
requestId: generateId(),
|
|
402
|
+
payload: encode(payload)
|
|
403
|
+
};
|
|
404
|
+
}
|
|
405
|
+
function createForwardBatchResultMessage(payload, sourceId, requestId) {
|
|
406
|
+
return {
|
|
407
|
+
type: ReplicationMessageTypes.FORWARD_BATCH,
|
|
408
|
+
sourceId,
|
|
409
|
+
requestId,
|
|
410
|
+
payload: encode(payload)
|
|
411
|
+
};
|
|
412
|
+
}
|
|
413
|
+
function createInsyncAddMessage(payload, sourceId) {
|
|
414
|
+
return {
|
|
415
|
+
type: ReplicationMessageTypes.INSYNC_ADD,
|
|
416
|
+
sourceId,
|
|
417
|
+
requestId: generateId(),
|
|
418
|
+
payload: encode(payload)
|
|
419
|
+
};
|
|
420
|
+
}
|
|
421
|
+
function createInsyncRemoveMessage(payload, sourceId) {
|
|
422
|
+
return {
|
|
423
|
+
type: ReplicationMessageTypes.INSYNC_REMOVE,
|
|
424
|
+
sourceId,
|
|
425
|
+
requestId: generateId(),
|
|
426
|
+
payload: encode(payload)
|
|
427
|
+
};
|
|
428
|
+
}
|
|
429
|
+
function createPrimaryResignMessage(payload, sourceId) {
|
|
430
|
+
return {
|
|
431
|
+
type: ReplicationMessageTypes.PRIMARY_RESIGN,
|
|
432
|
+
sourceId,
|
|
433
|
+
requestId: generateId(),
|
|
434
|
+
payload: encode(payload)
|
|
435
|
+
};
|
|
436
|
+
}
|
|
437
|
+
function createInsyncConfirmMessage(payload, sourceId, requestId) {
|
|
438
|
+
return {
|
|
439
|
+
type: ReplicationMessageTypes.INSYNC_CONFIRM,
|
|
440
|
+
sourceId,
|
|
441
|
+
requestId,
|
|
442
|
+
payload: encode(payload)
|
|
443
|
+
};
|
|
444
|
+
}
|
|
445
|
+
function isRecord(value) {
|
|
446
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
447
|
+
}
|
|
448
|
+
function optionalIndexUuid(value, payloadName) {
|
|
449
|
+
if (value === void 0 || value === null) {
|
|
450
|
+
return void 0;
|
|
451
|
+
}
|
|
452
|
+
if (typeof value !== "string" || value.length === 0) {
|
|
453
|
+
throw new Error(`Invalid ${payloadName}: "indexUuid" must be a non-empty string or absent`);
|
|
454
|
+
}
|
|
455
|
+
return value;
|
|
456
|
+
}
|
|
457
|
+
function isPlainObject(value) {
|
|
458
|
+
if (!isRecord(value)) {
|
|
459
|
+
return false;
|
|
460
|
+
}
|
|
461
|
+
const prototype = Object.getPrototypeOf(value);
|
|
462
|
+
return prototype === Object.prototype || prototype === null;
|
|
463
|
+
}
|
|
464
|
+
function validateEntryPayload(decoded) {
|
|
465
|
+
if (!isRecord(decoded) || !isRecord(decoded.entry)) {
|
|
466
|
+
throw new Error('Invalid EntryPayload: missing or invalid "entry" field');
|
|
467
|
+
}
|
|
468
|
+
const entry = decoded.entry;
|
|
469
|
+
if (typeof entry.seqNo !== "number") {
|
|
470
|
+
throw new Error('Invalid EntryPayload: "entry.seqNo" must be a number');
|
|
471
|
+
}
|
|
472
|
+
if (typeof entry.primaryTerm !== "number") {
|
|
473
|
+
throw new Error('Invalid EntryPayload: "entry.primaryTerm" must be a number');
|
|
474
|
+
}
|
|
475
|
+
if (entry.operation !== "INDEX" && entry.operation !== "DELETE") {
|
|
476
|
+
throw new Error('Invalid EntryPayload: "entry.operation" must be "INDEX" or "DELETE"');
|
|
477
|
+
}
|
|
478
|
+
if (typeof entry.partitionId !== "number") {
|
|
479
|
+
throw new Error('Invalid EntryPayload: "entry.partitionId" must be a number');
|
|
480
|
+
}
|
|
481
|
+
if (typeof entry.indexName !== "string") {
|
|
482
|
+
throw new Error('Invalid EntryPayload: "entry.indexName" must be a string');
|
|
483
|
+
}
|
|
484
|
+
if (typeof entry.documentId !== "string") {
|
|
485
|
+
throw new Error('Invalid EntryPayload: "entry.documentId" must be a string');
|
|
486
|
+
}
|
|
487
|
+
if (entry.document !== null && !(entry.document instanceof Uint8Array)) {
|
|
488
|
+
throw new Error('Invalid EntryPayload: "entry.document" must be Uint8Array or null');
|
|
489
|
+
}
|
|
490
|
+
if (typeof entry.checksum !== "number") {
|
|
491
|
+
throw new Error('Invalid EntryPayload: "entry.checksum" must be a number');
|
|
492
|
+
}
|
|
493
|
+
optionalIndexUuid(decoded.indexUuid, "EntryPayload");
|
|
494
|
+
return decoded;
|
|
495
|
+
}
|
|
496
|
+
function validateEntryBatchPayload(decoded) {
|
|
497
|
+
if (!isRecord(decoded) || !Array.isArray(decoded.entries)) {
|
|
498
|
+
throw new Error('Invalid EntryBatchPayload: missing or invalid "entries" field');
|
|
499
|
+
}
|
|
500
|
+
if (decoded.entries.length === 0) {
|
|
501
|
+
throw new Error('Invalid EntryBatchPayload: "entries" must not be empty');
|
|
502
|
+
}
|
|
503
|
+
const validated = decoded.entries.map((candidate) => validateEntryPayload({ entry: candidate }).entry);
|
|
504
|
+
const first = validated[0];
|
|
505
|
+
for (let index = 1; index < validated.length; index++) {
|
|
506
|
+
const entry = validated[index];
|
|
507
|
+
if (entry.indexName !== first.indexName || entry.partitionId !== first.partitionId) {
|
|
508
|
+
throw new Error("Invalid EntryBatchPayload: entries must belong to one partition of one index");
|
|
509
|
+
}
|
|
510
|
+
if (entry.primaryTerm !== first.primaryTerm) {
|
|
511
|
+
throw new Error("Invalid EntryBatchPayload: entries must share one primary term");
|
|
512
|
+
}
|
|
513
|
+
if (entry.seqNo !== validated[index - 1].seqNo + 1) {
|
|
514
|
+
throw new Error("Invalid EntryBatchPayload: entries must carry contiguous ascending sequence numbers");
|
|
515
|
+
}
|
|
516
|
+
}
|
|
517
|
+
const indexUuid = optionalIndexUuid(decoded.indexUuid, "EntryBatchPayload");
|
|
518
|
+
return indexUuid === void 0 ? { entries: validated } : { entries: validated, indexUuid };
|
|
519
|
+
}
|
|
520
|
+
function validateForwardBatchPayload(decoded) {
|
|
521
|
+
if (!isRecord(decoded)) {
|
|
522
|
+
throw new Error("Invalid ForwardBatchPayload: expected an object");
|
|
523
|
+
}
|
|
524
|
+
if (typeof decoded.indexName !== "string" || decoded.indexName.length === 0) {
|
|
525
|
+
throw new Error('Invalid ForwardBatchPayload: "indexName" must be a non-empty string');
|
|
526
|
+
}
|
|
527
|
+
if (!Array.isArray(decoded.operations)) {
|
|
528
|
+
throw new Error('Invalid ForwardBatchPayload: "operations" must be an array');
|
|
529
|
+
}
|
|
530
|
+
if (decoded.operations.length === 0) {
|
|
531
|
+
throw new Error('Invalid ForwardBatchPayload: "operations" must not be empty');
|
|
532
|
+
}
|
|
533
|
+
if (decoded.operations.length > MAX_FORWARD_BATCH_OPERATIONS) {
|
|
534
|
+
throw new Error(
|
|
535
|
+
`Invalid ForwardBatchPayload: "operations" exceeds maximum length of ${MAX_FORWARD_BATCH_OPERATIONS}`
|
|
536
|
+
);
|
|
537
|
+
}
|
|
538
|
+
for (let index = 0; index < decoded.operations.length; index++) {
|
|
539
|
+
const operation = decoded.operations[index];
|
|
540
|
+
if (!isRecord(operation)) {
|
|
541
|
+
throw new Error(`Invalid ForwardBatchPayload: "operations[${index}]" must be an object`);
|
|
542
|
+
}
|
|
543
|
+
if (typeof operation.documentId !== "string" || operation.documentId.length === 0) {
|
|
544
|
+
throw new Error(`Invalid ForwardBatchPayload: "operations[${index}].documentId" must be a non-empty string`);
|
|
545
|
+
}
|
|
546
|
+
if (operation.operation !== "insert" && operation.operation !== "remove" && operation.operation !== "update") {
|
|
547
|
+
throw new Error(
|
|
548
|
+
`Invalid ForwardBatchPayload: "operations[${index}].operation" must be "insert", "remove", or "update"`
|
|
549
|
+
);
|
|
550
|
+
}
|
|
551
|
+
if (operation.document !== null && !(operation.document instanceof Uint8Array)) {
|
|
552
|
+
throw new Error(`Invalid ForwardBatchPayload: "operations[${index}].document" must be Uint8Array or null`);
|
|
553
|
+
}
|
|
554
|
+
if (operation.operation === "insert" && operation.document === null) {
|
|
555
|
+
throw new Error(`Invalid ForwardBatchPayload: "operations[${index}]" requires a document`);
|
|
556
|
+
}
|
|
557
|
+
if (operation.operation === "update" && operation.document === null && !isRecord(operation.updateFields)) {
|
|
558
|
+
throw new Error(`Invalid ForwardBatchPayload: "operations[${index}]" requires a document or updateFields`);
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
optionalIndexUuid(decoded.indexUuid, "ForwardBatchPayload");
|
|
562
|
+
return decoded;
|
|
563
|
+
}
|
|
564
|
+
function validateForwardBatchResultPayload(decoded) {
|
|
565
|
+
if (!isRecord(decoded)) {
|
|
566
|
+
throw new Error("Invalid ForwardBatchResultPayload: expected an object");
|
|
567
|
+
}
|
|
568
|
+
if (!Array.isArray(decoded.results)) {
|
|
569
|
+
throw new Error('Invalid ForwardBatchResultPayload: "results" must be an array');
|
|
570
|
+
}
|
|
571
|
+
for (let index = 0; index < decoded.results.length; index++) {
|
|
572
|
+
const result = decoded.results[index];
|
|
573
|
+
if (!isRecord(result)) {
|
|
574
|
+
throw new Error(`Invalid ForwardBatchResultPayload: "results[${index}]" must be an object`);
|
|
575
|
+
}
|
|
576
|
+
if (typeof result.documentId !== "string") {
|
|
577
|
+
throw new Error(`Invalid ForwardBatchResultPayload: "results[${index}].documentId" must be a string`);
|
|
578
|
+
}
|
|
579
|
+
if (typeof result.success !== "boolean") {
|
|
580
|
+
throw new Error(`Invalid ForwardBatchResultPayload: "results[${index}].success" must be a boolean`);
|
|
581
|
+
}
|
|
582
|
+
if (result.errorCode !== null && typeof result.errorCode !== "string") {
|
|
583
|
+
throw new Error(`Invalid ForwardBatchResultPayload: "results[${index}].errorCode" must be a string or null`);
|
|
584
|
+
}
|
|
585
|
+
if (result.errorMessage !== null && typeof result.errorMessage !== "string") {
|
|
586
|
+
throw new Error(`Invalid ForwardBatchResultPayload: "results[${index}].errorMessage" must be a string or null`);
|
|
587
|
+
}
|
|
588
|
+
if (result.errorDetails !== void 0 && result.errorDetails !== null && !isPlainObject(result.errorDetails)) {
|
|
589
|
+
throw new Error(`Invalid ForwardBatchResultPayload: "results[${index}].errorDetails" must be an object or absent`);
|
|
590
|
+
}
|
|
591
|
+
}
|
|
592
|
+
return decoded;
|
|
593
|
+
}
|
|
594
|
+
function validateInsyncConfirmPayload(decoded) {
|
|
595
|
+
if (!isRecord(decoded)) {
|
|
596
|
+
throw new Error("Invalid InsyncConfirmPayload: expected an object");
|
|
597
|
+
}
|
|
598
|
+
if (typeof decoded.indexName !== "string") {
|
|
599
|
+
throw new Error('Invalid InsyncConfirmPayload: "indexName" must be a string');
|
|
600
|
+
}
|
|
601
|
+
if (typeof decoded.partitionId !== "number") {
|
|
602
|
+
throw new Error('Invalid InsyncConfirmPayload: "partitionId" must be a number');
|
|
603
|
+
}
|
|
604
|
+
if (typeof decoded.accepted !== "boolean") {
|
|
605
|
+
throw new Error('Invalid InsyncConfirmPayload: "accepted" must be a boolean');
|
|
606
|
+
}
|
|
607
|
+
return decoded;
|
|
608
|
+
}
|
|
609
|
+
function validateAckPayload(decoded) {
|
|
610
|
+
if (!isRecord(decoded)) {
|
|
611
|
+
throw new Error("Invalid AckPayload: expected an object");
|
|
612
|
+
}
|
|
613
|
+
if (typeof decoded.seqNo !== "number") {
|
|
614
|
+
throw new Error('Invalid AckPayload: "seqNo" must be a number');
|
|
615
|
+
}
|
|
616
|
+
if (typeof decoded.partitionId !== "number") {
|
|
617
|
+
throw new Error('Invalid AckPayload: "partitionId" must be a number');
|
|
618
|
+
}
|
|
619
|
+
if (typeof decoded.indexName !== "string") {
|
|
620
|
+
throw new Error('Invalid AckPayload: "indexName" must be a string');
|
|
621
|
+
}
|
|
622
|
+
return decoded;
|
|
623
|
+
}
|
|
624
|
+
function computeEntryChecksum(entry) {
|
|
625
|
+
const payload = encode([
|
|
626
|
+
entry.seqNo,
|
|
627
|
+
entry.primaryTerm,
|
|
628
|
+
entry.operation,
|
|
629
|
+
entry.partitionId,
|
|
630
|
+
entry.indexName,
|
|
631
|
+
entry.documentId,
|
|
632
|
+
entry.document
|
|
633
|
+
]);
|
|
634
|
+
return crc32(payload);
|
|
635
|
+
}
|
|
636
|
+
function buildEntry(fields) {
|
|
637
|
+
return { ...fields, checksum: computeEntryChecksum(fields) };
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
// src/persistence/durability/wal-framing.ts
|
|
641
|
+
var WAL_MAGIC = new Uint8Array([78, 82, 83, 87]);
|
|
642
|
+
var WAL_FORMAT_VERSION = 1;
|
|
643
|
+
var SEGMENT_HEADER_SIZE = 8;
|
|
644
|
+
var RECORD_LENGTH_SIZE = 4;
|
|
645
|
+
var FRAME_CRC_SIZE = 4;
|
|
646
|
+
var MAX_WAL_RECORD_BYTES = 268435456;
|
|
647
|
+
function writeSegmentHeader() {
|
|
648
|
+
const header2 = new Uint8Array(SEGMENT_HEADER_SIZE);
|
|
649
|
+
header2.set(WAL_MAGIC, 0);
|
|
650
|
+
header2[4] = WAL_FORMAT_VERSION & 255;
|
|
651
|
+
return header2;
|
|
652
|
+
}
|
|
653
|
+
function frameRecord(entry) {
|
|
654
|
+
const payload = encode(entry);
|
|
655
|
+
const frame = new Uint8Array(RECORD_LENGTH_SIZE + payload.length + FRAME_CRC_SIZE);
|
|
656
|
+
const view = new DataView(frame.buffer);
|
|
657
|
+
view.setUint32(0, payload.length, false);
|
|
658
|
+
frame.set(payload, RECORD_LENGTH_SIZE);
|
|
659
|
+
view.setUint32(RECORD_LENGTH_SIZE + payload.length, crc32(payload), false);
|
|
660
|
+
return frame;
|
|
661
|
+
}
|
|
662
|
+
function checkSegmentHeader(data) {
|
|
663
|
+
if (data.length < SEGMENT_HEADER_SIZE) {
|
|
664
|
+
return { ok: false, reason: "segment shorter than the 8-byte header" };
|
|
665
|
+
}
|
|
666
|
+
for (let i = 0; i < WAL_MAGIC.length; i += 1) {
|
|
667
|
+
if (data[i] !== WAL_MAGIC[i]) {
|
|
668
|
+
return { ok: false, reason: "missing NRSW magic" };
|
|
669
|
+
}
|
|
670
|
+
}
|
|
671
|
+
const formatVersion = data[4];
|
|
672
|
+
if (formatVersion > WAL_FORMAT_VERSION) {
|
|
673
|
+
return {
|
|
674
|
+
ok: false,
|
|
675
|
+
reason: `WAL format version ${formatVersion} is newer than supported version ${WAL_FORMAT_VERSION}`
|
|
676
|
+
};
|
|
677
|
+
}
|
|
678
|
+
return { ok: true };
|
|
679
|
+
}
|
|
680
|
+
function decodeEntry(payload) {
|
|
681
|
+
const decoded = decode(payload);
|
|
682
|
+
const { entry } = validateEntryPayload({ entry: decoded });
|
|
683
|
+
if (computeEntryChecksum(entry) !== entry.checksum) {
|
|
684
|
+
throw new Error("WAL record failed its logical entry checksum");
|
|
685
|
+
}
|
|
686
|
+
return entry;
|
|
687
|
+
}
|
|
688
|
+
function corrupt(message, details) {
|
|
689
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_WAL_CORRUPT, message, details);
|
|
690
|
+
}
|
|
691
|
+
function readDurableRegion(data, durableByteLength, maxRecordBytes = MAX_WAL_RECORD_BYTES) {
|
|
692
|
+
const header2 = checkSegmentHeader(data);
|
|
693
|
+
if (!header2.ok) {
|
|
694
|
+
corrupt(`WAL segment header invalid in durable region: ${header2.reason}`, { reason: header2.reason });
|
|
695
|
+
}
|
|
696
|
+
if (durableByteLength < SEGMENT_HEADER_SIZE || durableByteLength > data.length) {
|
|
697
|
+
corrupt("Commit marker durable byte length is outside the segment", {
|
|
698
|
+
durableByteLength,
|
|
699
|
+
segmentLength: data.length
|
|
700
|
+
});
|
|
701
|
+
}
|
|
702
|
+
const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
|
|
703
|
+
const entries = [];
|
|
704
|
+
let offset = SEGMENT_HEADER_SIZE;
|
|
705
|
+
while (offset < durableByteLength) {
|
|
706
|
+
if (offset + RECORD_LENGTH_SIZE > durableByteLength) {
|
|
707
|
+
corrupt("WAL record length field overruns the durable region", { offset, durableByteLength });
|
|
708
|
+
}
|
|
709
|
+
const recordLength = view.getUint32(offset, false);
|
|
710
|
+
if (recordLength === 0 || recordLength > maxRecordBytes) {
|
|
711
|
+
corrupt("WAL record length is zero or exceeds the maximum record size", { offset, recordLength });
|
|
712
|
+
}
|
|
713
|
+
const payloadStart = offset + RECORD_LENGTH_SIZE;
|
|
714
|
+
const crcStart = payloadStart + recordLength;
|
|
715
|
+
const frameEnd = crcStart + FRAME_CRC_SIZE;
|
|
716
|
+
if (frameEnd > durableByteLength) {
|
|
717
|
+
corrupt("WAL record overruns the durable region", { offset, frameEnd, durableByteLength });
|
|
718
|
+
}
|
|
719
|
+
const payload = data.subarray(payloadStart, crcStart);
|
|
720
|
+
const storedCrc = view.getUint32(crcStart, false);
|
|
721
|
+
if (crc32(payload) !== storedCrc) {
|
|
722
|
+
corrupt("WAL record failed its frame checksum inside the durable region", { offset });
|
|
723
|
+
}
|
|
724
|
+
let entry;
|
|
725
|
+
try {
|
|
726
|
+
entry = decodeEntry(payload);
|
|
727
|
+
} catch (err) {
|
|
728
|
+
corrupt("WAL record payload failed to decode inside the durable region", {
|
|
729
|
+
offset,
|
|
730
|
+
cause: err instanceof Error ? err.message : String(err)
|
|
731
|
+
});
|
|
732
|
+
}
|
|
733
|
+
if (entries.length > 0 && entry.seqNo <= entries[entries.length - 1].seqNo) {
|
|
734
|
+
corrupt("WAL sequence numbers are not strictly increasing", {
|
|
735
|
+
previousSeqNo: entries[entries.length - 1].seqNo,
|
|
736
|
+
seqNo: entry.seqNo
|
|
737
|
+
});
|
|
738
|
+
}
|
|
739
|
+
entries.push(entry);
|
|
740
|
+
offset = frameEnd;
|
|
741
|
+
}
|
|
742
|
+
return entries;
|
|
743
|
+
}
|
|
744
|
+
function readTailBeyondFrontier(data, durableByteLength, lastDurableSeqNo, maxRecordBytes = MAX_WAL_RECORD_BYTES) {
|
|
745
|
+
const start = Math.max(durableByteLength, SEGMENT_HEADER_SIZE);
|
|
746
|
+
const entries = [];
|
|
747
|
+
if (start >= data.length) {
|
|
748
|
+
return { entries, cleanEnd: start };
|
|
749
|
+
}
|
|
750
|
+
const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
|
|
751
|
+
let offset = start;
|
|
752
|
+
let cleanEnd = start;
|
|
753
|
+
let previousSeqNo = lastDurableSeqNo;
|
|
754
|
+
while (offset < data.length) {
|
|
755
|
+
if (offset + RECORD_LENGTH_SIZE > data.length) {
|
|
756
|
+
break;
|
|
757
|
+
}
|
|
758
|
+
const recordLength = view.getUint32(offset, false);
|
|
759
|
+
if (recordLength === 0 || recordLength > maxRecordBytes) {
|
|
760
|
+
break;
|
|
761
|
+
}
|
|
762
|
+
const payloadStart = offset + RECORD_LENGTH_SIZE;
|
|
763
|
+
const crcStart = payloadStart + recordLength;
|
|
764
|
+
const frameEnd = crcStart + FRAME_CRC_SIZE;
|
|
765
|
+
if (frameEnd > data.length) {
|
|
766
|
+
break;
|
|
767
|
+
}
|
|
768
|
+
const payload = data.subarray(payloadStart, crcStart);
|
|
769
|
+
if (crc32(payload) !== view.getUint32(crcStart, false)) {
|
|
770
|
+
break;
|
|
771
|
+
}
|
|
772
|
+
let entry;
|
|
773
|
+
try {
|
|
774
|
+
entry = decodeEntry(payload);
|
|
775
|
+
} catch {
|
|
776
|
+
break;
|
|
777
|
+
}
|
|
778
|
+
if (entry.seqNo <= previousSeqNo) {
|
|
779
|
+
break;
|
|
780
|
+
}
|
|
781
|
+
entries.push(entry);
|
|
782
|
+
previousSeqNo = entry.seqNo;
|
|
783
|
+
offset = frameEnd;
|
|
784
|
+
cleanEnd = frameEnd;
|
|
785
|
+
}
|
|
786
|
+
return { entries, cleanEnd };
|
|
787
|
+
}
|
|
788
|
+
|
|
789
|
+
// src/persistence/durability/wal-segments.ts
|
|
790
|
+
function walPrefix(indexName, partitionId) {
|
|
791
|
+
return `${indexName}/wal/${partitionId}/`;
|
|
792
|
+
}
|
|
793
|
+
function segmentStartSeqNo(key, prefix) {
|
|
794
|
+
if (!key.startsWith(prefix)) {
|
|
795
|
+
return null;
|
|
796
|
+
}
|
|
797
|
+
const tail = key.slice(prefix.length);
|
|
798
|
+
if (!/^\d{16}$/.test(tail)) {
|
|
799
|
+
return null;
|
|
800
|
+
}
|
|
801
|
+
const value = Number.parseInt(tail, 10);
|
|
802
|
+
return Number.isSafeInteger(value) ? value : null;
|
|
803
|
+
}
|
|
804
|
+
async function collectSegments(directory, prefix) {
|
|
805
|
+
const keys = await directory.list(prefix);
|
|
806
|
+
const refs = [];
|
|
807
|
+
for (const key of keys) {
|
|
808
|
+
const startSeqNo = segmentStartSeqNo(key, prefix);
|
|
809
|
+
if (startSeqNo !== null) {
|
|
810
|
+
refs.push({ key, startSeqNo });
|
|
811
|
+
}
|
|
812
|
+
}
|
|
813
|
+
refs.sort((a, b) => a.startSeqNo - b.startSeqNo);
|
|
814
|
+
return refs;
|
|
815
|
+
}
|
|
816
|
+
async function openWal(directory, indexName, partitionId) {
|
|
817
|
+
const prefix = walPrefix(indexName, partitionId);
|
|
818
|
+
const markerBytes = await directory.read(`${prefix}commit`);
|
|
819
|
+
const marker = markerBytes === null ? null : readCommitMarker(markerBytes);
|
|
820
|
+
if (marker === null) {
|
|
821
|
+
return null;
|
|
822
|
+
}
|
|
823
|
+
return { marker, segments: await collectSegments(directory, prefix) };
|
|
824
|
+
}
|
|
825
|
+
async function* readSegmentsInOrder(directory, wal) {
|
|
826
|
+
const { marker, segments } = wal;
|
|
827
|
+
let highestRead = 0;
|
|
828
|
+
for (const { key, startSeqNo } of segments) {
|
|
829
|
+
if (startSeqNo > marker.state.activeSegmentSeqNo) {
|
|
830
|
+
continue;
|
|
831
|
+
}
|
|
832
|
+
const bytes = await directory.read(key);
|
|
833
|
+
if (bytes === null) {
|
|
834
|
+
continue;
|
|
835
|
+
}
|
|
836
|
+
if (startSeqNo < marker.state.activeSegmentSeqNo) {
|
|
837
|
+
const header2 = checkSegmentHeader(bytes);
|
|
838
|
+
if (!header2.ok) {
|
|
839
|
+
throw new NarsilError(
|
|
840
|
+
ErrorCodes.PERSISTENCE_WAL_CORRUPT,
|
|
841
|
+
`Sealed WAL segment header invalid: ${header2.reason}`,
|
|
842
|
+
{
|
|
843
|
+
key,
|
|
844
|
+
reason: header2.reason
|
|
845
|
+
}
|
|
846
|
+
);
|
|
847
|
+
}
|
|
848
|
+
const durableEntries2 = readDurableRegion(bytes, bytes.length);
|
|
849
|
+
for (const entry of durableEntries2) highestRead = Math.max(highestRead, entry.seqNo);
|
|
850
|
+
yield { durableEntries: durableEntries2, activeTail: null };
|
|
851
|
+
continue;
|
|
852
|
+
}
|
|
853
|
+
const durableEntries = readDurableRegion(bytes, marker.state.durableByteLength);
|
|
854
|
+
for (const entry of durableEntries) highestRead = Math.max(highestRead, entry.seqNo);
|
|
855
|
+
const tail = readTailBeyondFrontier(
|
|
856
|
+
bytes,
|
|
857
|
+
marker.state.durableByteLength,
|
|
858
|
+
Math.max(highestRead, marker.state.highestDurableSeqNo)
|
|
859
|
+
);
|
|
860
|
+
yield {
|
|
861
|
+
durableEntries,
|
|
862
|
+
activeTail: { key, entries: tail.entries, cleanEnd: tail.cleanEnd, segmentLength: bytes.length }
|
|
863
|
+
};
|
|
864
|
+
}
|
|
865
|
+
}
|
|
866
|
+
async function readWalSegments(directory, indexName, partitionId) {
|
|
867
|
+
const wal = await openWal(directory, indexName, partitionId);
|
|
868
|
+
if (wal === null) {
|
|
869
|
+
return null;
|
|
870
|
+
}
|
|
871
|
+
const durableEntries = [];
|
|
872
|
+
let highestReadFromWal = 0;
|
|
873
|
+
let activeTail = null;
|
|
874
|
+
for await (const segment of readSegmentsInOrder(directory, wal)) {
|
|
875
|
+
for (const entry of segment.durableEntries) {
|
|
876
|
+
durableEntries.push(entry);
|
|
877
|
+
highestReadFromWal = Math.max(highestReadFromWal, entry.seqNo);
|
|
878
|
+
}
|
|
879
|
+
activeTail = segment.activeTail ?? activeTail;
|
|
880
|
+
}
|
|
881
|
+
return { marker: wal.marker, segments: wal.segments, durableEntries, activeTail, highestReadFromWal };
|
|
882
|
+
}
|
|
883
|
+
function durableRecordMissing(indexName, partitionId, highestRead, marker) {
|
|
884
|
+
return new NarsilError(
|
|
885
|
+
ErrorCodes.PERSISTENCE_WAL_CORRUPT,
|
|
886
|
+
"A durable WAL record is missing: the highest recovered seqNo is below the commit marker",
|
|
887
|
+
{ indexName, partitionId, highestRead, highestDurable: marker.state.highestDurableSeqNo }
|
|
888
|
+
);
|
|
889
|
+
}
|
|
890
|
+
async function* walEntriesInRange(directory, indexName, partitionId, fromSeqNoExclusive, upToSeqNoInclusive) {
|
|
891
|
+
const wal = await openWal(directory, indexName, partitionId);
|
|
892
|
+
if (wal === null) {
|
|
893
|
+
return;
|
|
894
|
+
}
|
|
895
|
+
let highestRead = 0;
|
|
896
|
+
for await (const segment of readSegmentsInOrder(directory, wal)) {
|
|
897
|
+
for (const entry of segment.durableEntries) {
|
|
898
|
+
highestRead = Math.max(highestRead, entry.seqNo);
|
|
899
|
+
if (entry.seqNo > fromSeqNoExclusive && entry.seqNo <= upToSeqNoInclusive) {
|
|
900
|
+
yield entry;
|
|
901
|
+
}
|
|
902
|
+
}
|
|
903
|
+
}
|
|
904
|
+
if (Math.max(fromSeqNoExclusive, highestRead) < wal.marker.state.highestDurableSeqNo) {
|
|
905
|
+
throw durableRecordMissing(indexName, partitionId, highestRead, wal.marker);
|
|
906
|
+
}
|
|
907
|
+
}
|
|
908
|
+
function everyVectorFieldIsTopLevel(vectorFieldPaths) {
|
|
909
|
+
for (const fieldPath of vectorFieldPaths) {
|
|
910
|
+
if (fieldPath.includes(".")) return false;
|
|
911
|
+
}
|
|
912
|
+
return vectorFieldPaths.size > 0;
|
|
913
|
+
}
|
|
914
|
+
function documentWithoutVectors(encoded, vectorFieldPaths, leavesVectorsUndecoded) {
|
|
915
|
+
const skipped = leavesVectorsUndecoded ? decodeMapWithoutFields(encoded, vectorFieldPaths) : null;
|
|
916
|
+
if (skipped !== null) return skipped;
|
|
917
|
+
const document = decode(encoded);
|
|
918
|
+
restoreVectorFields(document, vectorFieldPaths);
|
|
919
|
+
return prepareDocumentVectors(document, vectorFieldPaths).partitionDoc;
|
|
920
|
+
}
|
|
921
|
+
async function buildSegmentFromEntries(input) {
|
|
922
|
+
const leavesVectorsUndecoded = everyVectorFieldIsTopLevel(input.vectorFieldPaths);
|
|
923
|
+
const live = /* @__PURE__ */ new Map();
|
|
924
|
+
const deleted = /* @__PURE__ */ new Set();
|
|
925
|
+
for await (const entry of input.entries) {
|
|
926
|
+
if (entry.operation === "DELETE") {
|
|
927
|
+
deleted.add(entry.documentId);
|
|
928
|
+
live.delete(entry.documentId);
|
|
929
|
+
continue;
|
|
930
|
+
}
|
|
931
|
+
if (entry.document === null) continue;
|
|
932
|
+
live.delete(entry.documentId);
|
|
933
|
+
live.set(entry.documentId, documentWithoutVectors(entry.document, input.vectorFieldPaths, leavesVectorsUndecoded));
|
|
934
|
+
}
|
|
935
|
+
const tombstones = [];
|
|
936
|
+
for (const docId of deleted) {
|
|
937
|
+
if (!live.has(docId)) tombstones.push(docId);
|
|
938
|
+
}
|
|
939
|
+
if (live.size === 0 && tombstones.length === 0) {
|
|
940
|
+
return null;
|
|
941
|
+
}
|
|
942
|
+
const documents = [];
|
|
943
|
+
for (const [docId, document] of live) documents.push({ docId, document });
|
|
944
|
+
const manager = createPartitionManager(input.indexName, input.config, input.language, createPartitionRouter(), 1);
|
|
945
|
+
const payload = buildSegmentPayload(
|
|
946
|
+
documents,
|
|
947
|
+
input.config.schema,
|
|
948
|
+
input.language,
|
|
949
|
+
resolvePartitionInsertOptions(input.config, manager.analysis),
|
|
950
|
+
input.config.trackPositions ?? true
|
|
951
|
+
);
|
|
952
|
+
const segment = createFrozenSegment(
|
|
953
|
+
payload,
|
|
954
|
+
documents.map((entry) => entry.document)
|
|
955
|
+
);
|
|
956
|
+
return { selections: [{ segment }], tombstones, docCount: live.size };
|
|
957
|
+
}
|
|
958
|
+
var FIXMAP = 128;
|
|
959
|
+
var MAP16 = 222;
|
|
960
|
+
var MAP32 = 223;
|
|
961
|
+
var FIXARRAY = 144;
|
|
962
|
+
var ARRAY16 = 220;
|
|
963
|
+
var ARRAY32 = 221;
|
|
964
|
+
var BIN8 = 196;
|
|
965
|
+
var BIN16 = 197;
|
|
966
|
+
var BIN32 = 198;
|
|
967
|
+
var FIX_COUNT_LIMIT = 16;
|
|
968
|
+
var UINT8_LIMIT = 256;
|
|
969
|
+
var UINT16_LIMIT = 65536;
|
|
970
|
+
function header(fixed, sixteen, thirtyTwo, count) {
|
|
971
|
+
if (count < FIX_COUNT_LIMIT) return Uint8Array.of(fixed | count);
|
|
972
|
+
if (count < UINT16_LIMIT) return Uint8Array.of(sixteen, count >> 8, count & 255);
|
|
973
|
+
const bytes = new Uint8Array(5);
|
|
974
|
+
bytes[0] = thirtyTwo;
|
|
975
|
+
new DataView(bytes.buffer).setUint32(1, count, false);
|
|
976
|
+
return bytes;
|
|
977
|
+
}
|
|
978
|
+
function binHeader(byteLength) {
|
|
979
|
+
if (byteLength < UINT8_LIMIT) return Uint8Array.of(BIN8, byteLength);
|
|
980
|
+
if (byteLength < UINT16_LIMIT) return Uint8Array.of(BIN16, byteLength >> 8, byteLength & 255);
|
|
981
|
+
const bytes = new Uint8Array(5);
|
|
982
|
+
bytes[0] = BIN32;
|
|
983
|
+
new DataView(bytes.buffer).setUint32(1, byteLength, false);
|
|
984
|
+
return bytes;
|
|
985
|
+
}
|
|
986
|
+
var MessagePackChunks = class {
|
|
987
|
+
chunks = [];
|
|
988
|
+
byteLength = 0;
|
|
989
|
+
push(chunk) {
|
|
990
|
+
if (chunk.byteLength === 0) return;
|
|
991
|
+
this.chunks.push(chunk);
|
|
992
|
+
this.byteLength += chunk.byteLength;
|
|
993
|
+
}
|
|
994
|
+
map(entries) {
|
|
995
|
+
this.push(header(FIXMAP, MAP16, MAP32, entries));
|
|
996
|
+
return this;
|
|
997
|
+
}
|
|
998
|
+
array(items) {
|
|
999
|
+
this.push(header(FIXARRAY, ARRAY16, ARRAY32, items));
|
|
1000
|
+
return this;
|
|
1001
|
+
}
|
|
1002
|
+
value(value) {
|
|
1003
|
+
this.push(encode(value));
|
|
1004
|
+
return this;
|
|
1005
|
+
}
|
|
1006
|
+
key(name) {
|
|
1007
|
+
return this.value(name);
|
|
1008
|
+
}
|
|
1009
|
+
binary(bytes) {
|
|
1010
|
+
this.push(binHeader(bytes.byteLength));
|
|
1011
|
+
this.push(bytes);
|
|
1012
|
+
return this;
|
|
1013
|
+
}
|
|
1014
|
+
binaryOrNil(bytes) {
|
|
1015
|
+
return bytes === null ? this.value(null) : this.binary(bytes);
|
|
1016
|
+
}
|
|
1017
|
+
};
|
|
1018
|
+
|
|
1019
|
+
// src/serialization/keyword-segment/token-order.ts
|
|
1020
|
+
function slotsAreInTokenOrder(payloadSlots) {
|
|
1021
|
+
for (let at = 0; at < payloadSlots.length; at++) {
|
|
1022
|
+
if (payloadSlots[at] !== at) return false;
|
|
1023
|
+
}
|
|
1024
|
+
return true;
|
|
1025
|
+
}
|
|
1026
|
+
function reorderedPositions(arrays, postingOffsets) {
|
|
1027
|
+
const { positionOffsets, positionValues, tokenTable } = arrays;
|
|
1028
|
+
if (positionOffsets === null || positionValues === null) return { positionOffsets: null, positionValues: null };
|
|
1029
|
+
const postings = postingOffsets[postingOffsets.length - 1];
|
|
1030
|
+
const offsets = new Uint32Array(postings + 1);
|
|
1031
|
+
const values = new Uint32Array(positionValues.length);
|
|
1032
|
+
let cursor = 0;
|
|
1033
|
+
for (let token = 0; token < tokenTable.payloadSlots.length; token++) {
|
|
1034
|
+
const slot = tokenTable.payloadSlots[token];
|
|
1035
|
+
for (let from = arrays.postingOffsets[slot]; from < arrays.postingOffsets[slot + 1]; from++) {
|
|
1036
|
+
const to = postingOffsets[token] + (from - arrays.postingOffsets[slot]);
|
|
1037
|
+
offsets[to] = cursor;
|
|
1038
|
+
const start = positionOffsets[from];
|
|
1039
|
+
const end = positionOffsets[from + 1];
|
|
1040
|
+
values.set(positionValues.subarray(start, end), cursor);
|
|
1041
|
+
cursor += end - start;
|
|
1042
|
+
}
|
|
1043
|
+
}
|
|
1044
|
+
offsets[postings] = cursor;
|
|
1045
|
+
return { positionOffsets: offsets, positionValues: values };
|
|
1046
|
+
}
|
|
1047
|
+
function postingsInTokenOrder(arrays) {
|
|
1048
|
+
const { tokenTable } = arrays;
|
|
1049
|
+
if (slotsAreInTokenOrder(tokenTable.payloadSlots)) return arrays;
|
|
1050
|
+
const tokens = tokenTable.payloadSlots.length;
|
|
1051
|
+
const postingOffsets = new Uint32Array(tokens + 1);
|
|
1052
|
+
for (let token = 0; token < tokens; token++) {
|
|
1053
|
+
const slot = tokenTable.payloadSlots[token];
|
|
1054
|
+
postingOffsets[token + 1] = postingOffsets[token] + arrays.postingOffsets[slot + 1] - arrays.postingOffsets[slot];
|
|
1055
|
+
}
|
|
1056
|
+
const postings = postingOffsets[tokens];
|
|
1057
|
+
const postingDocIds = new Uint32Array(postings);
|
|
1058
|
+
const postingFrequencies = new Uint16Array(postings);
|
|
1059
|
+
const postingFieldIndices = new Uint16Array(postings);
|
|
1060
|
+
for (let token = 0; token < tokens; token++) {
|
|
1061
|
+
const slot = tokenTable.payloadSlots[token];
|
|
1062
|
+
const start = arrays.postingOffsets[slot];
|
|
1063
|
+
const end = arrays.postingOffsets[slot + 1];
|
|
1064
|
+
postingDocIds.set(arrays.postingDocIds.subarray(start, end), postingOffsets[token]);
|
|
1065
|
+
postingFrequencies.set(arrays.postingFrequencies.subarray(start, end), postingOffsets[token]);
|
|
1066
|
+
postingFieldIndices.set(arrays.postingFieldIndices.subarray(start, end), postingOffsets[token]);
|
|
1067
|
+
}
|
|
1068
|
+
return {
|
|
1069
|
+
postingOffsets,
|
|
1070
|
+
postingDocIds,
|
|
1071
|
+
postingFrequencies,
|
|
1072
|
+
postingFieldIndices,
|
|
1073
|
+
...reorderedPositions(arrays, postingOffsets)
|
|
1074
|
+
};
|
|
1075
|
+
}
|
|
1076
|
+
|
|
1077
|
+
// src/serialization/keyword-segment/encode.ts
|
|
1078
|
+
var TOP_LEVEL_ENTRIES = 28;
|
|
1079
|
+
function writeFieldLengths(out, arrays) {
|
|
1080
|
+
out.array(arrays.fieldLengthNames.length);
|
|
1081
|
+
for (let f = 0; f < arrays.fieldLengthNames.length; f++) {
|
|
1082
|
+
out.map(2).key("field").value(arrays.fieldLengthNames[f]);
|
|
1083
|
+
out.key("lengths").binary(littleEndianBytes(arrays.fieldLengthColumns[f]));
|
|
1084
|
+
}
|
|
1085
|
+
}
|
|
1086
|
+
function writeValueColumns(out, columns) {
|
|
1087
|
+
out.array(columns.length);
|
|
1088
|
+
for (const column of columns) {
|
|
1089
|
+
out.map(3).key("field").value(column.fieldPath);
|
|
1090
|
+
out.key("documents").binary(littleEndianBytes(column.docIds));
|
|
1091
|
+
out.key("values").binary(littleEndianBytes(column.values));
|
|
1092
|
+
}
|
|
1093
|
+
}
|
|
1094
|
+
function writeBooleanColumns(out, columns) {
|
|
1095
|
+
out.array(columns.length);
|
|
1096
|
+
for (const column of columns) {
|
|
1097
|
+
out.map(3).key("field").value(column.fieldPath);
|
|
1098
|
+
out.key("trueDocuments").binary(littleEndianBytes(column.trueDocs));
|
|
1099
|
+
out.key("falseDocuments").binary(littleEndianBytes(column.falseDocs));
|
|
1100
|
+
}
|
|
1101
|
+
}
|
|
1102
|
+
function writeEnumColumns(out, columns) {
|
|
1103
|
+
out.array(columns.length);
|
|
1104
|
+
for (const column of columns) {
|
|
1105
|
+
out.map(4).key("field").value(column.fieldPath);
|
|
1106
|
+
out.key("values").value(column.values);
|
|
1107
|
+
out.key("offsets").binary(littleEndianBytes(column.offsets));
|
|
1108
|
+
out.key("documents").binary(littleEndianBytes(column.docIds));
|
|
1109
|
+
}
|
|
1110
|
+
}
|
|
1111
|
+
function writeGeopointColumns(out, columns) {
|
|
1112
|
+
out.array(columns.length);
|
|
1113
|
+
for (const column of columns) {
|
|
1114
|
+
out.map(4).key("field").value(column.fieldPath);
|
|
1115
|
+
out.key("documents").binary(littleEndianBytes(column.docIds));
|
|
1116
|
+
out.key("latitudes").binary(littleEndianBytes(column.latitudes));
|
|
1117
|
+
out.key("longitudes").binary(littleEndianBytes(column.longitudes));
|
|
1118
|
+
}
|
|
1119
|
+
}
|
|
1120
|
+
function writePatternColumns(out, columns) {
|
|
1121
|
+
out.array(columns.length);
|
|
1122
|
+
for (const column of columns) {
|
|
1123
|
+
out.map(5).key("field").value(column.fieldPath);
|
|
1124
|
+
out.key("documents").binary(littleEndianBytes(column.docIds));
|
|
1125
|
+
out.key("runs").binary(littleEndianBytes(column.runs));
|
|
1126
|
+
out.key("offsets").binary(littleEndianBytes(column.offsets));
|
|
1127
|
+
out.key("postings").binary(littleEndianBytes(column.postings));
|
|
1128
|
+
}
|
|
1129
|
+
}
|
|
1130
|
+
function writeSurfaceForms(out, arrays) {
|
|
1131
|
+
const table = arrays.surfaceTable;
|
|
1132
|
+
if (table === null || table.counts.length === 0) {
|
|
1133
|
+
out.value(null);
|
|
1134
|
+
return;
|
|
1135
|
+
}
|
|
1136
|
+
out.map(3).key("text").binary(table.blob);
|
|
1137
|
+
out.key("offsets").binary(littleEndianBytes(table.offsets));
|
|
1138
|
+
out.key("counts").binary(littleEndianBytes(table.counts));
|
|
1139
|
+
}
|
|
1140
|
+
function writePartialWords(out, arrays) {
|
|
1141
|
+
const table = arrays.partialWords;
|
|
1142
|
+
if (table === null) {
|
|
1143
|
+
out.value(null);
|
|
1144
|
+
return;
|
|
1145
|
+
}
|
|
1146
|
+
out.map(8).key("fields").value(table.fields);
|
|
1147
|
+
out.key("text").binary(table.spellings);
|
|
1148
|
+
out.key("offsets").binary(littleEndianBytes(table.spellingOffsets));
|
|
1149
|
+
out.key("tokens").binary(littleEndianBytes(table.spellingTokens));
|
|
1150
|
+
out.key("counts").binary(littleEndianBytes(table.spellingCounts));
|
|
1151
|
+
out.key("runs").binary(littleEndianBytes(table.runs));
|
|
1152
|
+
out.key("runOffsets").binary(littleEndianBytes(table.runOffsets));
|
|
1153
|
+
out.key("runEntries").binary(littleEndianBytes(table.runEntries));
|
|
1154
|
+
}
|
|
1155
|
+
function encodeKeywordSegment(contents) {
|
|
1156
|
+
const { arrays } = contents;
|
|
1157
|
+
const postings = postingsInTokenOrder(arrays);
|
|
1158
|
+
const out = new MessagePackChunks();
|
|
1159
|
+
out.map(TOP_LEVEL_ENTRIES);
|
|
1160
|
+
out.key("v").value(KEYWORD_SEGMENT_PAYLOAD_VERSION);
|
|
1161
|
+
out.key("schema").value(contents.schema);
|
|
1162
|
+
out.key("tombstones").value(contents.tombstones);
|
|
1163
|
+
out.key("documentCount").value(arrays.documentCount);
|
|
1164
|
+
out.key("ids").binary(arrays.idTable.blob);
|
|
1165
|
+
out.key("idOffsets").binary(littleEndianBytes(arrays.idTable.offsets));
|
|
1166
|
+
out.key("idOrder").binary(littleEndianBytes(arrays.idTable.sortedOrdinals));
|
|
1167
|
+
out.key("documents").binary(arrays.documentTable.blob);
|
|
1168
|
+
out.key("documentOffsets").binary(littleEndianBytes(arrays.documentTable.offsets));
|
|
1169
|
+
out.key("fieldNames").value(arrays.fieldNames);
|
|
1170
|
+
out.key("tokens").binary(arrays.tokenTable.blob);
|
|
1171
|
+
out.key("tokenOffsets").binary(littleEndianBytes(arrays.tokenTable.offsets));
|
|
1172
|
+
out.key("documentFrequencies").binary(littleEndianBytes(arrays.tokenTable.documentFrequencies));
|
|
1173
|
+
out.key("postingOffsets").binary(littleEndianBytes(postings.postingOffsets));
|
|
1174
|
+
out.key("postingDocuments").binary(littleEndianBytes(postings.postingDocIds));
|
|
1175
|
+
out.key("postingFrequencies").binary(littleEndianBytes(postings.postingFrequencies));
|
|
1176
|
+
out.key("postingFields").binary(littleEndianBytes(postings.postingFieldIndices));
|
|
1177
|
+
out.key("positionOffsets").binaryOrNil(postings.positionOffsets === null ? null : littleEndianBytes(postings.positionOffsets));
|
|
1178
|
+
out.key("positions").binaryOrNil(postings.positionValues === null ? null : littleEndianBytes(postings.positionValues));
|
|
1179
|
+
out.key("fieldLengths");
|
|
1180
|
+
writeFieldLengths(out, arrays);
|
|
1181
|
+
out.key("numeric");
|
|
1182
|
+
writeValueColumns(out, arrays.numeric);
|
|
1183
|
+
out.key("sizes");
|
|
1184
|
+
writeValueColumns(out, arrays.sizes);
|
|
1185
|
+
out.key("boolean");
|
|
1186
|
+
writeBooleanColumns(out, arrays.boolean);
|
|
1187
|
+
out.key("enum");
|
|
1188
|
+
writeEnumColumns(out, arrays.enums);
|
|
1189
|
+
out.key("geopoint");
|
|
1190
|
+
writeGeopointColumns(out, arrays.geo);
|
|
1191
|
+
out.key("pattern");
|
|
1192
|
+
writePatternColumns(out, arrays.pattern);
|
|
1193
|
+
out.key("surfaceForms");
|
|
1194
|
+
writeSurfaceForms(out, arrays);
|
|
1195
|
+
out.key("partialWords");
|
|
1196
|
+
writePartialWords(out, arrays);
|
|
1197
|
+
return out;
|
|
1198
|
+
}
|
|
1199
|
+
|
|
1200
|
+
// src/persistence/durability/segment/segment-file.ts
|
|
1201
|
+
function sharesMemoryWithWorkers() {
|
|
1202
|
+
return typeof SharedArrayBuffer === "function";
|
|
1203
|
+
}
|
|
1204
|
+
async function readSegmentFile(store, ref) {
|
|
1205
|
+
const bytes = await readListedFile(store, ref.key, { bytes: ref.bytes, what: "segment" });
|
|
1206
|
+
const { header: header2, payload } = await openChecksummedEnvelope(bytes, {
|
|
1207
|
+
envelopeFormatVersion: KEYWORD_SEGMENT_ENVELOPE_VERSION,
|
|
1208
|
+
what: `segment "${ref.key}"`,
|
|
1209
|
+
allowsCompression: true
|
|
1210
|
+
});
|
|
1211
|
+
requireListedChecksum("segment", ref.key, header2.checksum, ref.checksum);
|
|
1212
|
+
return decodeKeywordSegment(payload, sharesMemoryWithWorkers());
|
|
1213
|
+
}
|
|
1214
|
+
var ORDINAL_BYTES = 4;
|
|
1215
|
+
var POSTING_BYTES = 8;
|
|
1216
|
+
var VALUE_ENTRY_BYTES = 12;
|
|
1217
|
+
var GEOPOINT_ENTRY_BYTES = 20;
|
|
1218
|
+
var TOKEN_ENTRY_BYTES = 12;
|
|
1219
|
+
var STRING_HEADER_BYTES = 5;
|
|
1220
|
+
var encoder = new TextEncoder();
|
|
1221
|
+
function documentBytes(segment, ordinal) {
|
|
1222
|
+
const table = segment.arrays.documentTable;
|
|
1223
|
+
if (table !== null) return table.offsets[ordinal + 1] - table.offsets[ordinal];
|
|
1224
|
+
return messagePackUpperBoundBytes(segment.documentSource.docAt(ordinal));
|
|
1225
|
+
}
|
|
1226
|
+
function addPerEntry(costs, docIds, bytes) {
|
|
1227
|
+
for (let at = 0; at < docIds.length; at++) costs[docIds[at]] += bytes;
|
|
1228
|
+
}
|
|
1229
|
+
function chargePostings(segment, costs, chargesTokens) {
|
|
1230
|
+
const arrays = segment.arrays;
|
|
1231
|
+
const positions = arrays.positionOffsets;
|
|
1232
|
+
for (let p = 0; p < arrays.postingDocIds.length; p++) {
|
|
1233
|
+
let bytes = POSTING_BYTES;
|
|
1234
|
+
if (positions !== null) bytes += ORDINAL_BYTES * (1 + positions[p + 1] - positions[p]);
|
|
1235
|
+
costs[arrays.postingDocIds[p]] += bytes;
|
|
1236
|
+
}
|
|
1237
|
+
if (!chargesTokens) return;
|
|
1238
|
+
const table = arrays.tokenTable;
|
|
1239
|
+
for (let at = 0; at < table.size; at++) {
|
|
1240
|
+
const slot = table.payloadSlot(at);
|
|
1241
|
+
const bytes = table.byteLengthAt(at) + TOKEN_ENTRY_BYTES;
|
|
1242
|
+
for (let p = arrays.postingOffsets[slot]; p < arrays.postingOffsets[slot + 1]; p++) {
|
|
1243
|
+
costs[arrays.postingDocIds[p]] += bytes;
|
|
1244
|
+
}
|
|
1245
|
+
}
|
|
1246
|
+
}
|
|
1247
|
+
function chargeColumns(segment, costs) {
|
|
1248
|
+
const arrays = segment.arrays;
|
|
1249
|
+
for (const column of [...arrays.numeric, ...arrays.sizes]) addPerEntry(costs, column.docIds, VALUE_ENTRY_BYTES);
|
|
1250
|
+
for (const column of arrays.boolean) {
|
|
1251
|
+
addPerEntry(costs, column.trueDocs, ORDINAL_BYTES);
|
|
1252
|
+
addPerEntry(costs, column.falseDocs, ORDINAL_BYTES);
|
|
1253
|
+
}
|
|
1254
|
+
for (const column of arrays.enums) addPerEntry(costs, column.docIds, ORDINAL_BYTES);
|
|
1255
|
+
for (const column of arrays.geo) addPerEntry(costs, column.docIds, GEOPOINT_ENTRY_BYTES);
|
|
1256
|
+
for (const column of arrays.pattern) {
|
|
1257
|
+
addPerEntry(costs, column.docIds, ORDINAL_BYTES);
|
|
1258
|
+
addPerEntry(costs, column.postings, ORDINAL_BYTES);
|
|
1259
|
+
}
|
|
1260
|
+
}
|
|
1261
|
+
function stringListBytes(values) {
|
|
1262
|
+
let bytes = STRING_HEADER_BYTES;
|
|
1263
|
+
for (const value of values) bytes += STRING_HEADER_BYTES + encoder.encode(value).byteLength;
|
|
1264
|
+
return bytes;
|
|
1265
|
+
}
|
|
1266
|
+
function fixedBytesOf(segment, chargesTokens) {
|
|
1267
|
+
const arrays = segment.arrays;
|
|
1268
|
+
let bytes = stringListBytes(arrays.fieldNames);
|
|
1269
|
+
if (!chargesTokens) bytes += arrays.tokenTable.bytes;
|
|
1270
|
+
for (const column of arrays.enums) bytes += stringListBytes(column.values) + column.offsets.byteLength;
|
|
1271
|
+
for (const column of arrays.pattern) bytes += column.runs.byteLength + column.offsets.byteLength;
|
|
1272
|
+
const surfaces = segment.sharedSnapshot?.surfaceTable ?? null;
|
|
1273
|
+
if (surfaces !== null) bytes += surfaces.blob.byteLength + surfaces.offsets.byteLength + surfaces.counts.byteLength;
|
|
1274
|
+
else bytes += messagePackUpperBoundBytes(arrays.readSurfaceForms());
|
|
1275
|
+
return bytes + segment.partialWords.bytes();
|
|
1276
|
+
}
|
|
1277
|
+
function keptDocumentsOf(selection, chargesTokens) {
|
|
1278
|
+
const { segment } = selection;
|
|
1279
|
+
const count = segment.arrays.documentCount;
|
|
1280
|
+
const costs = new Float64Array(count);
|
|
1281
|
+
const perDocument = ORDINAL_BYTES * (3 + segment.arrays.fieldLengthColumns.length);
|
|
1282
|
+
for (let ordinal = 0; ordinal < count; ordinal++) {
|
|
1283
|
+
costs[ordinal] = perDocument + encoder.encode(segment.arrays.idTable.idAt(ordinal)).byteLength;
|
|
1284
|
+
}
|
|
1285
|
+
chargePostings(segment, costs, chargesTokens);
|
|
1286
|
+
chargeColumns(segment, costs);
|
|
1287
|
+
const ordinals = [];
|
|
1288
|
+
for (let ordinal = 0; ordinal < count; ordinal++) {
|
|
1289
|
+
if (!selectionKeeps(selection, ordinal)) continue;
|
|
1290
|
+
costs[ordinal] += documentBytes(segment, ordinal);
|
|
1291
|
+
ordinals.push(ordinal);
|
|
1292
|
+
}
|
|
1293
|
+
return { segment, ordinals, costs };
|
|
1294
|
+
}
|
|
1295
|
+
function selectionOf(segment, ordinals) {
|
|
1296
|
+
const marked = new Uint8Array(segment.arrays.documentCount);
|
|
1297
|
+
for (const ordinal of ordinals) marked[ordinal] = 1;
|
|
1298
|
+
return { segment, keep: (ordinal) => marked[ordinal] === 1 };
|
|
1299
|
+
}
|
|
1300
|
+
function tooLargeForOneFile(cost, budget) {
|
|
1301
|
+
throw new NarsilError(
|
|
1302
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
1303
|
+
`A document needs up to ${Math.ceil(cost)} bytes of a segment, more than the ${Math.max(0, Math.floor(budget))} that one file leaves it`,
|
|
1304
|
+
{ bytes: Math.ceil(cost), budget: Math.floor(budget), maximum: MAX_FILE_BYTES }
|
|
1305
|
+
);
|
|
1306
|
+
}
|
|
1307
|
+
function splitIntoSegmentFiles(selections, schema, tombstones) {
|
|
1308
|
+
let tokenBytes = 0;
|
|
1309
|
+
for (const { segment } of selections) tokenBytes += segment.arrays.tokenTable.bytes;
|
|
1310
|
+
const chargesTokens = tokenBytes > MAX_FILE_BYTES * SEGMENT_TOKEN_TABLE_SHARE_OF_FILE;
|
|
1311
|
+
let fixed = HEADER_SIZE + SEGMENT_FILE_FRAMING_SLACK_BYTES + encode(schema).byteLength + encode(tombstones).byteLength;
|
|
1312
|
+
for (const { segment } of selections) fixed += fixedBytesOf(segment, chargesTokens);
|
|
1313
|
+
const budget = MAX_FILE_BYTES - fixed;
|
|
1314
|
+
const groups = [];
|
|
1315
|
+
let current = [];
|
|
1316
|
+
let currentBytes = 0;
|
|
1317
|
+
let currentCount = 0;
|
|
1318
|
+
const flush = () => {
|
|
1319
|
+
if (currentCount === 0) return;
|
|
1320
|
+
groups.push({
|
|
1321
|
+
selections: current.map((entry) => selectionOf(entry.segment, entry.ordinals)),
|
|
1322
|
+
documentCount: currentCount
|
|
1323
|
+
});
|
|
1324
|
+
current = [];
|
|
1325
|
+
currentBytes = 0;
|
|
1326
|
+
currentCount = 0;
|
|
1327
|
+
};
|
|
1328
|
+
for (const selection of selections) {
|
|
1329
|
+
const kept = keptDocumentsOf(selection, chargesTokens);
|
|
1330
|
+
let entry = null;
|
|
1331
|
+
for (const ordinal of kept.ordinals) {
|
|
1332
|
+
const cost = kept.costs[ordinal];
|
|
1333
|
+
if (cost > budget) tooLargeForOneFile(cost, budget);
|
|
1334
|
+
if (currentBytes + cost > budget) {
|
|
1335
|
+
flush();
|
|
1336
|
+
entry = null;
|
|
1337
|
+
}
|
|
1338
|
+
if (entry === null) {
|
|
1339
|
+
entry = { segment: kept.segment, ordinals: [] };
|
|
1340
|
+
current.push(entry);
|
|
1341
|
+
}
|
|
1342
|
+
entry.ordinals.push(ordinal);
|
|
1343
|
+
currentBytes += cost;
|
|
1344
|
+
currentCount += 1;
|
|
1345
|
+
}
|
|
1346
|
+
}
|
|
1347
|
+
flush();
|
|
1348
|
+
return groups;
|
|
1349
|
+
}
|
|
1350
|
+
function halvesOf(group) {
|
|
1351
|
+
const flat = [];
|
|
1352
|
+
for (const selection of group.selections) {
|
|
1353
|
+
for (let ordinal = 0; ordinal < selection.segment.arrays.documentCount; ordinal++) {
|
|
1354
|
+
if (!selectionKeeps(selection, ordinal)) continue;
|
|
1355
|
+
flat.push({ segment: selection.segment, ordinal });
|
|
1356
|
+
}
|
|
1357
|
+
}
|
|
1358
|
+
const middle = Math.ceil(flat.length / 2);
|
|
1359
|
+
const groupOf = (entries) => {
|
|
1360
|
+
const bySegment = /* @__PURE__ */ new Map();
|
|
1361
|
+
for (const { segment, ordinal } of entries) {
|
|
1362
|
+
let ordinals = bySegment.get(segment);
|
|
1363
|
+
if (ordinals === void 0) {
|
|
1364
|
+
ordinals = [];
|
|
1365
|
+
bySegment.set(segment, ordinals);
|
|
1366
|
+
}
|
|
1367
|
+
ordinals.push(ordinal);
|
|
1368
|
+
}
|
|
1369
|
+
return {
|
|
1370
|
+
selections: [...bySegment].map(([segment, ordinals]) => selectionOf(segment, ordinals)),
|
|
1371
|
+
documentCount: entries.length
|
|
1372
|
+
};
|
|
1373
|
+
};
|
|
1374
|
+
return [groupOf(flat.slice(0, middle)), groupOf(flat.slice(middle))];
|
|
1375
|
+
}
|
|
1376
|
+
|
|
1377
|
+
// src/persistence/durability/segment/segment-write.ts
|
|
1378
|
+
function emptyPayload() {
|
|
1379
|
+
return {
|
|
1380
|
+
documentCount: 0,
|
|
1381
|
+
docIds: [],
|
|
1382
|
+
fieldNames: [],
|
|
1383
|
+
tokens: [],
|
|
1384
|
+
postingOffsets: new Uint32Array(1),
|
|
1385
|
+
postingDocIds: new Uint32Array(0),
|
|
1386
|
+
postingFrequencies: new Uint16Array(0),
|
|
1387
|
+
postingFieldIndices: new Uint16Array(0),
|
|
1388
|
+
positionOffsets: null,
|
|
1389
|
+
positionValues: null,
|
|
1390
|
+
fieldLengthNames: [],
|
|
1391
|
+
fieldLengthColumns: [],
|
|
1392
|
+
totalFieldLengths: {},
|
|
1393
|
+
docFrequencies: {},
|
|
1394
|
+
surfaceForms: null,
|
|
1395
|
+
partialSpellings: null,
|
|
1396
|
+
numeric: [],
|
|
1397
|
+
boolean: [],
|
|
1398
|
+
enums: [],
|
|
1399
|
+
geo: [],
|
|
1400
|
+
pattern: [],
|
|
1401
|
+
sizes: []
|
|
1402
|
+
};
|
|
1403
|
+
}
|
|
1404
|
+
function noSpellingsOf(analysis) {
|
|
1405
|
+
if (analysis === null || analysis.fields.length === 0) return null;
|
|
1406
|
+
return { fields: [...analysis.fields], tokens: [], spellings: [], counts: [] };
|
|
1407
|
+
}
|
|
1408
|
+
function contentsOfGroup(group, request, tombstones) {
|
|
1409
|
+
const { schema, partialWordAnalysis, textAnalysis } = request;
|
|
1410
|
+
if (group === null) {
|
|
1411
|
+
const payload = { ...emptyPayload(), partialSpellings: noSpellingsOf(partialWordAnalysis) };
|
|
1412
|
+
return {
|
|
1413
|
+
schema,
|
|
1414
|
+
tombstones,
|
|
1415
|
+
arrays: segmentArraysOf(payload, { blob: new Uint8Array(0), offsets: new Uint32Array(1) })
|
|
1416
|
+
};
|
|
1417
|
+
}
|
|
1418
|
+
const merged = mergeSegmentContents(group.selections, partialWordAnalysis, textAnalysis);
|
|
1419
|
+
return {
|
|
1420
|
+
schema,
|
|
1421
|
+
tombstones,
|
|
1422
|
+
arrays: segmentArraysOf(merged.payload, merged.documents, merged.reusedPartialWords)
|
|
1423
|
+
};
|
|
1424
|
+
}
|
|
1425
|
+
function requireRoomForAnotherSegment(state) {
|
|
1426
|
+
if (state.segments.length < MAX_SEGMENTS_PER_PARTITION) return;
|
|
1427
|
+
throw new NarsilError(
|
|
1428
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
1429
|
+
`Partition ${state.request.partitionId} would hold more than ${MAX_SEGMENTS_PER_PARTITION} segments`,
|
|
1430
|
+
{ partitionId: state.request.partitionId, maximum: MAX_SEGMENTS_PER_PARTITION }
|
|
1431
|
+
);
|
|
1432
|
+
}
|
|
1433
|
+
async function writeGroup(state, group) {
|
|
1434
|
+
const { request } = state;
|
|
1435
|
+
const tombstones = state.tombstonesWritten ? [] : [...request.tombstones];
|
|
1436
|
+
const contents = contentsOfGroup(group, request, tombstones);
|
|
1437
|
+
const encoded = encodeKeywordSegment(contents);
|
|
1438
|
+
if (HEADER_SIZE + encoded.byteLength > MAX_FILE_BYTES && group !== null && group.documentCount > 1) {
|
|
1439
|
+
const [first, second] = halvesOf(group);
|
|
1440
|
+
await writeGroup(state, first);
|
|
1441
|
+
await writeGroup(state, second);
|
|
1442
|
+
return;
|
|
1443
|
+
}
|
|
1444
|
+
requireRoomForAnotherSegment(state);
|
|
1445
|
+
const id = state.nextSegmentId;
|
|
1446
|
+
const key = segmentKey(request.indexName, request.partitionId, id);
|
|
1447
|
+
const packed = await packChecksummedEnvelope(encoded.chunks, KEYWORD_SEGMENT_ENVELOPE_VERSION, `segment "${key}"`);
|
|
1448
|
+
await request.store.write(key, packed.chunks);
|
|
1449
|
+
state.segments.push({
|
|
1450
|
+
id,
|
|
1451
|
+
key,
|
|
1452
|
+
bytes: packed.byteLength,
|
|
1453
|
+
checksum: packed.checksum,
|
|
1454
|
+
docCount: contents.arrays.documentCount,
|
|
1455
|
+
tombstoneCount: tombstones.length
|
|
1456
|
+
});
|
|
1457
|
+
state.nextSegmentId = id + 1;
|
|
1458
|
+
state.tombstonesWritten = true;
|
|
1459
|
+
}
|
|
1460
|
+
function keptDocumentCount(selections) {
|
|
1461
|
+
let count = 0;
|
|
1462
|
+
for (const selection of selections) {
|
|
1463
|
+
for (let ordinal = 0; ordinal < selection.segment.arrays.documentCount; ordinal++) {
|
|
1464
|
+
if (selectionKeeps(selection, ordinal)) count += 1;
|
|
1465
|
+
}
|
|
1466
|
+
}
|
|
1467
|
+
return count;
|
|
1468
|
+
}
|
|
1469
|
+
async function writeSegmentsOf(request) {
|
|
1470
|
+
const state = { request, segments: [], nextSegmentId: request.nextSegmentId, tombstonesWritten: false };
|
|
1471
|
+
const groups = request.mergesIntoOneFile === true ? [{ selections: [...request.selections], documentCount: keptDocumentCount(request.selections) }] : splitIntoSegmentFiles(request.selections, request.schema, request.tombstones);
|
|
1472
|
+
for (const group of groups) {
|
|
1473
|
+
if (group.documentCount > 0) await writeGroup(state, group);
|
|
1474
|
+
}
|
|
1475
|
+
if (!state.tombstonesWritten && request.tombstones.length > 0) await writeGroup(state, null);
|
|
1476
|
+
return { segments: state.segments, nextSegmentId: state.nextSegmentId };
|
|
1477
|
+
}
|
|
1478
|
+
|
|
1479
|
+
// src/persistence/durability/segment/compaction.ts
|
|
1480
|
+
function windowFits(segments, start, windowLength) {
|
|
1481
|
+
let documents = 0;
|
|
1482
|
+
let bytes = SEGMENT_FILE_FRAMING_SLACK_BYTES;
|
|
1483
|
+
for (let at = start; at < start + windowLength; at++) {
|
|
1484
|
+
documents += segments[at].docCount;
|
|
1485
|
+
bytes += segments[at].bytes;
|
|
1486
|
+
}
|
|
1487
|
+
return documents <= COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT && bytes <= MAX_FILE_BYTES;
|
|
1488
|
+
}
|
|
1489
|
+
function smallestFittingWindow(segments, windowLength) {
|
|
1490
|
+
let best = null;
|
|
1491
|
+
let bestBytes = Number.POSITIVE_INFINITY;
|
|
1492
|
+
for (let start = 0; start + windowLength <= segments.length; start++) {
|
|
1493
|
+
if (!windowFits(segments, start, windowLength)) continue;
|
|
1494
|
+
let bytes = 0;
|
|
1495
|
+
for (let at = start; at < start + windowLength; at++) bytes += segments[at].bytes;
|
|
1496
|
+
if (bytes < bestBytes) {
|
|
1497
|
+
bestBytes = bytes;
|
|
1498
|
+
best = { start, windowLength };
|
|
1499
|
+
}
|
|
1500
|
+
}
|
|
1501
|
+
return best;
|
|
1502
|
+
}
|
|
1503
|
+
function chooseWindow(segments, wantedLength) {
|
|
1504
|
+
for (let windowLength = Math.min(wantedLength, segments.length); windowLength >= 2; windowLength--) {
|
|
1505
|
+
const window = smallestFittingWindow(segments, windowLength);
|
|
1506
|
+
if (window !== null) return window;
|
|
1507
|
+
}
|
|
1508
|
+
return null;
|
|
1509
|
+
}
|
|
1510
|
+
function frozenOf(contents) {
|
|
1511
|
+
return createSharedFrozenSegment(shareSegmentArrays(contents.arrays, null));
|
|
1512
|
+
}
|
|
1513
|
+
function applyTimeOrder(frozen, contents) {
|
|
1514
|
+
for (let newer = 1; newer < frozen.length; newer++) {
|
|
1515
|
+
const replaced = new Set(contents[newer].tombstones);
|
|
1516
|
+
const ids = frozen[newer].arrays.idTable;
|
|
1517
|
+
for (let ordinal = 0; ordinal < ids.count; ordinal++) replaced.add(ids.idAt(ordinal));
|
|
1518
|
+
for (let older = 0; older < newer; older++) {
|
|
1519
|
+
for (const docId of replaced) frozen[older].tombstoneDocument(docId);
|
|
1520
|
+
}
|
|
1521
|
+
}
|
|
1522
|
+
}
|
|
1523
|
+
function retainedTombstones(frozen, contents) {
|
|
1524
|
+
const retained = /* @__PURE__ */ new Set();
|
|
1525
|
+
for (const segment of contents) {
|
|
1526
|
+
for (const docId of segment.tombstones) {
|
|
1527
|
+
if (!frozen.some((candidate) => candidate.hasDocument(docId))) retained.add(docId);
|
|
1528
|
+
}
|
|
1529
|
+
}
|
|
1530
|
+
return [...retained];
|
|
1531
|
+
}
|
|
1532
|
+
async function compactPartitionSegments(input) {
|
|
1533
|
+
const { segments, compactionThreshold } = input;
|
|
1534
|
+
if (segments.length <= compactionThreshold) {
|
|
1535
|
+
return { segments, nextSegmentId: input.nextSegmentId };
|
|
1536
|
+
}
|
|
1537
|
+
const window = chooseWindow(segments, segments.length - compactionThreshold + 1);
|
|
1538
|
+
if (window === null) {
|
|
1539
|
+
return { segments, nextSegmentId: input.nextSegmentId };
|
|
1540
|
+
}
|
|
1541
|
+
const windowRefs = segments.slice(window.start, window.start + window.windowLength);
|
|
1542
|
+
const contents = [];
|
|
1543
|
+
for (const ref of windowRefs) contents.push(await readSegmentFile(input.store, ref));
|
|
1544
|
+
const frozen = contents.map(frozenOf);
|
|
1545
|
+
applyTimeOrder(frozen, contents);
|
|
1546
|
+
const written = await writeSegmentsOf({
|
|
1547
|
+
store: input.store,
|
|
1548
|
+
indexName: input.indexName,
|
|
1549
|
+
partitionId: input.partitionId,
|
|
1550
|
+
nextSegmentId: input.nextSegmentId,
|
|
1551
|
+
schema: input.schema,
|
|
1552
|
+
partialWordAnalysis: input.partialWordAnalysis,
|
|
1553
|
+
textAnalysis: input.textAnalysis,
|
|
1554
|
+
selections: frozen.map((segment) => ({ segment })),
|
|
1555
|
+
tombstones: retainedTombstones(frozen, contents),
|
|
1556
|
+
mergesIntoOneFile: true
|
|
1557
|
+
});
|
|
1558
|
+
return {
|
|
1559
|
+
segments: [
|
|
1560
|
+
...segments.slice(0, window.start),
|
|
1561
|
+
...written.segments,
|
|
1562
|
+
...segments.slice(window.start + window.windowLength)
|
|
1563
|
+
],
|
|
1564
|
+
nextSegmentId: written.nextSegmentId
|
|
1565
|
+
};
|
|
1566
|
+
}
|
|
1567
|
+
|
|
1568
|
+
// src/persistence/durability/segment/write.ts
|
|
1569
|
+
function textAnalysisOf(config) {
|
|
1570
|
+
let resolved = null;
|
|
1571
|
+
const resolve = () => {
|
|
1572
|
+
resolved ??= {
|
|
1573
|
+
language: getLanguage(config.language ?? "english"),
|
|
1574
|
+
options: resolvePartitionInsertOptions(config, resolveIndexAnalysis(config))
|
|
1575
|
+
};
|
|
1576
|
+
return resolved;
|
|
1577
|
+
};
|
|
1578
|
+
return {
|
|
1579
|
+
get language() {
|
|
1580
|
+
return resolve().language;
|
|
1581
|
+
},
|
|
1582
|
+
get options() {
|
|
1583
|
+
return resolve().options;
|
|
1584
|
+
}
|
|
1585
|
+
};
|
|
1586
|
+
}
|
|
1587
|
+
function partialWordAnalysisOf(config, text) {
|
|
1588
|
+
const fields = partialFieldsOf(config.schema);
|
|
1589
|
+
if (fields.length === 0) return null;
|
|
1590
|
+
return {
|
|
1591
|
+
fields,
|
|
1592
|
+
get language() {
|
|
1593
|
+
return text.language;
|
|
1594
|
+
},
|
|
1595
|
+
get options() {
|
|
1596
|
+
return text.options;
|
|
1597
|
+
}
|
|
1598
|
+
};
|
|
1599
|
+
}
|
|
1600
|
+
function resolveCompactionThreshold(configured) {
|
|
1601
|
+
if (!Number.isInteger(configured) || configured <= 0 || configured > MAX_SEGMENTS_PER_PARTITION) {
|
|
1602
|
+
return DEFAULT_COMPACTION_THRESHOLD;
|
|
1603
|
+
}
|
|
1604
|
+
return configured;
|
|
1605
|
+
}
|
|
1606
|
+
function wholeContentsOf(input, partitionId) {
|
|
1607
|
+
const whole = input.wholePartitions?.find((candidate) => candidate.partitionId === partitionId);
|
|
1608
|
+
if (whole !== void 0) return whole;
|
|
1609
|
+
const captured = input.capturedPartitions?.find((candidate) => candidate.partitionId === partitionId);
|
|
1610
|
+
return captured === void 0 ? void 0 : contentsOfCaptured(captured);
|
|
1611
|
+
}
|
|
1612
|
+
async function writeWholePartition(context, partitionId, priorPartition, whole) {
|
|
1613
|
+
const written = await writeSegmentsOf({
|
|
1614
|
+
store: context.input.store,
|
|
1615
|
+
indexName: context.indexName,
|
|
1616
|
+
partitionId,
|
|
1617
|
+
nextSegmentId: priorPartition?.nextSegmentId ?? 0,
|
|
1618
|
+
schema: context.input.metadata.schema,
|
|
1619
|
+
partialWordAnalysis: context.partialWordAnalysis,
|
|
1620
|
+
textAnalysis: context.textAnalysis,
|
|
1621
|
+
selections: whole.selections,
|
|
1622
|
+
tombstones: []
|
|
1623
|
+
});
|
|
1624
|
+
return { partitionId, nextSegmentId: written.nextSegmentId, segments: written.segments };
|
|
1625
|
+
}
|
|
1626
|
+
async function writeLoggedChanges(context, partitionId, priorPartition, entries) {
|
|
1627
|
+
const built = await buildSegmentFromEntries({
|
|
1628
|
+
indexName: context.indexName,
|
|
1629
|
+
config: context.config,
|
|
1630
|
+
language: getLanguage(context.config.language ?? "english"),
|
|
1631
|
+
vectorFieldPaths: context.vectorFieldPaths,
|
|
1632
|
+
entries
|
|
1633
|
+
});
|
|
1634
|
+
let segments = priorPartition ? [...priorPartition.segments] : [];
|
|
1635
|
+
let nextSegmentId = priorPartition?.nextSegmentId ?? 0;
|
|
1636
|
+
if (built !== null) {
|
|
1637
|
+
const written = await writeSegmentsOf({
|
|
1638
|
+
store: context.input.store,
|
|
1639
|
+
indexName: context.indexName,
|
|
1640
|
+
partitionId,
|
|
1641
|
+
nextSegmentId,
|
|
1642
|
+
schema: context.input.metadata.schema,
|
|
1643
|
+
partialWordAnalysis: context.partialWordAnalysis,
|
|
1644
|
+
textAnalysis: context.textAnalysis,
|
|
1645
|
+
selections: built.selections,
|
|
1646
|
+
tombstones: built.tombstones
|
|
1647
|
+
});
|
|
1648
|
+
segments.push(...written.segments);
|
|
1649
|
+
nextSegmentId = written.nextSegmentId;
|
|
1650
|
+
}
|
|
1651
|
+
if (segments.length > MAX_SEGMENTS_PER_PARTITION) {
|
|
1652
|
+
throw new NarsilError(
|
|
1653
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
1654
|
+
`Partition ${partitionId} would exceed the maximum of ${MAX_SEGMENTS_PER_PARTITION} segments`,
|
|
1655
|
+
{ partitionId, segmentCount: segments.length, maximum: MAX_SEGMENTS_PER_PARTITION }
|
|
1656
|
+
);
|
|
1657
|
+
}
|
|
1658
|
+
const compacted = await compactPartitionSegments({
|
|
1659
|
+
store: context.input.store,
|
|
1660
|
+
indexName: context.indexName,
|
|
1661
|
+
partitionId,
|
|
1662
|
+
schema: context.input.metadata.schema,
|
|
1663
|
+
partialWordAnalysis: context.partialWordAnalysis,
|
|
1664
|
+
textAnalysis: context.textAnalysis,
|
|
1665
|
+
segments,
|
|
1666
|
+
nextSegmentId,
|
|
1667
|
+
compactionThreshold: context.compactionThreshold
|
|
1668
|
+
});
|
|
1669
|
+
segments = compacted.segments;
|
|
1670
|
+
nextSegmentId = compacted.nextSegmentId;
|
|
1671
|
+
return { partitionId, nextSegmentId, segments };
|
|
1672
|
+
}
|
|
1673
|
+
function carryForwardUncheckpointedPartitions(prior, targets, partitions, checkpointByPartition) {
|
|
1674
|
+
if (prior === null) return;
|
|
1675
|
+
const covered = new Set(targets.map((target) => target.partitionId));
|
|
1676
|
+
for (const priorPartition of prior.partitions) {
|
|
1677
|
+
if (covered.has(priorPartition.partitionId)) continue;
|
|
1678
|
+
partitions.push(priorPartition);
|
|
1679
|
+
const priorCheckpoint = prior.checkpoint.find((c) => c.partitionId === priorPartition.partitionId);
|
|
1680
|
+
if (priorCheckpoint !== void 0 && !checkpointByPartition.has(priorPartition.partitionId)) {
|
|
1681
|
+
checkpointByPartition.set(priorPartition.partitionId, priorCheckpoint);
|
|
1682
|
+
}
|
|
1683
|
+
}
|
|
1684
|
+
}
|
|
1685
|
+
async function writeCheckpointSegments(input) {
|
|
1686
|
+
const config = reconstructSchemaFromMetadata(input.metadata);
|
|
1687
|
+
const textAnalysis = textAnalysisOf(config);
|
|
1688
|
+
const context = {
|
|
1689
|
+
input,
|
|
1690
|
+
indexName: input.metadata.indexName,
|
|
1691
|
+
config,
|
|
1692
|
+
partialWordAnalysis: partialWordAnalysisOf(config, textAnalysis),
|
|
1693
|
+
textAnalysis,
|
|
1694
|
+
vectorFieldPaths: new Set(extractVectorFieldsFromSchema(config.schema).keys()),
|
|
1695
|
+
compactionThreshold: resolveCompactionThreshold(input.compactionThreshold)
|
|
1696
|
+
};
|
|
1697
|
+
const checkpointByPartition = /* @__PURE__ */ new Map();
|
|
1698
|
+
for (const target of input.targets) checkpointByPartition.set(target.partitionId, { ...target });
|
|
1699
|
+
const partitions = [];
|
|
1700
|
+
for (const target of input.targets) {
|
|
1701
|
+
const priorPartition = input.prior?.partitions.find((p) => p.partitionId === target.partitionId);
|
|
1702
|
+
const whole = wholeContentsOf(input, target.partitionId);
|
|
1703
|
+
if (whole !== void 0) {
|
|
1704
|
+
partitions.push(await writeWholePartition(context, target.partitionId, priorPartition, whole));
|
|
1705
|
+
continue;
|
|
1706
|
+
}
|
|
1707
|
+
if (input.directory === void 0) {
|
|
1708
|
+
throw new NarsilError(
|
|
1709
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
1710
|
+
`The checkpoint of partition ${target.partitionId} holds neither the partition's content nor a log to read it from`,
|
|
1711
|
+
{ partitionId: target.partitionId }
|
|
1712
|
+
);
|
|
1713
|
+
}
|
|
1714
|
+
const priorSeqNo = checkpointLastSeqNo(input.prior?.checkpoint ?? [], target.partitionId);
|
|
1715
|
+
const entries = walEntriesInRange(
|
|
1716
|
+
input.directory,
|
|
1717
|
+
context.indexName,
|
|
1718
|
+
target.partitionId,
|
|
1719
|
+
priorSeqNo,
|
|
1720
|
+
target.lastSeqNo
|
|
1721
|
+
);
|
|
1722
|
+
partitions.push(await writeLoggedChanges(context, target.partitionId, priorPartition, entries));
|
|
1723
|
+
}
|
|
1724
|
+
carryForwardUncheckpointedPartitions(input.prior, input.targets, partitions, checkpointByPartition);
|
|
1725
|
+
return { checkpoint: [...checkpointByPartition.values()], partitions };
|
|
1726
|
+
}
|
|
1727
|
+
async function commitCheckpointManifest(store, metadata, written, vectors) {
|
|
1728
|
+
const indexName = metadata.indexName;
|
|
1729
|
+
const manifest = {
|
|
1730
|
+
schema: metadata.schema,
|
|
1731
|
+
language: metadata.language,
|
|
1732
|
+
checkpoint: written.checkpoint,
|
|
1733
|
+
partitions: written.partitions,
|
|
1734
|
+
vectors
|
|
1735
|
+
};
|
|
1736
|
+
const packed = await encodeSegmentManifest(manifest);
|
|
1737
|
+
await store.write(manifestKey(indexName), packed.chunks);
|
|
1738
|
+
return { garbage: await unreferencedKeys(store, indexName, manifest) };
|
|
1739
|
+
}
|
|
1740
|
+
async function unreferencedKeys(store, indexName, manifest) {
|
|
1741
|
+
const referenced = manifestReferencedKeys(manifest);
|
|
1742
|
+
const stored = await store.list(segmentsPrefix(indexName));
|
|
1743
|
+
return [...stored.filter((key) => !referenced.has(key)), snapshotBundleKey(indexName)];
|
|
1744
|
+
}
|
|
1745
|
+
async function removeCheckpointGarbage(store, garbage) {
|
|
1746
|
+
for (const key of garbage) {
|
|
1747
|
+
await store.remove(key);
|
|
1748
|
+
}
|
|
1749
|
+
}
|
|
1750
|
+
|
|
1751
|
+
export { CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_HEARTBEAT_MS, CHECKPOINT_WORKER_IDLE_MS, CHECKPOINT_WORKER_TIMEOUT_MS, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_SEGMENT_MAX_BYTES, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS, DIRECTORY_LOCK_SETTLE_MS, EARLIER_SEGMENT_MANIFEST_VERSION, LOG_RECORDS_PER_CHECKPOINT, MANIFEST_ENVELOPE_VERSION, MAX_SEGMENTS_PER_PARTITION, MAX_SEGMENT_ID, SEGMENT_HEADER_SIZE, SEGMENT_MANIFEST_VERSION, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, adapterCheckpointStore, buildEntry, captureWholePartition, checkpointLastSeqNo, checkpointPrimaryTerm, commitCheckpointManifest, computeEntryChecksum, createAckMessage, createEntryBatchMessage, createEntryMessage, createForwardBatchMessage, createForwardBatchResultMessage, createForwardMessage, createInsyncAddMessage, createInsyncConfirmMessage, createInsyncRemoveMessage, createPrimaryResignMessage, directoryCheckpointStore, durableRecordMissing, encodeMarkerSlot, encodeSegmentManifest, frameRecord, graphPartKey, manifestFileRefs, manifestKey, manifestReferencedKeys, markerSlotOffset, nextMarkerSlot, optionalIndexUuid, partitionContentsOf, readCommitMarker, readListedFile, readSegmentFile, readWalSegments, removeCheckpointGarbage, requireListedChecksum, segmentKey, segmentsPrefix, snapshotBundleKey, validateAckPayload, validateEntryBatchPayload, validateEntryPayload, validateForwardBatchPayload, validateForwardBatchResultPayload, validateInsyncConfirmPayload, vectorFileKey, writeCheckpointSegments, writeSegmentHeader };
|