@delali/narsil 0.3.1-next.86 → 0.3.1-next.87
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/chunk-3BXQPIXK.mjs +602 -0
- package/dist/browser/chunk-3ZRMZIH4.mjs +3959 -0
- package/dist/browser/chunk-BBDAYUCO.mjs +2703 -0
- package/dist/browser/chunk-FHFFWLEK.mjs +669 -0
- package/dist/browser/chunk-HOZW3PKP.mjs +9 -0
- package/dist/browser/chunk-NQW2YXEX.mjs +346 -0
- package/dist/browser/chunk-TBMNPGFO.mjs +22910 -0
- package/dist/browser/durability-wiring-7RUN7LNS.mjs +3250 -0
- package/dist/browser/module-E25UDADM.mjs +2766 -0
- package/dist/browser/snapshot-A42DRERF.mjs +888 -0
- package/dist/browser/write-4U6Q26TJ.mjs +206 -0
- package/dist/{capabilities-CAdkp2P-.d.ts → capabilities-SJPttcgQ.d.ts} +2 -2
- package/dist/chunk-37CD3OKS.mjs +87 -0
- package/dist/chunk-AQU2QHSY.mjs +7247 -0
- package/dist/{chunk-OLXM65YH.mjs → chunk-AV4IJKD2.mjs} +1 -18
- package/dist/{chunk-4D6OH4EF.mjs → chunk-AXDAS2VA.mjs} +7 -5
- package/dist/{chunk-LS5IRM23.mjs → chunk-BVOJIYRV.mjs} +2 -25
- package/dist/chunk-C7SLA6CM.mjs +19 -0
- package/dist/{chunk-KJ7LB3TR.mjs → chunk-CVQWQVFX.mjs} +3 -22
- package/dist/chunk-DJHHF2L5.mjs +17 -0
- package/dist/chunk-FGMGUDHX.mjs +21 -0
- package/dist/chunk-IKST36MA.mjs +614 -0
- package/dist/{chunk-2453RJYK.mjs → chunk-J2BPXTD7.mjs} +167 -176
- package/dist/chunk-JFJBA3PD.mjs +522 -0
- package/dist/chunk-K4OWQ2M3.mjs +462 -0
- package/dist/chunk-L5ORED7V.mjs +25 -0
- package/dist/chunk-MKK4OZ5W.mjs +703 -0
- package/dist/chunk-MXS5HWIH.mjs +515 -0
- package/dist/chunk-P36MUFEJ.mjs +10 -0
- package/dist/chunk-PR4O7CAP.mjs +18 -0
- package/dist/{chunk-4VBG6PEH.mjs → chunk-PSUJCOSE.mjs} +3 -2
- package/dist/{chunk-ZCKPRCYI.mjs → chunk-Q5DC6WAB.mjs} +1 -1
- package/dist/{chunk-NFF5TDRG.mjs → chunk-QZN3MIJ3.mjs} +1 -16
- package/dist/chunk-R7OSDAFK.mjs +44 -0
- package/dist/chunk-RVM7AODE.mjs +73 -0
- package/dist/{chunk-XYDP7CCJ.mjs → chunk-SYRFSF6V.mjs} +6 -4
- package/dist/{chunk-SREPGC6O.mjs → chunk-T6CNO72J.mjs} +1 -1
- package/dist/chunk-T7DMUVTL.mjs +1714 -0
- package/dist/chunk-UKSUENHK.mjs +997 -0
- package/dist/{chunk-B3F7LZXM.mjs → chunk-ULRPNYAG.mjs} +1220 -2318
- package/dist/{chunk-GNYXCLJS.mjs → chunk-URGLSNKR.mjs} +4 -3
- package/dist/chunk-WQSHDZ7Z.mjs +631 -0
- package/dist/{chunk-XPK3GB2P.mjs → chunk-XBXKPWMR.mjs} +20 -2664
- package/dist/{chunk-RM6FRYEC.mjs → chunk-XSZVS4S3.mjs} +3 -2
- package/dist/chunk-Y2F66IQJ.mjs +74 -0
- package/dist/chunk-YCJLVNWR.mjs +262 -0
- package/dist/{client-CKUtkLz3.d.ts → client-BMK53IR2.d.ts} +3 -3
- package/dist/client.d.ts +8 -8
- package/dist/{config-CSDEPraV.d.ts → config-BUR-7eVQ.d.ts} +2 -2
- package/dist/{config-BuIkeM5C.d.ts → config-UJx1dyX1.d.ts} +2 -2
- package/dist/direct-executor-LD6JW57R.mjs +18 -0
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +30 -15
- package/dist/durability-wiring-5S32KDDC.mjs +2797 -0
- package/dist/{engine-DMNZJAUq.d.ts → engine-9X6W5naj.d.ts} +2 -2
- package/dist/{index-DzONZUKE.d.ts → index-vCMu8OeQ.d.ts} +1 -1
- package/dist/index.browser.mjs +2 -38128
- package/dist/index.d.ts +23 -9
- package/dist/index.mjs +18 -14
- package/dist/{memory-BPKNx9Tl.d.ts → memory-CV9NAxLY.d.ts} +1 -1
- package/dist/module-WIDYIJLY.mjs +1819 -0
- package/dist/partial-words.mjs +5 -3
- package/dist/pattern.mjs +3 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +1 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +23 -13
- package/dist/react.d.ts +7 -7
- package/dist/{schema-DgyDijhf.d.ts → schema-DtsgmNfU.d.ts} +1 -1
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +14 -9
- package/dist/snapshot-6BYSUPWL.mjs +469 -0
- package/dist/{tasks-DGyz586z.d.ts → tasks-BGaZJyoY.d.ts} +1 -1
- package/dist/{thread-EUUNJNKU.mjs → thread-GS65E6RM.mjs} +12 -9
- package/dist/vector/search-worker.d.ts +30 -3
- package/dist/vector/search-worker.mjs +6 -2
- package/dist/workers/entry.mjs +5 -3
- package/dist/write-HLEZTL2L.mjs +233 -0
- package/package.json +11 -7
- package/dist/chunk-IDLP54FH.mjs +0 -15198
- package/dist/direct-executor-SI2QV4YI.mjs +0 -19
- package/dist/simd-DW5AtJ6B.d.ts +0 -41
|
@@ -0,0 +1,522 @@
|
|
|
1
|
+
import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-YCJLVNWR.mjs';
|
|
2
|
+
import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, readSegmentFile, readListedFile, requireListedChecksum, vectorFileKey, graphPartKey } from './chunk-T7DMUVTL.mjs';
|
|
3
|
+
import { VERSION, openChecksummedEnvelope, HEADER_SIZE, packChecksummedEnvelope } from './chunk-AQU2QHSY.mjs';
|
|
4
|
+
import { decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays } from './chunk-XBXKPWMR.mjs';
|
|
5
|
+
import { MAX_FILE_BYTES } from './chunk-RVM7AODE.mjs';
|
|
6
|
+
import { flattenSchema, requireValidFieldTypes } from './chunk-CVQWQVFX.mjs';
|
|
7
|
+
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
8
|
+
import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
|
|
9
|
+
import { decode, encode } from '@msgpack/msgpack';
|
|
10
|
+
|
|
11
|
+
// src/engine/durability-metadata.ts
|
|
12
|
+
function metadataFromConfig(indexName, config, partitionCount, documentCount) {
|
|
13
|
+
const metadata = {
|
|
14
|
+
indexName,
|
|
15
|
+
schema: config.schema,
|
|
16
|
+
language: config.language,
|
|
17
|
+
partitionCount: partitionCount ?? config.partitionCount,
|
|
18
|
+
bm25Params: { k1: config.k1, b: config.b },
|
|
19
|
+
createdAt: Date.now(),
|
|
20
|
+
engineVersion: VERSION
|
|
21
|
+
};
|
|
22
|
+
metadata.documentCount = documentCount ?? config.documentCount;
|
|
23
|
+
if (config.embedding) {
|
|
24
|
+
metadata.embedding = config.embedding;
|
|
25
|
+
}
|
|
26
|
+
if (config.surfaceForms !== void 0) {
|
|
27
|
+
metadata.surfaceForms = config.surfaceForms;
|
|
28
|
+
}
|
|
29
|
+
if (config.analysisRevision !== void 0) {
|
|
30
|
+
metadata.analysisRevision = config.analysisRevision;
|
|
31
|
+
}
|
|
32
|
+
if (config.tokenizer !== void 0) {
|
|
33
|
+
metadata.tokenizer = config.tokenizer;
|
|
34
|
+
}
|
|
35
|
+
if (config.stopWords !== void 0) {
|
|
36
|
+
metadata.stopWords = config.stopWords;
|
|
37
|
+
}
|
|
38
|
+
if (config.stopWordList !== void 0) {
|
|
39
|
+
metadata.stopWordList = config.stopWordList;
|
|
40
|
+
}
|
|
41
|
+
if (config.partitionLimits !== void 0) {
|
|
42
|
+
metadata.partitionLimits = config.partitionLimits;
|
|
43
|
+
}
|
|
44
|
+
if (config.defaultScoring !== void 0) {
|
|
45
|
+
metadata.defaultScoring = config.defaultScoring;
|
|
46
|
+
}
|
|
47
|
+
if (config.trackPositions !== void 0) {
|
|
48
|
+
metadata.trackPositions = config.trackPositions;
|
|
49
|
+
}
|
|
50
|
+
if (config.strict !== void 0) {
|
|
51
|
+
metadata.strict = config.strict;
|
|
52
|
+
}
|
|
53
|
+
if (config.required !== void 0) {
|
|
54
|
+
metadata.required = config.required;
|
|
55
|
+
}
|
|
56
|
+
if (config.vectorPromotion !== void 0) {
|
|
57
|
+
metadata.vectorPromotion = config.vectorPromotion;
|
|
58
|
+
}
|
|
59
|
+
if (config.patternValueLimit !== void 0) {
|
|
60
|
+
metadata.patternValueLimit = config.patternValueLimit;
|
|
61
|
+
}
|
|
62
|
+
if (config.indexUuid !== void 0) {
|
|
63
|
+
metadata.indexUuid = config.indexUuid;
|
|
64
|
+
}
|
|
65
|
+
if (config.heldPartitions !== void 0) {
|
|
66
|
+
metadata.heldPartitions = config.heldPartitions;
|
|
67
|
+
}
|
|
68
|
+
return metadata;
|
|
69
|
+
}
|
|
70
|
+
function metadataConfigOf(entry) {
|
|
71
|
+
const embedding = entry.config.embedding ? {
|
|
72
|
+
fields: entry.config.embedding.fields,
|
|
73
|
+
...entry.embeddingAdapterName !== null ? { adapter: entry.embeddingAdapterName } : {}
|
|
74
|
+
} : void 0;
|
|
75
|
+
return {
|
|
76
|
+
...entry.indexUuid !== null ? { indexUuid: entry.indexUuid } : {},
|
|
77
|
+
...entry.heldPartitions !== null ? { heldPartitions: entry.heldPartitions } : {},
|
|
78
|
+
documentCount: entry.documentCount,
|
|
79
|
+
partitionCount: entry.partitionCount,
|
|
80
|
+
schema: flattenSchema(entry.config.schema),
|
|
81
|
+
language: entry.language.name,
|
|
82
|
+
k1: entry.config.bm25?.k1 ?? 1.2,
|
|
83
|
+
b: entry.config.bm25?.b ?? 0.75,
|
|
84
|
+
...embedding !== void 0 ? { embedding } : {},
|
|
85
|
+
surfaceForms: entry.config.surfaceForms !== false,
|
|
86
|
+
...entry.analysisRevision !== null ? { analysisRevision: entry.analysisRevision } : {},
|
|
87
|
+
...typeof entry.config.tokenizer === "string" ? { tokenizer: entry.config.tokenizer } : {},
|
|
88
|
+
...typeof entry.config.stopWords === "string" ? { stopWords: entry.config.stopWords } : {},
|
|
89
|
+
...entry.config.stopWords instanceof Set ? { stopWordList: [...entry.config.stopWords].sort(compareCodePoints) } : {},
|
|
90
|
+
...entry.config.partitions !== void 0 ? { partitionLimits: entry.config.partitions } : {},
|
|
91
|
+
...entry.config.defaultScoring !== void 0 ? { defaultScoring: entry.config.defaultScoring } : {},
|
|
92
|
+
...entry.config.trackPositions !== void 0 ? { trackPositions: entry.config.trackPositions } : {},
|
|
93
|
+
...entry.config.strict !== void 0 ? { strict: entry.config.strict } : {},
|
|
94
|
+
...entry.config.required !== void 0 ? { required: entry.config.required } : {},
|
|
95
|
+
...entry.config.vectorPromotion !== void 0 ? { vectorPromotion: entry.config.vectorPromotion } : {},
|
|
96
|
+
...entry.config.patternValueLimit !== void 0 ? { patternValueLimit: entry.config.patternValueLimit } : {}
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
function indexMetadataOf(indexName, entry, partitionCount) {
|
|
100
|
+
return metadataFromConfig(indexName, metadataConfigOf(entry), partitionCount);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// src/persistence/durability/segment/manifest-fields.ts
|
|
104
|
+
function invalidManifest(message, details) {
|
|
105
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `Segment manifest ${message}`, details);
|
|
106
|
+
}
|
|
107
|
+
function isRecord(value) {
|
|
108
|
+
return typeof value === "object" && value !== null && !Array.isArray(value) && !(value instanceof Uint8Array);
|
|
109
|
+
}
|
|
110
|
+
function isId(value) {
|
|
111
|
+
return typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= MAX_SEGMENT_ID;
|
|
112
|
+
}
|
|
113
|
+
function isUint32(value) {
|
|
114
|
+
return typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= 4294967295;
|
|
115
|
+
}
|
|
116
|
+
function isSequenceNumber(value) {
|
|
117
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
|
|
118
|
+
}
|
|
119
|
+
function readFileRef(raw, expectedKey, what) {
|
|
120
|
+
if (raw.key !== expectedKey) {
|
|
121
|
+
invalidManifest(`lists the ${what} under the key "${String(raw.key)}" where it belongs at "${expectedKey}"`, {
|
|
122
|
+
key: raw.key,
|
|
123
|
+
expected: expectedKey
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
const bytes = raw.bytes;
|
|
127
|
+
if (typeof bytes !== "number" || !Number.isInteger(bytes) || bytes < HEADER_SIZE || bytes > MAX_FILE_BYTES) {
|
|
128
|
+
invalidManifest(`gives the ${what} "${expectedKey}" an invalid length`, { key: expectedKey, bytes });
|
|
129
|
+
}
|
|
130
|
+
if (!isUint32(raw.checksum)) invalidManifest(`gives the ${what} "${expectedKey}" an invalid checksum`);
|
|
131
|
+
return { key: expectedKey, bytes, checksum: raw.checksum };
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
// src/persistence/durability/segment/manifest-vectors-decode.ts
|
|
135
|
+
function normalizeFile(raw, indexName, fieldPath, nextFileId, seenIds) {
|
|
136
|
+
if (!isRecord(raw) || !isId(raw.id)) invalidManifest("has an invalid vector file reference", { fieldPath });
|
|
137
|
+
if (raw.id >= nextFileId || seenIds.has(raw.id)) {
|
|
138
|
+
invalidManifest(`lists vector file ${raw.id} of "${fieldPath}" twice or at its next file id`, { fieldPath });
|
|
139
|
+
}
|
|
140
|
+
seenIds.add(raw.id);
|
|
141
|
+
const count = raw.count;
|
|
142
|
+
if (typeof count !== "number" || !Number.isInteger(count) || count < 0 || count > VECTOR_FILE_MAX_VECTORS) {
|
|
143
|
+
invalidManifest(`gives vector file ${raw.id} of "${fieldPath}" an invalid count`, { fieldPath, count });
|
|
144
|
+
}
|
|
145
|
+
const dead = raw.dead ?? null;
|
|
146
|
+
if (dead !== null && !(dead instanceof Uint8Array)) invalidManifest("has dead vectors that are not bytes");
|
|
147
|
+
if (dead !== null && dead.byteLength !== deadBytesFor(count)) {
|
|
148
|
+
invalidManifest(`marks the dead vectors of file ${raw.id} of "${fieldPath}" with the wrong number of bytes`);
|
|
149
|
+
}
|
|
150
|
+
const ref = readFileRef(raw, vectorFileKey(indexName, fieldPath, raw.id), "vector file");
|
|
151
|
+
return { ...ref, id: raw.id, count, dead };
|
|
152
|
+
}
|
|
153
|
+
function normalizeGraphParts(raw, indexName, fieldPath, generation) {
|
|
154
|
+
if (!Array.isArray(raw)) invalidManifest(`gives "${fieldPath}" no list of graph parts`, { fieldPath });
|
|
155
|
+
if (raw.length > 0 && generation === 0) invalidManifest(`lists graph parts of "${fieldPath}" at generation 0`);
|
|
156
|
+
return raw.map((part, at) => {
|
|
157
|
+
if (!isRecord(part)) invalidManifest(`has an invalid graph part of "${fieldPath}"`, { fieldPath });
|
|
158
|
+
return readFileRef(part, graphPartKey(indexName, fieldPath, generation, at), "graph part");
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
function normalizeField(raw, indexName, seenFields) {
|
|
162
|
+
if (!isRecord(raw) || typeof raw.fieldPath !== "string") invalidManifest("has an invalid vector reference");
|
|
163
|
+
const fieldPath = raw.fieldPath;
|
|
164
|
+
if (seenFields.has(fieldPath)) invalidManifest(`lists the vector field "${fieldPath}" twice`, { fieldPath });
|
|
165
|
+
seenFields.add(fieldPath);
|
|
166
|
+
if (!isId(raw.nextFileId)) invalidManifest(`gives "${fieldPath}" an invalid next file id`, { fieldPath });
|
|
167
|
+
if (!Array.isArray(raw.files)) invalidManifest(`gives "${fieldPath}" no list of vector files`, { fieldPath });
|
|
168
|
+
if (!isId(raw.graphGeneration)) invalidManifest(`gives "${fieldPath}" an invalid graph generation`, { fieldPath });
|
|
169
|
+
const nextFileId = raw.nextFileId;
|
|
170
|
+
const seenIds = /* @__PURE__ */ new Set();
|
|
171
|
+
return {
|
|
172
|
+
fieldPath,
|
|
173
|
+
nextFileId,
|
|
174
|
+
files: raw.files.map((file) => normalizeFile(file, indexName, fieldPath, nextFileId, seenIds)),
|
|
175
|
+
graphGeneration: raw.graphGeneration,
|
|
176
|
+
graphParts: normalizeGraphParts(raw.graphParts, indexName, fieldPath, raw.graphGeneration)
|
|
177
|
+
};
|
|
178
|
+
}
|
|
179
|
+
function normalizeVectorFields(raw, indexName) {
|
|
180
|
+
if (!Array.isArray(raw)) invalidManifest("has malformed vector references");
|
|
181
|
+
const seenFields = /* @__PURE__ */ new Set();
|
|
182
|
+
return raw.map((field) => normalizeField(field, indexName, seenFields));
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
// src/persistence/durability/segment/manifest-decode.ts
|
|
186
|
+
async function openManifestPayload(data) {
|
|
187
|
+
const { payload } = await openChecksummedEnvelope(data, {
|
|
188
|
+
envelopeFormatVersion: MANIFEST_ENVELOPE_VERSION,
|
|
189
|
+
what: "segment manifest",
|
|
190
|
+
allowsCompression: true
|
|
191
|
+
});
|
|
192
|
+
const raw = decode(payload);
|
|
193
|
+
if (!isRecord(raw)) invalidManifest("is not a map");
|
|
194
|
+
return raw;
|
|
195
|
+
}
|
|
196
|
+
function listedKeysOf(raw) {
|
|
197
|
+
const keys = [];
|
|
198
|
+
for (const partition of Array.isArray(raw.partitions) ? raw.partitions : []) {
|
|
199
|
+
if (!isRecord(partition) || !Array.isArray(partition.segments)) continue;
|
|
200
|
+
for (const segment of partition.segments)
|
|
201
|
+
if (isRecord(segment) && typeof segment.key === "string") keys.push(segment.key);
|
|
202
|
+
}
|
|
203
|
+
for (const field of Array.isArray(raw.vectors) ? raw.vectors : []) {
|
|
204
|
+
if (!isRecord(field)) continue;
|
|
205
|
+
for (const list of [field.files, field.graphParts]) {
|
|
206
|
+
if (!Array.isArray(list)) continue;
|
|
207
|
+
for (const file of list) if (isRecord(file) && typeof file.key === "string") keys.push(file.key);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
return keys;
|
|
211
|
+
}
|
|
212
|
+
async function decodeSegmentManifest(data, indexName) {
|
|
213
|
+
return decodeManifestPayload(await openManifestPayload(data), indexName);
|
|
214
|
+
}
|
|
215
|
+
function decodeManifestPayload(raw, indexName) {
|
|
216
|
+
if (raw.version === EARLIER_SEGMENT_MANIFEST_VERSION) return { kind: "earlier", payload: raw };
|
|
217
|
+
if (raw.version !== SEGMENT_MANIFEST_VERSION) {
|
|
218
|
+
throw new NarsilError(
|
|
219
|
+
ErrorCodes.ENVELOPE_VERSION_MISMATCH,
|
|
220
|
+
`The segment manifest of index "${indexName}" holds layout ${String(raw.version)}, which this release cannot open. Export the index with the release that wrote it and import the export`,
|
|
221
|
+
{ indexName, version: raw.version }
|
|
222
|
+
);
|
|
223
|
+
}
|
|
224
|
+
const schema = raw.schema;
|
|
225
|
+
if (!isRecord(schema)) invalidManifest("is missing its schema");
|
|
226
|
+
requireValidFieldTypes(schema);
|
|
227
|
+
if (typeof raw.language !== "string") invalidManifest("is missing its language");
|
|
228
|
+
return {
|
|
229
|
+
kind: "current",
|
|
230
|
+
manifest: {
|
|
231
|
+
version: SEGMENT_MANIFEST_VERSION,
|
|
232
|
+
schema,
|
|
233
|
+
language: raw.language,
|
|
234
|
+
checkpoint: normalizeCheckpoint(raw.checkpoint),
|
|
235
|
+
partitions: normalizePartitions(raw.partitions, indexName),
|
|
236
|
+
vectors: normalizeVectorFields(raw.vectors, indexName)
|
|
237
|
+
}
|
|
238
|
+
};
|
|
239
|
+
}
|
|
240
|
+
function normalizeCheckpoint(raw) {
|
|
241
|
+
if (!Array.isArray(raw)) invalidManifest("is missing its checkpoint list");
|
|
242
|
+
const seen = /* @__PURE__ */ new Set();
|
|
243
|
+
return raw.map((entry) => {
|
|
244
|
+
if (!isRecord(entry) || !isUint32(entry.partitionId) || !isSequenceNumber(entry.lastSeqNo) || !isSequenceNumber(entry.primaryTerm) || seen.has(entry.partitionId)) {
|
|
245
|
+
invalidManifest("has an invalid partition checkpoint", { entry });
|
|
246
|
+
}
|
|
247
|
+
seen.add(entry.partitionId);
|
|
248
|
+
return { partitionId: entry.partitionId, lastSeqNo: entry.lastSeqNo, primaryTerm: entry.primaryTerm };
|
|
249
|
+
});
|
|
250
|
+
}
|
|
251
|
+
function normalizePartitions(raw, indexName) {
|
|
252
|
+
if (!Array.isArray(raw)) invalidManifest("is missing partition entries");
|
|
253
|
+
const seen = /* @__PURE__ */ new Set();
|
|
254
|
+
return raw.map((entry) => {
|
|
255
|
+
if (!isRecord(entry) || !isUint32(entry.partitionId) || seen.has(entry.partitionId)) {
|
|
256
|
+
invalidManifest("has a partition without a valid id", { partitionId: isRecord(entry) ? entry.partitionId : null });
|
|
257
|
+
}
|
|
258
|
+
seen.add(entry.partitionId);
|
|
259
|
+
const segments = normalizeSegments(entry.segments, indexName, entry.partitionId);
|
|
260
|
+
return {
|
|
261
|
+
partitionId: entry.partitionId,
|
|
262
|
+
nextSegmentId: readNextSegmentId(entry.nextSegmentId, segments, entry.partitionId),
|
|
263
|
+
segments
|
|
264
|
+
};
|
|
265
|
+
});
|
|
266
|
+
}
|
|
267
|
+
function normalizeSegments(raw, indexName, partitionId) {
|
|
268
|
+
if (!Array.isArray(raw)) invalidManifest("has malformed segment references", { partitionId });
|
|
269
|
+
if (raw.length > MAX_SEGMENTS_PER_PARTITION) {
|
|
270
|
+
invalidManifest(`lists ${raw.length} segments for partition ${partitionId}, above ${MAX_SEGMENTS_PER_PARTITION}`);
|
|
271
|
+
}
|
|
272
|
+
const seenIds = /* @__PURE__ */ new Set();
|
|
273
|
+
return raw.map((segment) => {
|
|
274
|
+
if (!isRecord(segment) || !isId(segment.id) || !isUint32(segment.docCount) || !isUint32(segment.tombstoneCount)) {
|
|
275
|
+
invalidManifest("has an invalid segment reference", { partitionId });
|
|
276
|
+
}
|
|
277
|
+
if (seenIds.has(segment.id)) invalidManifest(`reuses segment id ${segment.id} in partition ${partitionId}`);
|
|
278
|
+
seenIds.add(segment.id);
|
|
279
|
+
const ref = readFileRef(segment, segmentKey(indexName, partitionId, segment.id), "segment");
|
|
280
|
+
return { ...ref, id: segment.id, docCount: segment.docCount, tombstoneCount: segment.tombstoneCount };
|
|
281
|
+
});
|
|
282
|
+
}
|
|
283
|
+
function readNextSegmentId(value, segments, partitionId) {
|
|
284
|
+
if (!isId(value)) invalidManifest("has an invalid next segment id", { partitionId });
|
|
285
|
+
for (const segment of segments) {
|
|
286
|
+
if (segment.id >= value) {
|
|
287
|
+
invalidManifest(`next segment id ${value} does not exceed segment id ${segment.id}`, { partitionId });
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
return value;
|
|
291
|
+
}
|
|
292
|
+
var VECTOR_FILE_ENVELOPE_VERSION = 2;
|
|
293
|
+
var GRAPH_PART_ENVELOPE_VERSION = 2;
|
|
294
|
+
function liveVectorsOf(ref) {
|
|
295
|
+
let live = 0;
|
|
296
|
+
for (const file of ref.files) live += file.count - countDead(file.dead, file.count);
|
|
297
|
+
return live;
|
|
298
|
+
}
|
|
299
|
+
async function openListedFile(store, ref, what, allowsCompression) {
|
|
300
|
+
const bytes = await readListedFile(store, ref.key, { bytes: ref.bytes, what });
|
|
301
|
+
const { header, payload } = await openChecksummedEnvelope(bytes, {
|
|
302
|
+
envelopeFormatVersion: VECTOR_FILE_ENVELOPE_VERSION,
|
|
303
|
+
what: `${what} "${ref.key}"`,
|
|
304
|
+
allowsCompression
|
|
305
|
+
});
|
|
306
|
+
requireListedChecksum(what, ref.key, header.checksum, ref.checksum);
|
|
307
|
+
return { payload, payloadLength: header.payloadLength };
|
|
308
|
+
}
|
|
309
|
+
async function readGraphParts(store, ref) {
|
|
310
|
+
if (ref.graphParts.length === 0) return null;
|
|
311
|
+
const parts = [];
|
|
312
|
+
for (const part of ref.graphParts) {
|
|
313
|
+
const { payload } = await openListedFile(store, part, "graph part", true);
|
|
314
|
+
const decoded = decodeVectorGraphPartPayload(decode(payload));
|
|
315
|
+
if (decoded.fieldName !== ref.fieldPath) {
|
|
316
|
+
throw new NarsilError(
|
|
317
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
318
|
+
`The graph part "${part.key}" names field "${decoded.fieldName}" where the manifest lists "${ref.fieldPath}"`,
|
|
319
|
+
{ key: part.key }
|
|
320
|
+
);
|
|
321
|
+
}
|
|
322
|
+
parts.push(decoded);
|
|
323
|
+
}
|
|
324
|
+
return assembleGraphParts(parts);
|
|
325
|
+
}
|
|
326
|
+
async function readVectorFile(store, file, fieldPath) {
|
|
327
|
+
const { payload: bytes, payloadLength } = await openListedFile(store, file, "vector file", false);
|
|
328
|
+
const payload = decodeVectorFilePayload(decode(bytes));
|
|
329
|
+
if (payload.docIds.length !== file.count) {
|
|
330
|
+
throw new NarsilError(
|
|
331
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
332
|
+
`The vector file "${file.key}" holds ${payload.docIds.length} vectors where the manifest counts ${file.count}`,
|
|
333
|
+
{ key: file.key, held: payload.docIds.length, count: file.count }
|
|
334
|
+
);
|
|
335
|
+
}
|
|
336
|
+
if (payload.fieldName !== fieldPath) {
|
|
337
|
+
throw new NarsilError(
|
|
338
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
339
|
+
`The vector file "${file.key}" names field "${payload.fieldName}" where the manifest lists "${fieldPath}"`,
|
|
340
|
+
{ key: file.key }
|
|
341
|
+
);
|
|
342
|
+
}
|
|
343
|
+
return { payload, payloadLength };
|
|
344
|
+
}
|
|
345
|
+
async function loadVectorField(store, ref, vectorIndex) {
|
|
346
|
+
const restore = vectorIndex.restoreCheckpoint({
|
|
347
|
+
liveVectors: liveVectorsOf(ref),
|
|
348
|
+
holdsGraph: ref.graphParts.length > 0,
|
|
349
|
+
graph: await readGraphParts(store, ref)
|
|
350
|
+
});
|
|
351
|
+
for (const file of ref.files) {
|
|
352
|
+
const { payload, payloadLength } = await readVectorFile(store, file, ref.fieldPath);
|
|
353
|
+
restore.addFile({
|
|
354
|
+
id: file.id,
|
|
355
|
+
key: file.key,
|
|
356
|
+
payload,
|
|
357
|
+
dead: file.dead,
|
|
358
|
+
location: store.pathOf === void 0 ? null : {
|
|
359
|
+
path: await store.pathOf(file.key),
|
|
360
|
+
vectorsOffset: HEADER_SIZE + payloadLength - payload.vectors.byteLength
|
|
361
|
+
}
|
|
362
|
+
});
|
|
363
|
+
}
|
|
364
|
+
restore.finish();
|
|
365
|
+
}
|
|
366
|
+
var EMPTY_BIN = new Uint8Array(0);
|
|
367
|
+
var EMPTY_BIN_BYTES = 2;
|
|
368
|
+
var BIN8 = 196;
|
|
369
|
+
var BIN16 = 197;
|
|
370
|
+
var BIN32 = 198;
|
|
371
|
+
var BIN8_MAX_BYTES = 255;
|
|
372
|
+
var BIN16_MAX_BYTES = 65535;
|
|
373
|
+
function binHeader(byteLength) {
|
|
374
|
+
if (byteLength <= BIN8_MAX_BYTES) return Uint8Array.of(BIN8, byteLength);
|
|
375
|
+
if (byteLength <= BIN16_MAX_BYTES) return Uint8Array.of(BIN16, byteLength >> 8, byteLength & 255);
|
|
376
|
+
const header = new Uint8Array(5);
|
|
377
|
+
header[0] = BIN32;
|
|
378
|
+
new DataView(header.buffer).setUint32(1, byteLength, false);
|
|
379
|
+
return header;
|
|
380
|
+
}
|
|
381
|
+
function chunksEndingInBytes(withTheBytes, withEmptyBytesInTheirPlace, bytes) {
|
|
382
|
+
const head = encode(withEmptyBytesInTheirPlace);
|
|
383
|
+
const endsWithTheEmptyBin = head[head.length - EMPTY_BIN_BYTES] === BIN8 && head[head.length - 1] === 0;
|
|
384
|
+
if (!endsWithTheEmptyBin) return [encode(withTheBytes)];
|
|
385
|
+
return [head.subarray(0, head.length - EMPTY_BIN_BYTES), binHeader(bytes.byteLength), bytes];
|
|
386
|
+
}
|
|
387
|
+
function vectorPartChunks(part) {
|
|
388
|
+
return chunksEndingInBytes(part, { ...part, vectors: EMPTY_BIN }, part.vectors);
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
// src/persistence/durability/segment/vector-write.ts
|
|
392
|
+
function keptFileRef(input, prior, kept) {
|
|
393
|
+
const listed = prior?.files.find((file) => file.key === kept.key) ?? input.priorFileRefs?.get(kept.key);
|
|
394
|
+
if (listed === void 0) {
|
|
395
|
+
throw new NarsilError(
|
|
396
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
397
|
+
`The checkpoint keeps the vector file "${kept.key}", which the previous manifest does not list`,
|
|
398
|
+
{ key: kept.key }
|
|
399
|
+
);
|
|
400
|
+
}
|
|
401
|
+
return { ...kept, bytes: listed.bytes, checksum: listed.checksum };
|
|
402
|
+
}
|
|
403
|
+
async function writeGraphParts(input, fieldPath, plan, generation) {
|
|
404
|
+
const parts = plan.graphParts ?? [];
|
|
405
|
+
const refs = [];
|
|
406
|
+
for (let part = 0; part < parts.length; part++) {
|
|
407
|
+
const key = graphPartKey(input.indexName, fieldPath, generation, part);
|
|
408
|
+
const packed = await packChecksummedEnvelope(
|
|
409
|
+
[encode(parts[part])],
|
|
410
|
+
GRAPH_PART_ENVELOPE_VERSION,
|
|
411
|
+
`graph part "${key}"`
|
|
412
|
+
);
|
|
413
|
+
await input.store.write(key, packed.chunks);
|
|
414
|
+
refs.push({ key, bytes: packed.byteLength, checksum: packed.checksum });
|
|
415
|
+
}
|
|
416
|
+
return refs;
|
|
417
|
+
}
|
|
418
|
+
async function writeField(input, fieldPath, plan, outcome) {
|
|
419
|
+
const prior = input.priorVectors.find((ref) => ref.fieldPath === fieldPath);
|
|
420
|
+
if (plan.unchanged && prior !== void 0) {
|
|
421
|
+
outcome.refs.push(prior);
|
|
422
|
+
return;
|
|
423
|
+
}
|
|
424
|
+
let nextFileId = prior?.nextFileId ?? 0;
|
|
425
|
+
const files = plan.kept.map((kept) => keptFileRef(input, prior, kept));
|
|
426
|
+
const written = [];
|
|
427
|
+
for (let index = 0; index < plan.newFiles; index++) {
|
|
428
|
+
const { payload, ordinals, dead } = plan.readNewFile(index);
|
|
429
|
+
const id = nextFileId;
|
|
430
|
+
nextFileId += 1;
|
|
431
|
+
const key = vectorFileKey(input.indexName, fieldPath, id);
|
|
432
|
+
const packed = await packChecksummedEnvelope(
|
|
433
|
+
vectorPartChunks(payload),
|
|
434
|
+
VECTOR_FILE_ENVELOPE_VERSION,
|
|
435
|
+
`vector file "${key}"`
|
|
436
|
+
);
|
|
437
|
+
await input.store.write(key, packed.chunks);
|
|
438
|
+
files.push({ id, key, count: payload.docIds.length, dead, bytes: packed.byteLength, checksum: packed.checksum });
|
|
439
|
+
written.push({ id, key, ordinals });
|
|
440
|
+
outcome.layouts.push({
|
|
441
|
+
fieldPath,
|
|
442
|
+
key,
|
|
443
|
+
docIds: payload.docIds,
|
|
444
|
+
dead,
|
|
445
|
+
vectorsOffset: packed.byteLength - payload.vectors.byteLength
|
|
446
|
+
});
|
|
447
|
+
}
|
|
448
|
+
const priorGeneration = prior?.graphGeneration ?? 0;
|
|
449
|
+
const holdsGraph = plan.graphParts !== null && plan.graphParts.length > 0;
|
|
450
|
+
const graphGeneration = holdsGraph ? priorGeneration + 1 : priorGeneration;
|
|
451
|
+
const graphParts = holdsGraph ? await writeGraphParts(input, fieldPath, plan, graphGeneration) : [];
|
|
452
|
+
outcome.refs.push({ fieldPath, nextFileId, files, graphGeneration, graphParts });
|
|
453
|
+
outcome.written.push({ fieldPath, plan, files: written });
|
|
454
|
+
}
|
|
455
|
+
async function writeLiveVectors(input) {
|
|
456
|
+
const outcome = { refs: [], layouts: [], written: [] };
|
|
457
|
+
for (const [fieldPath, plan] of input.plans) await writeField(input, fieldPath, plan, outcome);
|
|
458
|
+
return outcome;
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
// src/persistence/durability/seq-owner.ts
|
|
462
|
+
var SINGLE_NODE_PRIMARY_TERM = 1;
|
|
463
|
+
function createSeqOwner(startSeqNo, primaryTerm = SINGLE_NODE_PRIMARY_TERM) {
|
|
464
|
+
let current = startSeqNo;
|
|
465
|
+
return {
|
|
466
|
+
next() {
|
|
467
|
+
current += 1;
|
|
468
|
+
return current;
|
|
469
|
+
},
|
|
470
|
+
get current() {
|
|
471
|
+
return current;
|
|
472
|
+
},
|
|
473
|
+
get primaryTerm() {
|
|
474
|
+
return primaryTerm;
|
|
475
|
+
}
|
|
476
|
+
};
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
// src/persistence/durability/segment/load-partition.ts
|
|
480
|
+
function requireAgreedSchema(agreed, schema, key) {
|
|
481
|
+
for (const [field, type] of Object.entries(schema)) {
|
|
482
|
+
const existing = agreed[field];
|
|
483
|
+
if (existing !== void 0 && existing !== type) {
|
|
484
|
+
throw new NarsilError(
|
|
485
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
486
|
+
`The segment "${key}" gives field "${field}" the type "${type}", while the index gives it "${existing}"`,
|
|
487
|
+
{ key, field, existing, found: type }
|
|
488
|
+
);
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
function removeWhatTheSegmentReplaces(manager, segment, contents) {
|
|
493
|
+
manager.beginBatchRemove();
|
|
494
|
+
try {
|
|
495
|
+
for (const docId of contents.tombstones) {
|
|
496
|
+
if (manager.has(docId)) manager.remove(docId);
|
|
497
|
+
}
|
|
498
|
+
const ids = segment.arrays.idTable;
|
|
499
|
+
for (let ordinal = 0; ordinal < ids.count; ordinal++) {
|
|
500
|
+
const docId = ids.idAt(ordinal);
|
|
501
|
+
if (manager.has(docId)) manager.remove(docId);
|
|
502
|
+
}
|
|
503
|
+
} finally {
|
|
504
|
+
manager.endBatchRemove();
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
function yieldToEventLoop() {
|
|
508
|
+
return new Promise((resolve) => setTimeout(resolve, 0));
|
|
509
|
+
}
|
|
510
|
+
async function loadPartitionSegments(store, partition, manager, options) {
|
|
511
|
+
for (const ref of partition.segments) {
|
|
512
|
+
const contents = await readSegmentFile(store, ref);
|
|
513
|
+
requireAgreedSchema(options.schema, contents.schema, ref.key);
|
|
514
|
+
if (options.decodesEveryDocument) decodeEveryStoredDocument(contents.arrays.documentTable);
|
|
515
|
+
const segment = createSharedFrozenSegment(shareSegmentArrays(contents.arrays, manager.partialWordAnalysis));
|
|
516
|
+
removeWhatTheSegmentReplaces(manager, segment, contents);
|
|
517
|
+
if (contents.arrays.documentCount > 0) manager.attachFrozenSegment(partition.partitionId, segment);
|
|
518
|
+
await yieldToEventLoop();
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
export { SINGLE_NODE_PRIMARY_TERM, VECTOR_FILE_ENVELOPE_VERSION, createSeqOwner, decodeManifestPayload, decodeSegmentManifest, indexMetadataOf, invalidManifest, isId, isRecord, isSequenceNumber, isUint32, listedKeysOf, loadPartitionSegments, loadVectorField, metadataConfigOf, metadataFromConfig, openManifestPayload, readGraphParts, readVectorFile, writeLiveVectors };
|