@delali/narsil 0.3.1-next.181 → 0.3.1-next.183
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/{chunk-JHFBQKUF.mjs → chunk-ZZUXMJIK.mjs} +12 -14
- package/dist/browser/{durability-config-SGB5ZP5K.mjs → durability-config-75NSFCQN.mjs} +6 -3
- package/dist/browser/{optional-writes-TB34FYUW.mjs → optional-writes-IQFTXANH.mjs} +2 -0
- package/dist/browser/{storage-leaf-QFXIWDFK.mjs → storage-leaf-2CO6SQ7R.mjs} +859 -250
- package/dist/{capabilities-BQIdsaWy.d.ts → capabilities-DAHXphRn.d.ts} +2 -2
- package/dist/{chunk-QFJMRB7D.mjs → chunk-2DK74OQL.mjs} +1 -1
- package/dist/{chunk-F3FDHXCJ.mjs → chunk-3OEXOSWI.mjs} +251 -50
- package/dist/{chunk-GO6TRSJ6.mjs → chunk-632ABEIR.mjs} +1 -1
- package/dist/{chunk-34IOOYXA.mjs → chunk-6ICOWB6I.mjs} +1 -1
- package/dist/{chunk-XNFBYAXI.mjs → chunk-C3EINJNJ.mjs} +1 -1
- package/dist/{chunk-JCWC75NH.mjs → chunk-C77OBNPP.mjs} +1 -1
- package/dist/{chunk-YPV2L7M5.mjs → chunk-CKS4QQEC.mjs} +3 -3
- package/dist/{chunk-2C3BXZGA.mjs → chunk-CWXR7OP4.mjs} +18 -20
- package/dist/{chunk-BIIROQ5Q.mjs → chunk-EGWBZSHK.mjs} +2 -2
- package/dist/{chunk-XQQKETAF.mjs → chunk-F2UOTRY7.mjs} +2 -2
- package/dist/{chunk-XTSFRWAS.mjs → chunk-FIXUVFUC.mjs} +3 -3
- package/dist/{chunk-Y27E2MGC.mjs → chunk-GGG5R4CU.mjs} +1 -1
- package/dist/{chunk-2KWTQ2P7.mjs → chunk-JVR5KMO4.mjs} +2 -2
- package/dist/{chunk-XITVDRRH.mjs → chunk-M3H7CTIC.mjs} +6 -6
- package/dist/{chunk-YXZDTKQS.mjs → chunk-MCAJOLVK.mjs} +2 -2
- package/dist/{chunk-QDZJM7NI.mjs → chunk-MSIN4NM3.mjs} +3 -1
- package/dist/{chunk-UISV46WX.mjs → chunk-NPXORGMK.mjs} +2 -2
- package/dist/{chunk-R7PLSVK3.mjs → chunk-OQANB7SJ.mjs} +1 -1
- package/dist/{chunk-KHJPMZE2.mjs → chunk-PK3BGGHO.mjs} +1 -1
- package/dist/{chunk-ISJ35523.mjs → chunk-PXEGKQHS.mjs} +2 -2
- package/dist/{chunk-LOQHA6QK.mjs → chunk-SA7U67XF.mjs} +2 -2
- package/dist/{chunk-JDIATDPY.mjs → chunk-T3YI4KN6.mjs} +1 -1
- package/dist/{chunk-ZB5X5WAK.mjs → chunk-TTTBE4TB.mjs} +10 -5
- package/dist/{chunk-A4XYDUVN.mjs → chunk-V7F5DOSC.mjs} +2 -2
- package/dist/{chunk-26E5MO3Y.mjs → chunk-WUWSUJ2X.mjs} +4 -4
- package/dist/{chunk-XFOL77WH.mjs → chunk-YWSIRKKQ.mjs} +1 -1
- package/dist/{chunk-7FC2ZILY.mjs → chunk-ZVZ2V2PZ.mjs} +37 -67
- package/dist/{client-CeUxvIvd.d.ts → client-DpqtT2XG.d.ts} +3 -3
- package/dist/client.d.ts +6 -6
- package/dist/{config-DehU-VGL.d.ts → config-C687-reW.d.ts} +11 -1
- package/dist/{config-CIpLXXxm.d.ts → config-D9WR3MhO.d.ts} +1 -1
- package/dist/{direct-executor-2WQSGBN3.mjs → direct-executor-JXYIL4OQ.mjs} +1 -1
- package/dist/distribution.d.ts +4 -4
- package/dist/distribution.mjs +391 -138
- package/dist/{durability-config-XV4XR7Y4.mjs → durability-config-NWU2G2SJ.mjs} +6 -3
- package/dist/{engine-P-4xhUe4.d.ts → engine-De6JzTW8.d.ts} +123 -72
- package/dist/index.browser.mjs +1 -1
- package/dist/index.d.ts +4 -4
- package/dist/index.mjs +1 -1
- package/dist/index.react-native.mjs +1 -1
- package/dist/{long-query-thread-AAYTLYZK.mjs → long-query-thread-HPKGRYWM.mjs} +10 -10
- package/dist/{optional-writes-N2DFWWZ3.mjs → optional-writes-QE7QUI45.mjs} +5 -3
- package/dist/{paged-partition-archive-X7L27E5Z.mjs → paged-partition-archive-IK2UOY2G.mjs} +9 -9
- package/dist/partial-words.browser.mjs +1 -1
- package/dist/partial-words.mjs +1 -1
- package/dist/persistence/durability/checkpoint-worker.d.ts +9 -0
- package/dist/persistence/durability/checkpoint-worker.mjs +23 -12
- package/dist/{query-actions-2JOLPELC.mjs → query-actions-COEFVUH3.mjs} +1 -1
- package/dist/query-suggestions/merge-worker.mjs +4 -4
- package/dist/query-suggestions.browser.mjs +1 -1
- package/dist/query-suggestions.d.ts +1 -1
- package/dist/query-suggestions.mjs +2 -2
- package/dist/react-native/chunks/{chunk-F4MIGBBP.mjs → chunk-6A5XE2MA.mjs} +18 -20
- package/dist/react-native/chunks/{chunk-H344ZXRH.mjs → chunk-DN2GVVR4.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-GVWLACAT.mjs → chunk-DZ4FQW5K.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-57R74JWC.mjs → chunk-G4YYO5TC.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-F6BJYCF2.mjs → chunk-UPN4NUXU.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-GGBATERA.mjs → chunk-XNIH3WIB.mjs} +1 -1
- package/dist/react-native/chunks/{durability-config-3IOXIPYZ.mjs → durability-config-RVTWSXPH.mjs} +6 -3
- package/dist/react-native/chunks/{optional-writes-TXTJ6HUZ.mjs → optional-writes-YZ7WTR7V.mjs} +4 -2
- package/dist/react-native/chunks/{restore-queue-SZLWIL3B.mjs → restore-queue-IGKD6Q2Y.mjs} +4 -4
- package/dist/react-native/chunks/{segment-codec-leaf-WMUZ5P4F.mjs → segment-codec-leaf-JTTGDT3G.mjs} +2 -2
- package/dist/react-native/chunks/{segment-merge-leaf-Q2OEJIQ3.mjs → segment-merge-leaf-D5ZFJRUB.mjs} +3 -3
- package/dist/react-native/chunks/{storage-leaf-OXYV6O6P.mjs → storage-leaf-ZKIM6IF6.mjs} +864 -255
- package/dist/react-native/chunks/{store-leaf-ZMDXYK4E.mjs → store-leaf-PHBOTT5J.mjs} +3 -3
- package/dist/react-native/chunks/{suggestions-leaf-FIMGQZ2T.mjs → suggestions-leaf-R7MIH6Q3.mjs} +3 -3
- package/dist/react-native/chunks/{worker-copies-72TGFYQB.mjs → worker-copies-U35KPVJ7.mjs} +2 -2
- package/dist/react-native/partial-words.mjs +1 -1
- package/dist/react-native/query-suggestions.mjs +1 -1
- package/dist/react-native/sync.mjs +1 -1
- package/dist/react-native/synonyms.mjs +1 -1
- package/dist/react.d.ts +7 -7
- package/dist/{rerank-E2HFKKGJ.mjs → rerank-LYL3D7LU.mjs} +1 -1
- package/dist/{restore-queue-PUVBN6L3.mjs → restore-queue-LF4DTVJC.mjs} +4 -4
- package/dist/{segment-codec-leaf-24Z3MMQI.mjs → segment-codec-leaf-ACALKS2C.mjs} +2 -2
- package/dist/{segment-merge-leaf-FQQDU45E.mjs → segment-merge-leaf-SKMPASFK.mjs} +2 -2
- package/dist/server.d.ts +8 -8
- package/dist/server.mjs +16 -16
- package/dist/{storage-leaf-KIA7XO6Y.mjs → storage-leaf-OJ55R4C5.mjs} +588 -158
- package/dist/{store-leaf-ESVLL4JO.mjs → store-leaf-CWHHEDRK.mjs} +3 -3
- package/dist/{suggestions-leaf-FXF4AHAY.mjs → suggestions-leaf-ZTU3ZTWJ.mjs} +5 -5
- package/dist/sync/filesystem.d.ts +2 -2
- package/dist/sync/filesystem.mjs +6 -6
- package/dist/sync/indexeddb.d.ts +2 -2
- package/dist/sync.browser.mjs +1 -1
- package/dist/sync.d.ts +3 -3
- package/dist/sync.mjs +1 -1
- package/dist/synonyms.browser.mjs +1 -1
- package/dist/synonyms.mjs +1 -1
- package/dist/{tasks-CXtQ3Zpp.d.ts → tasks-X2IJQc8J.d.ts} +2 -2
- package/dist/{thread-OWXVP2ZA.mjs → thread-BEGFRULV.mjs} +9 -9
- package/dist/{types-CDN70Y_C.d.ts → types-D0YasjoX.d.ts} +1 -1
- package/dist/{worker-copies-XXX7PF4J.mjs → worker-copies-373P6IYZ.mjs} +4 -4
- package/dist/workers/entry.mjs +7 -7
- package/openapi.json +32 -0
- package/package.json +7 -7
|
@@ -2109,6 +2109,8 @@ var SEGMENT_TOKEN_TABLE_SHARE_OF_FILE = 0.25;
|
|
|
2109
2109
|
var WHOLE_PARTITION_MIN_CHANGED_SHARE = 0.5;
|
|
2110
2110
|
var WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT = 1e5;
|
|
2111
2111
|
var LOG_RECORDS_PER_CHECKPOINT = 1e5;
|
|
2112
|
+
var REWRITE_ABOVE_REMOVED_DOCUMENT_SHARE = 0.2;
|
|
2113
|
+
var DEFAULT_REWRITE_BYTES_PER_CHECKPOINT = 268435456;
|
|
2112
2114
|
var WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES = 2147483648;
|
|
2113
2115
|
var CHECKPOINT_WORKER_TIMEOUT_MS = 6e5;
|
|
2114
2116
|
var CHECKPOINT_WORKER_IDLE_MS = 5e3;
|
|
@@ -2691,6 +2693,9 @@ function metadataToWire(meta) {
|
|
|
2691
2693
|
if (isNonNegativeInteger(meta.documentCount) && Number.isSafeInteger(meta.documentCount)) {
|
|
2692
2694
|
wire.document_count = meta.documentCount;
|
|
2693
2695
|
}
|
|
2696
|
+
if (isNonNegativeInteger(meta.savedBytes) && Number.isSafeInteger(meta.savedBytes)) {
|
|
2697
|
+
wire.saved_bytes = meta.savedBytes;
|
|
2698
|
+
}
|
|
2694
2699
|
if (meta.embedding) {
|
|
2695
2700
|
wire.embedding = meta.embedding.adapter !== void 0 ? { adapter: meta.embedding.adapter, fields: meta.embedding.fields } : { fields: meta.embedding.fields };
|
|
2696
2701
|
}
|
|
@@ -2806,6 +2811,9 @@ function wireToMetadata(raw) {
|
|
|
2806
2811
|
if (isNonNegativeInteger(raw.document_count) && Number.isSafeInteger(raw.document_count)) {
|
|
2807
2812
|
meta.documentCount = raw.document_count;
|
|
2808
2813
|
}
|
|
2814
|
+
if (isNonNegativeInteger(raw.saved_bytes) && Number.isSafeInteger(raw.saved_bytes)) {
|
|
2815
|
+
meta.savedBytes = raw.saved_bytes;
|
|
2816
|
+
}
|
|
2809
2817
|
if (raw.embedding && typeof raw.embedding === "object" && typeof raw.embedding.fields === "object") {
|
|
2810
2818
|
meta.embedding = typeof raw.embedding.adapter === "string" ? { adapter: raw.embedding.adapter, fields: raw.embedding.fields } : { fields: raw.embedding.fields };
|
|
2811
2819
|
}
|
|
@@ -4179,6 +4187,12 @@ function createReadCache(capacity) {
|
|
|
4179
4187
|
function evictDownTo(limit) {
|
|
4180
4188
|
for (const [key, entry] of entries) {
|
|
4181
4189
|
if (held <= limit) return;
|
|
4190
|
+
if (entry.referenced) {
|
|
4191
|
+
entry.referenced = false;
|
|
4192
|
+
entries.delete(key);
|
|
4193
|
+
entries.set(key, entry);
|
|
4194
|
+
continue;
|
|
4195
|
+
}
|
|
4182
4196
|
drop(key, entry);
|
|
4183
4197
|
}
|
|
4184
4198
|
}
|
|
@@ -4233,8 +4247,7 @@ function createReadCache(capacity) {
|
|
|
4233
4247
|
const key = fileNumber * FILE_NUMBER_SCALE + entry * ENTRY_NUMBER_SCALE + kind;
|
|
4234
4248
|
const found = entries.get(key);
|
|
4235
4249
|
if (found === void 0) return void 0;
|
|
4236
|
-
|
|
4237
|
-
entries.set(key, found);
|
|
4250
|
+
found.referenced = true;
|
|
4238
4251
|
return found.value;
|
|
4239
4252
|
},
|
|
4240
4253
|
put(fileNumber, kind, entry, value, bytes) {
|
|
@@ -4242,7 +4255,7 @@ function createReadCache(capacity) {
|
|
|
4242
4255
|
const key = fileNumber * FILE_NUMBER_SCALE + entry * ENTRY_NUMBER_SCALE + kind;
|
|
4243
4256
|
const existing = entries.get(key);
|
|
4244
4257
|
if (existing !== void 0) drop(key, existing);
|
|
4245
|
-
entries.set(key, { value, bytes, fileNumber });
|
|
4258
|
+
entries.set(key, { value, bytes, fileNumber, referenced: true });
|
|
4246
4259
|
held += bytes;
|
|
4247
4260
|
perFile.set(fileNumber, (perFile.get(fileNumber) ?? 0) + bytes);
|
|
4248
4261
|
evictDownTo(capacity - reserved);
|
|
@@ -5542,28 +5555,55 @@ async function pagedListingOf(directory, ref) {
|
|
|
5542
5555
|
function yieldToEventLoop2() {
|
|
5543
5556
|
return new Promise((resolve) => setTimeout(resolve, 0));
|
|
5544
5557
|
}
|
|
5558
|
+
function newerTombstonesOf(prepared, at) {
|
|
5559
|
+
const docIds = /* @__PURE__ */ new Set();
|
|
5560
|
+
for (let newer = at + 1; newer < prepared.length; newer++) {
|
|
5561
|
+
const access = prepared[newer].segment.paged;
|
|
5562
|
+
if (access === null) continue;
|
|
5563
|
+
for (const docId of access.snapshot.tombstones) docIds.add(docId);
|
|
5564
|
+
}
|
|
5565
|
+
return docIds;
|
|
5566
|
+
}
|
|
5567
|
+
function scansEveryId(segment, tombstoned) {
|
|
5568
|
+
const held = segment.arrays.idTable.count;
|
|
5569
|
+
return tombstoned.size * Math.ceil(Math.log2(held + 1)) > held;
|
|
5570
|
+
}
|
|
5571
|
+
function tombstoneScannedOrdinal(segment, tombstoned, ordinal) {
|
|
5572
|
+
const docId = segment.arrays.idTable.idAt(ordinal);
|
|
5573
|
+
if (tombstoned.has(docId)) segment.prepareTombstone(docId, void 0, void 0, ordinal)?.publish();
|
|
5574
|
+
}
|
|
5545
5575
|
function applyNewerTombstones(prepared) {
|
|
5546
5576
|
for (let at = 0; at < prepared.length; at++) {
|
|
5547
5577
|
if (!prepared[at].isNew) continue;
|
|
5548
|
-
|
|
5549
|
-
|
|
5550
|
-
|
|
5551
|
-
for (const docId of
|
|
5578
|
+
const { segment } = prepared[at];
|
|
5579
|
+
const tombstoned = newerTombstonesOf(prepared, at);
|
|
5580
|
+
if (!scansEveryId(segment, tombstoned)) {
|
|
5581
|
+
for (const docId of tombstoned) segment.tombstoneDocument(docId);
|
|
5582
|
+
continue;
|
|
5552
5583
|
}
|
|
5584
|
+
const held = segment.arrays.idTable.count;
|
|
5585
|
+
for (let ordinal = 0; ordinal < held; ordinal++) tombstoneScannedOrdinal(segment, tombstoned, ordinal);
|
|
5553
5586
|
}
|
|
5554
5587
|
}
|
|
5555
5588
|
async function applyNewerTombstonesInTurns(prepared) {
|
|
5556
|
-
let
|
|
5589
|
+
let steps = 0;
|
|
5557
5590
|
for (let at = 0; at < prepared.length; at++) {
|
|
5558
5591
|
if (!prepared[at].isNew) continue;
|
|
5559
|
-
|
|
5560
|
-
|
|
5561
|
-
|
|
5562
|
-
for (const docId of
|
|
5563
|
-
|
|
5564
|
-
|
|
5565
|
-
if (
|
|
5592
|
+
const { segment } = prepared[at];
|
|
5593
|
+
const tombstoned = newerTombstonesOf(prepared, at);
|
|
5594
|
+
if (!scansEveryId(segment, tombstoned)) {
|
|
5595
|
+
for (const docId of tombstoned) {
|
|
5596
|
+
segment.tombstoneDocument(docId);
|
|
5597
|
+
steps += 1;
|
|
5598
|
+
if (steps % TOMBSTONES_BETWEEN_YIELDS === 0) await yieldToEventLoop2();
|
|
5566
5599
|
}
|
|
5600
|
+
continue;
|
|
5601
|
+
}
|
|
5602
|
+
const held = segment.arrays.idTable.count;
|
|
5603
|
+
for (let ordinal = 0; ordinal < held; ordinal++) {
|
|
5604
|
+
tombstoneScannedOrdinal(segment, tombstoned, ordinal);
|
|
5605
|
+
steps += 1;
|
|
5606
|
+
if (steps % TOMBSTONES_BETWEEN_YIELDS === 0) await yieldToEventLoop2();
|
|
5567
5607
|
}
|
|
5568
5608
|
}
|
|
5569
5609
|
}
|
|
@@ -8237,6 +8277,9 @@ function wholePartitionsWhereMostChanged(manager, rules) {
|
|
|
8237
8277
|
return whole;
|
|
8238
8278
|
};
|
|
8239
8279
|
}
|
|
8280
|
+
function liveDocumentsOf(manager, targets) {
|
|
8281
|
+
return targets.map(({ partitionId }) => ({ partitionId, documentCount: manager.getPartition(partitionId).count() }));
|
|
8282
|
+
}
|
|
8240
8283
|
async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartitionsOf = noWholePartitions, priorCheckpointThatLimitsTheRecords, listedVectorFields = [], termHistoryOf, onCaptured) {
|
|
8241
8284
|
const whileNoBuildPassOrMutationRuns = (capture3) => whileGraphsStayComplete(vectorIndexes, () => whileNoMutationApplies([...indexState.partitions.values()], capture3));
|
|
8242
8285
|
return whileNoBuildPassOrMutationRuns(
|
|
@@ -8279,7 +8322,8 @@ async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartit
|
|
|
8279
8322
|
vectorPlans,
|
|
8280
8323
|
wholePartitions,
|
|
8281
8324
|
recordsLeftInLog,
|
|
8282
|
-
unreplayedCutoffs: cutoffs
|
|
8325
|
+
unreplayedCutoffs: cutoffs,
|
|
8326
|
+
liveDocuments: memoryMatchesTheTargets && cutoffs.size === 0 ? liveDocumentsOf(manager, targets) : null
|
|
8283
8327
|
};
|
|
8284
8328
|
} catch (error) {
|
|
8285
8329
|
for (const plan of vectorPlans.values()) plan.release();
|
|
@@ -8542,6 +8586,25 @@ async function pagedTombstonesFor(directory, older, written, deleted) {
|
|
|
8542
8586
|
return [...tombstones];
|
|
8543
8587
|
}
|
|
8544
8588
|
|
|
8589
|
+
// src/persistence/durability/stored-bytes.ts
|
|
8590
|
+
var NOTHING_SAVED = { bytes: 0, vectorBytes: 0, partitions: [] };
|
|
8591
|
+
function savedBytesOfManifest(manifest) {
|
|
8592
|
+
let bytes = 0;
|
|
8593
|
+
const partitions = [];
|
|
8594
|
+
for (const partition of manifest.partitions) {
|
|
8595
|
+
let partitionBytes = 0;
|
|
8596
|
+
for (const segment of partition.segments) partitionBytes += segment.bytes;
|
|
8597
|
+
partitions.push({ partitionId: partition.partitionId, bytes: partitionBytes });
|
|
8598
|
+
bytes += partitionBytes;
|
|
8599
|
+
}
|
|
8600
|
+
let vectorBytes = 0;
|
|
8601
|
+
for (const field of manifest.vectors) {
|
|
8602
|
+
for (const file of field.files) vectorBytes += file.bytes;
|
|
8603
|
+
for (const part of field.graphParts) vectorBytes += part.bytes;
|
|
8604
|
+
}
|
|
8605
|
+
return { bytes: bytes + vectorBytes, vectorBytes, partitions };
|
|
8606
|
+
}
|
|
8607
|
+
|
|
8545
8608
|
// src/persistence/durability/segment/build-segment.ts
|
|
8546
8609
|
function writeRecordsOf(documents, written) {
|
|
8547
8610
|
const seqNos = new Float64Array(documents.length);
|
|
@@ -10487,6 +10550,9 @@ async function mergePagedWindow(input) {
|
|
|
10487
10550
|
for (let older = 0; older < newer; older++) sources[older].segment.tombstoneDocument(docId);
|
|
10488
10551
|
}
|
|
10489
10552
|
}
|
|
10553
|
+
for (const docId of input.replacedAfterWindow ?? []) {
|
|
10554
|
+
for (const source of sources) source.segment.tombstoneDocument(docId);
|
|
10555
|
+
}
|
|
10490
10556
|
const tombstones = /* @__PURE__ */ new Set();
|
|
10491
10557
|
if (!input.dropsEveryTombstone) {
|
|
10492
10558
|
for (const source of sources) {
|
|
@@ -10580,49 +10646,34 @@ function retainedTombstones(frozen, contents) {
|
|
|
10580
10646
|
}
|
|
10581
10647
|
return [...retained];
|
|
10582
10648
|
}
|
|
10583
|
-
async function
|
|
10584
|
-
|
|
10585
|
-
if (segments.length <= compactionThreshold) {
|
|
10586
|
-
return { segments, nextSegmentId: input.nextSegmentId };
|
|
10587
|
-
}
|
|
10588
|
-
const paged = input.paged === true;
|
|
10589
|
-
const window = chooseWindow(segments, segments.length - compactionThreshold + 1, paged);
|
|
10590
|
-
if (window === null) {
|
|
10591
|
-
return { segments, nextSegmentId: input.nextSegmentId };
|
|
10592
|
-
}
|
|
10593
|
-
const windowRefs = segments.slice(window.start, window.start + window.windowLength);
|
|
10594
|
-
const replacing = (written2) => ({
|
|
10595
|
-
segments: [
|
|
10596
|
-
...segments.slice(0, window.start),
|
|
10597
|
-
...written2.segments,
|
|
10598
|
-
...segments.slice(window.start + window.windowLength)
|
|
10599
|
-
],
|
|
10600
|
-
nextSegmentId: written2.nextSegmentId
|
|
10601
|
-
});
|
|
10602
|
-
if (paged) {
|
|
10649
|
+
async function mergeSegmentWindow(input, window) {
|
|
10650
|
+
if (input.paged === true) {
|
|
10603
10651
|
const { directory } = input.store;
|
|
10604
|
-
if (directory === void 0) return
|
|
10605
|
-
return
|
|
10606
|
-
|
|
10607
|
-
|
|
10608
|
-
|
|
10609
|
-
|
|
10610
|
-
|
|
10611
|
-
|
|
10612
|
-
|
|
10613
|
-
|
|
10614
|
-
|
|
10615
|
-
|
|
10616
|
-
|
|
10617
|
-
|
|
10618
|
-
);
|
|
10652
|
+
if (directory === void 0) return null;
|
|
10653
|
+
return mergePagedWindow({
|
|
10654
|
+
store: input.store,
|
|
10655
|
+
directory,
|
|
10656
|
+
indexName: input.indexName,
|
|
10657
|
+
partitionId: input.partitionId,
|
|
10658
|
+
schema: input.schema,
|
|
10659
|
+
partialWordAnalysis: input.partialWordAnalysis,
|
|
10660
|
+
textAnalysis: input.textAnalysis,
|
|
10661
|
+
nextSegmentId: input.nextSegmentId,
|
|
10662
|
+
refs: window.refs,
|
|
10663
|
+
dropsEveryTombstone: window.startsTheList,
|
|
10664
|
+
...window.replacedAfterWindow === void 0 ? {} : { replacedAfterWindow: window.replacedAfterWindow }
|
|
10665
|
+
});
|
|
10619
10666
|
}
|
|
10620
10667
|
const contents = [];
|
|
10621
|
-
for (const ref of
|
|
10668
|
+
for (const ref of window.refs) contents.push(await readSegmentFile(input.store, ref));
|
|
10622
10669
|
const frozen = contents.map(frozenOf);
|
|
10623
10670
|
applyTimeOrder(frozen, contents);
|
|
10671
|
+
for (const docId of window.replacedAfterWindow ?? []) {
|
|
10672
|
+
for (const segment of frozen) segment.tombstoneDocument(docId);
|
|
10673
|
+
}
|
|
10624
10674
|
const writeRecords = writeRecordsOf2(frozen, contents);
|
|
10625
|
-
const
|
|
10675
|
+
const keepsTombstones = !(window.startsTheList && window.replacedAfterWindow !== void 0);
|
|
10676
|
+
return writeSegmentsOf({
|
|
10626
10677
|
store: input.store,
|
|
10627
10678
|
indexName: input.indexName,
|
|
10628
10679
|
partitionId: input.partitionId,
|
|
@@ -10631,11 +10682,33 @@ async function compactPartitionSegments(input) {
|
|
|
10631
10682
|
partialWordAnalysis: input.partialWordAnalysis,
|
|
10632
10683
|
textAnalysis: input.textAnalysis,
|
|
10633
10684
|
selections: frozen.map((segment) => ({ segment })),
|
|
10634
|
-
tombstones: retainedTombstones(frozen, contents),
|
|
10685
|
+
tombstones: keepsTombstones ? retainedTombstones(frozen, contents) : [],
|
|
10635
10686
|
mergesIntoOneFile: true,
|
|
10636
10687
|
...writeRecords === void 0 ? {} : { writeRecords }
|
|
10637
10688
|
});
|
|
10638
|
-
|
|
10689
|
+
}
|
|
10690
|
+
async function compactPartitionSegments(input) {
|
|
10691
|
+
const { segments, compactionThreshold } = input;
|
|
10692
|
+
if (segments.length <= compactionThreshold) {
|
|
10693
|
+
return { segments, nextSegmentId: input.nextSegmentId };
|
|
10694
|
+
}
|
|
10695
|
+
const window = chooseWindow(segments, segments.length - compactionThreshold + 1, input.paged === true);
|
|
10696
|
+
if (window === null) {
|
|
10697
|
+
return { segments, nextSegmentId: input.nextSegmentId };
|
|
10698
|
+
}
|
|
10699
|
+
const written = await mergeSegmentWindow(input, {
|
|
10700
|
+
refs: segments.slice(window.start, window.start + window.windowLength),
|
|
10701
|
+
startsTheList: window.start === 0
|
|
10702
|
+
});
|
|
10703
|
+
if (written === null) return { segments, nextSegmentId: input.nextSegmentId };
|
|
10704
|
+
return {
|
|
10705
|
+
segments: [
|
|
10706
|
+
...segments.slice(0, window.start),
|
|
10707
|
+
...written.segments,
|
|
10708
|
+
...segments.slice(window.start + window.windowLength)
|
|
10709
|
+
],
|
|
10710
|
+
nextSegmentId: written.nextSegmentId
|
|
10711
|
+
};
|
|
10639
10712
|
}
|
|
10640
10713
|
|
|
10641
10714
|
// src/persistence/durability/segment/manifest-encode.ts
|
|
@@ -10684,6 +10757,161 @@ function encodeSegmentManifest(manifest) {
|
|
|
10684
10757
|
return packChecksummedEnvelope([payload], MANIFEST_ENVELOPE_VERSION, "segment manifest");
|
|
10685
10758
|
}
|
|
10686
10759
|
|
|
10760
|
+
// src/persistence/paged/segment-scan.ts
|
|
10761
|
+
async function openPagedSegmentScan(directory, ref) {
|
|
10762
|
+
const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
|
|
10763
|
+
const cache = createReadCache(MERGE_READ_CACHE_BYTES);
|
|
10764
|
+
const segment = createPagedFrozenSegment(snapshot, cache);
|
|
10765
|
+
const { paged } = segment;
|
|
10766
|
+
if (paged === null) loadFailed(`The segment "${ref.key}" opened without its file`);
|
|
10767
|
+
const ids = segment.arrays.idTable;
|
|
10768
|
+
const tombstones = new Set(snapshot.tombstones);
|
|
10769
|
+
return {
|
|
10770
|
+
count: snapshot.documentCount,
|
|
10771
|
+
schema: snapshot.schema,
|
|
10772
|
+
tombstones,
|
|
10773
|
+
idAt: (ordinal) => ids.idAt(ordinal),
|
|
10774
|
+
holds: (docId) => ids.ordinalOf(docId) >= 0,
|
|
10775
|
+
storedDocumentAt: (ordinal) => decodeStoredDocument(paged.encodedDocumentAt(ordinal), snapshot.discardedFields),
|
|
10776
|
+
close: () => {
|
|
10777
|
+
paged.close();
|
|
10778
|
+
cache.clear();
|
|
10779
|
+
}
|
|
10780
|
+
};
|
|
10781
|
+
}
|
|
10782
|
+
async function checkPagedSegmentFully(directory, ref) {
|
|
10783
|
+
const listing = await pagedListingOf(directory, ref);
|
|
10784
|
+
await requirePagedPayloadChecksum(directory, listing);
|
|
10785
|
+
const snapshot = openPagedSnapshot(listing);
|
|
10786
|
+
const cache = createReadCache(MERGE_READ_CACHE_BYTES);
|
|
10787
|
+
const segment = createPagedFrozenSegment(snapshot, cache);
|
|
10788
|
+
const { paged } = segment;
|
|
10789
|
+
if (paged === null) loadFailed(`The segment "${ref.key}" opened without its file`);
|
|
10790
|
+
try {
|
|
10791
|
+
await checkPagedContents(segment, cache);
|
|
10792
|
+
return snapshot.schema;
|
|
10793
|
+
} finally {
|
|
10794
|
+
paged.close();
|
|
10795
|
+
cache.clear();
|
|
10796
|
+
}
|
|
10797
|
+
}
|
|
10798
|
+
async function bytesThatSegmentsHoldInMemory(directory, manifest) {
|
|
10799
|
+
let bytes = 0;
|
|
10800
|
+
for (const partition of manifest.partitions) {
|
|
10801
|
+
for (const ref of partition.segments) {
|
|
10802
|
+
const paged = await isPagedSegmentFile(directory, ref.key);
|
|
10803
|
+
bytes += paged ? bytesOfResidentSections(await pagedListingOf(directory, ref)) : ref.bytes;
|
|
10804
|
+
}
|
|
10805
|
+
}
|
|
10806
|
+
return bytes;
|
|
10807
|
+
}
|
|
10808
|
+
function countsAsLatestHolder(scans, at, docId) {
|
|
10809
|
+
for (let later = at + 1; later < scans.length; later++) {
|
|
10810
|
+
if (scans[later].tombstones.has(docId) || scans[later].holds(docId)) return false;
|
|
10811
|
+
}
|
|
10812
|
+
return true;
|
|
10813
|
+
}
|
|
10814
|
+
|
|
10815
|
+
// src/persistence/durability/segment/removed-documents.ts
|
|
10816
|
+
var decoder2 = new TextDecoder();
|
|
10817
|
+
async function readSegmentIds(store, ref) {
|
|
10818
|
+
const { directory } = store;
|
|
10819
|
+
if (directory !== void 0 && await isPagedSegmentFile(directory, ref.key)) {
|
|
10820
|
+
const scan = await openPagedSegmentScan(directory, ref);
|
|
10821
|
+
try {
|
|
10822
|
+
const ids2 = new Array(scan.count);
|
|
10823
|
+
for (let ordinal = 0; ordinal < scan.count; ordinal++) ids2[ordinal] = scan.idAt(ordinal);
|
|
10824
|
+
return { ids: ids2, tombstones: [...scan.tombstones] };
|
|
10825
|
+
} finally {
|
|
10826
|
+
scan.close();
|
|
10827
|
+
}
|
|
10828
|
+
}
|
|
10829
|
+
const contents = await readSegmentFile(store, ref);
|
|
10830
|
+
const table = contents.arrays.idTable;
|
|
10831
|
+
const ids = new Array(contents.arrays.documentCount);
|
|
10832
|
+
for (let ordinal = 0; ordinal < ids.length; ordinal++) {
|
|
10833
|
+
ids[ordinal] = decoder2.decode(table.blob.subarray(table.offsets[ordinal], table.offsets[ordinal + 1]));
|
|
10834
|
+
}
|
|
10835
|
+
return { ids, tombstones: contents.tombstones };
|
|
10836
|
+
}
|
|
10837
|
+
async function documentsReplacedLater(store, segments) {
|
|
10838
|
+
const later = /* @__PURE__ */ new Set();
|
|
10839
|
+
const replaced = new Array(segments.length);
|
|
10840
|
+
for (let at = segments.length - 1; at >= 0; at--) {
|
|
10841
|
+
const { ids, tombstones } = await readSegmentIds(store, segments[at]);
|
|
10842
|
+
const found = [];
|
|
10843
|
+
for (const docId of ids) if (later.has(docId)) found.push(docId);
|
|
10844
|
+
replaced[at] = found;
|
|
10845
|
+
if (at === 0) break;
|
|
10846
|
+
for (const docId of ids) later.add(docId);
|
|
10847
|
+
for (const docId of tombstones) later.add(docId);
|
|
10848
|
+
}
|
|
10849
|
+
return replaced;
|
|
10850
|
+
}
|
|
10851
|
+
function exceedsRemovedShare(removed, held) {
|
|
10852
|
+
return held > 0 && removed > held * REWRITE_ABOVE_REMOVED_DOCUMENT_SHARE;
|
|
10853
|
+
}
|
|
10854
|
+
function freedBytesOf(ref, replacedCount) {
|
|
10855
|
+
return ref.docCount === 0 ? 0 : ref.bytes * replacedCount / ref.docCount;
|
|
10856
|
+
}
|
|
10857
|
+
function chooseRewrites(segments, replaced, budget) {
|
|
10858
|
+
let held = 0;
|
|
10859
|
+
let removed = 0;
|
|
10860
|
+
for (let at = 0; at < segments.length; at++) {
|
|
10861
|
+
held += segments[at].docCount;
|
|
10862
|
+
removed += replaced[at].length;
|
|
10863
|
+
}
|
|
10864
|
+
const candidates = [];
|
|
10865
|
+
for (let at = 0; at < segments.length; at++) if (replaced[at].length > 0) candidates.push(at);
|
|
10866
|
+
candidates.sort(
|
|
10867
|
+
(left, right) => freedBytesOf(segments[right], replaced[right].length) - freedBytesOf(segments[left], replaced[left].length)
|
|
10868
|
+
);
|
|
10869
|
+
const chosen = [];
|
|
10870
|
+
for (const at of candidates) {
|
|
10871
|
+
if (!exceedsRemovedShare(removed, held)) break;
|
|
10872
|
+
const bytes = segments[at].bytes;
|
|
10873
|
+
if (chosen.length > 0 && budget.spentBytes + bytes > budget.limitBytes) continue;
|
|
10874
|
+
chosen.push(at);
|
|
10875
|
+
budget.spentBytes += bytes;
|
|
10876
|
+
held -= replaced[at].length;
|
|
10877
|
+
removed -= replaced[at].length;
|
|
10878
|
+
}
|
|
10879
|
+
return chosen.sort((left, right) => left - right);
|
|
10880
|
+
}
|
|
10881
|
+
function holdsSomething(ref) {
|
|
10882
|
+
return ref.docCount > 0 || ref.tombstoneCount > 0;
|
|
10883
|
+
}
|
|
10884
|
+
async function rewriteRemovedDocuments(input) {
|
|
10885
|
+
const { segments, liveDocumentCount } = input;
|
|
10886
|
+
const unchanged = { segments, nextSegmentId: input.nextSegmentId };
|
|
10887
|
+
if (liveDocumentCount === void 0) return unchanged;
|
|
10888
|
+
let held = 0;
|
|
10889
|
+
for (const ref of segments) held += ref.docCount;
|
|
10890
|
+
if (!exceedsRemovedShare(held - liveDocumentCount, held)) return unchanged;
|
|
10891
|
+
const replaced = await documentsReplacedLater(input.store, segments);
|
|
10892
|
+
const chosen = new Set(chooseRewrites(segments, replaced, input.budget));
|
|
10893
|
+
if (chosen.size === 0) return unchanged;
|
|
10894
|
+
const rewritten = [];
|
|
10895
|
+
let nextSegmentId = input.nextSegmentId;
|
|
10896
|
+
for (let at = 0; at < segments.length; at++) {
|
|
10897
|
+
if (!chosen.has(at)) {
|
|
10898
|
+
rewritten.push(segments[at]);
|
|
10899
|
+
continue;
|
|
10900
|
+
}
|
|
10901
|
+
const written = await mergeSegmentWindow(
|
|
10902
|
+
{ ...input, nextSegmentId },
|
|
10903
|
+
{ refs: [segments[at]], startsTheList: rewritten.length === 0, replacedAfterWindow: replaced[at] }
|
|
10904
|
+
);
|
|
10905
|
+
if (written === null) {
|
|
10906
|
+
rewritten.push(segments[at]);
|
|
10907
|
+
continue;
|
|
10908
|
+
}
|
|
10909
|
+
nextSegmentId = written.nextSegmentId;
|
|
10910
|
+
rewritten.push(...written.segments.filter(holdsSomething));
|
|
10911
|
+
}
|
|
10912
|
+
return { segments: rewritten, nextSegmentId };
|
|
10913
|
+
}
|
|
10914
|
+
|
|
10687
10915
|
// src/persistence/durability/segment/write.ts
|
|
10688
10916
|
function textAnalysisOf(config) {
|
|
10689
10917
|
let resolved = null;
|
|
@@ -10806,8 +11034,21 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
|
|
|
10806
11034
|
compactionThreshold: context.compactionThreshold,
|
|
10807
11035
|
paged: context.input.metadata.textStorage === "disk"
|
|
10808
11036
|
});
|
|
10809
|
-
|
|
10810
|
-
|
|
11037
|
+
const rewritten = await rewriteRemovedDocuments({
|
|
11038
|
+
store: context.input.store,
|
|
11039
|
+
indexName: context.indexName,
|
|
11040
|
+
partitionId,
|
|
11041
|
+
schema: context.input.metadata.schema,
|
|
11042
|
+
partialWordAnalysis: context.partialWordAnalysis,
|
|
11043
|
+
textAnalysis: context.textAnalysis,
|
|
11044
|
+
segments: compacted.segments,
|
|
11045
|
+
nextSegmentId: compacted.nextSegmentId,
|
|
11046
|
+
paged: context.input.metadata.textStorage === "disk",
|
|
11047
|
+
liveDocumentCount: context.input.liveDocuments?.find((entry) => entry.partitionId === partitionId)?.documentCount,
|
|
11048
|
+
budget: context.rewriteBudget
|
|
11049
|
+
});
|
|
11050
|
+
segments = rewritten.segments;
|
|
11051
|
+
nextSegmentId = rewritten.nextSegmentId;
|
|
10811
11052
|
return { partitionId, nextSegmentId, segments };
|
|
10812
11053
|
}
|
|
10813
11054
|
function carryForwardUncheckpointedPartitions(prior, targets, partitions, checkpointByPartition) {
|
|
@@ -10832,7 +11073,11 @@ async function writeCheckpointSegments(input) {
|
|
|
10832
11073
|
partialWordAnalysis: partialWordAnalysisOf(config, textAnalysis),
|
|
10833
11074
|
textAnalysis,
|
|
10834
11075
|
vectorFieldPaths: new Set(extractVectorFieldsFromSchema(config.schema).keys()),
|
|
10835
|
-
compactionThreshold: resolveCompactionThreshold(input.compactionThreshold)
|
|
11076
|
+
compactionThreshold: resolveCompactionThreshold(input.compactionThreshold),
|
|
11077
|
+
rewriteBudget: {
|
|
11078
|
+
spentBytes: 0,
|
|
11079
|
+
limitBytes: input.rewriteBytesPerCheckpoint ?? DEFAULT_REWRITE_BYTES_PER_CHECKPOINT
|
|
11080
|
+
}
|
|
10836
11081
|
};
|
|
10837
11082
|
const checkpointByPartition = /* @__PURE__ */ new Map();
|
|
10838
11083
|
for (const target of input.targets) checkpointByPartition.set(target.partitionId, { ...target });
|
|
@@ -10869,6 +11114,7 @@ async function writeCheckpointSegments(input) {
|
|
|
10869
11114
|
async function commitCheckpointManifest(store, metadata, written, vectors) {
|
|
10870
11115
|
const indexName = metadata.indexName;
|
|
10871
11116
|
const manifest = {
|
|
11117
|
+
version: SEGMENT_MANIFEST_VERSION,
|
|
10872
11118
|
schema: metadata.schema,
|
|
10873
11119
|
language: metadata.language,
|
|
10874
11120
|
...metadata.analysisGeneration ? { analysisGeneration: metadata.analysisGeneration } : {},
|
|
@@ -10878,9 +11124,9 @@ async function commitCheckpointManifest(store, metadata, written, vectors) {
|
|
|
10878
11124
|
};
|
|
10879
11125
|
const packed = await encodeSegmentManifest(manifest);
|
|
10880
11126
|
await store.write(manifestKey(indexName), packed.chunks);
|
|
10881
|
-
return {
|
|
11127
|
+
return { manifest, saved: savedBytesOfManifest(manifest) };
|
|
10882
11128
|
}
|
|
10883
|
-
async function
|
|
11129
|
+
async function checkpointGarbage(store, indexName, manifest) {
|
|
10884
11130
|
const referenced = manifestReferencedKeys(manifest);
|
|
10885
11131
|
const stored = await store.list(segmentsPrefix(indexName));
|
|
10886
11132
|
return [...stored.filter((key) => !referenced.has(key)), snapshotBundleKey(indexName)];
|
|
@@ -10902,7 +11148,8 @@ function workerCanWriteEverySegment(input) {
|
|
|
10902
11148
|
}
|
|
10903
11149
|
async function writeIndexCheckpointSegments(input) {
|
|
10904
11150
|
const { directory, metadata, targets, compactionThreshold, prior, wholePartitions, capturedPartitions } = input;
|
|
10905
|
-
const { keepsWriteRecords } = input;
|
|
11151
|
+
const { keepsWriteRecords, liveDocuments, rewriteBytesPerCheckpoint } = input;
|
|
11152
|
+
const counted = liveDocuments === null ? {} : { liveDocuments, rewriteBytesPerCheckpoint };
|
|
10906
11153
|
const offloaded = workerCanWriteEverySegment(input) ? await runCheckpointOnWorker({
|
|
10907
11154
|
root: directory.root,
|
|
10908
11155
|
metadata,
|
|
@@ -10910,7 +11157,8 @@ async function writeIndexCheckpointSegments(input) {
|
|
|
10910
11157
|
compactionThreshold,
|
|
10911
11158
|
prior,
|
|
10912
11159
|
capturedPartitions: [...capturedPartitions],
|
|
10913
|
-
keepsWriteRecords
|
|
11160
|
+
keepsWriteRecords,
|
|
11161
|
+
...counted
|
|
10914
11162
|
}) : null;
|
|
10915
11163
|
if (offloaded !== null) {
|
|
10916
11164
|
return offloaded;
|
|
@@ -10924,7 +11172,8 @@ async function writeIndexCheckpointSegments(input) {
|
|
|
10924
11172
|
prior,
|
|
10925
11173
|
wholePartitions,
|
|
10926
11174
|
capturedPartitions,
|
|
10927
|
-
keepsWriteRecords
|
|
11175
|
+
keepsWriteRecords,
|
|
11176
|
+
...counted
|
|
10928
11177
|
});
|
|
10929
11178
|
}
|
|
10930
11179
|
|
|
@@ -11007,61 +11256,6 @@ async function startReadingOpeningSectionsAhead(directory, manifest) {
|
|
|
11007
11256
|
}
|
|
11008
11257
|
}
|
|
11009
11258
|
|
|
11010
|
-
// src/persistence/paged/segment-scan.ts
|
|
11011
|
-
async function openPagedSegmentScan(directory, ref) {
|
|
11012
|
-
const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
|
|
11013
|
-
const cache = createReadCache(MERGE_READ_CACHE_BYTES);
|
|
11014
|
-
const segment = createPagedFrozenSegment(snapshot, cache);
|
|
11015
|
-
const { paged } = segment;
|
|
11016
|
-
if (paged === null) loadFailed(`The segment "${ref.key}" opened without its file`);
|
|
11017
|
-
const ids = segment.arrays.idTable;
|
|
11018
|
-
const tombstones = new Set(snapshot.tombstones);
|
|
11019
|
-
return {
|
|
11020
|
-
count: snapshot.documentCount,
|
|
11021
|
-
schema: snapshot.schema,
|
|
11022
|
-
tombstones,
|
|
11023
|
-
idAt: (ordinal) => ids.idAt(ordinal),
|
|
11024
|
-
holds: (docId) => ids.ordinalOf(docId) >= 0,
|
|
11025
|
-
storedDocumentAt: (ordinal) => decodeStoredDocument(paged.encodedDocumentAt(ordinal), snapshot.discardedFields),
|
|
11026
|
-
close: () => {
|
|
11027
|
-
paged.close();
|
|
11028
|
-
cache.clear();
|
|
11029
|
-
}
|
|
11030
|
-
};
|
|
11031
|
-
}
|
|
11032
|
-
async function checkPagedSegmentFully(directory, ref) {
|
|
11033
|
-
const listing = await pagedListingOf(directory, ref);
|
|
11034
|
-
await requirePagedPayloadChecksum(directory, listing);
|
|
11035
|
-
const snapshot = openPagedSnapshot(listing);
|
|
11036
|
-
const cache = createReadCache(MERGE_READ_CACHE_BYTES);
|
|
11037
|
-
const segment = createPagedFrozenSegment(snapshot, cache);
|
|
11038
|
-
const { paged } = segment;
|
|
11039
|
-
if (paged === null) loadFailed(`The segment "${ref.key}" opened without its file`);
|
|
11040
|
-
try {
|
|
11041
|
-
await checkPagedContents(segment, cache);
|
|
11042
|
-
return snapshot.schema;
|
|
11043
|
-
} finally {
|
|
11044
|
-
paged.close();
|
|
11045
|
-
cache.clear();
|
|
11046
|
-
}
|
|
11047
|
-
}
|
|
11048
|
-
async function bytesThatSegmentsHoldInMemory(directory, manifest) {
|
|
11049
|
-
let bytes = 0;
|
|
11050
|
-
for (const partition of manifest.partitions) {
|
|
11051
|
-
for (const ref of partition.segments) {
|
|
11052
|
-
const paged = await isPagedSegmentFile(directory, ref.key);
|
|
11053
|
-
bytes += paged ? bytesOfResidentSections(await pagedListingOf(directory, ref)) : ref.bytes;
|
|
11054
|
-
}
|
|
11055
|
-
}
|
|
11056
|
-
return bytes;
|
|
11057
|
-
}
|
|
11058
|
-
function countsAsLatestHolder(scans, at, docId) {
|
|
11059
|
-
for (let later = at + 1; later < scans.length; later++) {
|
|
11060
|
-
if (scans[later].tombstones.has(docId) || scans[later].holds(docId)) return false;
|
|
11061
|
-
}
|
|
11062
|
-
return true;
|
|
11063
|
-
}
|
|
11064
|
-
|
|
11065
11259
|
// src/persistence/durability/segment/load-partition.ts
|
|
11066
11260
|
function requireAgreedSchema(agreed, schema, key) {
|
|
11067
11261
|
for (const [field, type] of Object.entries(schema)) {
|
|
@@ -11075,6 +11269,20 @@ function requireAgreedSchema(agreed, schema, key) {
|
|
|
11075
11269
|
}
|
|
11076
11270
|
}
|
|
11077
11271
|
}
|
|
11272
|
+
function requireListedCounts(ref, documentCount, tombstoneCount) {
|
|
11273
|
+
if (ref.docCount === documentCount && ref.tombstoneCount === tombstoneCount) return;
|
|
11274
|
+
throw new NarsilError(
|
|
11275
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
11276
|
+
`The segment "${ref.key}" holds ${documentCount} documents and ${tombstoneCount} tombstones where the manifest lists ${ref.docCount} and ${ref.tombstoneCount}`,
|
|
11277
|
+
{
|
|
11278
|
+
key: ref.key,
|
|
11279
|
+
documentCount,
|
|
11280
|
+
tombstoneCount,
|
|
11281
|
+
listedDocCount: ref.docCount,
|
|
11282
|
+
listedTombstoneCount: ref.tombstoneCount
|
|
11283
|
+
}
|
|
11284
|
+
);
|
|
11285
|
+
}
|
|
11078
11286
|
function removeWhatTheSegmentReplaces(manager, segment, contents) {
|
|
11079
11287
|
manager.beginBatchRemove();
|
|
11080
11288
|
try {
|
|
@@ -11109,6 +11317,7 @@ function removeFromPartition(manager, partitionId, docIds) {
|
|
|
11109
11317
|
async function attachPagedSegment(directory, ref, partitionId, manager, options) {
|
|
11110
11318
|
if (options.decodesEveryDocument) await checkPagedSegmentFully(directory, ref);
|
|
11111
11319
|
const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
|
|
11320
|
+
requireListedCounts(ref, snapshot.documentCount, snapshot.tombstones.length);
|
|
11112
11321
|
requireAgreedSchema(options.schema, snapshot.schema, ref.key);
|
|
11113
11322
|
const segment = createPagedFrozenSegment(snapshot);
|
|
11114
11323
|
removeFromPartition(manager, partitionId, snapshot.tombstones);
|
|
@@ -11154,6 +11363,7 @@ async function attachPartitionSegments(store, partition, manager, options) {
|
|
|
11154
11363
|
}
|
|
11155
11364
|
}
|
|
11156
11365
|
const contents = await readSegmentFile(store, ref);
|
|
11366
|
+
requireListedCounts(ref, contents.arrays.documentCount, contents.tombstones.length);
|
|
11157
11367
|
requireAgreedSchema(options.schema, contents.schema, ref.key);
|
|
11158
11368
|
if (options.decodesEveryDocument) decodeEveryStoredDocument(contents.arrays.documentTable);
|
|
11159
11369
|
const segment = createSharedFrozenSegment(shareSegmentArrays(contents.arrays, manager.partialWordAnalysis));
|
|
@@ -11612,23 +11822,28 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
|
|
|
11612
11822
|
metadata,
|
|
11613
11823
|
targets,
|
|
11614
11824
|
compactionThreshold: input.compactionThreshold,
|
|
11825
|
+
rewriteBytesPerCheckpoint: input.rewriteBytesPerCheckpoint,
|
|
11615
11826
|
canOffload: input.canOffload,
|
|
11616
11827
|
prior: priorForLayout(priorManifest, capture3.partitionCount),
|
|
11617
11828
|
wholePartitions: capture3.wholePartitions.contents,
|
|
11618
11829
|
capturedPartitions: capture3.wholePartitions.captured,
|
|
11619
|
-
keepsWriteRecords: writeRecords !== null
|
|
11830
|
+
keepsWriteRecords: writeRecords !== null,
|
|
11831
|
+
liveDocuments: capture3.liveDocuments
|
|
11620
11832
|
})
|
|
11621
11833
|
);
|
|
11622
11834
|
await makeEveryAppliedMutationDurable(input.indexState, input.failPartition);
|
|
11623
11835
|
if (aTargetFailed(input.indexState, targets)) return 0;
|
|
11624
11836
|
const written = await commitCheckpointManifest(store, metadata, segments, liveVectors.vectors);
|
|
11837
|
+
input.recordSavedBytes(written.saved);
|
|
11625
11838
|
input.hooks.onManifestCommitted(input.indexName, metadata.analysisGeneration ?? 0);
|
|
11626
11839
|
recordWrittenVectors(vectorIndexes, liveVectors.written);
|
|
11627
11840
|
await adoptVectorLayouts(store, vectorIndexes, liveVectors.layouts);
|
|
11628
11841
|
const changes = changesSinceCapture();
|
|
11629
11842
|
const droppedThisCheckpoint = changes !== null && capture3.recordsLeftInLog === 0 ? new Set(await adoptWrittenSegments(input, manager, segments.partitions, changes)) : /* @__PURE__ */ new Set();
|
|
11843
|
+
await writeCheckpointMetadata(input, documentCount, capture3.partitionCount, written.saved.bytes);
|
|
11630
11844
|
if (!input.holdsCheckpointFiles()) {
|
|
11631
|
-
const
|
|
11845
|
+
const garbage = await checkpointGarbage(store, input.indexName, written.manifest);
|
|
11846
|
+
const removable = await garbageThatNoFieldStillReads(store, vectorIndexes, garbage);
|
|
11632
11847
|
const stillRead = metadata.textStorage === "disk" ? keysThatSegmentsStillRead(manager) : /* @__PURE__ */ new Set();
|
|
11633
11848
|
const keptForSync = await input.keepsCheckpointKey();
|
|
11634
11849
|
await removeCheckpointGarbage(
|
|
@@ -11636,19 +11851,6 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
|
|
|
11636
11851
|
removable.filter((key) => !droppedThisCheckpoint.has(key) && !stillRead.has(key) && !keptForSync(key))
|
|
11637
11852
|
);
|
|
11638
11853
|
}
|
|
11639
|
-
const checkpointDocumentCount = documentCount;
|
|
11640
|
-
await input.queueMetadataWrite(input.indexName, async () => {
|
|
11641
|
-
const checkpointMetadata = input.hooks.buildMetadata(input.indexName, checkpointDocumentCount);
|
|
11642
|
-
if (checkpointMetadata === void 0) {
|
|
11643
|
-
return;
|
|
11644
|
-
}
|
|
11645
|
-
const bytes = await writeMetadataEnvelope(
|
|
11646
|
-
{ ...checkpointMetadata, partitionCount: capture3.partitionCount },
|
|
11647
|
-
{ checksum: true }
|
|
11648
|
-
);
|
|
11649
|
-
await input.directory.atomicWrite(`${input.indexName}/meta`, bytes);
|
|
11650
|
-
input.hooks.recordCheckpoint?.(input.indexName, checkpointDocumentCount, capture3.partitionCount);
|
|
11651
|
-
});
|
|
11652
11854
|
const movedIntoHistoryAtMs = input.hooks.keepsSyncHistory?.(input.indexName) === true ? Date.now() : null;
|
|
11653
11855
|
await truncateCoveredSegments(input.directory, input.indexName, targets, movedIntoHistoryAtMs);
|
|
11654
11856
|
if (movedIntoHistoryAtMs !== null) input.hooks.onHistoryMoved?.(input.indexName);
|
|
@@ -11666,6 +11868,18 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
|
|
|
11666
11868
|
const anotherIsDue = input.mutationThreshold > 0 && eligibleBeyond >= input.mutationThreshold;
|
|
11667
11869
|
return anotherIsDue ? eligibleBeyond : capture3.recordsLeftInLog;
|
|
11668
11870
|
}
|
|
11871
|
+
async function writeCheckpointMetadata(input, documentCount, partitionCount, savedBytes) {
|
|
11872
|
+
const { indexName, hooks } = input;
|
|
11873
|
+
await input.queueMetadataWrite(indexName, async () => {
|
|
11874
|
+
const checkpointMetadata = hooks.buildMetadata(indexName, documentCount, savedBytes);
|
|
11875
|
+
if (checkpointMetadata === void 0) {
|
|
11876
|
+
return;
|
|
11877
|
+
}
|
|
11878
|
+
const bytes = await writeMetadataEnvelope({ ...checkpointMetadata, partitionCount }, { checksum: true });
|
|
11879
|
+
await input.directory.atomicWrite(`${indexName}/meta`, bytes);
|
|
11880
|
+
hooks.recordCheckpoint?.(indexName, documentCount, partitionCount, savedBytes);
|
|
11881
|
+
});
|
|
11882
|
+
}
|
|
11669
11883
|
function recordsBeyond(indexState, targets, cutoffs) {
|
|
11670
11884
|
let records = 0;
|
|
11671
11885
|
for (const [partitionId, partition] of indexState.partitions) {
|
|
@@ -11830,6 +12044,258 @@ function createHeldCheckpoints(hooks) {
|
|
|
11830
12044
|
};
|
|
11831
12045
|
}
|
|
11832
12046
|
|
|
12047
|
+
// src/persistence/durability/index-stored-sizes.ts
|
|
12048
|
+
function refreshStoredSizes(sizes) {
|
|
12049
|
+
const known = sizes.saved !== null || !sizes.measuring && sizes.held;
|
|
12050
|
+
sizes.known = known;
|
|
12051
|
+
if (known) {
|
|
12052
|
+
sizes.savedBytes = sizes.saved?.bytes ?? 0;
|
|
12053
|
+
sizes.vectorSavedBytes = sizes.saved?.vectorBytes ?? 0;
|
|
12054
|
+
sizes.logBytes = sizes.tracksLog ? sizes.logged?.bytes ?? 0 : void 0;
|
|
12055
|
+
return;
|
|
12056
|
+
}
|
|
12057
|
+
sizes.savedBytes = sizes.measuring ? void 0 : sizes.recorded;
|
|
12058
|
+
sizes.vectorSavedBytes = void 0;
|
|
12059
|
+
sizes.logBytes = void 0;
|
|
12060
|
+
}
|
|
12061
|
+
function savedBytesByPartitionOf(saved) {
|
|
12062
|
+
const byPartition = [];
|
|
12063
|
+
if (saved === null) return byPartition;
|
|
12064
|
+
for (const { partitionId, bytes } of saved.partitions) byPartition[partitionId] = bytes;
|
|
12065
|
+
return byPartition;
|
|
12066
|
+
}
|
|
12067
|
+
function createTrackedSizes(tracksLog) {
|
|
12068
|
+
const sizes = {
|
|
12069
|
+
savedBytes: void 0,
|
|
12070
|
+
logBytes: void 0,
|
|
12071
|
+
vectorSavedBytes: void 0,
|
|
12072
|
+
saved: null,
|
|
12073
|
+
savedBytesByPartition: [],
|
|
12074
|
+
measuring: false,
|
|
12075
|
+
held: false,
|
|
12076
|
+
recorded: void 0,
|
|
12077
|
+
logged: null,
|
|
12078
|
+
known: false,
|
|
12079
|
+
tracksLog,
|
|
12080
|
+
addTo(stats) {
|
|
12081
|
+
if (!sizes.known) return;
|
|
12082
|
+
const saved = sizes.savedBytesByPartition;
|
|
12083
|
+
const logged = sizes.tracksLog ? sizes.logged?.bytesByPartition ?? [] : null;
|
|
12084
|
+
for (const partition of stats) {
|
|
12085
|
+
partition.savedBytes = saved[partition.partitionId] ?? 0;
|
|
12086
|
+
if (logged !== null) partition.logBytes = logged[partition.partitionId] ?? 0;
|
|
12087
|
+
}
|
|
12088
|
+
},
|
|
12089
|
+
recordSavedBytes(bytes) {
|
|
12090
|
+
sizes.recorded = bytes;
|
|
12091
|
+
refreshStoredSizes(sizes);
|
|
12092
|
+
}
|
|
12093
|
+
};
|
|
12094
|
+
return sizes;
|
|
12095
|
+
}
|
|
12096
|
+
function createStoredSizesRecord(tracksLog) {
|
|
12097
|
+
const sizesByIndex = /* @__PURE__ */ new Map();
|
|
12098
|
+
function of(indexName) {
|
|
12099
|
+
let sizes = sizesByIndex.get(indexName);
|
|
12100
|
+
if (sizes === void 0) {
|
|
12101
|
+
sizes = createTrackedSizes(tracksLog);
|
|
12102
|
+
sizesByIndex.set(indexName, sizes);
|
|
12103
|
+
}
|
|
12104
|
+
return sizes;
|
|
12105
|
+
}
|
|
12106
|
+
function forget(indexName) {
|
|
12107
|
+
const sizes = sizesByIndex.get(indexName);
|
|
12108
|
+
if (sizes === void 0) return;
|
|
12109
|
+
sizes.saved = null;
|
|
12110
|
+
sizes.savedBytesByPartition = [];
|
|
12111
|
+
sizes.measuring = false;
|
|
12112
|
+
sizes.held = false;
|
|
12113
|
+
refreshStoredSizes(sizes);
|
|
12114
|
+
}
|
|
12115
|
+
return {
|
|
12116
|
+
of,
|
|
12117
|
+
setSaved(indexName, saved) {
|
|
12118
|
+
const sizes = of(indexName);
|
|
12119
|
+
sizes.saved = saved;
|
|
12120
|
+
sizes.savedBytesByPartition = savedBytesByPartitionOf(saved);
|
|
12121
|
+
refreshStoredSizes(sizes);
|
|
12122
|
+
},
|
|
12123
|
+
setMeasuring(indexName, measuring) {
|
|
12124
|
+
const sizes = of(indexName);
|
|
12125
|
+
sizes.measuring = measuring;
|
|
12126
|
+
refreshStoredSizes(sizes);
|
|
12127
|
+
},
|
|
12128
|
+
hold(indexName) {
|
|
12129
|
+
const sizes = of(indexName);
|
|
12130
|
+
if (!sizes.held) {
|
|
12131
|
+
sizes.held = true;
|
|
12132
|
+
refreshStoredSizes(sizes);
|
|
12133
|
+
}
|
|
12134
|
+
return sizes;
|
|
12135
|
+
},
|
|
12136
|
+
forget,
|
|
12137
|
+
drop(indexName) {
|
|
12138
|
+
forget(indexName);
|
|
12139
|
+
sizesByIndex.delete(indexName);
|
|
12140
|
+
}
|
|
12141
|
+
};
|
|
12142
|
+
}
|
|
12143
|
+
|
|
12144
|
+
// src/persistence/durability/log-bytes.ts
|
|
12145
|
+
var LOG_SEGMENT_KEY = /^(.+)\/wal\/(\d+)\/\d{16}$/;
|
|
12146
|
+
var HISTORY_SEGMENT_KEY = /^(.+)\/history\/(\d+)\/\d{16}-\d{16}$/;
|
|
12147
|
+
function loggedFileOf(key) {
|
|
12148
|
+
const match = LOG_SEGMENT_KEY.exec(key) ?? HISTORY_SEGMENT_KEY.exec(key);
|
|
12149
|
+
if (match === null) return null;
|
|
12150
|
+
const partitionId = Number.parseInt(match[2], 10);
|
|
12151
|
+
return Number.isSafeInteger(partitionId) ? { indexName: match[1], partitionId } : null;
|
|
12152
|
+
}
|
|
12153
|
+
function emptyLog() {
|
|
12154
|
+
return { files: /* @__PURE__ */ new Map(), bytes: 0, bytesByPartition: [], sizes: null };
|
|
12155
|
+
}
|
|
12156
|
+
function addBytes(log, partitionId, delta) {
|
|
12157
|
+
log.bytes += delta;
|
|
12158
|
+
log.bytesByPartition[partitionId] = (log.bytesByPartition[partitionId] ?? 0) + delta;
|
|
12159
|
+
if (log.sizes !== null) refreshStoredSizes(log.sizes);
|
|
12160
|
+
}
|
|
12161
|
+
function makeCurrent(previous, log, sizes) {
|
|
12162
|
+
if (previous !== void 0 && previous !== log) previous.sizes = null;
|
|
12163
|
+
log.sizes = sizes;
|
|
12164
|
+
sizes.logged = log;
|
|
12165
|
+
refreshStoredSizes(sizes);
|
|
12166
|
+
}
|
|
12167
|
+
function setFileBytes(log, key, partitionId, bytes) {
|
|
12168
|
+
const previous = log.files.get(key);
|
|
12169
|
+
if (previous !== void 0) addBytes(log, previous.partitionId, -previous.bytes);
|
|
12170
|
+
log.files.set(key, { partitionId, bytes });
|
|
12171
|
+
addBytes(log, partitionId, bytes);
|
|
12172
|
+
}
|
|
12173
|
+
function removeFile(log, key) {
|
|
12174
|
+
const previous = log.files.get(key);
|
|
12175
|
+
if (previous === void 0) return void 0;
|
|
12176
|
+
log.files.delete(key);
|
|
12177
|
+
addBytes(log, previous.partitionId, -previous.bytes);
|
|
12178
|
+
return previous;
|
|
12179
|
+
}
|
|
12180
|
+
function createLogBytesLedger(base, sizes) {
|
|
12181
|
+
const logs = /* @__PURE__ */ new Map();
|
|
12182
|
+
function logOf(indexName) {
|
|
12183
|
+
let log = logs.get(indexName);
|
|
12184
|
+
if (log === void 0) {
|
|
12185
|
+
log = emptyLog();
|
|
12186
|
+
logs.set(indexName, log);
|
|
12187
|
+
makeCurrent(void 0, log, sizes.of(indexName));
|
|
12188
|
+
}
|
|
12189
|
+
return log;
|
|
12190
|
+
}
|
|
12191
|
+
function track(key, bytes) {
|
|
12192
|
+
const file = loggedFileOf(key);
|
|
12193
|
+
if (file !== null) setFileBytes(logOf(file.indexName), key, file.partitionId, bytes);
|
|
12194
|
+
}
|
|
12195
|
+
function untrack(key) {
|
|
12196
|
+
const file = loggedFileOf(key);
|
|
12197
|
+
if (file === null) return void 0;
|
|
12198
|
+
const log = logs.get(file.indexName);
|
|
12199
|
+
return log === void 0 ? void 0 : removeFile(log, key);
|
|
12200
|
+
}
|
|
12201
|
+
async function currentLength(key) {
|
|
12202
|
+
return await base.byteLength?.(key) ?? (await base.read(key))?.byteLength ?? 0;
|
|
12203
|
+
}
|
|
12204
|
+
function trackedHandle(key, file, handle) {
|
|
12205
|
+
async function remeasure() {
|
|
12206
|
+
const log = logs.get(file.indexName);
|
|
12207
|
+
if (log?.files.has(key) === true) setFileBytes(log, key, file.partitionId, await handle.size());
|
|
12208
|
+
}
|
|
12209
|
+
return {
|
|
12210
|
+
async append(bytes) {
|
|
12211
|
+
try {
|
|
12212
|
+
await handle.append(bytes);
|
|
12213
|
+
} catch (error) {
|
|
12214
|
+
await remeasure().catch(() => void 0);
|
|
12215
|
+
throw error;
|
|
12216
|
+
}
|
|
12217
|
+
const log = logs.get(file.indexName);
|
|
12218
|
+
const entry = log?.files.get(key);
|
|
12219
|
+
if (log === void 0 || entry === void 0) return;
|
|
12220
|
+
entry.bytes += bytes.byteLength;
|
|
12221
|
+
addBytes(log, file.partitionId, bytes.byteLength);
|
|
12222
|
+
},
|
|
12223
|
+
sync: () => handle.sync(),
|
|
12224
|
+
async truncate(byteLength2) {
|
|
12225
|
+
try {
|
|
12226
|
+
await handle.truncate(byteLength2);
|
|
12227
|
+
} finally {
|
|
12228
|
+
await remeasure().catch(() => void 0);
|
|
12229
|
+
}
|
|
12230
|
+
},
|
|
12231
|
+
size: () => handle.size(),
|
|
12232
|
+
close: () => handle.close()
|
|
12233
|
+
};
|
|
12234
|
+
}
|
|
12235
|
+
const directory = {
|
|
12236
|
+
root: base.root,
|
|
12237
|
+
async appendHandle(key) {
|
|
12238
|
+
const handle = await base.appendHandle(key);
|
|
12239
|
+
const file = loggedFileOf(key);
|
|
12240
|
+
if (file === null) return handle;
|
|
12241
|
+
if (logs.get(file.indexName)?.files.has(key) !== true) track(key, await handle.size());
|
|
12242
|
+
return trackedHandle(key, file, handle);
|
|
12243
|
+
},
|
|
12244
|
+
markerHandle: (key) => base.markerHandle(key),
|
|
12245
|
+
async atomicWrite(key, data) {
|
|
12246
|
+
await base.atomicWrite(key, data);
|
|
12247
|
+
if (loggedFileOf(key) === null) return;
|
|
12248
|
+
let bytes = 0;
|
|
12249
|
+
for (const chunk of Array.isArray(data) ? data : [data]) bytes += chunk.byteLength;
|
|
12250
|
+
track(key, bytes);
|
|
12251
|
+
},
|
|
12252
|
+
syncDirectoryOf: (key) => base.syncDirectoryOf(key),
|
|
12253
|
+
read: (key) => base.read(key),
|
|
12254
|
+
async remove(key) {
|
|
12255
|
+
await base.remove(key);
|
|
12256
|
+
untrack(key);
|
|
12257
|
+
},
|
|
12258
|
+
async move(from, to) {
|
|
12259
|
+
await base.move(from, to);
|
|
12260
|
+
const moved = untrack(from);
|
|
12261
|
+
if (loggedFileOf(to) !== null) track(to, moved?.bytes ?? await currentLength(to));
|
|
12262
|
+
},
|
|
12263
|
+
list: (prefix) => base.list(prefix),
|
|
12264
|
+
pathOf: (key) => base.pathOf(key)
|
|
12265
|
+
};
|
|
12266
|
+
const { readRange, byteLength, identity, lock } = base;
|
|
12267
|
+
if (readRange !== void 0) directory.readRange = (key, offset, length) => readRange.call(base, key, offset, length);
|
|
12268
|
+
if (byteLength !== void 0) directory.byteLength = (key) => byteLength.call(base, key);
|
|
12269
|
+
if (identity !== void 0) directory.identity = () => identity.call(base);
|
|
12270
|
+
if (lock !== void 0) directory.lock = () => lock.call(base);
|
|
12271
|
+
return {
|
|
12272
|
+
directory,
|
|
12273
|
+
async seed(indexName) {
|
|
12274
|
+
const log = emptyLog();
|
|
12275
|
+
for (const prefix of [`${indexName}/wal/`, `${indexName}/history/`]) {
|
|
12276
|
+
for (const key of await base.list(prefix)) {
|
|
12277
|
+
const file = loggedFileOf(key);
|
|
12278
|
+
if (file === null || file.indexName !== indexName) continue;
|
|
12279
|
+
setFileBytes(log, key, file.partitionId, await currentLength(key));
|
|
12280
|
+
}
|
|
12281
|
+
}
|
|
12282
|
+
const previous = logs.get(indexName);
|
|
12283
|
+
logs.set(indexName, log);
|
|
12284
|
+
makeCurrent(previous, log, sizes.of(indexName));
|
|
12285
|
+
},
|
|
12286
|
+
forget(indexName) {
|
|
12287
|
+
const log = logs.get(indexName);
|
|
12288
|
+
if (log === void 0) return;
|
|
12289
|
+
logs.delete(indexName);
|
|
12290
|
+
if (log.sizes !== null) {
|
|
12291
|
+
log.sizes.logged = null;
|
|
12292
|
+
refreshStoredSizes(log.sizes);
|
|
12293
|
+
log.sizes = null;
|
|
12294
|
+
}
|
|
12295
|
+
}
|
|
12296
|
+
};
|
|
12297
|
+
}
|
|
12298
|
+
|
|
11833
12299
|
// src/persistence/durability/manager-state.ts
|
|
11834
12300
|
function closePartitionWriter(partition) {
|
|
11835
12301
|
return partition.failed === null ? partition.walWriter.close() : partition.walWriter.abandon();
|
|
@@ -12244,6 +12710,7 @@ function createIndexUnloading(deps) {
|
|
|
12244
12710
|
}
|
|
12245
12711
|
await removeQueryHistoryOf({ kind: "directory", directory }, indexName);
|
|
12246
12712
|
deps.forgetSyncState(indexName);
|
|
12713
|
+
deps.dropStoredBytes(indexName);
|
|
12247
12714
|
},
|
|
12248
12715
|
async unloadIndex(indexName) {
|
|
12249
12716
|
const indexState = indexes.get(indexName);
|
|
@@ -12254,6 +12721,7 @@ function createIndexUnloading(deps) {
|
|
|
12254
12721
|
indexes.delete(indexName);
|
|
12255
12722
|
metadataWrites.forget(indexName);
|
|
12256
12723
|
deps.forgetSyncState(indexName);
|
|
12724
|
+
deps.forgetStoredBytes(indexName);
|
|
12257
12725
|
}
|
|
12258
12726
|
};
|
|
12259
12727
|
}
|
|
@@ -12721,7 +13189,7 @@ function createSyncStorage(deps) {
|
|
|
12721
13189
|
|
|
12722
13190
|
// src/persistence/durability/manager-storage.ts
|
|
12723
13191
|
function createIndexStorageAccess(deps) {
|
|
12724
|
-
const { directory,
|
|
13192
|
+
const { directory, logBytes, sizes } = deps;
|
|
12725
13193
|
const archiveAccess = createArchiveCheckpointAccess({
|
|
12726
13194
|
directory,
|
|
12727
13195
|
held: deps.heldCheckpointFiles,
|
|
@@ -12733,7 +13201,7 @@ function createIndexStorageAccess(deps) {
|
|
|
12733
13201
|
directory,
|
|
12734
13202
|
openArchiveCheckpoint: archiveAccess.openArchiveCheckpoint,
|
|
12735
13203
|
inCheckpointTurn: (indexName, work) => {
|
|
12736
|
-
const indexState = indexes.get(indexName);
|
|
13204
|
+
const indexState = deps.indexes.get(indexName);
|
|
12737
13205
|
if (indexState === void 0) return work();
|
|
12738
13206
|
return checkpointInTurn(indexState, {
|
|
12739
13207
|
joinsInFlight: false,
|
|
@@ -12746,12 +13214,54 @@ function createIndexStorageAccess(deps) {
|
|
|
12746
13214
|
const unloading = createIndexUnloading({
|
|
12747
13215
|
directory,
|
|
12748
13216
|
hooks: deps.hooks,
|
|
12749
|
-
indexes,
|
|
13217
|
+
indexes: deps.indexes,
|
|
12750
13218
|
metadataWrites: deps.metadataWrites,
|
|
12751
13219
|
closeWriterReportingFailure: deps.closeWriterReportingFailure,
|
|
12752
|
-
forgetSyncState: syncStorage.forget
|
|
13220
|
+
forgetSyncState: syncStorage.forget,
|
|
13221
|
+
forgetStoredBytes: (indexName) => {
|
|
13222
|
+
sizes.forget(indexName);
|
|
13223
|
+
logBytes.forget(indexName);
|
|
13224
|
+
},
|
|
13225
|
+
dropStoredBytes: (indexName) => {
|
|
13226
|
+
sizes.drop(indexName);
|
|
13227
|
+
logBytes.forget(indexName);
|
|
13228
|
+
}
|
|
12753
13229
|
});
|
|
12754
|
-
|
|
13230
|
+
async function measureStoredBytes(indexName, saved) {
|
|
13231
|
+
sizes.setSaved(indexName, saved);
|
|
13232
|
+
await logBytes.seed(indexName);
|
|
13233
|
+
}
|
|
13234
|
+
const recoveryDeps = {
|
|
13235
|
+
directory,
|
|
13236
|
+
hooks: deps.hooks,
|
|
13237
|
+
compactionThreshold: deps.compactionThreshold,
|
|
13238
|
+
registerPartition: deps.registerPartition,
|
|
13239
|
+
holdsCheckpointFiles: (indexName) => deps.heldCheckpointFiles.has(indexName),
|
|
13240
|
+
queueMetadataWrite: deps.metadataWrites.queue,
|
|
13241
|
+
syncStorage,
|
|
13242
|
+
measureStoredBytes
|
|
13243
|
+
};
|
|
13244
|
+
const storedBytesAccess = {
|
|
13245
|
+
storedSizesOf: sizes.of,
|
|
13246
|
+
addStoredSizes(indexName, stats) {
|
|
13247
|
+
sizes.of(indexName).addTo(stats);
|
|
13248
|
+
},
|
|
13249
|
+
vectorSavedBytes(indexName) {
|
|
13250
|
+
return sizes.of(indexName).vectorSavedBytes;
|
|
13251
|
+
},
|
|
13252
|
+
async measureRestoredStorage(indexName, restored) {
|
|
13253
|
+
if (restored !== void 0) {
|
|
13254
|
+
await measureStoredBytes(indexName, restored);
|
|
13255
|
+
return;
|
|
13256
|
+
}
|
|
13257
|
+
const decoded = await readSegmentManifest(directoryCheckpointStore(directory), indexName);
|
|
13258
|
+
await measureStoredBytes(
|
|
13259
|
+
indexName,
|
|
13260
|
+
decoded?.kind === "current" ? savedBytesOfManifest(decoded.manifest) : NOTHING_SAVED
|
|
13261
|
+
);
|
|
13262
|
+
}
|
|
13263
|
+
};
|
|
13264
|
+
return { archiveAccess, syncStorage, unloading, storedBytesAccess, recoveryDeps };
|
|
12755
13265
|
}
|
|
12756
13266
|
|
|
12757
13267
|
// src/persistence/durability/metadata-write-queue.ts
|
|
@@ -13277,7 +13787,7 @@ function createWalWriter(directory, config) {
|
|
|
13277
13787
|
}
|
|
13278
13788
|
|
|
13279
13789
|
// src/persistence/durability/partition-registry.ts
|
|
13280
|
-
function createPartitionRegistry(directory, segmentMaxBytes) {
|
|
13790
|
+
function createPartitionRegistry(directory, segmentMaxBytes, onIndexHeld) {
|
|
13281
13791
|
const indexes = /* @__PURE__ */ new Map();
|
|
13282
13792
|
function indexState(indexName) {
|
|
13283
13793
|
let state = indexes.get(indexName);
|
|
@@ -13291,6 +13801,7 @@ function createPartitionRegistry(directory, segmentMaxBytes) {
|
|
|
13291
13801
|
stalledWrites: []
|
|
13292
13802
|
};
|
|
13293
13803
|
indexes.set(indexName, state);
|
|
13804
|
+
onIndexHeld(indexName);
|
|
13294
13805
|
}
|
|
13295
13806
|
return state;
|
|
13296
13807
|
}
|
|
@@ -13826,7 +14337,7 @@ async function readEarlierSegmentDocuments(store, key) {
|
|
|
13826
14337
|
}
|
|
13827
14338
|
|
|
13828
14339
|
// src/persistence/durability/checkpoint-count.ts
|
|
13829
|
-
var
|
|
14340
|
+
var decoder3 = new TextDecoder();
|
|
13830
14341
|
async function countPagedPartition(store, partition) {
|
|
13831
14342
|
const { directory } = store;
|
|
13832
14343
|
if (directory === void 0 || partition.segments.length === 0) return null;
|
|
@@ -13859,7 +14370,7 @@ async function countCurrentLayout(store, manifest) {
|
|
|
13859
14370
|
for (const docId of contents.tombstones) live.delete(docId);
|
|
13860
14371
|
const ids = contents.arrays.idTable;
|
|
13861
14372
|
for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) {
|
|
13862
|
-
live.add(
|
|
14373
|
+
live.add(decoder3.decode(ids.blob.subarray(ids.offsets[ordinal], ids.offsets[ordinal + 1])));
|
|
13863
14374
|
}
|
|
13864
14375
|
}
|
|
13865
14376
|
total += live.size;
|
|
@@ -14068,19 +14579,18 @@ async function writeCurrentLayout(deps, earlier, targets, fileRefs) {
|
|
|
14068
14579
|
const committed = await commitCheckpointManifest(deps.store, deps.metadata, written, vectors.refs);
|
|
14069
14580
|
recordWrittenVectors(deps.vectorIndexes, vectors.written);
|
|
14070
14581
|
await adoptVectorLayouts(deps.store, deps.vectorIndexes, vectors.layouts);
|
|
14071
|
-
await
|
|
14072
|
-
|
|
14073
|
-
|
|
14074
|
-
);
|
|
14582
|
+
const garbage = await checkpointGarbage(deps.store, deps.indexName, committed.manifest);
|
|
14583
|
+
await removeCheckpointGarbage(deps.store, await garbageThatNoFieldStillReads(deps.store, deps.vectorIndexes, garbage));
|
|
14584
|
+
return committed.saved;
|
|
14075
14585
|
}
|
|
14076
14586
|
async function upgradeEarlierLayout(deps, earlier) {
|
|
14077
14587
|
deps.onLayoutUpgrade?.({ indexName: deps.indexName, status: "started", layout: earlier.kind });
|
|
14078
14588
|
try {
|
|
14079
14589
|
const loaded = await loadEarlier(deps, earlier);
|
|
14080
14590
|
const targets = await deps.replay(loaded.checkpoint);
|
|
14081
|
-
await writeCurrentLayout(deps, earlier, targets, loaded.fileRefs);
|
|
14591
|
+
const saved = await writeCurrentLayout(deps, earlier, targets, loaded.fileRefs);
|
|
14082
14592
|
deps.onLayoutUpgrade?.({ indexName: deps.indexName, status: "completed", layout: earlier.kind });
|
|
14083
|
-
return { checkpoint: targets, documentCount: deps.manager.countDocuments() };
|
|
14593
|
+
return { checkpoint: targets, documentCount: deps.manager.countDocuments(), saved };
|
|
14084
14594
|
} catch (error) {
|
|
14085
14595
|
const failure = error instanceof Error ? error : new Error(String(error));
|
|
14086
14596
|
deps.onLayoutUpgrade?.({ indexName: deps.indexName, status: "failed", layout: earlier.kind, error: failure });
|
|
@@ -14100,11 +14610,11 @@ async function recoverCheckpoint(deps) {
|
|
|
14100
14610
|
const earlier = await earlierLayoutOf(deps, decoded);
|
|
14101
14611
|
if (earlier !== null) {
|
|
14102
14612
|
const upgraded = await upgradeEarlierLayout(deps, earlier);
|
|
14103
|
-
return { checkpoint: upgraded.checkpoint, upgradedDocumentCount: upgraded.documentCount };
|
|
14613
|
+
return { checkpoint: upgraded.checkpoint, upgradedDocumentCount: upgraded.documentCount, saved: upgraded.saved };
|
|
14104
14614
|
}
|
|
14105
14615
|
if (decoded === null || decoded.kind !== "current") {
|
|
14106
14616
|
if (!deps.keepsUnlistedFiles) await reclaimUnlistedKeys(deps.store, deps.indexName, deps.keptForSync);
|
|
14107
|
-
return { checkpoint: await deps.replay([]), upgradedDocumentCount: null };
|
|
14617
|
+
return { checkpoint: await deps.replay([]), upgradedDocumentCount: null, saved: NOTHING_SAVED };
|
|
14108
14618
|
}
|
|
14109
14619
|
await deps.store.remove(snapshotBundleKey(deps.indexName));
|
|
14110
14620
|
if (!deps.keepsUnlistedFiles) {
|
|
@@ -14117,7 +14627,11 @@ async function recoverCheckpoint(deps) {
|
|
|
14117
14627
|
vectorIndexes: deps.vectorIndexes,
|
|
14118
14628
|
writeRecords: deps.writeRecords ?? null
|
|
14119
14629
|
});
|
|
14120
|
-
return {
|
|
14630
|
+
return {
|
|
14631
|
+
checkpoint: await deps.replay(checkpoint),
|
|
14632
|
+
upgradedDocumentCount: null,
|
|
14633
|
+
saved: savedBytesOfManifest(decoded.manifest)
|
|
14634
|
+
};
|
|
14121
14635
|
}
|
|
14122
14636
|
|
|
14123
14637
|
// src/persistence/durability/layout-transition/recovery.ts
|
|
@@ -14219,9 +14733,9 @@ async function settleStoredAnalysis(store, metadata, metadataOnly) {
|
|
|
14219
14733
|
}
|
|
14220
14734
|
|
|
14221
14735
|
// src/persistence/durability/recover-index.ts
|
|
14222
|
-
async function writeCountedMetadata(deps, indexName, documentCount) {
|
|
14736
|
+
async function writeCountedMetadata(deps, indexName, documentCount, savedBytes) {
|
|
14223
14737
|
await deps.queueMetadataWrite(indexName, async () => {
|
|
14224
|
-
const metadata = deps.hooks.buildMetadata(indexName, documentCount);
|
|
14738
|
+
const metadata = deps.hooks.buildMetadata(indexName, documentCount, savedBytes);
|
|
14225
14739
|
if (metadata === void 0) return;
|
|
14226
14740
|
const bytes = await writeMetadataEnvelope(metadata, { checksum: true });
|
|
14227
14741
|
await deps.directory.atomicWrite(`${indexName}/meta`, bytes);
|
|
@@ -14316,8 +14830,14 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
|
|
|
14316
14830
|
for (const { partitionId, lastSeqNo, primaryTerm } of recovered.checkpoint) {
|
|
14317
14831
|
deps.registerPartition(indexName, partitionId, lastSeqNo, primaryTerm);
|
|
14318
14832
|
}
|
|
14833
|
+
await deps.measureStoredBytes(indexName, recovered.saved);
|
|
14319
14834
|
if (layout.transition === null && (recovered.upgradedDocumentCount !== null || promoted || layout.changedMetadata)) {
|
|
14320
|
-
await writeCountedMetadata(
|
|
14835
|
+
await writeCountedMetadata(
|
|
14836
|
+
deps,
|
|
14837
|
+
indexName,
|
|
14838
|
+
recovered.upgradedDocumentCount ?? manager.countDocuments(),
|
|
14839
|
+
recovered.saved.bytes
|
|
14840
|
+
);
|
|
14321
14841
|
}
|
|
14322
14842
|
return layout.transition === null ? null : removeUnread;
|
|
14323
14843
|
}
|
|
@@ -14333,15 +14853,22 @@ async function recoverWithLayoutCheckpoint(deps, indexName, metadataOnly, checkp
|
|
|
14333
14853
|
|
|
14334
14854
|
// src/persistence/durability/manager.ts
|
|
14335
14855
|
function createDurabilityManager(config, hooks, directoryOverride) {
|
|
14336
|
-
const
|
|
14856
|
+
const sizes = createStoredSizesRecord(true);
|
|
14857
|
+
const logBytes = createLogBytesLedger(createDurableDirectory(config.directory ?? ""), sizes);
|
|
14858
|
+
const { directory } = logBytes;
|
|
14337
14859
|
const canOffloadCheckpoint = directoryOverride === void 0;
|
|
14338
14860
|
const segmentMaxBytes = config.segmentMaxBytes ?? DEFAULT_SEGMENT_MAX_BYTES;
|
|
14339
14861
|
const checkpointIntervalMs = config.checkpointIntervalMs ?? DEFAULT_CHECKPOINT_INTERVAL_MS;
|
|
14340
14862
|
const checkpointMutationThreshold = config.checkpointMutationThreshold ?? DEFAULT_CHECKPOINT_MUTATION_THRESHOLD;
|
|
14341
14863
|
const compactionThreshold = config.compactionThreshold ?? DEFAULT_COMPACTION_THRESHOLD;
|
|
14864
|
+
const rewriteBytesPerCheckpoint = config.rewriteBytesPerCheckpoint ?? DEFAULT_REWRITE_BYTES_PER_CHECKPOINT;
|
|
14342
14865
|
const mode = config.mode ?? "sync";
|
|
14343
14866
|
const flushIntervalMs = config.flushIntervalMs ?? DEFAULT_ASYNC_FLUSH_INTERVAL_MS;
|
|
14344
|
-
const registry = createPartitionRegistry(
|
|
14867
|
+
const registry = createPartitionRegistry(
|
|
14868
|
+
directory,
|
|
14869
|
+
segmentMaxBytes,
|
|
14870
|
+
(indexName) => hooks.linkStoredSizes(indexName, sizes.hold(indexName))
|
|
14871
|
+
);
|
|
14345
14872
|
const indexes = registry.indexes;
|
|
14346
14873
|
const getOrCreateIndexState = registry.indexState;
|
|
14347
14874
|
const getOrCreatePartition = registry.partition;
|
|
@@ -14432,12 +14959,14 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
14432
14959
|
indexName,
|
|
14433
14960
|
indexState,
|
|
14434
14961
|
compactionThreshold,
|
|
14962
|
+
rewriteBytesPerCheckpoint,
|
|
14435
14963
|
canOffload: canOffloadCheckpoint,
|
|
14436
14964
|
fromMemory,
|
|
14437
14965
|
mutationThreshold: checkpointMutationThreshold,
|
|
14438
14966
|
mayContinue: () => !indexState.unloading && !shuttingDown && fatalError === null,
|
|
14439
14967
|
holdsCheckpointFiles: () => heldCheckpointFiles.has(indexName),
|
|
14440
14968
|
keepsCheckpointKey: () => syncStorage.keepsCheckpointKey(indexName),
|
|
14969
|
+
recordSavedBytes: (saved) => sizes.setSaved(indexName, saved),
|
|
14441
14970
|
queueMetadataWrite,
|
|
14442
14971
|
markFatal,
|
|
14443
14972
|
failPartition: partitionFailureFor(indexName, partitionBatchDeps)
|
|
@@ -14492,28 +15021,22 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
14492
15021
|
startAsyncFlushTimer();
|
|
14493
15022
|
return outcomes;
|
|
14494
15023
|
}
|
|
14495
|
-
const { archiveAccess, syncStorage, unloading } = createIndexStorageAccess({
|
|
15024
|
+
const { archiveAccess, syncStorage, unloading, storedBytesAccess, recoveryDeps } = createIndexStorageAccess({
|
|
14496
15025
|
directory,
|
|
15026
|
+
logBytes,
|
|
15027
|
+
sizes,
|
|
14497
15028
|
hooks,
|
|
14498
15029
|
indexes,
|
|
14499
15030
|
metadataWrites,
|
|
14500
15031
|
heldCheckpointFiles,
|
|
14501
|
-
checkpoint: (name, pinsVectors) => requestedCheckpoint(name, false, pinsVectors),
|
|
14502
|
-
registerPartition: (name, partitionId, startSeqNo, primaryTerm) => getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm),
|
|
14503
|
-
markFatal,
|
|
14504
|
-
closeWriterReportingFailure
|
|
14505
|
-
});
|
|
14506
|
-
const recoveryDeps = {
|
|
14507
|
-
directory,
|
|
14508
|
-
hooks,
|
|
14509
15032
|
compactionThreshold,
|
|
15033
|
+
checkpoint: (name, pinsVectors) => requestedCheckpoint(name, false, pinsVectors),
|
|
14510
15034
|
registerPartition: (name, partitionId, startSeqNo, primaryTerm) => {
|
|
14511
15035
|
getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm);
|
|
14512
15036
|
},
|
|
14513
|
-
|
|
14514
|
-
|
|
14515
|
-
|
|
14516
|
-
};
|
|
15037
|
+
markFatal,
|
|
15038
|
+
closeWriterReportingFailure
|
|
15039
|
+
});
|
|
14517
15040
|
function recoverIndex(indexName, metadataOnly = false, startsGraphRepairs = true) {
|
|
14518
15041
|
const recovered = recoverWithLayoutCheckpoint(
|
|
14519
15042
|
recoveryDeps,
|
|
@@ -14610,6 +15133,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
14610
15133
|
syncStorage: () => syncStorage,
|
|
14611
15134
|
...queryHistoryAccess({ kind: "directory", directory }),
|
|
14612
15135
|
...archiveAccess,
|
|
15136
|
+
...storedBytesAccess,
|
|
14613
15137
|
async shutdown() {
|
|
14614
15138
|
shuttingDown = true;
|
|
14615
15139
|
stopTimers();
|
|
@@ -14628,6 +15152,31 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
14628
15152
|
};
|
|
14629
15153
|
}
|
|
14630
15154
|
|
|
15155
|
+
// src/persistence/durability/snapshot-only/index-state.ts
|
|
15156
|
+
function createSnapshotIndexState() {
|
|
15157
|
+
return {
|
|
15158
|
+
mutationsSinceCheckpoint: 0,
|
|
15159
|
+
checkpointInFlight: null,
|
|
15160
|
+
applyChain: Promise.resolve(),
|
|
15161
|
+
unloading: false,
|
|
15162
|
+
appliedSeqNoByPartition: /* @__PURE__ */ new Map()
|
|
15163
|
+
};
|
|
15164
|
+
}
|
|
15165
|
+
function metadataKey(indexName) {
|
|
15166
|
+
return `${indexName}/meta`;
|
|
15167
|
+
}
|
|
15168
|
+
function noteSnapshotCommitted(state, positions, partitionCount) {
|
|
15169
|
+
let recordsLeft = 0;
|
|
15170
|
+
for (const [partitionId, sequence] of state.appliedSeqNoByPartition) {
|
|
15171
|
+
if (partitionId >= partitionCount) {
|
|
15172
|
+
state.appliedSeqNoByPartition.delete(partitionId);
|
|
15173
|
+
continue;
|
|
15174
|
+
}
|
|
15175
|
+
recordsLeft += Math.max(0, sequence - (positions.get(partitionId) ?? 0));
|
|
15176
|
+
}
|
|
15177
|
+
state.mutationsSinceCheckpoint = recordsLeft;
|
|
15178
|
+
}
|
|
15179
|
+
|
|
14631
15180
|
// src/persistence/durability/snapshot-only/checkpoint.ts
|
|
14632
15181
|
function capture(input, prior, cutoffs) {
|
|
14633
15182
|
const listedIds = new Set((prior?.partitions ?? []).map((partition) => partition.partitionId));
|
|
@@ -14686,10 +15235,21 @@ async function writeCaptured(input, store, prior, captured) {
|
|
|
14686
15235
|
});
|
|
14687
15236
|
const committed = await commitCheckpointManifest(store, input.metadata, written, vectors.refs);
|
|
14688
15237
|
recordWrittenVectors(input.vectorIndexes, vectors.written);
|
|
14689
|
-
await
|
|
14690
|
-
|
|
14691
|
-
|
|
14692
|
-
|
|
15238
|
+
await input.onCommitted({
|
|
15239
|
+
saved: committed.saved,
|
|
15240
|
+
documentCount: captured.documentCount,
|
|
15241
|
+
partitionCount: captured.partitionCount
|
|
15242
|
+
});
|
|
15243
|
+
const garbage = await checkpointGarbage(store, input.indexName, committed.manifest);
|
|
15244
|
+
await removeCheckpointGarbage(store, await garbageThatNoFieldStillReads(store, input.vectorIndexes, garbage));
|
|
15245
|
+
}
|
|
15246
|
+
async function saveCheckpointMetadata(adapter, hooks, indexName, committed) {
|
|
15247
|
+
const { saved, documentCount, partitionCount } = committed;
|
|
15248
|
+
const checkpointMetadata = hooks.buildMetadata(indexName, documentCount, saved.bytes);
|
|
15249
|
+
if (checkpointMetadata === void 0) return;
|
|
15250
|
+
checkpointMetadata.partitionCount = partitionCount;
|
|
15251
|
+
await adapter.save(metadataKey(indexName), await writeMetadataEnvelope(checkpointMetadata, { checksum: true }));
|
|
15252
|
+
hooks.recordCheckpoint?.(indexName, documentCount, partitionCount, saved.bytes);
|
|
14693
15253
|
}
|
|
14694
15254
|
async function writeSnapshotOnlyCheckpoint(input) {
|
|
14695
15255
|
const store = adapterCheckpointStore(input.adapter);
|
|
@@ -14700,41 +15260,12 @@ async function writeSnapshotOnlyCheckpoint(input) {
|
|
|
14700
15260
|
});
|
|
14701
15261
|
try {
|
|
14702
15262
|
await writeCaptured(input, store, prior, captured);
|
|
14703
|
-
return {
|
|
14704
|
-
documentCount: captured.documentCount,
|
|
14705
|
-
positions: captured.positions,
|
|
14706
|
-
partitionCount: captured.partitionCount
|
|
14707
|
-
};
|
|
15263
|
+
return { positions: captured.positions, partitionCount: captured.partitionCount };
|
|
14708
15264
|
} finally {
|
|
14709
15265
|
for (const plan of captured.plans.values()) plan.release();
|
|
14710
15266
|
}
|
|
14711
15267
|
}
|
|
14712
15268
|
|
|
14713
|
-
// src/persistence/durability/snapshot-only/index-state.ts
|
|
14714
|
-
function createSnapshotIndexState() {
|
|
14715
|
-
return {
|
|
14716
|
-
mutationsSinceCheckpoint: 0,
|
|
14717
|
-
checkpointInFlight: null,
|
|
14718
|
-
applyChain: Promise.resolve(),
|
|
14719
|
-
unloading: false,
|
|
14720
|
-
appliedSeqNoByPartition: /* @__PURE__ */ new Map()
|
|
14721
|
-
};
|
|
14722
|
-
}
|
|
14723
|
-
function metadataKey(indexName) {
|
|
14724
|
-
return `${indexName}/meta`;
|
|
14725
|
-
}
|
|
14726
|
-
function noteSnapshotCommitted(state, positions, partitionCount) {
|
|
14727
|
-
let recordsLeft = 0;
|
|
14728
|
-
for (const [partitionId, sequence] of state.appliedSeqNoByPartition) {
|
|
14729
|
-
if (partitionId >= partitionCount) {
|
|
14730
|
-
state.appliedSeqNoByPartition.delete(partitionId);
|
|
14731
|
-
continue;
|
|
14732
|
-
}
|
|
14733
|
-
recordsLeft += Math.max(0, sequence - (positions.get(partitionId) ?? 0));
|
|
14734
|
-
}
|
|
14735
|
-
state.mutationsSinceCheckpoint = recordsLeft;
|
|
14736
|
-
}
|
|
14737
|
-
|
|
14738
15269
|
// src/persistence/durability/snapshot-only/restores.ts
|
|
14739
15270
|
function toError6(error) {
|
|
14740
15271
|
return error instanceof Error ? error : new Error(String(error));
|
|
@@ -14772,15 +15303,67 @@ function createPendingRestores(onFatalError) {
|
|
|
14772
15303
|
};
|
|
14773
15304
|
}
|
|
14774
15305
|
|
|
15306
|
+
// src/persistence/durability/snapshot-only/saved-size.ts
|
|
15307
|
+
function createSnapshotSavedSize(adapter, settleRestore) {
|
|
15308
|
+
const sizes = createStoredSizesRecord(false);
|
|
15309
|
+
const generations = /* @__PURE__ */ new Map();
|
|
15310
|
+
const store = adapterCheckpointStore(adapter);
|
|
15311
|
+
function replace(indexName, saved) {
|
|
15312
|
+
const generation = (generations.get(indexName) ?? 0) + 1;
|
|
15313
|
+
generations.set(indexName, generation);
|
|
15314
|
+
sizes.setSaved(indexName, saved);
|
|
15315
|
+
sizes.setMeasuring(indexName, saved === null);
|
|
15316
|
+
return generation;
|
|
15317
|
+
}
|
|
15318
|
+
async function measureAfterRestore(indexName, generation) {
|
|
15319
|
+
await settleRestore(indexName);
|
|
15320
|
+
const decoded = await readSegmentManifest(store, indexName);
|
|
15321
|
+
if (generations.get(indexName) !== generation) return;
|
|
15322
|
+
replace(indexName, decoded?.kind === "current" ? savedBytesOfManifest(decoded.manifest) : NOTHING_SAVED);
|
|
15323
|
+
}
|
|
15324
|
+
return {
|
|
15325
|
+
record(indexName, saved) {
|
|
15326
|
+
replace(indexName, saved);
|
|
15327
|
+
},
|
|
15328
|
+
hold: sizes.hold,
|
|
15329
|
+
forget(indexName) {
|
|
15330
|
+
generations.delete(indexName);
|
|
15331
|
+
sizes.forget(indexName);
|
|
15332
|
+
},
|
|
15333
|
+
drop(indexName) {
|
|
15334
|
+
generations.delete(indexName);
|
|
15335
|
+
sizes.drop(indexName);
|
|
15336
|
+
},
|
|
15337
|
+
access: {
|
|
15338
|
+
storedSizesOf: sizes.of,
|
|
15339
|
+
addStoredSizes(indexName, stats) {
|
|
15340
|
+
sizes.of(indexName).addTo(stats);
|
|
15341
|
+
},
|
|
15342
|
+
vectorSavedBytes(indexName) {
|
|
15343
|
+
return sizes.of(indexName).vectorSavedBytes;
|
|
15344
|
+
},
|
|
15345
|
+
async measureRestoredStorage(indexName, restored) {
|
|
15346
|
+
if (restored !== void 0) {
|
|
15347
|
+
replace(indexName, restored);
|
|
15348
|
+
return;
|
|
15349
|
+
}
|
|
15350
|
+
const generation = replace(indexName, null);
|
|
15351
|
+
void measureAfterRestore(indexName, generation).catch(() => void 0);
|
|
15352
|
+
}
|
|
15353
|
+
}
|
|
15354
|
+
};
|
|
15355
|
+
}
|
|
15356
|
+
|
|
14775
15357
|
// src/persistence/durability/snapshot-only/index.ts
|
|
14776
15358
|
function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
14777
15359
|
const checkpointIntervalMs = config.checkpointIntervalMs ?? DEFAULT_CHECKPOINT_INTERVAL_MS;
|
|
14778
15360
|
const checkpointMutationThreshold = config.checkpointMutationThreshold ?? DEFAULT_CHECKPOINT_MUTATION_THRESHOLD;
|
|
14779
15361
|
const indexes = /* @__PURE__ */ new Map();
|
|
14780
|
-
const metadataWrites =
|
|
15362
|
+
const metadataWrites = createMetadataWriteQueue();
|
|
14781
15363
|
const loads = createIndexLoads(hooks.getVectorIndexes);
|
|
14782
15364
|
const restoreStorage = adapterRestoreStorage(adapter);
|
|
14783
15365
|
const restores = createPendingRestores(hooks.onFatalError);
|
|
15366
|
+
const savedSize = createSnapshotSavedSize(adapter, settleRestore);
|
|
14784
15367
|
let checkpointTimer = null;
|
|
14785
15368
|
let shuttingDown = false;
|
|
14786
15369
|
function settleRestore(indexName) {
|
|
@@ -14791,6 +15374,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
14791
15374
|
if (state === void 0) {
|
|
14792
15375
|
state = createSnapshotIndexState();
|
|
14793
15376
|
indexes.set(indexName, state);
|
|
15377
|
+
hooks.linkStoredSizes(indexName, savedSize.hold(indexName));
|
|
14794
15378
|
}
|
|
14795
15379
|
return state;
|
|
14796
15380
|
}
|
|
@@ -14819,23 +15403,19 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
14819
15403
|
const manager = hooks.getManager(indexName);
|
|
14820
15404
|
const metadata = hooks.buildTermsMetadata(indexName);
|
|
14821
15405
|
if (manager === void 0 || metadata === void 0) return;
|
|
14822
|
-
const {
|
|
15406
|
+
const { positions, partitionCount } = await writeSnapshotOnlyCheckpoint({
|
|
14823
15407
|
adapter,
|
|
14824
15408
|
indexName,
|
|
14825
15409
|
metadata,
|
|
14826
15410
|
manager,
|
|
14827
15411
|
vectorIndexes: hooks.getVectorIndexes(indexName),
|
|
14828
15412
|
appliedSeqNoByPartition: indexState.appliedSeqNoByPartition,
|
|
14829
|
-
rewritesEveryPartition: fromMemory || hooks.mustRewriteTerms(indexName)
|
|
14830
|
-
|
|
14831
|
-
|
|
14832
|
-
|
|
14833
|
-
|
|
14834
|
-
|
|
14835
|
-
checkpointMetadata.partitionCount = partitionCount;
|
|
14836
|
-
const metadataBytes = await writeMetadataEnvelope(checkpointMetadata, { checksum: true });
|
|
14837
|
-
await adapter.save(metadataKey(indexName), metadataBytes);
|
|
14838
|
-
hooks.recordCheckpoint?.(indexName, documentCount, partitionCount);
|
|
15413
|
+
rewritesEveryPartition: fromMemory || hooks.mustRewriteTerms(indexName),
|
|
15414
|
+
onCommitted: async (committed) => {
|
|
15415
|
+
savedSize.record(indexName, committed.saved);
|
|
15416
|
+
hooks.onManifestCommitted(indexName, metadata.analysisGeneration ?? 0);
|
|
15417
|
+
await queueMetadataWrite(indexName, () => saveCheckpointMetadata(adapter, hooks, indexName, committed));
|
|
15418
|
+
}
|
|
14839
15419
|
});
|
|
14840
15420
|
noteSnapshotCommitted(indexState, positions, partitionCount);
|
|
14841
15421
|
if (publisher !== void 0) {
|
|
@@ -14861,16 +15441,10 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
14861
15441
|
);
|
|
14862
15442
|
}
|
|
14863
15443
|
function queueMetadataWrite(indexName, write) {
|
|
14864
|
-
|
|
14865
|
-
const queued = previous.then(() => settleRestore(indexName)).then(write);
|
|
14866
|
-
metadataWrites.set(
|
|
14867
|
-
indexName,
|
|
14868
|
-
queued.catch(() => void 0)
|
|
14869
|
-
);
|
|
14870
|
-
return queued;
|
|
15444
|
+
return metadataWrites.queue(indexName, () => settleRestore(indexName).then(write));
|
|
14871
15445
|
}
|
|
14872
|
-
async function saveMetadata(indexName, documentCount) {
|
|
14873
|
-
const metadata = hooks.buildMetadata(indexName, documentCount);
|
|
15446
|
+
async function saveMetadata(indexName, documentCount, savedBytes) {
|
|
15447
|
+
const metadata = hooks.buildMetadata(indexName, documentCount, savedBytes);
|
|
14874
15448
|
if (metadata === void 0) return;
|
|
14875
15449
|
await adapter.save(metadataKey(indexName), await writeMetadataEnvelope(metadata, { checksum: true }));
|
|
14876
15450
|
}
|
|
@@ -14880,9 +15454,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
14880
15454
|
async function loadIndex(indexName, metadataOnly) {
|
|
14881
15455
|
await settleRestore(indexName);
|
|
14882
15456
|
const metaBytes = await adapter.load(metadataKey(indexName));
|
|
14883
|
-
if (metaBytes === null)
|
|
14884
|
-
return;
|
|
14885
|
-
}
|
|
15457
|
+
if (metaBytes === null) return;
|
|
14886
15458
|
const { metadata } = await readMetadataEnvelope(metaBytes);
|
|
14887
15459
|
const promoted = await settleStoredAnalysis(adapterCheckpointStore(adapter), metadata, metadataOnly);
|
|
14888
15460
|
let derivedCount;
|
|
@@ -14921,6 +15493,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
14921
15493
|
onLayoutUpgrade: hooks.onLayoutUpgrade
|
|
14922
15494
|
});
|
|
14923
15495
|
const indexState = getOrCreateIndexState(indexName);
|
|
15496
|
+
savedSize.record(indexName, recovered.saved);
|
|
14924
15497
|
for (let partitionId = 0; partitionId < manager.partitionCount; partitionId += 1) {
|
|
14925
15498
|
const persistedSeqNo = snapshotCheckpointFor(recovered.checkpoint, partitionId);
|
|
14926
15499
|
if (persistedSeqNo > 0) {
|
|
@@ -14929,7 +15502,9 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
14929
15502
|
}
|
|
14930
15503
|
if (recovered.upgradedDocumentCount !== null || promoted) {
|
|
14931
15504
|
const upgradedCount = recovered.upgradedDocumentCount ?? manager.countDocuments();
|
|
14932
|
-
await queueMetadataWrite(indexName, () => saveMetadata(indexName, upgradedCount)).catch(
|
|
15505
|
+
await queueMetadataWrite(indexName, () => saveMetadata(indexName, upgradedCount, recovered.saved.bytes)).catch(
|
|
15506
|
+
() => void 0
|
|
15507
|
+
);
|
|
14933
15508
|
}
|
|
14934
15509
|
}
|
|
14935
15510
|
async function recordMutation(record) {
|
|
@@ -14961,14 +15536,15 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
14961
15536
|
try {
|
|
14962
15537
|
await indexState.applyChain;
|
|
14963
15538
|
if (indexState.checkpointInFlight !== null) await indexState.checkpointInFlight;
|
|
14964
|
-
await metadataWrites.
|
|
15539
|
+
await metadataWrites.pending(indexName);
|
|
14965
15540
|
} catch (error) {
|
|
14966
15541
|
indexState.unloading = false;
|
|
14967
15542
|
throw error;
|
|
14968
15543
|
}
|
|
14969
15544
|
}
|
|
14970
15545
|
indexes.delete(indexName);
|
|
14971
|
-
metadataWrites.
|
|
15546
|
+
metadataWrites.forget(indexName);
|
|
15547
|
+
savedSize.forget(indexName);
|
|
14972
15548
|
}
|
|
14973
15549
|
return {
|
|
14974
15550
|
isActive() {
|
|
@@ -15050,6 +15626,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
15050
15626
|
}
|
|
15051
15627
|
},
|
|
15052
15628
|
...queryHistoryAccess({ kind: "adapter", adapter }),
|
|
15629
|
+
...savedSize.access,
|
|
15053
15630
|
async removeIndex(indexName) {
|
|
15054
15631
|
await settleRestore(indexName).catch(() => void 0);
|
|
15055
15632
|
restores.forget(indexName);
|
|
@@ -15059,6 +15636,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
15059
15636
|
await adapter.delete(key);
|
|
15060
15637
|
}
|
|
15061
15638
|
await removeQueryHistoryOf({ kind: "adapter", adapter }, indexName);
|
|
15639
|
+
savedSize.drop(indexName);
|
|
15062
15640
|
},
|
|
15063
15641
|
async unloadIndex(indexName) {
|
|
15064
15642
|
await settleRestore(indexName);
|
|
@@ -15103,7 +15681,7 @@ function vectorFieldsOf(schema, promotion) {
|
|
|
15103
15681
|
}
|
|
15104
15682
|
return Object.keys(fields).length === 0 ? null : fields;
|
|
15105
15683
|
}
|
|
15106
|
-
function metadataFromConfig(indexName, config, partitionCount, documentCount) {
|
|
15684
|
+
function metadataFromConfig(indexName, config, partitionCount, documentCount, savedBytes) {
|
|
15107
15685
|
const metadata = {
|
|
15108
15686
|
indexName,
|
|
15109
15687
|
schema: config.schema,
|
|
@@ -15118,6 +15696,8 @@ function metadataFromConfig(indexName, config, partitionCount, documentCount) {
|
|
|
15118
15696
|
engineVersion: VERSION
|
|
15119
15697
|
};
|
|
15120
15698
|
metadata.documentCount = documentCount ?? config.documentCount;
|
|
15699
|
+
const recordedSavedBytes = savedBytes ?? config.savedBytes;
|
|
15700
|
+
if (recordedSavedBytes !== void 0) metadata.savedBytes = recordedSavedBytes;
|
|
15121
15701
|
const vectorFields = vectorFieldsOf(config.schema, config.vectorPromotion);
|
|
15122
15702
|
if (vectorFields !== null) {
|
|
15123
15703
|
metadata.vectorFields = vectorFields;
|
|
@@ -15197,6 +15777,7 @@ function metadataConfigOf(entry) {
|
|
|
15197
15777
|
...entry.indexUuid !== null ? { indexUuid: entry.indexUuid } : {},
|
|
15198
15778
|
...entry.heldPartitions !== null ? { heldPartitions: entry.heldPartitions } : {},
|
|
15199
15779
|
documentCount: entry.documentCount,
|
|
15780
|
+
savedBytes: entry.savedBytes,
|
|
15200
15781
|
partitionCount: entry.partitionCount,
|
|
15201
15782
|
schema: flattenSchema(entry.config.schema),
|
|
15202
15783
|
language: entry.language.name,
|
|
@@ -15275,7 +15856,7 @@ function indexMetadataOf(indexName, entry, partitionCount) {
|
|
|
15275
15856
|
}
|
|
15276
15857
|
|
|
15277
15858
|
// src/engine/durability-integration.ts
|
|
15278
|
-
function buildMetadata(indexName, hooks, documentCount) {
|
|
15859
|
+
function buildMetadata(indexName, hooks, documentCount, savedBytes) {
|
|
15279
15860
|
const config = hooks.getIndexConfig(indexName);
|
|
15280
15861
|
if (config === void 0) {
|
|
15281
15862
|
return void 0;
|
|
@@ -15285,7 +15866,8 @@ function buildMetadata(indexName, hooks, documentCount) {
|
|
|
15285
15866
|
indexName,
|
|
15286
15867
|
config,
|
|
15287
15868
|
manager === void 0 ? void 0 : metadataPartitionCount(manager),
|
|
15288
|
-
documentCount
|
|
15869
|
+
documentCount,
|
|
15870
|
+
savedBytes
|
|
15289
15871
|
);
|
|
15290
15872
|
}
|
|
15291
15873
|
function createDurabilityIntegration(tier, hooks) {
|
|
@@ -15293,7 +15875,7 @@ function createDurabilityIntegration(tier, hooks) {
|
|
|
15293
15875
|
getManager: hooks.getManager,
|
|
15294
15876
|
getVectorFieldPaths: hooks.getVectorFieldPaths,
|
|
15295
15877
|
getVectorIndexes: hooks.getVectorIndexes,
|
|
15296
|
-
buildMetadata: (indexName, documentCount) => buildMetadata(indexName, hooks, documentCount),
|
|
15878
|
+
buildMetadata: (indexName, documentCount, savedBytes) => buildMetadata(indexName, hooks, documentCount, savedBytes),
|
|
15297
15879
|
buildTermsMetadata(indexName) {
|
|
15298
15880
|
const config = hooks.getTermsConfig(indexName);
|
|
15299
15881
|
return config === void 0 ? void 0 : metadataFromConfig(indexName, config, hooks.getManager(indexName)?.partitionCount);
|
|
@@ -15308,7 +15890,8 @@ function createDurabilityIntegration(tier, hooks) {
|
|
|
15308
15890
|
onCheckpointHeld: hooks.onCheckpointHeld,
|
|
15309
15891
|
onFatalError: hooks.onFatalError,
|
|
15310
15892
|
...hooks.keepsSyncHistory === void 0 ? {} : { keepsSyncHistory: hooks.keepsSyncHistory },
|
|
15311
|
-
...hooks.onHistoryMoved === void 0 ? {} : { onHistoryMoved: hooks.onHistoryMoved }
|
|
15893
|
+
...hooks.onHistoryMoved === void 0 ? {} : { onHistoryMoved: hooks.onHistoryMoved },
|
|
15894
|
+
linkStoredSizes: hooks.linkStoredSizes
|
|
15312
15895
|
};
|
|
15313
15896
|
const manager = tier.kind === "wal" ? createDurabilityManager(tier.config, managerHooks) : createSnapshotOnlyManager(tier.adapter, tier.config, managerHooks, hooks.checkpointPublisher);
|
|
15314
15897
|
async function recordMutation(indexName, docId, operation, document, apply) {
|
|
@@ -15429,7 +16012,7 @@ var OLD_REGULAR_FILE = 0;
|
|
|
15429
16012
|
var PAX_EXTENDED_HEADER = 120;
|
|
15430
16013
|
var PAX_HEADER_NAME = "PaxHeader";
|
|
15431
16014
|
var encoder3 = new TextEncoder();
|
|
15432
|
-
var
|
|
16015
|
+
var decoder4 = new TextDecoder("utf-8", { fatal: true });
|
|
15433
16016
|
function writeString(block, offset, length, value) {
|
|
15434
16017
|
const bytes = encoder3.encode(value);
|
|
15435
16018
|
block.set(bytes.subarray(0, length), offset);
|
|
@@ -15507,7 +16090,7 @@ function readString(block, offset, length) {
|
|
|
15507
16090
|
let end = offset;
|
|
15508
16091
|
while (end < offset + length && block[end] !== 0) end++;
|
|
15509
16092
|
try {
|
|
15510
|
-
return
|
|
16093
|
+
return decoder4.decode(block.subarray(offset, end));
|
|
15511
16094
|
} catch {
|
|
15512
16095
|
invalidArchive("an entry header holds a name that is not UTF-8");
|
|
15513
16096
|
}
|
|
@@ -15544,7 +16127,7 @@ var EQUALS = 61;
|
|
|
15544
16127
|
var NEWLINE = 10;
|
|
15545
16128
|
function decodeRecordText(bytes) {
|
|
15546
16129
|
try {
|
|
15547
|
-
return
|
|
16130
|
+
return decoder4.decode(bytes);
|
|
15548
16131
|
} catch {
|
|
15549
16132
|
invalidArchive("a pax extended header is not UTF-8");
|
|
15550
16133
|
}
|
|
@@ -16232,7 +16815,7 @@ function createInvalidationIntegration(deps) {
|
|
|
16232
16815
|
|
|
16233
16816
|
// src/engine/durability-wiring.ts
|
|
16234
16817
|
function createDurabilityFromTier(tier, wiring) {
|
|
16235
|
-
|
|
16818
|
+
const integration = createDurabilityIntegration(tier, {
|
|
16236
16819
|
checkpointPublisher: { publishPartitions: wiring.publishCheckpointedPartitions },
|
|
16237
16820
|
getManager: (indexName) => wiring.indexRegistry.has(indexName) ? wiring.getManager(indexName) : void 0,
|
|
16238
16821
|
getVectorFieldPaths: (indexName) => wiring.indexRegistry.get(indexName)?.vectorFieldPaths ?? /* @__PURE__ */ new Set(),
|
|
@@ -16252,8 +16835,25 @@ function createDurabilityFromTier(tier, wiring) {
|
|
|
16252
16835
|
},
|
|
16253
16836
|
queueDiskAdoption: wiring.queueDiskAdoption,
|
|
16254
16837
|
awaitCopiesOfIndex: wiring.awaitCopiesOfIndex,
|
|
16255
|
-
|
|
16256
|
-
|
|
16838
|
+
async createIndexFromMetadata(metadata, loadData) {
|
|
16839
|
+
await wiring.createIndexFromMetadata(metadata, loadData);
|
|
16840
|
+
const entry = wiring.indexRegistry.get(metadata.indexName);
|
|
16841
|
+
if (entry === void 0) return;
|
|
16842
|
+
entry.storedSizes = integration.manager.storedSizesOf(metadata.indexName);
|
|
16843
|
+
entry.storedSizes.recordSavedBytes(entry.savedBytes);
|
|
16844
|
+
},
|
|
16845
|
+
recordCheckpoint(indexName, documentCount, partitionCount, savedBytes) {
|
|
16846
|
+
const entry = wiring.indexRegistry.get(indexName);
|
|
16847
|
+
if (entry === void 0) return;
|
|
16848
|
+
entry.documentCount = documentCount;
|
|
16849
|
+
entry.partitionCount = partitionCount;
|
|
16850
|
+
entry.savedBytes = savedBytes;
|
|
16851
|
+
entry.storedSizes?.recordSavedBytes(savedBytes);
|
|
16852
|
+
},
|
|
16853
|
+
linkStoredSizes(indexName, sizes) {
|
|
16854
|
+
const entry = wiring.indexRegistry.get(indexName);
|
|
16855
|
+
if (entry !== void 0) entry.storedSizes = sizes;
|
|
16856
|
+
},
|
|
16257
16857
|
onLayoutUpgrade: wiring.emitLayoutUpgrade,
|
|
16258
16858
|
onCheckpointHeld: wiring.emitCheckpointHeld,
|
|
16259
16859
|
onFatalError: wiring.emitFatalError,
|
|
@@ -16267,6 +16867,7 @@ function createDurabilityFromTier(tier, wiring) {
|
|
|
16267
16867
|
}
|
|
16268
16868
|
}
|
|
16269
16869
|
});
|
|
16870
|
+
return integration;
|
|
16270
16871
|
}
|
|
16271
16872
|
async function reloadIndexAndCopies(orchestrator, durability, indexName) {
|
|
16272
16873
|
const manager = durability?.manager;
|
|
@@ -16612,7 +17213,7 @@ function joined2(parts, total) {
|
|
|
16612
17213
|
return bytes;
|
|
16613
17214
|
}
|
|
16614
17215
|
async function* exportLines(source, maxLineBytes) {
|
|
16615
|
-
const
|
|
17216
|
+
const decoder6 = new TextDecoder("utf-8", { fatal: true });
|
|
16616
17217
|
let parts = [];
|
|
16617
17218
|
let pending = 0;
|
|
16618
17219
|
let overflowing = false;
|
|
@@ -16629,7 +17230,7 @@ async function* exportLines(source, maxLineBytes) {
|
|
|
16629
17230
|
parts = [];
|
|
16630
17231
|
pending = 0;
|
|
16631
17232
|
try {
|
|
16632
|
-
return { line, text:
|
|
17233
|
+
return { line, text: decoder6.decode(bytes) };
|
|
16633
17234
|
} catch {
|
|
16634
17235
|
return { line, refused: "not-utf8" };
|
|
16635
17236
|
}
|
|
@@ -17164,10 +17765,10 @@ async function readSegment(checkpoint, ref) {
|
|
|
17164
17765
|
});
|
|
17165
17766
|
return decodeKeywordSegment(payload, false);
|
|
17166
17767
|
}
|
|
17167
|
-
var
|
|
17768
|
+
var decoder5 = new TextDecoder();
|
|
17168
17769
|
function idAt(contents, ordinal) {
|
|
17169
17770
|
const ids = contents.arrays.idTable;
|
|
17170
|
-
return
|
|
17771
|
+
return decoder5.decode(ids.blob.subarray(ids.offsets[ordinal], ids.offsets[ordinal + 1]));
|
|
17171
17772
|
}
|
|
17172
17773
|
async function winnersOf(checkpoint, partition) {
|
|
17173
17774
|
const winners = /* @__PURE__ */ new Map();
|
|
@@ -17688,8 +18289,9 @@ async function prepareDetachedIndex(indexName, prepared, partitionCount, deps) {
|
|
|
17688
18289
|
partitionCount ?? void 0
|
|
17689
18290
|
);
|
|
17690
18291
|
}
|
|
17691
|
-
function restoredRegistryEntry(prepared, manager, deps) {
|
|
18292
|
+
function restoredRegistryEntry(indexName, prepared, manager, deps) {
|
|
17692
18293
|
return {
|
|
18294
|
+
storedSizes: deps.durability?.manager.storedSizesOf(indexName),
|
|
17693
18295
|
config: prepared.config,
|
|
17694
18296
|
language: prepared.language,
|
|
17695
18297
|
embeddingAdapter: prepared.embeddingAdapter,
|
|
@@ -17789,13 +18391,17 @@ async function commitToStorage(indexName, restored, previous, deps) {
|
|
|
17789
18391
|
for (const vectorIndex of vectorIndexes) await vectorIndex.releaseVectorFiles();
|
|
17790
18392
|
await staging.commit();
|
|
17791
18393
|
} catch (error) {
|
|
17792
|
-
if (previousPartitions > 0)
|
|
18394
|
+
if (previousPartitions > 0) {
|
|
18395
|
+
durable.adoptRestoredCheckpoint?.(indexName, previousCheckpoint);
|
|
18396
|
+
await durable.measureRestoredStorage?.(indexName).catch(() => void 0);
|
|
18397
|
+
}
|
|
17793
18398
|
throw error;
|
|
17794
18399
|
}
|
|
17795
18400
|
for (const paged of pagedSegmentsOf(previous)) paged.close();
|
|
17796
18401
|
const restoredSegments = pagedSegmentsOf(restored.detached.manager);
|
|
17797
18402
|
for (const paged of restoredSegments) paged.suspend();
|
|
17798
18403
|
await staging.finish();
|
|
18404
|
+
await durable.measureRestoredStorage?.(indexName, restored.saved ?? void 0).catch(() => void 0);
|
|
17799
18405
|
const relocate = await staging.relocation().catch(() => null);
|
|
17800
18406
|
for (const paged of restoredSegments) paged.resume(relocate ?? samePath);
|
|
17801
18407
|
if (relocate === null) return;
|
|
@@ -17804,6 +18410,7 @@ async function commitToStorage(indexName, restored, previous, deps) {
|
|
|
17804
18410
|
function activate(indexName, restored, entry, deps) {
|
|
17805
18411
|
deps.executor.installDetachedIndex(restored.detached);
|
|
17806
18412
|
deps.indexRegistry.set(indexName, entry);
|
|
18413
|
+
entry.storedSizes?.recordSavedBytes(entry.savedBytes);
|
|
17807
18414
|
restored.detached.attach();
|
|
17808
18415
|
deps.clearAnalysisStale(indexName);
|
|
17809
18416
|
const { prepared } = restored;
|
|
@@ -17843,6 +18450,7 @@ async function createFresh(indexName, restored, entry, deps) {
|
|
|
17843
18450
|
deps.copies.desyncIndex(indexName);
|
|
17844
18451
|
deps.executor.installDetachedIndex(restored.detached);
|
|
17845
18452
|
deps.indexRegistry.set(indexName, entry);
|
|
18453
|
+
entry.storedSizes?.recordSavedBytes(entry.savedBytes);
|
|
17846
18454
|
try {
|
|
17847
18455
|
await deps.indexState.registerReplacing(indexName, async () => {
|
|
17848
18456
|
await commitToStorage(indexName, restored, void 0, deps);
|
|
@@ -17861,9 +18469,10 @@ async function stageMetadata(indexName, staging, entry) {
|
|
|
17861
18469
|
await staging.store.write(`${indexName}/meta`, [await writeMetadataEnvelope(metadata, { checksum: true })]);
|
|
17862
18470
|
}
|
|
17863
18471
|
async function installRestoredIndex(indexName, restored, deps, history = null) {
|
|
17864
|
-
const entry = restoredRegistryEntry(restored.prepared, restored.detached.manager, deps);
|
|
18472
|
+
const entry = restoredRegistryEntry(indexName, restored.prepared, restored.detached.manager, deps);
|
|
17865
18473
|
const previousSync = deps.indexRegistry.get(indexName)?.sync ?? null;
|
|
17866
18474
|
if (previousSync !== null) entry.sync = { ...previousSync, active: false, epoch: newSyncEpoch() };
|
|
18475
|
+
if (restored.saved !== null) entry.savedBytes = restored.saved.bytes;
|
|
17867
18476
|
let replaced;
|
|
17868
18477
|
try {
|
|
17869
18478
|
if (deps.indexRegistry.has(indexName)) await deps.abandonAnalysisChange(indexName);
|
|
@@ -17996,7 +18605,7 @@ async function loadStagedIndex(indexName, staged, staging, store, deps) {
|
|
|
17996
18605
|
deps.executor.discardDetachedIndex(detached);
|
|
17997
18606
|
throw error;
|
|
17998
18607
|
}
|
|
17999
|
-
return { prepared, detached, staging, checkpoint: manifest.checkpoint };
|
|
18608
|
+
return { prepared, detached, staging, checkpoint: manifest.checkpoint, saved: savedBytesOfManifest(manifest) };
|
|
18000
18609
|
}
|
|
18001
18610
|
async function restoreArchive(indexName, reader, deps) {
|
|
18002
18611
|
const staging = deps.durability?.manager.stageRestore?.(indexName) ?? null;
|
|
@@ -18097,7 +18706,7 @@ function insertSnapshotDocuments(envelope, manager) {
|
|
|
18097
18706
|
for (const vectorIndex of manager.getVectorIndexes().values()) vectorIndex.scheduleBuild();
|
|
18098
18707
|
}
|
|
18099
18708
|
async function stageBuiltIndex(indexName, prepared, built, staging, deps) {
|
|
18100
|
-
const entry = restoredRegistryEntry(prepared, built.manager, deps);
|
|
18709
|
+
const entry = restoredRegistryEntry(indexName, prepared, built.manager, deps);
|
|
18101
18710
|
const archive = await buildArchiveFromMemory({
|
|
18102
18711
|
indexName,
|
|
18103
18712
|
metadata: indexMetadataOf(indexName, entry, built.manager.partitionCount),
|
|
@@ -18121,7 +18730,7 @@ async function buildRestoredIndex(indexName, envelope, prepared, staging, deps)
|
|
|
18121
18730
|
let manifest;
|
|
18122
18731
|
try {
|
|
18123
18732
|
insertSnapshotDocuments(envelope, built.manager);
|
|
18124
|
-
if (staging === null) return { prepared, detached: built, staging: null, checkpoint: [] };
|
|
18733
|
+
if (staging === null) return { prepared, detached: built, staging: null, checkpoint: [], saved: null };
|
|
18125
18734
|
manifest = await stageBuiltIndex(indexName, prepared, built, staging, deps);
|
|
18126
18735
|
} catch (err) {
|
|
18127
18736
|
deps.executor.discardDetachedIndex(built);
|