@delali/narsil 0.3.1-next.91 → 0.3.1-next.94
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/browser/{chunk-B6XE7KXY.mjs → chunk-4JTBL63P.mjs} +4 -4
- package/dist/browser/{chunk-3ZRMZIH4.mjs → chunk-5S74HQRT.mjs} +208 -78
- package/dist/browser/{chunk-FHFFWLEK.mjs → chunk-HARBSUWP.mjs} +3 -4
- package/dist/browser/{chunk-NOUZ6RH3.mjs → chunk-HC6DMR4T.mjs} +238 -114
- package/dist/browser/{chunk-66KYAOIK.mjs → chunk-PP6N5CCS.mjs} +2 -2
- package/dist/browser/{chunk-NQW2YXEX.mjs → chunk-WKPJ4E73.mjs} +1 -1
- package/dist/browser/{durability-wiring-F4OGQUSU.mjs → durability-wiring-7DKYHKT5.mjs} +498 -71
- package/dist/browser/{module-E25UDADM.mjs → module-O5WH32EL.mjs} +102 -54
- package/dist/browser/release-76MXRPGV.mjs +33 -0
- package/dist/browser/replace-RURXTGVG.mjs +57 -0
- package/dist/browser/restore-queue-Z7FZZNKH.mjs +35 -0
- package/dist/browser/{snapshot-5EJ7I7QY.mjs → snapshot-PPTZ565G.mjs} +362 -103
- package/dist/browser/{write-K5V4XQ72.mjs → write-2ZAA7UDH.mjs} +4 -4
- package/dist/{capabilities-CvExBThq.d.ts → capabilities-xsJfdFoQ.d.ts} +2 -2
- package/dist/{chunk-NVL4ZR3K.mjs → chunk-2GZGBOQI.mjs} +2 -2
- package/dist/{chunk-CVQWQVFX.mjs → chunk-4ZJWRETC.mjs} +2 -2
- package/dist/{chunk-ZIJ63TBT.mjs → chunk-7KC56574.mjs} +1 -1
- package/dist/{chunk-MXS5HWIH.mjs → chunk-7VALMHZF.mjs} +89 -17
- package/dist/{chunk-UKSUENHK.mjs → chunk-AF2CIKYC.mjs} +40 -26
- package/dist/{chunk-BVOJIYRV.mjs → chunk-BB7MASRQ.mjs} +1 -1
- package/dist/{chunk-A4G7JSYO.mjs → chunk-C45FUYPI.mjs} +98 -49
- package/dist/{chunk-6VZ3Q73N.mjs → chunk-CHOGQSHK.mjs} +44 -25
- package/dist/{chunk-QZN3MIJ3.mjs → chunk-ES33L3HO.mjs} +2 -2
- package/dist/{chunk-67Z2V6TC.mjs → chunk-EWBXO45L.mjs} +3 -3
- package/dist/{chunk-3TTYK5A7.mjs → chunk-EX62BQT6.mjs} +9 -9
- package/dist/{chunk-3RDS4UHC.mjs → chunk-EZD4MWOU.mjs} +4 -4
- package/dist/{chunk-YCJLVNWR.mjs → chunk-F52QGNZ2.mjs} +1 -1
- package/dist/{chunk-43ORORN6.mjs → chunk-GJZHU4FC.mjs} +3 -3
- package/dist/{chunk-7OQHSDK6.mjs → chunk-J53IHLG7.mjs} +7 -7
- package/dist/{chunk-R7OSDAFK.mjs → chunk-K6A4FCVR.mjs} +1 -1
- package/dist/chunk-KOKM4PQN.mjs +52 -0
- package/dist/{chunk-T6CNO72J.mjs → chunk-QM3VPX4Z.mjs} +2 -2
- package/dist/{chunk-T4DB5YFB.mjs → chunk-QWWCLRXX.mjs} +6 -6
- package/dist/chunk-QY7GC2MW.mjs +33 -0
- package/dist/{chunk-BJ2YKRNX.mjs → chunk-R36MIGBU.mjs} +86 -54
- package/dist/{chunk-RVM7AODE.mjs → chunk-RDJKYKMQ.mjs} +2 -2
- package/dist/{chunk-UXQE2ROB.mjs → chunk-RUIXODYU.mjs} +3 -2
- package/dist/{chunk-5ER245FI.mjs → chunk-SBCEHIPL.mjs} +3 -3
- package/dist/{chunk-TPJU7NDJ.mjs → chunk-SYWGORQB.mjs} +1 -0
- package/dist/{chunk-XLJ2S2BH.mjs → chunk-UAUZGXJP.mjs} +40 -21
- package/dist/{chunk-EONIHG3V.mjs → chunk-WBS3EYC4.mjs} +7 -7
- package/dist/{chunk-CO4MRKST.mjs → chunk-WLOF6HSP.mjs} +2 -2
- package/dist/{chunk-ULRPNYAG.mjs → chunk-Z3MOCHG5.mjs} +120 -62
- package/dist/{chunk-MKWUXWQH.mjs → chunk-ZHS44YKF.mjs} +1 -1
- package/dist/{client-Cc42b42j.d.ts → client-CgayPw2R.d.ts} +2 -2
- package/dist/client.d.ts +6 -6
- package/dist/client.mjs +1 -0
- package/dist/{config-DyRowvVP.d.ts → config-3kBsQ0Pi.d.ts} +1 -1
- package/dist/{config-BYne-yCI.d.ts → config-B-RnO6dr.d.ts} +1 -1
- package/dist/direct-executor-IJTRU3YN.mjs +22 -0
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +5 -5
- package/dist/distribution.mjs +26 -26
- package/dist/{durability-wiring-6AN4QTT3.mjs → durability-wiring-Y55SCPFC.mjs} +518 -91
- package/dist/embeddings/openai.mjs +1 -1
- package/dist/{engine-96_kLtSY.d.ts → engine-DSUfY2F0.d.ts} +11 -11
- package/dist/index.browser.mjs +2 -2
- package/dist/index.d.ts +5 -5
- package/dist/index.mjs +18 -18
- package/dist/{memory-dd3GESdc.d.ts → memory-wuXcYOD1.d.ts} +96 -76
- package/dist/{module-WIDYIJLY.mjs → module-VITS4ZR5.mjs} +72 -38
- package/dist/partial-words.mjs +5 -5
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.mjs +21 -21
- package/dist/react.d.ts +6 -6
- package/dist/react.mjs +1 -0
- package/dist/release-BH3DKW3T.mjs +33 -0
- package/dist/replace-CEO5KJJ3.mjs +57 -0
- package/dist/restore-queue-4SG45BLF.mjs +35 -0
- package/dist/server.d.ts +8 -8
- package/dist/server.mjs +10 -10
- package/dist/snapshot-XBKE7PUG.mjs +738 -0
- package/dist/synonyms.mjs +3 -3
- package/dist/{tasks-DA1Ff0G4.d.ts → tasks-Bj9WBlFG.d.ts} +1 -1
- package/dist/{thread-VTDLLYXC.mjs → thread-U7XYJWGG.mjs} +15 -12
- package/dist/vector/search-worker.mjs +6 -6
- package/dist/workers/entry.mjs +5 -5
- package/dist/{write-OJR3H4BI.mjs → write-BMNV7P2H.mjs} +20 -20
- package/package.json +7 -7
- package/dist/chunk-Y2F66IQJ.mjs +0 -74
- package/dist/chunk-ZJ4CSAB2.mjs +0 -13
- package/dist/direct-executor-CHXVCBN7.mjs +0 -22
- package/dist/snapshot-JFZVOGZV.mjs +0 -479
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { nodeThreadId, spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
|
|
2
|
-
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, decodeSegmentManifest, readListedFile,
|
|
3
|
-
import { applyDeleteEntry, applyIndexEntry } from './chunk-
|
|
4
|
-
import { encodeDocument, fnv1a, requireValidFieldTypes, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-
|
|
5
|
-
import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-
|
|
6
|
-
import { NarsilError, ErrorCodes, compareCodePoints, detectRuntime, MIN_M, resolveWorkerEntry, MAX_FILE_BYTES } from './chunk-
|
|
2
|
+
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, decodeSegmentManifest, readListedFile, adapterCheckpointStore, readMetadataEnvelope, readSegmentManifest, reclaimUnlistedKeys, snapshotBundleKey, manifestReferencedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, writeMetadataEnvelope, metadataFromConfig, DIRECTORY_LOCK_ATTEMPTS, createSeqOwner, readCurrentManifest, manifestFileRefs, DIRECTORY_LOCK_SETTLE_MS, commitCheckpointManifest, removeCheckpointGarbage, readSegmentFile, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, SINGLE_NODE_PRIMARY_TERM, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, frameRecord, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, readCommitMarker, isId, isUint32, isSequenceNumber, readWalSegments, durableRecordMissing, growPartitionsTo, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, MAX_SEGMENTS_PER_PARTITION, VECTOR_FILE_ENVELOPE_VERSION, writeSegmentHeader, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, SEGMENT_HEADER_SIZE, documentsOfPayloadV2, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, LOG_RECORDS_PER_CHECKPOINT, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, metadataConfigOf, buildEntry } from './chunk-4JTBL63P.mjs';
|
|
3
|
+
import { applyDeleteEntry, applyIndexEntry } from './chunk-PP6N5CCS.mjs';
|
|
4
|
+
import { encodeDocument, fnv1a, requireValidFieldTypes, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-HC6DMR4T.mjs';
|
|
5
|
+
import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-WKPJ4E73.mjs';
|
|
6
|
+
import { NarsilError, ErrorCodes, compareCodePoints, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, detectRuntime, MIN_M, resolveWorkerEntry, MAX_FILE_BYTES } from './chunk-HARBSUWP.mjs';
|
|
7
7
|
import { decode } from '@msgpack/msgpack';
|
|
8
8
|
|
|
9
9
|
// src/persistence/durability/node-fs.browser.ts
|
|
@@ -491,40 +491,415 @@ async function listRecursive(dir, base, pathMod, fs) {
|
|
|
491
491
|
return results;
|
|
492
492
|
}
|
|
493
493
|
|
|
494
|
-
// src/persistence/durability/
|
|
494
|
+
// src/persistence/durability/sha256.ts
|
|
495
|
+
var ROUND_CONSTANTS = new Uint32Array([
|
|
496
|
+
1116352408,
|
|
497
|
+
1899447441,
|
|
498
|
+
3049323471,
|
|
499
|
+
3921009573,
|
|
500
|
+
961987163,
|
|
501
|
+
1508970993,
|
|
502
|
+
2453635748,
|
|
503
|
+
2870763221,
|
|
504
|
+
3624381080,
|
|
505
|
+
310598401,
|
|
506
|
+
607225278,
|
|
507
|
+
1426881987,
|
|
508
|
+
1925078388,
|
|
509
|
+
2162078206,
|
|
510
|
+
2614888103,
|
|
511
|
+
3248222580,
|
|
512
|
+
3835390401,
|
|
513
|
+
4022224774,
|
|
514
|
+
264347078,
|
|
515
|
+
604807628,
|
|
516
|
+
770255983,
|
|
517
|
+
1249150122,
|
|
518
|
+
1555081692,
|
|
519
|
+
1996064986,
|
|
520
|
+
2554220882,
|
|
521
|
+
2821834349,
|
|
522
|
+
2952996808,
|
|
523
|
+
3210313671,
|
|
524
|
+
3336571891,
|
|
525
|
+
3584528711,
|
|
526
|
+
113926993,
|
|
527
|
+
338241895,
|
|
528
|
+
666307205,
|
|
529
|
+
773529912,
|
|
530
|
+
1294757372,
|
|
531
|
+
1396182291,
|
|
532
|
+
1695183700,
|
|
533
|
+
1986661051,
|
|
534
|
+
2177026350,
|
|
535
|
+
2456956037,
|
|
536
|
+
2730485921,
|
|
537
|
+
2820302411,
|
|
538
|
+
3259730800,
|
|
539
|
+
3345764771,
|
|
540
|
+
3516065817,
|
|
541
|
+
3600352804,
|
|
542
|
+
4094571909,
|
|
543
|
+
275423344,
|
|
544
|
+
430227734,
|
|
545
|
+
506948616,
|
|
546
|
+
659060556,
|
|
547
|
+
883997877,
|
|
548
|
+
958139571,
|
|
549
|
+
1322822218,
|
|
550
|
+
1537002063,
|
|
551
|
+
1747873779,
|
|
552
|
+
1955562222,
|
|
553
|
+
2024104815,
|
|
554
|
+
2227730452,
|
|
555
|
+
2361852424,
|
|
556
|
+
2428436474,
|
|
557
|
+
2756734187,
|
|
558
|
+
3204031479,
|
|
559
|
+
3329325298
|
|
560
|
+
]);
|
|
561
|
+
var INITIAL_STATE = [1779033703, 3144134277, 1013904242, 2773480762, 1359893119, 2600822924, 528734635, 1541459225];
|
|
562
|
+
var BLOCK_BYTES = 64;
|
|
563
|
+
var LENGTH_BYTES = 8;
|
|
564
|
+
function rotateRight(value, bits) {
|
|
565
|
+
return value >>> bits | value << 32 - bits;
|
|
566
|
+
}
|
|
567
|
+
function padded(bytes) {
|
|
568
|
+
const total = Math.ceil((bytes.byteLength + 1 + LENGTH_BYTES) / BLOCK_BYTES) * BLOCK_BYTES;
|
|
569
|
+
const message = new Uint8Array(total);
|
|
570
|
+
message.set(bytes);
|
|
571
|
+
message[bytes.byteLength] = 128;
|
|
572
|
+
const view = new DataView(message.buffer);
|
|
573
|
+
const bitLength = bytes.byteLength * 8;
|
|
574
|
+
view.setUint32(total - LENGTH_BYTES, Math.floor(bitLength / 4294967296), false);
|
|
575
|
+
view.setUint32(total - 4, bitLength >>> 0, false);
|
|
576
|
+
return view;
|
|
577
|
+
}
|
|
578
|
+
function sha256(bytes) {
|
|
579
|
+
const view = padded(bytes);
|
|
580
|
+
const state = INITIAL_STATE.slice();
|
|
581
|
+
const words = new Uint32Array(64);
|
|
582
|
+
for (let block = 0; block < view.byteLength; block += BLOCK_BYTES) {
|
|
583
|
+
for (let i = 0; i < 16; i++) words[i] = view.getUint32(block + i * 4, false);
|
|
584
|
+
for (let i = 16; i < 64; i++) {
|
|
585
|
+
const w15 = words[i - 15];
|
|
586
|
+
const w2 = words[i - 2];
|
|
587
|
+
const s0 = rotateRight(w15, 7) ^ rotateRight(w15, 18) ^ w15 >>> 3;
|
|
588
|
+
const s1 = rotateRight(w2, 17) ^ rotateRight(w2, 19) ^ w2 >>> 10;
|
|
589
|
+
words[i] = words[i - 16] + s0 + words[i - 7] + s1 >>> 0;
|
|
590
|
+
}
|
|
591
|
+
let [a, b, c, d, e, f, g, h] = state;
|
|
592
|
+
for (let i = 0; i < 64; i++) {
|
|
593
|
+
const s1 = rotateRight(e, 6) ^ rotateRight(e, 11) ^ rotateRight(e, 25);
|
|
594
|
+
const choice = e & f ^ ~e & g;
|
|
595
|
+
const t1 = h + s1 + choice + ROUND_CONSTANTS[i] + words[i] >>> 0;
|
|
596
|
+
const s0 = rotateRight(a, 2) ^ rotateRight(a, 13) ^ rotateRight(a, 22);
|
|
597
|
+
const majority = a & b ^ a & c ^ b & c;
|
|
598
|
+
const t2 = s0 + majority >>> 0;
|
|
599
|
+
h = g;
|
|
600
|
+
g = f;
|
|
601
|
+
f = e;
|
|
602
|
+
e = d + t1 >>> 0;
|
|
603
|
+
d = c;
|
|
604
|
+
c = b;
|
|
605
|
+
b = a;
|
|
606
|
+
a = t1 + t2 >>> 0;
|
|
607
|
+
}
|
|
608
|
+
state[0] = state[0] + a >>> 0;
|
|
609
|
+
state[1] = state[1] + b >>> 0;
|
|
610
|
+
state[2] = state[2] + c >>> 0;
|
|
611
|
+
state[3] = state[3] + d >>> 0;
|
|
612
|
+
state[4] = state[4] + e >>> 0;
|
|
613
|
+
state[5] = state[5] + f >>> 0;
|
|
614
|
+
state[6] = state[6] + g >>> 0;
|
|
615
|
+
state[7] = state[7] + h >>> 0;
|
|
616
|
+
}
|
|
617
|
+
const digest = new Uint8Array(32);
|
|
618
|
+
const out = new DataView(digest.buffer);
|
|
619
|
+
for (let i = 0; i < 8; i++) out.setUint32(i * 4, state[i], false);
|
|
620
|
+
return digest;
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
// src/persistence/durability/restore-journal.ts
|
|
624
|
+
var RESTORE_JOURNAL_PREFIX = ".swap-";
|
|
495
625
|
var RESTORE_STAGING_PREFIX = ".restore-";
|
|
496
|
-
|
|
497
|
-
|
|
626
|
+
var RETIRED_INDEX_PREFIX = ".retired-";
|
|
627
|
+
var LOCK_FILE_KEY = ".narsil.lock";
|
|
628
|
+
var STAGING_PREFIX_PATTERN = /^\.[A-Za-z0-9][A-Za-z0-9._-]*\/$/;
|
|
629
|
+
function digestOf(indexName) {
|
|
630
|
+
let hex = "";
|
|
631
|
+
for (const byte of sha256(new TextEncoder().encode(indexName))) hex += byte.toString(16).padStart(2, "0");
|
|
632
|
+
return hex;
|
|
633
|
+
}
|
|
634
|
+
function journalKey(indexName) {
|
|
635
|
+
return `${RESTORE_JOURNAL_PREFIX}${digestOf(indexName)}`;
|
|
636
|
+
}
|
|
637
|
+
function randomSuffix() {
|
|
638
|
+
return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
|
|
639
|
+
}
|
|
640
|
+
function newStagingPrefix() {
|
|
641
|
+
return `${RESTORE_STAGING_PREFIX}${randomSuffix()}/`;
|
|
642
|
+
}
|
|
643
|
+
function newRetiredPrefix() {
|
|
644
|
+
return `${RETIRED_INDEX_PREFIX}${randomSuffix()}/`;
|
|
645
|
+
}
|
|
646
|
+
function encodeJournal(indexName, stagingPrefix) {
|
|
647
|
+
return new TextEncoder().encode(`${indexName}
|
|
648
|
+
${stagingPrefix}`);
|
|
649
|
+
}
|
|
650
|
+
function isStagingPrefix(prefix) {
|
|
651
|
+
return STAGING_PREFIX_PATTERN.test(prefix) && !prefix.includes("..");
|
|
652
|
+
}
|
|
653
|
+
function corruptJournal(key, reason) {
|
|
654
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The restore journal "${key}" ${reason}`, { key });
|
|
655
|
+
}
|
|
656
|
+
function readJournal(key, bytes) {
|
|
657
|
+
const text = new TextDecoder().decode(bytes);
|
|
658
|
+
const lineEnd = text.indexOf("\n");
|
|
659
|
+
const indexName = lineEnd === -1 ? text : text.slice(0, lineEnd);
|
|
660
|
+
if (key.slice(RESTORE_JOURNAL_PREFIX.length) !== digestOf(indexName)) return { kind: "leftover" };
|
|
661
|
+
if (indexName.length > MAX_INDEX_NAME_LENGTH || !INDEX_NAME_PATTERN.test(indexName) || indexName.includes("..")) {
|
|
662
|
+
corruptJournal(key, "names an index that no index name can match");
|
|
663
|
+
}
|
|
664
|
+
const stagingPrefix = lineEnd === -1 ? "" : text.slice(lineEnd + 1);
|
|
665
|
+
if (!isStagingPrefix(stagingPrefix) || stagingPrefix === `${LOCK_FILE_KEY}/`) {
|
|
666
|
+
corruptJournal(key, "names no staging prefix that a restore writes");
|
|
667
|
+
}
|
|
668
|
+
return { kind: "journal", indexName, stagingPrefix };
|
|
669
|
+
}
|
|
670
|
+
async function listedKeys(storage, indexName, stagingPrefix) {
|
|
671
|
+
const manifestBytes = await storage.read(`${stagingPrefix}manifest`) ?? await storage.read(manifestKey(indexName));
|
|
672
|
+
if (manifestBytes === null) {
|
|
673
|
+
throw new NarsilError(
|
|
674
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
675
|
+
`The restore of index "${indexName}" lost its manifest, both from staging and from the index`,
|
|
676
|
+
{ indexName, stagingPrefix }
|
|
677
|
+
);
|
|
678
|
+
}
|
|
679
|
+
const decoded = await decodeSegmentManifest(manifestBytes, indexName);
|
|
680
|
+
if (decoded.kind !== "current") {
|
|
681
|
+
throw new NarsilError(
|
|
682
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
683
|
+
`The restore of index "${indexName}" staged a manifest in the layout of an earlier release`,
|
|
684
|
+
{ indexName }
|
|
685
|
+
);
|
|
686
|
+
}
|
|
687
|
+
const root = `${indexName}/`;
|
|
688
|
+
const keys = /* @__PURE__ */ new Set(["meta", "manifest"]);
|
|
689
|
+
for (const ref of manifestFileRefs(decoded.manifest)) keys.add(ref.key.slice(root.length));
|
|
690
|
+
return keys;
|
|
691
|
+
}
|
|
692
|
+
function withoutTrailingSlash(prefix) {
|
|
693
|
+
return prefix.slice(0, -1);
|
|
694
|
+
}
|
|
695
|
+
async function finishJournal(storage, indexName, stagingPrefix) {
|
|
696
|
+
const root = `${indexName}/`;
|
|
697
|
+
const listed = await listedKeys(storage, indexName, stagingPrefix);
|
|
698
|
+
const staged = await storage.list(stagingPrefix);
|
|
699
|
+
let retiredPrefix = null;
|
|
700
|
+
if (storage.moveTree !== void 0 && staged.length > 0) {
|
|
701
|
+
retiredPrefix = newRetiredPrefix();
|
|
702
|
+
await storage.moveTree(indexName, withoutTrailingSlash(retiredPrefix));
|
|
703
|
+
await storage.moveTree(withoutTrailingSlash(stagingPrefix), indexName);
|
|
704
|
+
} else {
|
|
705
|
+
const stagedKeys = new Set(staged);
|
|
706
|
+
for (const key of listed) {
|
|
707
|
+
if (stagedKeys.has(`${stagingPrefix}${key}`)) await storage.moveKey(`${stagingPrefix}${key}`, `${root}${key}`);
|
|
708
|
+
}
|
|
709
|
+
}
|
|
710
|
+
for (const key of await storage.list(root)) {
|
|
711
|
+
if (!listed.has(key.slice(root.length))) await storage.remove(key);
|
|
712
|
+
}
|
|
713
|
+
await storage.remove(journalKey(indexName));
|
|
714
|
+
return retiredPrefix;
|
|
715
|
+
}
|
|
716
|
+
async function removePrefix(storage, prefix) {
|
|
717
|
+
if (storage.removeTree !== void 0) {
|
|
718
|
+
await storage.removeTree(prefix);
|
|
719
|
+
return;
|
|
720
|
+
}
|
|
721
|
+
for (const key of await storage.list(prefix)) await storage.remove(key);
|
|
722
|
+
}
|
|
723
|
+
async function commitJournal(storage, indexName, stagingPrefix) {
|
|
724
|
+
await storage.writeJournal(journalKey(indexName), encodeJournal(indexName, stagingPrefix));
|
|
725
|
+
}
|
|
726
|
+
function isTopLevelJournal(key) {
|
|
727
|
+
return key.startsWith(RESTORE_JOURNAL_PREFIX) && !key.includes("/");
|
|
728
|
+
}
|
|
729
|
+
async function removeStagingPrefixes(storage) {
|
|
730
|
+
const prefixes = /* @__PURE__ */ new Set();
|
|
731
|
+
for (const key of await storage.list(".")) {
|
|
732
|
+
const slash = key.indexOf("/");
|
|
733
|
+
if (slash !== -1 && isStagingPrefix(key.slice(0, slash + 1))) prefixes.add(key.slice(0, slash + 1));
|
|
734
|
+
}
|
|
735
|
+
for (const prefix of prefixes) await removePrefix(storage, prefix);
|
|
736
|
+
}
|
|
737
|
+
async function recoverRestoreJournals(storage) {
|
|
738
|
+
for (const key of await storage.list(RESTORE_JOURNAL_PREFIX)) {
|
|
739
|
+
if (!isTopLevelJournal(key)) continue;
|
|
740
|
+
const bytes = await storage.read(key);
|
|
741
|
+
if (bytes === null) continue;
|
|
742
|
+
const reading = readJournal(key, bytes);
|
|
743
|
+
if (reading.kind === "leftover") {
|
|
744
|
+
await storage.remove(key);
|
|
745
|
+
continue;
|
|
746
|
+
}
|
|
747
|
+
await finishJournal(storage, reading.indexName, reading.stagingPrefix);
|
|
748
|
+
}
|
|
749
|
+
await removeStagingPrefixes(storage);
|
|
750
|
+
}
|
|
751
|
+
|
|
752
|
+
// src/persistence/durability/restore-directory.ts
|
|
753
|
+
function missing(error) {
|
|
754
|
+
return error.code === "ENOENT";
|
|
755
|
+
}
|
|
756
|
+
function storageFailed(code, message, key, error) {
|
|
757
|
+
return new NarsilError(code, message, { key, cause: error instanceof Error ? error.message : String(error) });
|
|
758
|
+
}
|
|
759
|
+
function directoryRestoreStorage(directory) {
|
|
760
|
+
return {
|
|
761
|
+
read: (key) => directory.read(key),
|
|
762
|
+
write: (key, chunks) => directory.atomicWrite(key, chunks),
|
|
763
|
+
writeJournal: (key, bytes) => directory.atomicWrite(key, bytes),
|
|
764
|
+
remove: (key) => directory.remove(key),
|
|
765
|
+
list: (prefix) => directory.list(prefix),
|
|
766
|
+
moveKey: (from, to) => directory.move(from, to),
|
|
767
|
+
pathOf: (key) => directory.pathOf(key),
|
|
768
|
+
async moveTree(from, to) {
|
|
769
|
+
const source = await directory.pathOf(from);
|
|
770
|
+
const target = await directory.pathOf(to);
|
|
771
|
+
const fs = await getFs();
|
|
772
|
+
try {
|
|
773
|
+
await fs.rename(source, target);
|
|
774
|
+
} catch (error) {
|
|
775
|
+
if (missing(error)) return;
|
|
776
|
+
throw storageFailed(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Failed to move "${from}" to "${to}"`, from, error);
|
|
777
|
+
}
|
|
778
|
+
await directory.syncDirectoryOf(to);
|
|
779
|
+
},
|
|
780
|
+
async removeTree(prefix) {
|
|
781
|
+
const key = prefix.endsWith("/") ? prefix.slice(0, -1) : prefix;
|
|
782
|
+
const fs = await getFs();
|
|
783
|
+
try {
|
|
784
|
+
await fs.rm(await directory.pathOf(key), { recursive: true, force: true });
|
|
785
|
+
} catch (error) {
|
|
786
|
+
throw storageFailed(ErrorCodes.PERSISTENCE_DELETE_FAILED, `Failed to delete "${key}"`, key, error);
|
|
787
|
+
}
|
|
788
|
+
await directory.syncDirectoryOf(key);
|
|
789
|
+
}
|
|
790
|
+
};
|
|
791
|
+
}
|
|
792
|
+
function recoverDirectoryRestores(directory) {
|
|
793
|
+
return recoverRestoreJournals(directoryRestoreStorage(directory));
|
|
794
|
+
}
|
|
795
|
+
|
|
796
|
+
// src/persistence/durability/restore-staging.ts
|
|
797
|
+
function toError(error) {
|
|
798
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
799
|
+
}
|
|
800
|
+
function pathUnder(path, root) {
|
|
801
|
+
if (!path.startsWith(root)) return false;
|
|
802
|
+
const next = path.charAt(root.length);
|
|
803
|
+
return next === "" || next === "/" || next === "\\";
|
|
804
|
+
}
|
|
805
|
+
function createRestoreStaging(storage, indexName, options = {}) {
|
|
806
|
+
const stagingPrefix = newStagingPrefix();
|
|
498
807
|
const indexRoot = `${indexName}/`;
|
|
808
|
+
let committed = false;
|
|
499
809
|
const staged = (key) => {
|
|
500
810
|
if (!key.startsWith(indexRoot)) {
|
|
501
811
|
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `A restore of "${indexName}" cannot stage "${key}"`, {
|
|
502
812
|
key
|
|
503
813
|
});
|
|
504
814
|
}
|
|
505
|
-
return `${
|
|
815
|
+
return `${stagingPrefix}${key.slice(indexRoot.length)}`;
|
|
506
816
|
};
|
|
507
|
-
const unstaged = (key) => `${indexRoot}${key.slice(
|
|
508
|
-
const
|
|
509
|
-
read: (key) =>
|
|
510
|
-
write: (key, chunks) =>
|
|
511
|
-
remove: (key) =>
|
|
512
|
-
list: async (prefix) => (await
|
|
513
|
-
pathOf: (key) => directory.pathOf(staged(key))
|
|
817
|
+
const unstaged = (key) => `${indexRoot}${key.slice(stagingPrefix.length)}`;
|
|
818
|
+
const store = {
|
|
819
|
+
read: (key) => storage.read(staged(key)),
|
|
820
|
+
write: (key, chunks) => storage.write(staged(key), chunks),
|
|
821
|
+
remove: (key) => storage.remove(staged(key)),
|
|
822
|
+
list: async (prefix) => (await storage.list(staged(prefix))).map(unstaged)
|
|
514
823
|
};
|
|
824
|
+
const pathOf = storage.pathOf;
|
|
825
|
+
if (pathOf !== void 0) store.pathOf = (key) => pathOf.call(storage, staged(key));
|
|
826
|
+
let retiredPrefix = null;
|
|
827
|
+
async function finishOnStorage() {
|
|
828
|
+
retiredPrefix = await finishJournal(storage, indexName, stagingPrefix);
|
|
829
|
+
}
|
|
515
830
|
return {
|
|
516
|
-
store
|
|
831
|
+
store,
|
|
832
|
+
async relocation() {
|
|
833
|
+
if (pathOf === void 0 || (await storage.list(stagingPrefix)).length > 0) return null;
|
|
834
|
+
const from = await pathOf.call(storage, stagingPrefix.slice(0, -1));
|
|
835
|
+
const to = await pathOf.call(storage, indexName);
|
|
836
|
+
return (path) => pathUnder(path, from) ? `${to}${path.slice(from.length)}` : path;
|
|
837
|
+
},
|
|
517
838
|
async commit() {
|
|
518
|
-
|
|
519
|
-
|
|
839
|
+
try {
|
|
840
|
+
await commitJournal(storage, indexName, stagingPrefix);
|
|
841
|
+
} catch (error) {
|
|
842
|
+
try {
|
|
843
|
+
await storage.remove(journalKey(indexName));
|
|
844
|
+
} catch (removal) {
|
|
845
|
+
committed = true;
|
|
846
|
+
options.onStorageLost?.(toError(removal));
|
|
847
|
+
}
|
|
848
|
+
throw error;
|
|
849
|
+
}
|
|
850
|
+
committed = true;
|
|
851
|
+
},
|
|
852
|
+
async finish() {
|
|
853
|
+
try {
|
|
854
|
+
await (options.finish === void 0 ? finishOnStorage() : options.finish(finishOnStorage));
|
|
855
|
+
} catch (error) {
|
|
856
|
+
options.onStorageLost?.(toError(error));
|
|
857
|
+
}
|
|
858
|
+
},
|
|
859
|
+
async removeRetired() {
|
|
860
|
+
const prefix = retiredPrefix;
|
|
861
|
+
retiredPrefix = null;
|
|
862
|
+
if (prefix !== null) await removePrefix(storage, prefix);
|
|
520
863
|
},
|
|
521
864
|
async discard() {
|
|
522
|
-
|
|
865
|
+
if (committed) return;
|
|
866
|
+
await removePrefix(storage, stagingPrefix);
|
|
523
867
|
}
|
|
524
868
|
};
|
|
525
869
|
}
|
|
526
|
-
|
|
527
|
-
|
|
870
|
+
function adapterRestoreStorage(adapter) {
|
|
871
|
+
const store = adapterCheckpointStore(adapter);
|
|
872
|
+
return {
|
|
873
|
+
read: (key) => store.read(key),
|
|
874
|
+
write: (key, chunks) => store.write(key, chunks),
|
|
875
|
+
writeJournal: (key, bytes) => store.write(key, [bytes]),
|
|
876
|
+
remove: (key) => store.remove(key),
|
|
877
|
+
list: (prefix) => store.list(prefix),
|
|
878
|
+
async moveKey(from, to) {
|
|
879
|
+
const bytes = await store.read(from);
|
|
880
|
+
if (bytes === null) return;
|
|
881
|
+
await store.write(to, [bytes]);
|
|
882
|
+
await store.remove(from);
|
|
883
|
+
}
|
|
884
|
+
};
|
|
885
|
+
}
|
|
886
|
+
|
|
887
|
+
// src/persistence/durability/archive-checkpoint.ts
|
|
888
|
+
function toError2(error) {
|
|
889
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
890
|
+
}
|
|
891
|
+
function stageDirectoryRestore(deps, indexName) {
|
|
892
|
+
return createRestoreStaging(directoryRestoreStorage(deps.directory), indexName, {
|
|
893
|
+
async finish(run) {
|
|
894
|
+
try {
|
|
895
|
+
await run();
|
|
896
|
+
} catch (error) {
|
|
897
|
+
deps.markFatal(toError2(error));
|
|
898
|
+
throw error;
|
|
899
|
+
}
|
|
900
|
+
},
|
|
901
|
+
onStorageLost: deps.markFatal
|
|
902
|
+
});
|
|
528
903
|
}
|
|
529
904
|
function releaser(held, indexName) {
|
|
530
905
|
let released = false;
|
|
@@ -575,7 +950,7 @@ function createArchiveCheckpointAccess(deps) {
|
|
|
575
950
|
}
|
|
576
951
|
},
|
|
577
952
|
stageRestore(indexName) {
|
|
578
|
-
return
|
|
953
|
+
return stageDirectoryRestore(deps, indexName);
|
|
579
954
|
},
|
|
580
955
|
adoptRestoredCheckpoint(indexName, checkpoint) {
|
|
581
956
|
for (const { partitionId, lastSeqNo } of checkpoint) deps.registerPartition(indexName, partitionId, lastSeqNo);
|
|
@@ -833,8 +1208,8 @@ async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartit
|
|
|
833
1208
|
const vectorPlans = /* @__PURE__ */ new Map();
|
|
834
1209
|
for (const [fieldPath, vectorIndex] of vectorIndexes) {
|
|
835
1210
|
const listed = listedVectorFields.find((ref) => ref.fieldPath === fieldPath);
|
|
836
|
-
const
|
|
837
|
-
vectorPlans.set(fieldPath, vectorIndex.planCheckpoint(
|
|
1211
|
+
const listedKeys2 = listed === void 0 ? null : listed.files.map((file) => file.key);
|
|
1212
|
+
vectorPlans.set(fieldPath, vectorIndex.planCheckpoint(listedKeys2));
|
|
838
1213
|
}
|
|
839
1214
|
const memoryMatchesTheTargets = recordsLeftInLog === 0;
|
|
840
1215
|
const wholePartitions = memoryMatchesTheTargets ? wholePartitionsOf(targets) : noWholePartitions();
|
|
@@ -1328,7 +1703,7 @@ async function drainIndexStateForUnload(indexState, pendingMetadataWrite, closeW
|
|
|
1328
1703
|
}
|
|
1329
1704
|
|
|
1330
1705
|
// src/persistence/durability/mutation-batch.ts
|
|
1331
|
-
function
|
|
1706
|
+
function toError3(err) {
|
|
1332
1707
|
return err instanceof Error ? err : new Error(String(err));
|
|
1333
1708
|
}
|
|
1334
1709
|
function failUnlessAlreadyFailed(outcomes, count, error) {
|
|
@@ -1340,7 +1715,7 @@ function failUnlessAlreadyFailed(outcomes, count, error) {
|
|
|
1340
1715
|
}
|
|
1341
1716
|
}
|
|
1342
1717
|
function failPartition(partition, deps, err) {
|
|
1343
|
-
const error =
|
|
1718
|
+
const error = toError3(err);
|
|
1344
1719
|
partition.failed = error;
|
|
1345
1720
|
deps.markFatal(error);
|
|
1346
1721
|
return error;
|
|
@@ -1420,7 +1795,7 @@ async function recordPartitionBatch(partition, records, deps) {
|
|
|
1420
1795
|
}
|
|
1421
1796
|
}
|
|
1422
1797
|
} catch (err) {
|
|
1423
|
-
failUnlessAlreadyFailed(outcomes, records.length,
|
|
1798
|
+
failUnlessAlreadyFailed(outcomes, records.length, toError3(err));
|
|
1424
1799
|
}
|
|
1425
1800
|
return outcomes;
|
|
1426
1801
|
}
|
|
@@ -2257,8 +2632,8 @@ async function createMarkerWriter(directory, indexName, partitionId) {
|
|
|
2257
2632
|
// src/persistence/durability/wal-writer.ts
|
|
2258
2633
|
var SEGMENT_TAIL_PATTERN = /^\d{16}$/;
|
|
2259
2634
|
function segmentKey(indexName, partitionId, startSeqNo) {
|
|
2260
|
-
const
|
|
2261
|
-
return `${indexName}/wal/${partitionId}/${
|
|
2635
|
+
const padded2 = startSeqNo.toString().padStart(16, "0");
|
|
2636
|
+
return `${indexName}/wal/${partitionId}/${padded2}`;
|
|
2262
2637
|
}
|
|
2263
2638
|
function parseSegmentStartSeqNo2(key) {
|
|
2264
2639
|
const tail = key.slice(key.lastIndexOf("/") + 1);
|
|
@@ -2504,14 +2879,14 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
2504
2879
|
stopTimers();
|
|
2505
2880
|
hooks.onFatalError(error);
|
|
2506
2881
|
}
|
|
2507
|
-
function
|
|
2882
|
+
function toError5(err) {
|
|
2508
2883
|
return err instanceof Error ? err : new Error(String(err));
|
|
2509
2884
|
}
|
|
2510
2885
|
async function closeWriterReportingFailure(closePromise) {
|
|
2511
2886
|
try {
|
|
2512
2887
|
await closePromise;
|
|
2513
2888
|
} catch (err) {
|
|
2514
|
-
markFatal(
|
|
2889
|
+
markFatal(toError5(err));
|
|
2515
2890
|
}
|
|
2516
2891
|
}
|
|
2517
2892
|
function getOrCreateIndexState(indexName) {
|
|
@@ -2577,7 +2952,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
2577
2952
|
try {
|
|
2578
2953
|
await partition.walWriter.commit();
|
|
2579
2954
|
} catch (err) {
|
|
2580
|
-
partition.failed =
|
|
2955
|
+
partition.failed = toError5(err);
|
|
2581
2956
|
markFatal(partition.failed);
|
|
2582
2957
|
}
|
|
2583
2958
|
}
|
|
@@ -2591,7 +2966,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
2591
2966
|
try {
|
|
2592
2967
|
await checkpointIndex(indexName);
|
|
2593
2968
|
} catch (err) {
|
|
2594
|
-
markFatal(
|
|
2969
|
+
markFatal(toError5(err));
|
|
2595
2970
|
}
|
|
2596
2971
|
}
|
|
2597
2972
|
}
|
|
@@ -2672,7 +3047,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
2672
3047
|
indexState.documentBytesSinceCheckpoint += recorded.documentBytes;
|
|
2673
3048
|
if (checkpointIsDue(indexState, checkpointMutationThreshold)) {
|
|
2674
3049
|
void checkpointIndex(indexName).catch((err) => {
|
|
2675
|
-
markFatal(
|
|
3050
|
+
markFatal(toError5(err));
|
|
2676
3051
|
});
|
|
2677
3052
|
}
|
|
2678
3053
|
await stallWhileCheckpointsFallBehind(indexState);
|
|
@@ -2704,12 +3079,10 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
2704
3079
|
);
|
|
2705
3080
|
}
|
|
2706
3081
|
return {
|
|
2707
|
-
isActive()
|
|
2708
|
-
return true;
|
|
2709
|
-
},
|
|
3082
|
+
isActive: () => true,
|
|
2710
3083
|
async recover(metadataOnly = false) {
|
|
2711
3084
|
await directoryWatch.claim();
|
|
2712
|
-
await
|
|
3085
|
+
await recoverDirectoryRestores(directory);
|
|
2713
3086
|
const names = await listPersistedIndexes(directory);
|
|
2714
3087
|
for (const indexName of names) {
|
|
2715
3088
|
await recoverIndex(indexName, metadataOnly);
|
|
@@ -2779,7 +3152,8 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
2779
3152
|
directory,
|
|
2780
3153
|
held: heldCheckpointFiles,
|
|
2781
3154
|
checkpoint: (name) => checkpointIndex(name),
|
|
2782
|
-
registerPartition: (name, partitionId, startSeqNo) => getOrCreatePartition(name, partitionId, startSeqNo)
|
|
3155
|
+
registerPartition: (name, partitionId, startSeqNo) => getOrCreatePartition(name, partitionId, startSeqNo),
|
|
3156
|
+
markFatal
|
|
2783
3157
|
}),
|
|
2784
3158
|
async shutdown() {
|
|
2785
3159
|
shuttingDown = true;
|
|
@@ -2799,7 +3173,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
2799
3173
|
};
|
|
2800
3174
|
}
|
|
2801
3175
|
|
|
2802
|
-
// src/persistence/durability/snapshot-only
|
|
3176
|
+
// src/persistence/durability/snapshot-only/checkpoint.ts
|
|
2803
3177
|
function capture(input, prior) {
|
|
2804
3178
|
const listedIds = new Set((prior?.partitions ?? []).map((partition) => partition.partitionId));
|
|
2805
3179
|
const partitionCountChanged = prior !== null && listedIds.size !== input.manager.partitionCount;
|
|
@@ -2855,7 +3229,44 @@ async function writeSnapshotOnlyCheckpoint(input) {
|
|
|
2855
3229
|
return captured.documentCount;
|
|
2856
3230
|
}
|
|
2857
3231
|
|
|
2858
|
-
// src/persistence/durability/snapshot-only.ts
|
|
3232
|
+
// src/persistence/durability/snapshot-only/restores.ts
|
|
3233
|
+
function toError4(error) {
|
|
3234
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
3235
|
+
}
|
|
3236
|
+
function createPendingRestores(onFatalError) {
|
|
3237
|
+
const pending = /* @__PURE__ */ new Map();
|
|
3238
|
+
return {
|
|
3239
|
+
finishLater(indexName, run) {
|
|
3240
|
+
let running = null;
|
|
3241
|
+
const finish = () => {
|
|
3242
|
+
running ??= run().then(
|
|
3243
|
+
() => {
|
|
3244
|
+
if (pending.get(indexName) === finish) pending.delete(indexName);
|
|
3245
|
+
},
|
|
3246
|
+
(error) => {
|
|
3247
|
+
running = null;
|
|
3248
|
+
throw error;
|
|
3249
|
+
}
|
|
3250
|
+
);
|
|
3251
|
+
return running;
|
|
3252
|
+
};
|
|
3253
|
+
pending.set(indexName, finish);
|
|
3254
|
+
void finish().catch((error) => onFatalError(toError4(error)));
|
|
3255
|
+
return Promise.resolve();
|
|
3256
|
+
},
|
|
3257
|
+
async settle(indexName) {
|
|
3258
|
+
await pending.get(indexName)?.();
|
|
3259
|
+
},
|
|
3260
|
+
forget(indexName) {
|
|
3261
|
+
pending.delete(indexName);
|
|
3262
|
+
},
|
|
3263
|
+
async settleAll() {
|
|
3264
|
+
for (const finish of [...pending.values()]) await finish().catch(() => void 0);
|
|
3265
|
+
}
|
|
3266
|
+
};
|
|
3267
|
+
}
|
|
3268
|
+
|
|
3269
|
+
// src/persistence/durability/snapshot-only/index.ts
|
|
2859
3270
|
function metadataKey(indexName) {
|
|
2860
3271
|
return `${indexName}/meta`;
|
|
2861
3272
|
}
|
|
@@ -2864,8 +3275,13 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
2864
3275
|
const checkpointMutationThreshold = config.checkpointMutationThreshold ?? DEFAULT_CHECKPOINT_MUTATION_THRESHOLD;
|
|
2865
3276
|
const indexes = /* @__PURE__ */ new Map();
|
|
2866
3277
|
const metadataWrites = /* @__PURE__ */ new Map();
|
|
3278
|
+
const restoreStorage = adapterRestoreStorage(adapter);
|
|
3279
|
+
const restores = createPendingRestores(hooks.onFatalError);
|
|
2867
3280
|
let checkpointTimer = null;
|
|
2868
3281
|
let shuttingDown = false;
|
|
3282
|
+
function settleRestore(indexName) {
|
|
3283
|
+
return restores.settle(indexName);
|
|
3284
|
+
}
|
|
2869
3285
|
function getOrCreateIndexState(indexName) {
|
|
2870
3286
|
let state = indexes.get(indexName);
|
|
2871
3287
|
if (state === void 0) {
|
|
@@ -2901,6 +3317,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
2901
3317
|
}
|
|
2902
3318
|
}
|
|
2903
3319
|
async function performCheckpoint(indexName, indexState, fromMemory) {
|
|
3320
|
+
await settleRestore(indexName);
|
|
2904
3321
|
const manager = hooks.getManager(indexName);
|
|
2905
3322
|
const metadata = hooks.buildMetadata(indexName);
|
|
2906
3323
|
if (manager === void 0 || metadata === void 0) {
|
|
@@ -2950,7 +3367,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
2950
3367
|
}
|
|
2951
3368
|
function queueMetadataWrite(indexName, write) {
|
|
2952
3369
|
const previous = metadataWrites.get(indexName) ?? Promise.resolve();
|
|
2953
|
-
const queued = previous.then(write);
|
|
3370
|
+
const queued = previous.then(() => settleRestore(indexName)).then(write);
|
|
2954
3371
|
metadataWrites.set(
|
|
2955
3372
|
indexName,
|
|
2956
3373
|
queued.catch(() => void 0)
|
|
@@ -2958,6 +3375,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
2958
3375
|
return queued;
|
|
2959
3376
|
}
|
|
2960
3377
|
async function recoverIndex(indexName, metadataOnly = false) {
|
|
3378
|
+
await settleRestore(indexName);
|
|
2961
3379
|
const metaBytes = await adapter.load(metadataKey(indexName));
|
|
2962
3380
|
if (metaBytes === null) {
|
|
2963
3381
|
return;
|
|
@@ -3044,11 +3462,28 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
3044
3462
|
}
|
|
3045
3463
|
return appliedSeqNo;
|
|
3046
3464
|
}
|
|
3465
|
+
async function drainIndex(indexName) {
|
|
3466
|
+
const indexState = indexes.get(indexName);
|
|
3467
|
+
if (indexState !== void 0) {
|
|
3468
|
+
indexState.unloading = true;
|
|
3469
|
+
try {
|
|
3470
|
+
await indexState.applyChain;
|
|
3471
|
+
if (indexState.checkpointInFlight !== null) await indexState.checkpointInFlight;
|
|
3472
|
+
await metadataWrites.get(indexName);
|
|
3473
|
+
} catch (error) {
|
|
3474
|
+
indexState.unloading = false;
|
|
3475
|
+
throw error;
|
|
3476
|
+
}
|
|
3477
|
+
}
|
|
3478
|
+
indexes.delete(indexName);
|
|
3479
|
+
metadataWrites.delete(indexName);
|
|
3480
|
+
}
|
|
3047
3481
|
return {
|
|
3048
3482
|
isActive() {
|
|
3049
3483
|
return true;
|
|
3050
3484
|
},
|
|
3051
3485
|
async recover(metadataOnly = false) {
|
|
3486
|
+
await recoverRestoreJournals(restoreStorage);
|
|
3052
3487
|
const keys = await adapter.list("");
|
|
3053
3488
|
for (const key of keys) {
|
|
3054
3489
|
if (key.endsWith("/meta")) {
|
|
@@ -3097,40 +3532,31 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
3097
3532
|
await checkpointIndex(indexName);
|
|
3098
3533
|
}
|
|
3099
3534
|
},
|
|
3100
|
-
|
|
3101
|
-
|
|
3102
|
-
|
|
3103
|
-
|
|
3104
|
-
|
|
3105
|
-
|
|
3106
|
-
|
|
3107
|
-
|
|
3108
|
-
|
|
3109
|
-
|
|
3110
|
-
|
|
3111
|
-
}
|
|
3535
|
+
stageRestore(indexName) {
|
|
3536
|
+
return createRestoreStaging(restoreStorage, indexName, {
|
|
3537
|
+
finish: (run) => restores.finishLater(indexName, run),
|
|
3538
|
+
onStorageLost: hooks.onFatalError
|
|
3539
|
+
});
|
|
3540
|
+
},
|
|
3541
|
+
adoptRestoredCheckpoint(indexName, checkpoint) {
|
|
3542
|
+
const indexState = getOrCreateIndexState(indexName);
|
|
3543
|
+
indexState.appliedSeqNoByPartition.clear();
|
|
3544
|
+
for (const { partitionId, lastSeqNo } of checkpoint) {
|
|
3545
|
+
if (lastSeqNo > 0) indexState.appliedSeqNoByPartition.set(partitionId, lastSeqNo);
|
|
3112
3546
|
}
|
|
3113
|
-
|
|
3114
|
-
|
|
3547
|
+
},
|
|
3548
|
+
async removeIndex(indexName) {
|
|
3549
|
+
await settleRestore(indexName).catch(() => void 0);
|
|
3550
|
+
restores.forget(indexName);
|
|
3551
|
+
await restoreStorage.remove(journalKey(indexName));
|
|
3552
|
+
await drainIndex(indexName);
|
|
3115
3553
|
for (const key of await adapter.list(`${indexName}/`)) {
|
|
3116
3554
|
await adapter.delete(key);
|
|
3117
3555
|
}
|
|
3118
3556
|
},
|
|
3119
3557
|
async unloadIndex(indexName) {
|
|
3120
|
-
|
|
3121
|
-
|
|
3122
|
-
indexState.unloading = true;
|
|
3123
|
-
try {
|
|
3124
|
-
await indexState.applyChain;
|
|
3125
|
-
if (indexState.checkpointInFlight !== null) await indexState.checkpointInFlight;
|
|
3126
|
-
await metadataWrites.get(indexName);
|
|
3127
|
-
} catch (error) {
|
|
3128
|
-
indexState.unloading = false;
|
|
3129
|
-
throw error;
|
|
3130
|
-
}
|
|
3131
|
-
}
|
|
3132
|
-
indexes.delete(indexName);
|
|
3133
|
-
metadataWrites.delete(indexName);
|
|
3558
|
+
await settleRestore(indexName);
|
|
3559
|
+
await drainIndex(indexName);
|
|
3134
3560
|
},
|
|
3135
3561
|
async reloadIndex(indexName) {
|
|
3136
3562
|
const indexState = getOrCreateIndexState(indexName);
|
|
@@ -3144,6 +3570,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
3144
3570
|
clearInterval(checkpointTimer);
|
|
3145
3571
|
checkpointTimer = null;
|
|
3146
3572
|
}
|
|
3573
|
+
await restores.settleAll();
|
|
3147
3574
|
for (const indexState of indexes.values()) {
|
|
3148
3575
|
if (indexState.checkpointInFlight !== null) {
|
|
3149
3576
|
await indexState.checkpointInFlight.catch(() => void 0);
|