@delali/narsil 0.3.1-next.90 → 0.3.1-next.94
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/browser/{chunk-3GZQKGVQ.mjs → chunk-4JTBL63P.mjs} +13 -5
- package/dist/browser/{chunk-3ZRMZIH4.mjs → chunk-5S74HQRT.mjs} +208 -78
- package/dist/browser/{chunk-FHFFWLEK.mjs → chunk-HARBSUWP.mjs} +3 -4
- package/dist/browser/{chunk-RAHUQOJL.mjs → chunk-HC6DMR4T.mjs} +2445 -2107
- package/dist/browser/{chunk-JACCR3BJ.mjs → chunk-PP6N5CCS.mjs} +2 -2
- package/dist/browser/{chunk-NQW2YXEX.mjs → chunk-WKPJ4E73.mjs} +1 -1
- package/dist/browser/{durability-wiring-VSUBLUDK.mjs → durability-wiring-7DKYHKT5.mjs} +498 -71
- package/dist/browser/{module-E25UDADM.mjs → module-O5WH32EL.mjs} +102 -54
- package/dist/browser/release-76MXRPGV.mjs +33 -0
- package/dist/browser/replace-RURXTGVG.mjs +57 -0
- package/dist/browser/restore-queue-Z7FZZNKH.mjs +35 -0
- package/dist/browser/{snapshot-5MXBEDXF.mjs → snapshot-PPTZ565G.mjs} +368 -103
- package/dist/browser/{write-6SBYTS62.mjs → write-2ZAA7UDH.mjs} +4 -4
- package/dist/{capabilities-SJPttcgQ.d.ts → capabilities-xsJfdFoQ.d.ts} +11 -3
- package/dist/chunk-2GZGBOQI.mjs +426 -0
- package/dist/{chunk-CVQWQVFX.mjs → chunk-4ZJWRETC.mjs} +2 -2
- package/dist/chunk-7KC56574.mjs +507 -0
- package/dist/{chunk-MXS5HWIH.mjs → chunk-7VALMHZF.mjs} +89 -17
- package/dist/chunk-7X5JST5L.mjs +114 -0
- package/dist/{chunk-UKSUENHK.mjs → chunk-AF2CIKYC.mjs} +40 -26
- package/dist/{chunk-BVOJIYRV.mjs → chunk-BB7MASRQ.mjs} +1 -1
- package/dist/{chunk-R6BMMQXH.mjs → chunk-C45FUYPI.mjs} +193 -79
- package/dist/{chunk-EMXAYPTQ.mjs → chunk-CHOGQSHK.mjs} +2326 -3503
- package/dist/{chunk-QZN3MIJ3.mjs → chunk-ES33L3HO.mjs} +2 -2
- package/dist/{chunk-VY4DXZ5P.mjs → chunk-EWBXO45L.mjs} +3 -3
- package/dist/{chunk-QAMUC2BJ.mjs → chunk-EX62BQT6.mjs} +10 -9
- package/dist/{chunk-MWJIAREY.mjs → chunk-EZD4MWOU.mjs} +4 -4
- package/dist/{chunk-YCJLVNWR.mjs → chunk-F52QGNZ2.mjs} +1 -1
- package/dist/{chunk-URGLSNKR.mjs → chunk-GJZHU4FC.mjs} +202 -16
- package/dist/{chunk-S4LQDSUL.mjs → chunk-J53IHLG7.mjs} +7 -7
- package/dist/{chunk-R7OSDAFK.mjs → chunk-K6A4FCVR.mjs} +1 -1
- package/dist/chunk-KOKM4PQN.mjs +52 -0
- package/dist/{chunk-GCM3JFVA.mjs → chunk-OHP7QR33.mjs} +2 -1
- package/dist/{chunk-T6CNO72J.mjs → chunk-QM3VPX4Z.mjs} +2 -2
- package/dist/{chunk-V7YUSKKZ.mjs → chunk-QWWCLRXX.mjs} +10 -6
- package/dist/chunk-QY7GC2MW.mjs +33 -0
- package/dist/{chunk-DCK4P5BI.mjs → chunk-R36MIGBU.mjs} +100 -59
- package/dist/{chunk-YRX6XMOQ.mjs → chunk-R6PR4WYZ.mjs} +1 -0
- package/dist/{chunk-RVM7AODE.mjs → chunk-RDJKYKMQ.mjs} +2 -2
- package/dist/{chunk-OOSIDHA3.mjs → chunk-RSRVGAIQ.mjs} +1 -1
- package/dist/{chunk-XSZVS4S3.mjs → chunk-RUIXODYU.mjs} +9 -4
- package/dist/{chunk-A3UB2MXL.mjs → chunk-SBCEHIPL.mjs} +4 -507
- package/dist/{chunk-TPJU7NDJ.mjs → chunk-SYWGORQB.mjs} +1 -0
- package/dist/chunk-UAUZGXJP.mjs +3591 -0
- package/dist/{chunk-CL4OM7GB.mjs → chunk-WBS3EYC4.mjs} +12 -8
- package/dist/{chunk-J2BPXTD7.mjs → chunk-WLOF6HSP.mjs} +3 -3
- package/dist/chunk-X5CV2E2B.mjs +1 -0
- package/dist/{chunk-ULRPNYAG.mjs → chunk-Z3MOCHG5.mjs} +120 -62
- package/dist/chunk-ZHS44YKF.mjs +35 -0
- package/dist/{client-BMK53IR2.d.ts → client-CgayPw2R.d.ts} +27 -3
- package/dist/client.d.ts +9 -8
- package/dist/client.mjs +16 -2
- package/dist/{config-UJx1dyX1.d.ts → config-3kBsQ0Pi.d.ts} +2 -2
- package/dist/{config-BUR-7eVQ.d.ts → config-B-RnO6dr.d.ts} +2 -2
- package/dist/direct-executor-IJTRU3YN.mjs +22 -0
- package/dist/distribution/coordinator/etcd.d.ts +3 -2
- package/dist/distribution/coordinator/etcd.mjs +5 -1
- package/dist/distribution/coordinator/in-memory.d.ts +3 -2
- package/dist/distribution/coordinator/in-memory.mjs +4 -0
- package/dist/distribution/transport/grpc.mjs +3 -3
- package/dist/distribution/transport/in-memory.mjs +1 -1
- package/dist/distribution/transport/tcp.mjs +2 -2
- package/dist/distribution.d.ts +43 -9
- package/dist/distribution.mjs +4620 -4293
- package/dist/{durability-wiring-ZLRQU6FG.mjs → durability-wiring-Y55SCPFC.mjs} +524 -93
- package/dist/embeddings/openai.mjs +1 -1
- package/dist/{engine-9X6W5naj.d.ts → engine-DSUfY2F0.d.ts} +53 -17
- package/dist/{index-vCMu8OeQ.d.ts → index-ONAEaqtx.d.ts} +8 -1
- package/dist/index.browser.mjs +2 -2
- package/dist/index.d.ts +44 -25
- package/dist/index.mjs +22 -18
- package/dist/{memory-CV9NAxLY.d.ts → memory-wuXcYOD1.d.ts} +99 -79
- package/dist/{module-WIDYIJLY.mjs → module-VITS4ZR5.mjs} +72 -38
- package/dist/partial-words.mjs +5 -4
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +60 -58
- package/dist/persistence/durability/checkpoint-worker.mjs +27 -23
- package/dist/react.d.ts +8 -7
- package/dist/react.mjs +1 -0
- package/dist/release-BH3DKW3T.mjs +33 -0
- package/dist/replace-CEO5KJJ3.mjs +57 -0
- package/dist/restore-queue-4SG45BLF.mjs +35 -0
- package/dist/{schema-DtsgmNfU.d.ts → schema-DttzM-ln.d.ts} +12 -0
- package/dist/server.d.ts +11 -10
- package/dist/server.mjs +38 -11
- package/dist/snapshot-XBKE7PUG.mjs +738 -0
- package/dist/synonyms-CpOAHPa-.d.ts +52 -0
- package/dist/synonyms.d.ts +22 -0
- package/dist/synonyms.mjs +12 -0
- package/dist/{tasks-BGaZJyoY.d.ts → tasks-Bj9WBlFG.d.ts} +4 -1
- package/dist/{thread-ZP3UMIZT.mjs → thread-U7XYJWGG.mjs} +17 -12
- package/dist/vector/search-worker.d.ts +2 -1
- package/dist/vector/search-worker.mjs +6 -6
- package/dist/workers/entry.mjs +6 -6
- package/dist/{write-PXDKPG2R.mjs → write-BMNV7P2H.mjs} +24 -20
- package/package.json +15 -7
- package/dist/chunk-PR4O7CAP.mjs +0 -18
- package/dist/chunk-X4SEXL4U.mjs +0 -2310
- package/dist/chunk-Y2F66IQJ.mjs +0 -74
- package/dist/chunk-ZJ4CSAB2.mjs +0 -13
- package/dist/direct-executor-EGESD26S.mjs +0 -18
- package/dist/snapshot-22CUEYYW.mjs +0 -469
|
@@ -1,77 +1,456 @@
|
|
|
1
|
-
import { readMetadataEnvelope, readSegmentManifest, reclaimUnlistedKeys, loadSegmentedCheckpoint, writeMetadataEnvelope, readCurrentManifest, deserializePayloadV2, unpackEnvelopeBytes, growPartitionsTo, documentsOfPayloadV2 } from './chunk-
|
|
2
|
-
import { decodeSegmentManifest, metadataFromConfig, createSeqOwner, isRecord, invalidManifest, SINGLE_NODE_PRIMARY_TERM, writeLiveVectors, isId, isUint32, isSequenceNumber, VECTOR_FILE_ENVELOPE_VERSION, metadataConfigOf } from './chunk-
|
|
3
|
-
import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-
|
|
1
|
+
import { readMetadataEnvelope, readSegmentManifest, reclaimUnlistedKeys, loadSegmentedCheckpoint, writeMetadataEnvelope, readCurrentManifest, deserializePayloadV2, unpackEnvelopeBytes, growPartitionsTo, documentsOfPayloadV2 } from './chunk-QWWCLRXX.mjs';
|
|
2
|
+
import { decodeSegmentManifest, metadataFromConfig, createSeqOwner, isRecord, invalidManifest, SINGLE_NODE_PRIMARY_TERM, writeLiveVectors, isId, isUint32, isSequenceNumber, VECTOR_FILE_ENVELOPE_VERSION, metadataConfigOf } from './chunk-WBS3EYC4.mjs';
|
|
3
|
+
import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-F52QGNZ2.mjs';
|
|
4
4
|
import './chunk-37CD3OKS.mjs';
|
|
5
|
-
import { createDurableDirectory } from './chunk-
|
|
6
|
-
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, readListedFile, adapterCheckpointStore, snapshotBundleKey, manifestReferencedKeys, checkpointLastSeqNo, commitCheckpointManifest, removeCheckpointGarbage, readSegmentFile, checkpointPrimaryTerm, writeCheckpointSegments, partitionContentsOf, frameRecord, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, readCommitMarker, readWalSegments, durableRecordMissing, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, MAX_SEGMENTS_PER_PARTITION, writeSegmentHeader, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, SEGMENT_HEADER_SIZE, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, LOG_RECORDS_PER_CHECKPOINT, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, buildEntry } from './chunk-QAMUC2BJ.mjs';
|
|
7
|
-
import { applyDeleteEntry, applyIndexEntry } from './chunk-MWJIAREY.mjs';
|
|
8
|
-
import './chunk-GCM3JFVA.mjs';
|
|
9
|
-
import './chunk-P36MUFEJ.mjs';
|
|
5
|
+
import { createDurableDirectory, getFs } from './chunk-EWBXO45L.mjs';
|
|
10
6
|
import { spawnNodeWorker } from './chunk-C7SLA6CM.mjs';
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
13
|
-
import './chunk-
|
|
14
|
-
import
|
|
15
|
-
import './chunk-
|
|
16
|
-
import './chunk-
|
|
17
|
-
import { resolveWorkerEntry, MAX_FILE_BYTES } from './chunk-
|
|
7
|
+
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, readListedFile, adapterCheckpointStore, snapshotBundleKey, manifestReferencedKeys, checkpointLastSeqNo, manifestFileRefs, commitCheckpointManifest, removeCheckpointGarbage, readSegmentFile, checkpointPrimaryTerm, writeCheckpointSegments, partitionContentsOf, frameRecord, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, readCommitMarker, readWalSegments, durableRecordMissing, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, MAX_SEGMENTS_PER_PARTITION, writeSegmentHeader, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, SEGMENT_HEADER_SIZE, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, LOG_RECORDS_PER_CHECKPOINT, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, buildEntry } from './chunk-EX62BQT6.mjs';
|
|
8
|
+
import { applyDeleteEntry, applyIndexEntry } from './chunk-EZD4MWOU.mjs';
|
|
9
|
+
import './chunk-OHP7QR33.mjs';
|
|
10
|
+
import './chunk-P36MUFEJ.mjs';
|
|
11
|
+
import { openChecksummedEnvelope, HEADER_SIZE } from './chunk-C45FUYPI.mjs';
|
|
12
|
+
import './chunk-R36MIGBU.mjs';
|
|
13
|
+
import { resolveWorkerEntry, MAX_FILE_BYTES } from './chunk-RDJKYKMQ.mjs';
|
|
14
|
+
import { buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-UAUZGXJP.mjs';
|
|
15
|
+
import { fnv1a } from './chunk-CHOGQSHK.mjs';
|
|
16
|
+
import './chunk-WLOF6HSP.mjs';
|
|
18
17
|
import { detectRuntime } from './chunk-AV4IJKD2.mjs';
|
|
19
|
-
import './chunk-Q5DC6WAB.mjs';
|
|
18
|
+
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-Q5DC6WAB.mjs';
|
|
19
|
+
import './chunk-ZHS44YKF.mjs';
|
|
20
20
|
import './chunk-TGP5M5R7.mjs';
|
|
21
|
-
import './chunk-
|
|
22
|
-
import './chunk-
|
|
23
|
-
import './chunk-
|
|
24
|
-
import './chunk-
|
|
21
|
+
import './chunk-U2BKYEC7.mjs';
|
|
22
|
+
import './chunk-KOKM4PQN.mjs';
|
|
23
|
+
import './chunk-QY7GC2MW.mjs';
|
|
24
|
+
import './chunk-ES33L3HO.mjs';
|
|
25
|
+
import './chunk-QM3VPX4Z.mjs';
|
|
25
26
|
import './chunk-WQBCBKY3.mjs';
|
|
26
|
-
import './chunk-
|
|
27
|
-
import { encodeDocument, requireValidFieldTypes } from './chunk-
|
|
27
|
+
import './chunk-SBCEHIPL.mjs';
|
|
28
|
+
import { encodeDocument, requireValidFieldTypes } from './chunk-4ZJWRETC.mjs';
|
|
28
29
|
import './chunk-FGMGUDHX.mjs';
|
|
29
30
|
import { MIN_M } from './chunk-KEHK2O6P.mjs';
|
|
30
31
|
import './chunk-GES7U2PM.mjs';
|
|
31
|
-
import './chunk-
|
|
32
|
-
import './chunk-
|
|
32
|
+
import './chunk-BB7MASRQ.mjs';
|
|
33
|
+
import './chunk-2GZGBOQI.mjs';
|
|
34
|
+
import './chunk-X5CV2E2B.mjs';
|
|
35
|
+
import './chunk-7X5JST5L.mjs';
|
|
36
|
+
import './chunk-7KC56574.mjs';
|
|
33
37
|
import './chunk-R6A773WK.mjs';
|
|
34
38
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
35
|
-
import './chunk-
|
|
36
|
-
import './chunk-
|
|
39
|
+
import './chunk-R6PR4WYZ.mjs';
|
|
40
|
+
import './chunk-K6A4FCVR.mjs';
|
|
37
41
|
import './chunk-L5ORED7V.mjs';
|
|
38
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
42
|
+
import { NarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
|
|
39
43
|
import { decode } from '@msgpack/msgpack';
|
|
40
44
|
|
|
41
|
-
// src/persistence/durability/
|
|
45
|
+
// src/persistence/durability/sha256.ts
|
|
46
|
+
var ROUND_CONSTANTS = new Uint32Array([
|
|
47
|
+
1116352408,
|
|
48
|
+
1899447441,
|
|
49
|
+
3049323471,
|
|
50
|
+
3921009573,
|
|
51
|
+
961987163,
|
|
52
|
+
1508970993,
|
|
53
|
+
2453635748,
|
|
54
|
+
2870763221,
|
|
55
|
+
3624381080,
|
|
56
|
+
310598401,
|
|
57
|
+
607225278,
|
|
58
|
+
1426881987,
|
|
59
|
+
1925078388,
|
|
60
|
+
2162078206,
|
|
61
|
+
2614888103,
|
|
62
|
+
3248222580,
|
|
63
|
+
3835390401,
|
|
64
|
+
4022224774,
|
|
65
|
+
264347078,
|
|
66
|
+
604807628,
|
|
67
|
+
770255983,
|
|
68
|
+
1249150122,
|
|
69
|
+
1555081692,
|
|
70
|
+
1996064986,
|
|
71
|
+
2554220882,
|
|
72
|
+
2821834349,
|
|
73
|
+
2952996808,
|
|
74
|
+
3210313671,
|
|
75
|
+
3336571891,
|
|
76
|
+
3584528711,
|
|
77
|
+
113926993,
|
|
78
|
+
338241895,
|
|
79
|
+
666307205,
|
|
80
|
+
773529912,
|
|
81
|
+
1294757372,
|
|
82
|
+
1396182291,
|
|
83
|
+
1695183700,
|
|
84
|
+
1986661051,
|
|
85
|
+
2177026350,
|
|
86
|
+
2456956037,
|
|
87
|
+
2730485921,
|
|
88
|
+
2820302411,
|
|
89
|
+
3259730800,
|
|
90
|
+
3345764771,
|
|
91
|
+
3516065817,
|
|
92
|
+
3600352804,
|
|
93
|
+
4094571909,
|
|
94
|
+
275423344,
|
|
95
|
+
430227734,
|
|
96
|
+
506948616,
|
|
97
|
+
659060556,
|
|
98
|
+
883997877,
|
|
99
|
+
958139571,
|
|
100
|
+
1322822218,
|
|
101
|
+
1537002063,
|
|
102
|
+
1747873779,
|
|
103
|
+
1955562222,
|
|
104
|
+
2024104815,
|
|
105
|
+
2227730452,
|
|
106
|
+
2361852424,
|
|
107
|
+
2428436474,
|
|
108
|
+
2756734187,
|
|
109
|
+
3204031479,
|
|
110
|
+
3329325298
|
|
111
|
+
]);
|
|
112
|
+
var INITIAL_STATE = [1779033703, 3144134277, 1013904242, 2773480762, 1359893119, 2600822924, 528734635, 1541459225];
|
|
113
|
+
var BLOCK_BYTES = 64;
|
|
114
|
+
var LENGTH_BYTES = 8;
|
|
115
|
+
function rotateRight(value, bits) {
|
|
116
|
+
return value >>> bits | value << 32 - bits;
|
|
117
|
+
}
|
|
118
|
+
function padded(bytes) {
|
|
119
|
+
const total = Math.ceil((bytes.byteLength + 1 + LENGTH_BYTES) / BLOCK_BYTES) * BLOCK_BYTES;
|
|
120
|
+
const message = new Uint8Array(total);
|
|
121
|
+
message.set(bytes);
|
|
122
|
+
message[bytes.byteLength] = 128;
|
|
123
|
+
const view = new DataView(message.buffer);
|
|
124
|
+
const bitLength = bytes.byteLength * 8;
|
|
125
|
+
view.setUint32(total - LENGTH_BYTES, Math.floor(bitLength / 4294967296), false);
|
|
126
|
+
view.setUint32(total - 4, bitLength >>> 0, false);
|
|
127
|
+
return view;
|
|
128
|
+
}
|
|
129
|
+
function sha256(bytes) {
|
|
130
|
+
const view = padded(bytes);
|
|
131
|
+
const state = INITIAL_STATE.slice();
|
|
132
|
+
const words = new Uint32Array(64);
|
|
133
|
+
for (let block = 0; block < view.byteLength; block += BLOCK_BYTES) {
|
|
134
|
+
for (let i = 0; i < 16; i++) words[i] = view.getUint32(block + i * 4, false);
|
|
135
|
+
for (let i = 16; i < 64; i++) {
|
|
136
|
+
const w15 = words[i - 15];
|
|
137
|
+
const w2 = words[i - 2];
|
|
138
|
+
const s0 = rotateRight(w15, 7) ^ rotateRight(w15, 18) ^ w15 >>> 3;
|
|
139
|
+
const s1 = rotateRight(w2, 17) ^ rotateRight(w2, 19) ^ w2 >>> 10;
|
|
140
|
+
words[i] = words[i - 16] + s0 + words[i - 7] + s1 >>> 0;
|
|
141
|
+
}
|
|
142
|
+
let [a, b, c, d, e, f, g, h] = state;
|
|
143
|
+
for (let i = 0; i < 64; i++) {
|
|
144
|
+
const s1 = rotateRight(e, 6) ^ rotateRight(e, 11) ^ rotateRight(e, 25);
|
|
145
|
+
const choice = e & f ^ ~e & g;
|
|
146
|
+
const t1 = h + s1 + choice + ROUND_CONSTANTS[i] + words[i] >>> 0;
|
|
147
|
+
const s0 = rotateRight(a, 2) ^ rotateRight(a, 13) ^ rotateRight(a, 22);
|
|
148
|
+
const majority = a & b ^ a & c ^ b & c;
|
|
149
|
+
const t2 = s0 + majority >>> 0;
|
|
150
|
+
h = g;
|
|
151
|
+
g = f;
|
|
152
|
+
f = e;
|
|
153
|
+
e = d + t1 >>> 0;
|
|
154
|
+
d = c;
|
|
155
|
+
c = b;
|
|
156
|
+
b = a;
|
|
157
|
+
a = t1 + t2 >>> 0;
|
|
158
|
+
}
|
|
159
|
+
state[0] = state[0] + a >>> 0;
|
|
160
|
+
state[1] = state[1] + b >>> 0;
|
|
161
|
+
state[2] = state[2] + c >>> 0;
|
|
162
|
+
state[3] = state[3] + d >>> 0;
|
|
163
|
+
state[4] = state[4] + e >>> 0;
|
|
164
|
+
state[5] = state[5] + f >>> 0;
|
|
165
|
+
state[6] = state[6] + g >>> 0;
|
|
166
|
+
state[7] = state[7] + h >>> 0;
|
|
167
|
+
}
|
|
168
|
+
const digest = new Uint8Array(32);
|
|
169
|
+
const out = new DataView(digest.buffer);
|
|
170
|
+
for (let i = 0; i < 8; i++) out.setUint32(i * 4, state[i], false);
|
|
171
|
+
return digest;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
// src/persistence/durability/restore-journal.ts
|
|
175
|
+
var RESTORE_JOURNAL_PREFIX = ".swap-";
|
|
42
176
|
var RESTORE_STAGING_PREFIX = ".restore-";
|
|
43
|
-
|
|
44
|
-
|
|
177
|
+
var RETIRED_INDEX_PREFIX = ".retired-";
|
|
178
|
+
var LOCK_FILE_KEY = ".narsil.lock";
|
|
179
|
+
var STAGING_PREFIX_PATTERN = /^\.[A-Za-z0-9][A-Za-z0-9._-]*\/$/;
|
|
180
|
+
function digestOf(indexName) {
|
|
181
|
+
let hex = "";
|
|
182
|
+
for (const byte of sha256(new TextEncoder().encode(indexName))) hex += byte.toString(16).padStart(2, "0");
|
|
183
|
+
return hex;
|
|
184
|
+
}
|
|
185
|
+
function journalKey(indexName) {
|
|
186
|
+
return `${RESTORE_JOURNAL_PREFIX}${digestOf(indexName)}`;
|
|
187
|
+
}
|
|
188
|
+
function randomSuffix() {
|
|
189
|
+
return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
|
|
190
|
+
}
|
|
191
|
+
function newStagingPrefix() {
|
|
192
|
+
return `${RESTORE_STAGING_PREFIX}${randomSuffix()}/`;
|
|
193
|
+
}
|
|
194
|
+
function newRetiredPrefix() {
|
|
195
|
+
return `${RETIRED_INDEX_PREFIX}${randomSuffix()}/`;
|
|
196
|
+
}
|
|
197
|
+
function encodeJournal(indexName, stagingPrefix) {
|
|
198
|
+
return new TextEncoder().encode(`${indexName}
|
|
199
|
+
${stagingPrefix}`);
|
|
200
|
+
}
|
|
201
|
+
function isStagingPrefix(prefix) {
|
|
202
|
+
return STAGING_PREFIX_PATTERN.test(prefix) && !prefix.includes("..");
|
|
203
|
+
}
|
|
204
|
+
function corruptJournal(key, reason) {
|
|
205
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The restore journal "${key}" ${reason}`, { key });
|
|
206
|
+
}
|
|
207
|
+
function readJournal(key, bytes) {
|
|
208
|
+
const text = new TextDecoder().decode(bytes);
|
|
209
|
+
const lineEnd = text.indexOf("\n");
|
|
210
|
+
const indexName = lineEnd === -1 ? text : text.slice(0, lineEnd);
|
|
211
|
+
if (key.slice(RESTORE_JOURNAL_PREFIX.length) !== digestOf(indexName)) return { kind: "leftover" };
|
|
212
|
+
if (indexName.length > MAX_INDEX_NAME_LENGTH || !INDEX_NAME_PATTERN.test(indexName) || indexName.includes("..")) {
|
|
213
|
+
corruptJournal(key, "names an index that no index name can match");
|
|
214
|
+
}
|
|
215
|
+
const stagingPrefix = lineEnd === -1 ? "" : text.slice(lineEnd + 1);
|
|
216
|
+
if (!isStagingPrefix(stagingPrefix) || stagingPrefix === `${LOCK_FILE_KEY}/`) {
|
|
217
|
+
corruptJournal(key, "names no staging prefix that a restore writes");
|
|
218
|
+
}
|
|
219
|
+
return { kind: "journal", indexName, stagingPrefix };
|
|
220
|
+
}
|
|
221
|
+
async function listedKeys(storage, indexName, stagingPrefix) {
|
|
222
|
+
const manifestBytes = await storage.read(`${stagingPrefix}manifest`) ?? await storage.read(manifestKey(indexName));
|
|
223
|
+
if (manifestBytes === null) {
|
|
224
|
+
throw new NarsilError(
|
|
225
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
226
|
+
`The restore of index "${indexName}" lost its manifest, both from staging and from the index`,
|
|
227
|
+
{ indexName, stagingPrefix }
|
|
228
|
+
);
|
|
229
|
+
}
|
|
230
|
+
const decoded = await decodeSegmentManifest(manifestBytes, indexName);
|
|
231
|
+
if (decoded.kind !== "current") {
|
|
232
|
+
throw new NarsilError(
|
|
233
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
234
|
+
`The restore of index "${indexName}" staged a manifest in the layout of an earlier release`,
|
|
235
|
+
{ indexName }
|
|
236
|
+
);
|
|
237
|
+
}
|
|
238
|
+
const root = `${indexName}/`;
|
|
239
|
+
const keys = /* @__PURE__ */ new Set(["meta", "manifest"]);
|
|
240
|
+
for (const ref of manifestFileRefs(decoded.manifest)) keys.add(ref.key.slice(root.length));
|
|
241
|
+
return keys;
|
|
242
|
+
}
|
|
243
|
+
function withoutTrailingSlash(prefix) {
|
|
244
|
+
return prefix.slice(0, -1);
|
|
245
|
+
}
|
|
246
|
+
async function finishJournal(storage, indexName, stagingPrefix) {
|
|
247
|
+
const root = `${indexName}/`;
|
|
248
|
+
const listed = await listedKeys(storage, indexName, stagingPrefix);
|
|
249
|
+
const staged = await storage.list(stagingPrefix);
|
|
250
|
+
let retiredPrefix = null;
|
|
251
|
+
if (storage.moveTree !== void 0 && staged.length > 0) {
|
|
252
|
+
retiredPrefix = newRetiredPrefix();
|
|
253
|
+
await storage.moveTree(indexName, withoutTrailingSlash(retiredPrefix));
|
|
254
|
+
await storage.moveTree(withoutTrailingSlash(stagingPrefix), indexName);
|
|
255
|
+
} else {
|
|
256
|
+
const stagedKeys = new Set(staged);
|
|
257
|
+
for (const key of listed) {
|
|
258
|
+
if (stagedKeys.has(`${stagingPrefix}${key}`)) await storage.moveKey(`${stagingPrefix}${key}`, `${root}${key}`);
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
for (const key of await storage.list(root)) {
|
|
262
|
+
if (!listed.has(key.slice(root.length))) await storage.remove(key);
|
|
263
|
+
}
|
|
264
|
+
await storage.remove(journalKey(indexName));
|
|
265
|
+
return retiredPrefix;
|
|
266
|
+
}
|
|
267
|
+
async function removePrefix(storage, prefix) {
|
|
268
|
+
if (storage.removeTree !== void 0) {
|
|
269
|
+
await storage.removeTree(prefix);
|
|
270
|
+
return;
|
|
271
|
+
}
|
|
272
|
+
for (const key of await storage.list(prefix)) await storage.remove(key);
|
|
273
|
+
}
|
|
274
|
+
async function commitJournal(storage, indexName, stagingPrefix) {
|
|
275
|
+
await storage.writeJournal(journalKey(indexName), encodeJournal(indexName, stagingPrefix));
|
|
276
|
+
}
|
|
277
|
+
function isTopLevelJournal(key) {
|
|
278
|
+
return key.startsWith(RESTORE_JOURNAL_PREFIX) && !key.includes("/");
|
|
279
|
+
}
|
|
280
|
+
async function removeStagingPrefixes(storage) {
|
|
281
|
+
const prefixes = /* @__PURE__ */ new Set();
|
|
282
|
+
for (const key of await storage.list(".")) {
|
|
283
|
+
const slash = key.indexOf("/");
|
|
284
|
+
if (slash !== -1 && isStagingPrefix(key.slice(0, slash + 1))) prefixes.add(key.slice(0, slash + 1));
|
|
285
|
+
}
|
|
286
|
+
for (const prefix of prefixes) await removePrefix(storage, prefix);
|
|
287
|
+
}
|
|
288
|
+
async function recoverRestoreJournals(storage) {
|
|
289
|
+
for (const key of await storage.list(RESTORE_JOURNAL_PREFIX)) {
|
|
290
|
+
if (!isTopLevelJournal(key)) continue;
|
|
291
|
+
const bytes = await storage.read(key);
|
|
292
|
+
if (bytes === null) continue;
|
|
293
|
+
const reading = readJournal(key, bytes);
|
|
294
|
+
if (reading.kind === "leftover") {
|
|
295
|
+
await storage.remove(key);
|
|
296
|
+
continue;
|
|
297
|
+
}
|
|
298
|
+
await finishJournal(storage, reading.indexName, reading.stagingPrefix);
|
|
299
|
+
}
|
|
300
|
+
await removeStagingPrefixes(storage);
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
// src/persistence/durability/restore-directory.ts
|
|
304
|
+
function missing(error) {
|
|
305
|
+
return error.code === "ENOENT";
|
|
306
|
+
}
|
|
307
|
+
function storageFailed(code, message, key, error) {
|
|
308
|
+
return new NarsilError(code, message, { key, cause: error instanceof Error ? error.message : String(error) });
|
|
309
|
+
}
|
|
310
|
+
function directoryRestoreStorage(directory) {
|
|
311
|
+
return {
|
|
312
|
+
read: (key) => directory.read(key),
|
|
313
|
+
write: (key, chunks) => directory.atomicWrite(key, chunks),
|
|
314
|
+
writeJournal: (key, bytes) => directory.atomicWrite(key, bytes),
|
|
315
|
+
remove: (key) => directory.remove(key),
|
|
316
|
+
list: (prefix) => directory.list(prefix),
|
|
317
|
+
moveKey: (from, to) => directory.move(from, to),
|
|
318
|
+
pathOf: (key) => directory.pathOf(key),
|
|
319
|
+
async moveTree(from, to) {
|
|
320
|
+
const source = await directory.pathOf(from);
|
|
321
|
+
const target = await directory.pathOf(to);
|
|
322
|
+
const fs = await getFs();
|
|
323
|
+
try {
|
|
324
|
+
await fs.rename(source, target);
|
|
325
|
+
} catch (error) {
|
|
326
|
+
if (missing(error)) return;
|
|
327
|
+
throw storageFailed(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Failed to move "${from}" to "${to}"`, from, error);
|
|
328
|
+
}
|
|
329
|
+
await directory.syncDirectoryOf(to);
|
|
330
|
+
},
|
|
331
|
+
async removeTree(prefix) {
|
|
332
|
+
const key = prefix.endsWith("/") ? prefix.slice(0, -1) : prefix;
|
|
333
|
+
const fs = await getFs();
|
|
334
|
+
try {
|
|
335
|
+
await fs.rm(await directory.pathOf(key), { recursive: true, force: true });
|
|
336
|
+
} catch (error) {
|
|
337
|
+
throw storageFailed(ErrorCodes.PERSISTENCE_DELETE_FAILED, `Failed to delete "${key}"`, key, error);
|
|
338
|
+
}
|
|
339
|
+
await directory.syncDirectoryOf(key);
|
|
340
|
+
}
|
|
341
|
+
};
|
|
342
|
+
}
|
|
343
|
+
function recoverDirectoryRestores(directory) {
|
|
344
|
+
return recoverRestoreJournals(directoryRestoreStorage(directory));
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
// src/persistence/durability/restore-staging.ts
|
|
348
|
+
function toError(error) {
|
|
349
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
350
|
+
}
|
|
351
|
+
function pathUnder(path, root) {
|
|
352
|
+
if (!path.startsWith(root)) return false;
|
|
353
|
+
const next = path.charAt(root.length);
|
|
354
|
+
return next === "" || next === "/" || next === "\\";
|
|
355
|
+
}
|
|
356
|
+
function createRestoreStaging(storage, indexName, options = {}) {
|
|
357
|
+
const stagingPrefix = newStagingPrefix();
|
|
45
358
|
const indexRoot = `${indexName}/`;
|
|
359
|
+
let committed = false;
|
|
46
360
|
const staged = (key) => {
|
|
47
361
|
if (!key.startsWith(indexRoot)) {
|
|
48
362
|
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `A restore of "${indexName}" cannot stage "${key}"`, {
|
|
49
363
|
key
|
|
50
364
|
});
|
|
51
365
|
}
|
|
52
|
-
return `${
|
|
366
|
+
return `${stagingPrefix}${key.slice(indexRoot.length)}`;
|
|
53
367
|
};
|
|
54
|
-
const unstaged = (key) => `${indexRoot}${key.slice(
|
|
55
|
-
const
|
|
56
|
-
read: (key) =>
|
|
57
|
-
write: (key, chunks) =>
|
|
58
|
-
remove: (key) =>
|
|
59
|
-
list: async (prefix) => (await
|
|
60
|
-
pathOf: (key) => directory.pathOf(staged(key))
|
|
368
|
+
const unstaged = (key) => `${indexRoot}${key.slice(stagingPrefix.length)}`;
|
|
369
|
+
const store = {
|
|
370
|
+
read: (key) => storage.read(staged(key)),
|
|
371
|
+
write: (key, chunks) => storage.write(staged(key), chunks),
|
|
372
|
+
remove: (key) => storage.remove(staged(key)),
|
|
373
|
+
list: async (prefix) => (await storage.list(staged(prefix))).map(unstaged)
|
|
61
374
|
};
|
|
375
|
+
const pathOf = storage.pathOf;
|
|
376
|
+
if (pathOf !== void 0) store.pathOf = (key) => pathOf.call(storage, staged(key));
|
|
377
|
+
let retiredPrefix = null;
|
|
378
|
+
async function finishOnStorage() {
|
|
379
|
+
retiredPrefix = await finishJournal(storage, indexName, stagingPrefix);
|
|
380
|
+
}
|
|
62
381
|
return {
|
|
63
|
-
store
|
|
382
|
+
store,
|
|
383
|
+
async relocation() {
|
|
384
|
+
if (pathOf === void 0 || (await storage.list(stagingPrefix)).length > 0) return null;
|
|
385
|
+
const from = await pathOf.call(storage, stagingPrefix.slice(0, -1));
|
|
386
|
+
const to = await pathOf.call(storage, indexName);
|
|
387
|
+
return (path) => pathUnder(path, from) ? `${to}${path.slice(from.length)}` : path;
|
|
388
|
+
},
|
|
64
389
|
async commit() {
|
|
65
|
-
|
|
66
|
-
|
|
390
|
+
try {
|
|
391
|
+
await commitJournal(storage, indexName, stagingPrefix);
|
|
392
|
+
} catch (error) {
|
|
393
|
+
try {
|
|
394
|
+
await storage.remove(journalKey(indexName));
|
|
395
|
+
} catch (removal) {
|
|
396
|
+
committed = true;
|
|
397
|
+
options.onStorageLost?.(toError(removal));
|
|
398
|
+
}
|
|
399
|
+
throw error;
|
|
400
|
+
}
|
|
401
|
+
committed = true;
|
|
402
|
+
},
|
|
403
|
+
async finish() {
|
|
404
|
+
try {
|
|
405
|
+
await (options.finish === void 0 ? finishOnStorage() : options.finish(finishOnStorage));
|
|
406
|
+
} catch (error) {
|
|
407
|
+
options.onStorageLost?.(toError(error));
|
|
408
|
+
}
|
|
409
|
+
},
|
|
410
|
+
async removeRetired() {
|
|
411
|
+
const prefix = retiredPrefix;
|
|
412
|
+
retiredPrefix = null;
|
|
413
|
+
if (prefix !== null) await removePrefix(storage, prefix);
|
|
67
414
|
},
|
|
68
415
|
async discard() {
|
|
69
|
-
|
|
416
|
+
if (committed) return;
|
|
417
|
+
await removePrefix(storage, stagingPrefix);
|
|
418
|
+
}
|
|
419
|
+
};
|
|
420
|
+
}
|
|
421
|
+
function adapterRestoreStorage(adapter) {
|
|
422
|
+
const store = adapterCheckpointStore(adapter);
|
|
423
|
+
return {
|
|
424
|
+
read: (key) => store.read(key),
|
|
425
|
+
write: (key, chunks) => store.write(key, chunks),
|
|
426
|
+
writeJournal: (key, bytes) => store.write(key, [bytes]),
|
|
427
|
+
remove: (key) => store.remove(key),
|
|
428
|
+
list: (prefix) => store.list(prefix),
|
|
429
|
+
async moveKey(from, to) {
|
|
430
|
+
const bytes = await store.read(from);
|
|
431
|
+
if (bytes === null) return;
|
|
432
|
+
await store.write(to, [bytes]);
|
|
433
|
+
await store.remove(from);
|
|
70
434
|
}
|
|
71
435
|
};
|
|
72
436
|
}
|
|
73
|
-
|
|
74
|
-
|
|
437
|
+
|
|
438
|
+
// src/persistence/durability/archive-checkpoint.ts
|
|
439
|
+
function toError2(error) {
|
|
440
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
441
|
+
}
|
|
442
|
+
function stageDirectoryRestore(deps, indexName) {
|
|
443
|
+
return createRestoreStaging(directoryRestoreStorage(deps.directory), indexName, {
|
|
444
|
+
async finish(run) {
|
|
445
|
+
try {
|
|
446
|
+
await run();
|
|
447
|
+
} catch (error) {
|
|
448
|
+
deps.markFatal(toError2(error));
|
|
449
|
+
throw error;
|
|
450
|
+
}
|
|
451
|
+
},
|
|
452
|
+
onStorageLost: deps.markFatal
|
|
453
|
+
});
|
|
75
454
|
}
|
|
76
455
|
function releaser(held, indexName) {
|
|
77
456
|
let released = false;
|
|
@@ -122,7 +501,7 @@ function createArchiveCheckpointAccess(deps) {
|
|
|
122
501
|
}
|
|
123
502
|
},
|
|
124
503
|
stageRestore(indexName) {
|
|
125
|
-
return
|
|
504
|
+
return stageDirectoryRestore(deps, indexName);
|
|
126
505
|
},
|
|
127
506
|
adoptRestoredCheckpoint(indexName, checkpoint) {
|
|
128
507
|
for (const { partitionId, lastSeqNo } of checkpoint) deps.registerPartition(indexName, partitionId, lastSeqNo);
|
|
@@ -380,8 +759,8 @@ async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartit
|
|
|
380
759
|
const vectorPlans = /* @__PURE__ */ new Map();
|
|
381
760
|
for (const [fieldPath, vectorIndex] of vectorIndexes) {
|
|
382
761
|
const listed = listedVectorFields.find((ref) => ref.fieldPath === fieldPath);
|
|
383
|
-
const
|
|
384
|
-
vectorPlans.set(fieldPath, vectorIndex.planCheckpoint(
|
|
762
|
+
const listedKeys2 = listed === void 0 ? null : listed.files.map((file) => file.key);
|
|
763
|
+
vectorPlans.set(fieldPath, vectorIndex.planCheckpoint(listedKeys2));
|
|
385
764
|
}
|
|
386
765
|
const memoryMatchesTheTargets = recordsLeftInLog === 0;
|
|
387
766
|
const wholePartitions = memoryMatchesTheTargets ? wholePartitionsOf(targets) : noWholePartitions();
|
|
@@ -875,7 +1254,7 @@ async function drainIndexStateForUnload(indexState, pendingMetadataWrite, closeW
|
|
|
875
1254
|
}
|
|
876
1255
|
|
|
877
1256
|
// src/persistence/durability/mutation-batch.ts
|
|
878
|
-
function
|
|
1257
|
+
function toError3(err) {
|
|
879
1258
|
return err instanceof Error ? err : new Error(String(err));
|
|
880
1259
|
}
|
|
881
1260
|
function failUnlessAlreadyFailed(outcomes, count, error) {
|
|
@@ -887,7 +1266,7 @@ function failUnlessAlreadyFailed(outcomes, count, error) {
|
|
|
887
1266
|
}
|
|
888
1267
|
}
|
|
889
1268
|
function failPartition(partition, deps, err) {
|
|
890
|
-
const error =
|
|
1269
|
+
const error = toError3(err);
|
|
891
1270
|
partition.failed = error;
|
|
892
1271
|
deps.markFatal(error);
|
|
893
1272
|
return error;
|
|
@@ -967,7 +1346,7 @@ async function recordPartitionBatch(partition, records, deps) {
|
|
|
967
1346
|
}
|
|
968
1347
|
}
|
|
969
1348
|
} catch (err) {
|
|
970
|
-
failUnlessAlreadyFailed(outcomes, records.length,
|
|
1349
|
+
failUnlessAlreadyFailed(outcomes, records.length, toError3(err));
|
|
971
1350
|
}
|
|
972
1351
|
return outcomes;
|
|
973
1352
|
}
|
|
@@ -1804,8 +2183,8 @@ async function createMarkerWriter(directory, indexName, partitionId) {
|
|
|
1804
2183
|
// src/persistence/durability/wal-writer.ts
|
|
1805
2184
|
var SEGMENT_TAIL_PATTERN = /^\d{16}$/;
|
|
1806
2185
|
function segmentKey(indexName, partitionId, startSeqNo) {
|
|
1807
|
-
const
|
|
1808
|
-
return `${indexName}/wal/${partitionId}/${
|
|
2186
|
+
const padded2 = startSeqNo.toString().padStart(16, "0");
|
|
2187
|
+
return `${indexName}/wal/${partitionId}/${padded2}`;
|
|
1809
2188
|
}
|
|
1810
2189
|
function parseSegmentStartSeqNo2(key) {
|
|
1811
2190
|
const tail = key.slice(key.lastIndexOf("/") + 1);
|
|
@@ -2051,14 +2430,14 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
2051
2430
|
stopTimers();
|
|
2052
2431
|
hooks.onFatalError(error);
|
|
2053
2432
|
}
|
|
2054
|
-
function
|
|
2433
|
+
function toError5(err) {
|
|
2055
2434
|
return err instanceof Error ? err : new Error(String(err));
|
|
2056
2435
|
}
|
|
2057
2436
|
async function closeWriterReportingFailure(closePromise) {
|
|
2058
2437
|
try {
|
|
2059
2438
|
await closePromise;
|
|
2060
2439
|
} catch (err) {
|
|
2061
|
-
markFatal(
|
|
2440
|
+
markFatal(toError5(err));
|
|
2062
2441
|
}
|
|
2063
2442
|
}
|
|
2064
2443
|
function getOrCreateIndexState(indexName) {
|
|
@@ -2124,7 +2503,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
2124
2503
|
try {
|
|
2125
2504
|
await partition.walWriter.commit();
|
|
2126
2505
|
} catch (err) {
|
|
2127
|
-
partition.failed =
|
|
2506
|
+
partition.failed = toError5(err);
|
|
2128
2507
|
markFatal(partition.failed);
|
|
2129
2508
|
}
|
|
2130
2509
|
}
|
|
@@ -2138,7 +2517,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
2138
2517
|
try {
|
|
2139
2518
|
await checkpointIndex(indexName);
|
|
2140
2519
|
} catch (err) {
|
|
2141
|
-
markFatal(
|
|
2520
|
+
markFatal(toError5(err));
|
|
2142
2521
|
}
|
|
2143
2522
|
}
|
|
2144
2523
|
}
|
|
@@ -2219,7 +2598,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
2219
2598
|
indexState.documentBytesSinceCheckpoint += recorded.documentBytes;
|
|
2220
2599
|
if (checkpointIsDue(indexState, checkpointMutationThreshold)) {
|
|
2221
2600
|
void checkpointIndex(indexName).catch((err) => {
|
|
2222
|
-
markFatal(
|
|
2601
|
+
markFatal(toError5(err));
|
|
2223
2602
|
});
|
|
2224
2603
|
}
|
|
2225
2604
|
await stallWhileCheckpointsFallBehind(indexState);
|
|
@@ -2251,12 +2630,10 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
2251
2630
|
);
|
|
2252
2631
|
}
|
|
2253
2632
|
return {
|
|
2254
|
-
isActive()
|
|
2255
|
-
return true;
|
|
2256
|
-
},
|
|
2633
|
+
isActive: () => true,
|
|
2257
2634
|
async recover(metadataOnly = false) {
|
|
2258
2635
|
await directoryWatch.claim();
|
|
2259
|
-
await
|
|
2636
|
+
await recoverDirectoryRestores(directory);
|
|
2260
2637
|
const names = await listPersistedIndexes(directory);
|
|
2261
2638
|
for (const indexName of names) {
|
|
2262
2639
|
await recoverIndex(indexName, metadataOnly);
|
|
@@ -2326,7 +2703,8 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
2326
2703
|
directory,
|
|
2327
2704
|
held: heldCheckpointFiles,
|
|
2328
2705
|
checkpoint: (name) => checkpointIndex(name),
|
|
2329
|
-
registerPartition: (name, partitionId, startSeqNo) => getOrCreatePartition(name, partitionId, startSeqNo)
|
|
2706
|
+
registerPartition: (name, partitionId, startSeqNo) => getOrCreatePartition(name, partitionId, startSeqNo),
|
|
2707
|
+
markFatal
|
|
2330
2708
|
}),
|
|
2331
2709
|
async shutdown() {
|
|
2332
2710
|
shuttingDown = true;
|
|
@@ -2346,7 +2724,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
2346
2724
|
};
|
|
2347
2725
|
}
|
|
2348
2726
|
|
|
2349
|
-
// src/persistence/durability/snapshot-only
|
|
2727
|
+
// src/persistence/durability/snapshot-only/checkpoint.ts
|
|
2350
2728
|
function capture(input, prior) {
|
|
2351
2729
|
const listedIds = new Set((prior?.partitions ?? []).map((partition) => partition.partitionId));
|
|
2352
2730
|
const partitionCountChanged = prior !== null && listedIds.size !== input.manager.partitionCount;
|
|
@@ -2402,7 +2780,44 @@ async function writeSnapshotOnlyCheckpoint(input) {
|
|
|
2402
2780
|
return captured.documentCount;
|
|
2403
2781
|
}
|
|
2404
2782
|
|
|
2405
|
-
// src/persistence/durability/snapshot-only.ts
|
|
2783
|
+
// src/persistence/durability/snapshot-only/restores.ts
|
|
2784
|
+
function toError4(error) {
|
|
2785
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
2786
|
+
}
|
|
2787
|
+
function createPendingRestores(onFatalError) {
|
|
2788
|
+
const pending = /* @__PURE__ */ new Map();
|
|
2789
|
+
return {
|
|
2790
|
+
finishLater(indexName, run) {
|
|
2791
|
+
let running = null;
|
|
2792
|
+
const finish = () => {
|
|
2793
|
+
running ??= run().then(
|
|
2794
|
+
() => {
|
|
2795
|
+
if (pending.get(indexName) === finish) pending.delete(indexName);
|
|
2796
|
+
},
|
|
2797
|
+
(error) => {
|
|
2798
|
+
running = null;
|
|
2799
|
+
throw error;
|
|
2800
|
+
}
|
|
2801
|
+
);
|
|
2802
|
+
return running;
|
|
2803
|
+
};
|
|
2804
|
+
pending.set(indexName, finish);
|
|
2805
|
+
void finish().catch((error) => onFatalError(toError4(error)));
|
|
2806
|
+
return Promise.resolve();
|
|
2807
|
+
},
|
|
2808
|
+
async settle(indexName) {
|
|
2809
|
+
await pending.get(indexName)?.();
|
|
2810
|
+
},
|
|
2811
|
+
forget(indexName) {
|
|
2812
|
+
pending.delete(indexName);
|
|
2813
|
+
},
|
|
2814
|
+
async settleAll() {
|
|
2815
|
+
for (const finish of [...pending.values()]) await finish().catch(() => void 0);
|
|
2816
|
+
}
|
|
2817
|
+
};
|
|
2818
|
+
}
|
|
2819
|
+
|
|
2820
|
+
// src/persistence/durability/snapshot-only/index.ts
|
|
2406
2821
|
function metadataKey(indexName) {
|
|
2407
2822
|
return `${indexName}/meta`;
|
|
2408
2823
|
}
|
|
@@ -2411,8 +2826,13 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
2411
2826
|
const checkpointMutationThreshold = config.checkpointMutationThreshold ?? DEFAULT_CHECKPOINT_MUTATION_THRESHOLD;
|
|
2412
2827
|
const indexes = /* @__PURE__ */ new Map();
|
|
2413
2828
|
const metadataWrites = /* @__PURE__ */ new Map();
|
|
2829
|
+
const restoreStorage = adapterRestoreStorage(adapter);
|
|
2830
|
+
const restores = createPendingRestores(hooks.onFatalError);
|
|
2414
2831
|
let checkpointTimer = null;
|
|
2415
2832
|
let shuttingDown = false;
|
|
2833
|
+
function settleRestore(indexName) {
|
|
2834
|
+
return restores.settle(indexName);
|
|
2835
|
+
}
|
|
2416
2836
|
function getOrCreateIndexState(indexName) {
|
|
2417
2837
|
let state = indexes.get(indexName);
|
|
2418
2838
|
if (state === void 0) {
|
|
@@ -2448,6 +2868,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
2448
2868
|
}
|
|
2449
2869
|
}
|
|
2450
2870
|
async function performCheckpoint(indexName, indexState, fromMemory) {
|
|
2871
|
+
await settleRestore(indexName);
|
|
2451
2872
|
const manager = hooks.getManager(indexName);
|
|
2452
2873
|
const metadata = hooks.buildMetadata(indexName);
|
|
2453
2874
|
if (manager === void 0 || metadata === void 0) {
|
|
@@ -2497,7 +2918,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
2497
2918
|
}
|
|
2498
2919
|
function queueMetadataWrite(indexName, write) {
|
|
2499
2920
|
const previous = metadataWrites.get(indexName) ?? Promise.resolve();
|
|
2500
|
-
const queued = previous.then(write);
|
|
2921
|
+
const queued = previous.then(() => settleRestore(indexName)).then(write);
|
|
2501
2922
|
metadataWrites.set(
|
|
2502
2923
|
indexName,
|
|
2503
2924
|
queued.catch(() => void 0)
|
|
@@ -2505,6 +2926,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
2505
2926
|
return queued;
|
|
2506
2927
|
}
|
|
2507
2928
|
async function recoverIndex(indexName, metadataOnly = false) {
|
|
2929
|
+
await settleRestore(indexName);
|
|
2508
2930
|
const metaBytes = await adapter.load(metadataKey(indexName));
|
|
2509
2931
|
if (metaBytes === null) {
|
|
2510
2932
|
return;
|
|
@@ -2591,11 +3013,28 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
2591
3013
|
}
|
|
2592
3014
|
return appliedSeqNo;
|
|
2593
3015
|
}
|
|
3016
|
+
async function drainIndex(indexName) {
|
|
3017
|
+
const indexState = indexes.get(indexName);
|
|
3018
|
+
if (indexState !== void 0) {
|
|
3019
|
+
indexState.unloading = true;
|
|
3020
|
+
try {
|
|
3021
|
+
await indexState.applyChain;
|
|
3022
|
+
if (indexState.checkpointInFlight !== null) await indexState.checkpointInFlight;
|
|
3023
|
+
await metadataWrites.get(indexName);
|
|
3024
|
+
} catch (error) {
|
|
3025
|
+
indexState.unloading = false;
|
|
3026
|
+
throw error;
|
|
3027
|
+
}
|
|
3028
|
+
}
|
|
3029
|
+
indexes.delete(indexName);
|
|
3030
|
+
metadataWrites.delete(indexName);
|
|
3031
|
+
}
|
|
2594
3032
|
return {
|
|
2595
3033
|
isActive() {
|
|
2596
3034
|
return true;
|
|
2597
3035
|
},
|
|
2598
3036
|
async recover(metadataOnly = false) {
|
|
3037
|
+
await recoverRestoreJournals(restoreStorage);
|
|
2599
3038
|
const keys = await adapter.list("");
|
|
2600
3039
|
for (const key of keys) {
|
|
2601
3040
|
if (key.endsWith("/meta")) {
|
|
@@ -2644,40 +3083,31 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
2644
3083
|
await checkpointIndex(indexName);
|
|
2645
3084
|
}
|
|
2646
3085
|
},
|
|
2647
|
-
|
|
2648
|
-
|
|
2649
|
-
|
|
2650
|
-
|
|
2651
|
-
|
|
2652
|
-
|
|
2653
|
-
|
|
2654
|
-
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
|
|
2658
|
-
}
|
|
3086
|
+
stageRestore(indexName) {
|
|
3087
|
+
return createRestoreStaging(restoreStorage, indexName, {
|
|
3088
|
+
finish: (run) => restores.finishLater(indexName, run),
|
|
3089
|
+
onStorageLost: hooks.onFatalError
|
|
3090
|
+
});
|
|
3091
|
+
},
|
|
3092
|
+
adoptRestoredCheckpoint(indexName, checkpoint) {
|
|
3093
|
+
const indexState = getOrCreateIndexState(indexName);
|
|
3094
|
+
indexState.appliedSeqNoByPartition.clear();
|
|
3095
|
+
for (const { partitionId, lastSeqNo } of checkpoint) {
|
|
3096
|
+
if (lastSeqNo > 0) indexState.appliedSeqNoByPartition.set(partitionId, lastSeqNo);
|
|
2659
3097
|
}
|
|
2660
|
-
|
|
2661
|
-
|
|
3098
|
+
},
|
|
3099
|
+
async removeIndex(indexName) {
|
|
3100
|
+
await settleRestore(indexName).catch(() => void 0);
|
|
3101
|
+
restores.forget(indexName);
|
|
3102
|
+
await restoreStorage.remove(journalKey(indexName));
|
|
3103
|
+
await drainIndex(indexName);
|
|
2662
3104
|
for (const key of await adapter.list(`${indexName}/`)) {
|
|
2663
3105
|
await adapter.delete(key);
|
|
2664
3106
|
}
|
|
2665
3107
|
},
|
|
2666
3108
|
async unloadIndex(indexName) {
|
|
2667
|
-
|
|
2668
|
-
|
|
2669
|
-
indexState.unloading = true;
|
|
2670
|
-
try {
|
|
2671
|
-
await indexState.applyChain;
|
|
2672
|
-
if (indexState.checkpointInFlight !== null) await indexState.checkpointInFlight;
|
|
2673
|
-
await metadataWrites.get(indexName);
|
|
2674
|
-
} catch (error) {
|
|
2675
|
-
indexState.unloading = false;
|
|
2676
|
-
throw error;
|
|
2677
|
-
}
|
|
2678
|
-
}
|
|
2679
|
-
indexes.delete(indexName);
|
|
2680
|
-
metadataWrites.delete(indexName);
|
|
3109
|
+
await settleRestore(indexName);
|
|
3110
|
+
await drainIndex(indexName);
|
|
2681
3111
|
},
|
|
2682
3112
|
async reloadIndex(indexName) {
|
|
2683
3113
|
const indexState = getOrCreateIndexState(indexName);
|
|
@@ -2691,6 +3121,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
2691
3121
|
clearInterval(checkpointTimer);
|
|
2692
3122
|
checkpointTimer = null;
|
|
2693
3123
|
}
|
|
3124
|
+
await restores.settleAll();
|
|
2694
3125
|
for (const indexState of indexes.values()) {
|
|
2695
3126
|
if (indexState.checkpointInFlight !== null) {
|
|
2696
3127
|
await indexState.checkpointInFlight.catch(() => void 0);
|