@delali/narsil 0.3.1-next.91 → 0.3.1-next.95
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/browser/{chunk-66KYAOIK.mjs → chunk-3U636ADD.mjs} +2 -2
- package/dist/browser/{chunk-NOUZ6RH3.mjs → chunk-D3CC2NMZ.mjs} +238 -114
- package/dist/browser/{chunk-3ZRMZIH4.mjs → chunk-FYFQJKFW.mjs} +208 -78
- package/dist/browser/{chunk-FHFFWLEK.mjs → chunk-HARBSUWP.mjs} +3 -4
- package/dist/browser/{chunk-B6XE7KXY.mjs → chunk-LULHQXCK.mjs} +4 -4
- package/dist/browser/{chunk-NQW2YXEX.mjs → chunk-WKPJ4E73.mjs} +1 -1
- package/dist/browser/{durability-wiring-F4OGQUSU.mjs → durability-wiring-3ZRKSPJD.mjs} +498 -71
- package/dist/browser/{module-E25UDADM.mjs → module-IEOQI5GI.mjs} +102 -54
- package/dist/browser/release-76MXRPGV.mjs +33 -0
- package/dist/browser/replace-RURXTGVG.mjs +57 -0
- package/dist/browser/restore-queue-RBULSOVX.mjs +35 -0
- package/dist/browser/{snapshot-5EJ7I7QY.mjs → snapshot-Y3NGUJHX.mjs} +362 -103
- package/dist/browser/{write-K5V4XQ72.mjs → write-ZUBDNLCH.mjs} +4 -4
- package/dist/{capabilities-CvExBThq.d.ts → capabilities-BvxvFzVA.d.ts} +2 -2
- package/dist/{chunk-NVL4ZR3K.mjs → chunk-2GZGBOQI.mjs} +2 -2
- package/dist/{chunk-XLJ2S2BH.mjs → chunk-3LYY62M6.mjs} +40 -21
- package/dist/{chunk-CVQWQVFX.mjs → chunk-4ZJWRETC.mjs} +2 -2
- package/dist/{chunk-T4DB5YFB.mjs → chunk-54HYC5FT.mjs} +6 -6
- package/dist/{chunk-ZIJ63TBT.mjs → chunk-7KC56574.mjs} +1 -1
- package/dist/{chunk-67Z2V6TC.mjs → chunk-7QONURHW.mjs} +3 -3
- package/dist/{chunk-BVOJIYRV.mjs → chunk-BB7MASRQ.mjs} +1 -1
- package/dist/{chunk-6VZ3Q73N.mjs → chunk-CHOGQSHK.mjs} +44 -25
- package/dist/{chunk-QZN3MIJ3.mjs → chunk-ES33L3HO.mjs} +2 -2
- package/dist/{chunk-YCJLVNWR.mjs → chunk-F52QGNZ2.mjs} +1 -1
- package/dist/{chunk-43ORORN6.mjs → chunk-GJZHU4FC.mjs} +3 -3
- package/dist/{chunk-R7OSDAFK.mjs → chunk-HBVLF3HA.mjs} +1 -1
- package/dist/{chunk-ULRPNYAG.mjs → chunk-ICL2AELD.mjs} +120 -62
- package/dist/{chunk-3RDS4UHC.mjs → chunk-J7SDAOTP.mjs} +4 -4
- package/dist/{chunk-A4G7JSYO.mjs → chunk-JFUE6F2Y.mjs} +98 -49
- package/dist/chunk-KOKM4PQN.mjs +52 -0
- package/dist/{chunk-7OQHSDK6.mjs → chunk-L26LEDHO.mjs} +7 -7
- package/dist/{chunk-EONIHG3V.mjs → chunk-MASE2LTJ.mjs} +7 -7
- package/dist/{chunk-MXS5HWIH.mjs → chunk-MWT5WCJ5.mjs} +89 -17
- package/dist/{chunk-3TTYK5A7.mjs → chunk-Q4UTU5ZE.mjs} +9 -9
- package/dist/{chunk-T6CNO72J.mjs → chunk-QM3VPX4Z.mjs} +2 -2
- package/dist/chunk-QY7GC2MW.mjs +33 -0
- package/dist/{chunk-BJ2YKRNX.mjs → chunk-R36MIGBU.mjs} +86 -54
- package/dist/{chunk-RVM7AODE.mjs → chunk-RDJKYKMQ.mjs} +2 -2
- package/dist/{chunk-UXQE2ROB.mjs → chunk-RUIXODYU.mjs} +3 -2
- package/dist/{chunk-5ER245FI.mjs → chunk-SBCEHIPL.mjs} +3 -3
- package/dist/{chunk-TPJU7NDJ.mjs → chunk-SYWGORQB.mjs} +1 -0
- package/dist/{chunk-UKSUENHK.mjs → chunk-T2QN2FA4.mjs} +40 -26
- package/dist/{chunk-CO4MRKST.mjs → chunk-WLOF6HSP.mjs} +2 -2
- package/dist/{chunk-MKWUXWQH.mjs → chunk-ZHS44YKF.mjs} +1 -1
- package/dist/{client-Cc42b42j.d.ts → client-DKwterMb.d.ts} +2 -2
- package/dist/client.d.ts +6 -6
- package/dist/client.mjs +1 -0
- package/dist/{config-DyRowvVP.d.ts → config-6YBtxekG.d.ts} +1 -1
- package/dist/{config-BYne-yCI.d.ts → config-DPphZfuM.d.ts} +1 -1
- package/dist/direct-executor-5KJGVQHM.mjs +22 -0
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +5 -5
- package/dist/distribution.mjs +26 -26
- package/dist/{durability-wiring-6AN4QTT3.mjs → durability-wiring-FRPV6JLZ.mjs} +518 -91
- package/dist/embeddings/openai.mjs +1 -1
- package/dist/{engine-96_kLtSY.d.ts → engine-BGt9lI1V.d.ts} +11 -11
- package/dist/index.browser.mjs +2 -2
- package/dist/index.d.ts +5 -5
- package/dist/index.mjs +18 -18
- package/dist/{memory-dd3GESdc.d.ts → memory-COdoGthn.d.ts} +96 -76
- package/dist/{module-WIDYIJLY.mjs → module-IUTNXWMV.mjs} +72 -38
- package/dist/partial-words.mjs +5 -5
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.mjs +21 -21
- package/dist/react.d.ts +6 -6
- package/dist/react.mjs +1 -0
- package/dist/release-BH3DKW3T.mjs +33 -0
- package/dist/replace-CEO5KJJ3.mjs +57 -0
- package/dist/restore-queue-I5SI6AUQ.mjs +35 -0
- package/dist/server.d.ts +8 -8
- package/dist/server.mjs +10 -10
- package/dist/snapshot-YC6TS4OP.mjs +738 -0
- package/dist/synonyms.mjs +3 -3
- package/dist/{tasks-DA1Ff0G4.d.ts → tasks-BJmamiC4.d.ts} +1 -1
- package/dist/{thread-VTDLLYXC.mjs → thread-U7XYJWGG.mjs} +15 -12
- package/dist/vector/search-worker.mjs +6 -6
- package/dist/workers/entry.mjs +5 -5
- package/dist/{write-OJR3H4BI.mjs → write-M2OALSA4.mjs} +20 -20
- package/package.json +7 -7
- package/dist/chunk-Y2F66IQJ.mjs +0 -74
- package/dist/chunk-ZJ4CSAB2.mjs +0 -13
- package/dist/direct-executor-CHXVCBN7.mjs +0 -22
- package/dist/snapshot-JFZVOGZV.mjs +0 -479
|
@@ -1,81 +1,456 @@
|
|
|
1
|
-
import { readMetadataEnvelope, readSegmentManifest, reclaimUnlistedKeys, loadSegmentedCheckpoint, writeMetadataEnvelope, readCurrentManifest, deserializePayloadV2, unpackEnvelopeBytes, growPartitionsTo, documentsOfPayloadV2 } from './chunk-
|
|
2
|
-
import { decodeSegmentManifest, metadataFromConfig, createSeqOwner, isRecord, invalidManifest, SINGLE_NODE_PRIMARY_TERM, writeLiveVectors, isId, isUint32, isSequenceNumber, VECTOR_FILE_ENVELOPE_VERSION, metadataConfigOf } from './chunk-
|
|
3
|
-
import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-
|
|
1
|
+
import { readMetadataEnvelope, readSegmentManifest, reclaimUnlistedKeys, loadSegmentedCheckpoint, writeMetadataEnvelope, readCurrentManifest, deserializePayloadV2, unpackEnvelopeBytes, growPartitionsTo, documentsOfPayloadV2 } from './chunk-54HYC5FT.mjs';
|
|
2
|
+
import { decodeSegmentManifest, metadataFromConfig, createSeqOwner, isRecord, invalidManifest, SINGLE_NODE_PRIMARY_TERM, writeLiveVectors, isId, isUint32, isSequenceNumber, VECTOR_FILE_ENVELOPE_VERSION, metadataConfigOf } from './chunk-MASE2LTJ.mjs';
|
|
3
|
+
import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-F52QGNZ2.mjs';
|
|
4
4
|
import './chunk-37CD3OKS.mjs';
|
|
5
|
-
import { createDurableDirectory } from './chunk-
|
|
5
|
+
import { createDurableDirectory, getFs } from './chunk-7QONURHW.mjs';
|
|
6
6
|
import { spawnNodeWorker } from './chunk-C7SLA6CM.mjs';
|
|
7
|
-
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, readListedFile, adapterCheckpointStore, snapshotBundleKey, manifestReferencedKeys, checkpointLastSeqNo, commitCheckpointManifest, removeCheckpointGarbage, readSegmentFile, checkpointPrimaryTerm, writeCheckpointSegments, partitionContentsOf, frameRecord, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, readCommitMarker, readWalSegments, durableRecordMissing, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, MAX_SEGMENTS_PER_PARTITION, writeSegmentHeader, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, SEGMENT_HEADER_SIZE, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, LOG_RECORDS_PER_CHECKPOINT, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, buildEntry } from './chunk-
|
|
8
|
-
import { applyDeleteEntry, applyIndexEntry } from './chunk-
|
|
7
|
+
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, readListedFile, adapterCheckpointStore, snapshotBundleKey, manifestReferencedKeys, checkpointLastSeqNo, manifestFileRefs, commitCheckpointManifest, removeCheckpointGarbage, readSegmentFile, checkpointPrimaryTerm, writeCheckpointSegments, partitionContentsOf, frameRecord, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, readCommitMarker, readWalSegments, durableRecordMissing, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, MAX_SEGMENTS_PER_PARTITION, writeSegmentHeader, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, SEGMENT_HEADER_SIZE, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, LOG_RECORDS_PER_CHECKPOINT, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, buildEntry } from './chunk-Q4UTU5ZE.mjs';
|
|
8
|
+
import { applyDeleteEntry, applyIndexEntry } from './chunk-J7SDAOTP.mjs';
|
|
9
9
|
import './chunk-OHP7QR33.mjs';
|
|
10
10
|
import './chunk-P36MUFEJ.mjs';
|
|
11
|
-
import { openChecksummedEnvelope, HEADER_SIZE } from './chunk-
|
|
12
|
-
import
|
|
13
|
-
import {
|
|
14
|
-
import './chunk-
|
|
15
|
-
import { fnv1a } from './chunk-
|
|
16
|
-
import './chunk-
|
|
11
|
+
import { openChecksummedEnvelope, HEADER_SIZE } from './chunk-JFUE6F2Y.mjs';
|
|
12
|
+
import './chunk-R36MIGBU.mjs';
|
|
13
|
+
import { resolveWorkerEntry, MAX_FILE_BYTES } from './chunk-RDJKYKMQ.mjs';
|
|
14
|
+
import { buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-3LYY62M6.mjs';
|
|
15
|
+
import { fnv1a } from './chunk-CHOGQSHK.mjs';
|
|
16
|
+
import './chunk-WLOF6HSP.mjs';
|
|
17
17
|
import { detectRuntime } from './chunk-AV4IJKD2.mjs';
|
|
18
|
-
import './chunk-
|
|
19
|
-
import './chunk-
|
|
18
|
+
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-Q5DC6WAB.mjs';
|
|
19
|
+
import './chunk-ZHS44YKF.mjs';
|
|
20
20
|
import './chunk-TGP5M5R7.mjs';
|
|
21
21
|
import './chunk-U2BKYEC7.mjs';
|
|
22
|
-
import './chunk-
|
|
23
|
-
import './chunk-
|
|
24
|
-
import './chunk-
|
|
25
|
-
import './chunk-
|
|
22
|
+
import './chunk-KOKM4PQN.mjs';
|
|
23
|
+
import './chunk-QY7GC2MW.mjs';
|
|
24
|
+
import './chunk-ES33L3HO.mjs';
|
|
25
|
+
import './chunk-QM3VPX4Z.mjs';
|
|
26
26
|
import './chunk-WQBCBKY3.mjs';
|
|
27
|
-
import './chunk-
|
|
28
|
-
import { encodeDocument, requireValidFieldTypes } from './chunk-
|
|
27
|
+
import './chunk-SBCEHIPL.mjs';
|
|
28
|
+
import { encodeDocument, requireValidFieldTypes } from './chunk-4ZJWRETC.mjs';
|
|
29
29
|
import './chunk-FGMGUDHX.mjs';
|
|
30
30
|
import { MIN_M } from './chunk-KEHK2O6P.mjs';
|
|
31
31
|
import './chunk-GES7U2PM.mjs';
|
|
32
|
-
import './chunk-
|
|
33
|
-
import './chunk-
|
|
32
|
+
import './chunk-BB7MASRQ.mjs';
|
|
33
|
+
import './chunk-2GZGBOQI.mjs';
|
|
34
34
|
import './chunk-X5CV2E2B.mjs';
|
|
35
35
|
import './chunk-7X5JST5L.mjs';
|
|
36
|
-
import './chunk-
|
|
36
|
+
import './chunk-7KC56574.mjs';
|
|
37
37
|
import './chunk-R6A773WK.mjs';
|
|
38
38
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
39
39
|
import './chunk-R6PR4WYZ.mjs';
|
|
40
|
-
import './chunk-
|
|
40
|
+
import './chunk-HBVLF3HA.mjs';
|
|
41
41
|
import './chunk-L5ORED7V.mjs';
|
|
42
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
42
|
+
import { NarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
|
|
43
43
|
import { decode } from '@msgpack/msgpack';
|
|
44
44
|
|
|
45
|
-
// src/persistence/durability/
|
|
45
|
+
// src/persistence/durability/sha256.ts
|
|
46
|
+
var ROUND_CONSTANTS = new Uint32Array([
|
|
47
|
+
1116352408,
|
|
48
|
+
1899447441,
|
|
49
|
+
3049323471,
|
|
50
|
+
3921009573,
|
|
51
|
+
961987163,
|
|
52
|
+
1508970993,
|
|
53
|
+
2453635748,
|
|
54
|
+
2870763221,
|
|
55
|
+
3624381080,
|
|
56
|
+
310598401,
|
|
57
|
+
607225278,
|
|
58
|
+
1426881987,
|
|
59
|
+
1925078388,
|
|
60
|
+
2162078206,
|
|
61
|
+
2614888103,
|
|
62
|
+
3248222580,
|
|
63
|
+
3835390401,
|
|
64
|
+
4022224774,
|
|
65
|
+
264347078,
|
|
66
|
+
604807628,
|
|
67
|
+
770255983,
|
|
68
|
+
1249150122,
|
|
69
|
+
1555081692,
|
|
70
|
+
1996064986,
|
|
71
|
+
2554220882,
|
|
72
|
+
2821834349,
|
|
73
|
+
2952996808,
|
|
74
|
+
3210313671,
|
|
75
|
+
3336571891,
|
|
76
|
+
3584528711,
|
|
77
|
+
113926993,
|
|
78
|
+
338241895,
|
|
79
|
+
666307205,
|
|
80
|
+
773529912,
|
|
81
|
+
1294757372,
|
|
82
|
+
1396182291,
|
|
83
|
+
1695183700,
|
|
84
|
+
1986661051,
|
|
85
|
+
2177026350,
|
|
86
|
+
2456956037,
|
|
87
|
+
2730485921,
|
|
88
|
+
2820302411,
|
|
89
|
+
3259730800,
|
|
90
|
+
3345764771,
|
|
91
|
+
3516065817,
|
|
92
|
+
3600352804,
|
|
93
|
+
4094571909,
|
|
94
|
+
275423344,
|
|
95
|
+
430227734,
|
|
96
|
+
506948616,
|
|
97
|
+
659060556,
|
|
98
|
+
883997877,
|
|
99
|
+
958139571,
|
|
100
|
+
1322822218,
|
|
101
|
+
1537002063,
|
|
102
|
+
1747873779,
|
|
103
|
+
1955562222,
|
|
104
|
+
2024104815,
|
|
105
|
+
2227730452,
|
|
106
|
+
2361852424,
|
|
107
|
+
2428436474,
|
|
108
|
+
2756734187,
|
|
109
|
+
3204031479,
|
|
110
|
+
3329325298
|
|
111
|
+
]);
|
|
112
|
+
var INITIAL_STATE = [1779033703, 3144134277, 1013904242, 2773480762, 1359893119, 2600822924, 528734635, 1541459225];
|
|
113
|
+
var BLOCK_BYTES = 64;
|
|
114
|
+
var LENGTH_BYTES = 8;
|
|
115
|
+
function rotateRight(value, bits) {
|
|
116
|
+
return value >>> bits | value << 32 - bits;
|
|
117
|
+
}
|
|
118
|
+
function padded(bytes) {
|
|
119
|
+
const total = Math.ceil((bytes.byteLength + 1 + LENGTH_BYTES) / BLOCK_BYTES) * BLOCK_BYTES;
|
|
120
|
+
const message = new Uint8Array(total);
|
|
121
|
+
message.set(bytes);
|
|
122
|
+
message[bytes.byteLength] = 128;
|
|
123
|
+
const view = new DataView(message.buffer);
|
|
124
|
+
const bitLength = bytes.byteLength * 8;
|
|
125
|
+
view.setUint32(total - LENGTH_BYTES, Math.floor(bitLength / 4294967296), false);
|
|
126
|
+
view.setUint32(total - 4, bitLength >>> 0, false);
|
|
127
|
+
return view;
|
|
128
|
+
}
|
|
129
|
+
function sha256(bytes) {
|
|
130
|
+
const view = padded(bytes);
|
|
131
|
+
const state = INITIAL_STATE.slice();
|
|
132
|
+
const words = new Uint32Array(64);
|
|
133
|
+
for (let block = 0; block < view.byteLength; block += BLOCK_BYTES) {
|
|
134
|
+
for (let i = 0; i < 16; i++) words[i] = view.getUint32(block + i * 4, false);
|
|
135
|
+
for (let i = 16; i < 64; i++) {
|
|
136
|
+
const w15 = words[i - 15];
|
|
137
|
+
const w2 = words[i - 2];
|
|
138
|
+
const s0 = rotateRight(w15, 7) ^ rotateRight(w15, 18) ^ w15 >>> 3;
|
|
139
|
+
const s1 = rotateRight(w2, 17) ^ rotateRight(w2, 19) ^ w2 >>> 10;
|
|
140
|
+
words[i] = words[i - 16] + s0 + words[i - 7] + s1 >>> 0;
|
|
141
|
+
}
|
|
142
|
+
let [a, b, c, d, e, f, g, h] = state;
|
|
143
|
+
for (let i = 0; i < 64; i++) {
|
|
144
|
+
const s1 = rotateRight(e, 6) ^ rotateRight(e, 11) ^ rotateRight(e, 25);
|
|
145
|
+
const choice = e & f ^ ~e & g;
|
|
146
|
+
const t1 = h + s1 + choice + ROUND_CONSTANTS[i] + words[i] >>> 0;
|
|
147
|
+
const s0 = rotateRight(a, 2) ^ rotateRight(a, 13) ^ rotateRight(a, 22);
|
|
148
|
+
const majority = a & b ^ a & c ^ b & c;
|
|
149
|
+
const t2 = s0 + majority >>> 0;
|
|
150
|
+
h = g;
|
|
151
|
+
g = f;
|
|
152
|
+
f = e;
|
|
153
|
+
e = d + t1 >>> 0;
|
|
154
|
+
d = c;
|
|
155
|
+
c = b;
|
|
156
|
+
b = a;
|
|
157
|
+
a = t1 + t2 >>> 0;
|
|
158
|
+
}
|
|
159
|
+
state[0] = state[0] + a >>> 0;
|
|
160
|
+
state[1] = state[1] + b >>> 0;
|
|
161
|
+
state[2] = state[2] + c >>> 0;
|
|
162
|
+
state[3] = state[3] + d >>> 0;
|
|
163
|
+
state[4] = state[4] + e >>> 0;
|
|
164
|
+
state[5] = state[5] + f >>> 0;
|
|
165
|
+
state[6] = state[6] + g >>> 0;
|
|
166
|
+
state[7] = state[7] + h >>> 0;
|
|
167
|
+
}
|
|
168
|
+
const digest = new Uint8Array(32);
|
|
169
|
+
const out = new DataView(digest.buffer);
|
|
170
|
+
for (let i = 0; i < 8; i++) out.setUint32(i * 4, state[i], false);
|
|
171
|
+
return digest;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
// src/persistence/durability/restore-journal.ts
|
|
175
|
+
var RESTORE_JOURNAL_PREFIX = ".swap-";
|
|
46
176
|
var RESTORE_STAGING_PREFIX = ".restore-";
|
|
47
|
-
|
|
48
|
-
|
|
177
|
+
var RETIRED_INDEX_PREFIX = ".retired-";
|
|
178
|
+
var LOCK_FILE_KEY = ".narsil.lock";
|
|
179
|
+
var STAGING_PREFIX_PATTERN = /^\.[A-Za-z0-9][A-Za-z0-9._-]*\/$/;
|
|
180
|
+
function digestOf(indexName) {
|
|
181
|
+
let hex = "";
|
|
182
|
+
for (const byte of sha256(new TextEncoder().encode(indexName))) hex += byte.toString(16).padStart(2, "0");
|
|
183
|
+
return hex;
|
|
184
|
+
}
|
|
185
|
+
function journalKey(indexName) {
|
|
186
|
+
return `${RESTORE_JOURNAL_PREFIX}${digestOf(indexName)}`;
|
|
187
|
+
}
|
|
188
|
+
function randomSuffix() {
|
|
189
|
+
return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
|
|
190
|
+
}
|
|
191
|
+
function newStagingPrefix() {
|
|
192
|
+
return `${RESTORE_STAGING_PREFIX}${randomSuffix()}/`;
|
|
193
|
+
}
|
|
194
|
+
function newRetiredPrefix() {
|
|
195
|
+
return `${RETIRED_INDEX_PREFIX}${randomSuffix()}/`;
|
|
196
|
+
}
|
|
197
|
+
function encodeJournal(indexName, stagingPrefix) {
|
|
198
|
+
return new TextEncoder().encode(`${indexName}
|
|
199
|
+
${stagingPrefix}`);
|
|
200
|
+
}
|
|
201
|
+
function isStagingPrefix(prefix) {
|
|
202
|
+
return STAGING_PREFIX_PATTERN.test(prefix) && !prefix.includes("..");
|
|
203
|
+
}
|
|
204
|
+
function corruptJournal(key, reason) {
|
|
205
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The restore journal "${key}" ${reason}`, { key });
|
|
206
|
+
}
|
|
207
|
+
function readJournal(key, bytes) {
|
|
208
|
+
const text = new TextDecoder().decode(bytes);
|
|
209
|
+
const lineEnd = text.indexOf("\n");
|
|
210
|
+
const indexName = lineEnd === -1 ? text : text.slice(0, lineEnd);
|
|
211
|
+
if (key.slice(RESTORE_JOURNAL_PREFIX.length) !== digestOf(indexName)) return { kind: "leftover" };
|
|
212
|
+
if (indexName.length > MAX_INDEX_NAME_LENGTH || !INDEX_NAME_PATTERN.test(indexName) || indexName.includes("..")) {
|
|
213
|
+
corruptJournal(key, "names an index that no index name can match");
|
|
214
|
+
}
|
|
215
|
+
const stagingPrefix = lineEnd === -1 ? "" : text.slice(lineEnd + 1);
|
|
216
|
+
if (!isStagingPrefix(stagingPrefix) || stagingPrefix === `${LOCK_FILE_KEY}/`) {
|
|
217
|
+
corruptJournal(key, "names no staging prefix that a restore writes");
|
|
218
|
+
}
|
|
219
|
+
return { kind: "journal", indexName, stagingPrefix };
|
|
220
|
+
}
|
|
221
|
+
async function listedKeys(storage, indexName, stagingPrefix) {
|
|
222
|
+
const manifestBytes = await storage.read(`${stagingPrefix}manifest`) ?? await storage.read(manifestKey(indexName));
|
|
223
|
+
if (manifestBytes === null) {
|
|
224
|
+
throw new NarsilError(
|
|
225
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
226
|
+
`The restore of index "${indexName}" lost its manifest, both from staging and from the index`,
|
|
227
|
+
{ indexName, stagingPrefix }
|
|
228
|
+
);
|
|
229
|
+
}
|
|
230
|
+
const decoded = await decodeSegmentManifest(manifestBytes, indexName);
|
|
231
|
+
if (decoded.kind !== "current") {
|
|
232
|
+
throw new NarsilError(
|
|
233
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
234
|
+
`The restore of index "${indexName}" staged a manifest in the layout of an earlier release`,
|
|
235
|
+
{ indexName }
|
|
236
|
+
);
|
|
237
|
+
}
|
|
238
|
+
const root = `${indexName}/`;
|
|
239
|
+
const keys = /* @__PURE__ */ new Set(["meta", "manifest"]);
|
|
240
|
+
for (const ref of manifestFileRefs(decoded.manifest)) keys.add(ref.key.slice(root.length));
|
|
241
|
+
return keys;
|
|
242
|
+
}
|
|
243
|
+
function withoutTrailingSlash(prefix) {
|
|
244
|
+
return prefix.slice(0, -1);
|
|
245
|
+
}
|
|
246
|
+
async function finishJournal(storage, indexName, stagingPrefix) {
|
|
247
|
+
const root = `${indexName}/`;
|
|
248
|
+
const listed = await listedKeys(storage, indexName, stagingPrefix);
|
|
249
|
+
const staged = await storage.list(stagingPrefix);
|
|
250
|
+
let retiredPrefix = null;
|
|
251
|
+
if (storage.moveTree !== void 0 && staged.length > 0) {
|
|
252
|
+
retiredPrefix = newRetiredPrefix();
|
|
253
|
+
await storage.moveTree(indexName, withoutTrailingSlash(retiredPrefix));
|
|
254
|
+
await storage.moveTree(withoutTrailingSlash(stagingPrefix), indexName);
|
|
255
|
+
} else {
|
|
256
|
+
const stagedKeys = new Set(staged);
|
|
257
|
+
for (const key of listed) {
|
|
258
|
+
if (stagedKeys.has(`${stagingPrefix}${key}`)) await storage.moveKey(`${stagingPrefix}${key}`, `${root}${key}`);
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
for (const key of await storage.list(root)) {
|
|
262
|
+
if (!listed.has(key.slice(root.length))) await storage.remove(key);
|
|
263
|
+
}
|
|
264
|
+
await storage.remove(journalKey(indexName));
|
|
265
|
+
return retiredPrefix;
|
|
266
|
+
}
|
|
267
|
+
async function removePrefix(storage, prefix) {
|
|
268
|
+
if (storage.removeTree !== void 0) {
|
|
269
|
+
await storage.removeTree(prefix);
|
|
270
|
+
return;
|
|
271
|
+
}
|
|
272
|
+
for (const key of await storage.list(prefix)) await storage.remove(key);
|
|
273
|
+
}
|
|
274
|
+
async function commitJournal(storage, indexName, stagingPrefix) {
|
|
275
|
+
await storage.writeJournal(journalKey(indexName), encodeJournal(indexName, stagingPrefix));
|
|
276
|
+
}
|
|
277
|
+
function isTopLevelJournal(key) {
|
|
278
|
+
return key.startsWith(RESTORE_JOURNAL_PREFIX) && !key.includes("/");
|
|
279
|
+
}
|
|
280
|
+
async function removeStagingPrefixes(storage) {
|
|
281
|
+
const prefixes = /* @__PURE__ */ new Set();
|
|
282
|
+
for (const key of await storage.list(".")) {
|
|
283
|
+
const slash = key.indexOf("/");
|
|
284
|
+
if (slash !== -1 && isStagingPrefix(key.slice(0, slash + 1))) prefixes.add(key.slice(0, slash + 1));
|
|
285
|
+
}
|
|
286
|
+
for (const prefix of prefixes) await removePrefix(storage, prefix);
|
|
287
|
+
}
|
|
288
|
+
async function recoverRestoreJournals(storage) {
|
|
289
|
+
for (const key of await storage.list(RESTORE_JOURNAL_PREFIX)) {
|
|
290
|
+
if (!isTopLevelJournal(key)) continue;
|
|
291
|
+
const bytes = await storage.read(key);
|
|
292
|
+
if (bytes === null) continue;
|
|
293
|
+
const reading = readJournal(key, bytes);
|
|
294
|
+
if (reading.kind === "leftover") {
|
|
295
|
+
await storage.remove(key);
|
|
296
|
+
continue;
|
|
297
|
+
}
|
|
298
|
+
await finishJournal(storage, reading.indexName, reading.stagingPrefix);
|
|
299
|
+
}
|
|
300
|
+
await removeStagingPrefixes(storage);
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
// src/persistence/durability/restore-directory.ts
|
|
304
|
+
function missing(error) {
|
|
305
|
+
return error.code === "ENOENT";
|
|
306
|
+
}
|
|
307
|
+
function storageFailed(code, message, key, error) {
|
|
308
|
+
return new NarsilError(code, message, { key, cause: error instanceof Error ? error.message : String(error) });
|
|
309
|
+
}
|
|
310
|
+
function directoryRestoreStorage(directory) {
|
|
311
|
+
return {
|
|
312
|
+
read: (key) => directory.read(key),
|
|
313
|
+
write: (key, chunks) => directory.atomicWrite(key, chunks),
|
|
314
|
+
writeJournal: (key, bytes) => directory.atomicWrite(key, bytes),
|
|
315
|
+
remove: (key) => directory.remove(key),
|
|
316
|
+
list: (prefix) => directory.list(prefix),
|
|
317
|
+
moveKey: (from, to) => directory.move(from, to),
|
|
318
|
+
pathOf: (key) => directory.pathOf(key),
|
|
319
|
+
async moveTree(from, to) {
|
|
320
|
+
const source = await directory.pathOf(from);
|
|
321
|
+
const target = await directory.pathOf(to);
|
|
322
|
+
const fs = await getFs();
|
|
323
|
+
try {
|
|
324
|
+
await fs.rename(source, target);
|
|
325
|
+
} catch (error) {
|
|
326
|
+
if (missing(error)) return;
|
|
327
|
+
throw storageFailed(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Failed to move "${from}" to "${to}"`, from, error);
|
|
328
|
+
}
|
|
329
|
+
await directory.syncDirectoryOf(to);
|
|
330
|
+
},
|
|
331
|
+
async removeTree(prefix) {
|
|
332
|
+
const key = prefix.endsWith("/") ? prefix.slice(0, -1) : prefix;
|
|
333
|
+
const fs = await getFs();
|
|
334
|
+
try {
|
|
335
|
+
await fs.rm(await directory.pathOf(key), { recursive: true, force: true });
|
|
336
|
+
} catch (error) {
|
|
337
|
+
throw storageFailed(ErrorCodes.PERSISTENCE_DELETE_FAILED, `Failed to delete "${key}"`, key, error);
|
|
338
|
+
}
|
|
339
|
+
await directory.syncDirectoryOf(key);
|
|
340
|
+
}
|
|
341
|
+
};
|
|
342
|
+
}
|
|
343
|
+
function recoverDirectoryRestores(directory) {
|
|
344
|
+
return recoverRestoreJournals(directoryRestoreStorage(directory));
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
// src/persistence/durability/restore-staging.ts
|
|
348
|
+
function toError(error) {
|
|
349
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
350
|
+
}
|
|
351
|
+
function pathUnder(path, root) {
|
|
352
|
+
if (!path.startsWith(root)) return false;
|
|
353
|
+
const next = path.charAt(root.length);
|
|
354
|
+
return next === "" || next === "/" || next === "\\";
|
|
355
|
+
}
|
|
356
|
+
function createRestoreStaging(storage, indexName, options = {}) {
|
|
357
|
+
const stagingPrefix = newStagingPrefix();
|
|
49
358
|
const indexRoot = `${indexName}/`;
|
|
359
|
+
let committed = false;
|
|
50
360
|
const staged = (key) => {
|
|
51
361
|
if (!key.startsWith(indexRoot)) {
|
|
52
362
|
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `A restore of "${indexName}" cannot stage "${key}"`, {
|
|
53
363
|
key
|
|
54
364
|
});
|
|
55
365
|
}
|
|
56
|
-
return `${
|
|
366
|
+
return `${stagingPrefix}${key.slice(indexRoot.length)}`;
|
|
57
367
|
};
|
|
58
|
-
const unstaged = (key) => `${indexRoot}${key.slice(
|
|
59
|
-
const
|
|
60
|
-
read: (key) =>
|
|
61
|
-
write: (key, chunks) =>
|
|
62
|
-
remove: (key) =>
|
|
63
|
-
list: async (prefix) => (await
|
|
64
|
-
pathOf: (key) => directory.pathOf(staged(key))
|
|
368
|
+
const unstaged = (key) => `${indexRoot}${key.slice(stagingPrefix.length)}`;
|
|
369
|
+
const store = {
|
|
370
|
+
read: (key) => storage.read(staged(key)),
|
|
371
|
+
write: (key, chunks) => storage.write(staged(key), chunks),
|
|
372
|
+
remove: (key) => storage.remove(staged(key)),
|
|
373
|
+
list: async (prefix) => (await storage.list(staged(prefix))).map(unstaged)
|
|
65
374
|
};
|
|
375
|
+
const pathOf = storage.pathOf;
|
|
376
|
+
if (pathOf !== void 0) store.pathOf = (key) => pathOf.call(storage, staged(key));
|
|
377
|
+
let retiredPrefix = null;
|
|
378
|
+
async function finishOnStorage() {
|
|
379
|
+
retiredPrefix = await finishJournal(storage, indexName, stagingPrefix);
|
|
380
|
+
}
|
|
66
381
|
return {
|
|
67
|
-
store
|
|
382
|
+
store,
|
|
383
|
+
async relocation() {
|
|
384
|
+
if (pathOf === void 0 || (await storage.list(stagingPrefix)).length > 0) return null;
|
|
385
|
+
const from = await pathOf.call(storage, stagingPrefix.slice(0, -1));
|
|
386
|
+
const to = await pathOf.call(storage, indexName);
|
|
387
|
+
return (path) => pathUnder(path, from) ? `${to}${path.slice(from.length)}` : path;
|
|
388
|
+
},
|
|
68
389
|
async commit() {
|
|
69
|
-
|
|
70
|
-
|
|
390
|
+
try {
|
|
391
|
+
await commitJournal(storage, indexName, stagingPrefix);
|
|
392
|
+
} catch (error) {
|
|
393
|
+
try {
|
|
394
|
+
await storage.remove(journalKey(indexName));
|
|
395
|
+
} catch (removal) {
|
|
396
|
+
committed = true;
|
|
397
|
+
options.onStorageLost?.(toError(removal));
|
|
398
|
+
}
|
|
399
|
+
throw error;
|
|
400
|
+
}
|
|
401
|
+
committed = true;
|
|
402
|
+
},
|
|
403
|
+
async finish() {
|
|
404
|
+
try {
|
|
405
|
+
await (options.finish === void 0 ? finishOnStorage() : options.finish(finishOnStorage));
|
|
406
|
+
} catch (error) {
|
|
407
|
+
options.onStorageLost?.(toError(error));
|
|
408
|
+
}
|
|
409
|
+
},
|
|
410
|
+
async removeRetired() {
|
|
411
|
+
const prefix = retiredPrefix;
|
|
412
|
+
retiredPrefix = null;
|
|
413
|
+
if (prefix !== null) await removePrefix(storage, prefix);
|
|
71
414
|
},
|
|
72
415
|
async discard() {
|
|
73
|
-
|
|
416
|
+
if (committed) return;
|
|
417
|
+
await removePrefix(storage, stagingPrefix);
|
|
74
418
|
}
|
|
75
419
|
};
|
|
76
420
|
}
|
|
77
|
-
|
|
78
|
-
|
|
421
|
+
function adapterRestoreStorage(adapter) {
|
|
422
|
+
const store = adapterCheckpointStore(adapter);
|
|
423
|
+
return {
|
|
424
|
+
read: (key) => store.read(key),
|
|
425
|
+
write: (key, chunks) => store.write(key, chunks),
|
|
426
|
+
writeJournal: (key, bytes) => store.write(key, [bytes]),
|
|
427
|
+
remove: (key) => store.remove(key),
|
|
428
|
+
list: (prefix) => store.list(prefix),
|
|
429
|
+
async moveKey(from, to) {
|
|
430
|
+
const bytes = await store.read(from);
|
|
431
|
+
if (bytes === null) return;
|
|
432
|
+
await store.write(to, [bytes]);
|
|
433
|
+
await store.remove(from);
|
|
434
|
+
}
|
|
435
|
+
};
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
// src/persistence/durability/archive-checkpoint.ts
|
|
439
|
+
function toError2(error) {
|
|
440
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
441
|
+
}
|
|
442
|
+
function stageDirectoryRestore(deps, indexName) {
|
|
443
|
+
return createRestoreStaging(directoryRestoreStorage(deps.directory), indexName, {
|
|
444
|
+
async finish(run) {
|
|
445
|
+
try {
|
|
446
|
+
await run();
|
|
447
|
+
} catch (error) {
|
|
448
|
+
deps.markFatal(toError2(error));
|
|
449
|
+
throw error;
|
|
450
|
+
}
|
|
451
|
+
},
|
|
452
|
+
onStorageLost: deps.markFatal
|
|
453
|
+
});
|
|
79
454
|
}
|
|
80
455
|
function releaser(held, indexName) {
|
|
81
456
|
let released = false;
|
|
@@ -126,7 +501,7 @@ function createArchiveCheckpointAccess(deps) {
|
|
|
126
501
|
}
|
|
127
502
|
},
|
|
128
503
|
stageRestore(indexName) {
|
|
129
|
-
return
|
|
504
|
+
return stageDirectoryRestore(deps, indexName);
|
|
130
505
|
},
|
|
131
506
|
adoptRestoredCheckpoint(indexName, checkpoint) {
|
|
132
507
|
for (const { partitionId, lastSeqNo } of checkpoint) deps.registerPartition(indexName, partitionId, lastSeqNo);
|
|
@@ -384,8 +759,8 @@ async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartit
|
|
|
384
759
|
const vectorPlans = /* @__PURE__ */ new Map();
|
|
385
760
|
for (const [fieldPath, vectorIndex] of vectorIndexes) {
|
|
386
761
|
const listed = listedVectorFields.find((ref) => ref.fieldPath === fieldPath);
|
|
387
|
-
const
|
|
388
|
-
vectorPlans.set(fieldPath, vectorIndex.planCheckpoint(
|
|
762
|
+
const listedKeys2 = listed === void 0 ? null : listed.files.map((file) => file.key);
|
|
763
|
+
vectorPlans.set(fieldPath, vectorIndex.planCheckpoint(listedKeys2));
|
|
389
764
|
}
|
|
390
765
|
const memoryMatchesTheTargets = recordsLeftInLog === 0;
|
|
391
766
|
const wholePartitions = memoryMatchesTheTargets ? wholePartitionsOf(targets) : noWholePartitions();
|
|
@@ -879,7 +1254,7 @@ async function drainIndexStateForUnload(indexState, pendingMetadataWrite, closeW
|
|
|
879
1254
|
}
|
|
880
1255
|
|
|
881
1256
|
// src/persistence/durability/mutation-batch.ts
|
|
882
|
-
function
|
|
1257
|
+
function toError3(err) {
|
|
883
1258
|
return err instanceof Error ? err : new Error(String(err));
|
|
884
1259
|
}
|
|
885
1260
|
function failUnlessAlreadyFailed(outcomes, count, error) {
|
|
@@ -891,7 +1266,7 @@ function failUnlessAlreadyFailed(outcomes, count, error) {
|
|
|
891
1266
|
}
|
|
892
1267
|
}
|
|
893
1268
|
function failPartition(partition, deps, err) {
|
|
894
|
-
const error =
|
|
1269
|
+
const error = toError3(err);
|
|
895
1270
|
partition.failed = error;
|
|
896
1271
|
deps.markFatal(error);
|
|
897
1272
|
return error;
|
|
@@ -971,7 +1346,7 @@ async function recordPartitionBatch(partition, records, deps) {
|
|
|
971
1346
|
}
|
|
972
1347
|
}
|
|
973
1348
|
} catch (err) {
|
|
974
|
-
failUnlessAlreadyFailed(outcomes, records.length,
|
|
1349
|
+
failUnlessAlreadyFailed(outcomes, records.length, toError3(err));
|
|
975
1350
|
}
|
|
976
1351
|
return outcomes;
|
|
977
1352
|
}
|
|
@@ -1808,8 +2183,8 @@ async function createMarkerWriter(directory, indexName, partitionId) {
|
|
|
1808
2183
|
// src/persistence/durability/wal-writer.ts
|
|
1809
2184
|
var SEGMENT_TAIL_PATTERN = /^\d{16}$/;
|
|
1810
2185
|
function segmentKey(indexName, partitionId, startSeqNo) {
|
|
1811
|
-
const
|
|
1812
|
-
return `${indexName}/wal/${partitionId}/${
|
|
2186
|
+
const padded2 = startSeqNo.toString().padStart(16, "0");
|
|
2187
|
+
return `${indexName}/wal/${partitionId}/${padded2}`;
|
|
1813
2188
|
}
|
|
1814
2189
|
function parseSegmentStartSeqNo2(key) {
|
|
1815
2190
|
const tail = key.slice(key.lastIndexOf("/") + 1);
|
|
@@ -2055,14 +2430,14 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
2055
2430
|
stopTimers();
|
|
2056
2431
|
hooks.onFatalError(error);
|
|
2057
2432
|
}
|
|
2058
|
-
function
|
|
2433
|
+
function toError5(err) {
|
|
2059
2434
|
return err instanceof Error ? err : new Error(String(err));
|
|
2060
2435
|
}
|
|
2061
2436
|
async function closeWriterReportingFailure(closePromise) {
|
|
2062
2437
|
try {
|
|
2063
2438
|
await closePromise;
|
|
2064
2439
|
} catch (err) {
|
|
2065
|
-
markFatal(
|
|
2440
|
+
markFatal(toError5(err));
|
|
2066
2441
|
}
|
|
2067
2442
|
}
|
|
2068
2443
|
function getOrCreateIndexState(indexName) {
|
|
@@ -2128,7 +2503,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
2128
2503
|
try {
|
|
2129
2504
|
await partition.walWriter.commit();
|
|
2130
2505
|
} catch (err) {
|
|
2131
|
-
partition.failed =
|
|
2506
|
+
partition.failed = toError5(err);
|
|
2132
2507
|
markFatal(partition.failed);
|
|
2133
2508
|
}
|
|
2134
2509
|
}
|
|
@@ -2142,7 +2517,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
2142
2517
|
try {
|
|
2143
2518
|
await checkpointIndex(indexName);
|
|
2144
2519
|
} catch (err) {
|
|
2145
|
-
markFatal(
|
|
2520
|
+
markFatal(toError5(err));
|
|
2146
2521
|
}
|
|
2147
2522
|
}
|
|
2148
2523
|
}
|
|
@@ -2223,7 +2598,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
2223
2598
|
indexState.documentBytesSinceCheckpoint += recorded.documentBytes;
|
|
2224
2599
|
if (checkpointIsDue(indexState, checkpointMutationThreshold)) {
|
|
2225
2600
|
void checkpointIndex(indexName).catch((err) => {
|
|
2226
|
-
markFatal(
|
|
2601
|
+
markFatal(toError5(err));
|
|
2227
2602
|
});
|
|
2228
2603
|
}
|
|
2229
2604
|
await stallWhileCheckpointsFallBehind(indexState);
|
|
@@ -2255,12 +2630,10 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
2255
2630
|
);
|
|
2256
2631
|
}
|
|
2257
2632
|
return {
|
|
2258
|
-
isActive()
|
|
2259
|
-
return true;
|
|
2260
|
-
},
|
|
2633
|
+
isActive: () => true,
|
|
2261
2634
|
async recover(metadataOnly = false) {
|
|
2262
2635
|
await directoryWatch.claim();
|
|
2263
|
-
await
|
|
2636
|
+
await recoverDirectoryRestores(directory);
|
|
2264
2637
|
const names = await listPersistedIndexes(directory);
|
|
2265
2638
|
for (const indexName of names) {
|
|
2266
2639
|
await recoverIndex(indexName, metadataOnly);
|
|
@@ -2330,7 +2703,8 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
2330
2703
|
directory,
|
|
2331
2704
|
held: heldCheckpointFiles,
|
|
2332
2705
|
checkpoint: (name) => checkpointIndex(name),
|
|
2333
|
-
registerPartition: (name, partitionId, startSeqNo) => getOrCreatePartition(name, partitionId, startSeqNo)
|
|
2706
|
+
registerPartition: (name, partitionId, startSeqNo) => getOrCreatePartition(name, partitionId, startSeqNo),
|
|
2707
|
+
markFatal
|
|
2334
2708
|
}),
|
|
2335
2709
|
async shutdown() {
|
|
2336
2710
|
shuttingDown = true;
|
|
@@ -2350,7 +2724,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
2350
2724
|
};
|
|
2351
2725
|
}
|
|
2352
2726
|
|
|
2353
|
-
// src/persistence/durability/snapshot-only
|
|
2727
|
+
// src/persistence/durability/snapshot-only/checkpoint.ts
|
|
2354
2728
|
function capture(input, prior) {
|
|
2355
2729
|
const listedIds = new Set((prior?.partitions ?? []).map((partition) => partition.partitionId));
|
|
2356
2730
|
const partitionCountChanged = prior !== null && listedIds.size !== input.manager.partitionCount;
|
|
@@ -2406,7 +2780,44 @@ async function writeSnapshotOnlyCheckpoint(input) {
|
|
|
2406
2780
|
return captured.documentCount;
|
|
2407
2781
|
}
|
|
2408
2782
|
|
|
2409
|
-
// src/persistence/durability/snapshot-only.ts
|
|
2783
|
+
// src/persistence/durability/snapshot-only/restores.ts
|
|
2784
|
+
function toError4(error) {
|
|
2785
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
2786
|
+
}
|
|
2787
|
+
function createPendingRestores(onFatalError) {
|
|
2788
|
+
const pending = /* @__PURE__ */ new Map();
|
|
2789
|
+
return {
|
|
2790
|
+
finishLater(indexName, run) {
|
|
2791
|
+
let running = null;
|
|
2792
|
+
const finish = () => {
|
|
2793
|
+
running ??= run().then(
|
|
2794
|
+
() => {
|
|
2795
|
+
if (pending.get(indexName) === finish) pending.delete(indexName);
|
|
2796
|
+
},
|
|
2797
|
+
(error) => {
|
|
2798
|
+
running = null;
|
|
2799
|
+
throw error;
|
|
2800
|
+
}
|
|
2801
|
+
);
|
|
2802
|
+
return running;
|
|
2803
|
+
};
|
|
2804
|
+
pending.set(indexName, finish);
|
|
2805
|
+
void finish().catch((error) => onFatalError(toError4(error)));
|
|
2806
|
+
return Promise.resolve();
|
|
2807
|
+
},
|
|
2808
|
+
async settle(indexName) {
|
|
2809
|
+
await pending.get(indexName)?.();
|
|
2810
|
+
},
|
|
2811
|
+
forget(indexName) {
|
|
2812
|
+
pending.delete(indexName);
|
|
2813
|
+
},
|
|
2814
|
+
async settleAll() {
|
|
2815
|
+
for (const finish of [...pending.values()]) await finish().catch(() => void 0);
|
|
2816
|
+
}
|
|
2817
|
+
};
|
|
2818
|
+
}
|
|
2819
|
+
|
|
2820
|
+
// src/persistence/durability/snapshot-only/index.ts
|
|
2410
2821
|
function metadataKey(indexName) {
|
|
2411
2822
|
return `${indexName}/meta`;
|
|
2412
2823
|
}
|
|
@@ -2415,8 +2826,13 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
2415
2826
|
const checkpointMutationThreshold = config.checkpointMutationThreshold ?? DEFAULT_CHECKPOINT_MUTATION_THRESHOLD;
|
|
2416
2827
|
const indexes = /* @__PURE__ */ new Map();
|
|
2417
2828
|
const metadataWrites = /* @__PURE__ */ new Map();
|
|
2829
|
+
const restoreStorage = adapterRestoreStorage(adapter);
|
|
2830
|
+
const restores = createPendingRestores(hooks.onFatalError);
|
|
2418
2831
|
let checkpointTimer = null;
|
|
2419
2832
|
let shuttingDown = false;
|
|
2833
|
+
function settleRestore(indexName) {
|
|
2834
|
+
return restores.settle(indexName);
|
|
2835
|
+
}
|
|
2420
2836
|
function getOrCreateIndexState(indexName) {
|
|
2421
2837
|
let state = indexes.get(indexName);
|
|
2422
2838
|
if (state === void 0) {
|
|
@@ -2452,6 +2868,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
2452
2868
|
}
|
|
2453
2869
|
}
|
|
2454
2870
|
async function performCheckpoint(indexName, indexState, fromMemory) {
|
|
2871
|
+
await settleRestore(indexName);
|
|
2455
2872
|
const manager = hooks.getManager(indexName);
|
|
2456
2873
|
const metadata = hooks.buildMetadata(indexName);
|
|
2457
2874
|
if (manager === void 0 || metadata === void 0) {
|
|
@@ -2501,7 +2918,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
2501
2918
|
}
|
|
2502
2919
|
function queueMetadataWrite(indexName, write) {
|
|
2503
2920
|
const previous = metadataWrites.get(indexName) ?? Promise.resolve();
|
|
2504
|
-
const queued = previous.then(write);
|
|
2921
|
+
const queued = previous.then(() => settleRestore(indexName)).then(write);
|
|
2505
2922
|
metadataWrites.set(
|
|
2506
2923
|
indexName,
|
|
2507
2924
|
queued.catch(() => void 0)
|
|
@@ -2509,6 +2926,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
2509
2926
|
return queued;
|
|
2510
2927
|
}
|
|
2511
2928
|
async function recoverIndex(indexName, metadataOnly = false) {
|
|
2929
|
+
await settleRestore(indexName);
|
|
2512
2930
|
const metaBytes = await adapter.load(metadataKey(indexName));
|
|
2513
2931
|
if (metaBytes === null) {
|
|
2514
2932
|
return;
|
|
@@ -2595,11 +3013,28 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
2595
3013
|
}
|
|
2596
3014
|
return appliedSeqNo;
|
|
2597
3015
|
}
|
|
3016
|
+
async function drainIndex(indexName) {
|
|
3017
|
+
const indexState = indexes.get(indexName);
|
|
3018
|
+
if (indexState !== void 0) {
|
|
3019
|
+
indexState.unloading = true;
|
|
3020
|
+
try {
|
|
3021
|
+
await indexState.applyChain;
|
|
3022
|
+
if (indexState.checkpointInFlight !== null) await indexState.checkpointInFlight;
|
|
3023
|
+
await metadataWrites.get(indexName);
|
|
3024
|
+
} catch (error) {
|
|
3025
|
+
indexState.unloading = false;
|
|
3026
|
+
throw error;
|
|
3027
|
+
}
|
|
3028
|
+
}
|
|
3029
|
+
indexes.delete(indexName);
|
|
3030
|
+
metadataWrites.delete(indexName);
|
|
3031
|
+
}
|
|
2598
3032
|
return {
|
|
2599
3033
|
isActive() {
|
|
2600
3034
|
return true;
|
|
2601
3035
|
},
|
|
2602
3036
|
async recover(metadataOnly = false) {
|
|
3037
|
+
await recoverRestoreJournals(restoreStorage);
|
|
2603
3038
|
const keys = await adapter.list("");
|
|
2604
3039
|
for (const key of keys) {
|
|
2605
3040
|
if (key.endsWith("/meta")) {
|
|
@@ -2648,40 +3083,31 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
2648
3083
|
await checkpointIndex(indexName);
|
|
2649
3084
|
}
|
|
2650
3085
|
},
|
|
2651
|
-
|
|
2652
|
-
|
|
2653
|
-
|
|
2654
|
-
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
|
|
2658
|
-
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
|
|
2662
|
-
}
|
|
3086
|
+
stageRestore(indexName) {
|
|
3087
|
+
return createRestoreStaging(restoreStorage, indexName, {
|
|
3088
|
+
finish: (run) => restores.finishLater(indexName, run),
|
|
3089
|
+
onStorageLost: hooks.onFatalError
|
|
3090
|
+
});
|
|
3091
|
+
},
|
|
3092
|
+
adoptRestoredCheckpoint(indexName, checkpoint) {
|
|
3093
|
+
const indexState = getOrCreateIndexState(indexName);
|
|
3094
|
+
indexState.appliedSeqNoByPartition.clear();
|
|
3095
|
+
for (const { partitionId, lastSeqNo } of checkpoint) {
|
|
3096
|
+
if (lastSeqNo > 0) indexState.appliedSeqNoByPartition.set(partitionId, lastSeqNo);
|
|
2663
3097
|
}
|
|
2664
|
-
|
|
2665
|
-
|
|
3098
|
+
},
|
|
3099
|
+
async removeIndex(indexName) {
|
|
3100
|
+
await settleRestore(indexName).catch(() => void 0);
|
|
3101
|
+
restores.forget(indexName);
|
|
3102
|
+
await restoreStorage.remove(journalKey(indexName));
|
|
3103
|
+
await drainIndex(indexName);
|
|
2666
3104
|
for (const key of await adapter.list(`${indexName}/`)) {
|
|
2667
3105
|
await adapter.delete(key);
|
|
2668
3106
|
}
|
|
2669
3107
|
},
|
|
2670
3108
|
async unloadIndex(indexName) {
|
|
2671
|
-
|
|
2672
|
-
|
|
2673
|
-
indexState.unloading = true;
|
|
2674
|
-
try {
|
|
2675
|
-
await indexState.applyChain;
|
|
2676
|
-
if (indexState.checkpointInFlight !== null) await indexState.checkpointInFlight;
|
|
2677
|
-
await metadataWrites.get(indexName);
|
|
2678
|
-
} catch (error) {
|
|
2679
|
-
indexState.unloading = false;
|
|
2680
|
-
throw error;
|
|
2681
|
-
}
|
|
2682
|
-
}
|
|
2683
|
-
indexes.delete(indexName);
|
|
2684
|
-
metadataWrites.delete(indexName);
|
|
3109
|
+
await settleRestore(indexName);
|
|
3110
|
+
await drainIndex(indexName);
|
|
2685
3111
|
},
|
|
2686
3112
|
async reloadIndex(indexName) {
|
|
2687
3113
|
const indexState = getOrCreateIndexState(indexName);
|
|
@@ -2695,6 +3121,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
2695
3121
|
clearInterval(checkpointTimer);
|
|
2696
3122
|
checkpointTimer = null;
|
|
2697
3123
|
}
|
|
3124
|
+
await restores.settleAll();
|
|
2698
3125
|
for (const indexState of indexes.values()) {
|
|
2699
3126
|
if (indexState.checkpointInFlight !== null) {
|
|
2700
3127
|
await indexState.checkpointInFlight.catch(() => void 0);
|