@delali/narsil 0.3.1-next.91 → 0.3.1-next.95

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/dist/adapters/filesystem.mjs +1 -1
  2. package/dist/adapters/indexeddb.mjs +1 -1
  3. package/dist/browser/{chunk-66KYAOIK.mjs → chunk-3U636ADD.mjs} +2 -2
  4. package/dist/browser/{chunk-NOUZ6RH3.mjs → chunk-D3CC2NMZ.mjs} +238 -114
  5. package/dist/browser/{chunk-3ZRMZIH4.mjs → chunk-FYFQJKFW.mjs} +208 -78
  6. package/dist/browser/{chunk-FHFFWLEK.mjs → chunk-HARBSUWP.mjs} +3 -4
  7. package/dist/browser/{chunk-B6XE7KXY.mjs → chunk-LULHQXCK.mjs} +4 -4
  8. package/dist/browser/{chunk-NQW2YXEX.mjs → chunk-WKPJ4E73.mjs} +1 -1
  9. package/dist/browser/{durability-wiring-F4OGQUSU.mjs → durability-wiring-3ZRKSPJD.mjs} +498 -71
  10. package/dist/browser/{module-E25UDADM.mjs → module-IEOQI5GI.mjs} +102 -54
  11. package/dist/browser/release-76MXRPGV.mjs +33 -0
  12. package/dist/browser/replace-RURXTGVG.mjs +57 -0
  13. package/dist/browser/restore-queue-RBULSOVX.mjs +35 -0
  14. package/dist/browser/{snapshot-5EJ7I7QY.mjs → snapshot-Y3NGUJHX.mjs} +362 -103
  15. package/dist/browser/{write-K5V4XQ72.mjs → write-ZUBDNLCH.mjs} +4 -4
  16. package/dist/{capabilities-CvExBThq.d.ts → capabilities-BvxvFzVA.d.ts} +2 -2
  17. package/dist/{chunk-NVL4ZR3K.mjs → chunk-2GZGBOQI.mjs} +2 -2
  18. package/dist/{chunk-XLJ2S2BH.mjs → chunk-3LYY62M6.mjs} +40 -21
  19. package/dist/{chunk-CVQWQVFX.mjs → chunk-4ZJWRETC.mjs} +2 -2
  20. package/dist/{chunk-T4DB5YFB.mjs → chunk-54HYC5FT.mjs} +6 -6
  21. package/dist/{chunk-ZIJ63TBT.mjs → chunk-7KC56574.mjs} +1 -1
  22. package/dist/{chunk-67Z2V6TC.mjs → chunk-7QONURHW.mjs} +3 -3
  23. package/dist/{chunk-BVOJIYRV.mjs → chunk-BB7MASRQ.mjs} +1 -1
  24. package/dist/{chunk-6VZ3Q73N.mjs → chunk-CHOGQSHK.mjs} +44 -25
  25. package/dist/{chunk-QZN3MIJ3.mjs → chunk-ES33L3HO.mjs} +2 -2
  26. package/dist/{chunk-YCJLVNWR.mjs → chunk-F52QGNZ2.mjs} +1 -1
  27. package/dist/{chunk-43ORORN6.mjs → chunk-GJZHU4FC.mjs} +3 -3
  28. package/dist/{chunk-R7OSDAFK.mjs → chunk-HBVLF3HA.mjs} +1 -1
  29. package/dist/{chunk-ULRPNYAG.mjs → chunk-ICL2AELD.mjs} +120 -62
  30. package/dist/{chunk-3RDS4UHC.mjs → chunk-J7SDAOTP.mjs} +4 -4
  31. package/dist/{chunk-A4G7JSYO.mjs → chunk-JFUE6F2Y.mjs} +98 -49
  32. package/dist/chunk-KOKM4PQN.mjs +52 -0
  33. package/dist/{chunk-7OQHSDK6.mjs → chunk-L26LEDHO.mjs} +7 -7
  34. package/dist/{chunk-EONIHG3V.mjs → chunk-MASE2LTJ.mjs} +7 -7
  35. package/dist/{chunk-MXS5HWIH.mjs → chunk-MWT5WCJ5.mjs} +89 -17
  36. package/dist/{chunk-3TTYK5A7.mjs → chunk-Q4UTU5ZE.mjs} +9 -9
  37. package/dist/{chunk-T6CNO72J.mjs → chunk-QM3VPX4Z.mjs} +2 -2
  38. package/dist/chunk-QY7GC2MW.mjs +33 -0
  39. package/dist/{chunk-BJ2YKRNX.mjs → chunk-R36MIGBU.mjs} +86 -54
  40. package/dist/{chunk-RVM7AODE.mjs → chunk-RDJKYKMQ.mjs} +2 -2
  41. package/dist/{chunk-UXQE2ROB.mjs → chunk-RUIXODYU.mjs} +3 -2
  42. package/dist/{chunk-5ER245FI.mjs → chunk-SBCEHIPL.mjs} +3 -3
  43. package/dist/{chunk-TPJU7NDJ.mjs → chunk-SYWGORQB.mjs} +1 -0
  44. package/dist/{chunk-UKSUENHK.mjs → chunk-T2QN2FA4.mjs} +40 -26
  45. package/dist/{chunk-CO4MRKST.mjs → chunk-WLOF6HSP.mjs} +2 -2
  46. package/dist/{chunk-MKWUXWQH.mjs → chunk-ZHS44YKF.mjs} +1 -1
  47. package/dist/{client-Cc42b42j.d.ts → client-DKwterMb.d.ts} +2 -2
  48. package/dist/client.d.ts +6 -6
  49. package/dist/client.mjs +1 -0
  50. package/dist/{config-DyRowvVP.d.ts → config-6YBtxekG.d.ts} +1 -1
  51. package/dist/{config-BYne-yCI.d.ts → config-DPphZfuM.d.ts} +1 -1
  52. package/dist/direct-executor-5KJGVQHM.mjs +22 -0
  53. package/dist/distribution/coordinator/etcd.mjs +1 -1
  54. package/dist/distribution/transport/grpc.mjs +1 -1
  55. package/dist/distribution.d.ts +5 -5
  56. package/dist/distribution.mjs +26 -26
  57. package/dist/{durability-wiring-6AN4QTT3.mjs → durability-wiring-FRPV6JLZ.mjs} +518 -91
  58. package/dist/embeddings/openai.mjs +1 -1
  59. package/dist/{engine-96_kLtSY.d.ts → engine-BGt9lI1V.d.ts} +11 -11
  60. package/dist/index.browser.mjs +2 -2
  61. package/dist/index.d.ts +5 -5
  62. package/dist/index.mjs +18 -18
  63. package/dist/{memory-dd3GESdc.d.ts → memory-COdoGthn.d.ts} +96 -76
  64. package/dist/{module-WIDYIJLY.mjs → module-IUTNXWMV.mjs} +72 -38
  65. package/dist/partial-words.mjs +5 -5
  66. package/dist/pattern.mjs +3 -3
  67. package/dist/persistence/durability/checkpoint-worker.mjs +21 -21
  68. package/dist/react.d.ts +6 -6
  69. package/dist/react.mjs +1 -0
  70. package/dist/release-BH3DKW3T.mjs +33 -0
  71. package/dist/replace-CEO5KJJ3.mjs +57 -0
  72. package/dist/restore-queue-I5SI6AUQ.mjs +35 -0
  73. package/dist/server.d.ts +8 -8
  74. package/dist/server.mjs +10 -10
  75. package/dist/snapshot-YC6TS4OP.mjs +738 -0
  76. package/dist/synonyms.mjs +3 -3
  77. package/dist/{tasks-DA1Ff0G4.d.ts → tasks-BJmamiC4.d.ts} +1 -1
  78. package/dist/{thread-VTDLLYXC.mjs → thread-U7XYJWGG.mjs} +15 -12
  79. package/dist/vector/search-worker.mjs +6 -6
  80. package/dist/workers/entry.mjs +5 -5
  81. package/dist/{write-OJR3H4BI.mjs → write-M2OALSA4.mjs} +20 -20
  82. package/package.json +7 -7
  83. package/dist/chunk-Y2F66IQJ.mjs +0 -74
  84. package/dist/chunk-ZJ4CSAB2.mjs +0 -13
  85. package/dist/direct-executor-CHXVCBN7.mjs +0 -22
  86. package/dist/snapshot-JFZVOGZV.mjs +0 -479
@@ -1,81 +1,456 @@
1
- import { readMetadataEnvelope, readSegmentManifest, reclaimUnlistedKeys, loadSegmentedCheckpoint, writeMetadataEnvelope, readCurrentManifest, deserializePayloadV2, unpackEnvelopeBytes, growPartitionsTo, documentsOfPayloadV2 } from './chunk-T4DB5YFB.mjs';
2
- import { decodeSegmentManifest, metadataFromConfig, createSeqOwner, isRecord, invalidManifest, SINGLE_NODE_PRIMARY_TERM, writeLiveVectors, isId, isUint32, isSequenceNumber, VECTOR_FILE_ENVELOPE_VERSION, metadataConfigOf } from './chunk-EONIHG3V.mjs';
3
- import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-YCJLVNWR.mjs';
1
+ import { readMetadataEnvelope, readSegmentManifest, reclaimUnlistedKeys, loadSegmentedCheckpoint, writeMetadataEnvelope, readCurrentManifest, deserializePayloadV2, unpackEnvelopeBytes, growPartitionsTo, documentsOfPayloadV2 } from './chunk-54HYC5FT.mjs';
2
+ import { decodeSegmentManifest, metadataFromConfig, createSeqOwner, isRecord, invalidManifest, SINGLE_NODE_PRIMARY_TERM, writeLiveVectors, isId, isUint32, isSequenceNumber, VECTOR_FILE_ENVELOPE_VERSION, metadataConfigOf } from './chunk-MASE2LTJ.mjs';
3
+ import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-F52QGNZ2.mjs';
4
4
  import './chunk-37CD3OKS.mjs';
5
- import { createDurableDirectory } from './chunk-67Z2V6TC.mjs';
5
+ import { createDurableDirectory, getFs } from './chunk-7QONURHW.mjs';
6
6
  import { spawnNodeWorker } from './chunk-C7SLA6CM.mjs';
7
- import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, readListedFile, adapterCheckpointStore, snapshotBundleKey, manifestReferencedKeys, checkpointLastSeqNo, commitCheckpointManifest, removeCheckpointGarbage, readSegmentFile, checkpointPrimaryTerm, writeCheckpointSegments, partitionContentsOf, frameRecord, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, readCommitMarker, readWalSegments, durableRecordMissing, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, MAX_SEGMENTS_PER_PARTITION, writeSegmentHeader, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, SEGMENT_HEADER_SIZE, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, LOG_RECORDS_PER_CHECKPOINT, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, buildEntry } from './chunk-3TTYK5A7.mjs';
8
- import { applyDeleteEntry, applyIndexEntry } from './chunk-3RDS4UHC.mjs';
7
+ import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, readListedFile, adapterCheckpointStore, snapshotBundleKey, manifestReferencedKeys, checkpointLastSeqNo, manifestFileRefs, commitCheckpointManifest, removeCheckpointGarbage, readSegmentFile, checkpointPrimaryTerm, writeCheckpointSegments, partitionContentsOf, frameRecord, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, readCommitMarker, readWalSegments, durableRecordMissing, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, MAX_SEGMENTS_PER_PARTITION, writeSegmentHeader, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, SEGMENT_HEADER_SIZE, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, LOG_RECORDS_PER_CHECKPOINT, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, buildEntry } from './chunk-Q4UTU5ZE.mjs';
8
+ import { applyDeleteEntry, applyIndexEntry } from './chunk-J7SDAOTP.mjs';
9
9
  import './chunk-OHP7QR33.mjs';
10
10
  import './chunk-P36MUFEJ.mjs';
11
- import { openChecksummedEnvelope, HEADER_SIZE } from './chunk-A4G7JSYO.mjs';
12
- import { resolveWorkerEntry, MAX_FILE_BYTES } from './chunk-RVM7AODE.mjs';
13
- import { buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-XLJ2S2BH.mjs';
14
- import './chunk-BJ2YKRNX.mjs';
15
- import { fnv1a } from './chunk-6VZ3Q73N.mjs';
16
- import './chunk-CO4MRKST.mjs';
11
+ import { openChecksummedEnvelope, HEADER_SIZE } from './chunk-JFUE6F2Y.mjs';
12
+ import './chunk-R36MIGBU.mjs';
13
+ import { resolveWorkerEntry, MAX_FILE_BYTES } from './chunk-RDJKYKMQ.mjs';
14
+ import { buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-3LYY62M6.mjs';
15
+ import { fnv1a } from './chunk-CHOGQSHK.mjs';
16
+ import './chunk-WLOF6HSP.mjs';
17
17
  import { detectRuntime } from './chunk-AV4IJKD2.mjs';
18
- import './chunk-MKWUXWQH.mjs';
19
- import './chunk-Q5DC6WAB.mjs';
18
+ import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-Q5DC6WAB.mjs';
19
+ import './chunk-ZHS44YKF.mjs';
20
20
  import './chunk-TGP5M5R7.mjs';
21
21
  import './chunk-U2BKYEC7.mjs';
22
- import './chunk-Y2F66IQJ.mjs';
23
- import './chunk-ZJ4CSAB2.mjs';
24
- import './chunk-QZN3MIJ3.mjs';
25
- import './chunk-T6CNO72J.mjs';
22
+ import './chunk-KOKM4PQN.mjs';
23
+ import './chunk-QY7GC2MW.mjs';
24
+ import './chunk-ES33L3HO.mjs';
25
+ import './chunk-QM3VPX4Z.mjs';
26
26
  import './chunk-WQBCBKY3.mjs';
27
- import './chunk-5ER245FI.mjs';
28
- import { encodeDocument, requireValidFieldTypes } from './chunk-CVQWQVFX.mjs';
27
+ import './chunk-SBCEHIPL.mjs';
28
+ import { encodeDocument, requireValidFieldTypes } from './chunk-4ZJWRETC.mjs';
29
29
  import './chunk-FGMGUDHX.mjs';
30
30
  import { MIN_M } from './chunk-KEHK2O6P.mjs';
31
31
  import './chunk-GES7U2PM.mjs';
32
- import './chunk-BVOJIYRV.mjs';
33
- import './chunk-NVL4ZR3K.mjs';
32
+ import './chunk-BB7MASRQ.mjs';
33
+ import './chunk-2GZGBOQI.mjs';
34
34
  import './chunk-X5CV2E2B.mjs';
35
35
  import './chunk-7X5JST5L.mjs';
36
- import './chunk-ZIJ63TBT.mjs';
36
+ import './chunk-7KC56574.mjs';
37
37
  import './chunk-R6A773WK.mjs';
38
38
  import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
39
39
  import './chunk-R6PR4WYZ.mjs';
40
- import './chunk-R7OSDAFK.mjs';
40
+ import './chunk-HBVLF3HA.mjs';
41
41
  import './chunk-L5ORED7V.mjs';
42
- import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
42
+ import { NarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
43
43
  import { decode } from '@msgpack/msgpack';
44
44
 
45
- // src/persistence/durability/archive-checkpoint.ts
45
+ // src/persistence/durability/sha256.ts
46
+ var ROUND_CONSTANTS = new Uint32Array([
47
+ 1116352408,
48
+ 1899447441,
49
+ 3049323471,
50
+ 3921009573,
51
+ 961987163,
52
+ 1508970993,
53
+ 2453635748,
54
+ 2870763221,
55
+ 3624381080,
56
+ 310598401,
57
+ 607225278,
58
+ 1426881987,
59
+ 1925078388,
60
+ 2162078206,
61
+ 2614888103,
62
+ 3248222580,
63
+ 3835390401,
64
+ 4022224774,
65
+ 264347078,
66
+ 604807628,
67
+ 770255983,
68
+ 1249150122,
69
+ 1555081692,
70
+ 1996064986,
71
+ 2554220882,
72
+ 2821834349,
73
+ 2952996808,
74
+ 3210313671,
75
+ 3336571891,
76
+ 3584528711,
77
+ 113926993,
78
+ 338241895,
79
+ 666307205,
80
+ 773529912,
81
+ 1294757372,
82
+ 1396182291,
83
+ 1695183700,
84
+ 1986661051,
85
+ 2177026350,
86
+ 2456956037,
87
+ 2730485921,
88
+ 2820302411,
89
+ 3259730800,
90
+ 3345764771,
91
+ 3516065817,
92
+ 3600352804,
93
+ 4094571909,
94
+ 275423344,
95
+ 430227734,
96
+ 506948616,
97
+ 659060556,
98
+ 883997877,
99
+ 958139571,
100
+ 1322822218,
101
+ 1537002063,
102
+ 1747873779,
103
+ 1955562222,
104
+ 2024104815,
105
+ 2227730452,
106
+ 2361852424,
107
+ 2428436474,
108
+ 2756734187,
109
+ 3204031479,
110
+ 3329325298
111
+ ]);
112
+ var INITIAL_STATE = [1779033703, 3144134277, 1013904242, 2773480762, 1359893119, 2600822924, 528734635, 1541459225];
113
+ var BLOCK_BYTES = 64;
114
+ var LENGTH_BYTES = 8;
115
+ function rotateRight(value, bits) {
116
+ return value >>> bits | value << 32 - bits;
117
+ }
118
+ function padded(bytes) {
119
+ const total = Math.ceil((bytes.byteLength + 1 + LENGTH_BYTES) / BLOCK_BYTES) * BLOCK_BYTES;
120
+ const message = new Uint8Array(total);
121
+ message.set(bytes);
122
+ message[bytes.byteLength] = 128;
123
+ const view = new DataView(message.buffer);
124
+ const bitLength = bytes.byteLength * 8;
125
+ view.setUint32(total - LENGTH_BYTES, Math.floor(bitLength / 4294967296), false);
126
+ view.setUint32(total - 4, bitLength >>> 0, false);
127
+ return view;
128
+ }
129
+ function sha256(bytes) {
130
+ const view = padded(bytes);
131
+ const state = INITIAL_STATE.slice();
132
+ const words = new Uint32Array(64);
133
+ for (let block = 0; block < view.byteLength; block += BLOCK_BYTES) {
134
+ for (let i = 0; i < 16; i++) words[i] = view.getUint32(block + i * 4, false);
135
+ for (let i = 16; i < 64; i++) {
136
+ const w15 = words[i - 15];
137
+ const w2 = words[i - 2];
138
+ const s0 = rotateRight(w15, 7) ^ rotateRight(w15, 18) ^ w15 >>> 3;
139
+ const s1 = rotateRight(w2, 17) ^ rotateRight(w2, 19) ^ w2 >>> 10;
140
+ words[i] = words[i - 16] + s0 + words[i - 7] + s1 >>> 0;
141
+ }
142
+ let [a, b, c, d, e, f, g, h] = state;
143
+ for (let i = 0; i < 64; i++) {
144
+ const s1 = rotateRight(e, 6) ^ rotateRight(e, 11) ^ rotateRight(e, 25);
145
+ const choice = e & f ^ ~e & g;
146
+ const t1 = h + s1 + choice + ROUND_CONSTANTS[i] + words[i] >>> 0;
147
+ const s0 = rotateRight(a, 2) ^ rotateRight(a, 13) ^ rotateRight(a, 22);
148
+ const majority = a & b ^ a & c ^ b & c;
149
+ const t2 = s0 + majority >>> 0;
150
+ h = g;
151
+ g = f;
152
+ f = e;
153
+ e = d + t1 >>> 0;
154
+ d = c;
155
+ c = b;
156
+ b = a;
157
+ a = t1 + t2 >>> 0;
158
+ }
159
+ state[0] = state[0] + a >>> 0;
160
+ state[1] = state[1] + b >>> 0;
161
+ state[2] = state[2] + c >>> 0;
162
+ state[3] = state[3] + d >>> 0;
163
+ state[4] = state[4] + e >>> 0;
164
+ state[5] = state[5] + f >>> 0;
165
+ state[6] = state[6] + g >>> 0;
166
+ state[7] = state[7] + h >>> 0;
167
+ }
168
+ const digest = new Uint8Array(32);
169
+ const out = new DataView(digest.buffer);
170
+ for (let i = 0; i < 8; i++) out.setUint32(i * 4, state[i], false);
171
+ return digest;
172
+ }
173
+
174
+ // src/persistence/durability/restore-journal.ts
175
+ var RESTORE_JOURNAL_PREFIX = ".swap-";
46
176
  var RESTORE_STAGING_PREFIX = ".restore-";
47
- function createRestoreStaging(directory, indexStore, indexName) {
48
- const stagingRoot = `${RESTORE_STAGING_PREFIX}${indexName}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}/`;
177
+ var RETIRED_INDEX_PREFIX = ".retired-";
178
+ var LOCK_FILE_KEY = ".narsil.lock";
179
+ var STAGING_PREFIX_PATTERN = /^\.[A-Za-z0-9][A-Za-z0-9._-]*\/$/;
180
+ function digestOf(indexName) {
181
+ let hex = "";
182
+ for (const byte of sha256(new TextEncoder().encode(indexName))) hex += byte.toString(16).padStart(2, "0");
183
+ return hex;
184
+ }
185
+ function journalKey(indexName) {
186
+ return `${RESTORE_JOURNAL_PREFIX}${digestOf(indexName)}`;
187
+ }
188
+ function randomSuffix() {
189
+ return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
190
+ }
191
+ function newStagingPrefix() {
192
+ return `${RESTORE_STAGING_PREFIX}${randomSuffix()}/`;
193
+ }
194
+ function newRetiredPrefix() {
195
+ return `${RETIRED_INDEX_PREFIX}${randomSuffix()}/`;
196
+ }
197
+ function encodeJournal(indexName, stagingPrefix) {
198
+ return new TextEncoder().encode(`${indexName}
199
+ ${stagingPrefix}`);
200
+ }
201
+ function isStagingPrefix(prefix) {
202
+ return STAGING_PREFIX_PATTERN.test(prefix) && !prefix.includes("..");
203
+ }
204
+ function corruptJournal(key, reason) {
205
+ throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The restore journal "${key}" ${reason}`, { key });
206
+ }
207
+ function readJournal(key, bytes) {
208
+ const text = new TextDecoder().decode(bytes);
209
+ const lineEnd = text.indexOf("\n");
210
+ const indexName = lineEnd === -1 ? text : text.slice(0, lineEnd);
211
+ if (key.slice(RESTORE_JOURNAL_PREFIX.length) !== digestOf(indexName)) return { kind: "leftover" };
212
+ if (indexName.length > MAX_INDEX_NAME_LENGTH || !INDEX_NAME_PATTERN.test(indexName) || indexName.includes("..")) {
213
+ corruptJournal(key, "names an index that no index name can match");
214
+ }
215
+ const stagingPrefix = lineEnd === -1 ? "" : text.slice(lineEnd + 1);
216
+ if (!isStagingPrefix(stagingPrefix) || stagingPrefix === `${LOCK_FILE_KEY}/`) {
217
+ corruptJournal(key, "names no staging prefix that a restore writes");
218
+ }
219
+ return { kind: "journal", indexName, stagingPrefix };
220
+ }
221
+ async function listedKeys(storage, indexName, stagingPrefix) {
222
+ const manifestBytes = await storage.read(`${stagingPrefix}manifest`) ?? await storage.read(manifestKey(indexName));
223
+ if (manifestBytes === null) {
224
+ throw new NarsilError(
225
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
226
+ `The restore of index "${indexName}" lost its manifest, both from staging and from the index`,
227
+ { indexName, stagingPrefix }
228
+ );
229
+ }
230
+ const decoded = await decodeSegmentManifest(manifestBytes, indexName);
231
+ if (decoded.kind !== "current") {
232
+ throw new NarsilError(
233
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
234
+ `The restore of index "${indexName}" staged a manifest in the layout of an earlier release`,
235
+ { indexName }
236
+ );
237
+ }
238
+ const root = `${indexName}/`;
239
+ const keys = /* @__PURE__ */ new Set(["meta", "manifest"]);
240
+ for (const ref of manifestFileRefs(decoded.manifest)) keys.add(ref.key.slice(root.length));
241
+ return keys;
242
+ }
243
+ function withoutTrailingSlash(prefix) {
244
+ return prefix.slice(0, -1);
245
+ }
246
+ async function finishJournal(storage, indexName, stagingPrefix) {
247
+ const root = `${indexName}/`;
248
+ const listed = await listedKeys(storage, indexName, stagingPrefix);
249
+ const staged = await storage.list(stagingPrefix);
250
+ let retiredPrefix = null;
251
+ if (storage.moveTree !== void 0 && staged.length > 0) {
252
+ retiredPrefix = newRetiredPrefix();
253
+ await storage.moveTree(indexName, withoutTrailingSlash(retiredPrefix));
254
+ await storage.moveTree(withoutTrailingSlash(stagingPrefix), indexName);
255
+ } else {
256
+ const stagedKeys = new Set(staged);
257
+ for (const key of listed) {
258
+ if (stagedKeys.has(`${stagingPrefix}${key}`)) await storage.moveKey(`${stagingPrefix}${key}`, `${root}${key}`);
259
+ }
260
+ }
261
+ for (const key of await storage.list(root)) {
262
+ if (!listed.has(key.slice(root.length))) await storage.remove(key);
263
+ }
264
+ await storage.remove(journalKey(indexName));
265
+ return retiredPrefix;
266
+ }
267
+ async function removePrefix(storage, prefix) {
268
+ if (storage.removeTree !== void 0) {
269
+ await storage.removeTree(prefix);
270
+ return;
271
+ }
272
+ for (const key of await storage.list(prefix)) await storage.remove(key);
273
+ }
274
+ async function commitJournal(storage, indexName, stagingPrefix) {
275
+ await storage.writeJournal(journalKey(indexName), encodeJournal(indexName, stagingPrefix));
276
+ }
277
+ function isTopLevelJournal(key) {
278
+ return key.startsWith(RESTORE_JOURNAL_PREFIX) && !key.includes("/");
279
+ }
280
+ async function removeStagingPrefixes(storage) {
281
+ const prefixes = /* @__PURE__ */ new Set();
282
+ for (const key of await storage.list(".")) {
283
+ const slash = key.indexOf("/");
284
+ if (slash !== -1 && isStagingPrefix(key.slice(0, slash + 1))) prefixes.add(key.slice(0, slash + 1));
285
+ }
286
+ for (const prefix of prefixes) await removePrefix(storage, prefix);
287
+ }
288
+ async function recoverRestoreJournals(storage) {
289
+ for (const key of await storage.list(RESTORE_JOURNAL_PREFIX)) {
290
+ if (!isTopLevelJournal(key)) continue;
291
+ const bytes = await storage.read(key);
292
+ if (bytes === null) continue;
293
+ const reading = readJournal(key, bytes);
294
+ if (reading.kind === "leftover") {
295
+ await storage.remove(key);
296
+ continue;
297
+ }
298
+ await finishJournal(storage, reading.indexName, reading.stagingPrefix);
299
+ }
300
+ await removeStagingPrefixes(storage);
301
+ }
302
+
303
+ // src/persistence/durability/restore-directory.ts
304
+ function missing(error) {
305
+ return error.code === "ENOENT";
306
+ }
307
+ function storageFailed(code, message, key, error) {
308
+ return new NarsilError(code, message, { key, cause: error instanceof Error ? error.message : String(error) });
309
+ }
310
+ function directoryRestoreStorage(directory) {
311
+ return {
312
+ read: (key) => directory.read(key),
313
+ write: (key, chunks) => directory.atomicWrite(key, chunks),
314
+ writeJournal: (key, bytes) => directory.atomicWrite(key, bytes),
315
+ remove: (key) => directory.remove(key),
316
+ list: (prefix) => directory.list(prefix),
317
+ moveKey: (from, to) => directory.move(from, to),
318
+ pathOf: (key) => directory.pathOf(key),
319
+ async moveTree(from, to) {
320
+ const source = await directory.pathOf(from);
321
+ const target = await directory.pathOf(to);
322
+ const fs = await getFs();
323
+ try {
324
+ await fs.rename(source, target);
325
+ } catch (error) {
326
+ if (missing(error)) return;
327
+ throw storageFailed(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Failed to move "${from}" to "${to}"`, from, error);
328
+ }
329
+ await directory.syncDirectoryOf(to);
330
+ },
331
+ async removeTree(prefix) {
332
+ const key = prefix.endsWith("/") ? prefix.slice(0, -1) : prefix;
333
+ const fs = await getFs();
334
+ try {
335
+ await fs.rm(await directory.pathOf(key), { recursive: true, force: true });
336
+ } catch (error) {
337
+ throw storageFailed(ErrorCodes.PERSISTENCE_DELETE_FAILED, `Failed to delete "${key}"`, key, error);
338
+ }
339
+ await directory.syncDirectoryOf(key);
340
+ }
341
+ };
342
+ }
343
+ function recoverDirectoryRestores(directory) {
344
+ return recoverRestoreJournals(directoryRestoreStorage(directory));
345
+ }
346
+
347
+ // src/persistence/durability/restore-staging.ts
348
+ function toError(error) {
349
+ return error instanceof Error ? error : new Error(String(error));
350
+ }
351
+ function pathUnder(path, root) {
352
+ if (!path.startsWith(root)) return false;
353
+ const next = path.charAt(root.length);
354
+ return next === "" || next === "/" || next === "\\";
355
+ }
356
+ function createRestoreStaging(storage, indexName, options = {}) {
357
+ const stagingPrefix = newStagingPrefix();
49
358
  const indexRoot = `${indexName}/`;
359
+ let committed = false;
50
360
  const staged = (key) => {
51
361
  if (!key.startsWith(indexRoot)) {
52
362
  throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `A restore of "${indexName}" cannot stage "${key}"`, {
53
363
  key
54
364
  });
55
365
  }
56
- return `${stagingRoot}${key.slice(indexRoot.length)}`;
366
+ return `${stagingPrefix}${key.slice(indexRoot.length)}`;
57
367
  };
58
- const unstaged = (key) => `${indexRoot}${key.slice(stagingRoot.length)}`;
59
- const stagingStore = {
60
- read: (key) => directory.read(staged(key)),
61
- write: (key, chunks) => directory.atomicWrite(staged(key), chunks),
62
- remove: (key) => directory.remove(staged(key)),
63
- list: async (prefix) => (await directory.list(staged(prefix))).map(unstaged),
64
- pathOf: (key) => directory.pathOf(staged(key))
368
+ const unstaged = (key) => `${indexRoot}${key.slice(stagingPrefix.length)}`;
369
+ const store = {
370
+ read: (key) => storage.read(staged(key)),
371
+ write: (key, chunks) => storage.write(staged(key), chunks),
372
+ remove: (key) => storage.remove(staged(key)),
373
+ list: async (prefix) => (await storage.list(staged(prefix))).map(unstaged)
65
374
  };
375
+ const pathOf = storage.pathOf;
376
+ if (pathOf !== void 0) store.pathOf = (key) => pathOf.call(storage, staged(key));
377
+ let retiredPrefix = null;
378
+ async function finishOnStorage() {
379
+ retiredPrefix = await finishJournal(storage, indexName, stagingPrefix);
380
+ }
66
381
  return {
67
- store: stagingStore,
382
+ store,
383
+ async relocation() {
384
+ if (pathOf === void 0 || (await storage.list(stagingPrefix)).length > 0) return null;
385
+ const from = await pathOf.call(storage, stagingPrefix.slice(0, -1));
386
+ const to = await pathOf.call(storage, indexName);
387
+ return (path) => pathUnder(path, from) ? `${to}${path.slice(from.length)}` : path;
388
+ },
68
389
  async commit() {
69
- for (const key of await directory.list(stagingRoot)) await directory.move(key, unstaged(key));
70
- return indexStore;
390
+ try {
391
+ await commitJournal(storage, indexName, stagingPrefix);
392
+ } catch (error) {
393
+ try {
394
+ await storage.remove(journalKey(indexName));
395
+ } catch (removal) {
396
+ committed = true;
397
+ options.onStorageLost?.(toError(removal));
398
+ }
399
+ throw error;
400
+ }
401
+ committed = true;
402
+ },
403
+ async finish() {
404
+ try {
405
+ await (options.finish === void 0 ? finishOnStorage() : options.finish(finishOnStorage));
406
+ } catch (error) {
407
+ options.onStorageLost?.(toError(error));
408
+ }
409
+ },
410
+ async removeRetired() {
411
+ const prefix = retiredPrefix;
412
+ retiredPrefix = null;
413
+ if (prefix !== null) await removePrefix(storage, prefix);
71
414
  },
72
415
  async discard() {
73
- for (const key of await directory.list(stagingRoot)) await directory.remove(key);
416
+ if (committed) return;
417
+ await removePrefix(storage, stagingPrefix);
74
418
  }
75
419
  };
76
420
  }
77
- async function removeAbandonedRestores(directory) {
78
- for (const key of await directory.list(RESTORE_STAGING_PREFIX)) await directory.remove(key);
421
+ function adapterRestoreStorage(adapter) {
422
+ const store = adapterCheckpointStore(adapter);
423
+ return {
424
+ read: (key) => store.read(key),
425
+ write: (key, chunks) => store.write(key, chunks),
426
+ writeJournal: (key, bytes) => store.write(key, [bytes]),
427
+ remove: (key) => store.remove(key),
428
+ list: (prefix) => store.list(prefix),
429
+ async moveKey(from, to) {
430
+ const bytes = await store.read(from);
431
+ if (bytes === null) return;
432
+ await store.write(to, [bytes]);
433
+ await store.remove(from);
434
+ }
435
+ };
436
+ }
437
+
438
+ // src/persistence/durability/archive-checkpoint.ts
439
+ function toError2(error) {
440
+ return error instanceof Error ? error : new Error(String(error));
441
+ }
442
+ function stageDirectoryRestore(deps, indexName) {
443
+ return createRestoreStaging(directoryRestoreStorage(deps.directory), indexName, {
444
+ async finish(run) {
445
+ try {
446
+ await run();
447
+ } catch (error) {
448
+ deps.markFatal(toError2(error));
449
+ throw error;
450
+ }
451
+ },
452
+ onStorageLost: deps.markFatal
453
+ });
79
454
  }
80
455
  function releaser(held, indexName) {
81
456
  let released = false;
@@ -126,7 +501,7 @@ function createArchiveCheckpointAccess(deps) {
126
501
  }
127
502
  },
128
503
  stageRestore(indexName) {
129
- return createRestoreStaging(deps.directory, store, indexName);
504
+ return stageDirectoryRestore(deps, indexName);
130
505
  },
131
506
  adoptRestoredCheckpoint(indexName, checkpoint) {
132
507
  for (const { partitionId, lastSeqNo } of checkpoint) deps.registerPartition(indexName, partitionId, lastSeqNo);
@@ -384,8 +759,8 @@ async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartit
384
759
  const vectorPlans = /* @__PURE__ */ new Map();
385
760
  for (const [fieldPath, vectorIndex] of vectorIndexes) {
386
761
  const listed = listedVectorFields.find((ref) => ref.fieldPath === fieldPath);
387
- const listedKeys = listed === void 0 ? null : listed.files.map((file) => file.key);
388
- vectorPlans.set(fieldPath, vectorIndex.planCheckpoint(listedKeys));
762
+ const listedKeys2 = listed === void 0 ? null : listed.files.map((file) => file.key);
763
+ vectorPlans.set(fieldPath, vectorIndex.planCheckpoint(listedKeys2));
389
764
  }
390
765
  const memoryMatchesTheTargets = recordsLeftInLog === 0;
391
766
  const wholePartitions = memoryMatchesTheTargets ? wholePartitionsOf(targets) : noWholePartitions();
@@ -879,7 +1254,7 @@ async function drainIndexStateForUnload(indexState, pendingMetadataWrite, closeW
879
1254
  }
880
1255
 
881
1256
  // src/persistence/durability/mutation-batch.ts
882
- function toError(err) {
1257
+ function toError3(err) {
883
1258
  return err instanceof Error ? err : new Error(String(err));
884
1259
  }
885
1260
  function failUnlessAlreadyFailed(outcomes, count, error) {
@@ -891,7 +1266,7 @@ function failUnlessAlreadyFailed(outcomes, count, error) {
891
1266
  }
892
1267
  }
893
1268
  function failPartition(partition, deps, err) {
894
- const error = toError(err);
1269
+ const error = toError3(err);
895
1270
  partition.failed = error;
896
1271
  deps.markFatal(error);
897
1272
  return error;
@@ -971,7 +1346,7 @@ async function recordPartitionBatch(partition, records, deps) {
971
1346
  }
972
1347
  }
973
1348
  } catch (err) {
974
- failUnlessAlreadyFailed(outcomes, records.length, toError(err));
1349
+ failUnlessAlreadyFailed(outcomes, records.length, toError3(err));
975
1350
  }
976
1351
  return outcomes;
977
1352
  }
@@ -1808,8 +2183,8 @@ async function createMarkerWriter(directory, indexName, partitionId) {
1808
2183
  // src/persistence/durability/wal-writer.ts
1809
2184
  var SEGMENT_TAIL_PATTERN = /^\d{16}$/;
1810
2185
  function segmentKey(indexName, partitionId, startSeqNo) {
1811
- const padded = startSeqNo.toString().padStart(16, "0");
1812
- return `${indexName}/wal/${partitionId}/${padded}`;
2186
+ const padded2 = startSeqNo.toString().padStart(16, "0");
2187
+ return `${indexName}/wal/${partitionId}/${padded2}`;
1813
2188
  }
1814
2189
  function parseSegmentStartSeqNo2(key) {
1815
2190
  const tail = key.slice(key.lastIndexOf("/") + 1);
@@ -2055,14 +2430,14 @@ function createDurabilityManager(config, hooks, directoryOverride) {
2055
2430
  stopTimers();
2056
2431
  hooks.onFatalError(error);
2057
2432
  }
2058
- function toError2(err) {
2433
+ function toError5(err) {
2059
2434
  return err instanceof Error ? err : new Error(String(err));
2060
2435
  }
2061
2436
  async function closeWriterReportingFailure(closePromise) {
2062
2437
  try {
2063
2438
  await closePromise;
2064
2439
  } catch (err) {
2065
- markFatal(toError2(err));
2440
+ markFatal(toError5(err));
2066
2441
  }
2067
2442
  }
2068
2443
  function getOrCreateIndexState(indexName) {
@@ -2128,7 +2503,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
2128
2503
  try {
2129
2504
  await partition.walWriter.commit();
2130
2505
  } catch (err) {
2131
- partition.failed = toError2(err);
2506
+ partition.failed = toError5(err);
2132
2507
  markFatal(partition.failed);
2133
2508
  }
2134
2509
  }
@@ -2142,7 +2517,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
2142
2517
  try {
2143
2518
  await checkpointIndex(indexName);
2144
2519
  } catch (err) {
2145
- markFatal(toError2(err));
2520
+ markFatal(toError5(err));
2146
2521
  }
2147
2522
  }
2148
2523
  }
@@ -2223,7 +2598,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
2223
2598
  indexState.documentBytesSinceCheckpoint += recorded.documentBytes;
2224
2599
  if (checkpointIsDue(indexState, checkpointMutationThreshold)) {
2225
2600
  void checkpointIndex(indexName).catch((err) => {
2226
- markFatal(toError2(err));
2601
+ markFatal(toError5(err));
2227
2602
  });
2228
2603
  }
2229
2604
  await stallWhileCheckpointsFallBehind(indexState);
@@ -2255,12 +2630,10 @@ function createDurabilityManager(config, hooks, directoryOverride) {
2255
2630
  );
2256
2631
  }
2257
2632
  return {
2258
- isActive() {
2259
- return true;
2260
- },
2633
+ isActive: () => true,
2261
2634
  async recover(metadataOnly = false) {
2262
2635
  await directoryWatch.claim();
2263
- await removeAbandonedRestores(directory);
2636
+ await recoverDirectoryRestores(directory);
2264
2637
  const names = await listPersistedIndexes(directory);
2265
2638
  for (const indexName of names) {
2266
2639
  await recoverIndex(indexName, metadataOnly);
@@ -2330,7 +2703,8 @@ function createDurabilityManager(config, hooks, directoryOverride) {
2330
2703
  directory,
2331
2704
  held: heldCheckpointFiles,
2332
2705
  checkpoint: (name) => checkpointIndex(name),
2333
- registerPartition: (name, partitionId, startSeqNo) => getOrCreatePartition(name, partitionId, startSeqNo)
2706
+ registerPartition: (name, partitionId, startSeqNo) => getOrCreatePartition(name, partitionId, startSeqNo),
2707
+ markFatal
2334
2708
  }),
2335
2709
  async shutdown() {
2336
2710
  shuttingDown = true;
@@ -2350,7 +2724,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
2350
2724
  };
2351
2725
  }
2352
2726
 
2353
- // src/persistence/durability/snapshot-only-checkpoint.ts
2727
+ // src/persistence/durability/snapshot-only/checkpoint.ts
2354
2728
  function capture(input, prior) {
2355
2729
  const listedIds = new Set((prior?.partitions ?? []).map((partition) => partition.partitionId));
2356
2730
  const partitionCountChanged = prior !== null && listedIds.size !== input.manager.partitionCount;
@@ -2406,7 +2780,44 @@ async function writeSnapshotOnlyCheckpoint(input) {
2406
2780
  return captured.documentCount;
2407
2781
  }
2408
2782
 
2409
- // src/persistence/durability/snapshot-only.ts
2783
+ // src/persistence/durability/snapshot-only/restores.ts
2784
+ function toError4(error) {
2785
+ return error instanceof Error ? error : new Error(String(error));
2786
+ }
2787
+ function createPendingRestores(onFatalError) {
2788
+ const pending = /* @__PURE__ */ new Map();
2789
+ return {
2790
+ finishLater(indexName, run) {
2791
+ let running = null;
2792
+ const finish = () => {
2793
+ running ??= run().then(
2794
+ () => {
2795
+ if (pending.get(indexName) === finish) pending.delete(indexName);
2796
+ },
2797
+ (error) => {
2798
+ running = null;
2799
+ throw error;
2800
+ }
2801
+ );
2802
+ return running;
2803
+ };
2804
+ pending.set(indexName, finish);
2805
+ void finish().catch((error) => onFatalError(toError4(error)));
2806
+ return Promise.resolve();
2807
+ },
2808
+ async settle(indexName) {
2809
+ await pending.get(indexName)?.();
2810
+ },
2811
+ forget(indexName) {
2812
+ pending.delete(indexName);
2813
+ },
2814
+ async settleAll() {
2815
+ for (const finish of [...pending.values()]) await finish().catch(() => void 0);
2816
+ }
2817
+ };
2818
+ }
2819
+
2820
+ // src/persistence/durability/snapshot-only/index.ts
2410
2821
  function metadataKey(indexName) {
2411
2822
  return `${indexName}/meta`;
2412
2823
  }
@@ -2415,8 +2826,13 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
2415
2826
  const checkpointMutationThreshold = config.checkpointMutationThreshold ?? DEFAULT_CHECKPOINT_MUTATION_THRESHOLD;
2416
2827
  const indexes = /* @__PURE__ */ new Map();
2417
2828
  const metadataWrites = /* @__PURE__ */ new Map();
2829
+ const restoreStorage = adapterRestoreStorage(adapter);
2830
+ const restores = createPendingRestores(hooks.onFatalError);
2418
2831
  let checkpointTimer = null;
2419
2832
  let shuttingDown = false;
2833
+ function settleRestore(indexName) {
2834
+ return restores.settle(indexName);
2835
+ }
2420
2836
  function getOrCreateIndexState(indexName) {
2421
2837
  let state = indexes.get(indexName);
2422
2838
  if (state === void 0) {
@@ -2452,6 +2868,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
2452
2868
  }
2453
2869
  }
2454
2870
  async function performCheckpoint(indexName, indexState, fromMemory) {
2871
+ await settleRestore(indexName);
2455
2872
  const manager = hooks.getManager(indexName);
2456
2873
  const metadata = hooks.buildMetadata(indexName);
2457
2874
  if (manager === void 0 || metadata === void 0) {
@@ -2501,7 +2918,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
2501
2918
  }
2502
2919
  function queueMetadataWrite(indexName, write) {
2503
2920
  const previous = metadataWrites.get(indexName) ?? Promise.resolve();
2504
- const queued = previous.then(write);
2921
+ const queued = previous.then(() => settleRestore(indexName)).then(write);
2505
2922
  metadataWrites.set(
2506
2923
  indexName,
2507
2924
  queued.catch(() => void 0)
@@ -2509,6 +2926,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
2509
2926
  return queued;
2510
2927
  }
2511
2928
  async function recoverIndex(indexName, metadataOnly = false) {
2929
+ await settleRestore(indexName);
2512
2930
  const metaBytes = await adapter.load(metadataKey(indexName));
2513
2931
  if (metaBytes === null) {
2514
2932
  return;
@@ -2595,11 +3013,28 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
2595
3013
  }
2596
3014
  return appliedSeqNo;
2597
3015
  }
3016
+ async function drainIndex(indexName) {
3017
+ const indexState = indexes.get(indexName);
3018
+ if (indexState !== void 0) {
3019
+ indexState.unloading = true;
3020
+ try {
3021
+ await indexState.applyChain;
3022
+ if (indexState.checkpointInFlight !== null) await indexState.checkpointInFlight;
3023
+ await metadataWrites.get(indexName);
3024
+ } catch (error) {
3025
+ indexState.unloading = false;
3026
+ throw error;
3027
+ }
3028
+ }
3029
+ indexes.delete(indexName);
3030
+ metadataWrites.delete(indexName);
3031
+ }
2598
3032
  return {
2599
3033
  isActive() {
2600
3034
  return true;
2601
3035
  },
2602
3036
  async recover(metadataOnly = false) {
3037
+ await recoverRestoreJournals(restoreStorage);
2603
3038
  const keys = await adapter.list("");
2604
3039
  for (const key of keys) {
2605
3040
  if (key.endsWith("/meta")) {
@@ -2648,40 +3083,31 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
2648
3083
  await checkpointIndex(indexName);
2649
3084
  }
2650
3085
  },
2651
- async removeIndex(indexName) {
2652
- const indexState = indexes.get(indexName);
2653
- if (indexState !== void 0) {
2654
- indexState.unloading = true;
2655
- try {
2656
- await indexState.applyChain;
2657
- if (indexState.checkpointInFlight !== null) await indexState.checkpointInFlight;
2658
- await metadataWrites.get(indexName);
2659
- } catch (error) {
2660
- indexState.unloading = false;
2661
- throw error;
2662
- }
3086
+ stageRestore(indexName) {
3087
+ return createRestoreStaging(restoreStorage, indexName, {
3088
+ finish: (run) => restores.finishLater(indexName, run),
3089
+ onStorageLost: hooks.onFatalError
3090
+ });
3091
+ },
3092
+ adoptRestoredCheckpoint(indexName, checkpoint) {
3093
+ const indexState = getOrCreateIndexState(indexName);
3094
+ indexState.appliedSeqNoByPartition.clear();
3095
+ for (const { partitionId, lastSeqNo } of checkpoint) {
3096
+ if (lastSeqNo > 0) indexState.appliedSeqNoByPartition.set(partitionId, lastSeqNo);
2663
3097
  }
2664
- indexes.delete(indexName);
2665
- metadataWrites.delete(indexName);
3098
+ },
3099
+ async removeIndex(indexName) {
3100
+ await settleRestore(indexName).catch(() => void 0);
3101
+ restores.forget(indexName);
3102
+ await restoreStorage.remove(journalKey(indexName));
3103
+ await drainIndex(indexName);
2666
3104
  for (const key of await adapter.list(`${indexName}/`)) {
2667
3105
  await adapter.delete(key);
2668
3106
  }
2669
3107
  },
2670
3108
  async unloadIndex(indexName) {
2671
- const indexState = indexes.get(indexName);
2672
- if (indexState !== void 0) {
2673
- indexState.unloading = true;
2674
- try {
2675
- await indexState.applyChain;
2676
- if (indexState.checkpointInFlight !== null) await indexState.checkpointInFlight;
2677
- await metadataWrites.get(indexName);
2678
- } catch (error) {
2679
- indexState.unloading = false;
2680
- throw error;
2681
- }
2682
- }
2683
- indexes.delete(indexName);
2684
- metadataWrites.delete(indexName);
3109
+ await settleRestore(indexName);
3110
+ await drainIndex(indexName);
2685
3111
  },
2686
3112
  async reloadIndex(indexName) {
2687
3113
  const indexState = getOrCreateIndexState(indexName);
@@ -2695,6 +3121,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
2695
3121
  clearInterval(checkpointTimer);
2696
3122
  checkpointTimer = null;
2697
3123
  }
3124
+ await restores.settleAll();
2698
3125
  for (const indexState of indexes.values()) {
2699
3126
  if (indexState.checkpointInFlight !== null) {
2700
3127
  await indexState.checkpointInFlight.catch(() => void 0);