@delali/narsil 0.3.1-next.90 → 0.3.1-next.94

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/dist/adapters/filesystem.mjs +1 -1
  2. package/dist/adapters/indexeddb.mjs +1 -1
  3. package/dist/browser/{chunk-3GZQKGVQ.mjs → chunk-4JTBL63P.mjs} +13 -5
  4. package/dist/browser/{chunk-3ZRMZIH4.mjs → chunk-5S74HQRT.mjs} +208 -78
  5. package/dist/browser/{chunk-FHFFWLEK.mjs → chunk-HARBSUWP.mjs} +3 -4
  6. package/dist/browser/{chunk-RAHUQOJL.mjs → chunk-HC6DMR4T.mjs} +2445 -2107
  7. package/dist/browser/{chunk-JACCR3BJ.mjs → chunk-PP6N5CCS.mjs} +2 -2
  8. package/dist/browser/{chunk-NQW2YXEX.mjs → chunk-WKPJ4E73.mjs} +1 -1
  9. package/dist/browser/{durability-wiring-VSUBLUDK.mjs → durability-wiring-7DKYHKT5.mjs} +498 -71
  10. package/dist/browser/{module-E25UDADM.mjs → module-O5WH32EL.mjs} +102 -54
  11. package/dist/browser/release-76MXRPGV.mjs +33 -0
  12. package/dist/browser/replace-RURXTGVG.mjs +57 -0
  13. package/dist/browser/restore-queue-Z7FZZNKH.mjs +35 -0
  14. package/dist/browser/{snapshot-5MXBEDXF.mjs → snapshot-PPTZ565G.mjs} +368 -103
  15. package/dist/browser/{write-6SBYTS62.mjs → write-2ZAA7UDH.mjs} +4 -4
  16. package/dist/{capabilities-SJPttcgQ.d.ts → capabilities-xsJfdFoQ.d.ts} +11 -3
  17. package/dist/chunk-2GZGBOQI.mjs +426 -0
  18. package/dist/{chunk-CVQWQVFX.mjs → chunk-4ZJWRETC.mjs} +2 -2
  19. package/dist/chunk-7KC56574.mjs +507 -0
  20. package/dist/{chunk-MXS5HWIH.mjs → chunk-7VALMHZF.mjs} +89 -17
  21. package/dist/chunk-7X5JST5L.mjs +114 -0
  22. package/dist/{chunk-UKSUENHK.mjs → chunk-AF2CIKYC.mjs} +40 -26
  23. package/dist/{chunk-BVOJIYRV.mjs → chunk-BB7MASRQ.mjs} +1 -1
  24. package/dist/{chunk-R6BMMQXH.mjs → chunk-C45FUYPI.mjs} +193 -79
  25. package/dist/{chunk-EMXAYPTQ.mjs → chunk-CHOGQSHK.mjs} +2326 -3503
  26. package/dist/{chunk-QZN3MIJ3.mjs → chunk-ES33L3HO.mjs} +2 -2
  27. package/dist/{chunk-VY4DXZ5P.mjs → chunk-EWBXO45L.mjs} +3 -3
  28. package/dist/{chunk-QAMUC2BJ.mjs → chunk-EX62BQT6.mjs} +10 -9
  29. package/dist/{chunk-MWJIAREY.mjs → chunk-EZD4MWOU.mjs} +4 -4
  30. package/dist/{chunk-YCJLVNWR.mjs → chunk-F52QGNZ2.mjs} +1 -1
  31. package/dist/{chunk-URGLSNKR.mjs → chunk-GJZHU4FC.mjs} +202 -16
  32. package/dist/{chunk-S4LQDSUL.mjs → chunk-J53IHLG7.mjs} +7 -7
  33. package/dist/{chunk-R7OSDAFK.mjs → chunk-K6A4FCVR.mjs} +1 -1
  34. package/dist/chunk-KOKM4PQN.mjs +52 -0
  35. package/dist/{chunk-GCM3JFVA.mjs → chunk-OHP7QR33.mjs} +2 -1
  36. package/dist/{chunk-T6CNO72J.mjs → chunk-QM3VPX4Z.mjs} +2 -2
  37. package/dist/{chunk-V7YUSKKZ.mjs → chunk-QWWCLRXX.mjs} +10 -6
  38. package/dist/chunk-QY7GC2MW.mjs +33 -0
  39. package/dist/{chunk-DCK4P5BI.mjs → chunk-R36MIGBU.mjs} +100 -59
  40. package/dist/{chunk-YRX6XMOQ.mjs → chunk-R6PR4WYZ.mjs} +1 -0
  41. package/dist/{chunk-RVM7AODE.mjs → chunk-RDJKYKMQ.mjs} +2 -2
  42. package/dist/{chunk-OOSIDHA3.mjs → chunk-RSRVGAIQ.mjs} +1 -1
  43. package/dist/{chunk-XSZVS4S3.mjs → chunk-RUIXODYU.mjs} +9 -4
  44. package/dist/{chunk-A3UB2MXL.mjs → chunk-SBCEHIPL.mjs} +4 -507
  45. package/dist/{chunk-TPJU7NDJ.mjs → chunk-SYWGORQB.mjs} +1 -0
  46. package/dist/chunk-UAUZGXJP.mjs +3591 -0
  47. package/dist/{chunk-CL4OM7GB.mjs → chunk-WBS3EYC4.mjs} +12 -8
  48. package/dist/{chunk-J2BPXTD7.mjs → chunk-WLOF6HSP.mjs} +3 -3
  49. package/dist/chunk-X5CV2E2B.mjs +1 -0
  50. package/dist/{chunk-ULRPNYAG.mjs → chunk-Z3MOCHG5.mjs} +120 -62
  51. package/dist/chunk-ZHS44YKF.mjs +35 -0
  52. package/dist/{client-BMK53IR2.d.ts → client-CgayPw2R.d.ts} +27 -3
  53. package/dist/client.d.ts +9 -8
  54. package/dist/client.mjs +16 -2
  55. package/dist/{config-UJx1dyX1.d.ts → config-3kBsQ0Pi.d.ts} +2 -2
  56. package/dist/{config-BUR-7eVQ.d.ts → config-B-RnO6dr.d.ts} +2 -2
  57. package/dist/direct-executor-IJTRU3YN.mjs +22 -0
  58. package/dist/distribution/coordinator/etcd.d.ts +3 -2
  59. package/dist/distribution/coordinator/etcd.mjs +5 -1
  60. package/dist/distribution/coordinator/in-memory.d.ts +3 -2
  61. package/dist/distribution/coordinator/in-memory.mjs +4 -0
  62. package/dist/distribution/transport/grpc.mjs +3 -3
  63. package/dist/distribution/transport/in-memory.mjs +1 -1
  64. package/dist/distribution/transport/tcp.mjs +2 -2
  65. package/dist/distribution.d.ts +43 -9
  66. package/dist/distribution.mjs +4620 -4293
  67. package/dist/{durability-wiring-ZLRQU6FG.mjs → durability-wiring-Y55SCPFC.mjs} +524 -93
  68. package/dist/embeddings/openai.mjs +1 -1
  69. package/dist/{engine-9X6W5naj.d.ts → engine-DSUfY2F0.d.ts} +53 -17
  70. package/dist/{index-vCMu8OeQ.d.ts → index-ONAEaqtx.d.ts} +8 -1
  71. package/dist/index.browser.mjs +2 -2
  72. package/dist/index.d.ts +44 -25
  73. package/dist/index.mjs +22 -18
  74. package/dist/{memory-CV9NAxLY.d.ts → memory-wuXcYOD1.d.ts} +99 -79
  75. package/dist/{module-WIDYIJLY.mjs → module-VITS4ZR5.mjs} +72 -38
  76. package/dist/partial-words.mjs +5 -4
  77. package/dist/pattern.mjs +3 -3
  78. package/dist/persistence/durability/checkpoint-worker.d.ts +60 -58
  79. package/dist/persistence/durability/checkpoint-worker.mjs +27 -23
  80. package/dist/react.d.ts +8 -7
  81. package/dist/react.mjs +1 -0
  82. package/dist/release-BH3DKW3T.mjs +33 -0
  83. package/dist/replace-CEO5KJJ3.mjs +57 -0
  84. package/dist/restore-queue-4SG45BLF.mjs +35 -0
  85. package/dist/{schema-DtsgmNfU.d.ts → schema-DttzM-ln.d.ts} +12 -0
  86. package/dist/server.d.ts +11 -10
  87. package/dist/server.mjs +38 -11
  88. package/dist/snapshot-XBKE7PUG.mjs +738 -0
  89. package/dist/synonyms-CpOAHPa-.d.ts +52 -0
  90. package/dist/synonyms.d.ts +22 -0
  91. package/dist/synonyms.mjs +12 -0
  92. package/dist/{tasks-BGaZJyoY.d.ts → tasks-Bj9WBlFG.d.ts} +4 -1
  93. package/dist/{thread-ZP3UMIZT.mjs → thread-U7XYJWGG.mjs} +17 -12
  94. package/dist/vector/search-worker.d.ts +2 -1
  95. package/dist/vector/search-worker.mjs +6 -6
  96. package/dist/workers/entry.mjs +6 -6
  97. package/dist/{write-PXDKPG2R.mjs → write-BMNV7P2H.mjs} +24 -20
  98. package/package.json +15 -7
  99. package/dist/chunk-PR4O7CAP.mjs +0 -18
  100. package/dist/chunk-X4SEXL4U.mjs +0 -2310
  101. package/dist/chunk-Y2F66IQJ.mjs +0 -74
  102. package/dist/chunk-ZJ4CSAB2.mjs +0 -13
  103. package/dist/direct-executor-EGESD26S.mjs +0 -18
  104. package/dist/snapshot-22CUEYYW.mjs +0 -469
@@ -1,77 +1,456 @@
1
- import { readMetadataEnvelope, readSegmentManifest, reclaimUnlistedKeys, loadSegmentedCheckpoint, writeMetadataEnvelope, readCurrentManifest, deserializePayloadV2, unpackEnvelopeBytes, growPartitionsTo, documentsOfPayloadV2 } from './chunk-V7YUSKKZ.mjs';
2
- import { decodeSegmentManifest, metadataFromConfig, createSeqOwner, isRecord, invalidManifest, SINGLE_NODE_PRIMARY_TERM, writeLiveVectors, isId, isUint32, isSequenceNumber, VECTOR_FILE_ENVELOPE_VERSION, metadataConfigOf } from './chunk-CL4OM7GB.mjs';
3
- import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-YCJLVNWR.mjs';
1
+ import { readMetadataEnvelope, readSegmentManifest, reclaimUnlistedKeys, loadSegmentedCheckpoint, writeMetadataEnvelope, readCurrentManifest, deserializePayloadV2, unpackEnvelopeBytes, growPartitionsTo, documentsOfPayloadV2 } from './chunk-QWWCLRXX.mjs';
2
+ import { decodeSegmentManifest, metadataFromConfig, createSeqOwner, isRecord, invalidManifest, SINGLE_NODE_PRIMARY_TERM, writeLiveVectors, isId, isUint32, isSequenceNumber, VECTOR_FILE_ENVELOPE_VERSION, metadataConfigOf } from './chunk-WBS3EYC4.mjs';
3
+ import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-F52QGNZ2.mjs';
4
4
  import './chunk-37CD3OKS.mjs';
5
- import { createDurableDirectory } from './chunk-VY4DXZ5P.mjs';
6
- import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, readListedFile, adapterCheckpointStore, snapshotBundleKey, manifestReferencedKeys, checkpointLastSeqNo, commitCheckpointManifest, removeCheckpointGarbage, readSegmentFile, checkpointPrimaryTerm, writeCheckpointSegments, partitionContentsOf, frameRecord, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, readCommitMarker, readWalSegments, durableRecordMissing, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, MAX_SEGMENTS_PER_PARTITION, writeSegmentHeader, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, SEGMENT_HEADER_SIZE, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, LOG_RECORDS_PER_CHECKPOINT, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, buildEntry } from './chunk-QAMUC2BJ.mjs';
7
- import { applyDeleteEntry, applyIndexEntry } from './chunk-MWJIAREY.mjs';
8
- import './chunk-GCM3JFVA.mjs';
9
- import './chunk-P36MUFEJ.mjs';
5
+ import { createDurableDirectory, getFs } from './chunk-EWBXO45L.mjs';
10
6
  import { spawnNodeWorker } from './chunk-C7SLA6CM.mjs';
11
- import { openChecksummedEnvelope, HEADER_SIZE } from './chunk-R6BMMQXH.mjs';
12
- import { buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-EMXAYPTQ.mjs';
13
- import './chunk-DCK4P5BI.mjs';
14
- import { fnv1a } from './chunk-X4SEXL4U.mjs';
15
- import './chunk-J2BPXTD7.mjs';
16
- import './chunk-PR4O7CAP.mjs';
17
- import { resolveWorkerEntry, MAX_FILE_BYTES } from './chunk-RVM7AODE.mjs';
7
+ import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, readListedFile, adapterCheckpointStore, snapshotBundleKey, manifestReferencedKeys, checkpointLastSeqNo, manifestFileRefs, commitCheckpointManifest, removeCheckpointGarbage, readSegmentFile, checkpointPrimaryTerm, writeCheckpointSegments, partitionContentsOf, frameRecord, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, readCommitMarker, readWalSegments, durableRecordMissing, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, MAX_SEGMENTS_PER_PARTITION, writeSegmentHeader, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, SEGMENT_HEADER_SIZE, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, LOG_RECORDS_PER_CHECKPOINT, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, buildEntry } from './chunk-EX62BQT6.mjs';
8
+ import { applyDeleteEntry, applyIndexEntry } from './chunk-EZD4MWOU.mjs';
9
+ import './chunk-OHP7QR33.mjs';
10
+ import './chunk-P36MUFEJ.mjs';
11
+ import { openChecksummedEnvelope, HEADER_SIZE } from './chunk-C45FUYPI.mjs';
12
+ import './chunk-R36MIGBU.mjs';
13
+ import { resolveWorkerEntry, MAX_FILE_BYTES } from './chunk-RDJKYKMQ.mjs';
14
+ import { buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-UAUZGXJP.mjs';
15
+ import { fnv1a } from './chunk-CHOGQSHK.mjs';
16
+ import './chunk-WLOF6HSP.mjs';
18
17
  import { detectRuntime } from './chunk-AV4IJKD2.mjs';
19
- import './chunk-Q5DC6WAB.mjs';
18
+ import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-Q5DC6WAB.mjs';
19
+ import './chunk-ZHS44YKF.mjs';
20
20
  import './chunk-TGP5M5R7.mjs';
21
- import './chunk-Y2F66IQJ.mjs';
22
- import './chunk-ZJ4CSAB2.mjs';
23
- import './chunk-QZN3MIJ3.mjs';
24
- import './chunk-T6CNO72J.mjs';
21
+ import './chunk-U2BKYEC7.mjs';
22
+ import './chunk-KOKM4PQN.mjs';
23
+ import './chunk-QY7GC2MW.mjs';
24
+ import './chunk-ES33L3HO.mjs';
25
+ import './chunk-QM3VPX4Z.mjs';
25
26
  import './chunk-WQBCBKY3.mjs';
26
- import './chunk-A3UB2MXL.mjs';
27
- import { encodeDocument, requireValidFieldTypes } from './chunk-CVQWQVFX.mjs';
27
+ import './chunk-SBCEHIPL.mjs';
28
+ import { encodeDocument, requireValidFieldTypes } from './chunk-4ZJWRETC.mjs';
28
29
  import './chunk-FGMGUDHX.mjs';
29
30
  import { MIN_M } from './chunk-KEHK2O6P.mjs';
30
31
  import './chunk-GES7U2PM.mjs';
31
- import './chunk-BVOJIYRV.mjs';
32
- import './chunk-U2BKYEC7.mjs';
32
+ import './chunk-BB7MASRQ.mjs';
33
+ import './chunk-2GZGBOQI.mjs';
34
+ import './chunk-X5CV2E2B.mjs';
35
+ import './chunk-7X5JST5L.mjs';
36
+ import './chunk-7KC56574.mjs';
33
37
  import './chunk-R6A773WK.mjs';
34
38
  import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
35
- import './chunk-YRX6XMOQ.mjs';
36
- import './chunk-R7OSDAFK.mjs';
39
+ import './chunk-R6PR4WYZ.mjs';
40
+ import './chunk-K6A4FCVR.mjs';
37
41
  import './chunk-L5ORED7V.mjs';
38
- import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
42
+ import { NarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
39
43
  import { decode } from '@msgpack/msgpack';
40
44
 
41
- // src/persistence/durability/archive-checkpoint.ts
45
+ // src/persistence/durability/sha256.ts
46
+ var ROUND_CONSTANTS = new Uint32Array([
47
+ 1116352408,
48
+ 1899447441,
49
+ 3049323471,
50
+ 3921009573,
51
+ 961987163,
52
+ 1508970993,
53
+ 2453635748,
54
+ 2870763221,
55
+ 3624381080,
56
+ 310598401,
57
+ 607225278,
58
+ 1426881987,
59
+ 1925078388,
60
+ 2162078206,
61
+ 2614888103,
62
+ 3248222580,
63
+ 3835390401,
64
+ 4022224774,
65
+ 264347078,
66
+ 604807628,
67
+ 770255983,
68
+ 1249150122,
69
+ 1555081692,
70
+ 1996064986,
71
+ 2554220882,
72
+ 2821834349,
73
+ 2952996808,
74
+ 3210313671,
75
+ 3336571891,
76
+ 3584528711,
77
+ 113926993,
78
+ 338241895,
79
+ 666307205,
80
+ 773529912,
81
+ 1294757372,
82
+ 1396182291,
83
+ 1695183700,
84
+ 1986661051,
85
+ 2177026350,
86
+ 2456956037,
87
+ 2730485921,
88
+ 2820302411,
89
+ 3259730800,
90
+ 3345764771,
91
+ 3516065817,
92
+ 3600352804,
93
+ 4094571909,
94
+ 275423344,
95
+ 430227734,
96
+ 506948616,
97
+ 659060556,
98
+ 883997877,
99
+ 958139571,
100
+ 1322822218,
101
+ 1537002063,
102
+ 1747873779,
103
+ 1955562222,
104
+ 2024104815,
105
+ 2227730452,
106
+ 2361852424,
107
+ 2428436474,
108
+ 2756734187,
109
+ 3204031479,
110
+ 3329325298
111
+ ]);
112
+ var INITIAL_STATE = [1779033703, 3144134277, 1013904242, 2773480762, 1359893119, 2600822924, 528734635, 1541459225];
113
+ var BLOCK_BYTES = 64;
114
+ var LENGTH_BYTES = 8;
115
+ function rotateRight(value, bits) {
116
+ return value >>> bits | value << 32 - bits;
117
+ }
118
+ function padded(bytes) {
119
+ const total = Math.ceil((bytes.byteLength + 1 + LENGTH_BYTES) / BLOCK_BYTES) * BLOCK_BYTES;
120
+ const message = new Uint8Array(total);
121
+ message.set(bytes);
122
+ message[bytes.byteLength] = 128;
123
+ const view = new DataView(message.buffer);
124
+ const bitLength = bytes.byteLength * 8;
125
+ view.setUint32(total - LENGTH_BYTES, Math.floor(bitLength / 4294967296), false);
126
+ view.setUint32(total - 4, bitLength >>> 0, false);
127
+ return view;
128
+ }
129
+ function sha256(bytes) {
130
+ const view = padded(bytes);
131
+ const state = INITIAL_STATE.slice();
132
+ const words = new Uint32Array(64);
133
+ for (let block = 0; block < view.byteLength; block += BLOCK_BYTES) {
134
+ for (let i = 0; i < 16; i++) words[i] = view.getUint32(block + i * 4, false);
135
+ for (let i = 16; i < 64; i++) {
136
+ const w15 = words[i - 15];
137
+ const w2 = words[i - 2];
138
+ const s0 = rotateRight(w15, 7) ^ rotateRight(w15, 18) ^ w15 >>> 3;
139
+ const s1 = rotateRight(w2, 17) ^ rotateRight(w2, 19) ^ w2 >>> 10;
140
+ words[i] = words[i - 16] + s0 + words[i - 7] + s1 >>> 0;
141
+ }
142
+ let [a, b, c, d, e, f, g, h] = state;
143
+ for (let i = 0; i < 64; i++) {
144
+ const s1 = rotateRight(e, 6) ^ rotateRight(e, 11) ^ rotateRight(e, 25);
145
+ const choice = e & f ^ ~e & g;
146
+ const t1 = h + s1 + choice + ROUND_CONSTANTS[i] + words[i] >>> 0;
147
+ const s0 = rotateRight(a, 2) ^ rotateRight(a, 13) ^ rotateRight(a, 22);
148
+ const majority = a & b ^ a & c ^ b & c;
149
+ const t2 = s0 + majority >>> 0;
150
+ h = g;
151
+ g = f;
152
+ f = e;
153
+ e = d + t1 >>> 0;
154
+ d = c;
155
+ c = b;
156
+ b = a;
157
+ a = t1 + t2 >>> 0;
158
+ }
159
+ state[0] = state[0] + a >>> 0;
160
+ state[1] = state[1] + b >>> 0;
161
+ state[2] = state[2] + c >>> 0;
162
+ state[3] = state[3] + d >>> 0;
163
+ state[4] = state[4] + e >>> 0;
164
+ state[5] = state[5] + f >>> 0;
165
+ state[6] = state[6] + g >>> 0;
166
+ state[7] = state[7] + h >>> 0;
167
+ }
168
+ const digest = new Uint8Array(32);
169
+ const out = new DataView(digest.buffer);
170
+ for (let i = 0; i < 8; i++) out.setUint32(i * 4, state[i], false);
171
+ return digest;
172
+ }
173
+
174
+ // src/persistence/durability/restore-journal.ts
175
+ var RESTORE_JOURNAL_PREFIX = ".swap-";
42
176
  var RESTORE_STAGING_PREFIX = ".restore-";
43
- function createRestoreStaging(directory, indexStore, indexName) {
44
- const stagingRoot = `${RESTORE_STAGING_PREFIX}${indexName}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}/`;
177
+ var RETIRED_INDEX_PREFIX = ".retired-";
178
+ var LOCK_FILE_KEY = ".narsil.lock";
179
+ var STAGING_PREFIX_PATTERN = /^\.[A-Za-z0-9][A-Za-z0-9._-]*\/$/;
180
+ function digestOf(indexName) {
181
+ let hex = "";
182
+ for (const byte of sha256(new TextEncoder().encode(indexName))) hex += byte.toString(16).padStart(2, "0");
183
+ return hex;
184
+ }
185
+ function journalKey(indexName) {
186
+ return `${RESTORE_JOURNAL_PREFIX}${digestOf(indexName)}`;
187
+ }
188
+ function randomSuffix() {
189
+ return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
190
+ }
191
+ function newStagingPrefix() {
192
+ return `${RESTORE_STAGING_PREFIX}${randomSuffix()}/`;
193
+ }
194
+ function newRetiredPrefix() {
195
+ return `${RETIRED_INDEX_PREFIX}${randomSuffix()}/`;
196
+ }
197
+ function encodeJournal(indexName, stagingPrefix) {
198
+ return new TextEncoder().encode(`${indexName}
199
+ ${stagingPrefix}`);
200
+ }
201
+ function isStagingPrefix(prefix) {
202
+ return STAGING_PREFIX_PATTERN.test(prefix) && !prefix.includes("..");
203
+ }
204
+ function corruptJournal(key, reason) {
205
+ throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The restore journal "${key}" ${reason}`, { key });
206
+ }
207
+ function readJournal(key, bytes) {
208
+ const text = new TextDecoder().decode(bytes);
209
+ const lineEnd = text.indexOf("\n");
210
+ const indexName = lineEnd === -1 ? text : text.slice(0, lineEnd);
211
+ if (key.slice(RESTORE_JOURNAL_PREFIX.length) !== digestOf(indexName)) return { kind: "leftover" };
212
+ if (indexName.length > MAX_INDEX_NAME_LENGTH || !INDEX_NAME_PATTERN.test(indexName) || indexName.includes("..")) {
213
+ corruptJournal(key, "names an index that no index name can match");
214
+ }
215
+ const stagingPrefix = lineEnd === -1 ? "" : text.slice(lineEnd + 1);
216
+ if (!isStagingPrefix(stagingPrefix) || stagingPrefix === `${LOCK_FILE_KEY}/`) {
217
+ corruptJournal(key, "names no staging prefix that a restore writes");
218
+ }
219
+ return { kind: "journal", indexName, stagingPrefix };
220
+ }
221
+ async function listedKeys(storage, indexName, stagingPrefix) {
222
+ const manifestBytes = await storage.read(`${stagingPrefix}manifest`) ?? await storage.read(manifestKey(indexName));
223
+ if (manifestBytes === null) {
224
+ throw new NarsilError(
225
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
226
+ `The restore of index "${indexName}" lost its manifest, both from staging and from the index`,
227
+ { indexName, stagingPrefix }
228
+ );
229
+ }
230
+ const decoded = await decodeSegmentManifest(manifestBytes, indexName);
231
+ if (decoded.kind !== "current") {
232
+ throw new NarsilError(
233
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
234
+ `The restore of index "${indexName}" staged a manifest in the layout of an earlier release`,
235
+ { indexName }
236
+ );
237
+ }
238
+ const root = `${indexName}/`;
239
+ const keys = /* @__PURE__ */ new Set(["meta", "manifest"]);
240
+ for (const ref of manifestFileRefs(decoded.manifest)) keys.add(ref.key.slice(root.length));
241
+ return keys;
242
+ }
243
+ function withoutTrailingSlash(prefix) {
244
+ return prefix.slice(0, -1);
245
+ }
246
+ async function finishJournal(storage, indexName, stagingPrefix) {
247
+ const root = `${indexName}/`;
248
+ const listed = await listedKeys(storage, indexName, stagingPrefix);
249
+ const staged = await storage.list(stagingPrefix);
250
+ let retiredPrefix = null;
251
+ if (storage.moveTree !== void 0 && staged.length > 0) {
252
+ retiredPrefix = newRetiredPrefix();
253
+ await storage.moveTree(indexName, withoutTrailingSlash(retiredPrefix));
254
+ await storage.moveTree(withoutTrailingSlash(stagingPrefix), indexName);
255
+ } else {
256
+ const stagedKeys = new Set(staged);
257
+ for (const key of listed) {
258
+ if (stagedKeys.has(`${stagingPrefix}${key}`)) await storage.moveKey(`${stagingPrefix}${key}`, `${root}${key}`);
259
+ }
260
+ }
261
+ for (const key of await storage.list(root)) {
262
+ if (!listed.has(key.slice(root.length))) await storage.remove(key);
263
+ }
264
+ await storage.remove(journalKey(indexName));
265
+ return retiredPrefix;
266
+ }
267
+ async function removePrefix(storage, prefix) {
268
+ if (storage.removeTree !== void 0) {
269
+ await storage.removeTree(prefix);
270
+ return;
271
+ }
272
+ for (const key of await storage.list(prefix)) await storage.remove(key);
273
+ }
274
+ async function commitJournal(storage, indexName, stagingPrefix) {
275
+ await storage.writeJournal(journalKey(indexName), encodeJournal(indexName, stagingPrefix));
276
+ }
277
+ function isTopLevelJournal(key) {
278
+ return key.startsWith(RESTORE_JOURNAL_PREFIX) && !key.includes("/");
279
+ }
280
+ async function removeStagingPrefixes(storage) {
281
+ const prefixes = /* @__PURE__ */ new Set();
282
+ for (const key of await storage.list(".")) {
283
+ const slash = key.indexOf("/");
284
+ if (slash !== -1 && isStagingPrefix(key.slice(0, slash + 1))) prefixes.add(key.slice(0, slash + 1));
285
+ }
286
+ for (const prefix of prefixes) await removePrefix(storage, prefix);
287
+ }
288
+ async function recoverRestoreJournals(storage) {
289
+ for (const key of await storage.list(RESTORE_JOURNAL_PREFIX)) {
290
+ if (!isTopLevelJournal(key)) continue;
291
+ const bytes = await storage.read(key);
292
+ if (bytes === null) continue;
293
+ const reading = readJournal(key, bytes);
294
+ if (reading.kind === "leftover") {
295
+ await storage.remove(key);
296
+ continue;
297
+ }
298
+ await finishJournal(storage, reading.indexName, reading.stagingPrefix);
299
+ }
300
+ await removeStagingPrefixes(storage);
301
+ }
302
+
303
+ // src/persistence/durability/restore-directory.ts
304
+ function missing(error) {
305
+ return error.code === "ENOENT";
306
+ }
307
+ function storageFailed(code, message, key, error) {
308
+ return new NarsilError(code, message, { key, cause: error instanceof Error ? error.message : String(error) });
309
+ }
310
+ function directoryRestoreStorage(directory) {
311
+ return {
312
+ read: (key) => directory.read(key),
313
+ write: (key, chunks) => directory.atomicWrite(key, chunks),
314
+ writeJournal: (key, bytes) => directory.atomicWrite(key, bytes),
315
+ remove: (key) => directory.remove(key),
316
+ list: (prefix) => directory.list(prefix),
317
+ moveKey: (from, to) => directory.move(from, to),
318
+ pathOf: (key) => directory.pathOf(key),
319
+ async moveTree(from, to) {
320
+ const source = await directory.pathOf(from);
321
+ const target = await directory.pathOf(to);
322
+ const fs = await getFs();
323
+ try {
324
+ await fs.rename(source, target);
325
+ } catch (error) {
326
+ if (missing(error)) return;
327
+ throw storageFailed(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Failed to move "${from}" to "${to}"`, from, error);
328
+ }
329
+ await directory.syncDirectoryOf(to);
330
+ },
331
+ async removeTree(prefix) {
332
+ const key = prefix.endsWith("/") ? prefix.slice(0, -1) : prefix;
333
+ const fs = await getFs();
334
+ try {
335
+ await fs.rm(await directory.pathOf(key), { recursive: true, force: true });
336
+ } catch (error) {
337
+ throw storageFailed(ErrorCodes.PERSISTENCE_DELETE_FAILED, `Failed to delete "${key}"`, key, error);
338
+ }
339
+ await directory.syncDirectoryOf(key);
340
+ }
341
+ };
342
+ }
343
+ function recoverDirectoryRestores(directory) {
344
+ return recoverRestoreJournals(directoryRestoreStorage(directory));
345
+ }
346
+
347
+ // src/persistence/durability/restore-staging.ts
348
+ function toError(error) {
349
+ return error instanceof Error ? error : new Error(String(error));
350
+ }
351
+ function pathUnder(path, root) {
352
+ if (!path.startsWith(root)) return false;
353
+ const next = path.charAt(root.length);
354
+ return next === "" || next === "/" || next === "\\";
355
+ }
356
+ function createRestoreStaging(storage, indexName, options = {}) {
357
+ const stagingPrefix = newStagingPrefix();
45
358
  const indexRoot = `${indexName}/`;
359
+ let committed = false;
46
360
  const staged = (key) => {
47
361
  if (!key.startsWith(indexRoot)) {
48
362
  throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `A restore of "${indexName}" cannot stage "${key}"`, {
49
363
  key
50
364
  });
51
365
  }
52
- return `${stagingRoot}${key.slice(indexRoot.length)}`;
366
+ return `${stagingPrefix}${key.slice(indexRoot.length)}`;
53
367
  };
54
- const unstaged = (key) => `${indexRoot}${key.slice(stagingRoot.length)}`;
55
- const stagingStore = {
56
- read: (key) => directory.read(staged(key)),
57
- write: (key, chunks) => directory.atomicWrite(staged(key), chunks),
58
- remove: (key) => directory.remove(staged(key)),
59
- list: async (prefix) => (await directory.list(staged(prefix))).map(unstaged),
60
- pathOf: (key) => directory.pathOf(staged(key))
368
+ const unstaged = (key) => `${indexRoot}${key.slice(stagingPrefix.length)}`;
369
+ const store = {
370
+ read: (key) => storage.read(staged(key)),
371
+ write: (key, chunks) => storage.write(staged(key), chunks),
372
+ remove: (key) => storage.remove(staged(key)),
373
+ list: async (prefix) => (await storage.list(staged(prefix))).map(unstaged)
61
374
  };
375
+ const pathOf = storage.pathOf;
376
+ if (pathOf !== void 0) store.pathOf = (key) => pathOf.call(storage, staged(key));
377
+ let retiredPrefix = null;
378
+ async function finishOnStorage() {
379
+ retiredPrefix = await finishJournal(storage, indexName, stagingPrefix);
380
+ }
62
381
  return {
63
- store: stagingStore,
382
+ store,
383
+ async relocation() {
384
+ if (pathOf === void 0 || (await storage.list(stagingPrefix)).length > 0) return null;
385
+ const from = await pathOf.call(storage, stagingPrefix.slice(0, -1));
386
+ const to = await pathOf.call(storage, indexName);
387
+ return (path) => pathUnder(path, from) ? `${to}${path.slice(from.length)}` : path;
388
+ },
64
389
  async commit() {
65
- for (const key of await directory.list(stagingRoot)) await directory.move(key, unstaged(key));
66
- return indexStore;
390
+ try {
391
+ await commitJournal(storage, indexName, stagingPrefix);
392
+ } catch (error) {
393
+ try {
394
+ await storage.remove(journalKey(indexName));
395
+ } catch (removal) {
396
+ committed = true;
397
+ options.onStorageLost?.(toError(removal));
398
+ }
399
+ throw error;
400
+ }
401
+ committed = true;
402
+ },
403
+ async finish() {
404
+ try {
405
+ await (options.finish === void 0 ? finishOnStorage() : options.finish(finishOnStorage));
406
+ } catch (error) {
407
+ options.onStorageLost?.(toError(error));
408
+ }
409
+ },
410
+ async removeRetired() {
411
+ const prefix = retiredPrefix;
412
+ retiredPrefix = null;
413
+ if (prefix !== null) await removePrefix(storage, prefix);
67
414
  },
68
415
  async discard() {
69
- for (const key of await directory.list(stagingRoot)) await directory.remove(key);
416
+ if (committed) return;
417
+ await removePrefix(storage, stagingPrefix);
418
+ }
419
+ };
420
+ }
421
+ function adapterRestoreStorage(adapter) {
422
+ const store = adapterCheckpointStore(adapter);
423
+ return {
424
+ read: (key) => store.read(key),
425
+ write: (key, chunks) => store.write(key, chunks),
426
+ writeJournal: (key, bytes) => store.write(key, [bytes]),
427
+ remove: (key) => store.remove(key),
428
+ list: (prefix) => store.list(prefix),
429
+ async moveKey(from, to) {
430
+ const bytes = await store.read(from);
431
+ if (bytes === null) return;
432
+ await store.write(to, [bytes]);
433
+ await store.remove(from);
70
434
  }
71
435
  };
72
436
  }
73
- async function removeAbandonedRestores(directory) {
74
- for (const key of await directory.list(RESTORE_STAGING_PREFIX)) await directory.remove(key);
437
+
438
+ // src/persistence/durability/archive-checkpoint.ts
439
+ function toError2(error) {
440
+ return error instanceof Error ? error : new Error(String(error));
441
+ }
442
+ function stageDirectoryRestore(deps, indexName) {
443
+ return createRestoreStaging(directoryRestoreStorage(deps.directory), indexName, {
444
+ async finish(run) {
445
+ try {
446
+ await run();
447
+ } catch (error) {
448
+ deps.markFatal(toError2(error));
449
+ throw error;
450
+ }
451
+ },
452
+ onStorageLost: deps.markFatal
453
+ });
75
454
  }
76
455
  function releaser(held, indexName) {
77
456
  let released = false;
@@ -122,7 +501,7 @@ function createArchiveCheckpointAccess(deps) {
122
501
  }
123
502
  },
124
503
  stageRestore(indexName) {
125
- return createRestoreStaging(deps.directory, store, indexName);
504
+ return stageDirectoryRestore(deps, indexName);
126
505
  },
127
506
  adoptRestoredCheckpoint(indexName, checkpoint) {
128
507
  for (const { partitionId, lastSeqNo } of checkpoint) deps.registerPartition(indexName, partitionId, lastSeqNo);
@@ -380,8 +759,8 @@ async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartit
380
759
  const vectorPlans = /* @__PURE__ */ new Map();
381
760
  for (const [fieldPath, vectorIndex] of vectorIndexes) {
382
761
  const listed = listedVectorFields.find((ref) => ref.fieldPath === fieldPath);
383
- const listedKeys = listed === void 0 ? null : listed.files.map((file) => file.key);
384
- vectorPlans.set(fieldPath, vectorIndex.planCheckpoint(listedKeys));
762
+ const listedKeys2 = listed === void 0 ? null : listed.files.map((file) => file.key);
763
+ vectorPlans.set(fieldPath, vectorIndex.planCheckpoint(listedKeys2));
385
764
  }
386
765
  const memoryMatchesTheTargets = recordsLeftInLog === 0;
387
766
  const wholePartitions = memoryMatchesTheTargets ? wholePartitionsOf(targets) : noWholePartitions();
@@ -875,7 +1254,7 @@ async function drainIndexStateForUnload(indexState, pendingMetadataWrite, closeW
875
1254
  }
876
1255
 
877
1256
  // src/persistence/durability/mutation-batch.ts
878
- function toError(err) {
1257
+ function toError3(err) {
879
1258
  return err instanceof Error ? err : new Error(String(err));
880
1259
  }
881
1260
  function failUnlessAlreadyFailed(outcomes, count, error) {
@@ -887,7 +1266,7 @@ function failUnlessAlreadyFailed(outcomes, count, error) {
887
1266
  }
888
1267
  }
889
1268
  function failPartition(partition, deps, err) {
890
- const error = toError(err);
1269
+ const error = toError3(err);
891
1270
  partition.failed = error;
892
1271
  deps.markFatal(error);
893
1272
  return error;
@@ -967,7 +1346,7 @@ async function recordPartitionBatch(partition, records, deps) {
967
1346
  }
968
1347
  }
969
1348
  } catch (err) {
970
- failUnlessAlreadyFailed(outcomes, records.length, toError(err));
1349
+ failUnlessAlreadyFailed(outcomes, records.length, toError3(err));
971
1350
  }
972
1351
  return outcomes;
973
1352
  }
@@ -1804,8 +2183,8 @@ async function createMarkerWriter(directory, indexName, partitionId) {
1804
2183
  // src/persistence/durability/wal-writer.ts
1805
2184
  var SEGMENT_TAIL_PATTERN = /^\d{16}$/;
1806
2185
  function segmentKey(indexName, partitionId, startSeqNo) {
1807
- const padded = startSeqNo.toString().padStart(16, "0");
1808
- return `${indexName}/wal/${partitionId}/${padded}`;
2186
+ const padded2 = startSeqNo.toString().padStart(16, "0");
2187
+ return `${indexName}/wal/${partitionId}/${padded2}`;
1809
2188
  }
1810
2189
  function parseSegmentStartSeqNo2(key) {
1811
2190
  const tail = key.slice(key.lastIndexOf("/") + 1);
@@ -2051,14 +2430,14 @@ function createDurabilityManager(config, hooks, directoryOverride) {
2051
2430
  stopTimers();
2052
2431
  hooks.onFatalError(error);
2053
2432
  }
2054
- function toError2(err) {
2433
+ function toError5(err) {
2055
2434
  return err instanceof Error ? err : new Error(String(err));
2056
2435
  }
2057
2436
  async function closeWriterReportingFailure(closePromise) {
2058
2437
  try {
2059
2438
  await closePromise;
2060
2439
  } catch (err) {
2061
- markFatal(toError2(err));
2440
+ markFatal(toError5(err));
2062
2441
  }
2063
2442
  }
2064
2443
  function getOrCreateIndexState(indexName) {
@@ -2124,7 +2503,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
2124
2503
  try {
2125
2504
  await partition.walWriter.commit();
2126
2505
  } catch (err) {
2127
- partition.failed = toError2(err);
2506
+ partition.failed = toError5(err);
2128
2507
  markFatal(partition.failed);
2129
2508
  }
2130
2509
  }
@@ -2138,7 +2517,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
2138
2517
  try {
2139
2518
  await checkpointIndex(indexName);
2140
2519
  } catch (err) {
2141
- markFatal(toError2(err));
2520
+ markFatal(toError5(err));
2142
2521
  }
2143
2522
  }
2144
2523
  }
@@ -2219,7 +2598,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
2219
2598
  indexState.documentBytesSinceCheckpoint += recorded.documentBytes;
2220
2599
  if (checkpointIsDue(indexState, checkpointMutationThreshold)) {
2221
2600
  void checkpointIndex(indexName).catch((err) => {
2222
- markFatal(toError2(err));
2601
+ markFatal(toError5(err));
2223
2602
  });
2224
2603
  }
2225
2604
  await stallWhileCheckpointsFallBehind(indexState);
@@ -2251,12 +2630,10 @@ function createDurabilityManager(config, hooks, directoryOverride) {
2251
2630
  );
2252
2631
  }
2253
2632
  return {
2254
- isActive() {
2255
- return true;
2256
- },
2633
+ isActive: () => true,
2257
2634
  async recover(metadataOnly = false) {
2258
2635
  await directoryWatch.claim();
2259
- await removeAbandonedRestores(directory);
2636
+ await recoverDirectoryRestores(directory);
2260
2637
  const names = await listPersistedIndexes(directory);
2261
2638
  for (const indexName of names) {
2262
2639
  await recoverIndex(indexName, metadataOnly);
@@ -2326,7 +2703,8 @@ function createDurabilityManager(config, hooks, directoryOverride) {
2326
2703
  directory,
2327
2704
  held: heldCheckpointFiles,
2328
2705
  checkpoint: (name) => checkpointIndex(name),
2329
- registerPartition: (name, partitionId, startSeqNo) => getOrCreatePartition(name, partitionId, startSeqNo)
2706
+ registerPartition: (name, partitionId, startSeqNo) => getOrCreatePartition(name, partitionId, startSeqNo),
2707
+ markFatal
2330
2708
  }),
2331
2709
  async shutdown() {
2332
2710
  shuttingDown = true;
@@ -2346,7 +2724,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
2346
2724
  };
2347
2725
  }
2348
2726
 
2349
- // src/persistence/durability/snapshot-only-checkpoint.ts
2727
+ // src/persistence/durability/snapshot-only/checkpoint.ts
2350
2728
  function capture(input, prior) {
2351
2729
  const listedIds = new Set((prior?.partitions ?? []).map((partition) => partition.partitionId));
2352
2730
  const partitionCountChanged = prior !== null && listedIds.size !== input.manager.partitionCount;
@@ -2402,7 +2780,44 @@ async function writeSnapshotOnlyCheckpoint(input) {
2402
2780
  return captured.documentCount;
2403
2781
  }
2404
2782
 
2405
- // src/persistence/durability/snapshot-only.ts
2783
+ // src/persistence/durability/snapshot-only/restores.ts
2784
+ function toError4(error) {
2785
+ return error instanceof Error ? error : new Error(String(error));
2786
+ }
2787
+ function createPendingRestores(onFatalError) {
2788
+ const pending = /* @__PURE__ */ new Map();
2789
+ return {
2790
+ finishLater(indexName, run) {
2791
+ let running = null;
2792
+ const finish = () => {
2793
+ running ??= run().then(
2794
+ () => {
2795
+ if (pending.get(indexName) === finish) pending.delete(indexName);
2796
+ },
2797
+ (error) => {
2798
+ running = null;
2799
+ throw error;
2800
+ }
2801
+ );
2802
+ return running;
2803
+ };
2804
+ pending.set(indexName, finish);
2805
+ void finish().catch((error) => onFatalError(toError4(error)));
2806
+ return Promise.resolve();
2807
+ },
2808
+ async settle(indexName) {
2809
+ await pending.get(indexName)?.();
2810
+ },
2811
+ forget(indexName) {
2812
+ pending.delete(indexName);
2813
+ },
2814
+ async settleAll() {
2815
+ for (const finish of [...pending.values()]) await finish().catch(() => void 0);
2816
+ }
2817
+ };
2818
+ }
2819
+
2820
+ // src/persistence/durability/snapshot-only/index.ts
2406
2821
  function metadataKey(indexName) {
2407
2822
  return `${indexName}/meta`;
2408
2823
  }
@@ -2411,8 +2826,13 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
2411
2826
  const checkpointMutationThreshold = config.checkpointMutationThreshold ?? DEFAULT_CHECKPOINT_MUTATION_THRESHOLD;
2412
2827
  const indexes = /* @__PURE__ */ new Map();
2413
2828
  const metadataWrites = /* @__PURE__ */ new Map();
2829
+ const restoreStorage = adapterRestoreStorage(adapter);
2830
+ const restores = createPendingRestores(hooks.onFatalError);
2414
2831
  let checkpointTimer = null;
2415
2832
  let shuttingDown = false;
2833
+ function settleRestore(indexName) {
2834
+ return restores.settle(indexName);
2835
+ }
2416
2836
  function getOrCreateIndexState(indexName) {
2417
2837
  let state = indexes.get(indexName);
2418
2838
  if (state === void 0) {
@@ -2448,6 +2868,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
2448
2868
  }
2449
2869
  }
2450
2870
  async function performCheckpoint(indexName, indexState, fromMemory) {
2871
+ await settleRestore(indexName);
2451
2872
  const manager = hooks.getManager(indexName);
2452
2873
  const metadata = hooks.buildMetadata(indexName);
2453
2874
  if (manager === void 0 || metadata === void 0) {
@@ -2497,7 +2918,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
2497
2918
  }
2498
2919
  function queueMetadataWrite(indexName, write) {
2499
2920
  const previous = metadataWrites.get(indexName) ?? Promise.resolve();
2500
- const queued = previous.then(write);
2921
+ const queued = previous.then(() => settleRestore(indexName)).then(write);
2501
2922
  metadataWrites.set(
2502
2923
  indexName,
2503
2924
  queued.catch(() => void 0)
@@ -2505,6 +2926,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
2505
2926
  return queued;
2506
2927
  }
2507
2928
  async function recoverIndex(indexName, metadataOnly = false) {
2929
+ await settleRestore(indexName);
2508
2930
  const metaBytes = await adapter.load(metadataKey(indexName));
2509
2931
  if (metaBytes === null) {
2510
2932
  return;
@@ -2591,11 +3013,28 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
2591
3013
  }
2592
3014
  return appliedSeqNo;
2593
3015
  }
3016
+ async function drainIndex(indexName) {
3017
+ const indexState = indexes.get(indexName);
3018
+ if (indexState !== void 0) {
3019
+ indexState.unloading = true;
3020
+ try {
3021
+ await indexState.applyChain;
3022
+ if (indexState.checkpointInFlight !== null) await indexState.checkpointInFlight;
3023
+ await metadataWrites.get(indexName);
3024
+ } catch (error) {
3025
+ indexState.unloading = false;
3026
+ throw error;
3027
+ }
3028
+ }
3029
+ indexes.delete(indexName);
3030
+ metadataWrites.delete(indexName);
3031
+ }
2594
3032
  return {
2595
3033
  isActive() {
2596
3034
  return true;
2597
3035
  },
2598
3036
  async recover(metadataOnly = false) {
3037
+ await recoverRestoreJournals(restoreStorage);
2599
3038
  const keys = await adapter.list("");
2600
3039
  for (const key of keys) {
2601
3040
  if (key.endsWith("/meta")) {
@@ -2644,40 +3083,31 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
2644
3083
  await checkpointIndex(indexName);
2645
3084
  }
2646
3085
  },
2647
- async removeIndex(indexName) {
2648
- const indexState = indexes.get(indexName);
2649
- if (indexState !== void 0) {
2650
- indexState.unloading = true;
2651
- try {
2652
- await indexState.applyChain;
2653
- if (indexState.checkpointInFlight !== null) await indexState.checkpointInFlight;
2654
- await metadataWrites.get(indexName);
2655
- } catch (error) {
2656
- indexState.unloading = false;
2657
- throw error;
2658
- }
3086
+ stageRestore(indexName) {
3087
+ return createRestoreStaging(restoreStorage, indexName, {
3088
+ finish: (run) => restores.finishLater(indexName, run),
3089
+ onStorageLost: hooks.onFatalError
3090
+ });
3091
+ },
3092
+ adoptRestoredCheckpoint(indexName, checkpoint) {
3093
+ const indexState = getOrCreateIndexState(indexName);
3094
+ indexState.appliedSeqNoByPartition.clear();
3095
+ for (const { partitionId, lastSeqNo } of checkpoint) {
3096
+ if (lastSeqNo > 0) indexState.appliedSeqNoByPartition.set(partitionId, lastSeqNo);
2659
3097
  }
2660
- indexes.delete(indexName);
2661
- metadataWrites.delete(indexName);
3098
+ },
3099
+ async removeIndex(indexName) {
3100
+ await settleRestore(indexName).catch(() => void 0);
3101
+ restores.forget(indexName);
3102
+ await restoreStorage.remove(journalKey(indexName));
3103
+ await drainIndex(indexName);
2662
3104
  for (const key of await adapter.list(`${indexName}/`)) {
2663
3105
  await adapter.delete(key);
2664
3106
  }
2665
3107
  },
2666
3108
  async unloadIndex(indexName) {
2667
- const indexState = indexes.get(indexName);
2668
- if (indexState !== void 0) {
2669
- indexState.unloading = true;
2670
- try {
2671
- await indexState.applyChain;
2672
- if (indexState.checkpointInFlight !== null) await indexState.checkpointInFlight;
2673
- await metadataWrites.get(indexName);
2674
- } catch (error) {
2675
- indexState.unloading = false;
2676
- throw error;
2677
- }
2678
- }
2679
- indexes.delete(indexName);
2680
- metadataWrites.delete(indexName);
3109
+ await settleRestore(indexName);
3110
+ await drainIndex(indexName);
2681
3111
  },
2682
3112
  async reloadIndex(indexName) {
2683
3113
  const indexState = getOrCreateIndexState(indexName);
@@ -2691,6 +3121,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
2691
3121
  clearInterval(checkpointTimer);
2692
3122
  checkpointTimer = null;
2693
3123
  }
3124
+ await restores.settleAll();
2694
3125
  for (const indexState of indexes.values()) {
2695
3126
  if (indexState.checkpointInFlight !== null) {
2696
3127
  await indexState.checkpointInFlight.catch(() => void 0);