@platformatic/sqlite-js-stream 0.0.0-stage → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (162) hide show
  1. package/CHANGELOG.md +40 -0
  2. package/LICENSE +202 -0
  3. package/NOTICE +22 -0
  4. package/README.md +482 -2
  5. package/dist/backends/file/index.d.ts +4 -0
  6. package/dist/backends/file/index.d.ts.map +1 -0
  7. package/dist/backends/file/index.js +3 -0
  8. package/dist/backends/file/leaser.d.ts +20 -0
  9. package/dist/backends/file/leaser.d.ts.map +1 -0
  10. package/dist/backends/file/leaser.js +123 -0
  11. package/dist/backends/file/replica-client.d.ts +38 -0
  12. package/dist/backends/file/replica-client.d.ts.map +1 -0
  13. package/dist/backends/file/replica-client.js +209 -0
  14. package/dist/db.d.ts +80 -0
  15. package/dist/db.d.ts.map +1 -0
  16. package/dist/db.js +484 -0
  17. package/dist/errors.d.ts +42 -0
  18. package/dist/errors.d.ts.map +1 -0
  19. package/dist/errors.js +66 -0
  20. package/dist/index.d.ts +19 -0
  21. package/dist/index.d.ts.map +1 -0
  22. package/dist/index.js +22 -0
  23. package/dist/leaser.d.ts +55 -0
  24. package/dist/leaser.d.ts.map +1 -0
  25. package/dist/leaser.js +60 -0
  26. package/dist/ltx/byte-reader.d.ts +18 -0
  27. package/dist/ltx/byte-reader.d.ts.map +1 -0
  28. package/dist/ltx/byte-reader.js +38 -0
  29. package/dist/ltx/bytes.d.ts +20 -0
  30. package/dist/ltx/bytes.d.ts.map +1 -0
  31. package/dist/ltx/bytes.js +69 -0
  32. package/dist/ltx/checksum.d.ts +20 -0
  33. package/dist/ltx/checksum.d.ts.map +1 -0
  34. package/dist/ltx/checksum.js +42 -0
  35. package/dist/ltx/compaction-level.d.ts +58 -0
  36. package/dist/ltx/compaction-level.d.ts.map +1 -0
  37. package/dist/ltx/compaction-level.js +115 -0
  38. package/dist/ltx/compactor.d.ts +33 -0
  39. package/dist/ltx/compactor.d.ts.map +1 -0
  40. package/dist/ltx/compactor.js +137 -0
  41. package/dist/ltx/constants.d.ts +49 -0
  42. package/dist/ltx/constants.d.ts.map +1 -0
  43. package/dist/ltx/constants.js +58 -0
  44. package/dist/ltx/crc64.d.ts +29 -0
  45. package/dist/ltx/crc64.d.ts.map +1 -0
  46. package/dist/ltx/crc64.js +57 -0
  47. package/dist/ltx/decoder.d.ts +55 -0
  48. package/dist/ltx/decoder.d.ts.map +1 -0
  49. package/dist/ltx/decoder.js +236 -0
  50. package/dist/ltx/encoder.d.ts +49 -0
  51. package/dist/ltx/encoder.d.ts.map +1 -0
  52. package/dist/ltx/encoder.js +159 -0
  53. package/dist/ltx/file-info.d.ts +24 -0
  54. package/dist/ltx/file-info.d.ts.map +1 -0
  55. package/dist/ltx/file-info.js +74 -0
  56. package/dist/ltx/filename.d.ts +13 -0
  57. package/dist/ltx/filename.d.ts.map +1 -0
  58. package/dist/ltx/filename.js +23 -0
  59. package/dist/ltx/header.d.ts +17 -0
  60. package/dist/ltx/header.d.ts.map +1 -0
  61. package/dist/ltx/header.js +115 -0
  62. package/dist/ltx/index.d.ts +24 -0
  63. package/dist/ltx/index.d.ts.map +1 -0
  64. package/dist/ltx/index.js +25 -0
  65. package/dist/ltx/lz4.d.ts +41 -0
  66. package/dist/ltx/lz4.d.ts.map +1 -0
  67. package/dist/ltx/lz4.js +264 -0
  68. package/dist/ltx/page-index.d.ts +26 -0
  69. package/dist/ltx/page-index.d.ts.map +1 -0
  70. package/dist/ltx/page-index.js +118 -0
  71. package/dist/ltx/page.d.ts +27 -0
  72. package/dist/ltx/page.d.ts.map +1 -0
  73. package/dist/ltx/page.js +66 -0
  74. package/dist/ltx/sink.d.ts +17 -0
  75. package/dist/ltx/sink.d.ts.map +1 -0
  76. package/dist/ltx/sink.js +21 -0
  77. package/dist/ltx/stream.d.ts +64 -0
  78. package/dist/ltx/stream.d.ts.map +1 -0
  79. package/dist/ltx/stream.js +248 -0
  80. package/dist/ltx/streaming-compactor.d.ts +34 -0
  81. package/dist/ltx/streaming-compactor.d.ts.map +1 -0
  82. package/dist/ltx/streaming-compactor.js +140 -0
  83. package/dist/ltx/trailer.d.ts +12 -0
  84. package/dist/ltx/trailer.d.ts.map +1 -0
  85. package/dist/ltx/trailer.js +47 -0
  86. package/dist/ltx/types.d.ts +79 -0
  87. package/dist/ltx/types.d.ts.map +1 -0
  88. package/dist/ltx/types.js +70 -0
  89. package/dist/replica-client.d.ts +92 -0
  90. package/dist/replica-client.d.ts.map +1 -0
  91. package/dist/replica-client.js +122 -0
  92. package/dist/replica-url.d.ts +20 -0
  93. package/dist/replica-url.d.ts.map +1 -0
  94. package/dist/replica-url.js +49 -0
  95. package/dist/replica.d.ts +379 -0
  96. package/dist/replica.d.ts.map +1 -0
  97. package/dist/replica.js +1253 -0
  98. package/dist/store.d.ts +34 -0
  99. package/dist/store.d.ts.map +1 -0
  100. package/dist/store.js +60 -0
  101. package/dist/util/fs.d.ts +28 -0
  102. package/dist/util/fs.d.ts.map +1 -0
  103. package/dist/util/fs.js +73 -0
  104. package/dist/util/logger.d.ts +10 -0
  105. package/dist/util/logger.d.ts.map +1 -0
  106. package/dist/util/logger.js +7 -0
  107. package/dist/util/stream.d.ts +11 -0
  108. package/dist/util/stream.d.ts.map +1 -0
  109. package/dist/util/stream.js +25 -0
  110. package/dist/wal/checksum.d.ts +17 -0
  111. package/dist/wal/checksum.d.ts.map +1 -0
  112. package/dist/wal/checksum.js +30 -0
  113. package/dist/wal/constants.d.ts +8 -0
  114. package/dist/wal/constants.d.ts.map +1 -0
  115. package/dist/wal/constants.js +7 -0
  116. package/dist/wal/index.d.ts +5 -0
  117. package/dist/wal/index.d.ts.map +1 -0
  118. package/dist/wal/index.js +4 -0
  119. package/dist/wal/reader.d.ts +61 -0
  120. package/dist/wal/reader.d.ts.map +1 -0
  121. package/dist/wal/reader.js +207 -0
  122. package/package.json +68 -3
  123. package/src/backends/file/index.ts +3 -0
  124. package/src/backends/file/leaser.ts +145 -0
  125. package/src/backends/file/replica-client.ts +246 -0
  126. package/src/db.ts +553 -0
  127. package/src/errors.ts +74 -0
  128. package/src/index.ts +26 -0
  129. package/src/leaser.ts +86 -0
  130. package/src/ltx/byte-reader.ts +46 -0
  131. package/src/ltx/bytes.ts +87 -0
  132. package/src/ltx/checksum.ts +46 -0
  133. package/src/ltx/compaction-level.ts +135 -0
  134. package/src/ltx/compactor.ts +167 -0
  135. package/src/ltx/constants.ts +71 -0
  136. package/src/ltx/crc64.ts +65 -0
  137. package/src/ltx/decoder.ts +282 -0
  138. package/src/ltx/encoder.ts +200 -0
  139. package/src/ltx/file-info.ts +81 -0
  140. package/src/ltx/filename.ts +27 -0
  141. package/src/ltx/header.ts +126 -0
  142. package/src/ltx/index.ts +27 -0
  143. package/src/ltx/lz4.ts +279 -0
  144. package/src/ltx/page-index.ts +140 -0
  145. package/src/ltx/page.ts +80 -0
  146. package/src/ltx/sink.ts +33 -0
  147. package/src/ltx/stream.ts +286 -0
  148. package/src/ltx/streaming-compactor.ts +169 -0
  149. package/src/ltx/trailer.ts +58 -0
  150. package/src/ltx/types.ts +152 -0
  151. package/src/lz4js.d.ts +27 -0
  152. package/src/replica-client.ts +222 -0
  153. package/src/replica-url.ts +72 -0
  154. package/src/replica.ts +1480 -0
  155. package/src/store.ts +71 -0
  156. package/src/util/fs.ts +82 -0
  157. package/src/util/logger.ts +15 -0
  158. package/src/util/stream.ts +28 -0
  159. package/src/wal/checksum.ts +39 -0
  160. package/src/wal/constants.ts +10 -0
  161. package/src/wal/index.ts +5 -0
  162. package/src/wal/reader.ts +264 -0
@@ -0,0 +1,1253 @@
1
+ /**
2
+ * Replica manages replication of a single SQLite database (identified by its
3
+ * `path`) to a ReplicaClient.
4
+ *
5
+ * It reads the database + WAL directly from disk, writes level-0/level-9 LTX
6
+ * files into a local cache, uploads them to the client, tracks the replicated
7
+ * position, and can restore a database by merging LTX files. Mirrors
8
+ * litestream's `replica.go`, but is agnostic to the SQLite driver the
9
+ * application uses.
10
+ *
11
+ * For snapshot consistency it opens its own **independent read-only** SQLite
12
+ * connection (never the application's writable one) to pin the WAL while it
13
+ * reads. A single SQLite connection holds one transaction at a time, so
14
+ * reusing an application `DatabaseSync` handle would merge the read snapshot
15
+ * with the app's writes (and could read uncommitted rows). Keep the app's
16
+ * writer and the library's reader on separate connections to the same file.
17
+ */
18
+ import { createReadStream } from 'node:fs';
19
+ import { access, mkdir, open, readFile, rename, rm, stat } from 'node:fs/promises';
20
+ import nodePath from 'node:path';
21
+ import { setTimeout as sleep } from 'node:timers/promises';
22
+ import { lockPgno, StreamLtxReader, StreamingCompactor, CompactionLevels, nextCompactionAt, prevCompactionAt, defaultCompactionLevels, DefaultL0RetentionCheckIntervalMs, } from "./ltx/index.js";
23
+ import { LocalDatabase } from "./db.js";
24
+ import { SnapshotLevel } from "./replica-client.js";
25
+ import { DefaultLeaseRenewalBuffer, ErrLeaseNotHeld, Lease, LeaseExistsError, } from "./leaser.js";
26
+ import { FileSink } from "./util/fs.js";
27
+ import { noopLogger } from "./util/logger.js";
28
+ import { DatabaseError, InvalidArgumentError, LTXFormatError, PageSizeMismatchError, SqliteJsStreamError, } from "./errors.js";
29
+ /**
30
+ * Base class for errors thrown by {@link Replica.follow}. Carries a stable,
31
+ * Node-style `code` so callers can branch on the failure without matching
32
+ * message text, and can be caught via `instanceof`.
33
+ */
34
+ export class FollowError extends SqliteJsStreamError {
35
+ constructor(code, message) {
36
+ super(code, message);
37
+ this.name = 'FollowError';
38
+ }
39
+ }
40
+ /** The database exists but its `-txid` sidecar is missing, so we can't resume. */
41
+ export class FollowMissingTxidError extends FollowError {
42
+ constructor(message) {
43
+ super('ERR_FOLLOW_DB_WITHOUT_TXID', message);
44
+ this.name = 'FollowMissingTxidError';
45
+ }
46
+ }
47
+ /** The `-txid` sidecar is present but does not parse as a TXID. */
48
+ export class FollowInvalidTxidError extends FollowError {
49
+ constructor(message) {
50
+ super('ERR_FOLLOW_INVALID_TXID', message);
51
+ this.name = 'FollowInvalidTxidError';
52
+ }
53
+ }
54
+ /** The saved TXID is ahead of anything the replica holds; re-restore. */
55
+ export class FollowAheadOfLatestError extends FollowError {
56
+ constructor(message) {
57
+ super('ERR_FOLLOW_AHEAD_OF_LATEST', message);
58
+ this.name = 'FollowAheadOfLatestError';
59
+ }
60
+ }
61
+ /** The saved TXID is below the retained window; retention pruned the next file. */
62
+ export class FollowDataGapError extends FollowError {
63
+ constructor(message) {
64
+ super('ERR_FOLLOW_OLDER_THAN_EARLIEST', message);
65
+ this.name = 'FollowDataGapError';
66
+ }
67
+ }
68
+ /** Follow was aborted before the initial restore could complete. */
69
+ export class FollowAbortedError extends FollowError {
70
+ constructor(message) {
71
+ super('ERR_FOLLOW_ABORTED', message);
72
+ this.name = 'FollowAbortedError';
73
+ }
74
+ }
75
+ /**
76
+ * Default highest LTX compaction level (L0 -> L1 -> ... -> L{max}) for the
77
+ * legacy {@link CompactOptions.maxLevel} path. Prefer {@link CompactionLevels}.
78
+ */
79
+ export const DefaultMaxCompactionLevel = 3;
80
+ /** Returned by compaction scheduling when there is nothing to compact. */
81
+ export class ErrNoCompactionError extends SqliteJsStreamError {
82
+ constructor() {
83
+ super('ERR_NO_COMPACTION', 'no files to compact');
84
+ this.name = 'ErrNoCompactionError';
85
+ }
86
+ }
87
+ export const ErrNoCompaction = new ErrNoCompactionError();
88
+ /** Returned when a level was already compacted within its current time bucket. */
89
+ export class ErrCompactionTooEarlyError extends SqliteJsStreamError {
90
+ constructor() {
91
+ super('ERR_COMPACTION_TOO_EARLY', 'compaction scheduled too early');
92
+ this.name = 'ErrCompactionTooEarlyError';
93
+ }
94
+ }
95
+ export const ErrCompactionTooEarly = new ErrCompactionTooEarlyError();
96
+ /** Raised when a restore/plan cannot find LTX files at or below the target. */
97
+ export class NoMatchingBackupError extends SqliteJsStreamError {
98
+ constructor(message = 'no matching backup files available') {
99
+ super('ERR_NO_MATCHING_BACKUP', message);
100
+ this.name = 'NoMatchingBackupError';
101
+ }
102
+ }
103
+ /** Constant intervals for compaction levels beyond the 4-level default. */
104
+ const HIGHER_LEVEL_INTERVAL_MS = 3_600_000;
105
+ /** Default intervals for levels 1..3 (matches litestream's defaults). */
106
+ const DEFAULT_LEVEL_INTERVALS = [30_000, 300_000, 3_600_000];
107
+ /** Builds a CompactionLevels covering levels 0..max with sane intervals. */
108
+ function compactionLevelsForMax(max) {
109
+ const arr = [];
110
+ for (let i = 0; i <= max; i++) {
111
+ const intervalMs = i === 0 ? 0 : (DEFAULT_LEVEL_INTERVALS[i - 1] ?? HIGHER_LEVEL_INTERVAL_MS);
112
+ arr.push({ level: i, intervalMs });
113
+ }
114
+ return new CompactionLevels(arr);
115
+ }
116
+ /** Replica replicates a single database (by path) to a ReplicaClient. */
117
+ export class Replica {
118
+ path;
119
+ client;
120
+ logger;
121
+ posValue = null;
122
+ db = null;
123
+ compactionLevelsValue = defaultCompactionLevels();
124
+ leaserValue = null;
125
+ leasePrimary = true;
126
+ leaseCurrent = null;
127
+ leaseTimer = null;
128
+ runActive = false;
129
+ constructor(path, client, logger = noopLogger, options = {}) {
130
+ this.path = path;
131
+ this.client = client;
132
+ this.logger = logger;
133
+ if (options.compactionLevels) {
134
+ this.compactionLevelsValue = options.compactionLevels;
135
+ }
136
+ else if (options.maxLevel !== undefined) {
137
+ this.maxCompactionLevel = options.maxLevel;
138
+ }
139
+ if (options.leaser) {
140
+ this.leaserValue = options.leaser;
141
+ // A leaser is not held until acquired, so a fresh replica starts as a
142
+ // follower until {@link run} (or an explicit acquire) makes it primary.
143
+ this.leasePrimary = false;
144
+ }
145
+ }
146
+ /** Leaser optionally bounds this replica to a single primary across hosts. */
147
+ get leaser() {
148
+ return this.leaserValue;
149
+ }
150
+ set leaser(leaser) {
151
+ this.leaserValue = leaser;
152
+ // Mirror the constructor: a newly set leaser is not held until acquired.
153
+ // While running, {@link run} owns the primary state.
154
+ if (!this.runActive)
155
+ this.leasePrimary = leaser === null;
156
+ }
157
+ /**
158
+ * IsPrimary reports whether this replica may currently write. Always true
159
+ * when no {@link Leaser} is configured; false while another host holds the
160
+ * lease. Updated by {@link run}.
161
+ */
162
+ isPrimary() {
163
+ return this.leasePrimary;
164
+ }
165
+ /** @internal Throws if a leaser is set but this replica does not hold it. */
166
+ assertCanWrite() {
167
+ if (this.leaserValue !== null && !this.leasePrimary) {
168
+ throw new ErrLeaseNotHeld();
169
+ }
170
+ }
171
+ /** Per-level compaction schedule. Defaults to {@link defaultCompactionLevels}. */
172
+ get compactionLevels() {
173
+ return this.compactionLevelsValue;
174
+ }
175
+ set compactionLevels(levels) {
176
+ this.compactionLevelsValue = levels;
177
+ }
178
+ /** Highest non-snapshot compaction level this replica will produce. */
179
+ get maxCompactionLevel() {
180
+ return this.compactionLevelsValue.maxLevel();
181
+ }
182
+ set maxCompactionLevel(level) {
183
+ this.compactionLevelsValue = compactionLevelsForMax(level);
184
+ }
185
+ /** Lazily opens the local database reader (cached). */
186
+ async localDb() {
187
+ if (this.db === null) {
188
+ this.db = await LocalDatabase.open(this.path);
189
+ }
190
+ return this.db;
191
+ }
192
+ /** Pos returns the current replicated position (max level-0 maxTXID). */
193
+ async pos() {
194
+ if (this.posValue === null) {
195
+ this.posValue = await this.calcPos();
196
+ }
197
+ return this.posValue;
198
+ }
199
+ /**
200
+ * CalcPos finds the maximum TXID on the replica across all levels. Since
201
+ * compaction moves the newest data to higher levels and prunes level-0, the
202
+ * position must be derived from every level (and any snapshot).
203
+ */
204
+ async calcPos() {
205
+ let max = 0n;
206
+ for (let level = 0; level <= SnapshotLevel; level++) {
207
+ for await (const info of this.client.ltxFiles({ level, seek: 0n, useMetadata: false })) {
208
+ if (info.maxTXID > max)
209
+ max = info.maxTXID;
210
+ }
211
+ }
212
+ return max;
213
+ }
214
+ /** LocalPos returns the local cached position (max local level-0 maxTXID). */
215
+ async localPos() {
216
+ return (await this.localDb()).pos();
217
+ }
218
+ /**
219
+ * Sync builds a level-0 LTX file locally (if there are new WAL changes) and
220
+ * uploads any pending local files to the client. Returns the file that was
221
+ * written locally, or null if there was nothing new to do.
222
+ */
223
+ async sync() {
224
+ this.assertCanWrite();
225
+ await this.client.init();
226
+ const db = await this.localDb();
227
+ const info = await db.sync();
228
+ const dbPos = await db.pos();
229
+ const remotePos = await this.pos();
230
+ for (let txID = remotePos + 1n; txID <= dbPos; txID = (await this.pos()) + 1n) {
231
+ await this.uploadLtxFile(0, txID, txID);
232
+ this.posValue = txID;
233
+ }
234
+ return info;
235
+ }
236
+ /** SyncAndWait uploads all pending LTX files; harmless if already synced. */
237
+ async syncAndWait() {
238
+ await this.sync();
239
+ }
240
+ /**
241
+ * Run continuously drives the replica until `signal` is aborted. It syncs on
242
+ * `intervalMs`, compacts per level on each level's wall-clock bucket, takes
243
+ * snapshots on `snapshotIntervalMs`, and enforces retention. Errors are logged
244
+ * and the loop keeps going, so a transient failure does not stop replication.
245
+ *
246
+ * Compaction mirrors litestream: each level is compacted at most once per
247
+ * bucket (see {@link CompactionLevels}); the legacy `compactionIntervalMs`
248
+ * cascades all levels on a fixed interval instead.
249
+ */
250
+ async run(opt = {}) {
251
+ const { intervalMs = 1000, signal } = opt;
252
+ if (opt.maxLevel !== undefined)
253
+ this.maxCompactionLevel = opt.maxLevel;
254
+ const levels = opt.compactionLevels ?? this.compactionLevels;
255
+ const useLegacyCompaction = (opt.compactionIntervalMs ?? 0) > 0;
256
+ const compactionIntervalMs = opt.compactionIntervalMs ?? 0;
257
+ const retentionIntervalMs = opt.retentionIntervalMs ?? 0;
258
+ const snapshotIntervalMs = opt.snapshotIntervalMs ?? 0;
259
+ const l0RetentionMs = opt.l0RetentionMs ?? 0;
260
+ const l0RetentionCheckIntervalMs = opt.l0RetentionCheckIntervalMs ?? DefaultL0RetentionCheckIntervalMs;
261
+ // Try to acquire the lease up front. If held by another host we become a
262
+ // follower and poll until it is available. Without a leaser we are primary.
263
+ this.runActive = true;
264
+ const leaser = this.leaserValue;
265
+ let primary = leaser === null;
266
+ if (leaser !== null) {
267
+ try {
268
+ this.leaseCurrent = await leaser.acquireLease();
269
+ primary = true;
270
+ this.scheduleLeaseRenewal();
271
+ }
272
+ catch (err) {
273
+ if (err instanceof LeaseExistsError) {
274
+ this.logger.debug('lease held elsewhere; running as follower', err);
275
+ primary = false;
276
+ }
277
+ else {
278
+ this.runActive = false;
279
+ throw err;
280
+ }
281
+ }
282
+ }
283
+ this.leasePrimary = primary;
284
+ let lastCompaction = 0;
285
+ let lastRetention = 0;
286
+ let snapshotNext = snapshotIntervalMs > 0 ? nextCompactionAt(Date.now(), snapshotIntervalMs) : Infinity;
287
+ let l0RetentionNext = l0RetentionMs > 0 ? Date.now() + l0RetentionCheckIntervalMs : Infinity;
288
+ // Per-level next-bucket times (litestream-style scheduling).
289
+ const levelNext = new Map();
290
+ if (!useLegacyCompaction) {
291
+ for (const lvl of levels.toArray()) {
292
+ if (lvl.level > 0 && lvl.intervalMs > 0) {
293
+ levelNext.set(lvl.level, nextCompactionAt(Date.now(), lvl.intervalMs));
294
+ }
295
+ }
296
+ }
297
+ while (!signal?.aborted) {
298
+ const now = Date.now();
299
+ // Follower management: poll to acquire the lease. The primary never
300
+ // renews here — a dedicated timer ({@link scheduleLeaseRenewal}) renews it
301
+ // independently of the loop cadence, so a large `intervalMs` cannot let the
302
+ // lease expire.
303
+ if (leaser !== null && !primary) {
304
+ try {
305
+ this.leaseCurrent = await leaser.acquireLease();
306
+ primary = true;
307
+ this.leasePrimary = true;
308
+ this.scheduleLeaseRenewal();
309
+ }
310
+ catch (err) {
311
+ if (!(err instanceof LeaseExistsError)) {
312
+ this.logger.error('lease acquire error', err);
313
+ }
314
+ await sleep(intervalMs, undefined, { signal });
315
+ continue;
316
+ }
317
+ }
318
+ // Only the primary (or a leaser-less replica) performs writes.
319
+ if (leaser === null || primary) {
320
+ try {
321
+ await this.sync();
322
+ }
323
+ catch (err) {
324
+ this.logger.error('replica sync error', err);
325
+ }
326
+ }
327
+ if (signal?.aborted)
328
+ break;
329
+ if (leaser !== null && !primary) {
330
+ await sleep(intervalMs, undefined, { signal });
331
+ continue;
332
+ }
333
+ // Compaction.
334
+ if (useLegacyCompaction) {
335
+ if (compactionIntervalMs > 0 && now - lastCompaction >= compactionIntervalMs) {
336
+ try {
337
+ await this.compact();
338
+ }
339
+ catch (err) {
340
+ this.logger.error('replica compaction error', err);
341
+ }
342
+ lastCompaction = Date.now();
343
+ }
344
+ }
345
+ else {
346
+ for (const [level, next] of levelNext) {
347
+ if (now >= next) {
348
+ try {
349
+ await this.compactDB(level, now);
350
+ }
351
+ catch (err) {
352
+ if (err !== ErrNoCompaction && err !== ErrCompactionTooEarly) {
353
+ this.logger.error('replica compaction error', err);
354
+ }
355
+ }
356
+ levelNext.set(level, nextCompactionAt(now, levels.level(level).intervalMs));
357
+ }
358
+ }
359
+ }
360
+ // Snapshot + snapshot retention.
361
+ if (snapshotIntervalMs > 0 && now >= snapshotNext) {
362
+ try {
363
+ await this.compactSnapshot(now, snapshotIntervalMs);
364
+ const retention = opt.snapshotRetentionMs ?? 0;
365
+ if (retention > 0 || (opt.snapshotRetain ?? 0) > 0) {
366
+ await this.enforceSnapshotRetention(now, retention, opt.snapshotRetain ?? 0);
367
+ }
368
+ }
369
+ catch (err) {
370
+ this.logger.error('replica snapshot error', err);
371
+ }
372
+ snapshotNext = nextCompactionAt(now, snapshotIntervalMs);
373
+ }
374
+ // Retention (legacy interval-driven or L0 time-based).
375
+ if (retentionIntervalMs > 0 && now - lastRetention >= retentionIntervalMs) {
376
+ try {
377
+ await this.enforceRetention(opt.retention ?? {});
378
+ }
379
+ catch (err) {
380
+ this.logger.error('replica retention error', err);
381
+ }
382
+ lastRetention = Date.now();
383
+ }
384
+ if (l0RetentionMs > 0 && now >= l0RetentionNext) {
385
+ try {
386
+ await this.enforceL0RetentionByTime(now, l0RetentionMs);
387
+ }
388
+ catch (err) {
389
+ this.logger.error('replica l0 retention error', err);
390
+ }
391
+ l0RetentionNext = now + l0RetentionCheckIntervalMs;
392
+ }
393
+ if (signal?.aborted)
394
+ break;
395
+ try {
396
+ await sleep(intervalMs, undefined, { signal });
397
+ }
398
+ catch {
399
+ break; // aborted during the sleep
400
+ }
401
+ }
402
+ // Stop the renewal timer and release the lease so another host can take
403
+ // over promptly.
404
+ this.stopLeaseRenewal();
405
+ if (leaser !== null && this.leaseCurrent !== null) {
406
+ try {
407
+ await leaser.releaseLease(this.leaseCurrent);
408
+ }
409
+ catch (err) {
410
+ this.logger.debug('lease release error', err);
411
+ }
412
+ this.leaseCurrent = null;
413
+ this.leasePrimary = false;
414
+ }
415
+ this.runActive = false;
416
+ }
417
+ /**
418
+ * Schedules the next lease renewal to fire just before expiry, independent of
419
+ * the run-loop cadence. Reschedules after every successful renewal.
420
+ */
421
+ scheduleLeaseRenewal() {
422
+ this.stopLeaseRenewal();
423
+ if (this.leaserValue === null || this.leaseCurrent === null)
424
+ return;
425
+ const delay = Math.max(100, this.leaseCurrent.expiresAt - DefaultLeaseRenewalBuffer - Date.now());
426
+ this.leaseTimer = setTimeout(() => {
427
+ this.leaseTimer = null;
428
+ void this.renewLeaseLoop();
429
+ }, delay);
430
+ }
431
+ /** Renews the lease and reschedules; reverts to follower if we lost it. */
432
+ async renewLeaseLoop() {
433
+ if (!this.runActive || this.leaserValue === null || this.leaseCurrent === null)
434
+ return;
435
+ try {
436
+ this.leaseCurrent = await this.leaserValue.renewLease(this.leaseCurrent);
437
+ this.leasePrimary = true;
438
+ this.scheduleLeaseRenewal();
439
+ }
440
+ catch (err) {
441
+ if (err instanceof ErrLeaseNotHeld) {
442
+ this.logger.debug('lost lease; reverting to follower', err);
443
+ this.leasePrimary = false;
444
+ this.leaseCurrent = null;
445
+ this.stopLeaseRenewal();
446
+ }
447
+ else {
448
+ // Transient error: retry shortly rather than letting the lease expire.
449
+ this.logger.error('lease renew error', err);
450
+ this.scheduleLeaseRenewal();
451
+ }
452
+ }
453
+ }
454
+ /** Cancels any pending lease renewal. */
455
+ stopLeaseRenewal() {
456
+ if (this.leaseTimer !== null) {
457
+ clearTimeout(this.leaseTimer);
458
+ this.leaseTimer = null;
459
+ }
460
+ }
461
+ /** Snapshot writes and uploads a level-9 full-database LTX file. */
462
+ async snapshot() {
463
+ this.assertCanWrite();
464
+ const info = await (await this.localDb()).snapshot();
465
+ if (info) {
466
+ await this.uploadLtxFile(9, info.minTXID, info.maxTXID);
467
+ }
468
+ return info;
469
+ }
470
+ /** SyncStatus reports the local and remote positions. */
471
+ async syncStatus() {
472
+ const localTXID = await this.localPos();
473
+ const remoteTXID = await this.calcPos();
474
+ return { localTXID, remoteTXID, inSync: localTXID === remoteTXID };
475
+ }
476
+ /** Uploads a single local LTX file to the client. */
477
+ async uploadLtxFile(level, minTXID, maxTXID) {
478
+ const p = (await this.localDb()).ltxPath(level, minTXID, maxTXID);
479
+ // Stream the local LTX file straight to the client (no whole-file buffer).
480
+ const body = createReadStream(p);
481
+ const info = await this.client.writeLtxFile({ level, minTXID, maxTXID }, body);
482
+ this.logger.debug('ltx file uploaded', { level, minTXID, maxTXID, size: info.size });
483
+ return info;
484
+ }
485
+ /** EnsureExists restores the local database if it is missing. */
486
+ async ensureExists() {
487
+ try {
488
+ await access(this.path);
489
+ return;
490
+ }
491
+ catch {
492
+ /* missing; restore below */
493
+ }
494
+ await this.restore({ outputPath: this.path });
495
+ }
496
+ /** Restore reconstructs a SQLite database at `outputPath` from LTX files. */
497
+ async restore(opt) {
498
+ if (!opt.outputPath)
499
+ throw new InvalidArgumentError('output path required');
500
+ const plan = await calcRestorePlan(this.client, opt.txID ?? 0n);
501
+ if (plan.length === 0)
502
+ throw new NoMatchingBackupError();
503
+ await mkdir(nodePath.dirname(opt.outputPath), { recursive: true });
504
+ const tmp = `${opt.outputPath}.tmp`;
505
+ let fh = null;
506
+ const sources = [];
507
+ try {
508
+ fh = await open(tmp, 'w');
509
+ // The first plan file is the base (a snapshot). Stream its pages
510
+ // sequentially into the output, then overlay each later file's pages by
511
+ // offset, so a later version of a page wins. Peak memory stays bounded by
512
+ // one page buffer + the (small) page index, never the whole DB.
513
+ const openReader = (info) => this.openStreamReader(info, sources);
514
+ const base = await openReader(plan[0]);
515
+ await base.readHeader();
516
+ const pageSize = base.Header().pageSize;
517
+ let commit = base.Header().commit;
518
+ await writeSnapshotToFile(fh, base, pageSize, commit);
519
+ await base.verify();
520
+ for (let i = 1; i < plan.length; i++) {
521
+ const reader = await openReader(plan[i]);
522
+ await reader.readHeader();
523
+ const hdr = reader.Header();
524
+ if (hdr.pageSize !== pageSize) {
525
+ throw new PageSizeMismatchError(`mismatched page sizes: ${pageSize} != ${hdr.pageSize}`);
526
+ }
527
+ commit = hdr.commit;
528
+ await overlayPagesToFile(fh, reader, pageSize);
529
+ await reader.verify();
530
+ }
531
+ // The final database size is the last file's committed page count.
532
+ await fh.truncate(commit * pageSize);
533
+ await fh.sync();
534
+ await fh.close();
535
+ fh = null;
536
+ await rename(tmp, opt.outputPath);
537
+ }
538
+ catch (err) {
539
+ if (fh !== null)
540
+ await fh.close().catch(() => { });
541
+ await rm(tmp, { force: true }).catch(() => { });
542
+ throw err;
543
+ }
544
+ finally {
545
+ // Release any not-yet-drained input streams (avoids leaking fds/connections
546
+ // on a corrupt/truncated backup). Destroying an ended stream is a no-op.
547
+ for (const s of sources) {
548
+ try {
549
+ s.destroy?.();
550
+ }
551
+ catch {
552
+ /* ignore */
553
+ }
554
+ }
555
+ }
556
+ }
557
+ /**
558
+ * Follow continuously restores new LTX files into {@link FollowOptions.outputPath},
559
+ * keeping a read-only replica of the database up to date with the primary.
560
+ *
561
+ * On the first call (no output DB yet, or the replica is empty) it restores the
562
+ * latest backup as the base, then polls (every {@link FollowOptions.intervalMs})
563
+ * and applies any newly replicated LTX files as they appear. It records the
564
+ * applied TXID in a `-txid` sidecar after each batch so a later `follow()` can
565
+ * resume, and the loop runs until {@link FollowOptions.signal} is aborted.
566
+ *
567
+ * The restored database must only be opened **read-only**; writing to it while
568
+ * follow runs will conflict with the pages we overlay.
569
+ */
570
+ async follow(opt) {
571
+ const { outputPath, signal } = opt;
572
+ if (!outputPath)
573
+ throw new InvalidArgumentError('output path required');
574
+ const intervalMs = opt.intervalMs ?? 1000;
575
+ const txidPath = opt.txidFile ?? `${outputPath}-txid`;
576
+ if (intervalMs < 1)
577
+ throw new InvalidArgumentError('intervalMs must be >= 1');
578
+ // Determine the resume point. We must know the last applied TXID so we only
579
+ // apply files that come strictly after it. If the DB is missing we do a
580
+ // fresh latest restore; if it exists we require the -txid sidecar to know
581
+ // where to resume (otherwise we cannot prove the file matches the replica).
582
+ let currentMax;
583
+ let pageSize;
584
+ const dbExists = await statSafe(outputPath);
585
+ const txidExists = await statSafe(txidPath);
586
+ if (!dbExists) {
587
+ const plan = await this.waitForInitialRestore(outputPath, intervalMs, signal);
588
+ currentMax = plan[plan.length - 1].maxTXID;
589
+ await writeTxidFile(txidPath, currentMax);
590
+ pageSize = await readDbPageSize(outputPath);
591
+ }
592
+ else {
593
+ if (!txidExists) {
594
+ throw new FollowMissingTxidError(`database ${outputPath} exists but ${txidPath} is missing; ` +
595
+ 'delete the database and re-run follow');
596
+ }
597
+ const saved = await readTxidFile(txidPath);
598
+ if (saved === null)
599
+ throw new FollowInvalidTxidError(`invalid TXID in ${txidPath}`);
600
+ currentMax = saved;
601
+ pageSize = await readDbPageSize(outputPath);
602
+ await this.validateResumePoint(currentMax, outputPath, txidPath);
603
+ }
604
+ // Poll and apply new files until aborted. Batches are applied atomically-ish:
605
+ // overlay pages, truncate, fsync, THEN advance the -txid sidecar. If we crash
606
+ // mid-batch the sidecar is stale and the next run re-applies the same files
607
+ // (LTX files are immutable and page content is deterministic, so this is safe).
608
+ while (!signal?.aborted) {
609
+ const batch = await this.collectFollowBatch(currentMax);
610
+ if (batch.length > 0) {
611
+ currentMax = await this.applyFollowBatch(outputPath, batch, currentMax, pageSize);
612
+ await writeTxidFile(txidPath, currentMax);
613
+ }
614
+ if (signal?.aborted)
615
+ break;
616
+ try {
617
+ await sleep(intervalMs, undefined, { signal });
618
+ }
619
+ catch {
620
+ break; // aborted during the sleep
621
+ }
622
+ }
623
+ }
624
+ /**
625
+ * Guards a follow() resume against an inconsistent state: if the saved TXID is
626
+ * ahead of every TXID the replica holds, the DB/sidecar cannot be trusted and
627
+ * must be re-restored. (A saved TXID below the retained window is caught later
628
+ * as a data gap in {@link collectFollowBatch}.)
629
+ */
630
+ async validateResumePoint(saved, outputPath, txidPath) {
631
+ let latestMax = 0n;
632
+ for (let level = 0; level <= SnapshotLevel; level++) {
633
+ for await (const info of this.client.ltxFiles({ level, seek: 0n, useMetadata: false })) {
634
+ if (info.maxTXID > latestMax)
635
+ latestMax = info.maxTXID;
636
+ }
637
+ }
638
+ if (latestMax !== 0n && saved > latestMax) {
639
+ throw new FollowAheadOfLatestError(`saved TXID ${hexTxid(saved)} is ahead of the latest snapshot ` +
640
+ `(${hexTxid(latestMax)}); delete ${outputPath} and ${txidPath} then re-restore`);
641
+ }
642
+ }
643
+ /**
644
+ * Waits for the first backup to appear on an empty replica, then restores it
645
+ * as the follow base. Returns the plan that was applied.
646
+ */
647
+ async waitForInitialRestore(outputPath, intervalMs, signal) {
648
+ for (;;) {
649
+ if (signal?.aborted)
650
+ throw new FollowAbortedError('aborted before the initial restore');
651
+ try {
652
+ const plan = await calcRestorePlan(this.client, 0n);
653
+ await this.restore({ outputPath });
654
+ return plan;
655
+ }
656
+ catch (err) {
657
+ if (signal?.aborted)
658
+ throw err;
659
+ // An empty replica is the only case we retry; anything else is a real error.
660
+ if (!(err instanceof NoMatchingBackupError)) {
661
+ throw err;
662
+ }
663
+ await sleep(intervalMs, undefined, { signal });
664
+ }
665
+ }
666
+ }
667
+ /**
668
+ * Returns the contiguous run of LTX files (across all compaction levels) that
669
+ * extend the applied position `currentMax`, in application order. Prefers the
670
+ * highest (most compacted) level for a range and stops at the first gap. Throws
671
+ * if the very next file (currentMax + 1) is missing, e.g. retention pruned it.
672
+ */
673
+ async collectFollowBatch(currentMax) {
674
+ const candidates = [];
675
+ for (let level = 0; level <= SnapshotLevel; level++) {
676
+ for await (const info of this.client.ltxFiles({ level, seek: 0n, useMetadata: false })) {
677
+ if (info.maxTXID <= currentMax)
678
+ continue;
679
+ candidates.push(info);
680
+ }
681
+ }
682
+ candidates.sort((a, b) => a.minTXID !== b.minTXID ? (a.minTXID < b.minTXID ? -1 : 1) : b.level - a.level);
683
+ const batch = [];
684
+ let cursor = currentMax;
685
+ const seen = [];
686
+ for (const info of candidates) {
687
+ if (info.maxTXID <= cursor)
688
+ continue;
689
+ if (info.minTXID > cursor + 1n) {
690
+ if (batch.length === 0) {
691
+ throw new FollowDataGapError(`saved TXID ${hexTxid(currentMax)} is older than the earliest ` +
692
+ 'retained snapshot; retention has pruned the required history; delete the ' +
693
+ 'database and re-restore');
694
+ }
695
+ break;
696
+ }
697
+ if (seen.some(([lo, hi]) => lo <= info.minTXID && info.maxTXID <= hi))
698
+ continue;
699
+ batch.push(info);
700
+ cursor = info.maxTXID;
701
+ seen.push([info.minTXID, info.maxTXID]);
702
+ }
703
+ return batch;
704
+ }
705
+ /**
706
+ * Applies a contiguous batch of LTX files to the existing `outputPath` DB by
707
+ * overlaying each file's pages (a later page wins) and truncating to the final
708
+ * commit size. Returns the TXID now applied.
709
+ */
710
+ async applyFollowBatch(outputPath, batch, currentMax, pageSize) {
711
+ const sources = [];
712
+ let fh = null;
713
+ let commit = 0;
714
+ let appliedMax = currentMax;
715
+ try {
716
+ fh = await open(outputPath, 'r+');
717
+ for (const info of batch) {
718
+ const reader = await this.openStreamReader(info, sources);
719
+ await reader.readHeader();
720
+ const hdr = reader.Header();
721
+ if (hdr.pageSize !== pageSize) {
722
+ throw new PageSizeMismatchError(`mismatched page sizes: ${pageSize} != ${hdr.pageSize}`);
723
+ }
724
+ commit = hdr.commit;
725
+ await overlayPagesToFile(fh, reader, pageSize);
726
+ await reader.verify();
727
+ appliedMax = info.maxTXID;
728
+ }
729
+ await fh.truncate(commit * pageSize);
730
+ await fh.sync();
731
+ await fh.close();
732
+ fh = null;
733
+ }
734
+ catch (err) {
735
+ if (fh !== null)
736
+ await fh.close().catch(() => { });
737
+ throw err;
738
+ }
739
+ finally {
740
+ for (const s of sources) {
741
+ try {
742
+ s.destroy?.();
743
+ }
744
+ catch {
745
+ /* ignore */
746
+ }
747
+ }
748
+ }
749
+ return appliedMax;
750
+ }
751
+ /** Opens a streaming LTX reader over a plan file from the client. */
752
+ async openStreamReader(info, sources) {
753
+ const rc = await this.client.openLtxFile({
754
+ level: info.level,
755
+ minTXID: info.minTXID,
756
+ maxTXID: info.maxTXID,
757
+ offset: 0,
758
+ size: 0,
759
+ });
760
+ if (sources && typeof rc.destroy === 'function') {
761
+ sources.push({ destroy: () => rc.destroy() });
762
+ }
763
+ return new StreamLtxReader(rc);
764
+ }
765
+ /** Close releases any internal database connection. */
766
+ close() {
767
+ if (this.db !== null) {
768
+ this.db.close();
769
+ this.db = null;
770
+ }
771
+ }
772
+ /**
773
+ * Compact merges source LTX files into the next level. If `dstLevel` is
774
+ * given, compacts that level; otherwise compacts the first level (1..max)
775
+ * that has pending source files. Returns the written file, or null if there
776
+ * was nothing to compact.
777
+ */
778
+ async compact(options = {}) {
779
+ this.assertCanWrite();
780
+ if (options.dstLevel !== undefined) {
781
+ if (options.dstLevel < 1 || options.dstLevel > this.maxCompactionLevel)
782
+ return null;
783
+ return this.compactLevel(options.dstLevel);
784
+ }
785
+ for (let dst = 1; dst <= this.maxCompactionLevel; dst++) {
786
+ const info = await this.compactLevel(dst);
787
+ if (info)
788
+ return info;
789
+ }
790
+ return null;
791
+ }
792
+ /**
793
+ * CompactLevel compacts pending files at `dstLevel - 1` into a single file at
794
+ * `dstLevel`. It streams readers (local cache preferred, client fallback),
795
+ * writes the result locally via a file-backed sink, uploads it to the client,
796
+ * and returns the new file's metadata. Source files are left in place; GC is
797
+ * performed by {@link enforceRetention}.
798
+ */
799
+ async compactLevel(dstLevel) {
800
+ this.assertCanWrite();
801
+ if (dstLevel < 1 || dstLevel > this.maxCompactionLevel)
802
+ return null;
803
+ const srcLevel = dstLevel - 1;
804
+ // Determine the max TXID already present at the destination level; only
805
+ // source files after it remain to be compacted.
806
+ let dstMax = 0n;
807
+ for await (const info of this.client.ltxFiles({ level: dstLevel, seek: 0n, useMetadata: false })) {
808
+ if (info.maxTXID > dstMax)
809
+ dstMax = info.maxTXID;
810
+ }
811
+ const seek = dstMax + 1n;
812
+ const sources = [];
813
+ for await (const info of this.client.ltxFiles({ level: srcLevel, seek, useMetadata: false })) {
814
+ sources.push(info);
815
+ }
816
+ if (sources.length === 0)
817
+ return null;
818
+ let minTXID = sources[0].minTXID;
819
+ let maxTXID = sources[0].maxTXID;
820
+ for (const info of sources) {
821
+ if (info.minTXID < minTXID)
822
+ minTXID = info.minTXID;
823
+ if (info.maxTXID > maxTXID)
824
+ maxTXID = info.maxTXID;
825
+ }
826
+ if (minTXID < seek) {
827
+ throw new LTXFormatError(`compaction source ${minTXID}..${maxTXID} overlaps the destination max ${dstMax}`);
828
+ }
829
+ const cleanup = [];
830
+ const readers = [];
831
+ const db = await this.localDb();
832
+ const outPath = db.ltxPath(dstLevel, minTXID, maxTXID);
833
+ const tmpPath = `${outPath}.tmp`;
834
+ await mkdir(nodePath.dirname(outPath), { recursive: true });
835
+ const fh = await open(tmpPath, 'w');
836
+ try {
837
+ for (const info of sources) {
838
+ readers.push(await this.openSourceReader(info, cleanup));
839
+ }
840
+ const sink = new FileSink(fh);
841
+ const compactor = new StreamingCompactor(sink, readers);
842
+ await compactor.compact();
843
+ const cHeader = compactor.Header();
844
+ if (cHeader.pageSize !== (await this.localDb()).pageSize) {
845
+ throw new PageSizeMismatchError(`compacted page size ${cHeader.pageSize} does not match database ${(await this.localDb()).pageSize}`);
846
+ }
847
+ await sink.flush();
848
+ await fh.close();
849
+ await rename(tmpPath, outPath);
850
+ const size = (await stat(outPath)).size;
851
+ await this.uploadLtxFile(dstLevel, minTXID, maxTXID);
852
+ return { level: dstLevel, minTXID, maxTXID, size, createdAt: Date.now() };
853
+ }
854
+ catch (err) {
855
+ await fh.close().catch(() => { });
856
+ await rm(tmpPath, { force: true }).catch(() => { });
857
+ throw err;
858
+ }
859
+ finally {
860
+ for (const s of cleanup) {
861
+ try {
862
+ s.destroy?.();
863
+ }
864
+ catch {
865
+ /* ignore */
866
+ }
867
+ }
868
+ }
869
+ }
870
+ /**
871
+ * CompactDB compacts `dstLevel` if it is time to do so and there is pending
872
+ * source data. Mirrors litestream's `Store.CompactDB`: it refuses to re-compact
873
+ * within the level's current time bucket (throws {@link ErrCompactionTooEarly})
874
+ * and skips when there is nothing to do (throws {@link ErrNoCompaction}).
875
+ */
876
+ async compactDB(dstLevel, now) {
877
+ this.assertCanWrite();
878
+ if (dstLevel === SnapshotLevel)
879
+ return this.compactSnapshot(now, 0);
880
+ const lvl = this.compactionLevels.level(dstLevel);
881
+ const prevAt = prevCompactionAt(now, lvl.intervalMs);
882
+ const dstInfo = await this.maxLTXFileInfo(dstLevel);
883
+ if (dstInfo && dstInfo.createdAt > prevAt)
884
+ throw ErrCompactionTooEarly;
885
+ const srcLevel = this.compactionLevels.prevLevel(dstLevel);
886
+ const srcInfo = await this.maxLTXFileInfo(srcLevel);
887
+ if (srcInfo && dstInfo && srcInfo.maxTXID <= dstInfo.minTXID)
888
+ throw ErrNoCompaction;
889
+ const info = await this.compactLevel(dstLevel);
890
+ if (info === null)
891
+ throw ErrNoCompaction;
892
+ return info;
893
+ }
894
+ /**
895
+ * CompactSnapshot takes a level-9 snapshot if it is time to do so and the
896
+ * database has advanced past the current snapshot. Mirrors `Store.CompactDB`
897
+ * for the snapshot level.
898
+ */
899
+ async compactSnapshot(now, intervalMs) {
900
+ this.assertCanWrite();
901
+ const prevAt = prevCompactionAt(now, intervalMs);
902
+ const dstInfo = await this.maxLTXFileInfo(SnapshotLevel);
903
+ if (dstInfo && dstInfo.createdAt > prevAt)
904
+ throw ErrCompactionTooEarly;
905
+ const pos = await (await this.localDb()).pos();
906
+ if (dstInfo && dstInfo.maxTXID !== 0n && dstInfo.maxTXID >= pos)
907
+ throw ErrNoCompaction;
908
+ return this.snapshot();
909
+ }
910
+ /** MaxLTXFileInfo returns the highest-TXID file at a level, or null. */
911
+ async maxLTXFileInfo(level) {
912
+ let max = null;
913
+ for await (const info of this.client.ltxFiles({ level, seek: 0n, useMetadata: false })) {
914
+ if (max === null || info.maxTXID > max.maxTXID)
915
+ max = info;
916
+ }
917
+ return max;
918
+ }
919
+ /**
920
+ * EnforceRetention prunes old snapshots and superseded files.
921
+ *
922
+ * 1. Snapshots older than `snapshotRetentionMs` are deleted (the newest is
923
+ * always kept; `snapshotRetain` may cap how many newest are kept).
924
+ * 2. The resulting snapshot boundary is cascaded down through levels 0..max:
925
+ * any file whose maxTXID is below the oldest retained snapshot's maxTXID
926
+ * is deleted (it is superseded by that snapshot).
927
+ * 3. Level-0 files that have been compacted into a higher level and are older
928
+ * than `l0RetentionMs` are deleted (except the newest, the live head).
929
+ */
930
+ async enforceRetention(options = {}) {
931
+ this.assertCanWrite();
932
+ const now = Date.now();
933
+ const snapshotRetentionMs = options.snapshotRetentionMs ?? 0;
934
+ const snapshotRetain = options.snapshotRetain ?? 0;
935
+ const l0RetentionMs = options.l0RetentionMs ?? 0;
936
+ const { deleted: snapshotsDeleted, boundary } = await this.enforceSnapshotRetention(now, snapshotRetentionMs, snapshotRetain);
937
+ let filesDeleted = 0;
938
+ if (boundary > 0n) {
939
+ for (let level = 0; level <= this.maxCompactionLevel; level++) {
940
+ filesDeleted += await this.enforceRetentionByTXID(level, boundary);
941
+ }
942
+ }
943
+ filesDeleted += await this.enforceL0RetentionByTime(now, l0RetentionMs);
944
+ return { snapshotsDeleted, filesDeleted, boundary };
945
+ }
946
+ /**
947
+ * EnforceSnapshotRetention deletes snapshots older than `retentionMs` and
948
+ * returns the boundary (max maxTXID of the deleted snapshots). The newest
949
+ * snapshot is always kept; `retain` may keep additional newest snapshots.
950
+ */
951
+ async enforceSnapshotRetention(now, retentionMs, retain = 0) {
952
+ this.assertCanWrite();
953
+ const snapshots = await this.listRemoteLevel(SnapshotLevel);
954
+ snapshots.sort((a, b) => (a.maxTXID < b.maxTXID ? -1 : a.maxTXID > b.maxTXID ? 1 : 0));
955
+ if (snapshots.length === 0)
956
+ return { deleted: 0, boundary: 0n };
957
+ const cutoff = retentionMs > 0 ? now - retentionMs : 0;
958
+ let retained = snapshots.filter((s) => !(retentionMs > 0 && s.createdAt < cutoff));
959
+ const newest = snapshots[snapshots.length - 1];
960
+ if (!retained.includes(newest))
961
+ retained.push(newest);
962
+ retained.sort((a, b) => (a.maxTXID < b.maxTXID ? -1 : a.maxTXID > b.maxTXID ? 1 : 0));
963
+ if (retain > 0 && retained.length > retain)
964
+ retained = retained.slice(retained.length - retain);
965
+ const retainedSet = new Set(retained);
966
+ const deletedFiles = snapshots.filter((s) => !retainedSet.has(s));
967
+ await this.deleteFiles(deletedFiles);
968
+ let boundary = 0n;
969
+ for (const d of deletedFiles)
970
+ if (d.maxTXID > boundary)
971
+ boundary = d.maxTXID;
972
+ return { deleted: deletedFiles.length, boundary };
973
+ }
974
+ /**
975
+ * EnforceRetentionByTXID deletes files at `level` with maxTXID below `txID`,
976
+ * always keeping at least one file. Returns the number deleted.
977
+ */
978
+ async enforceRetentionByTXID(level, txID) {
979
+ this.assertCanWrite();
980
+ if (txID <= 0n)
981
+ return 0;
982
+ const files = await this.listRemoteLevel(level);
983
+ files.sort((a, b) => (a.maxTXID < b.maxTXID ? -1 : a.maxTXID > b.maxTXID ? 1 : 0));
984
+ if (files.length === 0)
985
+ return 0;
986
+ const last = files[files.length - 1];
987
+ const deletedFiles = files.filter((f) => f.maxTXID < txID);
988
+ if (deletedFiles.length > 0 && deletedFiles[deletedFiles.length - 1] === last) {
989
+ deletedFiles.pop(); // always keep at least one file
990
+ }
991
+ await this.deleteFiles(deletedFiles);
992
+ return deletedFiles.length;
993
+ }
994
+ /**
995
+ * EnforceL0RetentionByTime deletes level-0 files that have been compacted into
996
+ * a higher level and are older than `l0RetentionMs`. It never deletes the
997
+ * newest level-0 file (the live head) and preserves contiguous L0 coverage by
998
+ * stopping at the first file that is not old enough.
999
+ */
1000
+ async enforceL0RetentionByTime(now, l0RetentionMs) {
1001
+ this.assertCanWrite();
1002
+ const l0 = await this.listRemoteLevel(0);
1003
+ l0.sort((a, b) => (a.maxTXID < b.maxTXID ? -1 : a.maxTXID > b.maxTXID ? 1 : 0));
1004
+ if (l0.length === 0)
1005
+ return 0;
1006
+ const newest = l0[l0.length - 1];
1007
+ const coverers = [];
1008
+ for (let level = 1; level <= this.maxCompactionLevel; level++) {
1009
+ for await (const info of this.client.ltxFiles({ level, seek: 0n, useMetadata: false })) {
1010
+ coverers.push(info);
1011
+ }
1012
+ }
1013
+ const threshold = now - l0RetentionMs;
1014
+ const deletedFiles = [];
1015
+ for (const info of l0) {
1016
+ if (info === newest)
1017
+ continue; // never delete the live head
1018
+ if (info.createdAt > threshold)
1019
+ break; // preserve contiguous L0 coverage
1020
+ if (this.isCoveredByHigherLevel(info, coverers))
1021
+ deletedFiles.push(info);
1022
+ }
1023
+ await this.deleteFiles(deletedFiles);
1024
+ return deletedFiles.length;
1025
+ }
1026
+ /** Returns true if `info` is fully contained in a file at a higher level. */
1027
+ isCoveredByHigherLevel(info, coverers) {
1028
+ for (const g of coverers) {
1029
+ if (g.level <= info.level)
1030
+ continue;
1031
+ if (g.minTXID <= info.minTXID && info.maxTXID <= g.maxTXID)
1032
+ return true;
1033
+ }
1034
+ return false;
1035
+ }
1036
+ /** Lists LTX files at a remote level. */
1037
+ async listRemoteLevel(level) {
1038
+ const out = [];
1039
+ for await (const info of this.client.ltxFiles({ level, seek: 0n, useMetadata: false })) {
1040
+ out.push(info);
1041
+ }
1042
+ return out;
1043
+ }
1044
+ /** Deletes files from the client and the local cache. */
1045
+ async deleteFiles(files) {
1046
+ if (files.length === 0)
1047
+ return;
1048
+ this.logger.debug('deleting ltx files', { count: files.length });
1049
+ await this.client.deleteLtxFiles(files);
1050
+ const db = await this.localDb();
1051
+ for (const f of files) {
1052
+ const p = db.ltxPath(f.level, f.minTXID, f.maxTXID);
1053
+ await rm(p, { force: true }).catch(() => { });
1054
+ }
1055
+ }
1056
+ /**
1057
+ * Opens a streaming LTX reader for a source file, preferring the local cache
1058
+ * and falling back to the client. Collects cleanup callbacks for the streams.
1059
+ */
1060
+ async openSourceReader(info, cleanup) {
1061
+ const db = await this.localDb();
1062
+ const localPath = db.ltxPath(info.level, info.minTXID, info.maxTXID);
1063
+ try {
1064
+ await access(localPath);
1065
+ const s = createReadStream(localPath);
1066
+ cleanup.push({ destroy: () => s.destroy() });
1067
+ return new StreamLtxReader(s);
1068
+ }
1069
+ catch {
1070
+ /* fall back to the client */
1071
+ }
1072
+ const rc = await this.client.openLtxFile({
1073
+ level: info.level,
1074
+ minTXID: info.minTXID,
1075
+ maxTXID: info.maxTXID,
1076
+ offset: 0,
1077
+ size: 0,
1078
+ });
1079
+ if (typeof rc.destroy === 'function') {
1080
+ cleanup.push({ destroy: () => rc.destroy() });
1081
+ }
1082
+ return new StreamLtxReader(rc);
1083
+ }
1084
+ }
1085
+ /**
1086
+ * CalcRestorePlan returns LTX files to apply, in order, to reconstruct a
1087
+ * database up to `txID` (0 = latest). It prefers the latest level-9 snapshot
1088
+ * before the target as the base, then appends contiguous files across all
1089
+ * compaction levels (0..maxLevel) that extend the range.
1090
+ */
1091
+ export async function calcRestorePlan(client, txID) {
1092
+ const plan = [];
1093
+ let currentMax = 0n;
1094
+ // Pick the latest level-9 snapshot at or before the target as the base.
1095
+ let base = null;
1096
+ for await (const info of client.ltxFiles({ level: SnapshotLevel, seek: 0n, useMetadata: false })) {
1097
+ if (txID !== 0n && info.maxTXID > txID)
1098
+ continue;
1099
+ if (!base || info.maxTXID > base.maxTXID)
1100
+ base = info;
1101
+ }
1102
+ if (base) {
1103
+ plan.push(base);
1104
+ currentMax = base.maxTXID;
1105
+ }
1106
+ // Collect candidate files from every compaction level (0..SnapshotLevel-1).
1107
+ // Compaction moves data to higher levels, so a TXID range may be represented
1108
+ // by a file at any level; gather them all and prefer the highest level first.
1109
+ const candidates = [];
1110
+ for (let level = 0; level < SnapshotLevel; level++) {
1111
+ for await (const info of client.ltxFiles({ level, seek: 0n, useMetadata: false })) {
1112
+ if (txID !== 0n && info.maxTXID > txID)
1113
+ continue;
1114
+ candidates.push(info);
1115
+ }
1116
+ }
1117
+ // Order by minTXID, then by higher level (more compacted) first, so a file
1118
+ // at a higher level supersedes lower-level files spanning the same range.
1119
+ candidates.sort((a, b) => {
1120
+ if (a.minTXID !== b.minTXID)
1121
+ return a.minTXID < b.minTXID ? -1 : 1;
1122
+ return b.level - a.level;
1123
+ });
1124
+ const seenRanges = [];
1125
+ for (const info of candidates) {
1126
+ if (info.maxTXID <= currentMax)
1127
+ continue; // covered by the base or earlier
1128
+ if (info.minTXID > currentMax + 1n)
1129
+ break; // non-contiguous gap
1130
+ // Skip a candidate entirely covered by a higher-level file already in the plan.
1131
+ if (seenRanges.some(([lo, hi]) => lo <= info.minTXID && info.maxTXID <= hi))
1132
+ continue;
1133
+ plan.push(info);
1134
+ currentMax = info.maxTXID;
1135
+ seenRanges.push([info.minTXID, info.maxTXID]);
1136
+ }
1137
+ if (plan.length === 0) {
1138
+ throw new NoMatchingBackupError();
1139
+ }
1140
+ if (txID !== 0n && currentMax < txID) {
1141
+ throw new NoMatchingBackupError();
1142
+ }
1143
+ return plan;
1144
+ }
1145
+ /**
1146
+ * Streams a snapshot's pages (1..commit) into the output file. The lock page
1147
+ * is written as a zeroed page (SQLite requires it to exist but never uses it).
1148
+ */
1149
+ async function writeSnapshotToFile(fh, reader, pageSize, commit) {
1150
+ const lockPg = lockPgno(pageSize);
1151
+ const page = new Uint8Array(pageSize);
1152
+ const zero = new Uint8Array(pageSize);
1153
+ for (let pgno = 1; pgno <= commit; pgno++) {
1154
+ if (pgno === lockPg) {
1155
+ await writeAllAt(fh, zero, (pgno - 1) * pageSize);
1156
+ continue;
1157
+ }
1158
+ const hdr = await reader.readPage(page);
1159
+ if (hdr === null)
1160
+ throw new LTXFormatError(`unexpected end of pages at pgno ${pgno}`);
1161
+ if (hdr.pgno !== pgno) {
1162
+ throw new LTXFormatError(`unexpected pgno while decoding page: read ${hdr.pgno}, expected ${pgno}`);
1163
+ }
1164
+ await writeAllAt(fh, page, (pgno - 1) * pageSize);
1165
+ }
1166
+ // Expect one final read to be the end-of-pages marker.
1167
+ if ((await reader.readPage(page)) !== null) {
1168
+ throw new LTXFormatError('unexpected page after end of database');
1169
+ }
1170
+ }
1171
+ /** Overlays an incremental file's pages into the output file at their offsets. */
1172
+ async function overlayPagesToFile(fh, reader, pageSize) {
1173
+ const page = new Uint8Array(pageSize);
1174
+ for (;;) {
1175
+ const hdr = await reader.readPage(page);
1176
+ if (hdr === null)
1177
+ break;
1178
+ await writeAllAt(fh, page, (hdr.pgno - 1) * pageSize);
1179
+ }
1180
+ }
1181
+ /** Writes all of `bytes` to `handle` at an absolute `position`, looping on partial writes. */
1182
+ async function writeAllAt(handle, bytes, position) {
1183
+ let offset = 0;
1184
+ while (offset < bytes.length) {
1185
+ const { bytesWritten } = await handle.write(bytes, offset, bytes.length - offset, position + offset);
1186
+ offset += bytesWritten;
1187
+ }
1188
+ }
1189
+ /** Returns true if `path` exists. */
1190
+ async function statSafe(path) {
1191
+ try {
1192
+ await stat(path);
1193
+ return true;
1194
+ }
1195
+ catch {
1196
+ return false;
1197
+ }
1198
+ }
1199
+ /** Encodes a TXID as the 16-char lowercase hex used in LTX filenames. */
1200
+ function hexTxid(txid) {
1201
+ return txid.toString(16).padStart(16, '0');
1202
+ }
1203
+ /** Decodes a (possibly trimmed) 1..16 char hex TXID; null on malformed input. */
1204
+ function decodeTxid(s) {
1205
+ const t = s.trim();
1206
+ if (!/^[0-9a-fA-F]{1,16}$/.test(t))
1207
+ return null;
1208
+ try {
1209
+ return BigInt(`0x${t}`);
1210
+ }
1211
+ catch {
1212
+ return null;
1213
+ }
1214
+ }
1215
+ /** Reads the last applied TXID from a follow sidecar file, or null if absent/invalid. */
1216
+ async function readTxidFile(path) {
1217
+ try {
1218
+ return decodeTxid(await readFile(path, 'utf8'));
1219
+ }
1220
+ catch {
1221
+ return null;
1222
+ }
1223
+ }
1224
+ /** Atomically writes the applied TXID to a follow sidecar file (temp + rename). */
1225
+ async function writeTxidFile(path, txid) {
1226
+ const tmp = `${path}.tmp`;
1227
+ await mkdir(nodePath.dirname(path), { recursive: true });
1228
+ const fh = await open(tmp, 'w');
1229
+ try {
1230
+ await fh.writeFile(hexTxid(txid) + '\n');
1231
+ await fh.sync();
1232
+ }
1233
+ finally {
1234
+ await fh.close();
1235
+ }
1236
+ await rename(tmp, path);
1237
+ }
1238
+ /** Reads the SQLite page size from a database file header (offset 16, BE 16-bit). */
1239
+ async function readDbPageSize(path) {
1240
+ const fh = await open(path, 'r');
1241
+ try {
1242
+ const buf = Buffer.alloc(4);
1243
+ const { bytesRead } = await fh.read(buf, 0, 4, 16);
1244
+ if (bytesRead !== 4)
1245
+ throw new DatabaseError(`cannot read database header of ${path}`);
1246
+ const raw = buf.readUInt16BE(0);
1247
+ // SQLite encodes a 64 KiB page size as 1.
1248
+ return raw === 1 ? 65536 : raw;
1249
+ }
1250
+ finally {
1251
+ await fh.close();
1252
+ }
1253
+ }