bunqueue 2.8.42 → 2.8.44

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/README.md +3 -3
  2. package/dist/application/latencyTracker.js +3 -3
  3. package/dist/application/metricsExporter.d.ts +3 -1
  4. package/dist/application/metricsExporter.js +63 -16
  5. package/dist/application/prometheusOperationalMetrics.d.ts +31 -0
  6. package/dist/application/prometheusOperationalMetrics.js +53 -0
  7. package/dist/application/queueManager.d.ts +6 -0
  8. package/dist/application/queueManager.js +24 -1
  9. package/dist/application/types.d.ts +3 -0
  10. package/dist/application/types.js +1 -0
  11. package/dist/application/workerManager.d.ts +1 -0
  12. package/dist/application/workerManager.js +4 -1
  13. package/dist/cli/output.js +2 -1
  14. package/dist/config/resolve.d.ts +1 -0
  15. package/dist/config/resolve.js +10 -0
  16. package/dist/config/types.d.ts +6 -0
  17. package/dist/infrastructure/backup/backupTelemetry.d.ts +30 -0
  18. package/dist/infrastructure/backup/backupTelemetry.js +63 -0
  19. package/dist/infrastructure/backup/index.d.ts +1 -1
  20. package/dist/infrastructure/backup/index.js +1 -1
  21. package/dist/infrastructure/backup/s3Backup.d.ts +6 -2
  22. package/dist/infrastructure/backup/s3Backup.js +22 -9
  23. package/dist/infrastructure/backup/s3BackupConfig.d.ts +7 -0
  24. package/dist/infrastructure/backup/s3BackupConfig.js +5 -0
  25. package/dist/infrastructure/backup/s3BackupIo.d.ts +23 -0
  26. package/dist/infrastructure/backup/s3BackupIo.js +106 -0
  27. package/dist/infrastructure/backup/s3BackupOperations.d.ts +6 -15
  28. package/dist/infrastructure/backup/s3BackupOperations.js +139 -259
  29. package/dist/infrastructure/backup/sqliteBackupFiles.d.ts +22 -0
  30. package/dist/infrastructure/backup/sqliteBackupFiles.js +130 -0
  31. package/dist/infrastructure/cloud/statsRefresh.d.ts +1 -0
  32. package/dist/infrastructure/persistence/sqlite.js +6 -1
  33. package/dist/infrastructure/server/bootstrap.d.ts +2 -0
  34. package/dist/infrastructure/server/bootstrap.js +30 -1
  35. package/dist/infrastructure/server/http.d.ts +2 -0
  36. package/dist/infrastructure/server/http.js +9 -58
  37. package/dist/infrastructure/server/httpDashboardEndpoints.d.ts +10 -0
  38. package/dist/infrastructure/server/httpDashboardEndpoints.js +146 -0
  39. package/dist/infrastructure/server/httpEndpoints.d.ts +5 -9
  40. package/dist/infrastructure/server/httpEndpoints.js +16 -160
  41. package/dist/infrastructure/server/httpResponse.d.ts +2 -0
  42. package/dist/infrastructure/server/httpResponse.js +12 -0
  43. package/dist/infrastructure/server/httpRouter.d.ts +6 -0
  44. package/dist/infrastructure/server/httpRouter.js +50 -0
  45. package/dist/shared/histogram.d.ts +2 -2
  46. package/dist/shared/histogram.js +4 -4
  47. package/package.json +2 -2
@@ -1,334 +1,214 @@
1
1
  /**
2
- * S3 Backup Operations
3
- * Core backup, restore, list, and cleanup operations
2
+ * S3 backup, restore, listing, and retention operations.
4
3
  */
5
4
  import { backupLog } from '../../shared/logger';
6
5
  import { VERSION } from '../../shared/version';
7
- const DEFAULT_S3_TIMEOUT_MS = 30_000;
8
- /** Race a promise against a timeout; rejects with a descriptive error on timeout */
9
- function withTimeout(promise, ms, label) {
10
- return Promise.race([
11
- promise,
12
- new Promise((_, reject) => {
13
- setTimeout(() => {
14
- reject(new Error(`${label} timed out after ${ms}ms`));
15
- }, ms);
16
- }),
17
- ]);
6
+ import { createConsistentSnapshot, installDatabaseCandidate, removeDatabaseArtifacts, verifyDatabaseIntegrity, } from './sqliteBackupFiles';
7
+ import { cleanupFailedPayload, DEFAULT_S3_TIMEOUT_MS, gunzipAsync, gzipAsync, retryWithTimeout, sha256, } from './s3BackupIo';
8
+ function timeoutFor(config) {
9
+ return config.timeoutMs ?? DEFAULT_S3_TIMEOUT_MS;
18
10
  }
19
- /** Check if an error is transient and worth retrying */
20
- function isTransientError(error) {
21
- const message = error instanceof Error ? error.message : String(error);
22
- const lower = message.toLowerCase();
23
- return (lower.includes('connection reset') ||
24
- lower.includes('econnreset') ||
25
- lower.includes('timeout') ||
26
- lower.includes('etimedout') ||
27
- lower.includes('econnrefused') ||
28
- lower.includes('socket hang up') ||
29
- lower.includes('network') ||
30
- lower.includes('503') ||
31
- lower.includes('500') ||
32
- lower.includes('502') ||
33
- lower.includes('504') ||
34
- lower.includes('service unavailable') ||
35
- lower.includes('internal server error') ||
36
- lower.includes('bad gateway') ||
37
- lower.includes('gateway timeout') ||
38
- lower.includes('transient'));
11
+ function backupKey(prefix) {
12
+ const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
13
+ return `${prefix}bunqueue-${timestamp}-${crypto.randomUUID()}.db`;
39
14
  }
40
- /** Retry an async operation with exponential backoff (500ms, 1000ms, 2000ms) */
41
- async function withRetry(fn, label, maxRetries = 3, baseDelayMs = 500) {
42
- let lastError;
43
- for (let attempt = 0; attempt <= maxRetries; attempt++) {
44
- try {
45
- return await fn();
46
- }
47
- catch (error) {
48
- lastError = error;
49
- if (attempt < maxRetries && isTransientError(error)) {
50
- const delay = baseDelayMs * Math.pow(2, attempt);
51
- backupLog.warn(`${label} failed (attempt ${attempt + 1}/${maxRetries + 1}), retrying in ${delay}ms`, { error: error instanceof Error ? error.message : String(error) });
52
- await new Promise((resolve) => setTimeout(resolve, delay));
53
- }
54
- else {
55
- throw error;
56
- }
57
- }
15
+ function restoreCandidatePath(databasePath) {
16
+ return `${databasePath}.restore-${Date.now()}-${crypto.randomUUID()}.tmp`;
17
+ }
18
+ function isGzip(data) {
19
+ return data.byteLength >= 2 && data[0] === 0x1f && data[1] === 0x8b;
20
+ }
21
+ function validateMetadata(value, compressedSize) {
22
+ if (!value || typeof value !== 'object') {
23
+ throw new Error('Backup metadata is invalid');
24
+ }
25
+ const metadata = value;
26
+ if (typeof metadata.timestamp !== 'string' ||
27
+ typeof metadata.version !== 'string' ||
28
+ !Number.isSafeInteger(metadata.size) ||
29
+ (metadata.size ?? -1) < 0 ||
30
+ !Number.isSafeInteger(metadata.compressedSize) ||
31
+ metadata.compressedSize !== compressedSize ||
32
+ typeof metadata.checksum !== 'string' ||
33
+ !/^[a-f0-9]{64}$/i.test(metadata.checksum) ||
34
+ metadata.compressed !== true) {
35
+ throw new Error('Backup metadata is invalid or does not match the payload');
58
36
  }
59
- throw lastError;
37
+ return metadata;
60
38
  }
61
- /**
62
- * Verify database integrity via PRAGMA integrity_check.
63
- *
64
- * Validates the file at `databasePath` and throws on failure. This function
65
- * is intended to run against a TEMP file (never the live DB): on integrity
66
- * failure it deletes the file it inspected, so it must only ever be pointed
67
- * at a throwaway candidate, not the live database.
68
- */
69
- async function verifyDatabaseIntegrity(databasePath) {
70
- const { Database } = await import('bun:sqlite');
71
- let db = null;
39
+ async function cleanupLocalArtifacts(path, label) {
40
+ if (!path)
41
+ return;
72
42
  try {
73
- db = new Database(databasePath);
74
- const result = db.query('PRAGMA integrity_check').get();
75
- const status = result?.integrity_check ?? '';
76
- if (status !== 'ok') {
77
- throw new Error(`Database integrity check failed: ${status || 'unknown error'}`);
78
- }
43
+ await removeDatabaseArtifacts(path);
79
44
  }
80
45
  catch (error) {
81
- // Close before unlinking so the file handle is released.
82
- try {
83
- db?.close();
84
- }
85
- catch {
86
- /* already closed */
87
- }
88
- db = null;
89
- // Clean up the corrupt candidate file (this is a temp file, never the live DB).
90
- try {
91
- const { unlink } = await import('fs/promises');
92
- await unlink(databasePath);
93
- }
94
- catch {
95
- /* best effort */
96
- }
97
- if (error instanceof Error && error.message.includes('integrity check failed')) {
98
- throw error;
99
- }
100
- throw new Error('Database integrity check failed: corrupt or invalid database', {
101
- cause: error,
102
- });
46
+ backupLog.warn(`Failed to clean ${label} artifacts`, { path, error: String(error) });
103
47
  }
104
- finally {
105
- try {
106
- db?.close();
107
- }
108
- catch {
109
- /* already closed */
110
- }
111
- }
112
- }
113
- /** Async gzip compress using Web Streams API (non-blocking) */
114
- async function gzipAsync(data) {
115
- const stream = new Blob([data])
116
- .stream()
117
- .pipeThrough(new CompressionStream('gzip'));
118
- return new Uint8Array(await new Response(stream).arrayBuffer());
119
- }
120
- /** Async gzip decompress using Web Streams API (non-blocking) */
121
- async function gunzipAsync(data) {
122
- const stream = new Blob([data])
123
- .stream()
124
- .pipeThrough(new DecompressionStream('gzip'));
125
- return new Uint8Array(await new Response(stream).arrayBuffer());
126
48
  }
127
- /**
128
- * Perform a backup to S3
129
- */
49
+ /** Create and publish a consistent SQLite backup. */
130
50
  export async function performBackup(config, client) {
131
51
  const startTime = Date.now();
52
+ let snapshotPath = null;
53
+ let pendingPayloadKey = null;
132
54
  try {
133
- // Check if database file exists
134
- const dbFile = Bun.file(config.databasePath);
135
- const exists = await dbFile.exists();
136
- if (!exists) {
55
+ if (!(await Bun.file(config.databasePath).exists())) {
137
56
  throw new Error(`Database file not found: ${config.databasePath}`);
138
57
  }
139
- // Checkpoint WAL to ensure all data is in the main database file
140
- try {
141
- const { Database } = await import('bun:sqlite');
142
- const db = new Database(config.databasePath);
143
- db.run('PRAGMA wal_checkpoint(TRUNCATE)');
144
- db.close();
145
- }
146
- catch {
147
- // Ignore - database might be locked or not in WAL mode
148
- }
149
- // Generate backup key with timestamp
150
- const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
151
- const key = `${config.prefix}bunqueue-${timestamp}.db`;
152
- // Read database file
153
- const data = await Bun.file(config.databasePath).arrayBuffer();
154
- const originalSize = data.byteLength;
155
- // Compress with gzip for efficient storage
156
- const compressed = await gzipAsync(new Uint8Array(data));
157
- const compressedSize = compressed.byteLength;
158
- // Calculate checksum of original data (for integrity verification)
159
- const hasher = new Bun.CryptoHasher('sha256');
160
- hasher.update(new Uint8Array(data));
161
- const checksum = hasher.digest('hex');
162
- // Upload compressed backup to S3 (with retry and timeout)
163
- const timeoutMs = config.timeoutMs ?? DEFAULT_S3_TIMEOUT_MS;
164
- const s3File = client.file(key);
165
- await withTimeout(withRetry(() => s3File.write(compressed, { type: 'application/gzip' }), 'S3 backup upload'), timeoutMs, 'S3 backup upload');
166
- // Upload metadata (with retry for transient errors)
58
+ snapshotPath = await createConsistentSnapshot(config.databasePath);
59
+ const data = new Uint8Array(await Bun.file(snapshotPath).arrayBuffer());
60
+ const compressed = await gzipAsync(data);
61
+ const key = backupKey(config.prefix);
62
+ const metadataKey = `${key}.meta.json`;
63
+ const timeoutMs = timeoutFor(config);
167
64
  const metadata = {
168
65
  timestamp: new Date().toISOString(),
169
66
  version: VERSION,
170
- size: originalSize,
171
- compressedSize,
172
- checksum,
67
+ size: data.byteLength,
68
+ compressedSize: compressed.byteLength,
69
+ checksum: sha256(data),
173
70
  compressed: true,
174
71
  };
175
- const metadataKey = `${key}.meta.json`;
176
- await withTimeout(withRetry(() => client
72
+ // Metadata is published first. The data object is the visibility/commit
73
+ // point, so every successfully published current-format backup is paired.
74
+ await retryWithTimeout(() => client
177
75
  .file(metadataKey)
178
- .write(JSON.stringify(metadata, null, 2), { type: 'application/json' }), 'S3 metadata upload'), timeoutMs, 'S3 metadata upload');
179
- const duration = Date.now() - startTime;
180
- return { success: true, key, size: originalSize, duration };
76
+ .write(JSON.stringify(metadata, null, 2), { type: 'application/json' }), timeoutMs, 'S3 metadata upload');
77
+ pendingPayloadKey = key;
78
+ await retryWithTimeout(() => client.file(key).write(compressed, { type: 'application/gzip' }), timeoutMs, 'S3 backup upload');
79
+ pendingPayloadKey = null;
80
+ return {
81
+ success: true,
82
+ key,
83
+ size: data.byteLength,
84
+ compressedSize: compressed.byteLength,
85
+ duration: Date.now() - startTime,
86
+ };
181
87
  }
182
88
  catch (error) {
89
+ if (pendingPayloadKey) {
90
+ await cleanupFailedPayload(client, pendingPayloadKey, timeoutFor(config), error);
91
+ }
183
92
  const message = error instanceof Error ? error.message : String(error);
184
93
  backupLog.error('Backup failed', { error: message });
185
94
  return { success: false, error: message };
186
95
  }
96
+ finally {
97
+ await cleanupLocalArtifacts(snapshotPath, 'backup snapshot');
98
+ }
187
99
  }
188
- /**
189
- * List available backups
190
- */
100
+ /** List every backup object under the configured prefix. */
191
101
  export async function listBackups(config, client) {
192
102
  try {
193
- const allContents = [];
103
+ const contents = [];
104
+ const seenTokens = new Set();
105
+ const timeoutMs = timeoutFor(config);
194
106
  let continuationToken;
195
107
  do {
196
- const result = await client.list({
108
+ const result = await retryWithTimeout(() => client.list({
197
109
  prefix: config.prefix,
198
110
  maxKeys: 100,
199
111
  ...(continuationToken ? { continuationToken } : {}),
200
- });
201
- if (result.contents) {
202
- allContents.push(...result.contents);
112
+ }), timeoutMs, 'S3 backup list');
113
+ if (result.contents)
114
+ contents.push(...result.contents);
115
+ if (!result.isTruncated) {
116
+ continuationToken = undefined;
117
+ continue;
118
+ }
119
+ const next = result.nextContinuationToken;
120
+ if (!next || seenTokens.has(next)) {
121
+ throw new Error('S3 backup list returned an invalid continuation token');
203
122
  }
204
- continuationToken = result.isTruncated ? result.nextContinuationToken : undefined;
123
+ seenTokens.add(next);
124
+ continuationToken = next;
205
125
  } while (continuationToken);
206
- return allContents
207
- .filter((item) => typeof item.key === 'string' &&
208
- item.key.endsWith('.db') &&
209
- !item.key.endsWith('.meta.json'))
210
- .map((item) => {
211
- const lastMod = item.lastModified;
212
- const lastModDate = lastMod ? new Date(lastMod) : new Date();
213
- return {
214
- key: item.key,
215
- size: item.size ?? 0,
216
- lastModified: lastModDate,
217
- };
218
- })
126
+ return contents
127
+ .filter((item) => typeof item.key === 'string' && item.key.endsWith('.db'))
128
+ .map((item) => ({
129
+ key: item.key,
130
+ size: item.size ?? 0,
131
+ lastModified: item.lastModified ? new Date(item.lastModified) : new Date(),
132
+ }))
219
133
  .sort((a, b) => b.lastModified.getTime() - a.lastModified.getTime());
220
134
  }
221
135
  catch (error) {
222
- backupLog.error('Failed to list backups', { error: String(error) });
223
- return [];
136
+ const message = error instanceof Error ? error.message : String(error);
137
+ backupLog.error('Failed to list backups', { error: message });
138
+ throw new Error(`Failed to list backups: ${message}`, { cause: error });
224
139
  }
225
140
  }
226
- /**
227
- * Restore from a backup
228
- */
141
+ /** Download, validate, and atomically install a backup. */
229
142
  export async function restoreBackup(key, config, client) {
230
143
  const startTime = Date.now();
144
+ const timeoutMs = timeoutFor(config);
145
+ let candidatePath = null;
231
146
  try {
232
- // Verify backup exists
233
147
  const s3File = client.file(key);
234
- const exists = await s3File.exists();
235
- if (!exists) {
148
+ const exists = await retryWithTimeout(() => s3File.exists(), timeoutMs, 'S3 backup existence check');
149
+ if (!exists)
236
150
  throw new Error(`Backup not found: ${key}`);
237
- }
238
- // Download backup (with timeout)
239
- const timeoutMs = config.timeoutMs ?? DEFAULT_S3_TIMEOUT_MS;
240
- const compressedData = await withTimeout(s3File.arrayBuffer(), timeoutMs, 'S3 backup download');
241
- // Check metadata to determine if backup is compressed
242
- const metadataKey = `${key}.meta.json`;
243
- const metadataFile = client.file(metadataKey);
244
- const metadataExists = await metadataFile.exists();
245
- let metadataRaw = null;
151
+ const downloaded = new Uint8Array(await retryWithTimeout(() => s3File.arrayBuffer(), timeoutMs, 'S3 backup download'));
152
+ const metadataFile = client.file(`${key}.meta.json`);
153
+ const metadataExists = await retryWithTimeout(() => metadataFile.exists(), timeoutMs, 'S3 metadata existence check');
154
+ let data;
246
155
  if (metadataExists) {
247
- metadataRaw = (await metadataFile.json());
248
- }
249
- // Decompress if backup is compressed (new format) or try to detect gzip magic bytes
250
- const isCompressed = metadataRaw?.compressed ??
251
- (compressedData.byteLength >= 2 &&
252
- new Uint8Array(compressedData)[0] === 0x1f &&
253
- new Uint8Array(compressedData)[1] === 0x8b);
254
- const data = isCompressed
255
- ? await gunzipAsync(new Uint8Array(compressedData))
256
- : new Uint8Array(compressedData);
257
- // Verify checksum if metadata exists
258
- if (metadataRaw?.checksum) {
259
- const hasher = new Bun.CryptoHasher('sha256');
260
- hasher.update(data);
261
- const checksum = hasher.digest('hex');
262
- if (checksum !== metadataRaw.checksum) {
156
+ const metadata = validateMetadata(await retryWithTimeout(() => metadataFile.json(), timeoutMs, 'S3 metadata download'), downloaded.byteLength);
157
+ data = await gunzipAsync(downloaded);
158
+ if (data.byteLength !== metadata.size) {
159
+ throw new Error('Backup size mismatch - file may be corrupted');
160
+ }
161
+ if (sha256(data) !== metadata.checksum) {
263
162
  throw new Error('Backup checksum mismatch - file may be corrupted');
264
163
  }
265
164
  }
266
- // Validate SQLite format
165
+ else {
166
+ if (isGzip(downloaded)) {
167
+ throw new Error('Compressed backup is invalid because metadata is missing');
168
+ }
169
+ data = downloaded;
170
+ }
267
171
  const header = new TextDecoder().decode(data.slice(0, 16));
268
172
  if (!header.startsWith('SQLite format 3')) {
269
173
  throw new Error('Restored data is not a valid SQLite database');
270
174
  }
271
- // Atomic, validate-before-replace restore:
272
- // Write the payload to a TEMP file next to the target, validate integrity
273
- // on the temp file, and only swap the live DB into place on full success.
274
- // On any failure the temp file is removed and the live DB is left untouched.
275
- const tempPath = `${config.databasePath}.restore-${Date.now()}-${Math.random()
276
- .toString(36)
277
- .slice(2)}.tmp`;
278
- try {
279
- await Bun.write(tempPath, data);
280
- // Verify integrity on the temp candidate (this deletes the temp file on failure).
281
- await verifyDatabaseIntegrity(tempPath);
282
- // Validation passed: atomically replace the live DB with the temp file.
283
- const { rename } = await import('fs/promises');
284
- await rename(tempPath, config.databasePath);
285
- }
286
- catch (error) {
287
- // Best-effort cleanup of the temp file (may already be removed by the
288
- // integrity check). The live DB at config.databasePath is never touched.
289
- try {
290
- const { unlink } = await import('fs/promises');
291
- await unlink(tempPath);
292
- }
293
- catch {
294
- /* best effort — temp file may not exist */
295
- }
296
- throw error;
297
- }
298
- const duration = Date.now() - startTime;
299
- return { success: true, key, size: data.byteLength, duration };
175
+ candidatePath = restoreCandidatePath(config.databasePath);
176
+ await Bun.write(candidatePath, data);
177
+ verifyDatabaseIntegrity(candidatePath);
178
+ await installDatabaseCandidate(candidatePath, config.databasePath);
179
+ return {
180
+ success: true,
181
+ key,
182
+ size: data.byteLength,
183
+ ...(metadataExists && { compressedSize: downloaded.byteLength }),
184
+ duration: Date.now() - startTime,
185
+ };
300
186
  }
301
187
  catch (error) {
302
188
  const message = error instanceof Error ? error.message : String(error);
303
189
  backupLog.error('Restore failed', { error: message });
304
190
  return { success: false, error: message };
305
191
  }
192
+ finally {
193
+ await cleanupLocalArtifacts(candidatePath, 'restore candidate');
194
+ }
306
195
  }
307
- /**
308
- * Clean up old backups based on retention policy
309
- */
196
+ /** Delete backup pairs beyond the configured retention count. */
310
197
  export async function cleanupOldBackups(config, client) {
311
198
  try {
312
199
  const backups = await listBackups(config, client);
313
- const retention = Math.max(config.retention, 1);
314
- if (backups.length <= retention) {
315
- return;
316
- }
317
- // Sort by date (newest first) and get backups to delete
318
- const toDelete = backups.slice(retention);
200
+ const toDelete = backups.slice(Math.max(config.retention, 1));
201
+ const timeoutMs = timeoutFor(config);
319
202
  for (const backup of toDelete) {
320
203
  try {
321
- // Delete backup file
322
- await client.delete(backup.key);
323
- // Delete metadata file if exists
324
- const metadataKey = `${backup.key}.meta.json`;
325
- const metadataFile = client.file(metadataKey);
326
- if (await metadataFile.exists()) {
327
- await client.delete(metadataKey);
328
- }
204
+ await retryWithTimeout(() => client.delete(backup.key), timeoutMs, 'S3 backup delete');
205
+ await retryWithTimeout(() => client.delete(`${backup.key}.meta.json`), timeoutMs, 'S3 metadata delete');
329
206
  }
330
207
  catch (error) {
331
- backupLog.warn('Failed to delete old backup', { key: backup.key, error: String(error) });
208
+ backupLog.warn('Failed to delete old backup', {
209
+ key: backup.key,
210
+ error: String(error),
211
+ });
332
212
  }
333
213
  }
334
214
  }
@@ -0,0 +1,22 @@
1
+ /**
2
+ * SQLite snapshot creation, validation, and restore file transitions.
3
+ */
4
+ /** Remove a database candidate and any sidecars SQLite may have created for it. */
5
+ export declare function removeDatabaseArtifacts(databasePath: string): Promise<void>;
6
+ /** Validate all SQLite database pages. */
7
+ export declare function verifyDatabaseIntegrity(databasePath: string): void;
8
+ /**
9
+ * Produce a transactionally consistent, standalone SQLite snapshot.
10
+ *
11
+ * VACUUM INTO reads through SQLite rather than copying the live main file, so
12
+ * committed WAL frames are included even when an older reader blocks a WAL
13
+ * checkpoint.
14
+ */
15
+ export declare function createConsistentSnapshot(databasePath: string): Promise<string>;
16
+ /**
17
+ * Install an already validated candidate while quarantining stale sidecars.
18
+ *
19
+ * The live main database is not touched until all sidecars have moved. If a
20
+ * pre-swap operation fails, moved sidecars are restored before returning.
21
+ */
22
+ export declare function installDatabaseCandidate(candidatePath: string, databasePath: string): Promise<void>;
@@ -0,0 +1,130 @@
1
+ /**
2
+ * SQLite snapshot creation, validation, and restore file transitions.
3
+ */
4
+ import { Database } from 'bun:sqlite';
5
+ import { rename, rm } from 'node:fs/promises';
6
+ import { backupLog } from '../../shared/logger';
7
+ const SIDECAR_SUFFIXES = ['-wal', '-shm', '-journal'];
8
+ function uniqueSiblingPath(databasePath, purpose) {
9
+ return `${databasePath}.${purpose}-${Date.now()}-${crypto.randomUUID()}.tmp`;
10
+ }
11
+ function quoteSqliteString(value) {
12
+ return value.replaceAll("'", "''");
13
+ }
14
+ async function removeIfPresent(path) {
15
+ await rm(path, { force: true });
16
+ }
17
+ /** Remove a database candidate and any sidecars SQLite may have created for it. */
18
+ export async function removeDatabaseArtifacts(databasePath) {
19
+ await Promise.all([
20
+ removeIfPresent(databasePath),
21
+ ...SIDECAR_SUFFIXES.map((suffix) => removeIfPresent(`${databasePath}${suffix}`)),
22
+ ]);
23
+ }
24
+ /** Validate all SQLite database pages. */
25
+ export function verifyDatabaseIntegrity(databasePath) {
26
+ let db = null;
27
+ try {
28
+ db = new Database(databasePath, { readonly: true });
29
+ const result = db.query('PRAGMA integrity_check').get();
30
+ const status = result?.integrity_check ?? '';
31
+ if (status !== 'ok') {
32
+ throw new Error(`Database integrity check failed: ${status || 'unknown error'}`);
33
+ }
34
+ }
35
+ catch (error) {
36
+ if (error instanceof Error && error.message.includes('integrity check failed')) {
37
+ throw error;
38
+ }
39
+ throw new Error('Database integrity check failed: corrupt or invalid database', {
40
+ cause: error,
41
+ });
42
+ }
43
+ finally {
44
+ try {
45
+ db?.close();
46
+ }
47
+ catch {
48
+ // The failed open or check may already have closed the handle.
49
+ }
50
+ }
51
+ }
52
+ /**
53
+ * Produce a transactionally consistent, standalone SQLite snapshot.
54
+ *
55
+ * VACUUM INTO reads through SQLite rather than copying the live main file, so
56
+ * committed WAL frames are included even when an older reader blocks a WAL
57
+ * checkpoint.
58
+ */
59
+ export async function createConsistentSnapshot(databasePath) {
60
+ const snapshotPath = uniqueSiblingPath(databasePath, 'backup-snapshot');
61
+ let db = null;
62
+ try {
63
+ db = new Database(databasePath, { readonly: true });
64
+ db.exec('PRAGMA busy_timeout = 30000');
65
+ db.exec(`VACUUM INTO '${quoteSqliteString(snapshotPath)}'`);
66
+ db.close();
67
+ db = null;
68
+ verifyDatabaseIntegrity(snapshotPath);
69
+ return snapshotPath;
70
+ }
71
+ catch (error) {
72
+ try {
73
+ db?.close();
74
+ }
75
+ catch {
76
+ // The failed snapshot may already have closed the handle.
77
+ }
78
+ await removeDatabaseArtifacts(snapshotPath);
79
+ throw error;
80
+ }
81
+ }
82
+ /**
83
+ * Install an already validated candidate while quarantining stale sidecars.
84
+ *
85
+ * The live main database is not touched until all sidecars have moved. If a
86
+ * pre-swap operation fails, moved sidecars are restored before returning.
87
+ */
88
+ export async function installDatabaseCandidate(candidatePath, databasePath) {
89
+ const quarantineId = `${Date.now()}-${crypto.randomUUID()}`;
90
+ const movedSidecars = [];
91
+ try {
92
+ for (const suffix of SIDECAR_SUFFIXES) {
93
+ const live = `${databasePath}${suffix}`;
94
+ if (!(await Bun.file(live).exists())) {
95
+ continue;
96
+ }
97
+ const quarantine = `${live}.restore-quarantine-${quarantineId}`;
98
+ await rename(live, quarantine);
99
+ movedSidecars.push({ live, quarantine });
100
+ }
101
+ await rename(candidatePath, databasePath);
102
+ }
103
+ catch (error) {
104
+ const rollbackErrors = [];
105
+ for (const moved of movedSidecars.reverse()) {
106
+ try {
107
+ await rename(moved.quarantine, moved.live);
108
+ }
109
+ catch (rollbackError) {
110
+ rollbackErrors.push(String(rollbackError));
111
+ }
112
+ }
113
+ if (rollbackErrors.length > 0) {
114
+ throw new Error(`Restore failed and sidecar rollback failed: ${rollbackErrors.join('; ')}`, {
115
+ cause: error,
116
+ });
117
+ }
118
+ throw error;
119
+ }
120
+ const cleanup = await Promise.allSettled([
121
+ ...movedSidecars.map(({ quarantine }) => removeIfPresent(quarantine)),
122
+ ...SIDECAR_SUFFIXES.map((suffix) => removeIfPresent(`${candidatePath}${suffix}`)),
123
+ ]);
124
+ const failures = cleanup.filter((result) => result.status === 'rejected');
125
+ if (failures.length > 0) {
126
+ backupLog.warn('Restore committed but quarantine cleanup was incomplete', {
127
+ errors: failures.map(({ reason }) => String(reason)),
128
+ });
129
+ }
130
+ }
@@ -93,6 +93,7 @@ export declare function buildStatsRefresh(qm: QueueManager): {
93
93
  totalProcessed: number;
94
94
  totalFailed: number;
95
95
  activeJobs: number;
96
+ concurrencySlots: number;
96
97
  };
97
98
  cronCount: number;
98
99
  storage: {
@@ -203,7 +203,12 @@ export class SqliteStorage {
203
203
  }
204
204
  /** Flush write buffer to disk. Returns number of jobs flushed. */
205
205
  flushWriteBuffer() {
206
- return this.writeBuffer.flush();
206
+ const flushed = this.writeBuffer.flush();
207
+ const pending = this.writeBuffer.pendingCount;
208
+ if (pending > 0) {
209
+ throw new Error(`Cannot create a consistent backup while ${pending} write${pending === 1 ? '' : 's'} remains buffered`);
210
+ }
211
+ return flushed;
207
212
  }
208
213
  migrate() {
209
214
  this.db.run(MIGRATION_TABLE);
@@ -5,5 +5,7 @@
5
5
  * agent, stats interval, crash handlers and graceful shutdown are always on.
6
6
  */
7
7
  import { type BunqueueConfig, type ResolvedConfig } from '../../config';
8
+ /** Return a startup error when an enabled file backup has no file to snapshot. */
9
+ export declare function backupStartupError(config: Pick<ResolvedConfig, 's3BackupEnabled' | 'dataPath'>): string | null;
8
10
  /** Boot the full server from resolved configuration. Runs until shutdown. */
9
11
  export declare function bootServer(fileConfig: BunqueueConfig | null, config: ResolvedConfig): void;