bunqueue 2.8.42 → 2.8.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/application/latencyTracker.js +3 -3
- package/dist/application/metricsExporter.d.ts +3 -1
- package/dist/application/metricsExporter.js +63 -16
- package/dist/application/prometheusOperationalMetrics.d.ts +31 -0
- package/dist/application/prometheusOperationalMetrics.js +53 -0
- package/dist/application/queueManager.d.ts +6 -0
- package/dist/application/queueManager.js +24 -1
- package/dist/application/types.d.ts +3 -0
- package/dist/application/types.js +1 -0
- package/dist/application/workerManager.d.ts +1 -0
- package/dist/application/workerManager.js +4 -1
- package/dist/cli/output.js +2 -1
- package/dist/config/resolve.d.ts +1 -0
- package/dist/config/resolve.js +10 -0
- package/dist/config/types.d.ts +6 -0
- package/dist/infrastructure/backup/backupTelemetry.d.ts +30 -0
- package/dist/infrastructure/backup/backupTelemetry.js +63 -0
- package/dist/infrastructure/backup/index.d.ts +1 -1
- package/dist/infrastructure/backup/index.js +1 -1
- package/dist/infrastructure/backup/s3Backup.d.ts +6 -2
- package/dist/infrastructure/backup/s3Backup.js +22 -9
- package/dist/infrastructure/backup/s3BackupConfig.d.ts +7 -0
- package/dist/infrastructure/backup/s3BackupConfig.js +5 -0
- package/dist/infrastructure/backup/s3BackupIo.d.ts +23 -0
- package/dist/infrastructure/backup/s3BackupIo.js +106 -0
- package/dist/infrastructure/backup/s3BackupOperations.d.ts +6 -15
- package/dist/infrastructure/backup/s3BackupOperations.js +139 -259
- package/dist/infrastructure/backup/sqliteBackupFiles.d.ts +22 -0
- package/dist/infrastructure/backup/sqliteBackupFiles.js +130 -0
- package/dist/infrastructure/cloud/statsRefresh.d.ts +1 -0
- package/dist/infrastructure/persistence/sqlite.js +6 -1
- package/dist/infrastructure/server/bootstrap.d.ts +2 -0
- package/dist/infrastructure/server/bootstrap.js +30 -1
- package/dist/infrastructure/server/http.d.ts +2 -0
- package/dist/infrastructure/server/http.js +9 -58
- package/dist/infrastructure/server/httpDashboardEndpoints.d.ts +10 -0
- package/dist/infrastructure/server/httpDashboardEndpoints.js +146 -0
- package/dist/infrastructure/server/httpEndpoints.d.ts +5 -9
- package/dist/infrastructure/server/httpEndpoints.js +16 -160
- package/dist/infrastructure/server/httpResponse.d.ts +2 -0
- package/dist/infrastructure/server/httpResponse.js +12 -0
- package/dist/infrastructure/server/httpRouter.d.ts +6 -0
- package/dist/infrastructure/server/httpRouter.js +50 -0
- package/dist/shared/histogram.d.ts +2 -2
- package/dist/shared/histogram.js +4 -4
- package/package.json +2 -2
|
@@ -1,334 +1,214 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* S3
|
|
3
|
-
* Core backup, restore, list, and cleanup operations
|
|
2
|
+
* S3 backup, restore, listing, and retention operations.
|
|
4
3
|
*/
|
|
5
4
|
import { backupLog } from '../../shared/logger';
|
|
6
5
|
import { VERSION } from '../../shared/version';
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
function
|
|
10
|
-
return
|
|
11
|
-
promise,
|
|
12
|
-
new Promise((_, reject) => {
|
|
13
|
-
setTimeout(() => {
|
|
14
|
-
reject(new Error(`${label} timed out after ${ms}ms`));
|
|
15
|
-
}, ms);
|
|
16
|
-
}),
|
|
17
|
-
]);
|
|
6
|
+
import { createConsistentSnapshot, installDatabaseCandidate, removeDatabaseArtifacts, verifyDatabaseIntegrity, } from './sqliteBackupFiles';
|
|
7
|
+
import { cleanupFailedPayload, DEFAULT_S3_TIMEOUT_MS, gunzipAsync, gzipAsync, retryWithTimeout, sha256, } from './s3BackupIo';
|
|
8
|
+
function timeoutFor(config) {
|
|
9
|
+
return config.timeoutMs ?? DEFAULT_S3_TIMEOUT_MS;
|
|
18
10
|
}
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
const lower = message.toLowerCase();
|
|
23
|
-
return (lower.includes('connection reset') ||
|
|
24
|
-
lower.includes('econnreset') ||
|
|
25
|
-
lower.includes('timeout') ||
|
|
26
|
-
lower.includes('etimedout') ||
|
|
27
|
-
lower.includes('econnrefused') ||
|
|
28
|
-
lower.includes('socket hang up') ||
|
|
29
|
-
lower.includes('network') ||
|
|
30
|
-
lower.includes('503') ||
|
|
31
|
-
lower.includes('500') ||
|
|
32
|
-
lower.includes('502') ||
|
|
33
|
-
lower.includes('504') ||
|
|
34
|
-
lower.includes('service unavailable') ||
|
|
35
|
-
lower.includes('internal server error') ||
|
|
36
|
-
lower.includes('bad gateway') ||
|
|
37
|
-
lower.includes('gateway timeout') ||
|
|
38
|
-
lower.includes('transient'));
|
|
11
|
+
function backupKey(prefix) {
|
|
12
|
+
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
|
13
|
+
return `${prefix}bunqueue-${timestamp}-${crypto.randomUUID()}.db`;
|
|
39
14
|
}
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
15
|
+
function restoreCandidatePath(databasePath) {
|
|
16
|
+
return `${databasePath}.restore-${Date.now()}-${crypto.randomUUID()}.tmp`;
|
|
17
|
+
}
|
|
18
|
+
function isGzip(data) {
|
|
19
|
+
return data.byteLength >= 2 && data[0] === 0x1f && data[1] === 0x8b;
|
|
20
|
+
}
|
|
21
|
+
function validateMetadata(value, compressedSize) {
|
|
22
|
+
if (!value || typeof value !== 'object') {
|
|
23
|
+
throw new Error('Backup metadata is invalid');
|
|
24
|
+
}
|
|
25
|
+
const metadata = value;
|
|
26
|
+
if (typeof metadata.timestamp !== 'string' ||
|
|
27
|
+
typeof metadata.version !== 'string' ||
|
|
28
|
+
!Number.isSafeInteger(metadata.size) ||
|
|
29
|
+
(metadata.size ?? -1) < 0 ||
|
|
30
|
+
!Number.isSafeInteger(metadata.compressedSize) ||
|
|
31
|
+
metadata.compressedSize !== compressedSize ||
|
|
32
|
+
typeof metadata.checksum !== 'string' ||
|
|
33
|
+
!/^[a-f0-9]{64}$/i.test(metadata.checksum) ||
|
|
34
|
+
metadata.compressed !== true) {
|
|
35
|
+
throw new Error('Backup metadata is invalid or does not match the payload');
|
|
58
36
|
}
|
|
59
|
-
|
|
37
|
+
return metadata;
|
|
60
38
|
}
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
* Validates the file at `databasePath` and throws on failure. This function
|
|
65
|
-
* is intended to run against a TEMP file (never the live DB): on integrity
|
|
66
|
-
* failure it deletes the file it inspected, so it must only ever be pointed
|
|
67
|
-
* at a throwaway candidate, not the live database.
|
|
68
|
-
*/
|
|
69
|
-
async function verifyDatabaseIntegrity(databasePath) {
|
|
70
|
-
const { Database } = await import('bun:sqlite');
|
|
71
|
-
let db = null;
|
|
39
|
+
async function cleanupLocalArtifacts(path, label) {
|
|
40
|
+
if (!path)
|
|
41
|
+
return;
|
|
72
42
|
try {
|
|
73
|
-
|
|
74
|
-
const result = db.query('PRAGMA integrity_check').get();
|
|
75
|
-
const status = result?.integrity_check ?? '';
|
|
76
|
-
if (status !== 'ok') {
|
|
77
|
-
throw new Error(`Database integrity check failed: ${status || 'unknown error'}`);
|
|
78
|
-
}
|
|
43
|
+
await removeDatabaseArtifacts(path);
|
|
79
44
|
}
|
|
80
45
|
catch (error) {
|
|
81
|
-
|
|
82
|
-
try {
|
|
83
|
-
db?.close();
|
|
84
|
-
}
|
|
85
|
-
catch {
|
|
86
|
-
/* already closed */
|
|
87
|
-
}
|
|
88
|
-
db = null;
|
|
89
|
-
// Clean up the corrupt candidate file (this is a temp file, never the live DB).
|
|
90
|
-
try {
|
|
91
|
-
const { unlink } = await import('fs/promises');
|
|
92
|
-
await unlink(databasePath);
|
|
93
|
-
}
|
|
94
|
-
catch {
|
|
95
|
-
/* best effort */
|
|
96
|
-
}
|
|
97
|
-
if (error instanceof Error && error.message.includes('integrity check failed')) {
|
|
98
|
-
throw error;
|
|
99
|
-
}
|
|
100
|
-
throw new Error('Database integrity check failed: corrupt or invalid database', {
|
|
101
|
-
cause: error,
|
|
102
|
-
});
|
|
46
|
+
backupLog.warn(`Failed to clean ${label} artifacts`, { path, error: String(error) });
|
|
103
47
|
}
|
|
104
|
-
finally {
|
|
105
|
-
try {
|
|
106
|
-
db?.close();
|
|
107
|
-
}
|
|
108
|
-
catch {
|
|
109
|
-
/* already closed */
|
|
110
|
-
}
|
|
111
|
-
}
|
|
112
|
-
}
|
|
113
|
-
/** Async gzip compress using Web Streams API (non-blocking) */
|
|
114
|
-
async function gzipAsync(data) {
|
|
115
|
-
const stream = new Blob([data])
|
|
116
|
-
.stream()
|
|
117
|
-
.pipeThrough(new CompressionStream('gzip'));
|
|
118
|
-
return new Uint8Array(await new Response(stream).arrayBuffer());
|
|
119
|
-
}
|
|
120
|
-
/** Async gzip decompress using Web Streams API (non-blocking) */
|
|
121
|
-
async function gunzipAsync(data) {
|
|
122
|
-
const stream = new Blob([data])
|
|
123
|
-
.stream()
|
|
124
|
-
.pipeThrough(new DecompressionStream('gzip'));
|
|
125
|
-
return new Uint8Array(await new Response(stream).arrayBuffer());
|
|
126
48
|
}
|
|
127
|
-
/**
|
|
128
|
-
* Perform a backup to S3
|
|
129
|
-
*/
|
|
49
|
+
/** Create and publish a consistent SQLite backup. */
|
|
130
50
|
export async function performBackup(config, client) {
|
|
131
51
|
const startTime = Date.now();
|
|
52
|
+
let snapshotPath = null;
|
|
53
|
+
let pendingPayloadKey = null;
|
|
132
54
|
try {
|
|
133
|
-
|
|
134
|
-
const dbFile = Bun.file(config.databasePath);
|
|
135
|
-
const exists = await dbFile.exists();
|
|
136
|
-
if (!exists) {
|
|
55
|
+
if (!(await Bun.file(config.databasePath).exists())) {
|
|
137
56
|
throw new Error(`Database file not found: ${config.databasePath}`);
|
|
138
57
|
}
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
}
|
|
146
|
-
catch {
|
|
147
|
-
// Ignore - database might be locked or not in WAL mode
|
|
148
|
-
}
|
|
149
|
-
// Generate backup key with timestamp
|
|
150
|
-
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
|
151
|
-
const key = `${config.prefix}bunqueue-${timestamp}.db`;
|
|
152
|
-
// Read database file
|
|
153
|
-
const data = await Bun.file(config.databasePath).arrayBuffer();
|
|
154
|
-
const originalSize = data.byteLength;
|
|
155
|
-
// Compress with gzip for efficient storage
|
|
156
|
-
const compressed = await gzipAsync(new Uint8Array(data));
|
|
157
|
-
const compressedSize = compressed.byteLength;
|
|
158
|
-
// Calculate checksum of original data (for integrity verification)
|
|
159
|
-
const hasher = new Bun.CryptoHasher('sha256');
|
|
160
|
-
hasher.update(new Uint8Array(data));
|
|
161
|
-
const checksum = hasher.digest('hex');
|
|
162
|
-
// Upload compressed backup to S3 (with retry and timeout)
|
|
163
|
-
const timeoutMs = config.timeoutMs ?? DEFAULT_S3_TIMEOUT_MS;
|
|
164
|
-
const s3File = client.file(key);
|
|
165
|
-
await withTimeout(withRetry(() => s3File.write(compressed, { type: 'application/gzip' }), 'S3 backup upload'), timeoutMs, 'S3 backup upload');
|
|
166
|
-
// Upload metadata (with retry for transient errors)
|
|
58
|
+
snapshotPath = await createConsistentSnapshot(config.databasePath);
|
|
59
|
+
const data = new Uint8Array(await Bun.file(snapshotPath).arrayBuffer());
|
|
60
|
+
const compressed = await gzipAsync(data);
|
|
61
|
+
const key = backupKey(config.prefix);
|
|
62
|
+
const metadataKey = `${key}.meta.json`;
|
|
63
|
+
const timeoutMs = timeoutFor(config);
|
|
167
64
|
const metadata = {
|
|
168
65
|
timestamp: new Date().toISOString(),
|
|
169
66
|
version: VERSION,
|
|
170
|
-
size:
|
|
171
|
-
compressedSize,
|
|
172
|
-
checksum,
|
|
67
|
+
size: data.byteLength,
|
|
68
|
+
compressedSize: compressed.byteLength,
|
|
69
|
+
checksum: sha256(data),
|
|
173
70
|
compressed: true,
|
|
174
71
|
};
|
|
175
|
-
|
|
176
|
-
|
|
72
|
+
// Metadata is published first. The data object is the visibility/commit
|
|
73
|
+
// point, so every successfully published current-format backup is paired.
|
|
74
|
+
await retryWithTimeout(() => client
|
|
177
75
|
.file(metadataKey)
|
|
178
|
-
.write(JSON.stringify(metadata, null, 2), { type: 'application/json' }),
|
|
179
|
-
|
|
180
|
-
|
|
76
|
+
.write(JSON.stringify(metadata, null, 2), { type: 'application/json' }), timeoutMs, 'S3 metadata upload');
|
|
77
|
+
pendingPayloadKey = key;
|
|
78
|
+
await retryWithTimeout(() => client.file(key).write(compressed, { type: 'application/gzip' }), timeoutMs, 'S3 backup upload');
|
|
79
|
+
pendingPayloadKey = null;
|
|
80
|
+
return {
|
|
81
|
+
success: true,
|
|
82
|
+
key,
|
|
83
|
+
size: data.byteLength,
|
|
84
|
+
compressedSize: compressed.byteLength,
|
|
85
|
+
duration: Date.now() - startTime,
|
|
86
|
+
};
|
|
181
87
|
}
|
|
182
88
|
catch (error) {
|
|
89
|
+
if (pendingPayloadKey) {
|
|
90
|
+
await cleanupFailedPayload(client, pendingPayloadKey, timeoutFor(config), error);
|
|
91
|
+
}
|
|
183
92
|
const message = error instanceof Error ? error.message : String(error);
|
|
184
93
|
backupLog.error('Backup failed', { error: message });
|
|
185
94
|
return { success: false, error: message };
|
|
186
95
|
}
|
|
96
|
+
finally {
|
|
97
|
+
await cleanupLocalArtifacts(snapshotPath, 'backup snapshot');
|
|
98
|
+
}
|
|
187
99
|
}
|
|
188
|
-
/**
|
|
189
|
-
* List available backups
|
|
190
|
-
*/
|
|
100
|
+
/** List every backup object under the configured prefix. */
|
|
191
101
|
export async function listBackups(config, client) {
|
|
192
102
|
try {
|
|
193
|
-
const
|
|
103
|
+
const contents = [];
|
|
104
|
+
const seenTokens = new Set();
|
|
105
|
+
const timeoutMs = timeoutFor(config);
|
|
194
106
|
let continuationToken;
|
|
195
107
|
do {
|
|
196
|
-
const result = await client.list({
|
|
108
|
+
const result = await retryWithTimeout(() => client.list({
|
|
197
109
|
prefix: config.prefix,
|
|
198
110
|
maxKeys: 100,
|
|
199
111
|
...(continuationToken ? { continuationToken } : {}),
|
|
200
|
-
});
|
|
201
|
-
if (result.contents)
|
|
202
|
-
|
|
112
|
+
}), timeoutMs, 'S3 backup list');
|
|
113
|
+
if (result.contents)
|
|
114
|
+
contents.push(...result.contents);
|
|
115
|
+
if (!result.isTruncated) {
|
|
116
|
+
continuationToken = undefined;
|
|
117
|
+
continue;
|
|
118
|
+
}
|
|
119
|
+
const next = result.nextContinuationToken;
|
|
120
|
+
if (!next || seenTokens.has(next)) {
|
|
121
|
+
throw new Error('S3 backup list returned an invalid continuation token');
|
|
203
122
|
}
|
|
204
|
-
|
|
123
|
+
seenTokens.add(next);
|
|
124
|
+
continuationToken = next;
|
|
205
125
|
} while (continuationToken);
|
|
206
|
-
return
|
|
207
|
-
.filter((item) => typeof item.key === 'string' &&
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
.
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
return {
|
|
214
|
-
key: item.key,
|
|
215
|
-
size: item.size ?? 0,
|
|
216
|
-
lastModified: lastModDate,
|
|
217
|
-
};
|
|
218
|
-
})
|
|
126
|
+
return contents
|
|
127
|
+
.filter((item) => typeof item.key === 'string' && item.key.endsWith('.db'))
|
|
128
|
+
.map((item) => ({
|
|
129
|
+
key: item.key,
|
|
130
|
+
size: item.size ?? 0,
|
|
131
|
+
lastModified: item.lastModified ? new Date(item.lastModified) : new Date(),
|
|
132
|
+
}))
|
|
219
133
|
.sort((a, b) => b.lastModified.getTime() - a.lastModified.getTime());
|
|
220
134
|
}
|
|
221
135
|
catch (error) {
|
|
222
|
-
|
|
223
|
-
|
|
136
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
137
|
+
backupLog.error('Failed to list backups', { error: message });
|
|
138
|
+
throw new Error(`Failed to list backups: ${message}`, { cause: error });
|
|
224
139
|
}
|
|
225
140
|
}
|
|
226
|
-
/**
|
|
227
|
-
* Restore from a backup
|
|
228
|
-
*/
|
|
141
|
+
/** Download, validate, and atomically install a backup. */
|
|
229
142
|
export async function restoreBackup(key, config, client) {
|
|
230
143
|
const startTime = Date.now();
|
|
144
|
+
const timeoutMs = timeoutFor(config);
|
|
145
|
+
let candidatePath = null;
|
|
231
146
|
try {
|
|
232
|
-
// Verify backup exists
|
|
233
147
|
const s3File = client.file(key);
|
|
234
|
-
const exists = await s3File.exists();
|
|
235
|
-
if (!exists)
|
|
148
|
+
const exists = await retryWithTimeout(() => s3File.exists(), timeoutMs, 'S3 backup existence check');
|
|
149
|
+
if (!exists)
|
|
236
150
|
throw new Error(`Backup not found: ${key}`);
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
const
|
|
240
|
-
|
|
241
|
-
// Check metadata to determine if backup is compressed
|
|
242
|
-
const metadataKey = `${key}.meta.json`;
|
|
243
|
-
const metadataFile = client.file(metadataKey);
|
|
244
|
-
const metadataExists = await metadataFile.exists();
|
|
245
|
-
let metadataRaw = null;
|
|
151
|
+
const downloaded = new Uint8Array(await retryWithTimeout(() => s3File.arrayBuffer(), timeoutMs, 'S3 backup download'));
|
|
152
|
+
const metadataFile = client.file(`${key}.meta.json`);
|
|
153
|
+
const metadataExists = await retryWithTimeout(() => metadataFile.exists(), timeoutMs, 'S3 metadata existence check');
|
|
154
|
+
let data;
|
|
246
155
|
if (metadataExists) {
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
new Uint8Array(compressedData)[1] === 0x8b);
|
|
254
|
-
const data = isCompressed
|
|
255
|
-
? await gunzipAsync(new Uint8Array(compressedData))
|
|
256
|
-
: new Uint8Array(compressedData);
|
|
257
|
-
// Verify checksum if metadata exists
|
|
258
|
-
if (metadataRaw?.checksum) {
|
|
259
|
-
const hasher = new Bun.CryptoHasher('sha256');
|
|
260
|
-
hasher.update(data);
|
|
261
|
-
const checksum = hasher.digest('hex');
|
|
262
|
-
if (checksum !== metadataRaw.checksum) {
|
|
156
|
+
const metadata = validateMetadata(await retryWithTimeout(() => metadataFile.json(), timeoutMs, 'S3 metadata download'), downloaded.byteLength);
|
|
157
|
+
data = await gunzipAsync(downloaded);
|
|
158
|
+
if (data.byteLength !== metadata.size) {
|
|
159
|
+
throw new Error('Backup size mismatch - file may be corrupted');
|
|
160
|
+
}
|
|
161
|
+
if (sha256(data) !== metadata.checksum) {
|
|
263
162
|
throw new Error('Backup checksum mismatch - file may be corrupted');
|
|
264
163
|
}
|
|
265
164
|
}
|
|
266
|
-
|
|
165
|
+
else {
|
|
166
|
+
if (isGzip(downloaded)) {
|
|
167
|
+
throw new Error('Compressed backup is invalid because metadata is missing');
|
|
168
|
+
}
|
|
169
|
+
data = downloaded;
|
|
170
|
+
}
|
|
267
171
|
const header = new TextDecoder().decode(data.slice(0, 16));
|
|
268
172
|
if (!header.startsWith('SQLite format 3')) {
|
|
269
173
|
throw new Error('Restored data is not a valid SQLite database');
|
|
270
174
|
}
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
// Validation passed: atomically replace the live DB with the temp file.
|
|
283
|
-
const { rename } = await import('fs/promises');
|
|
284
|
-
await rename(tempPath, config.databasePath);
|
|
285
|
-
}
|
|
286
|
-
catch (error) {
|
|
287
|
-
// Best-effort cleanup of the temp file (may already be removed by the
|
|
288
|
-
// integrity check). The live DB at config.databasePath is never touched.
|
|
289
|
-
try {
|
|
290
|
-
const { unlink } = await import('fs/promises');
|
|
291
|
-
await unlink(tempPath);
|
|
292
|
-
}
|
|
293
|
-
catch {
|
|
294
|
-
/* best effort — temp file may not exist */
|
|
295
|
-
}
|
|
296
|
-
throw error;
|
|
297
|
-
}
|
|
298
|
-
const duration = Date.now() - startTime;
|
|
299
|
-
return { success: true, key, size: data.byteLength, duration };
|
|
175
|
+
candidatePath = restoreCandidatePath(config.databasePath);
|
|
176
|
+
await Bun.write(candidatePath, data);
|
|
177
|
+
verifyDatabaseIntegrity(candidatePath);
|
|
178
|
+
await installDatabaseCandidate(candidatePath, config.databasePath);
|
|
179
|
+
return {
|
|
180
|
+
success: true,
|
|
181
|
+
key,
|
|
182
|
+
size: data.byteLength,
|
|
183
|
+
...(metadataExists && { compressedSize: downloaded.byteLength }),
|
|
184
|
+
duration: Date.now() - startTime,
|
|
185
|
+
};
|
|
300
186
|
}
|
|
301
187
|
catch (error) {
|
|
302
188
|
const message = error instanceof Error ? error.message : String(error);
|
|
303
189
|
backupLog.error('Restore failed', { error: message });
|
|
304
190
|
return { success: false, error: message };
|
|
305
191
|
}
|
|
192
|
+
finally {
|
|
193
|
+
await cleanupLocalArtifacts(candidatePath, 'restore candidate');
|
|
194
|
+
}
|
|
306
195
|
}
|
|
307
|
-
/**
|
|
308
|
-
* Clean up old backups based on retention policy
|
|
309
|
-
*/
|
|
196
|
+
/** Delete backup pairs beyond the configured retention count. */
|
|
310
197
|
export async function cleanupOldBackups(config, client) {
|
|
311
198
|
try {
|
|
312
199
|
const backups = await listBackups(config, client);
|
|
313
|
-
const
|
|
314
|
-
|
|
315
|
-
return;
|
|
316
|
-
}
|
|
317
|
-
// Sort by date (newest first) and get backups to delete
|
|
318
|
-
const toDelete = backups.slice(retention);
|
|
200
|
+
const toDelete = backups.slice(Math.max(config.retention, 1));
|
|
201
|
+
const timeoutMs = timeoutFor(config);
|
|
319
202
|
for (const backup of toDelete) {
|
|
320
203
|
try {
|
|
321
|
-
|
|
322
|
-
await client.delete(backup.key);
|
|
323
|
-
// Delete metadata file if exists
|
|
324
|
-
const metadataKey = `${backup.key}.meta.json`;
|
|
325
|
-
const metadataFile = client.file(metadataKey);
|
|
326
|
-
if (await metadataFile.exists()) {
|
|
327
|
-
await client.delete(metadataKey);
|
|
328
|
-
}
|
|
204
|
+
await retryWithTimeout(() => client.delete(backup.key), timeoutMs, 'S3 backup delete');
|
|
205
|
+
await retryWithTimeout(() => client.delete(`${backup.key}.meta.json`), timeoutMs, 'S3 metadata delete');
|
|
329
206
|
}
|
|
330
207
|
catch (error) {
|
|
331
|
-
backupLog.warn('Failed to delete old backup', {
|
|
208
|
+
backupLog.warn('Failed to delete old backup', {
|
|
209
|
+
key: backup.key,
|
|
210
|
+
error: String(error),
|
|
211
|
+
});
|
|
332
212
|
}
|
|
333
213
|
}
|
|
334
214
|
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SQLite snapshot creation, validation, and restore file transitions.
|
|
3
|
+
*/
|
|
4
|
+
/** Remove a database candidate and any sidecars SQLite may have created for it. */
|
|
5
|
+
export declare function removeDatabaseArtifacts(databasePath: string): Promise<void>;
|
|
6
|
+
/** Validate all SQLite database pages. */
|
|
7
|
+
export declare function verifyDatabaseIntegrity(databasePath: string): void;
|
|
8
|
+
/**
|
|
9
|
+
* Produce a transactionally consistent, standalone SQLite snapshot.
|
|
10
|
+
*
|
|
11
|
+
* VACUUM INTO reads through SQLite rather than copying the live main file, so
|
|
12
|
+
* committed WAL frames are included even when an older reader blocks a WAL
|
|
13
|
+
* checkpoint.
|
|
14
|
+
*/
|
|
15
|
+
export declare function createConsistentSnapshot(databasePath: string): Promise<string>;
|
|
16
|
+
/**
|
|
17
|
+
* Install an already validated candidate while quarantining stale sidecars.
|
|
18
|
+
*
|
|
19
|
+
* The live main database is not touched until all sidecars have moved. If a
|
|
20
|
+
* pre-swap operation fails, moved sidecars are restored before returning.
|
|
21
|
+
*/
|
|
22
|
+
export declare function installDatabaseCandidate(candidatePath: string, databasePath: string): Promise<void>;
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SQLite snapshot creation, validation, and restore file transitions.
|
|
3
|
+
*/
|
|
4
|
+
import { Database } from 'bun:sqlite';
|
|
5
|
+
import { rename, rm } from 'node:fs/promises';
|
|
6
|
+
import { backupLog } from '../../shared/logger';
|
|
7
|
+
const SIDECAR_SUFFIXES = ['-wal', '-shm', '-journal'];
|
|
8
|
+
function uniqueSiblingPath(databasePath, purpose) {
|
|
9
|
+
return `${databasePath}.${purpose}-${Date.now()}-${crypto.randomUUID()}.tmp`;
|
|
10
|
+
}
|
|
11
|
+
function quoteSqliteString(value) {
|
|
12
|
+
return value.replaceAll("'", "''");
|
|
13
|
+
}
|
|
14
|
+
async function removeIfPresent(path) {
|
|
15
|
+
await rm(path, { force: true });
|
|
16
|
+
}
|
|
17
|
+
/** Remove a database candidate and any sidecars SQLite may have created for it. */
|
|
18
|
+
export async function removeDatabaseArtifacts(databasePath) {
|
|
19
|
+
await Promise.all([
|
|
20
|
+
removeIfPresent(databasePath),
|
|
21
|
+
...SIDECAR_SUFFIXES.map((suffix) => removeIfPresent(`${databasePath}${suffix}`)),
|
|
22
|
+
]);
|
|
23
|
+
}
|
|
24
|
+
/** Validate all SQLite database pages. */
|
|
25
|
+
export function verifyDatabaseIntegrity(databasePath) {
|
|
26
|
+
let db = null;
|
|
27
|
+
try {
|
|
28
|
+
db = new Database(databasePath, { readonly: true });
|
|
29
|
+
const result = db.query('PRAGMA integrity_check').get();
|
|
30
|
+
const status = result?.integrity_check ?? '';
|
|
31
|
+
if (status !== 'ok') {
|
|
32
|
+
throw new Error(`Database integrity check failed: ${status || 'unknown error'}`);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
catch (error) {
|
|
36
|
+
if (error instanceof Error && error.message.includes('integrity check failed')) {
|
|
37
|
+
throw error;
|
|
38
|
+
}
|
|
39
|
+
throw new Error('Database integrity check failed: corrupt or invalid database', {
|
|
40
|
+
cause: error,
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
finally {
|
|
44
|
+
try {
|
|
45
|
+
db?.close();
|
|
46
|
+
}
|
|
47
|
+
catch {
|
|
48
|
+
// The failed open or check may already have closed the handle.
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Produce a transactionally consistent, standalone SQLite snapshot.
|
|
54
|
+
*
|
|
55
|
+
* VACUUM INTO reads through SQLite rather than copying the live main file, so
|
|
56
|
+
* committed WAL frames are included even when an older reader blocks a WAL
|
|
57
|
+
* checkpoint.
|
|
58
|
+
*/
|
|
59
|
+
export async function createConsistentSnapshot(databasePath) {
|
|
60
|
+
const snapshotPath = uniqueSiblingPath(databasePath, 'backup-snapshot');
|
|
61
|
+
let db = null;
|
|
62
|
+
try {
|
|
63
|
+
db = new Database(databasePath, { readonly: true });
|
|
64
|
+
db.exec('PRAGMA busy_timeout = 30000');
|
|
65
|
+
db.exec(`VACUUM INTO '${quoteSqliteString(snapshotPath)}'`);
|
|
66
|
+
db.close();
|
|
67
|
+
db = null;
|
|
68
|
+
verifyDatabaseIntegrity(snapshotPath);
|
|
69
|
+
return snapshotPath;
|
|
70
|
+
}
|
|
71
|
+
catch (error) {
|
|
72
|
+
try {
|
|
73
|
+
db?.close();
|
|
74
|
+
}
|
|
75
|
+
catch {
|
|
76
|
+
// The failed snapshot may already have closed the handle.
|
|
77
|
+
}
|
|
78
|
+
await removeDatabaseArtifacts(snapshotPath);
|
|
79
|
+
throw error;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Install an already validated candidate while quarantining stale sidecars.
|
|
84
|
+
*
|
|
85
|
+
* The live main database is not touched until all sidecars have moved. If a
|
|
86
|
+
* pre-swap operation fails, moved sidecars are restored before returning.
|
|
87
|
+
*/
|
|
88
|
+
export async function installDatabaseCandidate(candidatePath, databasePath) {
|
|
89
|
+
const quarantineId = `${Date.now()}-${crypto.randomUUID()}`;
|
|
90
|
+
const movedSidecars = [];
|
|
91
|
+
try {
|
|
92
|
+
for (const suffix of SIDECAR_SUFFIXES) {
|
|
93
|
+
const live = `${databasePath}${suffix}`;
|
|
94
|
+
if (!(await Bun.file(live).exists())) {
|
|
95
|
+
continue;
|
|
96
|
+
}
|
|
97
|
+
const quarantine = `${live}.restore-quarantine-${quarantineId}`;
|
|
98
|
+
await rename(live, quarantine);
|
|
99
|
+
movedSidecars.push({ live, quarantine });
|
|
100
|
+
}
|
|
101
|
+
await rename(candidatePath, databasePath);
|
|
102
|
+
}
|
|
103
|
+
catch (error) {
|
|
104
|
+
const rollbackErrors = [];
|
|
105
|
+
for (const moved of movedSidecars.reverse()) {
|
|
106
|
+
try {
|
|
107
|
+
await rename(moved.quarantine, moved.live);
|
|
108
|
+
}
|
|
109
|
+
catch (rollbackError) {
|
|
110
|
+
rollbackErrors.push(String(rollbackError));
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
if (rollbackErrors.length > 0) {
|
|
114
|
+
throw new Error(`Restore failed and sidecar rollback failed: ${rollbackErrors.join('; ')}`, {
|
|
115
|
+
cause: error,
|
|
116
|
+
});
|
|
117
|
+
}
|
|
118
|
+
throw error;
|
|
119
|
+
}
|
|
120
|
+
const cleanup = await Promise.allSettled([
|
|
121
|
+
...movedSidecars.map(({ quarantine }) => removeIfPresent(quarantine)),
|
|
122
|
+
...SIDECAR_SUFFIXES.map((suffix) => removeIfPresent(`${candidatePath}${suffix}`)),
|
|
123
|
+
]);
|
|
124
|
+
const failures = cleanup.filter((result) => result.status === 'rejected');
|
|
125
|
+
if (failures.length > 0) {
|
|
126
|
+
backupLog.warn('Restore committed but quarantine cleanup was incomplete', {
|
|
127
|
+
errors: failures.map(({ reason }) => String(reason)),
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
}
|
|
@@ -203,7 +203,12 @@ export class SqliteStorage {
|
|
|
203
203
|
}
|
|
204
204
|
/** Flush write buffer to disk. Returns number of jobs flushed. */
|
|
205
205
|
flushWriteBuffer() {
|
|
206
|
-
|
|
206
|
+
const flushed = this.writeBuffer.flush();
|
|
207
|
+
const pending = this.writeBuffer.pendingCount;
|
|
208
|
+
if (pending > 0) {
|
|
209
|
+
throw new Error(`Cannot create a consistent backup while ${pending} write${pending === 1 ? '' : 's'} remains buffered`);
|
|
210
|
+
}
|
|
211
|
+
return flushed;
|
|
207
212
|
}
|
|
208
213
|
migrate() {
|
|
209
214
|
this.db.run(MIGRATION_TABLE);
|
|
@@ -5,5 +5,7 @@
|
|
|
5
5
|
* agent, stats interval, crash handlers and graceful shutdown are always on.
|
|
6
6
|
*/
|
|
7
7
|
import { type BunqueueConfig, type ResolvedConfig } from '../../config';
|
|
8
|
+
/** Return a startup error when an enabled file backup has no file to snapshot. */
|
|
9
|
+
export declare function backupStartupError(config: Pick<ResolvedConfig, 's3BackupEnabled' | 'dataPath'>): string | null;
|
|
8
10
|
/** Boot the full server from resolved configuration. Runs until shutdown. */
|
|
9
11
|
export declare function bootServer(fileConfig: BunqueueConfig | null, config: ResolvedConfig): void;
|