@delali/sirannon-db 0.2.3-next.32 → 0.2.3-next.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backup/index.d.ts +3 -3
- package/dist/backup-scheduler/index.d.ts +14 -1
- package/dist/backup-scheduler/index.mjs +1 -1
- package/dist/{chain-queries-CL_R_DuV.d.ts → chain-queries-BFRYrmNE.d.ts} +1 -1
- package/dist/{change-tracker-NPJjE1Sh.d.ts → change-tracker-DNisL4aw.d.ts} +1 -1
- package/dist/{chunk-MZM74XYK.mjs → chunk-7HVBKOII.mjs} +426 -19
- package/dist/{chunk-C33OO6KQ.mjs → chunk-JFZXO5S2.mjs} +72 -5
- package/dist/client/index.d.ts +8 -8
- package/dist/client/topology.d.ts +4 -4
- package/dist/{client-base-BDCz2KB1.d.ts → client-base-0QBh8JC4.d.ts} +2 -2
- package/dist/core/index.d.ts +9 -9
- package/dist/core/index.mjs +18 -8
- package/dist/core/writer-worker.mjs +1 -0
- package/dist/{database-Cy4ewMCg.d.ts → database-CKZP6SiT.d.ts} +11 -7
- package/dist/driver/better-sqlite3.d.ts +8 -1
- package/dist/driver/better-sqlite3.mjs +10 -5
- package/dist/driver/bun.d.ts +1 -1
- package/dist/driver/expo.d.ts +1 -1
- package/dist/driver/node.d.ts +8 -1
- package/dist/driver/node.mjs +14 -3
- package/dist/driver/wa-sqlite.d.ts +1 -1
- package/dist/file-migrations/index.d.ts +1 -1
- package/dist/{protocol-D9Yt7YOR.d.ts → protocol-2vVZ4ijz.d.ts} +1 -1
- package/dist/replication/index.d.ts +7 -7
- package/dist/server/index.d.ts +8 -8
- package/dist/{server-options-CosjHEle.d.ts → server-options-DxHqvZw6.d.ts} +1 -1
- package/dist/{sirannon-BMw6cs63.d.ts → sirannon-Clqx8Xyu.d.ts} +2 -2
- package/dist/transport/grpc.d.ts +3 -3
- package/dist/transport/memory.d.ts +3 -3
- package/dist/{types-Dm_POacb.d.ts → types-DAemUqLQ.d.ts} +2 -2
- package/dist/{types-CwIiWpVr.d.ts → types-GuxbxlwL.d.ts} +39 -13
- package/package.json +10 -1
package/dist/backup/index.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { e as BackupDestination, f as BackupRunReport } from '../types-
|
|
2
|
-
export { g as BackupChain, h as BackupChainBase, i as BackupChainChange, j as BackupChainPosition, k as BackupChainRecord, l as BackupCycle, m as BackupCycleOptions, n as BackupPiece, o as BackupProgress, p as BackupToDestinationOptions, q as DEFAULT_CHAIN_NAME, r as chainLogName, s as readBackupChains } from '../types-
|
|
3
|
-
export { B as BackupCapabilities, a as BackupRestorePlan, b as BackupSafeToDeleteOptions, c as backupPiecesSafeToDelete, p as planBackupRestore } from '../chain-queries-
|
|
1
|
+
import { e as BackupDestination, f as BackupRunReport } from '../types-GuxbxlwL.js';
|
|
2
|
+
export { g as BackupChain, h as BackupChainBase, i as BackupChainChange, j as BackupChainPosition, k as BackupChainRecord, l as BackupCycle, m as BackupCycleOptions, n as BackupPiece, o as BackupProgress, p as BackupToDestinationOptions, q as DEFAULT_CHAIN_NAME, r as chainLogName, s as readBackupChains } from '../types-GuxbxlwL.js';
|
|
3
|
+
export { B as BackupCapabilities, a as BackupRestorePlan, b as BackupSafeToDeleteOptions, c as backupPiecesSafeToDelete, p as planBackupRestore } from '../chain-queries-BFRYrmNE.js';
|
|
4
4
|
import '../query-types-Cv8N7pXj.js';
|
|
5
5
|
|
|
6
6
|
/** What one assembled file took to build.
|
|
@@ -1,9 +1,22 @@
|
|
|
1
|
-
import { t as SQLiteConnection, u as BackupRunRequest, f as BackupRunReport, v as BackupScheduleOptions } from '../types-
|
|
1
|
+
import { t as SQLiteConnection, u as BackupRunRequest, f as BackupRunReport, v as BackupScheduleOptions } from '../types-GuxbxlwL.js';
|
|
2
2
|
import '../query-types-Cv8N7pXj.js';
|
|
3
3
|
|
|
4
|
+
/** What one runtime needs before a copy can reach a destination without a local file.
|
|
5
|
+
* @internal
|
|
6
|
+
*/
|
|
7
|
+
interface BackupStreamingSupport {
|
|
8
|
+
/** Absolute path of the compiled extension that carries the bytes. */
|
|
9
|
+
extensionPath: string;
|
|
10
|
+
/** Opens the connection the extension's statements run on. */
|
|
11
|
+
openConnection(): Promise<SQLiteConnection>;
|
|
12
|
+
}
|
|
13
|
+
|
|
4
14
|
declare class BackupManager {
|
|
15
|
+
private readonly streaming?;
|
|
16
|
+
constructor(streaming?: BackupStreamingSupport | undefined);
|
|
5
17
|
backup(conn: SQLiteConnection, destPath: string, onFirstStep?: () => void): Promise<void>;
|
|
6
18
|
copyToDestination(conn: SQLiteConnection, request: BackupRunRequest): Promise<BackupRunReport>;
|
|
19
|
+
streamsToDestination(): boolean;
|
|
7
20
|
generateFilename(): string;
|
|
8
21
|
rotate(dir: string, maxFiles: number): void;
|
|
9
22
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { h as BackupChainBase, i as BackupChainChange, g as BackupChain, k as BackupChainRecord } from './types-
|
|
1
|
+
import { h as BackupChainBase, i as BackupChainChange, g as BackupChain, k as BackupChainRecord } from './types-GuxbxlwL.js';
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
4
|
* What one runtime supports of the backup operations, so a caller learns
|
|
@@ -1,19 +1,64 @@
|
|
|
1
1
|
import { startCopyWithoutHoldingWriter } from './chunk-W67A4H2E.mjs';
|
|
2
2
|
import { randomHex } from './chunk-V5FUJEYY.mjs';
|
|
3
|
-
import {
|
|
3
|
+
import { SirannonError, BackupError } from './chunk-PBRXXISQ.mjs';
|
|
4
4
|
import { existsSync, mkdirSync, rmSync, readdirSync, lstatSync } from 'fs';
|
|
5
5
|
import { resolve, dirname, join } from 'path';
|
|
6
6
|
import { open, mkdtemp, rm } from 'fs/promises';
|
|
7
7
|
import { tmpdir } from 'os';
|
|
8
8
|
import { createHash } from 'crypto';
|
|
9
9
|
|
|
10
|
-
|
|
10
|
+
// src/core/backup/destination-deadline.ts
|
|
11
|
+
var DEFAULT_DESTINATION_TIMEOUT_MS = 6e5;
|
|
12
|
+
async function withinDeadline(operation, action, timeoutMs) {
|
|
13
|
+
let timer = null;
|
|
14
|
+
const deadline = new Promise((_, reject) => {
|
|
15
|
+
timer = setTimeout(() => {
|
|
16
|
+
reject(
|
|
17
|
+
new SirannonError(
|
|
18
|
+
`The destination did not ${action} within ${timeoutMs}ms, so the run stopped`,
|
|
19
|
+
"BACKUP_DESTINATION_ERROR"
|
|
20
|
+
)
|
|
21
|
+
);
|
|
22
|
+
}, timeoutMs);
|
|
23
|
+
});
|
|
24
|
+
try {
|
|
25
|
+
return await Promise.race([operation, deadline]);
|
|
26
|
+
} finally {
|
|
27
|
+
if (timer) clearTimeout(timer);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
function destinationWithDeadline(destination, timeoutMs) {
|
|
31
|
+
if (!Number.isFinite(timeoutMs) || timeoutMs < 0) {
|
|
32
|
+
throw new SirannonError(
|
|
33
|
+
`The destination deadline must be a number of milliseconds that is zero or above, and it was ${timeoutMs}`,
|
|
34
|
+
"BACKUP_ERROR"
|
|
35
|
+
);
|
|
36
|
+
}
|
|
37
|
+
if (timeoutMs === 0) return destination;
|
|
38
|
+
return {
|
|
39
|
+
writePiece: (name, index, bytes) => withinDeadline(destination.writePiece(name, index, bytes), `store piece ${index} of '${name}'`, timeoutMs),
|
|
40
|
+
readPiece: (name, index) => withinDeadline(destination.readPiece(name, index), `return piece ${index} of '${name}'`, timeoutMs),
|
|
41
|
+
listPieces: (name) => withinDeadline(destination.listPieces(name), `list the pieces of '${name}'`, timeoutMs)
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
function destinationPieceError(name, index, err) {
|
|
11
45
|
if (err instanceof SirannonError) return err;
|
|
12
46
|
return new SirannonError(
|
|
13
47
|
`The destination refused piece ${index} of '${name}': ${err instanceof Error ? err.message : String(err)}`,
|
|
14
48
|
"BACKUP_DESTINATION_ERROR"
|
|
15
49
|
);
|
|
16
50
|
}
|
|
51
|
+
async function fingerprintStoredPieces(destination, name, pieceCount) {
|
|
52
|
+
const digest = createHash("sha256");
|
|
53
|
+
for (let index = 0; index < pieceCount; index++) {
|
|
54
|
+
try {
|
|
55
|
+
digest.update(await destination.readPiece(name, index));
|
|
56
|
+
} catch (err) {
|
|
57
|
+
throw destinationPieceError(name, index, err);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
return digest.digest("hex");
|
|
61
|
+
}
|
|
17
62
|
async function sendFileInPieces(sourcePath, destination, name, pieceBytes, fingerprint, report) {
|
|
18
63
|
const file = await open(sourcePath, "r");
|
|
19
64
|
const digest = fingerprint ? createHash("sha256") : null;
|
|
@@ -35,7 +80,7 @@ async function sendFileInPieces(sourcePath, destination, name, pieceBytes, finge
|
|
|
35
80
|
try {
|
|
36
81
|
await destination.writePiece(name, index, piece);
|
|
37
82
|
} catch (err) {
|
|
38
|
-
throw
|
|
83
|
+
throw destinationPieceError(name, index, err);
|
|
39
84
|
}
|
|
40
85
|
index++;
|
|
41
86
|
bytesWritten += filled;
|
|
@@ -50,6 +95,22 @@ async function sendFileInPieces(sourcePath, destination, name, pieceBytes, finge
|
|
|
50
95
|
|
|
51
96
|
// src/core/backup/report.ts
|
|
52
97
|
var DEFAULT_PIECE_BYTES = 16 * 1024 * 1024;
|
|
98
|
+
function defaultDestinationName() {
|
|
99
|
+
return `backup-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}.db`;
|
|
100
|
+
}
|
|
101
|
+
function assertPieceBytes(pieceBytes) {
|
|
102
|
+
if (!Number.isInteger(pieceBytes) || pieceBytes <= 0) {
|
|
103
|
+
throw new SirannonError(
|
|
104
|
+
`Piece size must be a positive whole number of bytes, and it was ${pieceBytes}`,
|
|
105
|
+
"BACKUP_ERROR"
|
|
106
|
+
);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
async function readPageSize(conn) {
|
|
110
|
+
const stmt = await conn.prepare("PRAGMA page_size");
|
|
111
|
+
const row = await stmt.get();
|
|
112
|
+
return row ? Number(row.page_size) : 0;
|
|
113
|
+
}
|
|
53
114
|
|
|
54
115
|
// src/core/backup/stepped-copy.ts
|
|
55
116
|
var DEFAULT_PAGES_PER_STEP = 256;
|
|
@@ -88,6 +149,7 @@ async function copyDatabaseStepwise(conn, options) {
|
|
|
88
149
|
"BACKUP_UNSUPPORTED"
|
|
89
150
|
);
|
|
90
151
|
}
|
|
152
|
+
const pagesPerStep = options.pagesPerStep ?? DEFAULT_PAGES_PER_STEP;
|
|
91
153
|
const restartLimit = options.restartLimit ?? DEFAULT_RESTART_LIMIT;
|
|
92
154
|
const stallTimeoutMs = options.stallTimeoutMs ?? DEFAULT_STALL_TIMEOUT_MS;
|
|
93
155
|
const noProgressStepLimit = options.noProgressStepLimit ?? DEFAULT_NO_PROGRESS_STEP_LIMIT;
|
|
@@ -95,6 +157,7 @@ async function copyDatabaseStepwise(conn, options) {
|
|
|
95
157
|
let restarts = 0;
|
|
96
158
|
let furthestCopied = -1;
|
|
97
159
|
let stepsWithoutProgress = 0;
|
|
160
|
+
let pausedPreviousStep = false;
|
|
98
161
|
let stopped = null;
|
|
99
162
|
let stallTimer = null;
|
|
100
163
|
let reportStall = () => {
|
|
@@ -106,6 +169,10 @@ async function copyDatabaseStepwise(conn, options) {
|
|
|
106
169
|
if (stallTimeoutMs <= 0) return;
|
|
107
170
|
if (stallTimer) clearTimeout(stallTimer);
|
|
108
171
|
stallTimer = setTimeout(() => {
|
|
172
|
+
if (options.pauseWhile?.() === true) {
|
|
173
|
+
armStall();
|
|
174
|
+
return;
|
|
175
|
+
}
|
|
109
176
|
stopped = stallError(stallTimeoutMs, options.destPath);
|
|
110
177
|
reportStall(stopped);
|
|
111
178
|
}, stallTimeoutMs);
|
|
@@ -114,10 +181,16 @@ async function copyDatabaseStepwise(conn, options) {
|
|
|
114
181
|
armStall();
|
|
115
182
|
const copy = conn.copyDatabase({
|
|
116
183
|
destPath: options.destPath,
|
|
117
|
-
pagesPerStep
|
|
184
|
+
pagesPerStep,
|
|
118
185
|
onStep: (step) => {
|
|
119
186
|
if (stopped) throw stopped;
|
|
120
187
|
armStall();
|
|
188
|
+
const pausing = options.pauseWhile?.() === true;
|
|
189
|
+
if (pausedPreviousStep && previous !== null && copiedPages(step) === copiedPages(previous)) {
|
|
190
|
+
pausedPreviousStep = pausing;
|
|
191
|
+
return pausing ? 0 : pagesPerStep;
|
|
192
|
+
}
|
|
193
|
+
pausedPreviousStep = pausing;
|
|
121
194
|
if (hasRestarted(step, previous)) {
|
|
122
195
|
restarts++;
|
|
123
196
|
if (restarts > restartLimit) {
|
|
@@ -134,6 +207,7 @@ async function copyDatabaseStepwise(conn, options) {
|
|
|
134
207
|
}
|
|
135
208
|
previous = step;
|
|
136
209
|
options.onStep?.({ totalPages: step.totalPages, remainingPages: step.remainingPages, restarts });
|
|
210
|
+
return pausing ? 0 : pagesPerStep;
|
|
137
211
|
}
|
|
138
212
|
}).catch((err) => {
|
|
139
213
|
if (stopped) throw stopped;
|
|
@@ -154,21 +228,11 @@ async function copyDatabaseStepwise(conn, options) {
|
|
|
154
228
|
|
|
155
229
|
// src/core/backup/staged-copy.ts
|
|
156
230
|
var STAGED_FILE_NAME = "copy.db";
|
|
157
|
-
async function readPageSize(conn) {
|
|
158
|
-
const stmt = await conn.prepare("PRAGMA page_size");
|
|
159
|
-
const row = await stmt.get();
|
|
160
|
-
return row ? Number(row.page_size) : 0;
|
|
161
|
-
}
|
|
162
231
|
async function copyToDestinationStaged(conn, request) {
|
|
163
232
|
const runId = randomHex(8);
|
|
164
|
-
const name = request.name ??
|
|
233
|
+
const name = request.name ?? defaultDestinationName();
|
|
165
234
|
const pieceBytes = request.pieceBytes ?? DEFAULT_PIECE_BYTES;
|
|
166
|
-
|
|
167
|
-
throw new SirannonError(
|
|
168
|
-
`Piece size must be a positive whole number of bytes, and it was ${pieceBytes}`,
|
|
169
|
-
"BACKUP_ERROR"
|
|
170
|
-
);
|
|
171
|
-
}
|
|
235
|
+
assertPieceBytes(pieceBytes);
|
|
172
236
|
const startedAt = Date.now();
|
|
173
237
|
const stagingRoot = await mkdtemp(join(request.stagingDir ?? tmpdir(), "sirannon-backup-"));
|
|
174
238
|
const stagedPath = join(stagingRoot, STAGED_FILE_NAME);
|
|
@@ -243,6 +307,336 @@ async function copyToDestinationStaged(conn, request) {
|
|
|
243
307
|
}
|
|
244
308
|
}
|
|
245
309
|
|
|
310
|
+
// src/core/system-catalog/backup-stream-functions.ts
|
|
311
|
+
var SELECT_NEW_STREAM_ID = "SELECT sirannon_stream_open(?, ?, ?) AS streamId";
|
|
312
|
+
var SELECT_NEXT_PIECE = "SELECT sirannon_stream_take(?) AS piece";
|
|
313
|
+
var SELECT_BYTES_WRITTEN = "SELECT sirannon_stream_written(?) AS bytes";
|
|
314
|
+
var SELECT_FAILURE = "SELECT sirannon_stream_error(?) AS failure";
|
|
315
|
+
var SELECT_FINISHED_BYTES = "SELECT sirannon_stream_finish(?) AS bytes";
|
|
316
|
+
var SELECT_RELEASED_BYTES = "SELECT sirannon_stream_close(?) AS bytes";
|
|
317
|
+
async function prepareBackupStreamStatements(conn) {
|
|
318
|
+
const [newStreamId, nextPiece, bytesWritten, failure, finishedBytes, releasedBytes] = await Promise.all([
|
|
319
|
+
conn.prepare(SELECT_NEW_STREAM_ID),
|
|
320
|
+
conn.prepare(SELECT_NEXT_PIECE),
|
|
321
|
+
conn.prepare(SELECT_BYTES_WRITTEN),
|
|
322
|
+
conn.prepare(SELECT_FAILURE),
|
|
323
|
+
conn.prepare(SELECT_FINISHED_BYTES),
|
|
324
|
+
conn.prepare(SELECT_RELEASED_BYTES)
|
|
325
|
+
]);
|
|
326
|
+
const readBytes = async (stmt, streamId) => {
|
|
327
|
+
const row = await stmt.get(streamId);
|
|
328
|
+
return row ? Number(row.bytes) : 0;
|
|
329
|
+
};
|
|
330
|
+
return {
|
|
331
|
+
async selectNewStreamId(pieceBytes, maxQueuedPieces, waitWhenFull) {
|
|
332
|
+
const row = await newStreamId.get(pieceBytes, maxQueuedPieces, waitWhenFull);
|
|
333
|
+
return row ? Number(row.streamId) : 0;
|
|
334
|
+
},
|
|
335
|
+
async selectNextPiece(streamId) {
|
|
336
|
+
const row = await nextPiece.get(streamId);
|
|
337
|
+
return row?.piece ?? null;
|
|
338
|
+
},
|
|
339
|
+
selectBytesWritten: (streamId) => readBytes(bytesWritten, streamId),
|
|
340
|
+
async selectFailure(streamId) {
|
|
341
|
+
const row = await failure.get(streamId);
|
|
342
|
+
return row?.failure ?? null;
|
|
343
|
+
},
|
|
344
|
+
selectFinishedBytes: (streamId) => readBytes(finishedBytes, streamId),
|
|
345
|
+
selectReleasedBytes: (streamId) => readBytes(releasedBytes, streamId)
|
|
346
|
+
};
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
// src/core/backup/vfs/stream-host.ts
|
|
350
|
+
var PIECE_HEADER_BYTES = 8;
|
|
351
|
+
function decodePiece(framed) {
|
|
352
|
+
if (framed.byteLength < PIECE_HEADER_BYTES) {
|
|
353
|
+
throw new SirannonError(
|
|
354
|
+
`The streaming extension returned ${framed.byteLength} bytes for a piece, which is less than its ${PIECE_HEADER_BYTES}-byte header`,
|
|
355
|
+
"BACKUP_ERROR"
|
|
356
|
+
);
|
|
357
|
+
}
|
|
358
|
+
const header = new DataView(framed.buffer, framed.byteOffset, PIECE_HEADER_BYTES);
|
|
359
|
+
const index = header.getUint32(0, true);
|
|
360
|
+
const length = header.getUint32(4, true);
|
|
361
|
+
const carried = framed.byteLength - PIECE_HEADER_BYTES;
|
|
362
|
+
if (carried < length) {
|
|
363
|
+
throw new SirannonError(
|
|
364
|
+
`The streaming extension declared ${length} bytes for piece ${index} and returned ${carried}`,
|
|
365
|
+
"BACKUP_ERROR"
|
|
366
|
+
);
|
|
367
|
+
}
|
|
368
|
+
return { index, bytes: framed.subarray(PIECE_HEADER_BYTES, PIECE_HEADER_BYTES + length) };
|
|
369
|
+
}
|
|
370
|
+
var BackupStreamHost = class _BackupStreamHost {
|
|
371
|
+
constructor(connection, statements) {
|
|
372
|
+
this.connection = connection;
|
|
373
|
+
this.statements = statements;
|
|
374
|
+
}
|
|
375
|
+
/**
|
|
376
|
+
* Opens a connection, loads the compiled extension into it, and compiles the
|
|
377
|
+
* statements one run needs.
|
|
378
|
+
*
|
|
379
|
+
* @param openConnection - Opens the connection the statements run on.
|
|
380
|
+
* @param extensionPath - Absolute path of the compiled extension.
|
|
381
|
+
* @returns A host ready to open a stream.
|
|
382
|
+
*/
|
|
383
|
+
static async start(openConnection, extensionPath) {
|
|
384
|
+
const connection = await openConnection();
|
|
385
|
+
try {
|
|
386
|
+
if (!connection.loadExtension) {
|
|
387
|
+
throw new SirannonError(
|
|
388
|
+
"This driver opens connections with no extension loading call, so it cannot stream a copy to a destination",
|
|
389
|
+
"BACKUP_UNSUPPORTED"
|
|
390
|
+
);
|
|
391
|
+
}
|
|
392
|
+
await connection.loadExtension(extensionPath);
|
|
393
|
+
return new _BackupStreamHost(connection, await prepareBackupStreamStatements(connection));
|
|
394
|
+
} catch (err) {
|
|
395
|
+
await connection.close().catch(() => void 0);
|
|
396
|
+
throw err;
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
/**
|
|
400
|
+
* Opens one stream and returns the identifier that names it in the
|
|
401
|
+
* destination URI.
|
|
402
|
+
*
|
|
403
|
+
* @param pieceBytes - Bytes one whole piece holds.
|
|
404
|
+
* @param maxQueuedPieces - Pieces the extension holds before it stops taking more.
|
|
405
|
+
* @param waitWhenFull - Whether the copy waits for the destination to catch up rather than queueing further pieces.
|
|
406
|
+
* @returns The identifier of the open stream.
|
|
407
|
+
*/
|
|
408
|
+
async open(pieceBytes, maxQueuedPieces, waitWhenFull) {
|
|
409
|
+
const streamId = await this.statements.selectNewStreamId(pieceBytes, maxQueuedPieces, waitWhenFull ? 1 : 0);
|
|
410
|
+
if (streamId === 0) {
|
|
411
|
+
throw new SirannonError("The streaming extension opened no stream for this run", "BACKUP_ERROR");
|
|
412
|
+
}
|
|
413
|
+
return streamId;
|
|
414
|
+
}
|
|
415
|
+
/**
|
|
416
|
+
* Takes the next whole piece the copy has produced.
|
|
417
|
+
*
|
|
418
|
+
* @param streamId - Stream to take from.
|
|
419
|
+
* @returns The piece, or null where the copy has produced none since the last call.
|
|
420
|
+
*/
|
|
421
|
+
async take(streamId) {
|
|
422
|
+
const framed = await this.statements.selectNextPiece(streamId);
|
|
423
|
+
return framed ? decodePiece(framed) : null;
|
|
424
|
+
}
|
|
425
|
+
/**
|
|
426
|
+
* Reports how many bytes of the copy have reached the extension.
|
|
427
|
+
*
|
|
428
|
+
* @param streamId - Stream to ask about.
|
|
429
|
+
* @returns Bytes SQLite has written to this stream.
|
|
430
|
+
*/
|
|
431
|
+
written(streamId) {
|
|
432
|
+
return this.statements.selectBytesWritten(streamId);
|
|
433
|
+
}
|
|
434
|
+
/**
|
|
435
|
+
* Reports what stopped a stream, where anything did.
|
|
436
|
+
*
|
|
437
|
+
* @param streamId - Stream to ask about.
|
|
438
|
+
* @returns The failure the extension recorded, or null where it recorded none.
|
|
439
|
+
*/
|
|
440
|
+
failure(streamId) {
|
|
441
|
+
return this.statements.selectFailure(streamId);
|
|
442
|
+
}
|
|
443
|
+
/**
|
|
444
|
+
* Closes the file to further writes and queues the pieces the extension was
|
|
445
|
+
* still holding, which are the first piece and the last one.
|
|
446
|
+
*
|
|
447
|
+
* @param streamId - Stream to finish.
|
|
448
|
+
* @returns Bytes the finished file holds.
|
|
449
|
+
*/
|
|
450
|
+
finish(streamId) {
|
|
451
|
+
return this.statements.selectFinishedBytes(streamId);
|
|
452
|
+
}
|
|
453
|
+
/**
|
|
454
|
+
* Releases a stream and everything it still held.
|
|
455
|
+
*
|
|
456
|
+
* @param streamId - Stream to release.
|
|
457
|
+
*/
|
|
458
|
+
async close(streamId) {
|
|
459
|
+
await this.statements.selectReleasedBytes(streamId);
|
|
460
|
+
}
|
|
461
|
+
/** Closes the connection the statements ran on. */
|
|
462
|
+
async stop() {
|
|
463
|
+
await this.connection.close();
|
|
464
|
+
}
|
|
465
|
+
};
|
|
466
|
+
|
|
467
|
+
// src/core/backup/streamed-copy.ts
|
|
468
|
+
var VFS_NAME = "sirannon";
|
|
469
|
+
var PIECE_BLOCK_BYTES = 512;
|
|
470
|
+
var DRAIN_POLL_MS = 1;
|
|
471
|
+
function destinationUri(streamId) {
|
|
472
|
+
return `file:sirannon-stream-${streamId}?vfs=${VFS_NAME}`;
|
|
473
|
+
}
|
|
474
|
+
function sizeTheSteps(requestedPagesPerStep, pieceBytes, pageSize) {
|
|
475
|
+
const pagesPerPiece = Math.max(1, Math.floor(pieceBytes / pageSize));
|
|
476
|
+
const pagesPerStep = Math.max(1, Math.min(requestedPagesPerStep, pagesPerPiece));
|
|
477
|
+
const piecesPerStep = Math.max(1, Math.ceil(pagesPerStep * pageSize / pieceBytes));
|
|
478
|
+
return { pagesPerStep, maxQueuedPieces: piecesPerStep + 1 };
|
|
479
|
+
}
|
|
480
|
+
function assertWholeBlocks(pieceBytes) {
|
|
481
|
+
if (pieceBytes % PIECE_BLOCK_BYTES !== 0) {
|
|
482
|
+
throw new SirannonError(
|
|
483
|
+
`A streamed copy hands SQLite whole ${PIECE_BLOCK_BYTES}-byte blocks, so its piece size must divide by ${PIECE_BLOCK_BYTES}, and it was ${pieceBytes}`,
|
|
484
|
+
"BACKUP_ERROR"
|
|
485
|
+
);
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
async function removeFilesNamedAfterTheUri(uri) {
|
|
489
|
+
const stray = join(process.cwd(), uri);
|
|
490
|
+
await rm(stray, { force: true }).catch(() => void 0);
|
|
491
|
+
await rm(`${stray}-journal`, { force: true }).catch(() => void 0);
|
|
492
|
+
}
|
|
493
|
+
async function assertTheStreamCarriedTheCopy(host, streamId, bytesStreamed, pageCount, uri) {
|
|
494
|
+
const failure = await recordedFailure(host, streamId);
|
|
495
|
+
if (failure) throw failure;
|
|
496
|
+
if (bytesStreamed === 0 && pageCount > 0) {
|
|
497
|
+
await removeFilesNamedAfterTheUri(uri);
|
|
498
|
+
throw new SirannonError(
|
|
499
|
+
"SQLite read the destination as an ordinary file name rather than as a URI, so the copy reached no destination. This runtime parses URI file names only where the operator sets SQLITE_USE_URI=1 in the environment before the SQLite module loads.",
|
|
500
|
+
"BACKUP_UNSUPPORTED"
|
|
501
|
+
);
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
async function recordedFailure(host, streamId) {
|
|
505
|
+
const failure = await host.failure(streamId).catch(() => null);
|
|
506
|
+
return failure ? new SirannonError(`The streamed copy stopped because ${failure}.`, "BACKUP_ERROR") : null;
|
|
507
|
+
}
|
|
508
|
+
async function copyToDestinationStreamed(conn, request, support) {
|
|
509
|
+
const runId = randomHex(8);
|
|
510
|
+
const name = request.name ?? defaultDestinationName();
|
|
511
|
+
const pieceBytes = request.pieceBytes ?? DEFAULT_PIECE_BYTES;
|
|
512
|
+
assertPieceBytes(pieceBytes);
|
|
513
|
+
assertWholeBlocks(pieceBytes);
|
|
514
|
+
const startedAt = Date.now();
|
|
515
|
+
const pageSize = await readPageSize(conn);
|
|
516
|
+
const { pagesPerStep, maxQueuedPieces } = sizeTheSteps(
|
|
517
|
+
request.pagesPerStep ?? DEFAULT_PAGES_PER_STEP,
|
|
518
|
+
pieceBytes,
|
|
519
|
+
pageSize
|
|
520
|
+
);
|
|
521
|
+
const host = await BackupStreamHost.start(support.openConnection, support.extensionPath);
|
|
522
|
+
let streamId;
|
|
523
|
+
try {
|
|
524
|
+
streamId = await host.open(pieceBytes, maxQueuedPieces, conn.copyRunsOffCallerThread === true);
|
|
525
|
+
} catch (err) {
|
|
526
|
+
await host.stop().catch(() => void 0);
|
|
527
|
+
throw err;
|
|
528
|
+
}
|
|
529
|
+
const uri = destinationUri(streamId);
|
|
530
|
+
let firstStepSeen = false;
|
|
531
|
+
let copyLeftRunning = null;
|
|
532
|
+
let writingPiece = false;
|
|
533
|
+
let copying = true;
|
|
534
|
+
let discarding = false;
|
|
535
|
+
let piecesWritten = 0;
|
|
536
|
+
let bytesWritten = 0;
|
|
537
|
+
let transferMs = 0;
|
|
538
|
+
let totalPages = 0;
|
|
539
|
+
let remainingPages = 0;
|
|
540
|
+
let restarts = 0;
|
|
541
|
+
let pumpFailure = null;
|
|
542
|
+
const emit = (progress) => request.onProgress?.({ runId, ...progress });
|
|
543
|
+
const pump = async () => {
|
|
544
|
+
for (; ; ) {
|
|
545
|
+
const piece = await host.take(streamId);
|
|
546
|
+
if (!piece) {
|
|
547
|
+
if (!copying) return;
|
|
548
|
+
await new Promise((resolve3) => setTimeout(resolve3, DRAIN_POLL_MS));
|
|
549
|
+
continue;
|
|
550
|
+
}
|
|
551
|
+
if (discarding) continue;
|
|
552
|
+
const writeStartedAt = Date.now();
|
|
553
|
+
writingPiece = true;
|
|
554
|
+
try {
|
|
555
|
+
await request.destination.writePiece(name, piece.index, piece.bytes);
|
|
556
|
+
} catch (err) {
|
|
557
|
+
pumpFailure = destinationPieceError(name, piece.index, err);
|
|
558
|
+
discarding = true;
|
|
559
|
+
continue;
|
|
560
|
+
} finally {
|
|
561
|
+
writingPiece = false;
|
|
562
|
+
transferMs += Date.now() - writeStartedAt;
|
|
563
|
+
}
|
|
564
|
+
piecesWritten++;
|
|
565
|
+
bytesWritten += piece.bytes.byteLength;
|
|
566
|
+
emit({ phase: "transfer", totalPages, remainingPages, restarts, piecesWritten, bytesWritten });
|
|
567
|
+
}
|
|
568
|
+
};
|
|
569
|
+
const pumping = pump().catch((err) => {
|
|
570
|
+
pumpFailure = err;
|
|
571
|
+
});
|
|
572
|
+
try {
|
|
573
|
+
const copyStartedAt = Date.now();
|
|
574
|
+
const copy = await copyDatabaseStepwise(conn, {
|
|
575
|
+
destPath: uri,
|
|
576
|
+
pagesPerStep,
|
|
577
|
+
...request.restartLimit === void 0 ? {} : { restartLimit: request.restartLimit },
|
|
578
|
+
...request.stallTimeoutMs === void 0 ? {} : { stallTimeoutMs: request.stallTimeoutMs },
|
|
579
|
+
...request.noProgressStepLimit === void 0 ? {} : { noProgressStepLimit: request.noProgressStepLimit },
|
|
580
|
+
pauseWhile: () => writingPiece,
|
|
581
|
+
onCopyLeftRunning: (running) => {
|
|
582
|
+
copyLeftRunning = running;
|
|
583
|
+
},
|
|
584
|
+
onStep: (step) => {
|
|
585
|
+
if (pumpFailure) throw pumpFailure;
|
|
586
|
+
if (!firstStepSeen) {
|
|
587
|
+
firstStepSeen = true;
|
|
588
|
+
request.onFirstStep?.();
|
|
589
|
+
}
|
|
590
|
+
totalPages = step.totalPages;
|
|
591
|
+
remainingPages = step.remainingPages;
|
|
592
|
+
restarts = step.restarts;
|
|
593
|
+
emit({ phase: "copy", ...step, piecesWritten, bytesWritten });
|
|
594
|
+
}
|
|
595
|
+
});
|
|
596
|
+
const copyMs = Date.now() - copyStartedAt;
|
|
597
|
+
const bytesStreamed = await host.finish(streamId);
|
|
598
|
+
copying = false;
|
|
599
|
+
await pumping;
|
|
600
|
+
if (pumpFailure) throw pumpFailure;
|
|
601
|
+
await assertTheStreamCarriedTheCopy(host, streamId, bytesStreamed, copy.pageCount, uri);
|
|
602
|
+
const fingerprint = request.fingerprint === false ? void 0 : await fingerprintStoredPieces(request.destination, name, piecesWritten);
|
|
603
|
+
const finishedAt = Date.now();
|
|
604
|
+
return {
|
|
605
|
+
runId,
|
|
606
|
+
databaseId: request.databaseId,
|
|
607
|
+
sourcePath: request.sourcePath,
|
|
608
|
+
kind: "full",
|
|
609
|
+
chainId: request.chainId ?? randomHex(8),
|
|
610
|
+
route: "streamed",
|
|
611
|
+
destinationName: name,
|
|
612
|
+
startedAt,
|
|
613
|
+
finishedAt,
|
|
614
|
+
durationMs: finishedAt - startedAt,
|
|
615
|
+
copyMs,
|
|
616
|
+
transferMs,
|
|
617
|
+
pageCount: copy.pageCount,
|
|
618
|
+
pageSize,
|
|
619
|
+
bytesWritten,
|
|
620
|
+
pieceCount: piecesWritten,
|
|
621
|
+
pieceBytes,
|
|
622
|
+
restarts: copy.restarts,
|
|
623
|
+
...fingerprint ? { fingerprint } : {}
|
|
624
|
+
};
|
|
625
|
+
} catch (err) {
|
|
626
|
+
throw await recordedFailure(host, streamId) ?? err;
|
|
627
|
+
} finally {
|
|
628
|
+
discarding = true;
|
|
629
|
+
const release = async () => {
|
|
630
|
+
copying = false;
|
|
631
|
+
await pumping;
|
|
632
|
+
await host.close(streamId).catch(() => void 0);
|
|
633
|
+
await host.stop().catch(() => void 0);
|
|
634
|
+
};
|
|
635
|
+
if (copyLeftRunning) void copyLeftRunning.then(release, release);
|
|
636
|
+
else await release();
|
|
637
|
+
}
|
|
638
|
+
}
|
|
639
|
+
|
|
246
640
|
// src/core/backup/backup.ts
|
|
247
641
|
var BACKUP_FILE_PREFIX = "backup";
|
|
248
642
|
function once(action) {
|
|
@@ -261,6 +655,9 @@ function hasControlCharacters(s) {
|
|
|
261
655
|
return false;
|
|
262
656
|
}
|
|
263
657
|
var BackupManager = class {
|
|
658
|
+
constructor(streaming) {
|
|
659
|
+
this.streaming = streaming;
|
|
660
|
+
}
|
|
264
661
|
async backup(conn, destPath, onFirstStep) {
|
|
265
662
|
if (hasControlCharacters(destPath)) {
|
|
266
663
|
throw new BackupError("Backup path contains invalid characters");
|
|
@@ -294,8 +691,18 @@ var BackupManager = class {
|
|
|
294
691
|
throw new BackupError(`Backup to '${destPath}' failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
295
692
|
}
|
|
296
693
|
}
|
|
297
|
-
copyToDestination(conn, request) {
|
|
298
|
-
|
|
694
|
+
async copyToDestination(conn, request) {
|
|
695
|
+
const bounded = {
|
|
696
|
+
...request,
|
|
697
|
+
destination: destinationWithDeadline(
|
|
698
|
+
request.destination,
|
|
699
|
+
request.destinationTimeoutMs ?? DEFAULT_DESTINATION_TIMEOUT_MS
|
|
700
|
+
)
|
|
701
|
+
};
|
|
702
|
+
return this.streaming ? copyToDestinationStreamed(conn, bounded, this.streaming) : copyToDestinationStaged(conn, bounded);
|
|
703
|
+
}
|
|
704
|
+
streamsToDestination() {
|
|
705
|
+
return this.streaming !== void 0;
|
|
299
706
|
}
|
|
300
707
|
generateFilename() {
|
|
301
708
|
const ts = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
@@ -731,4 +1138,4 @@ var BackupScheduler = class {
|
|
|
731
1138
|
}
|
|
732
1139
|
};
|
|
733
1140
|
|
|
734
|
-
export { BackupManager, BackupScheduler, DEFAULT_PIECE_BYTES, sendFileInPieces };
|
|
1141
|
+
export { BackupManager, BackupScheduler, DEFAULT_DESTINATION_TIMEOUT_MS, DEFAULT_PIECE_BYTES, destinationWithDeadline, sendFileInPieces };
|