@delali/sirannon-db 0.2.3-next.30 → 0.2.3-next.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backup/index.d.ts +30 -0
- package/dist/backup/index.mjs +52 -0
- package/dist/backup-scheduler/index.d.ts +5 -3
- package/dist/backup-scheduler/index.mjs +3 -1
- package/dist/capabilities-DagBnGk8.d.ts +21 -0
- package/dist/{change-tracker-rTXrPhQq.d.ts → change-tracker-7wXnBjUN.d.ts} +2 -2
- package/dist/{chunk-VOYGMAU7.mjs → chunk-23HKRYSM.mjs} +258 -5
- package/dist/{chunk-CCZK6LCB.mjs → chunk-7X6E7UUF.mjs} +2 -8
- package/dist/{chunk-BD3XGHFC.mjs → chunk-ADBGZA6L.mjs} +54 -11
- package/dist/chunk-V5FUJEYY.mjs +8 -0
- package/dist/chunk-W67A4H2E.mjs +23 -0
- package/dist/{chunk-4ISB7XMA.mjs → chunk-YLAVDZCS.mjs} +2 -1
- package/dist/client/index.d.ts +11 -8
- package/dist/client/topology.d.ts +7 -6
- package/dist/{client-base-DkjHphzB.d.ts → client-base-TJj2LTmq.d.ts} +4 -4
- package/dist/codegen/index.d.ts +2 -2
- package/dist/core/index.d.ts +13 -73
- package/dist/core/index.mjs +166 -65
- package/dist/core/writer-worker.mjs +38 -3
- package/dist/database-rRTtNLyK.d.ts +825 -0
- package/dist/driver/better-sqlite3.d.ts +3 -2
- package/dist/driver/better-sqlite3.mjs +24 -5
- package/dist/driver/bun.d.ts +3 -2
- package/dist/driver/bun.mjs +1 -1
- package/dist/driver/expo.d.ts +3 -2
- package/dist/driver/expo.mjs +1 -1
- package/dist/driver/node.d.ts +3 -2
- package/dist/driver/node.mjs +36 -5
- package/dist/driver/wa-sqlite.d.ts +3 -2
- package/dist/driver/wa-sqlite.mjs +1 -1
- package/dist/file-migrations/index.d.ts +3 -2
- package/dist/{operation-registry-6qErmUT2.d.ts → operation-registry-C1FmwQWR.d.ts} +1 -1
- package/dist/{protocol-BQMNEubg.d.ts → protocol-b7bwu6s4.d.ts} +2 -2
- package/dist/{query-types-BvkzxKQv.d.ts → query-types-Cv8N7pXj.d.ts} +1 -1
- package/dist/react/index.d.ts +2 -2
- package/dist/replication/index.d.ts +13 -60
- package/dist/report-BFsHdkKt.d.ts +132 -0
- package/dist/server/index.d.ts +11 -8
- package/dist/server/index.mjs +2 -1
- package/dist/{server-options-Bc6WuuFf.d.ts → server-options-Dgvo1Fhv.d.ts} +4 -4
- package/dist/{sirannon-Bs0WBW1o.d.ts → sirannon-Xu6bjxsA.d.ts} +2 -2
- package/dist/transport/grpc.d.ts +5 -4
- package/dist/transport/memory.d.ts +5 -4
- package/dist/{types-OGVLZjPS.d.ts → types-B1wQuNXV.d.ts} +39 -3
- package/dist/{types-BgVF0xhd.d.ts → types-B8dVULSQ.d.ts} +2 -2
- package/package.json +5 -1
- package/dist/database-DvURlr-n.d.ts +0 -380
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { B as BackupDestination, a as BackupRunReport } from '../report-BFsHdkKt.js';
|
|
2
|
+
export { b as BackupPiece, c as BackupProgress, d as BackupToDestinationOptions } from '../report-BFsHdkKt.js';
|
|
3
|
+
export { B as BackupCapabilities } from '../capabilities-DagBnGk8.js';
|
|
4
|
+
|
|
5
|
+
/** What one assembled file took to build.
|
|
6
|
+
* @public
|
|
7
|
+
*/
|
|
8
|
+
interface AssembleResult {
|
|
9
|
+
/** Bytes the assembly wrote. */
|
|
10
|
+
bytesWritten: number;
|
|
11
|
+
/** Pieces the assembly read. */
|
|
12
|
+
pieceCount: number;
|
|
13
|
+
/** SHA-256 of the assembled file, where the run recorded one to check it against. */
|
|
14
|
+
fingerprint?: string;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Builds a local file from the pieces a destination holds, fetching one piece
|
|
18
|
+
* at a time and writing it where its index places it, so a piece SQLite never
|
|
19
|
+
* wrote leaves zeros rather than moving every later byte.
|
|
20
|
+
*
|
|
21
|
+
* @param destination - Where the pieces are read from.
|
|
22
|
+
* @param report - What the run that wrote those pieces recorded, which the assembly checks its result against.
|
|
23
|
+
* @param destPath - Path the assembled file is written to.
|
|
24
|
+
* @returns The bytes and pieces the assembly wrote, and the fingerprint it computed.
|
|
25
|
+
*
|
|
26
|
+
* @public
|
|
27
|
+
*/
|
|
28
|
+
declare function assembleFromDestination(destination: BackupDestination, report: BackupRunReport, destPath: string): Promise<AssembleResult>;
|
|
29
|
+
|
|
30
|
+
export { type AssembleResult, BackupDestination, BackupRunReport, assembleFromDestination };
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { SirannonError } from '../chunk-PBRXXISQ.mjs';
|
|
2
|
+
import { createHash } from 'crypto';
|
|
3
|
+
import { open, rm } from 'fs/promises';
|
|
4
|
+
|
|
5
|
+
function destinationError(message) {
|
|
6
|
+
return new SirannonError(message, "BACKUP_DESTINATION_ERROR");
|
|
7
|
+
}
|
|
8
|
+
function assertChainIsWhole(pieces, report) {
|
|
9
|
+
if (pieces.length === 0) {
|
|
10
|
+
throw destinationError(`The destination holds no pieces named '${report.destinationName}'`);
|
|
11
|
+
}
|
|
12
|
+
for (let expected = 0; expected < report.pieceCount; expected++) {
|
|
13
|
+
if (pieces[expected]?.index !== expected) {
|
|
14
|
+
throw destinationError(
|
|
15
|
+
`The destination is missing piece ${expected} of '${report.destinationName}', so the file cannot be assembled`
|
|
16
|
+
);
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
if (pieces.length > report.pieceCount) {
|
|
20
|
+
throw destinationError(
|
|
21
|
+
`The destination holds ${pieces.length} pieces of '${report.destinationName}' where the run wrote ${report.pieceCount}, so a later piece belongs to a different run`
|
|
22
|
+
);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
async function assembleFromDestination(destination, report, destPath) {
|
|
26
|
+
const name = report.destinationName;
|
|
27
|
+
const pieces = [...await destination.listPieces(name)].sort((a, b) => a.index - b.index);
|
|
28
|
+
assertChainIsWhole(pieces, report);
|
|
29
|
+
const digest = report.fingerprint === void 0 ? null : createHash("sha256");
|
|
30
|
+
const file = await open(destPath, "w");
|
|
31
|
+
let bytesWritten = 0;
|
|
32
|
+
try {
|
|
33
|
+
for (const piece of pieces) {
|
|
34
|
+
const bytes = await destination.readPiece(name, piece.index);
|
|
35
|
+
await file.write(bytes, 0, bytes.byteLength, piece.index * report.pieceBytes);
|
|
36
|
+
digest?.update(bytes);
|
|
37
|
+
bytesWritten += bytes.byteLength;
|
|
38
|
+
}
|
|
39
|
+
} finally {
|
|
40
|
+
await file.close();
|
|
41
|
+
}
|
|
42
|
+
const fingerprint = digest?.digest("hex");
|
|
43
|
+
const failure = bytesWritten !== report.bytesWritten ? `The pieces of '${name}' hold ${bytesWritten} bytes where the run wrote ${report.bytesWritten}` : fingerprint !== void 0 && fingerprint !== report.fingerprint ? `The pieces of '${name}' do not match the fingerprint the run recorded` : null;
|
|
44
|
+
if (failure !== null) {
|
|
45
|
+
await rm(destPath, { force: true }).catch(() => {
|
|
46
|
+
});
|
|
47
|
+
throw destinationError(failure);
|
|
48
|
+
}
|
|
49
|
+
return { bytesWritten, pieceCount: pieces.length, ...fingerprint ? { fingerprint } : {} };
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export { assembleFromDestination };
|
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
import { e as SQLiteConnection, f as BackupScheduleOptions } from '../types-
|
|
2
|
-
import '../
|
|
1
|
+
import { e as SQLiteConnection, f as BackupScheduleOptions } from '../types-B1wQuNXV.js';
|
|
2
|
+
import { e as BackupRunRequest, a as BackupRunReport } from '../report-BFsHdkKt.js';
|
|
3
|
+
import '../query-types-Cv8N7pXj.js';
|
|
3
4
|
|
|
4
5
|
declare class BackupManager {
|
|
5
|
-
backup(conn: SQLiteConnection, destPath: string): Promise<void>;
|
|
6
|
+
backup(conn: SQLiteConnection, destPath: string, onFirstStep?: () => void): Promise<void>;
|
|
7
|
+
copyToDestination(conn: SQLiteConnection, request: BackupRunRequest): Promise<BackupRunReport>;
|
|
6
8
|
generateFilename(): string;
|
|
7
9
|
rotate(dir: string, maxFiles: number): void;
|
|
8
10
|
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What one runtime supports of the backup operations, so a caller learns
|
|
3
|
+
* before a run rather than at failure time. A runtime that hands over whole
|
|
4
|
+
* databases only reports no full copy at all.
|
|
5
|
+
*
|
|
6
|
+
* @public
|
|
7
|
+
*/
|
|
8
|
+
interface BackupCapabilities {
|
|
9
|
+
/** Whether this runtime copies an open database while writes continue. */
|
|
10
|
+
fullCopy: boolean;
|
|
11
|
+
/** Whether a full copy reaches the destination without a local file. */
|
|
12
|
+
streamedCopy: boolean;
|
|
13
|
+
/** Whether a full copy writes a local file and sends that file on. */
|
|
14
|
+
stagedCopy: boolean;
|
|
15
|
+
/** Local disk a full copy needs, which the staged route sets to the size of the backup. */
|
|
16
|
+
localDiskRequired: 'none' | 'equal-to-backup';
|
|
17
|
+
/** Whether this runtime repeats a full copy on a schedule. */
|
|
18
|
+
schedule: boolean;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export type { BackupCapabilities as B };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { e as SQLiteConnection } from './types-
|
|
2
|
-
import { C as ChangeEvent } from './query-types-
|
|
1
|
+
import { e as SQLiteConnection } from './types-B1wQuNXV.js';
|
|
2
|
+
import { C as ChangeEvent } from './query-types-Cv8N7pXj.js';
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* How long a change tracker keeps changes, and how much it reads at a time.
|
|
@@ -1,8 +1,255 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { startCopyWithoutHoldingWriter } from './chunk-W67A4H2E.mjs';
|
|
2
|
+
import { randomHex } from './chunk-V5FUJEYY.mjs';
|
|
3
|
+
import { BackupError, SirannonError } from './chunk-PBRXXISQ.mjs';
|
|
2
4
|
import { existsSync, mkdirSync, rmSync, readdirSync, lstatSync } from 'fs';
|
|
3
5
|
import { resolve, dirname, join } from 'path';
|
|
6
|
+
import { createHash } from 'crypto';
|
|
7
|
+
import { mkdtemp, open, rm } from 'fs/promises';
|
|
8
|
+
import { tmpdir } from 'os';
|
|
4
9
|
|
|
10
|
+
// src/core/backup/report.ts
|
|
11
|
+
var DEFAULT_PIECE_BYTES = 16 * 1024 * 1024;
|
|
12
|
+
|
|
13
|
+
// src/core/backup/stepped-copy.ts
|
|
14
|
+
var DEFAULT_PAGES_PER_STEP = 256;
|
|
15
|
+
var DEFAULT_RESTART_LIMIT = 3;
|
|
16
|
+
var DEFAULT_STALL_TIMEOUT_MS = 3e4;
|
|
17
|
+
var DEFAULT_NO_PROGRESS_STEP_LIMIT = 256;
|
|
18
|
+
function restartLimitError(restarts, destPath) {
|
|
19
|
+
return new SirannonError(
|
|
20
|
+
`The copy to '${destPath}' returned to page one ${restarts} times because another connection wrote to the source database or ran a RESTART or TRUNCATE checkpoint on it. Close any other connection that writes to this file, route its writes through Sirannon, and run the copy again.`,
|
|
21
|
+
"BACKUP_RESTARTED"
|
|
22
|
+
);
|
|
23
|
+
}
|
|
24
|
+
function noProgressError(steps, destPath) {
|
|
25
|
+
return new SirannonError(
|
|
26
|
+
`The copy to '${destPath}' moved no page it had not already moved across ${steps} steps. Another connection restarts the copy on every step, or the source grows faster than the copy moves it. Close any other connection that writes to this file, or run the copy when the write rate is lower.`,
|
|
27
|
+
"BACKUP_RESTARTED"
|
|
28
|
+
);
|
|
29
|
+
}
|
|
30
|
+
function stallError(stallTimeoutMs, destPath) {
|
|
31
|
+
return new SirannonError(
|
|
32
|
+
`The copy to '${destPath}' moved no pages for ${stallTimeoutMs}ms. SQLite steps the copy once per turn of the event loop, so a caller that never lets the loop reach its timers and immediates holds the copy still. Let the event loop run between writes, or raise the stall timeout for a host this slow.`,
|
|
33
|
+
"BACKUP_STALLED"
|
|
34
|
+
);
|
|
35
|
+
}
|
|
36
|
+
function copiedPages(step) {
|
|
37
|
+
return step.totalPages - step.remainingPages;
|
|
38
|
+
}
|
|
39
|
+
function hasRestarted(step, previous) {
|
|
40
|
+
if (!previous) return false;
|
|
41
|
+
return copiedPages(step) < copiedPages(previous);
|
|
42
|
+
}
|
|
43
|
+
async function copyDatabaseStepwise(conn, options) {
|
|
44
|
+
if (!conn.copyDatabase) {
|
|
45
|
+
throw new SirannonError(
|
|
46
|
+
"This driver opens connections without a stepped copy call, so it cannot copy a database while writes continue",
|
|
47
|
+
"BACKUP_UNSUPPORTED"
|
|
48
|
+
);
|
|
49
|
+
}
|
|
50
|
+
const restartLimit = options.restartLimit ?? DEFAULT_RESTART_LIMIT;
|
|
51
|
+
const stallTimeoutMs = options.stallTimeoutMs ?? DEFAULT_STALL_TIMEOUT_MS;
|
|
52
|
+
const noProgressStepLimit = options.noProgressStepLimit ?? DEFAULT_NO_PROGRESS_STEP_LIMIT;
|
|
53
|
+
let previous = null;
|
|
54
|
+
let restarts = 0;
|
|
55
|
+
let furthestCopied = -1;
|
|
56
|
+
let stepsWithoutProgress = 0;
|
|
57
|
+
let stopped = null;
|
|
58
|
+
let stallTimer = null;
|
|
59
|
+
let reportStall = () => {
|
|
60
|
+
};
|
|
61
|
+
const stalled = new Promise((_, reject) => {
|
|
62
|
+
reportStall = reject;
|
|
63
|
+
});
|
|
64
|
+
const armStall = () => {
|
|
65
|
+
if (stallTimeoutMs <= 0) return;
|
|
66
|
+
if (stallTimer) clearTimeout(stallTimer);
|
|
67
|
+
stallTimer = setTimeout(() => {
|
|
68
|
+
stopped = stallError(stallTimeoutMs, options.destPath);
|
|
69
|
+
reportStall(stopped);
|
|
70
|
+
}, stallTimeoutMs);
|
|
71
|
+
stallTimer.unref?.();
|
|
72
|
+
};
|
|
73
|
+
armStall();
|
|
74
|
+
const copy = conn.copyDatabase({
|
|
75
|
+
destPath: options.destPath,
|
|
76
|
+
pagesPerStep: options.pagesPerStep ?? DEFAULT_PAGES_PER_STEP,
|
|
77
|
+
onStep: (step) => {
|
|
78
|
+
if (stopped) throw stopped;
|
|
79
|
+
armStall();
|
|
80
|
+
if (hasRestarted(step, previous)) {
|
|
81
|
+
restarts++;
|
|
82
|
+
if (restarts > restartLimit) {
|
|
83
|
+
stopped = restartLimitError(restarts, options.destPath);
|
|
84
|
+
throw stopped;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
if (copiedPages(step) > furthestCopied) {
|
|
88
|
+
furthestCopied = copiedPages(step);
|
|
89
|
+
stepsWithoutProgress = 0;
|
|
90
|
+
} else if (step.remainingPages > 0 && ++stepsWithoutProgress > noProgressStepLimit) {
|
|
91
|
+
stopped = noProgressError(stepsWithoutProgress, options.destPath);
|
|
92
|
+
throw stopped;
|
|
93
|
+
}
|
|
94
|
+
previous = step;
|
|
95
|
+
options.onStep?.({ totalPages: step.totalPages, remainingPages: step.remainingPages, restarts });
|
|
96
|
+
}
|
|
97
|
+
}).catch((err) => {
|
|
98
|
+
if (stopped) throw stopped;
|
|
99
|
+
throw err instanceof SirannonError ? err : new BackupError(`Copy to '${options.destPath}' failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
100
|
+
});
|
|
101
|
+
copy.catch(() => {
|
|
102
|
+
});
|
|
103
|
+
try {
|
|
104
|
+
const final = await Promise.race([copy, stalled]);
|
|
105
|
+
return { pageCount: final.totalPages, restarts };
|
|
106
|
+
} catch (err) {
|
|
107
|
+
options.onCopyLeftRunning?.(copy);
|
|
108
|
+
throw err;
|
|
109
|
+
} finally {
|
|
110
|
+
if (stallTimer) clearTimeout(stallTimer);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
// src/core/backup/staged-copy.ts
|
|
115
|
+
var STAGED_FILE_NAME = "copy.db";
|
|
116
|
+
function destinationError(name, index, err) {
|
|
117
|
+
return new SirannonError(
|
|
118
|
+
`The destination refused piece ${index} of '${name}': ${err instanceof Error ? err.message : String(err)}`,
|
|
119
|
+
"BACKUP_DESTINATION_ERROR"
|
|
120
|
+
);
|
|
121
|
+
}
|
|
122
|
+
async function readPageSize(conn) {
|
|
123
|
+
const stmt = await conn.prepare("PRAGMA page_size");
|
|
124
|
+
const row = await stmt.get();
|
|
125
|
+
return row ? Number(row.page_size) : 0;
|
|
126
|
+
}
|
|
127
|
+
async function sendPieces(stagedPath, destination, name, pieceBytes, fingerprint, report) {
|
|
128
|
+
const file = await open(stagedPath, "r");
|
|
129
|
+
const digest = fingerprint ? createHash("sha256") : null;
|
|
130
|
+
let index = 0;
|
|
131
|
+
let bytesWritten = 0;
|
|
132
|
+
try {
|
|
133
|
+
const buffer = Buffer.allocUnsafe(pieceBytes);
|
|
134
|
+
for (; ; ) {
|
|
135
|
+
let filled = 0;
|
|
136
|
+
while (filled < pieceBytes) {
|
|
137
|
+
const { bytesRead } = await file.read(buffer, filled, pieceBytes - filled, index * pieceBytes + filled);
|
|
138
|
+
if (bytesRead === 0) break;
|
|
139
|
+
filled += bytesRead;
|
|
140
|
+
}
|
|
141
|
+
if (filled === 0) break;
|
|
142
|
+
const piece = new Uint8Array(filled);
|
|
143
|
+
piece.set(buffer.subarray(0, filled));
|
|
144
|
+
digest?.update(piece);
|
|
145
|
+
try {
|
|
146
|
+
await destination.writePiece(name, index, piece);
|
|
147
|
+
} catch (err) {
|
|
148
|
+
throw destinationError(name, index, err);
|
|
149
|
+
}
|
|
150
|
+
index++;
|
|
151
|
+
bytesWritten += filled;
|
|
152
|
+
report(index, bytesWritten);
|
|
153
|
+
if (filled < pieceBytes) break;
|
|
154
|
+
}
|
|
155
|
+
} finally {
|
|
156
|
+
await file.close();
|
|
157
|
+
}
|
|
158
|
+
return { pieceCount: index, bytesWritten, ...digest ? { fingerprint: digest.digest("hex") } : {} };
|
|
159
|
+
}
|
|
160
|
+
async function copyToDestinationStaged(conn, request) {
|
|
161
|
+
const runId = randomHex(8);
|
|
162
|
+
const name = request.name ?? `backup-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}.db`;
|
|
163
|
+
const pieceBytes = request.pieceBytes ?? DEFAULT_PIECE_BYTES;
|
|
164
|
+
if (!Number.isInteger(pieceBytes) || pieceBytes <= 0) {
|
|
165
|
+
throw new SirannonError(
|
|
166
|
+
`Piece size must be a positive whole number of bytes, and it was ${pieceBytes}`,
|
|
167
|
+
"BACKUP_ERROR"
|
|
168
|
+
);
|
|
169
|
+
}
|
|
170
|
+
const startedAt = Date.now();
|
|
171
|
+
const stagingRoot = await mkdtemp(join(request.stagingDir ?? tmpdir(), "sirannon-backup-"));
|
|
172
|
+
const stagedPath = join(stagingRoot, STAGED_FILE_NAME);
|
|
173
|
+
let firstStepSeen = false;
|
|
174
|
+
let copyLeftRunning = null;
|
|
175
|
+
const emit = (progress) => request.onProgress?.({ runId, ...progress });
|
|
176
|
+
try {
|
|
177
|
+
const pageSize = await readPageSize(conn);
|
|
178
|
+
const copyStartedAt = Date.now();
|
|
179
|
+
const copy = await copyDatabaseStepwise(conn, {
|
|
180
|
+
destPath: stagedPath,
|
|
181
|
+
...request.pagesPerStep === void 0 ? {} : { pagesPerStep: request.pagesPerStep },
|
|
182
|
+
...request.restartLimit === void 0 ? {} : { restartLimit: request.restartLimit },
|
|
183
|
+
...request.stallTimeoutMs === void 0 ? {} : { stallTimeoutMs: request.stallTimeoutMs },
|
|
184
|
+
...request.noProgressStepLimit === void 0 ? {} : { noProgressStepLimit: request.noProgressStepLimit },
|
|
185
|
+
onCopyLeftRunning: (copy2) => {
|
|
186
|
+
copyLeftRunning = copy2;
|
|
187
|
+
},
|
|
188
|
+
onStep: (step) => {
|
|
189
|
+
if (!firstStepSeen) {
|
|
190
|
+
firstStepSeen = true;
|
|
191
|
+
request.onFirstStep?.();
|
|
192
|
+
}
|
|
193
|
+
emit({ phase: "copy", ...step, piecesWritten: 0, bytesWritten: 0 });
|
|
194
|
+
}
|
|
195
|
+
});
|
|
196
|
+
const copyMs = Date.now() - copyStartedAt;
|
|
197
|
+
const transferStartedAt = Date.now();
|
|
198
|
+
const sent = await sendPieces(
|
|
199
|
+
stagedPath,
|
|
200
|
+
request.destination,
|
|
201
|
+
name,
|
|
202
|
+
pieceBytes,
|
|
203
|
+
request.fingerprint ?? true,
|
|
204
|
+
(piecesWritten, bytesWritten) => emit({
|
|
205
|
+
phase: "transfer",
|
|
206
|
+
totalPages: copy.pageCount,
|
|
207
|
+
remainingPages: 0,
|
|
208
|
+
restarts: copy.restarts,
|
|
209
|
+
piecesWritten,
|
|
210
|
+
bytesWritten
|
|
211
|
+
})
|
|
212
|
+
);
|
|
213
|
+
const transferMs = Date.now() - transferStartedAt;
|
|
214
|
+
const finishedAt = Date.now();
|
|
215
|
+
return {
|
|
216
|
+
runId,
|
|
217
|
+
databaseId: request.databaseId,
|
|
218
|
+
sourcePath: request.sourcePath,
|
|
219
|
+
kind: "full",
|
|
220
|
+
route: "staged",
|
|
221
|
+
destinationName: name,
|
|
222
|
+
startedAt,
|
|
223
|
+
finishedAt,
|
|
224
|
+
durationMs: finishedAt - startedAt,
|
|
225
|
+
copyMs,
|
|
226
|
+
transferMs,
|
|
227
|
+
pageCount: copy.pageCount,
|
|
228
|
+
pageSize,
|
|
229
|
+
bytesWritten: sent.bytesWritten,
|
|
230
|
+
pieceCount: sent.pieceCount,
|
|
231
|
+
pieceBytes,
|
|
232
|
+
restarts: copy.restarts,
|
|
233
|
+
...sent.fingerprint ? { fingerprint: sent.fingerprint } : {}
|
|
234
|
+
};
|
|
235
|
+
} finally {
|
|
236
|
+
const removeStaging = () => rm(stagingRoot, { recursive: true, force: true }).catch(() => {
|
|
237
|
+
});
|
|
238
|
+
if (copyLeftRunning) void copyLeftRunning.then(removeStaging, removeStaging);
|
|
239
|
+
else await removeStaging();
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
// src/core/backup/backup.ts
|
|
5
244
|
var BACKUP_FILE_PREFIX = "backup";
|
|
245
|
+
function once(action) {
|
|
246
|
+
let done = false;
|
|
247
|
+
return () => {
|
|
248
|
+
if (done) return;
|
|
249
|
+
done = true;
|
|
250
|
+
action();
|
|
251
|
+
};
|
|
252
|
+
}
|
|
6
253
|
function hasControlCharacters(s) {
|
|
7
254
|
for (let i = 0; i < s.length; i++) {
|
|
8
255
|
const code = s.charCodeAt(i);
|
|
@@ -11,7 +258,7 @@ function hasControlCharacters(s) {
|
|
|
11
258
|
return false;
|
|
12
259
|
}
|
|
13
260
|
var BackupManager = class {
|
|
14
|
-
async backup(conn, destPath) {
|
|
261
|
+
async backup(conn, destPath, onFirstStep) {
|
|
15
262
|
if (hasControlCharacters(destPath)) {
|
|
16
263
|
throw new BackupError("Backup path contains invalid characters");
|
|
17
264
|
}
|
|
@@ -33,17 +280,20 @@ var BackupManager = class {
|
|
|
33
280
|
if (existsSync(resolved)) {
|
|
34
281
|
throw new BackupError(`Backup destination '${destPath}' already exists`);
|
|
35
282
|
}
|
|
36
|
-
const escaped = resolved.replace(/'/g, "''");
|
|
37
283
|
try {
|
|
38
|
-
await conn
|
|
284
|
+
await copyDatabaseStepwise(conn, { destPath: resolved, onStep: onFirstStep ? once(onFirstStep) : void 0 });
|
|
39
285
|
} catch (err) {
|
|
40
286
|
try {
|
|
41
287
|
rmSync(resolved, { force: true });
|
|
42
288
|
} catch {
|
|
43
289
|
}
|
|
290
|
+
if (err instanceof SirannonError) throw err;
|
|
44
291
|
throw new BackupError(`Backup to '${destPath}' failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
45
292
|
}
|
|
46
293
|
}
|
|
294
|
+
copyToDestination(conn, request) {
|
|
295
|
+
return copyToDestinationStaged(conn, request);
|
|
296
|
+
}
|
|
47
297
|
generateFilename() {
|
|
48
298
|
const ts = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
49
299
|
return `${BACKUP_FILE_PREFIX}-${ts}.db`;
|
|
@@ -429,7 +679,10 @@ var BackupScheduler = class {
|
|
|
429
679
|
const runBackup = async () => {
|
|
430
680
|
try {
|
|
431
681
|
const destPath = join(resolvedDir, this.manager.generateFilename());
|
|
432
|
-
await
|
|
682
|
+
await startCopyWithoutHoldingWriter(
|
|
683
|
+
runExclusive,
|
|
684
|
+
(onFirstStep) => this.manager.backup(conn, destPath, onFirstStep)
|
|
685
|
+
);
|
|
433
686
|
this.manager.rotate(resolvedDir, maxFiles);
|
|
434
687
|
} catch (err) {
|
|
435
688
|
if (onError) {
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { tableColumnNames, tablePkColumns, selectMinChangeSeqSql, selectMaxChangeSeq, deleteChangesBeforeUpToSeqSql, deleteChangesBeforeSql, selectTableExists, ensureChangesTable, ensureMetaTable, insertMetaValueIfAbsent, selectMetaValue, selectChangesAfterSeqSql, selectTablesChangesInRangeSql } from './chunk-7FQRQH5Z.mjs';
|
|
2
2
|
import { CHANGES_TABLE, isReservedIdentifier, CDC_TRIGGER_PREFIX, encodeTaggedValues, decodeTaggedValues, SAFE_INT_BOUND_TEXT } from './chunk-7R4ER4FB.mjs';
|
|
3
3
|
import { synchronousPragmaValue } from './chunk-OUSWVNWT.mjs';
|
|
4
|
+
import { randomHex } from './chunk-V5FUJEYY.mjs';
|
|
4
5
|
import { SirannonError, ForbiddenSqlError, CDCError, MigrationError } from './chunk-PBRXXISQ.mjs';
|
|
5
6
|
|
|
6
7
|
// src/core/bulk-load.ts
|
|
@@ -549,13 +550,6 @@ function reconcileMigrationChecksums(migrations, applied) {
|
|
|
549
550
|
return backfills;
|
|
550
551
|
}
|
|
551
552
|
|
|
552
|
-
// src/core/random-hex.ts
|
|
553
|
-
function randomHex(byteLength) {
|
|
554
|
-
const bytes = new Uint8Array(byteLength);
|
|
555
|
-
globalThis.crypto.getRandomValues(bytes);
|
|
556
|
-
return Array.from(bytes, (value) => value.toString(16).padStart(2, "0")).join("");
|
|
557
|
-
}
|
|
558
|
-
|
|
559
553
|
// src/core/cdc/epoch.ts
|
|
560
554
|
var EPOCH_KEY = "cdc_epoch";
|
|
561
555
|
async function ensureCdcEpoch(conn) {
|
|
@@ -855,4 +849,4 @@ var TransactionGrouper = class {
|
|
|
855
849
|
}
|
|
856
850
|
};
|
|
857
851
|
|
|
858
|
-
export { ChangeTracker, PrimedSubscription, SubscriptionBuilderImpl, SubscriptionManager, TransactionGrouper, ensureCdcEpoch, filteredChange, isBulkLoadDurability, migrationChecksum, migrationContentChecksum, needsResync,
|
|
852
|
+
export { ChangeTracker, PrimedSubscription, SubscriptionBuilderImpl, SubscriptionManager, TransactionGrouper, ensureCdcEpoch, filteredChange, isBulkLoadDurability, migrationChecksum, migrationContentChecksum, needsResync, reconcileMigrationChecksums, runBulkLoad, startPolling };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { BackupManager, BackupScheduler } from './chunk-
|
|
2
|
-
import { WORKER_CANCELLED_CODE, deserializeError } from './chunk-
|
|
1
|
+
import { BackupManager, BackupScheduler } from './chunk-23HKRYSM.mjs';
|
|
2
|
+
import { WORKER_CANCELLED_CODE, deserializeError } from './chunk-YLAVDZCS.mjs';
|
|
3
3
|
import { SirannonError, ExtensionError } from './chunk-PBRXXISQ.mjs';
|
|
4
4
|
import { Worker } from 'worker_threads';
|
|
5
5
|
import { existsSync } from 'fs';
|
|
@@ -155,6 +155,10 @@ var WriterWorker = class _WriterWorker {
|
|
|
155
155
|
}
|
|
156
156
|
}
|
|
157
157
|
onResponse(res) {
|
|
158
|
+
if ("kind" in res) {
|
|
159
|
+
this.onCopyStep(res.id, res.step);
|
|
160
|
+
return;
|
|
161
|
+
}
|
|
158
162
|
const entry = this.pending.get(res.id);
|
|
159
163
|
if (!entry) return;
|
|
160
164
|
this.pending.delete(res.id);
|
|
@@ -174,6 +178,26 @@ var WriterWorker = class _WriterWorker {
|
|
|
174
178
|
}
|
|
175
179
|
entry.reject(deserializeError(res.error));
|
|
176
180
|
}
|
|
181
|
+
onCopyStep(id, step) {
|
|
182
|
+
const entry = this.pending.get(id);
|
|
183
|
+
if (!entry) return;
|
|
184
|
+
if (entry.timer) {
|
|
185
|
+
clearTimeout(entry.timer);
|
|
186
|
+
entry.timer = this.timeoutMs > 0 ? setTimeout(() => this.onDeadline(id), this.timeoutMs) : null;
|
|
187
|
+
entry.timer?.unref?.();
|
|
188
|
+
}
|
|
189
|
+
try {
|
|
190
|
+
entry.onStep?.(step);
|
|
191
|
+
} catch (err) {
|
|
192
|
+
this.pending.delete(id);
|
|
193
|
+
clearPendingTimers(entry);
|
|
194
|
+
try {
|
|
195
|
+
this.worker?.postMessage({ kind: "cancel", id });
|
|
196
|
+
} catch {
|
|
197
|
+
}
|
|
198
|
+
entry.reject(err instanceof Error ? err : new Error(String(err)));
|
|
199
|
+
}
|
|
200
|
+
}
|
|
177
201
|
rejectPending(id, err) {
|
|
178
202
|
const entry = this.pending.get(id);
|
|
179
203
|
if (!entry) return;
|
|
@@ -181,7 +205,13 @@ var WriterWorker = class _WriterWorker {
|
|
|
181
205
|
clearPendingTimers(entry);
|
|
182
206
|
entry.reject(err);
|
|
183
207
|
}
|
|
184
|
-
unresponsiveError(waitedMs) {
|
|
208
|
+
unresponsiveError(waitedMs, kind) {
|
|
209
|
+
if (kind === "copyDatabase") {
|
|
210
|
+
return new SirannonError(
|
|
211
|
+
`The writer worker moved no page of the copy for ${waitedMs}ms, so the copy was stopped and nothing it wrote is usable`,
|
|
212
|
+
"BACKUP_STALLED"
|
|
213
|
+
);
|
|
214
|
+
}
|
|
185
215
|
return new SirannonError(
|
|
186
216
|
`Writer worker did not respond within ${waitedMs}ms; the operation's outcome is unknown`,
|
|
187
217
|
"WRITER_WORKER_TIMEOUT"
|
|
@@ -196,17 +226,17 @@ var WriterWorker = class _WriterWorker {
|
|
|
196
226
|
if (!entry) return;
|
|
197
227
|
const worker = this.worker;
|
|
198
228
|
if (!entry.cancellable || !worker) {
|
|
199
|
-
this.rejectPending(id, this.unresponsiveError(this.timeoutMs));
|
|
229
|
+
this.rejectPending(id, this.unresponsiveError(this.timeoutMs, entry.kind));
|
|
200
230
|
return;
|
|
201
231
|
}
|
|
202
232
|
try {
|
|
203
233
|
worker.postMessage({ kind: "cancel", id });
|
|
204
234
|
} catch {
|
|
205
|
-
this.rejectPending(id, this.unresponsiveError(this.timeoutMs));
|
|
235
|
+
this.rejectPending(id, this.unresponsiveError(this.timeoutMs, entry.kind));
|
|
206
236
|
return;
|
|
207
237
|
}
|
|
208
238
|
entry.graceTimer = setTimeout(() => {
|
|
209
|
-
this.rejectPending(id, this.unresponsiveError(this.timeoutMs * 2));
|
|
239
|
+
this.rejectPending(id, this.unresponsiveError(this.timeoutMs * 2, entry.kind));
|
|
210
240
|
}, this.timeoutMs);
|
|
211
241
|
entry.graceTimer.unref?.();
|
|
212
242
|
}
|
|
@@ -242,7 +272,7 @@ var WriterWorker = class _WriterWorker {
|
|
|
242
272
|
}
|
|
243
273
|
this.spawn();
|
|
244
274
|
}
|
|
245
|
-
send(request) {
|
|
275
|
+
send(request, onStep) {
|
|
246
276
|
const worker = this.worker;
|
|
247
277
|
if (!worker) {
|
|
248
278
|
return Promise.reject(
|
|
@@ -258,7 +288,15 @@ var WriterWorker = class _WriterWorker {
|
|
|
258
288
|
timer.unref?.();
|
|
259
289
|
}
|
|
260
290
|
const cancellable = request.kind !== "open" && request.kind !== "close" && request.kind !== "loadExtension";
|
|
261
|
-
this.pending.set(id, {
|
|
291
|
+
this.pending.set(id, {
|
|
292
|
+
resolve: resolve2,
|
|
293
|
+
reject,
|
|
294
|
+
timer,
|
|
295
|
+
graceTimer: null,
|
|
296
|
+
cancellable,
|
|
297
|
+
kind: request.kind,
|
|
298
|
+
...onStep ? { onStep } : {}
|
|
299
|
+
});
|
|
262
300
|
try {
|
|
263
301
|
worker.postMessage(message);
|
|
264
302
|
} catch (err) {
|
|
@@ -273,10 +311,10 @@ var WriterWorker = class _WriterWorker {
|
|
|
273
311
|
}
|
|
274
312
|
});
|
|
275
313
|
}
|
|
276
|
-
request(request) {
|
|
314
|
+
request(request, onStep) {
|
|
277
315
|
if (this.fatal) return Promise.reject(this.fatal);
|
|
278
316
|
if (this.closed) return Promise.reject(new SirannonError("Writer worker is closed", "WRITER_WORKER_CLOSED"));
|
|
279
|
-
return this.ready.then(() => this.send(request)).then((value) => {
|
|
317
|
+
return this.ready.then(() => this.send(request, onStep)).then((value) => {
|
|
280
318
|
this.restarts = 0;
|
|
281
319
|
return value;
|
|
282
320
|
});
|
|
@@ -302,6 +340,10 @@ var WriterWorker = class _WriterWorker {
|
|
|
302
340
|
}))
|
|
303
341
|
}))
|
|
304
342
|
}),
|
|
343
|
+
copyDatabase: (request) => this.request(
|
|
344
|
+
{ kind: "copyDatabase", destPath: request.destPath, pagesPerStep: request.pagesPerStep },
|
|
345
|
+
request.onStep
|
|
346
|
+
),
|
|
305
347
|
loadExtension: async (extensionPath) => {
|
|
306
348
|
await this.request({ kind: "loadExtension", path: extensionPath });
|
|
307
349
|
if (!this.loadedExtensions.includes(extensionPath)) this.loadedExtensions.push(extensionPath);
|
|
@@ -369,7 +411,8 @@ function nodeBackupEngine() {
|
|
|
369
411
|
const manager = new BackupManager();
|
|
370
412
|
const scheduler = new BackupScheduler(manager);
|
|
371
413
|
return {
|
|
372
|
-
backup: (conn, destPath) => manager.backup(conn, destPath),
|
|
414
|
+
backup: (conn, destPath, onFirstStep) => manager.backup(conn, destPath, onFirstStep),
|
|
415
|
+
copyToDestination: (conn, request) => manager.copyToDestination(conn, request),
|
|
373
416
|
schedule: (conn, options, runExclusive) => scheduler.schedule(conn, options, runExclusive)
|
|
374
417
|
};
|
|
375
418
|
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
// src/core/backup/start-guard.ts
|
|
2
|
+
async function startCopyWithoutHoldingWriter(runExclusive, start) {
|
|
3
|
+
let begin;
|
|
4
|
+
let release;
|
|
5
|
+
const writerHeld = new Promise((resolve) => {
|
|
6
|
+
begin = resolve;
|
|
7
|
+
});
|
|
8
|
+
const firstStepDone = new Promise((resolve) => {
|
|
9
|
+
release = resolve;
|
|
10
|
+
});
|
|
11
|
+
const run = (async () => {
|
|
12
|
+
await writerHeld;
|
|
13
|
+
return start(release);
|
|
14
|
+
})();
|
|
15
|
+
run.then(release, release);
|
|
16
|
+
await runExclusive(async () => {
|
|
17
|
+
begin();
|
|
18
|
+
await firstStepDone;
|
|
19
|
+
});
|
|
20
|
+
return run;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export { startCopyWithoutHoldingWriter };
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
// src/core/worker/protocol.ts
|
|
2
2
|
var WORKER_CANCELLED_CODE = "WRITER_WORKER_CANCELLED";
|
|
3
|
+
var WORKER_COPY_ABORTED_CODE = "BACKUP_ABORTED";
|
|
3
4
|
function serializeError(err) {
|
|
4
5
|
if (err instanceof Error) {
|
|
5
6
|
const code = err.code;
|
|
@@ -18,4 +19,4 @@ function deserializeError(error) {
|
|
|
18
19
|
return err;
|
|
19
20
|
}
|
|
20
21
|
|
|
21
|
-
export { WORKER_CANCELLED_CODE, deserializeError, serializeError };
|
|
22
|
+
export { WORKER_CANCELLED_CODE, WORKER_COPY_ABORTED_CODE, deserializeError, serializeError };
|