@delali/sirannon-db 0.2.3-next.29 → 0.2.3-next.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backup/index.d.ts +30 -0
- package/dist/backup/index.mjs +52 -0
- package/dist/backup-scheduler/index.d.ts +5 -3
- package/dist/backup-scheduler/index.mjs +3 -1
- package/dist/capabilities-DagBnGk8.d.ts +21 -0
- package/dist/{change-tracker-C2Z8UbI0.d.ts → change-tracker-7wXnBjUN.d.ts} +2 -2
- package/dist/{chunk-VOYGMAU7.mjs → chunk-23HKRYSM.mjs} +258 -5
- package/dist/{chunk-CCZK6LCB.mjs → chunk-7X6E7UUF.mjs} +2 -8
- package/dist/{chunk-UPKKSUPA.mjs → chunk-ADBGZA6L.mjs} +78 -15
- package/dist/{chunk-DVWQD3GF.mjs → chunk-GVYLZVUZ.mjs} +10 -1
- package/dist/chunk-V5FUJEYY.mjs +8 -0
- package/dist/chunk-W67A4H2E.mjs +23 -0
- package/dist/{chunk-4ISB7XMA.mjs → chunk-YLAVDZCS.mjs} +2 -1
- package/dist/client/index.d.ts +11 -8
- package/dist/client/topology.d.ts +7 -6
- package/dist/{client-base-CqTMYzQP.d.ts → client-base-TJj2LTmq.d.ts} +4 -4
- package/dist/codegen/index.d.ts +2 -2
- package/dist/core/index.d.ts +13 -71
- package/dist/core/index.mjs +300 -93
- package/dist/core/writer-worker.mjs +49 -3
- package/dist/database-rRTtNLyK.d.ts +825 -0
- package/dist/driver/better-sqlite3.d.ts +3 -2
- package/dist/driver/better-sqlite3.mjs +27 -5
- package/dist/driver/bun.d.ts +3 -2
- package/dist/driver/bun.mjs +17 -2
- package/dist/driver/expo.d.ts +3 -2
- package/dist/driver/expo.mjs +8 -2
- package/dist/driver/node.d.ts +3 -2
- package/dist/driver/node.mjs +54 -6
- package/dist/driver/wa-sqlite.d.ts +3 -2
- package/dist/driver/wa-sqlite.mjs +16 -1
- package/dist/file-migrations/index.d.ts +3 -2
- package/dist/{operation-registry-6qErmUT2.d.ts → operation-registry-C1FmwQWR.d.ts} +1 -1
- package/dist/{protocol-CEBVo1tO.d.ts → protocol-b7bwu6s4.d.ts} +2 -2
- package/dist/{query-types-BvkzxKQv.d.ts → query-types-Cv8N7pXj.d.ts} +1 -1
- package/dist/react/index.d.ts +2 -2
- package/dist/replication/index.d.ts +13 -60
- package/dist/report-BFsHdkKt.d.ts +132 -0
- package/dist/server/index.d.ts +11 -8
- package/dist/server/index.mjs +2 -1
- package/dist/{server-options-BVAFmguz.d.ts → server-options-Dgvo1Fhv.d.ts} +4 -4
- package/dist/{sirannon-DVYfrNiq.d.ts → sirannon-Xu6bjxsA.d.ts} +2 -2
- package/dist/transport/grpc.d.ts +5 -4
- package/dist/transport/memory.d.ts +5 -4
- package/dist/{types-CXoPBeDM.d.ts → types-B1wQuNXV.d.ts} +48 -3
- package/dist/{types-DxoEm08T.d.ts → types-B8dVULSQ.d.ts} +2 -2
- package/package.json +5 -1
- package/dist/database-DvSvONb-.d.ts +0 -380
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { B as BackupDestination, a as BackupRunReport } from '../report-BFsHdkKt.js';
|
|
2
|
+
export { b as BackupPiece, c as BackupProgress, d as BackupToDestinationOptions } from '../report-BFsHdkKt.js';
|
|
3
|
+
export { B as BackupCapabilities } from '../capabilities-DagBnGk8.js';
|
|
4
|
+
|
|
5
|
+
/** What one assembled file took to build.
|
|
6
|
+
* @public
|
|
7
|
+
*/
|
|
8
|
+
interface AssembleResult {
|
|
9
|
+
/** Bytes the assembly wrote. */
|
|
10
|
+
bytesWritten: number;
|
|
11
|
+
/** Pieces the assembly read. */
|
|
12
|
+
pieceCount: number;
|
|
13
|
+
/** SHA-256 of the assembled file, where the run recorded one to check it against. */
|
|
14
|
+
fingerprint?: string;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Builds a local file from the pieces a destination holds, fetching one piece
|
|
18
|
+
* at a time and writing it where its index places it, so a piece SQLite never
|
|
19
|
+
* wrote leaves zeros rather than moving every later byte.
|
|
20
|
+
*
|
|
21
|
+
* @param destination - Where the pieces are read from.
|
|
22
|
+
* @param report - What the run that wrote those pieces recorded, which the assembly checks its result against.
|
|
23
|
+
* @param destPath - Path the assembled file is written to.
|
|
24
|
+
* @returns The bytes and pieces the assembly wrote, and the fingerprint it computed.
|
|
25
|
+
*
|
|
26
|
+
* @public
|
|
27
|
+
*/
|
|
28
|
+
declare function assembleFromDestination(destination: BackupDestination, report: BackupRunReport, destPath: string): Promise<AssembleResult>;
|
|
29
|
+
|
|
30
|
+
export { type AssembleResult, BackupDestination, BackupRunReport, assembleFromDestination };
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { SirannonError } from '../chunk-PBRXXISQ.mjs';
|
|
2
|
+
import { createHash } from 'crypto';
|
|
3
|
+
import { open, rm } from 'fs/promises';
|
|
4
|
+
|
|
5
|
+
function destinationError(message) {
|
|
6
|
+
return new SirannonError(message, "BACKUP_DESTINATION_ERROR");
|
|
7
|
+
}
|
|
8
|
+
function assertChainIsWhole(pieces, report) {
|
|
9
|
+
if (pieces.length === 0) {
|
|
10
|
+
throw destinationError(`The destination holds no pieces named '${report.destinationName}'`);
|
|
11
|
+
}
|
|
12
|
+
for (let expected = 0; expected < report.pieceCount; expected++) {
|
|
13
|
+
if (pieces[expected]?.index !== expected) {
|
|
14
|
+
throw destinationError(
|
|
15
|
+
`The destination is missing piece ${expected} of '${report.destinationName}', so the file cannot be assembled`
|
|
16
|
+
);
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
if (pieces.length > report.pieceCount) {
|
|
20
|
+
throw destinationError(
|
|
21
|
+
`The destination holds ${pieces.length} pieces of '${report.destinationName}' where the run wrote ${report.pieceCount}, so a later piece belongs to a different run`
|
|
22
|
+
);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
async function assembleFromDestination(destination, report, destPath) {
|
|
26
|
+
const name = report.destinationName;
|
|
27
|
+
const pieces = [...await destination.listPieces(name)].sort((a, b) => a.index - b.index);
|
|
28
|
+
assertChainIsWhole(pieces, report);
|
|
29
|
+
const digest = report.fingerprint === void 0 ? null : createHash("sha256");
|
|
30
|
+
const file = await open(destPath, "w");
|
|
31
|
+
let bytesWritten = 0;
|
|
32
|
+
try {
|
|
33
|
+
for (const piece of pieces) {
|
|
34
|
+
const bytes = await destination.readPiece(name, piece.index);
|
|
35
|
+
await file.write(bytes, 0, bytes.byteLength, piece.index * report.pieceBytes);
|
|
36
|
+
digest?.update(bytes);
|
|
37
|
+
bytesWritten += bytes.byteLength;
|
|
38
|
+
}
|
|
39
|
+
} finally {
|
|
40
|
+
await file.close();
|
|
41
|
+
}
|
|
42
|
+
const fingerprint = digest?.digest("hex");
|
|
43
|
+
const failure = bytesWritten !== report.bytesWritten ? `The pieces of '${name}' hold ${bytesWritten} bytes where the run wrote ${report.bytesWritten}` : fingerprint !== void 0 && fingerprint !== report.fingerprint ? `The pieces of '${name}' do not match the fingerprint the run recorded` : null;
|
|
44
|
+
if (failure !== null) {
|
|
45
|
+
await rm(destPath, { force: true }).catch(() => {
|
|
46
|
+
});
|
|
47
|
+
throw destinationError(failure);
|
|
48
|
+
}
|
|
49
|
+
return { bytesWritten, pieceCount: pieces.length, ...fingerprint ? { fingerprint } : {} };
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export { assembleFromDestination };
|
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
import { e as SQLiteConnection, f as BackupScheduleOptions } from '../types-
|
|
2
|
-
import '../
|
|
1
|
+
import { e as SQLiteConnection, f as BackupScheduleOptions } from '../types-B1wQuNXV.js';
|
|
2
|
+
import { e as BackupRunRequest, a as BackupRunReport } from '../report-BFsHdkKt.js';
|
|
3
|
+
import '../query-types-Cv8N7pXj.js';
|
|
3
4
|
|
|
4
5
|
declare class BackupManager {
|
|
5
|
-
backup(conn: SQLiteConnection, destPath: string): Promise<void>;
|
|
6
|
+
backup(conn: SQLiteConnection, destPath: string, onFirstStep?: () => void): Promise<void>;
|
|
7
|
+
copyToDestination(conn: SQLiteConnection, request: BackupRunRequest): Promise<BackupRunReport>;
|
|
6
8
|
generateFilename(): string;
|
|
7
9
|
rotate(dir: string, maxFiles: number): void;
|
|
8
10
|
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What one runtime supports of the backup operations, so a caller learns
|
|
3
|
+
* before a run rather than at failure time. A runtime that hands over whole
|
|
4
|
+
* databases only reports no full copy at all.
|
|
5
|
+
*
|
|
6
|
+
* @public
|
|
7
|
+
*/
|
|
8
|
+
interface BackupCapabilities {
|
|
9
|
+
/** Whether this runtime copies an open database while writes continue. */
|
|
10
|
+
fullCopy: boolean;
|
|
11
|
+
/** Whether a full copy reaches the destination without a local file. */
|
|
12
|
+
streamedCopy: boolean;
|
|
13
|
+
/** Whether a full copy writes a local file and sends that file on. */
|
|
14
|
+
stagedCopy: boolean;
|
|
15
|
+
/** Local disk a full copy needs, which the staged route sets to the size of the backup. */
|
|
16
|
+
localDiskRequired: 'none' | 'equal-to-backup';
|
|
17
|
+
/** Whether this runtime repeats a full copy on a schedule. */
|
|
18
|
+
schedule: boolean;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export type { BackupCapabilities as B };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { e as SQLiteConnection } from './types-
|
|
2
|
-
import { C as ChangeEvent } from './query-types-
|
|
1
|
+
import { e as SQLiteConnection } from './types-B1wQuNXV.js';
|
|
2
|
+
import { C as ChangeEvent } from './query-types-Cv8N7pXj.js';
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* How long a change tracker keeps changes, and how much it reads at a time.
|
|
@@ -1,8 +1,255 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { startCopyWithoutHoldingWriter } from './chunk-W67A4H2E.mjs';
|
|
2
|
+
import { randomHex } from './chunk-V5FUJEYY.mjs';
|
|
3
|
+
import { BackupError, SirannonError } from './chunk-PBRXXISQ.mjs';
|
|
2
4
|
import { existsSync, mkdirSync, rmSync, readdirSync, lstatSync } from 'fs';
|
|
3
5
|
import { resolve, dirname, join } from 'path';
|
|
6
|
+
import { createHash } from 'crypto';
|
|
7
|
+
import { mkdtemp, open, rm } from 'fs/promises';
|
|
8
|
+
import { tmpdir } from 'os';
|
|
4
9
|
|
|
10
|
+
// src/core/backup/report.ts
|
|
11
|
+
var DEFAULT_PIECE_BYTES = 16 * 1024 * 1024;
|
|
12
|
+
|
|
13
|
+
// src/core/backup/stepped-copy.ts
|
|
14
|
+
var DEFAULT_PAGES_PER_STEP = 256;
|
|
15
|
+
var DEFAULT_RESTART_LIMIT = 3;
|
|
16
|
+
var DEFAULT_STALL_TIMEOUT_MS = 3e4;
|
|
17
|
+
var DEFAULT_NO_PROGRESS_STEP_LIMIT = 256;
|
|
18
|
+
function restartLimitError(restarts, destPath) {
|
|
19
|
+
return new SirannonError(
|
|
20
|
+
`The copy to '${destPath}' returned to page one ${restarts} times because another connection wrote to the source database or ran a RESTART or TRUNCATE checkpoint on it. Close any other connection that writes to this file, route its writes through Sirannon, and run the copy again.`,
|
|
21
|
+
"BACKUP_RESTARTED"
|
|
22
|
+
);
|
|
23
|
+
}
|
|
24
|
+
function noProgressError(steps, destPath) {
|
|
25
|
+
return new SirannonError(
|
|
26
|
+
`The copy to '${destPath}' moved no page it had not already moved across ${steps} steps. Another connection restarts the copy on every step, or the source grows faster than the copy moves it. Close any other connection that writes to this file, or run the copy when the write rate is lower.`,
|
|
27
|
+
"BACKUP_RESTARTED"
|
|
28
|
+
);
|
|
29
|
+
}
|
|
30
|
+
function stallError(stallTimeoutMs, destPath) {
|
|
31
|
+
return new SirannonError(
|
|
32
|
+
`The copy to '${destPath}' moved no pages for ${stallTimeoutMs}ms. SQLite steps the copy once per turn of the event loop, so a caller that never lets the loop reach its timers and immediates holds the copy still. Let the event loop run between writes, or raise the stall timeout for a host this slow.`,
|
|
33
|
+
"BACKUP_STALLED"
|
|
34
|
+
);
|
|
35
|
+
}
|
|
36
|
+
function copiedPages(step) {
|
|
37
|
+
return step.totalPages - step.remainingPages;
|
|
38
|
+
}
|
|
39
|
+
function hasRestarted(step, previous) {
|
|
40
|
+
if (!previous) return false;
|
|
41
|
+
return copiedPages(step) < copiedPages(previous);
|
|
42
|
+
}
|
|
43
|
+
async function copyDatabaseStepwise(conn, options) {
|
|
44
|
+
if (!conn.copyDatabase) {
|
|
45
|
+
throw new SirannonError(
|
|
46
|
+
"This driver opens connections without a stepped copy call, so it cannot copy a database while writes continue",
|
|
47
|
+
"BACKUP_UNSUPPORTED"
|
|
48
|
+
);
|
|
49
|
+
}
|
|
50
|
+
const restartLimit = options.restartLimit ?? DEFAULT_RESTART_LIMIT;
|
|
51
|
+
const stallTimeoutMs = options.stallTimeoutMs ?? DEFAULT_STALL_TIMEOUT_MS;
|
|
52
|
+
const noProgressStepLimit = options.noProgressStepLimit ?? DEFAULT_NO_PROGRESS_STEP_LIMIT;
|
|
53
|
+
let previous = null;
|
|
54
|
+
let restarts = 0;
|
|
55
|
+
let furthestCopied = -1;
|
|
56
|
+
let stepsWithoutProgress = 0;
|
|
57
|
+
let stopped = null;
|
|
58
|
+
let stallTimer = null;
|
|
59
|
+
let reportStall = () => {
|
|
60
|
+
};
|
|
61
|
+
const stalled = new Promise((_, reject) => {
|
|
62
|
+
reportStall = reject;
|
|
63
|
+
});
|
|
64
|
+
const armStall = () => {
|
|
65
|
+
if (stallTimeoutMs <= 0) return;
|
|
66
|
+
if (stallTimer) clearTimeout(stallTimer);
|
|
67
|
+
stallTimer = setTimeout(() => {
|
|
68
|
+
stopped = stallError(stallTimeoutMs, options.destPath);
|
|
69
|
+
reportStall(stopped);
|
|
70
|
+
}, stallTimeoutMs);
|
|
71
|
+
stallTimer.unref?.();
|
|
72
|
+
};
|
|
73
|
+
armStall();
|
|
74
|
+
const copy = conn.copyDatabase({
|
|
75
|
+
destPath: options.destPath,
|
|
76
|
+
pagesPerStep: options.pagesPerStep ?? DEFAULT_PAGES_PER_STEP,
|
|
77
|
+
onStep: (step) => {
|
|
78
|
+
if (stopped) throw stopped;
|
|
79
|
+
armStall();
|
|
80
|
+
if (hasRestarted(step, previous)) {
|
|
81
|
+
restarts++;
|
|
82
|
+
if (restarts > restartLimit) {
|
|
83
|
+
stopped = restartLimitError(restarts, options.destPath);
|
|
84
|
+
throw stopped;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
if (copiedPages(step) > furthestCopied) {
|
|
88
|
+
furthestCopied = copiedPages(step);
|
|
89
|
+
stepsWithoutProgress = 0;
|
|
90
|
+
} else if (step.remainingPages > 0 && ++stepsWithoutProgress > noProgressStepLimit) {
|
|
91
|
+
stopped = noProgressError(stepsWithoutProgress, options.destPath);
|
|
92
|
+
throw stopped;
|
|
93
|
+
}
|
|
94
|
+
previous = step;
|
|
95
|
+
options.onStep?.({ totalPages: step.totalPages, remainingPages: step.remainingPages, restarts });
|
|
96
|
+
}
|
|
97
|
+
}).catch((err) => {
|
|
98
|
+
if (stopped) throw stopped;
|
|
99
|
+
throw err instanceof SirannonError ? err : new BackupError(`Copy to '${options.destPath}' failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
100
|
+
});
|
|
101
|
+
copy.catch(() => {
|
|
102
|
+
});
|
|
103
|
+
try {
|
|
104
|
+
const final = await Promise.race([copy, stalled]);
|
|
105
|
+
return { pageCount: final.totalPages, restarts };
|
|
106
|
+
} catch (err) {
|
|
107
|
+
options.onCopyLeftRunning?.(copy);
|
|
108
|
+
throw err;
|
|
109
|
+
} finally {
|
|
110
|
+
if (stallTimer) clearTimeout(stallTimer);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
// src/core/backup/staged-copy.ts
|
|
115
|
+
var STAGED_FILE_NAME = "copy.db";
|
|
116
|
+
function destinationError(name, index, err) {
|
|
117
|
+
return new SirannonError(
|
|
118
|
+
`The destination refused piece ${index} of '${name}': ${err instanceof Error ? err.message : String(err)}`,
|
|
119
|
+
"BACKUP_DESTINATION_ERROR"
|
|
120
|
+
);
|
|
121
|
+
}
|
|
122
|
+
async function readPageSize(conn) {
|
|
123
|
+
const stmt = await conn.prepare("PRAGMA page_size");
|
|
124
|
+
const row = await stmt.get();
|
|
125
|
+
return row ? Number(row.page_size) : 0;
|
|
126
|
+
}
|
|
127
|
+
async function sendPieces(stagedPath, destination, name, pieceBytes, fingerprint, report) {
|
|
128
|
+
const file = await open(stagedPath, "r");
|
|
129
|
+
const digest = fingerprint ? createHash("sha256") : null;
|
|
130
|
+
let index = 0;
|
|
131
|
+
let bytesWritten = 0;
|
|
132
|
+
try {
|
|
133
|
+
const buffer = Buffer.allocUnsafe(pieceBytes);
|
|
134
|
+
for (; ; ) {
|
|
135
|
+
let filled = 0;
|
|
136
|
+
while (filled < pieceBytes) {
|
|
137
|
+
const { bytesRead } = await file.read(buffer, filled, pieceBytes - filled, index * pieceBytes + filled);
|
|
138
|
+
if (bytesRead === 0) break;
|
|
139
|
+
filled += bytesRead;
|
|
140
|
+
}
|
|
141
|
+
if (filled === 0) break;
|
|
142
|
+
const piece = new Uint8Array(filled);
|
|
143
|
+
piece.set(buffer.subarray(0, filled));
|
|
144
|
+
digest?.update(piece);
|
|
145
|
+
try {
|
|
146
|
+
await destination.writePiece(name, index, piece);
|
|
147
|
+
} catch (err) {
|
|
148
|
+
throw destinationError(name, index, err);
|
|
149
|
+
}
|
|
150
|
+
index++;
|
|
151
|
+
bytesWritten += filled;
|
|
152
|
+
report(index, bytesWritten);
|
|
153
|
+
if (filled < pieceBytes) break;
|
|
154
|
+
}
|
|
155
|
+
} finally {
|
|
156
|
+
await file.close();
|
|
157
|
+
}
|
|
158
|
+
return { pieceCount: index, bytesWritten, ...digest ? { fingerprint: digest.digest("hex") } : {} };
|
|
159
|
+
}
|
|
160
|
+
async function copyToDestinationStaged(conn, request) {
|
|
161
|
+
const runId = randomHex(8);
|
|
162
|
+
const name = request.name ?? `backup-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}.db`;
|
|
163
|
+
const pieceBytes = request.pieceBytes ?? DEFAULT_PIECE_BYTES;
|
|
164
|
+
if (!Number.isInteger(pieceBytes) || pieceBytes <= 0) {
|
|
165
|
+
throw new SirannonError(
|
|
166
|
+
`Piece size must be a positive whole number of bytes, and it was ${pieceBytes}`,
|
|
167
|
+
"BACKUP_ERROR"
|
|
168
|
+
);
|
|
169
|
+
}
|
|
170
|
+
const startedAt = Date.now();
|
|
171
|
+
const stagingRoot = await mkdtemp(join(request.stagingDir ?? tmpdir(), "sirannon-backup-"));
|
|
172
|
+
const stagedPath = join(stagingRoot, STAGED_FILE_NAME);
|
|
173
|
+
let firstStepSeen = false;
|
|
174
|
+
let copyLeftRunning = null;
|
|
175
|
+
const emit = (progress) => request.onProgress?.({ runId, ...progress });
|
|
176
|
+
try {
|
|
177
|
+
const pageSize = await readPageSize(conn);
|
|
178
|
+
const copyStartedAt = Date.now();
|
|
179
|
+
const copy = await copyDatabaseStepwise(conn, {
|
|
180
|
+
destPath: stagedPath,
|
|
181
|
+
...request.pagesPerStep === void 0 ? {} : { pagesPerStep: request.pagesPerStep },
|
|
182
|
+
...request.restartLimit === void 0 ? {} : { restartLimit: request.restartLimit },
|
|
183
|
+
...request.stallTimeoutMs === void 0 ? {} : { stallTimeoutMs: request.stallTimeoutMs },
|
|
184
|
+
...request.noProgressStepLimit === void 0 ? {} : { noProgressStepLimit: request.noProgressStepLimit },
|
|
185
|
+
onCopyLeftRunning: (copy2) => {
|
|
186
|
+
copyLeftRunning = copy2;
|
|
187
|
+
},
|
|
188
|
+
onStep: (step) => {
|
|
189
|
+
if (!firstStepSeen) {
|
|
190
|
+
firstStepSeen = true;
|
|
191
|
+
request.onFirstStep?.();
|
|
192
|
+
}
|
|
193
|
+
emit({ phase: "copy", ...step, piecesWritten: 0, bytesWritten: 0 });
|
|
194
|
+
}
|
|
195
|
+
});
|
|
196
|
+
const copyMs = Date.now() - copyStartedAt;
|
|
197
|
+
const transferStartedAt = Date.now();
|
|
198
|
+
const sent = await sendPieces(
|
|
199
|
+
stagedPath,
|
|
200
|
+
request.destination,
|
|
201
|
+
name,
|
|
202
|
+
pieceBytes,
|
|
203
|
+
request.fingerprint ?? true,
|
|
204
|
+
(piecesWritten, bytesWritten) => emit({
|
|
205
|
+
phase: "transfer",
|
|
206
|
+
totalPages: copy.pageCount,
|
|
207
|
+
remainingPages: 0,
|
|
208
|
+
restarts: copy.restarts,
|
|
209
|
+
piecesWritten,
|
|
210
|
+
bytesWritten
|
|
211
|
+
})
|
|
212
|
+
);
|
|
213
|
+
const transferMs = Date.now() - transferStartedAt;
|
|
214
|
+
const finishedAt = Date.now();
|
|
215
|
+
return {
|
|
216
|
+
runId,
|
|
217
|
+
databaseId: request.databaseId,
|
|
218
|
+
sourcePath: request.sourcePath,
|
|
219
|
+
kind: "full",
|
|
220
|
+
route: "staged",
|
|
221
|
+
destinationName: name,
|
|
222
|
+
startedAt,
|
|
223
|
+
finishedAt,
|
|
224
|
+
durationMs: finishedAt - startedAt,
|
|
225
|
+
copyMs,
|
|
226
|
+
transferMs,
|
|
227
|
+
pageCount: copy.pageCount,
|
|
228
|
+
pageSize,
|
|
229
|
+
bytesWritten: sent.bytesWritten,
|
|
230
|
+
pieceCount: sent.pieceCount,
|
|
231
|
+
pieceBytes,
|
|
232
|
+
restarts: copy.restarts,
|
|
233
|
+
...sent.fingerprint ? { fingerprint: sent.fingerprint } : {}
|
|
234
|
+
};
|
|
235
|
+
} finally {
|
|
236
|
+
const removeStaging = () => rm(stagingRoot, { recursive: true, force: true }).catch(() => {
|
|
237
|
+
});
|
|
238
|
+
if (copyLeftRunning) void copyLeftRunning.then(removeStaging, removeStaging);
|
|
239
|
+
else await removeStaging();
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
// src/core/backup/backup.ts
|
|
5
244
|
var BACKUP_FILE_PREFIX = "backup";
|
|
245
|
+
function once(action) {
|
|
246
|
+
let done = false;
|
|
247
|
+
return () => {
|
|
248
|
+
if (done) return;
|
|
249
|
+
done = true;
|
|
250
|
+
action();
|
|
251
|
+
};
|
|
252
|
+
}
|
|
6
253
|
function hasControlCharacters(s) {
|
|
7
254
|
for (let i = 0; i < s.length; i++) {
|
|
8
255
|
const code = s.charCodeAt(i);
|
|
@@ -11,7 +258,7 @@ function hasControlCharacters(s) {
|
|
|
11
258
|
return false;
|
|
12
259
|
}
|
|
13
260
|
var BackupManager = class {
|
|
14
|
-
async backup(conn, destPath) {
|
|
261
|
+
async backup(conn, destPath, onFirstStep) {
|
|
15
262
|
if (hasControlCharacters(destPath)) {
|
|
16
263
|
throw new BackupError("Backup path contains invalid characters");
|
|
17
264
|
}
|
|
@@ -33,17 +280,20 @@ var BackupManager = class {
|
|
|
33
280
|
if (existsSync(resolved)) {
|
|
34
281
|
throw new BackupError(`Backup destination '${destPath}' already exists`);
|
|
35
282
|
}
|
|
36
|
-
const escaped = resolved.replace(/'/g, "''");
|
|
37
283
|
try {
|
|
38
|
-
await conn
|
|
284
|
+
await copyDatabaseStepwise(conn, { destPath: resolved, onStep: onFirstStep ? once(onFirstStep) : void 0 });
|
|
39
285
|
} catch (err) {
|
|
40
286
|
try {
|
|
41
287
|
rmSync(resolved, { force: true });
|
|
42
288
|
} catch {
|
|
43
289
|
}
|
|
290
|
+
if (err instanceof SirannonError) throw err;
|
|
44
291
|
throw new BackupError(`Backup to '${destPath}' failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
45
292
|
}
|
|
46
293
|
}
|
|
294
|
+
copyToDestination(conn, request) {
|
|
295
|
+
return copyToDestinationStaged(conn, request);
|
|
296
|
+
}
|
|
47
297
|
generateFilename() {
|
|
48
298
|
const ts = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
49
299
|
return `${BACKUP_FILE_PREFIX}-${ts}.db`;
|
|
@@ -429,7 +679,10 @@ var BackupScheduler = class {
|
|
|
429
679
|
const runBackup = async () => {
|
|
430
680
|
try {
|
|
431
681
|
const destPath = join(resolvedDir, this.manager.generateFilename());
|
|
432
|
-
await
|
|
682
|
+
await startCopyWithoutHoldingWriter(
|
|
683
|
+
runExclusive,
|
|
684
|
+
(onFirstStep) => this.manager.backup(conn, destPath, onFirstStep)
|
|
685
|
+
);
|
|
433
686
|
this.manager.rotate(resolvedDir, maxFiles);
|
|
434
687
|
} catch (err) {
|
|
435
688
|
if (onError) {
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { tableColumnNames, tablePkColumns, selectMinChangeSeqSql, selectMaxChangeSeq, deleteChangesBeforeUpToSeqSql, deleteChangesBeforeSql, selectTableExists, ensureChangesTable, ensureMetaTable, insertMetaValueIfAbsent, selectMetaValue, selectChangesAfterSeqSql, selectTablesChangesInRangeSql } from './chunk-7FQRQH5Z.mjs';
|
|
2
2
|
import { CHANGES_TABLE, isReservedIdentifier, CDC_TRIGGER_PREFIX, encodeTaggedValues, decodeTaggedValues, SAFE_INT_BOUND_TEXT } from './chunk-7R4ER4FB.mjs';
|
|
3
3
|
import { synchronousPragmaValue } from './chunk-OUSWVNWT.mjs';
|
|
4
|
+
import { randomHex } from './chunk-V5FUJEYY.mjs';
|
|
4
5
|
import { SirannonError, ForbiddenSqlError, CDCError, MigrationError } from './chunk-PBRXXISQ.mjs';
|
|
5
6
|
|
|
6
7
|
// src/core/bulk-load.ts
|
|
@@ -549,13 +550,6 @@ function reconcileMigrationChecksums(migrations, applied) {
|
|
|
549
550
|
return backfills;
|
|
550
551
|
}
|
|
551
552
|
|
|
552
|
-
// src/core/random-hex.ts
|
|
553
|
-
function randomHex(byteLength) {
|
|
554
|
-
const bytes = new Uint8Array(byteLength);
|
|
555
|
-
globalThis.crypto.getRandomValues(bytes);
|
|
556
|
-
return Array.from(bytes, (value) => value.toString(16).padStart(2, "0")).join("");
|
|
557
|
-
}
|
|
558
|
-
|
|
559
553
|
// src/core/cdc/epoch.ts
|
|
560
554
|
var EPOCH_KEY = "cdc_epoch";
|
|
561
555
|
async function ensureCdcEpoch(conn) {
|
|
@@ -855,4 +849,4 @@ var TransactionGrouper = class {
|
|
|
855
849
|
}
|
|
856
850
|
};
|
|
857
851
|
|
|
858
|
-
export { ChangeTracker, PrimedSubscription, SubscriptionBuilderImpl, SubscriptionManager, TransactionGrouper, ensureCdcEpoch, filteredChange, isBulkLoadDurability, migrationChecksum, migrationContentChecksum, needsResync,
|
|
852
|
+
export { ChangeTracker, PrimedSubscription, SubscriptionBuilderImpl, SubscriptionManager, TransactionGrouper, ensureCdcEpoch, filteredChange, isBulkLoadDurability, migrationChecksum, migrationContentChecksum, needsResync, reconcileMigrationChecksums, runBulkLoad, startPolling };
|
|
@@ -1,12 +1,22 @@
|
|
|
1
|
-
import { BackupManager, BackupScheduler } from './chunk-
|
|
2
|
-
import { WORKER_CANCELLED_CODE, deserializeError } from './chunk-
|
|
3
|
-
import { SirannonError } from './chunk-PBRXXISQ.mjs';
|
|
1
|
+
import { BackupManager, BackupScheduler } from './chunk-23HKRYSM.mjs';
|
|
2
|
+
import { WORKER_CANCELLED_CODE, deserializeError } from './chunk-YLAVDZCS.mjs';
|
|
3
|
+
import { SirannonError, ExtensionError } from './chunk-PBRXXISQ.mjs';
|
|
4
4
|
import { Worker } from 'worker_threads';
|
|
5
5
|
import { existsSync } from 'fs';
|
|
6
6
|
import { resolve, dirname, join, sep } from 'path';
|
|
7
7
|
import { fileURLToPath, pathToFileURL } from 'url';
|
|
8
8
|
import { AsyncLocalStorage } from 'async_hooks';
|
|
9
9
|
|
|
10
|
+
// src/core/driver/extension.ts
|
|
11
|
+
async function loadThroughRuntime(extensionPath, load) {
|
|
12
|
+
try {
|
|
13
|
+
load();
|
|
14
|
+
} catch (err) {
|
|
15
|
+
if (err instanceof SirannonError) throw err;
|
|
16
|
+
throw new ExtensionError(extensionPath, err instanceof Error ? err.message : String(err));
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
|
|
10
20
|
// src/core/driver/statement-cache.ts
|
|
11
21
|
var STATEMENT_CACHE_CAPACITY = 128;
|
|
12
22
|
function createStatementCache(prepare) {
|
|
@@ -102,6 +112,7 @@ var WriterWorker = class _WriterWorker {
|
|
|
102
112
|
closed = false;
|
|
103
113
|
fatal = null;
|
|
104
114
|
restarts = 0;
|
|
115
|
+
loadedExtensions = [];
|
|
105
116
|
connection;
|
|
106
117
|
static async start(entry, path, options, workerOptions) {
|
|
107
118
|
const worker = new _WriterWorker(
|
|
@@ -134,11 +145,20 @@ var WriterWorker = class _WriterWorker {
|
|
|
134
145
|
this.fault(new SirannonError(`Writer worker exited with code ${code}`, "WRITER_WORKER_EXIT"));
|
|
135
146
|
}
|
|
136
147
|
});
|
|
137
|
-
this.ready = this.send(this.openRequest).then(() =>
|
|
148
|
+
this.ready = this.send(this.openRequest).then(() => this.reloadExtensionsOntoRestartedConnection());
|
|
138
149
|
this.ready.catch(() => {
|
|
139
150
|
});
|
|
140
151
|
}
|
|
152
|
+
async reloadExtensionsOntoRestartedConnection() {
|
|
153
|
+
for (const path of this.loadedExtensions) {
|
|
154
|
+
await this.send({ kind: "loadExtension", path });
|
|
155
|
+
}
|
|
156
|
+
}
|
|
141
157
|
onResponse(res) {
|
|
158
|
+
if ("kind" in res) {
|
|
159
|
+
this.onCopyStep(res.id, res.step);
|
|
160
|
+
return;
|
|
161
|
+
}
|
|
142
162
|
const entry = this.pending.get(res.id);
|
|
143
163
|
if (!entry) return;
|
|
144
164
|
this.pending.delete(res.id);
|
|
@@ -158,6 +178,26 @@ var WriterWorker = class _WriterWorker {
|
|
|
158
178
|
}
|
|
159
179
|
entry.reject(deserializeError(res.error));
|
|
160
180
|
}
|
|
181
|
+
onCopyStep(id, step) {
|
|
182
|
+
const entry = this.pending.get(id);
|
|
183
|
+
if (!entry) return;
|
|
184
|
+
if (entry.timer) {
|
|
185
|
+
clearTimeout(entry.timer);
|
|
186
|
+
entry.timer = this.timeoutMs > 0 ? setTimeout(() => this.onDeadline(id), this.timeoutMs) : null;
|
|
187
|
+
entry.timer?.unref?.();
|
|
188
|
+
}
|
|
189
|
+
try {
|
|
190
|
+
entry.onStep?.(step);
|
|
191
|
+
} catch (err) {
|
|
192
|
+
this.pending.delete(id);
|
|
193
|
+
clearPendingTimers(entry);
|
|
194
|
+
try {
|
|
195
|
+
this.worker?.postMessage({ kind: "cancel", id });
|
|
196
|
+
} catch {
|
|
197
|
+
}
|
|
198
|
+
entry.reject(err instanceof Error ? err : new Error(String(err)));
|
|
199
|
+
}
|
|
200
|
+
}
|
|
161
201
|
rejectPending(id, err) {
|
|
162
202
|
const entry = this.pending.get(id);
|
|
163
203
|
if (!entry) return;
|
|
@@ -165,7 +205,13 @@ var WriterWorker = class _WriterWorker {
|
|
|
165
205
|
clearPendingTimers(entry);
|
|
166
206
|
entry.reject(err);
|
|
167
207
|
}
|
|
168
|
-
unresponsiveError(waitedMs) {
|
|
208
|
+
unresponsiveError(waitedMs, kind) {
|
|
209
|
+
if (kind === "copyDatabase") {
|
|
210
|
+
return new SirannonError(
|
|
211
|
+
`The writer worker moved no page of the copy for ${waitedMs}ms, so the copy was stopped and nothing it wrote is usable`,
|
|
212
|
+
"BACKUP_STALLED"
|
|
213
|
+
);
|
|
214
|
+
}
|
|
169
215
|
return new SirannonError(
|
|
170
216
|
`Writer worker did not respond within ${waitedMs}ms; the operation's outcome is unknown`,
|
|
171
217
|
"WRITER_WORKER_TIMEOUT"
|
|
@@ -180,17 +226,17 @@ var WriterWorker = class _WriterWorker {
|
|
|
180
226
|
if (!entry) return;
|
|
181
227
|
const worker = this.worker;
|
|
182
228
|
if (!entry.cancellable || !worker) {
|
|
183
|
-
this.rejectPending(id, this.unresponsiveError(this.timeoutMs));
|
|
229
|
+
this.rejectPending(id, this.unresponsiveError(this.timeoutMs, entry.kind));
|
|
184
230
|
return;
|
|
185
231
|
}
|
|
186
232
|
try {
|
|
187
233
|
worker.postMessage({ kind: "cancel", id });
|
|
188
234
|
} catch {
|
|
189
|
-
this.rejectPending(id, this.unresponsiveError(this.timeoutMs));
|
|
235
|
+
this.rejectPending(id, this.unresponsiveError(this.timeoutMs, entry.kind));
|
|
190
236
|
return;
|
|
191
237
|
}
|
|
192
238
|
entry.graceTimer = setTimeout(() => {
|
|
193
|
-
this.rejectPending(id, this.unresponsiveError(this.timeoutMs * 2));
|
|
239
|
+
this.rejectPending(id, this.unresponsiveError(this.timeoutMs * 2, entry.kind));
|
|
194
240
|
}, this.timeoutMs);
|
|
195
241
|
entry.graceTimer.unref?.();
|
|
196
242
|
}
|
|
@@ -226,7 +272,7 @@ var WriterWorker = class _WriterWorker {
|
|
|
226
272
|
}
|
|
227
273
|
this.spawn();
|
|
228
274
|
}
|
|
229
|
-
send(request) {
|
|
275
|
+
send(request, onStep) {
|
|
230
276
|
const worker = this.worker;
|
|
231
277
|
if (!worker) {
|
|
232
278
|
return Promise.reject(
|
|
@@ -241,8 +287,16 @@ var WriterWorker = class _WriterWorker {
|
|
|
241
287
|
timer = setTimeout(() => this.onDeadline(id), this.timeoutMs);
|
|
242
288
|
timer.unref?.();
|
|
243
289
|
}
|
|
244
|
-
const cancellable = request.kind !== "open" && request.kind !== "close";
|
|
245
|
-
this.pending.set(id, {
|
|
290
|
+
const cancellable = request.kind !== "open" && request.kind !== "close" && request.kind !== "loadExtension";
|
|
291
|
+
this.pending.set(id, {
|
|
292
|
+
resolve: resolve2,
|
|
293
|
+
reject,
|
|
294
|
+
timer,
|
|
295
|
+
graceTimer: null,
|
|
296
|
+
cancellable,
|
|
297
|
+
kind: request.kind,
|
|
298
|
+
...onStep ? { onStep } : {}
|
|
299
|
+
});
|
|
246
300
|
try {
|
|
247
301
|
worker.postMessage(message);
|
|
248
302
|
} catch (err) {
|
|
@@ -257,10 +311,10 @@ var WriterWorker = class _WriterWorker {
|
|
|
257
311
|
}
|
|
258
312
|
});
|
|
259
313
|
}
|
|
260
|
-
request(request) {
|
|
314
|
+
request(request, onStep) {
|
|
261
315
|
if (this.fatal) return Promise.reject(this.fatal);
|
|
262
316
|
if (this.closed) return Promise.reject(new SirannonError("Writer worker is closed", "WRITER_WORKER_CLOSED"));
|
|
263
|
-
return this.ready.then(() => this.send(request)).then((value) => {
|
|
317
|
+
return this.ready.then(() => this.send(request, onStep)).then((value) => {
|
|
264
318
|
this.restarts = 0;
|
|
265
319
|
return value;
|
|
266
320
|
});
|
|
@@ -286,6 +340,14 @@ var WriterWorker = class _WriterWorker {
|
|
|
286
340
|
}))
|
|
287
341
|
}))
|
|
288
342
|
}),
|
|
343
|
+
copyDatabase: (request) => this.request(
|
|
344
|
+
{ kind: "copyDatabase", destPath: request.destPath, pagesPerStep: request.pagesPerStep },
|
|
345
|
+
request.onStep
|
|
346
|
+
),
|
|
347
|
+
loadExtension: async (extensionPath) => {
|
|
348
|
+
await this.request({ kind: "loadExtension", path: extensionPath });
|
|
349
|
+
if (!this.loadedExtensions.includes(extensionPath)) this.loadedExtensions.push(extensionPath);
|
|
350
|
+
},
|
|
289
351
|
transaction: async (fn) => {
|
|
290
352
|
await conn.exec("BEGIN");
|
|
291
353
|
try {
|
|
@@ -349,9 +411,10 @@ function nodeBackupEngine() {
|
|
|
349
411
|
const manager = new BackupManager();
|
|
350
412
|
const scheduler = new BackupScheduler(manager);
|
|
351
413
|
return {
|
|
352
|
-
backup: (conn, destPath) => manager.backup(conn, destPath),
|
|
414
|
+
backup: (conn, destPath, onFirstStep) => manager.backup(conn, destPath, onFirstStep),
|
|
415
|
+
copyToDestination: (conn, request) => manager.copyToDestination(conn, request),
|
|
353
416
|
schedule: (conn, options, runExclusive) => scheduler.schedule(conn, options, runExclusive)
|
|
354
417
|
};
|
|
355
418
|
}
|
|
356
419
|
|
|
357
|
-
export { WriterWorker, createStatementCache, narrowRowIntegers, narrowRowsIntegers, narrowSafeBigInt, nodeBackupEngine, nodeResolveExtensionPath, nodeWriterContext };
|
|
420
|
+
export { WriterWorker, createStatementCache, loadThroughRuntime, narrowRowIntegers, narrowRowsIntegers, narrowSafeBigInt, nodeBackupEngine, nodeResolveExtensionPath, nodeWriterContext };
|
|
@@ -6,6 +6,15 @@ var SirannonError = class extends Error {
|
|
|
6
6
|
this.name = "SirannonError";
|
|
7
7
|
}
|
|
8
8
|
};
|
|
9
|
+
var ExtensionError = class extends SirannonError {
|
|
10
|
+
constructor(path, cause) {
|
|
11
|
+
super(
|
|
12
|
+
cause ? `Failed to load extension '${path}': ${cause}` : `Failed to load extension '${path}'`,
|
|
13
|
+
"EXTENSION_ERROR"
|
|
14
|
+
);
|
|
15
|
+
this.name = "ExtensionError";
|
|
16
|
+
}
|
|
17
|
+
};
|
|
9
18
|
|
|
10
19
|
// src/core/driver/define.ts
|
|
11
20
|
function defineDriver(config) {
|
|
@@ -46,4 +55,4 @@ function synchronousPragmaValue(level) {
|
|
|
46
55
|
return value;
|
|
47
56
|
}
|
|
48
57
|
|
|
49
|
-
export { defineDriver, synchronousPragmaValue };
|
|
58
|
+
export { ExtensionError, SirannonError, defineDriver, synchronousPragmaValue };
|