@delali/sirannon-db 0.2.3-next.37 → 0.2.3-next.38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backup/index.d.ts +6 -84
- package/dist/backup/index.mjs +6 -584
- package/dist/backup-scheduler/index.d.ts +1 -1
- package/dist/backup-scheduler/index.mjs +2 -2
- package/dist/{change-tracker-DXHPFe9J.d.ts → change-tracker-BLKRX8ic.d.ts} +1 -1
- package/dist/chunk-CR6HTXSC.mjs +41 -0
- package/dist/{chunk-ZKDBGX7H.mjs → chunk-G7AXGAEA.mjs} +250 -344
- package/dist/chunk-GJ6SX53Z.mjs +272 -0
- package/dist/{chunk-72I5UI5C.mjs → chunk-KSYFFYAP.mjs} +62 -1
- package/dist/{chunk-BQFQ65OL.mjs → chunk-LXCGPZPA.mjs} +9 -1
- package/dist/chunk-P53I2G6J.mjs +525 -0
- package/dist/client/index.d.ts +8 -8
- package/dist/client/topology.d.ts +4 -4
- package/dist/{client-base-CGBpjZUB.d.ts → client-base-DldBgJWS.d.ts} +2 -2
- package/dist/core/index.d.ts +9 -9
- package/dist/core/index.mjs +214 -43
- package/dist/{database-955qag_W.d.ts → database-DSbJl-k5.d.ts} +43 -37
- package/dist/{chain-queries-D5aZ-8v2.d.ts → database-backup-CliAhaEm.d.ts} +52 -2
- package/dist/driver/better-sqlite3.d.ts +1 -1
- package/dist/driver/better-sqlite3.mjs +8 -6
- package/dist/driver/bun.d.ts +1 -1
- package/dist/driver/expo.d.ts +1 -1
- package/dist/driver/node.d.ts +1 -1
- package/dist/driver/node.mjs +8 -6
- package/dist/driver/wa-sqlite.d.ts +1 -1
- package/dist/file-migrations/index.d.ts +1 -1
- package/dist/{protocol-BfiujBsE.d.ts → protocol-BK0Yxu5P.d.ts} +2 -2
- package/dist/replication/index.d.ts +7 -7
- package/dist/restore-options-O5gYlr3n.d.ts +83 -0
- package/dist/server/index.d.ts +102 -9
- package/dist/server/index.mjs +354 -3
- package/dist/{server-options-D4VmfTHw.d.ts → server-options-CFgBQDLz.d.ts} +8 -1
- package/dist/{sirannon-DaGUawHs.d.ts → sirannon-Bo0PX2Cj.d.ts} +48 -5
- package/dist/transport/grpc.d.ts +3 -3
- package/dist/transport/memory.d.ts +3 -3
- package/dist/{types-BEjS69rs.d.ts → types-CL8uxXIV.d.ts} +2 -2
- package/dist/{types-Xb0L3o3w.d.ts → types-CopTgHG3.d.ts} +117 -20
- package/package.json +7 -7
- package/dist/{chunk-BQTVNI5H.mjs → chunk-IPDHHIG2.mjs} +1 -1
package/dist/backup/index.d.ts
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
|
-
import {
|
|
2
|
-
export { g as BackupChain,
|
|
3
|
-
export { B as BackupCapabilities, a as
|
|
1
|
+
import { B as BackupDestination, f as BackupRunReport } from '../types-CopTgHG3.js';
|
|
2
|
+
export { g as BackupChain, h as BackupChainBase, i as BackupChainChange, j as BackupChainPosition, k as BackupChainRecord, l as BackupCycle, m as BackupCycleError, n as BackupCycleOptions, o as BackupCycleStatus, p as BackupGroupMembership, q as BackupGroupSource, r as BackupNodePreference, s as BackupPiece, t as BackupProgress, u as BackupSkip, v as BackupSkipReason, w as BackupToDestinationOptions, x as BackupVerifyResult, y as DEFAULT_CHAIN_NAME, z as chainLogName, C as readBackupChains, E as verifyBackupRecord } from '../types-CopTgHG3.js';
|
|
3
|
+
export { B as BackupCapabilities, a as BackupChainLocation, b as BackupRestorePlan, c as BackupSafeToDeleteOptions, d as backupPiecesSafeToDelete, p as planBackupRestore } from '../database-backup-CliAhaEm.js';
|
|
4
|
+
import { B as BackupRestoreOptions, a as BackupRestoreReport } from '../restore-options-O5gYlr3n.js';
|
|
5
|
+
export { b as BackupRestoreProgress } from '../restore-options-O5gYlr3n.js';
|
|
4
6
|
import '../query-types-Cv8N7pXj.js';
|
|
5
7
|
|
|
6
8
|
/** What one assembled file took to build.
|
|
@@ -27,86 +29,6 @@ interface AssembleResult {
|
|
|
27
29
|
*/
|
|
28
30
|
declare function assembleFromDestination(destination: BackupDestination, report: BackupRunReport, destPath: string): Promise<AssembleResult>;
|
|
29
31
|
|
|
30
|
-
/** How far one restore has got, reported after every piece it fetches.
|
|
31
|
-
* @public
|
|
32
|
-
*/
|
|
33
|
-
interface BackupRestoreProgress {
|
|
34
|
-
/** Whether the restore is fetching the full copy or replaying the change pieces on top of it. */
|
|
35
|
-
phase: 'full-copy' | 'changes';
|
|
36
|
-
/** Pieces the restore has fetched. */
|
|
37
|
-
piecesFetched: number;
|
|
38
|
-
/** Bytes it has fetched. */
|
|
39
|
-
bytesFetched: number;
|
|
40
|
-
/** Change pieces it has replayed. */
|
|
41
|
-
changesApplied: number;
|
|
42
|
-
/** Change pieces the plan holds altogether. */
|
|
43
|
-
changesTotal: number;
|
|
44
|
-
}
|
|
45
|
-
/**
|
|
46
|
-
* How Sirannon rebuilds a database from a moment you name.
|
|
47
|
-
*
|
|
48
|
-
* @public
|
|
49
|
-
*/
|
|
50
|
-
interface BackupRestoreOptions {
|
|
51
|
-
/** Where the backups and their records are stored. */
|
|
52
|
-
destination: BackupDestination;
|
|
53
|
-
/** Driver the restore opens the rebuilt database through, so it can fold each batch of changes in. */
|
|
54
|
-
driver: SQLiteDriver;
|
|
55
|
-
/** Path Sirannon writes the rebuilt database to. A file already there stops the restore unless you set {@link BackupRestoreOptions.replaceExisting}. */
|
|
56
|
-
destPath: string;
|
|
57
|
-
/**
|
|
58
|
-
* Whether to replace a database already at that path. It defaults to false,
|
|
59
|
-
* because a restore removes the write-ahead log beside the path it writes to
|
|
60
|
-
* and any commit that log still held would go with it. Set this where you
|
|
61
|
-
* mean to restore over a database you no longer want.
|
|
62
|
-
*/
|
|
63
|
-
replaceExisting?: boolean;
|
|
64
|
-
/** Epoch milliseconds you want back. Defaults to now, which reaches the newest backup the destination holds. */
|
|
65
|
-
moment?: number;
|
|
66
|
-
/** Name the list of chains is stored under. Defaults to `sirannon-backup-chain`. */
|
|
67
|
-
chainName?: string;
|
|
68
|
-
/**
|
|
69
|
-
* How many change pieces to replay between one checkpoint and the next.
|
|
70
|
-
* Defaults to 16. This is what bounds the log the restore writes beside the
|
|
71
|
-
* database, so lower it where disk is tight and raise it where a long chain
|
|
72
|
-
* takes too many checkpoints.
|
|
73
|
-
*/
|
|
74
|
-
batchSize?: number;
|
|
75
|
-
/** Milliseconds one call to the destination may take before the restore stops with an error. Defaults to 10 minutes, and zero leaves the calls unbounded. */
|
|
76
|
-
destinationTimeoutMs?: number;
|
|
77
|
-
/** Called after every piece the restore fetches. */
|
|
78
|
-
onProgress?: (progress: BackupRestoreProgress) => void;
|
|
79
|
-
}
|
|
80
|
-
/** What one finished restore produced.
|
|
81
|
-
* @public
|
|
82
|
-
*/
|
|
83
|
-
interface BackupRestoreReport {
|
|
84
|
-
/** The chain the restore read. */
|
|
85
|
-
chainId: string;
|
|
86
|
-
/** Path of the rebuilt database. */
|
|
87
|
-
destPath: string;
|
|
88
|
-
/** Name the full copy underneath it is stored under. */
|
|
89
|
-
baseName: string;
|
|
90
|
-
/** Epoch milliseconds the rebuilt database reflects, which is when the last piece replayed was captured. */
|
|
91
|
-
restoresTo: number;
|
|
92
|
-
/** Pieces the restore fetched, counting the full copy and every change piece. */
|
|
93
|
-
pieceCount: number;
|
|
94
|
-
/** Bytes it fetched. */
|
|
95
|
-
bytesFetched: number;
|
|
96
|
-
/** Change pieces it replayed. */
|
|
97
|
-
changesApplied: number;
|
|
98
|
-
/** Log frames those pieces held. */
|
|
99
|
-
framesApplied: number;
|
|
100
|
-
/** Batches it replayed them in, each one folded into the database by a checkpoint of its own. */
|
|
101
|
-
batchCount: number;
|
|
102
|
-
/** Epoch milliseconds the restore started. */
|
|
103
|
-
startedAt: number;
|
|
104
|
-
/** Epoch milliseconds it finished. */
|
|
105
|
-
finishedAt: number;
|
|
106
|
-
/** Milliseconds it took. */
|
|
107
|
-
durationMs: number;
|
|
108
|
-
}
|
|
109
|
-
|
|
110
32
|
/**
|
|
111
33
|
* Rebuilds a database from the moment you name and leaves it at a path of your
|
|
112
34
|
* choosing.
|
|
@@ -148,4 +70,4 @@ interface BackupRestoreReport {
|
|
|
148
70
|
*/
|
|
149
71
|
declare function restoreBackup(options: BackupRestoreOptions): Promise<BackupRestoreReport>;
|
|
150
72
|
|
|
151
|
-
export { type AssembleResult, BackupDestination,
|
|
73
|
+
export { type AssembleResult, BackupDestination, BackupRestoreOptions, BackupRestoreReport, BackupRunReport, assembleFromDestination, restoreBackup };
|
package/dist/backup/index.mjs
CHANGED
|
@@ -1,586 +1,8 @@
|
|
|
1
|
-
|
|
1
|
+
export { assembleFromDestination, restoreBackup } from '../chunk-P53I2G6J.mjs';
|
|
2
2
|
export { backupPiecesSafeToDelete, planBackupRestore } from '../chunk-YNTDOFTN.mjs';
|
|
3
|
-
|
|
4
|
-
import
|
|
5
|
-
import
|
|
3
|
+
export { verifyBackupRecord } from '../chunk-CR6HTXSC.mjs';
|
|
4
|
+
import '../chunk-KSYFFYAP.mjs';
|
|
5
|
+
import '../chunk-GBM4AU24.mjs';
|
|
6
6
|
export { DEFAULT_CHAIN_NAME, chainLogName, readBackupChains } from '../chunk-YVF2ZJTE.mjs';
|
|
7
|
-
import
|
|
8
|
-
import
|
|
9
|
-
import { rename, open, rm, stat } from 'fs/promises';
|
|
10
|
-
import { createHash } from 'crypto';
|
|
11
|
-
|
|
12
|
-
function destinationError(message, err) {
|
|
13
|
-
if (err instanceof SirannonError) return err;
|
|
14
|
-
const detail = err instanceof Error ? `: ${err.message}` : "";
|
|
15
|
-
return new SirannonError(`${message}${detail}`, "BACKUP_DESTINATION_ERROR");
|
|
16
|
-
}
|
|
17
|
-
function assertNoPieceIsMissing(pieces, file) {
|
|
18
|
-
if (pieces.length === 0) {
|
|
19
|
-
throw destinationError(`The destination holds no pieces named '${file.name}'`);
|
|
20
|
-
}
|
|
21
|
-
for (let expected = 0; expected < file.pieceCount; expected++) {
|
|
22
|
-
if (pieces[expected]?.index !== expected) {
|
|
23
|
-
throw destinationError(
|
|
24
|
-
`The destination is missing piece ${expected} of '${file.name}', so the file cannot be assembled`
|
|
25
|
-
);
|
|
26
|
-
}
|
|
27
|
-
}
|
|
28
|
-
if (pieces.length > file.pieceCount) {
|
|
29
|
-
throw destinationError(
|
|
30
|
-
`The destination holds ${pieces.length} pieces of '${file.name}' where the run wrote ${file.pieceCount}, so a later piece belongs to a different run`
|
|
31
|
-
);
|
|
32
|
-
}
|
|
33
|
-
}
|
|
34
|
-
async function listStoredFilePieces(destination, file) {
|
|
35
|
-
let listed;
|
|
36
|
-
try {
|
|
37
|
-
listed = [...await destination.listPieces(file.name)];
|
|
38
|
-
} catch (err) {
|
|
39
|
-
throw destinationError(`The destination could not list the pieces of '${file.name}'`, err);
|
|
40
|
-
}
|
|
41
|
-
listed.sort((left, right) => left.index - right.index);
|
|
42
|
-
assertNoPieceIsMissing(listed, file);
|
|
43
|
-
return listed;
|
|
44
|
-
}
|
|
45
|
-
async function fetchStoredFile(destination, file, pieces, take) {
|
|
46
|
-
const digest = file.fingerprint === void 0 ? null : createHash("sha256");
|
|
47
|
-
let bytesFetched = 0;
|
|
48
|
-
for (const piece of pieces) {
|
|
49
|
-
let bytes;
|
|
50
|
-
try {
|
|
51
|
-
bytes = await destination.readPiece(file.name, piece.index);
|
|
52
|
-
} catch (err) {
|
|
53
|
-
throw destinationError(`The destination could not return piece ${piece.index} of '${file.name}'`, err);
|
|
54
|
-
}
|
|
55
|
-
digest?.update(bytes);
|
|
56
|
-
bytesFetched += bytes.byteLength;
|
|
57
|
-
await take(piece.index, bytes);
|
|
58
|
-
}
|
|
59
|
-
const fingerprint = digest?.digest("hex");
|
|
60
|
-
if (bytesFetched !== file.bytesWritten) {
|
|
61
|
-
throw destinationError(
|
|
62
|
-
`The pieces of '${file.name}' hold ${bytesFetched} bytes where the run wrote ${file.bytesWritten}`
|
|
63
|
-
);
|
|
64
|
-
}
|
|
65
|
-
if (fingerprint !== void 0 && fingerprint !== file.fingerprint) {
|
|
66
|
-
throw destinationError(`The pieces of '${file.name}' do not match the fingerprint the run recorded`);
|
|
67
|
-
}
|
|
68
|
-
return { bytesFetched, pieceCount: pieces.length, ...fingerprint === void 0 ? {} : { fingerprint } };
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
// src/core/backup/write-fully.ts
|
|
72
|
-
async function writeFully(handle, path, bytes, byteLength, offset) {
|
|
73
|
-
let written = 0;
|
|
74
|
-
while (written < byteLength) {
|
|
75
|
-
const result = await handle.write(bytes, written, byteLength - written, offset + written);
|
|
76
|
-
if (result.bytesWritten === 0) {
|
|
77
|
-
throw new SirannonError(
|
|
78
|
-
`Writing to '${path}' stopped after ${written} of ${byteLength} bytes. Check the free space and the permissions on that directory.`,
|
|
79
|
-
"BACKUP_ERROR"
|
|
80
|
-
);
|
|
81
|
-
}
|
|
82
|
-
written += result.bytesWritten;
|
|
83
|
-
}
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
// src/core/backup/assemble.ts
|
|
87
|
-
async function assembleStoredFile(destination, file, destPath, onPiece) {
|
|
88
|
-
const pieces = await listStoredFilePieces(destination, file);
|
|
89
|
-
try {
|
|
90
|
-
const handle = await open(destPath, "w");
|
|
91
|
-
let piecesRead = 0;
|
|
92
|
-
let bytesWritten = 0;
|
|
93
|
-
try {
|
|
94
|
-
const fetched = await fetchStoredFile(destination, file, pieces, async (index, bytes) => {
|
|
95
|
-
await writeFully(handle, destPath, bytes, bytes.byteLength, index * file.pieceBytes);
|
|
96
|
-
piecesRead++;
|
|
97
|
-
bytesWritten += bytes.byteLength;
|
|
98
|
-
onPiece?.(piecesRead, bytesWritten);
|
|
99
|
-
});
|
|
100
|
-
return {
|
|
101
|
-
bytesWritten: fetched.bytesFetched,
|
|
102
|
-
pieceCount: fetched.pieceCount,
|
|
103
|
-
...fetched.fingerprint === void 0 ? {} : { fingerprint: fetched.fingerprint }
|
|
104
|
-
};
|
|
105
|
-
} finally {
|
|
106
|
-
await handle.close();
|
|
107
|
-
}
|
|
108
|
-
} catch (err) {
|
|
109
|
-
await rm(destPath, { force: true }).catch(() => {
|
|
110
|
-
});
|
|
111
|
-
throw err;
|
|
112
|
-
}
|
|
113
|
-
}
|
|
114
|
-
function assembleFromDestination(destination, report, destPath) {
|
|
115
|
-
return assembleStoredFile(destination, { ...report, name: report.destinationName }, destPath);
|
|
116
|
-
}
|
|
117
|
-
var DATABASE_HEADER_BYTES = 100;
|
|
118
|
-
var PAGE_SIZE_OFFSET = 16;
|
|
119
|
-
var PAGE_SIZE_FOR_65536 = 1;
|
|
120
|
-
var SMALLEST_PAGE_BYTES = 512;
|
|
121
|
-
var LARGEST_PAGE_BYTES = 65536;
|
|
122
|
-
var WRITE_FORMAT_OFFSET = 18;
|
|
123
|
-
var WRITE_AHEAD_LOGGING_FORMAT = 2;
|
|
124
|
-
var CHECKSUM_SEED = { first: 0, second: 0 };
|
|
125
|
-
function restoreError(message) {
|
|
126
|
-
return new SirannonError(message, "BACKUP_CHAIN_BROKEN");
|
|
127
|
-
}
|
|
128
|
-
async function readDatabaseHeader(path) {
|
|
129
|
-
const handle = await open(path, "r");
|
|
130
|
-
try {
|
|
131
|
-
const buffer = Buffer.allocUnsafe(DATABASE_HEADER_BYTES);
|
|
132
|
-
const { bytesRead } = await handle.read(buffer, 0, DATABASE_HEADER_BYTES, 0);
|
|
133
|
-
if (bytesRead < DATABASE_HEADER_BYTES) {
|
|
134
|
-
throw restoreError(`The full copy assembled into '${path}' is too short to be a SQLite database`);
|
|
135
|
-
}
|
|
136
|
-
const stated = buffer.readUInt16BE(PAGE_SIZE_OFFSET);
|
|
137
|
-
const pageSize = stated === PAGE_SIZE_FOR_65536 ? LARGEST_PAGE_BYTES : stated;
|
|
138
|
-
if (pageSize < SMALLEST_PAGE_BYTES || pageSize > LARGEST_PAGE_BYTES || (pageSize & pageSize - 1) !== 0) {
|
|
139
|
-
throw restoreError(
|
|
140
|
-
`The full copy assembled into '${path}' states a page size of ${pageSize}, which no SQLite database has`
|
|
141
|
-
);
|
|
142
|
-
}
|
|
143
|
-
return {
|
|
144
|
-
pageSize,
|
|
145
|
-
walMode: buffer.readUInt8(WRITE_FORMAT_OFFSET) === WRITE_AHEAD_LOGGING_FORMAT
|
|
146
|
-
};
|
|
147
|
-
} finally {
|
|
148
|
-
await handle.close();
|
|
149
|
-
}
|
|
150
|
-
}
|
|
151
|
-
function logHeaderBytes(pageSize, logSequence, salt1, salt2) {
|
|
152
|
-
const header = new Uint8Array(LOG_HEADER_BYTES);
|
|
153
|
-
const view = new DataView(header.buffer);
|
|
154
|
-
view.setUint32(0, MAGIC_WITH_LITTLE_ENDIAN_CONTENT, false);
|
|
155
|
-
view.setUint32(4, LOG_FORMAT_VERSION, false);
|
|
156
|
-
view.setUint32(8, pageSize, false);
|
|
157
|
-
view.setUint32(12, logSequence, false);
|
|
158
|
-
view.setUint32(16, salt1, false);
|
|
159
|
-
view.setUint32(20, salt2, false);
|
|
160
|
-
const checksum = foldLogChecksum(view, 0, 24, false, CHECKSUM_SEED);
|
|
161
|
-
view.setUint32(24, checksum.first, false);
|
|
162
|
-
view.setUint32(28, checksum.second, false);
|
|
163
|
-
return header;
|
|
164
|
-
}
|
|
165
|
-
function describeHeaderMismatch(captured, expected) {
|
|
166
|
-
if (captured.pageSize !== expected.pageSize) {
|
|
167
|
-
return `of ${captured.pageSize}-byte pages where its record says ${expected.pageSize}`;
|
|
168
|
-
}
|
|
169
|
-
if (captured.logSequence !== expected.logSequence) {
|
|
170
|
-
return `at checkpoint sequence ${captured.logSequence} where its record says ${expected.logSequence}`;
|
|
171
|
-
}
|
|
172
|
-
if (captured.salt1 !== expected.salt1 || captured.salt2 !== expected.salt2) {
|
|
173
|
-
return `salted ${captured.salt1} and ${captured.salt2} where its record says ${expected.salt1} and ${expected.salt2}`;
|
|
174
|
-
}
|
|
175
|
-
return void 0;
|
|
176
|
-
}
|
|
177
|
-
function randomSalt() {
|
|
178
|
-
return Number.parseInt(randomHex(4), 16);
|
|
179
|
-
}
|
|
180
|
-
var RestoreLogWriter = class _RestoreLogWriter {
|
|
181
|
-
constructor(handle, path, pageSize, salt1, salt2, checksum) {
|
|
182
|
-
this.handle = handle;
|
|
183
|
-
this.path = path;
|
|
184
|
-
this.pageSize = pageSize;
|
|
185
|
-
this.salt1 = salt1;
|
|
186
|
-
this.salt2 = salt2;
|
|
187
|
-
this.checksum = checksum;
|
|
188
|
-
}
|
|
189
|
-
carry = new Uint8Array(0);
|
|
190
|
-
checksum;
|
|
191
|
-
offset = LOG_HEADER_BYTES;
|
|
192
|
-
framesWritten = 0;
|
|
193
|
-
lastDatabasePages = 0;
|
|
194
|
-
skipRemaining = 0;
|
|
195
|
-
skipped = new Uint8Array(0);
|
|
196
|
-
expected;
|
|
197
|
-
closed = false;
|
|
198
|
-
/**
|
|
199
|
-
* Opens the log beside a restored database and writes its header.
|
|
200
|
-
*
|
|
201
|
-
* @param path - Path of the log file.
|
|
202
|
-
* @param pageSize - Size of one page of the database being restored, in bytes.
|
|
203
|
-
* @param logSequence - Checkpoint sequence number to record in the header.
|
|
204
|
-
* @returns The writer, ready for the first piece.
|
|
205
|
-
*/
|
|
206
|
-
static async create(path, pageSize, logSequence) {
|
|
207
|
-
const salt1 = randomSalt();
|
|
208
|
-
const salt2 = randomSalt();
|
|
209
|
-
const header = logHeaderBytes(pageSize, logSequence, salt1, salt2);
|
|
210
|
-
const view = new DataView(header.buffer);
|
|
211
|
-
const checksum = { first: view.getUint32(24, false), second: view.getUint32(28, false) };
|
|
212
|
-
const handle = await open(path, "w");
|
|
213
|
-
try {
|
|
214
|
-
await writeFully(handle, path, header, header.byteLength, 0);
|
|
215
|
-
} catch (err) {
|
|
216
|
-
await handle.close();
|
|
217
|
-
throw err;
|
|
218
|
-
}
|
|
219
|
-
return new _RestoreLogWriter(handle, path, pageSize, salt1, salt2, checksum);
|
|
220
|
-
}
|
|
221
|
-
/** Bytes one frame takes, being its header plus one page. */
|
|
222
|
-
get frameBytes() {
|
|
223
|
-
return LOG_FRAME_HEADER_BYTES + this.pageSize;
|
|
224
|
-
}
|
|
225
|
-
/**
|
|
226
|
-
* Starts a new change piece, naming how many bytes of log header stand in
|
|
227
|
-
* front of its first frame.
|
|
228
|
-
*
|
|
229
|
-
* Where the piece begins with a header, this reads it and checks it against
|
|
230
|
-
* the record that named the piece. A record claiming a page size the frames
|
|
231
|
-
* were never written at would otherwise have SQLite read every frame at the
|
|
232
|
-
* wrong length.
|
|
233
|
-
*
|
|
234
|
-
* @param headerBytes - Bytes of log header in front of the frames.
|
|
235
|
-
* @param expected - What the piece's chain record says that header holds.
|
|
236
|
-
*/
|
|
237
|
-
beginPiece(headerBytes, expected) {
|
|
238
|
-
if (this.carry.byteLength > 0) {
|
|
239
|
-
throw restoreError(
|
|
240
|
-
`The change piece before this one left ${this.carry.byteLength} bytes of a frame unwritten in '${this.path}', so the two would be joined into a frame neither of them holds`
|
|
241
|
-
);
|
|
242
|
-
}
|
|
243
|
-
this.skipRemaining = headerBytes;
|
|
244
|
-
this.skipped = new Uint8Array(0);
|
|
245
|
-
this.expected = expected;
|
|
246
|
-
}
|
|
247
|
-
/**
|
|
248
|
-
* Stamps every whole frame in the bytes given and appends them to the log.
|
|
249
|
-
* Where a frame is split across two pieces, this keeps the first part until
|
|
250
|
-
* the second one is added.
|
|
251
|
-
*
|
|
252
|
-
* @param bytes - The next run of bytes of the change piece.
|
|
253
|
-
*/
|
|
254
|
-
async add(bytes) {
|
|
255
|
-
let arriving = bytes;
|
|
256
|
-
if (this.skipRemaining > 0) {
|
|
257
|
-
const dropped = Math.min(this.skipRemaining, arriving.byteLength);
|
|
258
|
-
this.keepSkipped(arriving.subarray(0, dropped));
|
|
259
|
-
this.skipRemaining -= dropped;
|
|
260
|
-
arriving = arriving.subarray(dropped);
|
|
261
|
-
if (this.skipRemaining === 0) this.checkCapturedHeader();
|
|
262
|
-
}
|
|
263
|
-
if (arriving.byteLength === 0) return;
|
|
264
|
-
const working = new Uint8Array(this.carry.byteLength + arriving.byteLength);
|
|
265
|
-
working.set(this.carry, 0);
|
|
266
|
-
working.set(arriving, this.carry.byteLength);
|
|
267
|
-
const wholeFrames = Math.floor(working.byteLength / this.frameBytes);
|
|
268
|
-
if (wholeFrames === 0) {
|
|
269
|
-
this.carry = working;
|
|
270
|
-
return;
|
|
271
|
-
}
|
|
272
|
-
const stampedBytes = wholeFrames * this.frameBytes;
|
|
273
|
-
this.stamp(working, wholeFrames);
|
|
274
|
-
await writeFully(this.handle, this.path, working, stampedBytes, this.offset);
|
|
275
|
-
this.offset += stampedBytes;
|
|
276
|
-
this.carry = working.slice(stampedBytes);
|
|
277
|
-
}
|
|
278
|
-
keepSkipped(bytes) {
|
|
279
|
-
const kept = new Uint8Array(this.skipped.byteLength + bytes.byteLength);
|
|
280
|
-
kept.set(this.skipped, 0);
|
|
281
|
-
kept.set(bytes, this.skipped.byteLength);
|
|
282
|
-
this.skipped = kept;
|
|
283
|
-
}
|
|
284
|
-
checkCapturedHeader() {
|
|
285
|
-
const expected = this.expected;
|
|
286
|
-
if (!expected) return;
|
|
287
|
-
const captured = readLogHeader(this.skipped);
|
|
288
|
-
if (!captured) {
|
|
289
|
-
throw restoreError(
|
|
290
|
-
"The 32 bytes in front of the frames of a change piece are no log header SQLite wrote, so that piece reached the destination damaged"
|
|
291
|
-
);
|
|
292
|
-
}
|
|
293
|
-
const mismatch = describeHeaderMismatch(captured, expected);
|
|
294
|
-
if (mismatch) {
|
|
295
|
-
throw restoreError(
|
|
296
|
-
`A change piece was captured from a log ${mismatch}, so its record describes a different database and SQLite would read its frames at the wrong length`
|
|
297
|
-
);
|
|
298
|
-
}
|
|
299
|
-
}
|
|
300
|
-
stamp(working, wholeFrames) {
|
|
301
|
-
const view = new DataView(working.buffer, working.byteOffset, working.byteLength);
|
|
302
|
-
for (let frame = 0; frame < wholeFrames; frame++) {
|
|
303
|
-
const at = frame * this.frameBytes;
|
|
304
|
-
view.setUint32(at + 8, this.salt1, false);
|
|
305
|
-
view.setUint32(at + 12, this.salt2, false);
|
|
306
|
-
const overHeader = foldLogChecksum(view, at, 8, false, this.checksum);
|
|
307
|
-
const checksum = foldLogChecksum(view, at + LOG_FRAME_HEADER_BYTES, this.pageSize, false, overHeader);
|
|
308
|
-
view.setUint32(at + 16, checksum.first, false);
|
|
309
|
-
view.setUint32(at + 20, checksum.second, false);
|
|
310
|
-
this.checksum = checksum;
|
|
311
|
-
this.lastDatabasePages = view.getUint32(at + 4, false);
|
|
312
|
-
this.framesWritten++;
|
|
313
|
-
}
|
|
314
|
-
}
|
|
315
|
-
/**
|
|
316
|
-
* Closes the log and reports what it holds.
|
|
317
|
-
*
|
|
318
|
-
* @returns The frames the log holds, and the size the database reaches once SQLite replays them.
|
|
319
|
-
* @throws A `BACKUP_CHAIN_BROKEN` where the pieces ended part-way through a frame, or where the last frame commits nothing.
|
|
320
|
-
*/
|
|
321
|
-
async finish() {
|
|
322
|
-
await this.close();
|
|
323
|
-
if (this.carry.byteLength > 0) {
|
|
324
|
-
throw restoreError(
|
|
325
|
-
`The change pieces written into '${this.path}' end ${this.carry.byteLength} bytes into a frame of ${this.frameBytes}, so the chain is missing the rest of it`
|
|
326
|
-
);
|
|
327
|
-
}
|
|
328
|
-
if (this.lastDatabasePages === 0) {
|
|
329
|
-
throw restoreError(
|
|
330
|
-
`The last frame written into '${this.path}' commits no transaction, so SQLite would replay none of the batch`
|
|
331
|
-
);
|
|
332
|
-
}
|
|
333
|
-
return { frameCount: this.framesWritten, databasePages: this.lastDatabasePages };
|
|
334
|
-
}
|
|
335
|
-
/** Closes the log without checking it, which is what a failed batch calls before it removes the file. */
|
|
336
|
-
async abandon() {
|
|
337
|
-
await this.close().catch(() => {
|
|
338
|
-
});
|
|
339
|
-
}
|
|
340
|
-
async close() {
|
|
341
|
-
if (this.closed) return;
|
|
342
|
-
this.closed = true;
|
|
343
|
-
await this.handle.close();
|
|
344
|
-
}
|
|
345
|
-
};
|
|
346
|
-
|
|
347
|
-
// src/core/backup/restore-apply.ts
|
|
348
|
-
function chainError(message) {
|
|
349
|
-
return new SirannonError(message, "BACKUP_CHAIN_BROKEN");
|
|
350
|
-
}
|
|
351
|
-
function headerBytesOf(change) {
|
|
352
|
-
return change.position.firstFrame === 1 ? LOG_HEADER_BYTES : 0;
|
|
353
|
-
}
|
|
354
|
-
function expectedLogHeaderOf(change, pageSize) {
|
|
355
|
-
if (change.position.firstFrame !== 1) return void 0;
|
|
356
|
-
return {
|
|
357
|
-
pageSize,
|
|
358
|
-
logSequence: change.position.logSequence,
|
|
359
|
-
salt1: change.position.salt1,
|
|
360
|
-
salt2: change.position.salt2
|
|
361
|
-
};
|
|
362
|
-
}
|
|
363
|
-
function assertChangePiecesRunOn(changes, chainId) {
|
|
364
|
-
let previous;
|
|
365
|
-
for (const change of changes) {
|
|
366
|
-
if (previous === void 0) {
|
|
367
|
-
assertStartsAt(change, chainId, 1, "the full copy underneath it");
|
|
368
|
-
previous = change;
|
|
369
|
-
continue;
|
|
370
|
-
}
|
|
371
|
-
const sameRun = previous.position.logSequence === change.position.logSequence && previous.position.salt1 === change.position.salt1 && previous.position.salt2 === change.position.salt2;
|
|
372
|
-
const expected = sameRun ? previous.position.lastFrame + 1 : 1;
|
|
373
|
-
const follows = `change piece ${previous.sequence}, which ends at frame ${previous.position.lastFrame}`;
|
|
374
|
-
assertStartsAt(change, chainId, expected, follows);
|
|
375
|
-
previous = change;
|
|
376
|
-
}
|
|
377
|
-
}
|
|
378
|
-
function assertStartsAt(change, chainId, expected, follows) {
|
|
379
|
-
if (change.position.firstFrame === expected) return;
|
|
380
|
-
throw chainError(
|
|
381
|
-
`Change piece ${change.sequence} of chain '${chainId}' starts at frame ${change.position.firstFrame} where it has to start at frame ${expected} to follow ${follows}, so the frames in between are in no backup`
|
|
382
|
-
);
|
|
383
|
-
}
|
|
384
|
-
function assertPieceFits(change, chainId, frameBytes) {
|
|
385
|
-
const expected = headerBytesOf(change) + change.frameCount * frameBytes;
|
|
386
|
-
if (change.bytesWritten !== expected) {
|
|
387
|
-
throw chainError(
|
|
388
|
-
`Change piece ${change.sequence} of chain '${chainId}' holds ${change.bytesWritten} bytes where ${change.frameCount} frames of a database with ${frameBytes - LOG_FRAME_HEADER_BYTES}-byte pages come to ${expected}, so it was taken from a different database`
|
|
389
|
-
);
|
|
390
|
-
}
|
|
391
|
-
}
|
|
392
|
-
async function readPageCount(conn) {
|
|
393
|
-
const stmt = await conn.prepare("PRAGMA page_count");
|
|
394
|
-
const row = await stmt.get();
|
|
395
|
-
return row ? Number(row.page_count) : 0;
|
|
396
|
-
}
|
|
397
|
-
async function countDatabasePages(driver, destPath) {
|
|
398
|
-
const conn = await driver.open(destPath, { walMode: false, walAutoCheckpoint: 0 });
|
|
399
|
-
try {
|
|
400
|
-
return await readPageCount(conn);
|
|
401
|
-
} finally {
|
|
402
|
-
await conn.close();
|
|
403
|
-
}
|
|
404
|
-
}
|
|
405
|
-
async function foldLogIntoDatabase(driver, destPath, databasePages) {
|
|
406
|
-
const conn = await driver.open(destPath, { walMode: false, walAutoCheckpoint: 0 });
|
|
407
|
-
try {
|
|
408
|
-
const reached = await readPageCount(conn);
|
|
409
|
-
if (reached !== databasePages) {
|
|
410
|
-
throw chainError(
|
|
411
|
-
`Replaying the change pieces left '${destPath}' at ${reached} pages where the last frame of the batch commits ${databasePages}, so SQLite read none of those frames`
|
|
412
|
-
);
|
|
413
|
-
}
|
|
414
|
-
const checkpoint = await checkpointLog(conn);
|
|
415
|
-
if (!checkpoint.emptied) {
|
|
416
|
-
throw new SirannonError(
|
|
417
|
-
`The checkpoint after a batch left ${checkpoint.framesInLog} frames in the log beside '${destPath}', so the restore stopped rather than let the log grow`,
|
|
418
|
-
"BACKUP_ERROR"
|
|
419
|
-
);
|
|
420
|
-
}
|
|
421
|
-
} finally {
|
|
422
|
-
await conn.close();
|
|
423
|
-
}
|
|
424
|
-
}
|
|
425
|
-
async function applyChangeBatch(request) {
|
|
426
|
-
const logPath = `${request.destPath}-wal`;
|
|
427
|
-
await rm(`${request.destPath}-shm`, { force: true });
|
|
428
|
-
const writer = await RestoreLogWriter.create(logPath, request.pageSize, request.logSequence);
|
|
429
|
-
try {
|
|
430
|
-
for (const change of request.batch) {
|
|
431
|
-
assertPieceFits(change, change.chainId, writer.frameBytes);
|
|
432
|
-
const pieces = await listStoredFilePieces(request.destination, change);
|
|
433
|
-
writer.beginPiece(headerBytesOf(change), expectedLogHeaderOf(change, request.pageSize));
|
|
434
|
-
await fetchStoredFile(request.destination, change, pieces, async (_index, bytes) => {
|
|
435
|
-
await writer.add(bytes);
|
|
436
|
-
request.onPiece(bytes.byteLength);
|
|
437
|
-
});
|
|
438
|
-
}
|
|
439
|
-
const written = await writer.finish();
|
|
440
|
-
await foldLogIntoDatabase(request.driver, request.destPath, written.databasePages);
|
|
441
|
-
return written.frameCount;
|
|
442
|
-
} catch (err) {
|
|
443
|
-
await writer.abandon();
|
|
444
|
-
await rm(logPath, { force: true }).catch(() => {
|
|
445
|
-
});
|
|
446
|
-
throw err;
|
|
447
|
-
}
|
|
448
|
-
}
|
|
449
|
-
|
|
450
|
-
// src/core/backup/restore-options.ts
|
|
451
|
-
var DEFAULT_RESTORE_BATCH_SIZE = 16;
|
|
452
|
-
|
|
453
|
-
// src/core/backup/restore.ts
|
|
454
|
-
function assertBatchSize(batchSize) {
|
|
455
|
-
if (!Number.isInteger(batchSize) || batchSize < 1) {
|
|
456
|
-
throw new SirannonError(
|
|
457
|
-
`A restore replays at least one change piece per batch, and the batch size given was ${batchSize}`,
|
|
458
|
-
"BACKUP_ERROR"
|
|
459
|
-
);
|
|
460
|
-
}
|
|
461
|
-
}
|
|
462
|
-
var BUILD_SUFFIX = ".restoring";
|
|
463
|
-
async function removeLogBeside(path) {
|
|
464
|
-
await rm(`${path}-wal`, { force: true }).catch(() => {
|
|
465
|
-
});
|
|
466
|
-
await rm(`${path}-shm`, { force: true }).catch(() => {
|
|
467
|
-
});
|
|
468
|
-
}
|
|
469
|
-
async function removeDatabaseAndLog(path) {
|
|
470
|
-
await rm(path, { force: true }).catch(() => {
|
|
471
|
-
});
|
|
472
|
-
await removeLogBeside(path);
|
|
473
|
-
}
|
|
474
|
-
async function foldLogIntoDatabase2(driver, path) {
|
|
475
|
-
const existing = await stat(path).catch(() => void 0);
|
|
476
|
-
if (!existing) return true;
|
|
477
|
-
const log = await stat(`${path}-wal`).catch(() => void 0);
|
|
478
|
-
if (!log || log.size === 0) return true;
|
|
479
|
-
const conn = await driver.open(path, { walMode: false, walAutoCheckpoint: 0 }).catch(() => void 0);
|
|
480
|
-
if (!conn) return false;
|
|
481
|
-
const folded = await checkpointLog(conn).catch(() => void 0);
|
|
482
|
-
await conn.close().catch(() => void 0);
|
|
483
|
-
return folded?.emptied === true;
|
|
484
|
-
}
|
|
485
|
-
async function assertPathIsFree(destPath, replaceExisting) {
|
|
486
|
-
if (replaceExisting) return;
|
|
487
|
-
const existing = await stat(destPath).catch(() => void 0);
|
|
488
|
-
if (!existing) return;
|
|
489
|
-
throw new SirannonError(
|
|
490
|
-
`A restore would replace '${destPath}', and its write-ahead log with it. Pass replaceExisting to say that is what you want, or name a path this process holds nothing at.`,
|
|
491
|
-
"BACKUP_ERROR"
|
|
492
|
-
);
|
|
493
|
-
}
|
|
494
|
-
function batchesOf(changes, batchSize) {
|
|
495
|
-
const batches = [];
|
|
496
|
-
for (let start = 0; start < changes.length; start += batchSize) {
|
|
497
|
-
batches.push(changes.slice(start, start + batchSize));
|
|
498
|
-
}
|
|
499
|
-
return batches;
|
|
500
|
-
}
|
|
501
|
-
async function restoreBackup(options) {
|
|
502
|
-
const startedAt = Date.now();
|
|
503
|
-
const batchSize = options.batchSize ?? DEFAULT_RESTORE_BATCH_SIZE;
|
|
504
|
-
assertBatchSize(batchSize);
|
|
505
|
-
const destination = destinationWithDeadline(
|
|
506
|
-
options.destination,
|
|
507
|
-
options.destinationTimeoutMs ?? DEFAULT_DESTINATION_TIMEOUT_MS
|
|
508
|
-
);
|
|
509
|
-
const chains = await readBackupChains(destination, options.chainName ?? DEFAULT_CHAIN_NAME);
|
|
510
|
-
const plan = planBackupRestore(chains, options.moment ?? Date.now());
|
|
511
|
-
assertChangePiecesRunOn(plan.changes, plan.chainId);
|
|
512
|
-
const destPath = options.destPath;
|
|
513
|
-
await assertPathIsFree(destPath, options.replaceExisting === true);
|
|
514
|
-
const buildPath = `${destPath}${BUILD_SUFFIX}`;
|
|
515
|
-
await removeDatabaseAndLog(buildPath);
|
|
516
|
-
let piecesFetched = 0;
|
|
517
|
-
let bytesFetched = 0;
|
|
518
|
-
let changesApplied = 0;
|
|
519
|
-
let framesApplied = 0;
|
|
520
|
-
const batches = batchesOf(plan.changes, batchSize);
|
|
521
|
-
const reportProgress = (phase) => options.onProgress?.({
|
|
522
|
-
phase,
|
|
523
|
-
piecesFetched,
|
|
524
|
-
bytesFetched,
|
|
525
|
-
changesApplied,
|
|
526
|
-
changesTotal: plan.changes.length
|
|
527
|
-
});
|
|
528
|
-
try {
|
|
529
|
-
await assembleStoredFile(destination, plan.base, buildPath, (pieces, bytes) => {
|
|
530
|
-
piecesFetched = pieces;
|
|
531
|
-
bytesFetched = bytes;
|
|
532
|
-
reportProgress("full-copy");
|
|
533
|
-
});
|
|
534
|
-
const header = await readDatabaseHeader(buildPath);
|
|
535
|
-
if (!header.walMode && plan.changes.length > 0) {
|
|
536
|
-
throw new SirannonError(
|
|
537
|
-
`The full copy assembled into '${buildPath}' records no write-ahead log, so SQLite would ignore the change pieces rather than replay them`,
|
|
538
|
-
"BACKUP_CHAIN_BROKEN"
|
|
539
|
-
);
|
|
540
|
-
}
|
|
541
|
-
let logSequence = 0;
|
|
542
|
-
for (const batch of batches) {
|
|
543
|
-
logSequence++;
|
|
544
|
-
framesApplied += await applyChangeBatch({
|
|
545
|
-
destination,
|
|
546
|
-
driver: options.driver,
|
|
547
|
-
destPath: buildPath,
|
|
548
|
-
pageSize: header.pageSize,
|
|
549
|
-
logSequence,
|
|
550
|
-
batch,
|
|
551
|
-
onPiece: (byteLength) => {
|
|
552
|
-
piecesFetched++;
|
|
553
|
-
bytesFetched += byteLength;
|
|
554
|
-
reportProgress("changes");
|
|
555
|
-
}
|
|
556
|
-
});
|
|
557
|
-
changesApplied += batch.length;
|
|
558
|
-
reportProgress("changes");
|
|
559
|
-
}
|
|
560
|
-
await countDatabasePages(options.driver, buildPath);
|
|
561
|
-
const logFoldedIn = await foldLogIntoDatabase2(options.driver, destPath);
|
|
562
|
-
if (logFoldedIn) await removeLogBeside(destPath);
|
|
563
|
-
else await removeDatabaseAndLog(destPath);
|
|
564
|
-
await rename(buildPath, destPath);
|
|
565
|
-
} catch (err) {
|
|
566
|
-
await removeDatabaseAndLog(buildPath);
|
|
567
|
-
throw err;
|
|
568
|
-
}
|
|
569
|
-
const finishedAt = Date.now();
|
|
570
|
-
return {
|
|
571
|
-
chainId: plan.chainId,
|
|
572
|
-
destPath,
|
|
573
|
-
baseName: plan.base.name,
|
|
574
|
-
restoresTo: plan.restoresTo,
|
|
575
|
-
pieceCount: piecesFetched,
|
|
576
|
-
bytesFetched,
|
|
577
|
-
changesApplied,
|
|
578
|
-
framesApplied,
|
|
579
|
-
batchCount: batches.length,
|
|
580
|
-
startedAt,
|
|
581
|
-
finishedAt,
|
|
582
|
-
durationMs: finishedAt - startedAt
|
|
583
|
-
};
|
|
584
|
-
}
|
|
585
|
-
|
|
586
|
-
export { assembleFromDestination, restoreBackup };
|
|
7
|
+
import '../chunk-V5FUJEYY.mjs';
|
|
8
|
+
import '../chunk-PBRXXISQ.mjs';
|