@platformatic/sqlite-js-stream 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +40 -0
- package/LICENSE +202 -0
- package/NOTICE +22 -0
- package/README.md +482 -2
- package/dist/backends/file/index.d.ts +4 -0
- package/dist/backends/file/index.d.ts.map +1 -0
- package/dist/backends/file/index.js +3 -0
- package/dist/backends/file/leaser.d.ts +20 -0
- package/dist/backends/file/leaser.d.ts.map +1 -0
- package/dist/backends/file/leaser.js +123 -0
- package/dist/backends/file/replica-client.d.ts +38 -0
- package/dist/backends/file/replica-client.d.ts.map +1 -0
- package/dist/backends/file/replica-client.js +209 -0
- package/dist/db.d.ts +80 -0
- package/dist/db.d.ts.map +1 -0
- package/dist/db.js +484 -0
- package/dist/errors.d.ts +42 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +66 -0
- package/dist/index.d.ts +19 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +22 -0
- package/dist/leaser.d.ts +55 -0
- package/dist/leaser.d.ts.map +1 -0
- package/dist/leaser.js +60 -0
- package/dist/ltx/byte-reader.d.ts +18 -0
- package/dist/ltx/byte-reader.d.ts.map +1 -0
- package/dist/ltx/byte-reader.js +38 -0
- package/dist/ltx/bytes.d.ts +20 -0
- package/dist/ltx/bytes.d.ts.map +1 -0
- package/dist/ltx/bytes.js +69 -0
- package/dist/ltx/checksum.d.ts +20 -0
- package/dist/ltx/checksum.d.ts.map +1 -0
- package/dist/ltx/checksum.js +42 -0
- package/dist/ltx/compaction-level.d.ts +58 -0
- package/dist/ltx/compaction-level.d.ts.map +1 -0
- package/dist/ltx/compaction-level.js +115 -0
- package/dist/ltx/compactor.d.ts +33 -0
- package/dist/ltx/compactor.d.ts.map +1 -0
- package/dist/ltx/compactor.js +137 -0
- package/dist/ltx/constants.d.ts +49 -0
- package/dist/ltx/constants.d.ts.map +1 -0
- package/dist/ltx/constants.js +58 -0
- package/dist/ltx/crc64.d.ts +29 -0
- package/dist/ltx/crc64.d.ts.map +1 -0
- package/dist/ltx/crc64.js +57 -0
- package/dist/ltx/decoder.d.ts +55 -0
- package/dist/ltx/decoder.d.ts.map +1 -0
- package/dist/ltx/decoder.js +236 -0
- package/dist/ltx/encoder.d.ts +49 -0
- package/dist/ltx/encoder.d.ts.map +1 -0
- package/dist/ltx/encoder.js +159 -0
- package/dist/ltx/file-info.d.ts +24 -0
- package/dist/ltx/file-info.d.ts.map +1 -0
- package/dist/ltx/file-info.js +74 -0
- package/dist/ltx/filename.d.ts +13 -0
- package/dist/ltx/filename.d.ts.map +1 -0
- package/dist/ltx/filename.js +23 -0
- package/dist/ltx/header.d.ts +17 -0
- package/dist/ltx/header.d.ts.map +1 -0
- package/dist/ltx/header.js +115 -0
- package/dist/ltx/index.d.ts +24 -0
- package/dist/ltx/index.d.ts.map +1 -0
- package/dist/ltx/index.js +25 -0
- package/dist/ltx/lz4.d.ts +41 -0
- package/dist/ltx/lz4.d.ts.map +1 -0
- package/dist/ltx/lz4.js +264 -0
- package/dist/ltx/page-index.d.ts +26 -0
- package/dist/ltx/page-index.d.ts.map +1 -0
- package/dist/ltx/page-index.js +118 -0
- package/dist/ltx/page.d.ts +27 -0
- package/dist/ltx/page.d.ts.map +1 -0
- package/dist/ltx/page.js +66 -0
- package/dist/ltx/sink.d.ts +17 -0
- package/dist/ltx/sink.d.ts.map +1 -0
- package/dist/ltx/sink.js +21 -0
- package/dist/ltx/stream.d.ts +64 -0
- package/dist/ltx/stream.d.ts.map +1 -0
- package/dist/ltx/stream.js +248 -0
- package/dist/ltx/streaming-compactor.d.ts +34 -0
- package/dist/ltx/streaming-compactor.d.ts.map +1 -0
- package/dist/ltx/streaming-compactor.js +140 -0
- package/dist/ltx/trailer.d.ts +12 -0
- package/dist/ltx/trailer.d.ts.map +1 -0
- package/dist/ltx/trailer.js +47 -0
- package/dist/ltx/types.d.ts +79 -0
- package/dist/ltx/types.d.ts.map +1 -0
- package/dist/ltx/types.js +70 -0
- package/dist/replica-client.d.ts +92 -0
- package/dist/replica-client.d.ts.map +1 -0
- package/dist/replica-client.js +122 -0
- package/dist/replica-url.d.ts +20 -0
- package/dist/replica-url.d.ts.map +1 -0
- package/dist/replica-url.js +49 -0
- package/dist/replica.d.ts +379 -0
- package/dist/replica.d.ts.map +1 -0
- package/dist/replica.js +1253 -0
- package/dist/store.d.ts +34 -0
- package/dist/store.d.ts.map +1 -0
- package/dist/store.js +60 -0
- package/dist/util/fs.d.ts +28 -0
- package/dist/util/fs.d.ts.map +1 -0
- package/dist/util/fs.js +73 -0
- package/dist/util/logger.d.ts +10 -0
- package/dist/util/logger.d.ts.map +1 -0
- package/dist/util/logger.js +7 -0
- package/dist/util/stream.d.ts +11 -0
- package/dist/util/stream.d.ts.map +1 -0
- package/dist/util/stream.js +25 -0
- package/dist/wal/checksum.d.ts +17 -0
- package/dist/wal/checksum.d.ts.map +1 -0
- package/dist/wal/checksum.js +30 -0
- package/dist/wal/constants.d.ts +8 -0
- package/dist/wal/constants.d.ts.map +1 -0
- package/dist/wal/constants.js +7 -0
- package/dist/wal/index.d.ts +5 -0
- package/dist/wal/index.d.ts.map +1 -0
- package/dist/wal/index.js +4 -0
- package/dist/wal/reader.d.ts +61 -0
- package/dist/wal/reader.d.ts.map +1 -0
- package/dist/wal/reader.js +207 -0
- package/package.json +68 -3
- package/src/backends/file/index.ts +3 -0
- package/src/backends/file/leaser.ts +145 -0
- package/src/backends/file/replica-client.ts +246 -0
- package/src/db.ts +553 -0
- package/src/errors.ts +74 -0
- package/src/index.ts +26 -0
- package/src/leaser.ts +86 -0
- package/src/ltx/byte-reader.ts +46 -0
- package/src/ltx/bytes.ts +87 -0
- package/src/ltx/checksum.ts +46 -0
- package/src/ltx/compaction-level.ts +135 -0
- package/src/ltx/compactor.ts +167 -0
- package/src/ltx/constants.ts +71 -0
- package/src/ltx/crc64.ts +65 -0
- package/src/ltx/decoder.ts +282 -0
- package/src/ltx/encoder.ts +200 -0
- package/src/ltx/file-info.ts +81 -0
- package/src/ltx/filename.ts +27 -0
- package/src/ltx/header.ts +126 -0
- package/src/ltx/index.ts +27 -0
- package/src/ltx/lz4.ts +279 -0
- package/src/ltx/page-index.ts +140 -0
- package/src/ltx/page.ts +80 -0
- package/src/ltx/sink.ts +33 -0
- package/src/ltx/stream.ts +286 -0
- package/src/ltx/streaming-compactor.ts +169 -0
- package/src/ltx/trailer.ts +58 -0
- package/src/ltx/types.ts +152 -0
- package/src/lz4js.d.ts +27 -0
- package/src/replica-client.ts +222 -0
- package/src/replica-url.ts +72 -0
- package/src/replica.ts +1480 -0
- package/src/store.ts +71 -0
- package/src/util/fs.ts +82 -0
- package/src/util/logger.ts +15 -0
- package/src/util/stream.ts +28 -0
- package/src/wal/checksum.ts +39 -0
- package/src/wal/constants.ts +10 -0
- package/src/wal/index.ts +5 -0
- package/src/wal/reader.ts +264 -0
package/dist/db.js
ADDED
|
@@ -0,0 +1,484 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LocalDatabase is the internal engine that reads a SQLite database and its
|
|
3
|
+
* `-wal` file and writes LTX files into a local cache (`<path>-litestream/ltx/`).
|
|
4
|
+
*
|
|
5
|
+
* It is deliberately NOT part of the public API: `Replica`/`Store` take a
|
|
6
|
+
* database `path` and read directly, so the library is agnostic to whatever
|
|
7
|
+
* SQLite driver the application uses. To keep snapshots consistent, it holds a
|
|
8
|
+
* short-lived **read-only** `node:sqlite` read transaction that pins the WAL
|
|
9
|
+
* (SQLite will not checkpoint/truncate the WAL past an active reader), while
|
|
10
|
+
* reading the database + WAL files directly.
|
|
11
|
+
*/
|
|
12
|
+
import { DatabaseSync } from 'node:sqlite';
|
|
13
|
+
import { open, readFile, readdir, rename, rm, stat } from 'node:fs/promises';
|
|
14
|
+
import nodePath from 'node:path';
|
|
15
|
+
import { Decoder, Encoder, HeaderFlagNoChecksum, HeaderSize, formatFilename, lockPgno, parseFilename, } from "./ltx/index.js";
|
|
16
|
+
import { FileSink } from "./util/fs.js";
|
|
17
|
+
import { PrevFrameMismatchError, WALFrameHeaderSize, WALHeaderSize, WALReader, WALEOFError, } from "./wal/index.js";
|
|
18
|
+
import { SnapshotLevel } from "./replica-client.js";
|
|
19
|
+
import { mkdirAll } from "./util/fs.js";
|
|
20
|
+
import { DatabaseError } from "./errors.js";
|
|
21
|
+
/** Reads the page size from the SQLite database header (bytes 16-17). */
|
|
22
|
+
function headerPageSize(buf) {
|
|
23
|
+
const raw = buf.length >= 18 ? (buf[16] << 8) | buf[17] : 0;
|
|
24
|
+
if (raw === 1)
|
|
25
|
+
return 65536;
|
|
26
|
+
return raw || 4096;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* LocalDatabase reads a database + WAL from disk, tracks the local replicated
|
|
30
|
+
* position, and builds LTX files in the local cache.
|
|
31
|
+
*/
|
|
32
|
+
export class LocalDatabase {
|
|
33
|
+
path;
|
|
34
|
+
metaPath;
|
|
35
|
+
pageSize;
|
|
36
|
+
conn = null;
|
|
37
|
+
constructor(path, metaPath, pageSize) {
|
|
38
|
+
this.path = path;
|
|
39
|
+
this.metaPath = metaPath;
|
|
40
|
+
this.pageSize = pageSize;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Open opens a local database reader for `path`. It does not create the
|
|
44
|
+
* database — the application owns it. `pageSize` is read from the database
|
|
45
|
+
* header; a missing file falls back to the default page size.
|
|
46
|
+
*/
|
|
47
|
+
static async open(path) {
|
|
48
|
+
const metaPath = `${path}-litestream`;
|
|
49
|
+
await mkdirAll(nodePath.join(metaPath, 'ltx', '0'));
|
|
50
|
+
let pageSize = 4096;
|
|
51
|
+
try {
|
|
52
|
+
const fh = await open(path, 'r');
|
|
53
|
+
const buf = new Uint8Array(18);
|
|
54
|
+
await fh.read(buf, 0, 18, 0);
|
|
55
|
+
await fh.close();
|
|
56
|
+
pageSize = headerPageSize(buf);
|
|
57
|
+
}
|
|
58
|
+
catch {
|
|
59
|
+
/* database not present yet; use default page size */
|
|
60
|
+
}
|
|
61
|
+
return new LocalDatabase(path, metaPath, pageSize);
|
|
62
|
+
}
|
|
63
|
+
/** WalPath returns the path to the database's WAL file. */
|
|
64
|
+
walPath() {
|
|
65
|
+
return `${this.path}-wal`;
|
|
66
|
+
}
|
|
67
|
+
/** LTXDir returns the local LTX root directory. */
|
|
68
|
+
ltxDir() {
|
|
69
|
+
return nodePath.join(this.metaPath, 'ltx');
|
|
70
|
+
}
|
|
71
|
+
/** LTXLevelDir returns the local LTX directory for a level. */
|
|
72
|
+
ltxLevelDir(level) {
|
|
73
|
+
return nodePath.join(this.ltxDir(), String(level));
|
|
74
|
+
}
|
|
75
|
+
/** LTXPath returns the local path of a single LTX file. */
|
|
76
|
+
ltxPath(level, minTXID, maxTXID) {
|
|
77
|
+
return nodePath.join(this.ltxLevelDir(level), formatFilename(minTXID, maxTXID));
|
|
78
|
+
}
|
|
79
|
+
/** Lists local level-0 LTX files, sorted by TXID. */
|
|
80
|
+
async l0Files() {
|
|
81
|
+
return this.filesForLevel(0);
|
|
82
|
+
}
|
|
83
|
+
/** Lists local LTX files for any level, sorted by TXID. */
|
|
84
|
+
async filesForLevel(level) {
|
|
85
|
+
const dir = this.ltxLevelDir(level);
|
|
86
|
+
const out = [];
|
|
87
|
+
let entries;
|
|
88
|
+
try {
|
|
89
|
+
entries = await readdir(dir, { withFileTypes: true });
|
|
90
|
+
}
|
|
91
|
+
catch (err) {
|
|
92
|
+
if (err.code === 'ENOENT')
|
|
93
|
+
return out;
|
|
94
|
+
throw err;
|
|
95
|
+
}
|
|
96
|
+
for (const entry of entries) {
|
|
97
|
+
if (!entry.isFile())
|
|
98
|
+
continue;
|
|
99
|
+
let minTXID;
|
|
100
|
+
let maxTXID;
|
|
101
|
+
try {
|
|
102
|
+
({ minTXID, maxTXID } = parseFilename(entry.name));
|
|
103
|
+
}
|
|
104
|
+
catch {
|
|
105
|
+
continue;
|
|
106
|
+
}
|
|
107
|
+
const st = await stat(nodePath.join(dir, entry.name));
|
|
108
|
+
out.push({ level, minTXID, maxTXID, size: st.size, createdAt: st.mtimeMs });
|
|
109
|
+
}
|
|
110
|
+
out.sort((a, b) => (a.minTXID < b.minTXID ? -1 : a.minTXID > b.minTXID ? 1 : 0));
|
|
111
|
+
return out;
|
|
112
|
+
}
|
|
113
|
+
/** Pos returns the maximum TXID in the local level-0 cache (0 if none). */
|
|
114
|
+
async pos() {
|
|
115
|
+
const m = await this.maxFile();
|
|
116
|
+
return m === null ? 0n : m.maxTXID;
|
|
117
|
+
}
|
|
118
|
+
/** Returns the local LTX file with the highest maxTXID across all levels. */
|
|
119
|
+
async maxFile() {
|
|
120
|
+
let best = null;
|
|
121
|
+
for (let level = 0; level <= SnapshotLevel; level++) {
|
|
122
|
+
for (const f of await this.filesForLevel(level)) {
|
|
123
|
+
if (best === null || f.maxTXID > best.maxTXID)
|
|
124
|
+
best = f;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
return best;
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* Opens (and caches) a read-only connection used to hold a snapshot during
|
|
131
|
+
* sync/snapshot. Not part of the public API; the DB file must exist.
|
|
132
|
+
*/
|
|
133
|
+
ensureConn() {
|
|
134
|
+
if (this.conn === null) {
|
|
135
|
+
this.conn = new DatabaseSync(this.path, { readOnly: true });
|
|
136
|
+
this.conn.exec('PRAGMA busy_timeout=5000');
|
|
137
|
+
}
|
|
138
|
+
return this.conn;
|
|
139
|
+
}
|
|
140
|
+
/**
|
|
141
|
+
* Pins a WAL snapshot by starting a read transaction and issuing a read that
|
|
142
|
+
* touches the database (a plain `SELECT 1` does not acquire the WAL read lock).
|
|
143
|
+
*/
|
|
144
|
+
beginRead() {
|
|
145
|
+
const conn = this.ensureConn();
|
|
146
|
+
conn.exec('BEGIN');
|
|
147
|
+
conn.prepare('SELECT count(*) FROM sqlite_master').get();
|
|
148
|
+
}
|
|
149
|
+
/** Releases the pinned read snapshot. */
|
|
150
|
+
commitRead() {
|
|
151
|
+
if (this.conn !== null) {
|
|
152
|
+
this.conn.exec('COMMIT');
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
/** Current committed database size, in pages (as seen by the pinned reader). */
|
|
156
|
+
pageCount() {
|
|
157
|
+
const r = this.ensureConn().prepare('PRAGMA page_count').get();
|
|
158
|
+
return r.page_count;
|
|
159
|
+
}
|
|
160
|
+
/** Sync builds a level-0 LTX file and writes it to the local cache. */
|
|
161
|
+
async sync() {
|
|
162
|
+
const pos = await this.pos();
|
|
163
|
+
const txID = pos + 1n;
|
|
164
|
+
this.beginRead();
|
|
165
|
+
let dbHandle = null;
|
|
166
|
+
try {
|
|
167
|
+
const commit = this.pageCount();
|
|
168
|
+
if (commit === 0)
|
|
169
|
+
return null;
|
|
170
|
+
const pageSize = this.pageSize;
|
|
171
|
+
const lockPg = lockPgno(pageSize);
|
|
172
|
+
dbHandle = await open(this.path, 'r');
|
|
173
|
+
const walBuf = await this.readWalOrEmpty();
|
|
174
|
+
// Determine the previous sync state to resume incremental reading.
|
|
175
|
+
const prev = await this.prevSyncInfo();
|
|
176
|
+
let snapshotting = prev === null;
|
|
177
|
+
let reader = null;
|
|
178
|
+
let offset = WALHeaderSize;
|
|
179
|
+
let salt1 = 0;
|
|
180
|
+
let salt2 = 0;
|
|
181
|
+
let pageMap = new Map();
|
|
182
|
+
let maxOffset = 0;
|
|
183
|
+
let walCommit = 0;
|
|
184
|
+
let prevCommit = 0;
|
|
185
|
+
if (!snapshotting && prev !== null) {
|
|
186
|
+
reader = makeResumeReader(walBuf, prev.walOffset + prev.walSize, prev.salt1, prev.salt2);
|
|
187
|
+
if (reader === null) {
|
|
188
|
+
snapshotting = true; // WAL was reset; fall back to a snapshot base
|
|
189
|
+
}
|
|
190
|
+
else {
|
|
191
|
+
const pm = reader.pageMap();
|
|
192
|
+
pageMap = pm.pageIndex;
|
|
193
|
+
maxOffset = pm.maxOffset;
|
|
194
|
+
walCommit = pm.commit;
|
|
195
|
+
offset = prev.walOffset + prev.walSize;
|
|
196
|
+
salt1 = prev.salt1;
|
|
197
|
+
salt2 = prev.salt2;
|
|
198
|
+
prevCommit = prev.commit;
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
const walSize = snapshotting ? 0 : maxOffset > 0 ? maxOffset - offset : 0;
|
|
202
|
+
if (!snapshotting && walSize === 0) {
|
|
203
|
+
return null; // no new WAL pages since the last sync
|
|
204
|
+
}
|
|
205
|
+
if (snapshotting) {
|
|
206
|
+
// Read the WAL from the start to build a full page map for the snapshot.
|
|
207
|
+
try {
|
|
208
|
+
const full = new WALReader(walBuf);
|
|
209
|
+
const pm = full.pageMap();
|
|
210
|
+
pageMap = pm.pageIndex;
|
|
211
|
+
walCommit = pm.commit;
|
|
212
|
+
salt1 = full.salt1();
|
|
213
|
+
salt2 = full.salt2();
|
|
214
|
+
}
|
|
215
|
+
catch (err) {
|
|
216
|
+
if (!(err instanceof WALEOFError))
|
|
217
|
+
throw err;
|
|
218
|
+
}
|
|
219
|
+
if (walCommit > 0) {
|
|
220
|
+
offset = WALHeaderSize;
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
const finalCommit = Math.max(commit, walCommit);
|
|
224
|
+
// Stream the LTX output directly to a temp file (no whole-file buffer).
|
|
225
|
+
const dir = this.ltxLevelDir(0);
|
|
226
|
+
await mkdirAll(dir);
|
|
227
|
+
const filename = this.ltxPath(0, txID, txID);
|
|
228
|
+
const tmp = `${filename}.tmp`;
|
|
229
|
+
const fh = await open(tmp, 'w');
|
|
230
|
+
const sink = new FileSink(fh);
|
|
231
|
+
const enc = new Encoder(sink);
|
|
232
|
+
try {
|
|
233
|
+
enc.encodeHeader({
|
|
234
|
+
version: 3,
|
|
235
|
+
flags: HeaderFlagNoChecksum,
|
|
236
|
+
pageSize,
|
|
237
|
+
commit: finalCommit,
|
|
238
|
+
minTXID: txID,
|
|
239
|
+
maxTXID: txID,
|
|
240
|
+
timestamp: BigInt(Date.now()),
|
|
241
|
+
preApplyChecksum: 0n,
|
|
242
|
+
walOffset: BigInt(offset),
|
|
243
|
+
walSize: BigInt(snapshotting ? Math.max(0, walBuf.length - WALHeaderSize) : walSize),
|
|
244
|
+
walSalt1: salt1,
|
|
245
|
+
walSalt2: salt2,
|
|
246
|
+
nodeID: 0n,
|
|
247
|
+
});
|
|
248
|
+
if (snapshotting) {
|
|
249
|
+
await this.writePagesFromDB(enc, pageSize, finalCommit, lockPg, pageMap, walBuf, dbHandle);
|
|
250
|
+
}
|
|
251
|
+
else {
|
|
252
|
+
await this.writePagesFromWAL(enc, pageSize, finalCommit, prevCommit, lockPg, pageMap, walBuf, dbHandle);
|
|
253
|
+
}
|
|
254
|
+
enc.close();
|
|
255
|
+
await sink.flush();
|
|
256
|
+
await fh.close();
|
|
257
|
+
await rename(tmp, filename);
|
|
258
|
+
const size = (await stat(filename)).size;
|
|
259
|
+
return { level: 0, minTXID: txID, maxTXID: txID, size, createdAt: Date.now() };
|
|
260
|
+
}
|
|
261
|
+
catch (err) {
|
|
262
|
+
await fh.close().catch(() => { });
|
|
263
|
+
await rm(tmp, { force: true }).catch(() => { });
|
|
264
|
+
throw err;
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
finally {
|
|
268
|
+
if (dbHandle !== null)
|
|
269
|
+
await dbHandle.close();
|
|
270
|
+
this.commitRead();
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
/** Writes a full snapshot: every committed page, preferring WAL pages. */
|
|
274
|
+
async writePagesFromDB(enc, pageSize, commit, lockPg, pageMap, walBuf, dbHandle) {
|
|
275
|
+
for (let pgno = 1; pgno <= commit; pgno++) {
|
|
276
|
+
if (pgno === lockPg)
|
|
277
|
+
continue;
|
|
278
|
+
enc.encodePage({ pgno, flags: 0 }, await this.readPage(pgno, pageSize, pageMap, walBuf, dbHandle));
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
/** Writes an incremental file: changed WAL pages + DB growth pages. */
|
|
282
|
+
async writePagesFromWAL(enc, pageSize, commit, prevCommit, lockPg, pageMap, walBuf, dbHandle) {
|
|
283
|
+
// Sorted page numbers present in the WAL change-set.
|
|
284
|
+
const pgnos = [...pageMap.keys()].sort((a, b) => a - b);
|
|
285
|
+
const lockPgno = lockPg;
|
|
286
|
+
// Add DB growth pages (new pages beyond prevCommit not already in pageMap).
|
|
287
|
+
if (commit > prevCommit) {
|
|
288
|
+
for (let pgno = prevCommit + 1; pgno <= commit; pgno++) {
|
|
289
|
+
if (pgno === lockPgno)
|
|
290
|
+
continue;
|
|
291
|
+
if (pageMap.has(pgno))
|
|
292
|
+
continue;
|
|
293
|
+
pgnos.push(pgno);
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
for (const pgno of pgnos) {
|
|
297
|
+
if (pgno === lockPgno)
|
|
298
|
+
continue;
|
|
299
|
+
// Growth pages (not in pageMap) come directly from the database file.
|
|
300
|
+
if (!pageMap.has(pgno)) {
|
|
301
|
+
const page = new Uint8Array(pageSize);
|
|
302
|
+
await readExact(dbHandle, page, (pgno - 1) * pageSize);
|
|
303
|
+
enc.encodePage({ pgno, flags: 0 }, page);
|
|
304
|
+
continue;
|
|
305
|
+
}
|
|
306
|
+
enc.encodePage({ pgno, flags: 0 }, await this.readPage(pgno, pageSize, pageMap, walBuf, dbHandle));
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
/**
|
|
310
|
+
* Snapshot writes a full-database LTX file at `SnapshotLevel` (9) with
|
|
311
|
+
* MinTXID=1 and MaxTXID equal to the current local position. Returns null if
|
|
312
|
+
* a snapshot would cover no data (empty database).
|
|
313
|
+
*/
|
|
314
|
+
async snapshot() {
|
|
315
|
+
const pos = await this.pos();
|
|
316
|
+
const maxTXID = pos === 0n ? 1n : pos;
|
|
317
|
+
this.beginRead();
|
|
318
|
+
let dbHandle = null;
|
|
319
|
+
try {
|
|
320
|
+
const commit = this.pageCount();
|
|
321
|
+
if (commit === 0)
|
|
322
|
+
return null;
|
|
323
|
+
const pageSize = this.pageSize;
|
|
324
|
+
const lockPg = lockPgno(pageSize);
|
|
325
|
+
dbHandle = await open(this.path, 'r');
|
|
326
|
+
const walBuf = await this.readWalOrEmpty();
|
|
327
|
+
// Build the committed WAL page map for the overlay (may be empty).
|
|
328
|
+
let pageMap = new Map();
|
|
329
|
+
let walCommit = 0;
|
|
330
|
+
let salt1 = 0;
|
|
331
|
+
let salt2 = 0;
|
|
332
|
+
if (walBuf.length > 0) {
|
|
333
|
+
try {
|
|
334
|
+
const full = new WALReader(walBuf);
|
|
335
|
+
const pm = full.pageMap();
|
|
336
|
+
pageMap = pm.pageIndex;
|
|
337
|
+
walCommit = pm.commit;
|
|
338
|
+
salt1 = full.salt1();
|
|
339
|
+
salt2 = full.salt2();
|
|
340
|
+
}
|
|
341
|
+
catch (err) {
|
|
342
|
+
if (!(err instanceof WALEOFError))
|
|
343
|
+
throw err;
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
// The commit may only be present in the WAL (not yet checkpointed into
|
|
347
|
+
// the main database header), so take the larger of the two.
|
|
348
|
+
const finalCommit = Math.max(commit, walCommit);
|
|
349
|
+
if (finalCommit === 0)
|
|
350
|
+
return null;
|
|
351
|
+
// Stream the LTX output directly to a temp file (no whole-file buffer).
|
|
352
|
+
const dir = this.ltxLevelDir(9);
|
|
353
|
+
await mkdirAll(dir);
|
|
354
|
+
const filename = this.ltxPath(9, 1n, maxTXID);
|
|
355
|
+
const tmp = `${filename}.tmp`;
|
|
356
|
+
const fh = await open(tmp, 'w');
|
|
357
|
+
const sink = new FileSink(fh);
|
|
358
|
+
const enc = new Encoder(sink);
|
|
359
|
+
try {
|
|
360
|
+
enc.encodeHeader({
|
|
361
|
+
version: 3,
|
|
362
|
+
flags: HeaderFlagNoChecksum,
|
|
363
|
+
pageSize,
|
|
364
|
+
commit: finalCommit,
|
|
365
|
+
minTXID: 1n,
|
|
366
|
+
maxTXID,
|
|
367
|
+
timestamp: BigInt(Date.now()),
|
|
368
|
+
preApplyChecksum: 0n,
|
|
369
|
+
walOffset: BigInt(WALHeaderSize),
|
|
370
|
+
walSize: BigInt(Math.max(0, walBuf.length - WALHeaderSize)),
|
|
371
|
+
walSalt1: salt1,
|
|
372
|
+
walSalt2: salt2,
|
|
373
|
+
nodeID: 0n,
|
|
374
|
+
});
|
|
375
|
+
await this.writePagesFromDB(enc, pageSize, finalCommit, lockPg, pageMap, walBuf, dbHandle);
|
|
376
|
+
enc.close();
|
|
377
|
+
await sink.flush();
|
|
378
|
+
await fh.close();
|
|
379
|
+
}
|
|
380
|
+
catch (err) {
|
|
381
|
+
await fh.close().catch(() => { });
|
|
382
|
+
await rm(tmp, { force: true }).catch(() => { });
|
|
383
|
+
throw err;
|
|
384
|
+
}
|
|
385
|
+
await rename(tmp, filename);
|
|
386
|
+
const size = (await stat(filename)).size;
|
|
387
|
+
return { level: 9, minTXID: 1n, maxTXID, size, createdAt: Date.now() };
|
|
388
|
+
}
|
|
389
|
+
finally {
|
|
390
|
+
if (dbHandle !== null)
|
|
391
|
+
await dbHandle.close();
|
|
392
|
+
this.commitRead();
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
/** Reads a single page, preferring the WAL over the database file. */
|
|
396
|
+
async readPage(pgno, pageSize, pageMap, walBuf, dbHandle) {
|
|
397
|
+
const off = pageMap.get(pgno);
|
|
398
|
+
if (off !== undefined) {
|
|
399
|
+
const start = off + WALFrameHeaderSize;
|
|
400
|
+
return new Uint8Array(walBuf.subarray(start, start + pageSize));
|
|
401
|
+
}
|
|
402
|
+
const page = new Uint8Array(pageSize);
|
|
403
|
+
await readExact(dbHandle, page, (pgno - 1) * pageSize);
|
|
404
|
+
return page;
|
|
405
|
+
}
|
|
406
|
+
/** Reads the WAL file bytes, returning empty if it does not exist. */
|
|
407
|
+
async readWalOrEmpty() {
|
|
408
|
+
try {
|
|
409
|
+
return new Uint8Array(await readFile(this.walPath()));
|
|
410
|
+
}
|
|
411
|
+
catch {
|
|
412
|
+
return new Uint8Array(0);
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
/** Decodes the header of the max-TXID local L0 file (or null if none). */
|
|
416
|
+
async prevSyncInfo() {
|
|
417
|
+
const max = await this.maxFile();
|
|
418
|
+
if (max === null)
|
|
419
|
+
return null;
|
|
420
|
+
try {
|
|
421
|
+
// Only the 100-byte header is needed; do not read the whole file.
|
|
422
|
+
const fh = await open(this.ltxPath(max.level, max.minTXID, max.maxTXID), 'r');
|
|
423
|
+
try {
|
|
424
|
+
const buf = new Uint8Array(HeaderSize);
|
|
425
|
+
await readExact(fh, buf, 0);
|
|
426
|
+
const hdr = new Decoder(buf);
|
|
427
|
+
hdr.decodeHeader();
|
|
428
|
+
const h = hdr.Header();
|
|
429
|
+
return {
|
|
430
|
+
walOffset: Number(h.walOffset),
|
|
431
|
+
walSize: Number(h.walSize),
|
|
432
|
+
salt1: h.walSalt1,
|
|
433
|
+
salt2: h.walSalt2,
|
|
434
|
+
commit: h.commit,
|
|
435
|
+
};
|
|
436
|
+
}
|
|
437
|
+
finally {
|
|
438
|
+
await fh.close();
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
catch {
|
|
442
|
+
return null;
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
/** Close releases the internal read-only connection. */
|
|
446
|
+
close() {
|
|
447
|
+
if (this.conn !== null) {
|
|
448
|
+
this.conn.close();
|
|
449
|
+
this.conn = null;
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
/** Creates a WALReader positioned at `offset`, or null if it cannot resume. */
|
|
454
|
+
function makeResumeReader(walBuf, offset, salt1, salt2) {
|
|
455
|
+
if (walBuf.length === 0)
|
|
456
|
+
return null;
|
|
457
|
+
if (offset <= WALHeaderSize) {
|
|
458
|
+
// Starting fresh (or before any frames): read from the header.
|
|
459
|
+
try {
|
|
460
|
+
return new WALReader(walBuf);
|
|
461
|
+
}
|
|
462
|
+
catch {
|
|
463
|
+
return null;
|
|
464
|
+
}
|
|
465
|
+
}
|
|
466
|
+
try {
|
|
467
|
+
return WALReader.withOffset(walBuf, offset, salt1, salt2);
|
|
468
|
+
}
|
|
469
|
+
catch (err) {
|
|
470
|
+
if (err instanceof PrevFrameMismatchError)
|
|
471
|
+
return null;
|
|
472
|
+
throw err;
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
/** Reads exactly `buf.length` bytes into `buf` from `handle` at `position`. */
|
|
476
|
+
async function readExact(handle, buf, position) {
|
|
477
|
+
let offset = 0;
|
|
478
|
+
while (offset < buf.length) {
|
|
479
|
+
const { bytesRead } = await handle.read(buf, offset, buf.length - offset, position + offset);
|
|
480
|
+
if (bytesRead === 0)
|
|
481
|
+
throw new DatabaseError('unexpected end of file while reading a page');
|
|
482
|
+
offset += bytesRead;
|
|
483
|
+
}
|
|
484
|
+
}
|
package/dist/errors.d.ts
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared error base for sqlite-js-stream.
|
|
3
|
+
*
|
|
4
|
+
* Every typed library error extends {@link SqliteJsStreamError} and carries a
|
|
5
|
+
* stable, Node-style `code` string (e.g. `ERR_NO_MATCHING_BACKUP`). This lets
|
|
6
|
+
* callers catch any library error with a single `instanceof`, and branch on the
|
|
7
|
+
* precise failure via `err.code` without matching message text.
|
|
8
|
+
*/
|
|
9
|
+
export declare class SqliteJsStreamError extends Error {
|
|
10
|
+
/** Stable, machine-readable error code (Node.js convention). */
|
|
11
|
+
readonly code: string;
|
|
12
|
+
constructor(code: string, message: string);
|
|
13
|
+
}
|
|
14
|
+
/** A required function argument is missing or invalid (`ERR_INVALID_ARGUMENT`). */
|
|
15
|
+
export declare class InvalidArgumentError extends SqliteJsStreamError {
|
|
16
|
+
constructor(message: string);
|
|
17
|
+
}
|
|
18
|
+
/** An incremental LTX file's page size differs from the base database. */
|
|
19
|
+
export declare class PageSizeMismatchError extends SqliteJsStreamError {
|
|
20
|
+
constructor(message: string);
|
|
21
|
+
}
|
|
22
|
+
/** An LTX file is malformed/corrupt or violates the format (`ERR_LTX_FORMAT`). */
|
|
23
|
+
export declare class LTXFormatError extends SqliteJsStreamError {
|
|
24
|
+
constructor(message: string);
|
|
25
|
+
}
|
|
26
|
+
/** A SQLite WAL file is malformed/corrupt (`ERR_WAL_FORMAT`). */
|
|
27
|
+
export declare class WALFormatError extends SqliteJsStreamError {
|
|
28
|
+
constructor(message: string);
|
|
29
|
+
}
|
|
30
|
+
/** A replica URL is malformed or uses an unsupported scheme (`ERR_REPLICA_URL`). */
|
|
31
|
+
export declare class ReplicaURLError extends SqliteJsStreamError {
|
|
32
|
+
constructor(message: string);
|
|
33
|
+
}
|
|
34
|
+
/** A file/backend replica operation failed (`ERR_FILE_REPLICA`). */
|
|
35
|
+
export declare class FileReplicaError extends SqliteJsStreamError {
|
|
36
|
+
constructor(message: string);
|
|
37
|
+
}
|
|
38
|
+
/** A SQLite database file is missing/invalid or could not be read (`ERR_DATABASE`). */
|
|
39
|
+
export declare class DatabaseError extends SqliteJsStreamError {
|
|
40
|
+
constructor(message: string);
|
|
41
|
+
}
|
|
42
|
+
//# sourceMappingURL=errors.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,qBAAa,mBAAoB,SAAQ,KAAK;IAC5C,gEAAgE;IAChE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;gBAEV,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;CAK1C;AAED,mFAAmF;AACnF,qBAAa,oBAAqB,SAAQ,mBAAmB;gBAC/C,OAAO,EAAE,MAAM;CAI5B;AAED,0EAA0E;AAC1E,qBAAa,qBAAsB,SAAQ,mBAAmB;gBAChD,OAAO,EAAE,MAAM;CAI5B;AAED,kFAAkF;AAClF,qBAAa,cAAe,SAAQ,mBAAmB;gBACzC,OAAO,EAAE,MAAM;CAI5B;AAED,iEAAiE;AACjE,qBAAa,cAAe,SAAQ,mBAAmB;gBACzC,OAAO,EAAE,MAAM;CAI5B;AAED,oFAAoF;AACpF,qBAAa,eAAgB,SAAQ,mBAAmB;gBAC1C,OAAO,EAAE,MAAM;CAI5B;AAED,oEAAoE;AACpE,qBAAa,gBAAiB,SAAQ,mBAAmB;gBAC3C,OAAO,EAAE,MAAM;CAI5B;AAED,uFAAuF;AACvF,qBAAa,aAAc,SAAQ,mBAAmB;gBACxC,OAAO,EAAE,MAAM;CAI5B"}
|
package/dist/errors.js
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared error base for sqlite-js-stream.
|
|
3
|
+
*
|
|
4
|
+
* Every typed library error extends {@link SqliteJsStreamError} and carries a
|
|
5
|
+
* stable, Node-style `code` string (e.g. `ERR_NO_MATCHING_BACKUP`). This lets
|
|
6
|
+
* callers catch any library error with a single `instanceof`, and branch on the
|
|
7
|
+
* precise failure via `err.code` without matching message text.
|
|
8
|
+
*/
|
|
9
|
+
export class SqliteJsStreamError extends Error {
|
|
10
|
+
/** Stable, machine-readable error code (Node.js convention). */
|
|
11
|
+
code;
|
|
12
|
+
constructor(code, message) {
|
|
13
|
+
super(message);
|
|
14
|
+
this.name = 'SqliteJsStreamError';
|
|
15
|
+
this.code = code;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
/** A required function argument is missing or invalid (`ERR_INVALID_ARGUMENT`). */
|
|
19
|
+
export class InvalidArgumentError extends SqliteJsStreamError {
|
|
20
|
+
constructor(message) {
|
|
21
|
+
super('ERR_INVALID_ARGUMENT', message);
|
|
22
|
+
this.name = 'InvalidArgumentError';
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
/** An incremental LTX file's page size differs from the base database. */
|
|
26
|
+
export class PageSizeMismatchError extends SqliteJsStreamError {
|
|
27
|
+
constructor(message) {
|
|
28
|
+
super('ERR_PAGE_SIZE_MISMATCH', message);
|
|
29
|
+
this.name = 'PageSizeMismatchError';
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
/** An LTX file is malformed/corrupt or violates the format (`ERR_LTX_FORMAT`). */
|
|
33
|
+
export class LTXFormatError extends SqliteJsStreamError {
|
|
34
|
+
constructor(message) {
|
|
35
|
+
super('ERR_LTX_FORMAT', message);
|
|
36
|
+
this.name = 'LTXFormatError';
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
/** A SQLite WAL file is malformed/corrupt (`ERR_WAL_FORMAT`). */
|
|
40
|
+
export class WALFormatError extends SqliteJsStreamError {
|
|
41
|
+
constructor(message) {
|
|
42
|
+
super('ERR_WAL_FORMAT', message);
|
|
43
|
+
this.name = 'WALFormatError';
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
/** A replica URL is malformed or uses an unsupported scheme (`ERR_REPLICA_URL`). */
|
|
47
|
+
export class ReplicaURLError extends SqliteJsStreamError {
|
|
48
|
+
constructor(message) {
|
|
49
|
+
super('ERR_REPLICA_URL', message);
|
|
50
|
+
this.name = 'ReplicaURLError';
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
/** A file/backend replica operation failed (`ERR_FILE_REPLICA`). */
|
|
54
|
+
export class FileReplicaError extends SqliteJsStreamError {
|
|
55
|
+
constructor(message) {
|
|
56
|
+
super('ERR_FILE_REPLICA', message);
|
|
57
|
+
this.name = 'FileReplicaError';
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
/** A SQLite database file is missing/invalid or could not be read (`ERR_DATABASE`). */
|
|
61
|
+
export class DatabaseError extends SqliteJsStreamError {
|
|
62
|
+
constructor(message) {
|
|
63
|
+
super('ERR_DATABASE', message);
|
|
64
|
+
this.name = 'DatabaseError';
|
|
65
|
+
}
|
|
66
|
+
}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* sqlite-js-stream — a Node.js port of Litestream.
|
|
3
|
+
*
|
|
4
|
+
* This is the public entry point. For now it only exports the version marker
|
|
5
|
+
* and placeholder types; the LTX/WAL/DB layers are added in later milestones.
|
|
6
|
+
*/
|
|
7
|
+
export declare const VERSION = "0.1.0";
|
|
8
|
+
export type TXID = bigint;
|
|
9
|
+
export * from './errors.js';
|
|
10
|
+
export * from './ltx/index.js';
|
|
11
|
+
export * from './wal/index.js';
|
|
12
|
+
export * from './replica-client.js';
|
|
13
|
+
export * from './replica-url.js';
|
|
14
|
+
export * from './backends/file/index.js';
|
|
15
|
+
export * from './replica.js';
|
|
16
|
+
export * from './store.js';
|
|
17
|
+
export * from './leaser.js';
|
|
18
|
+
export * from './util/logger.js';
|
|
19
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,eAAO,MAAM,OAAO,UAAU,CAAC;AAE/B,MAAM,MAAM,IAAI,GAAG,MAAM,CAAC;AAE1B,cAAc,aAAa,CAAC;AAC5B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,qBAAqB,CAAC;AACpC,cAAc,kBAAkB,CAAC;AACjC,cAAc,0BAA0B,CAAC;AAIzC,cAAc,cAAc,CAAC;AAC7B,cAAc,YAAY,CAAC;AAC3B,cAAc,aAAa,CAAC;AAG5B,cAAc,kBAAkB,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* sqlite-js-stream — a Node.js port of Litestream.
|
|
3
|
+
*
|
|
4
|
+
* This is the public entry point. For now it only exports the version marker
|
|
5
|
+
* and placeholder types; the LTX/WAL/DB layers are added in later milestones.
|
|
6
|
+
*/
|
|
7
|
+
export const VERSION = '0.1.0';
|
|
8
|
+
export * from "./errors.js";
|
|
9
|
+
export * from "./ltx/index.js";
|
|
10
|
+
export * from "./wal/index.js";
|
|
11
|
+
export * from "./replica-client.js";
|
|
12
|
+
export * from "./replica-url.js";
|
|
13
|
+
export * from "./backends/file/index.js";
|
|
14
|
+
// The DB layer (`src/db.ts`) is intentionally NOT part of the public API.
|
|
15
|
+
// `Replica`/`Store` take a database path and read the DB + WAL directly, so the
|
|
16
|
+
// library is agnostic to whatever SQLite driver the application uses.
|
|
17
|
+
export * from "./replica.js";
|
|
18
|
+
export * from "./store.js";
|
|
19
|
+
export * from "./leaser.js";
|
|
20
|
+
// Logger is part of the public ReplicaClient contract; expose it so addons
|
|
21
|
+
// (e.g. the S3 backend) can implement the interface consistently.
|
|
22
|
+
export * from "./util/logger.js";
|