@platformatic/sqlite-js-stream 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +40 -0
- package/LICENSE +202 -0
- package/NOTICE +22 -0
- package/README.md +482 -2
- package/dist/backends/file/index.d.ts +4 -0
- package/dist/backends/file/index.d.ts.map +1 -0
- package/dist/backends/file/index.js +3 -0
- package/dist/backends/file/leaser.d.ts +20 -0
- package/dist/backends/file/leaser.d.ts.map +1 -0
- package/dist/backends/file/leaser.js +123 -0
- package/dist/backends/file/replica-client.d.ts +38 -0
- package/dist/backends/file/replica-client.d.ts.map +1 -0
- package/dist/backends/file/replica-client.js +209 -0
- package/dist/db.d.ts +80 -0
- package/dist/db.d.ts.map +1 -0
- package/dist/db.js +484 -0
- package/dist/errors.d.ts +42 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +66 -0
- package/dist/index.d.ts +19 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +22 -0
- package/dist/leaser.d.ts +55 -0
- package/dist/leaser.d.ts.map +1 -0
- package/dist/leaser.js +60 -0
- package/dist/ltx/byte-reader.d.ts +18 -0
- package/dist/ltx/byte-reader.d.ts.map +1 -0
- package/dist/ltx/byte-reader.js +38 -0
- package/dist/ltx/bytes.d.ts +20 -0
- package/dist/ltx/bytes.d.ts.map +1 -0
- package/dist/ltx/bytes.js +69 -0
- package/dist/ltx/checksum.d.ts +20 -0
- package/dist/ltx/checksum.d.ts.map +1 -0
- package/dist/ltx/checksum.js +42 -0
- package/dist/ltx/compaction-level.d.ts +58 -0
- package/dist/ltx/compaction-level.d.ts.map +1 -0
- package/dist/ltx/compaction-level.js +115 -0
- package/dist/ltx/compactor.d.ts +33 -0
- package/dist/ltx/compactor.d.ts.map +1 -0
- package/dist/ltx/compactor.js +137 -0
- package/dist/ltx/constants.d.ts +49 -0
- package/dist/ltx/constants.d.ts.map +1 -0
- package/dist/ltx/constants.js +58 -0
- package/dist/ltx/crc64.d.ts +29 -0
- package/dist/ltx/crc64.d.ts.map +1 -0
- package/dist/ltx/crc64.js +57 -0
- package/dist/ltx/decoder.d.ts +55 -0
- package/dist/ltx/decoder.d.ts.map +1 -0
- package/dist/ltx/decoder.js +236 -0
- package/dist/ltx/encoder.d.ts +49 -0
- package/dist/ltx/encoder.d.ts.map +1 -0
- package/dist/ltx/encoder.js +159 -0
- package/dist/ltx/file-info.d.ts +24 -0
- package/dist/ltx/file-info.d.ts.map +1 -0
- package/dist/ltx/file-info.js +74 -0
- package/dist/ltx/filename.d.ts +13 -0
- package/dist/ltx/filename.d.ts.map +1 -0
- package/dist/ltx/filename.js +23 -0
- package/dist/ltx/header.d.ts +17 -0
- package/dist/ltx/header.d.ts.map +1 -0
- package/dist/ltx/header.js +115 -0
- package/dist/ltx/index.d.ts +24 -0
- package/dist/ltx/index.d.ts.map +1 -0
- package/dist/ltx/index.js +25 -0
- package/dist/ltx/lz4.d.ts +41 -0
- package/dist/ltx/lz4.d.ts.map +1 -0
- package/dist/ltx/lz4.js +264 -0
- package/dist/ltx/page-index.d.ts +26 -0
- package/dist/ltx/page-index.d.ts.map +1 -0
- package/dist/ltx/page-index.js +118 -0
- package/dist/ltx/page.d.ts +27 -0
- package/dist/ltx/page.d.ts.map +1 -0
- package/dist/ltx/page.js +66 -0
- package/dist/ltx/sink.d.ts +17 -0
- package/dist/ltx/sink.d.ts.map +1 -0
- package/dist/ltx/sink.js +21 -0
- package/dist/ltx/stream.d.ts +64 -0
- package/dist/ltx/stream.d.ts.map +1 -0
- package/dist/ltx/stream.js +248 -0
- package/dist/ltx/streaming-compactor.d.ts +34 -0
- package/dist/ltx/streaming-compactor.d.ts.map +1 -0
- package/dist/ltx/streaming-compactor.js +140 -0
- package/dist/ltx/trailer.d.ts +12 -0
- package/dist/ltx/trailer.d.ts.map +1 -0
- package/dist/ltx/trailer.js +47 -0
- package/dist/ltx/types.d.ts +79 -0
- package/dist/ltx/types.d.ts.map +1 -0
- package/dist/ltx/types.js +70 -0
- package/dist/replica-client.d.ts +92 -0
- package/dist/replica-client.d.ts.map +1 -0
- package/dist/replica-client.js +122 -0
- package/dist/replica-url.d.ts +20 -0
- package/dist/replica-url.d.ts.map +1 -0
- package/dist/replica-url.js +49 -0
- package/dist/replica.d.ts +379 -0
- package/dist/replica.d.ts.map +1 -0
- package/dist/replica.js +1253 -0
- package/dist/store.d.ts +34 -0
- package/dist/store.d.ts.map +1 -0
- package/dist/store.js +60 -0
- package/dist/util/fs.d.ts +28 -0
- package/dist/util/fs.d.ts.map +1 -0
- package/dist/util/fs.js +73 -0
- package/dist/util/logger.d.ts +10 -0
- package/dist/util/logger.d.ts.map +1 -0
- package/dist/util/logger.js +7 -0
- package/dist/util/stream.d.ts +11 -0
- package/dist/util/stream.d.ts.map +1 -0
- package/dist/util/stream.js +25 -0
- package/dist/wal/checksum.d.ts +17 -0
- package/dist/wal/checksum.d.ts.map +1 -0
- package/dist/wal/checksum.js +30 -0
- package/dist/wal/constants.d.ts +8 -0
- package/dist/wal/constants.d.ts.map +1 -0
- package/dist/wal/constants.js +7 -0
- package/dist/wal/index.d.ts +5 -0
- package/dist/wal/index.d.ts.map +1 -0
- package/dist/wal/index.js +4 -0
- package/dist/wal/reader.d.ts +61 -0
- package/dist/wal/reader.d.ts.map +1 -0
- package/dist/wal/reader.js +207 -0
- package/package.json +68 -3
- package/src/backends/file/index.ts +3 -0
- package/src/backends/file/leaser.ts +145 -0
- package/src/backends/file/replica-client.ts +246 -0
- package/src/db.ts +553 -0
- package/src/errors.ts +74 -0
- package/src/index.ts +26 -0
- package/src/leaser.ts +86 -0
- package/src/ltx/byte-reader.ts +46 -0
- package/src/ltx/bytes.ts +87 -0
- package/src/ltx/checksum.ts +46 -0
- package/src/ltx/compaction-level.ts +135 -0
- package/src/ltx/compactor.ts +167 -0
- package/src/ltx/constants.ts +71 -0
- package/src/ltx/crc64.ts +65 -0
- package/src/ltx/decoder.ts +282 -0
- package/src/ltx/encoder.ts +200 -0
- package/src/ltx/file-info.ts +81 -0
- package/src/ltx/filename.ts +27 -0
- package/src/ltx/header.ts +126 -0
- package/src/ltx/index.ts +27 -0
- package/src/ltx/lz4.ts +279 -0
- package/src/ltx/page-index.ts +140 -0
- package/src/ltx/page.ts +80 -0
- package/src/ltx/sink.ts +33 -0
- package/src/ltx/stream.ts +286 -0
- package/src/ltx/streaming-compactor.ts +169 -0
- package/src/ltx/trailer.ts +58 -0
- package/src/ltx/types.ts +152 -0
- package/src/lz4js.d.ts +27 -0
- package/src/replica-client.ts +222 -0
- package/src/replica-url.ts +72 -0
- package/src/replica.ts +1480 -0
- package/src/store.ts +71 -0
- package/src/util/fs.ts +82 -0
- package/src/util/logger.ts +15 -0
- package/src/util/stream.ts +28 -0
- package/src/wal/checksum.ts +39 -0
- package/src/wal/constants.ts +10 -0
- package/src/wal/index.ts +5 -0
- package/src/wal/reader.ts +264 -0
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Streaming LTX reader.
|
|
3
|
+
*
|
|
4
|
+
* The in-memory {@link Decoder} is convenient but requires the whole LTX file in
|
|
5
|
+
* a single buffer. For large databases we read files from a streaming source (a
|
|
6
|
+
* `Readable`) and decode page-by-page, so peak memory stays bounded by one page
|
|
7
|
+
* buffer plus the small page index, never by the full database size.
|
|
8
|
+
*
|
|
9
|
+
* {@link StreamLtxReader} mirrors the synchronous `Decoder` in shape: read the
|
|
10
|
+
* header, then read each page into a reusable buffer, then `verify()` the page
|
|
11
|
+
* index + trailer (including the rolling checksum) once the end-of-pages marker
|
|
12
|
+
* is reached. It is used by streaming restore and compact.
|
|
13
|
+
*/
|
|
14
|
+
import { ChecksumFlag, HeaderSize, PageHeaderFlagSize, PageHeaderSize, TrailerChecksumOffset, TrailerSize, lockPgno, } from "./constants.js";
|
|
15
|
+
import { Crc64 } from "./crc64.js";
|
|
16
|
+
import { u32BE } from "./bytes.js";
|
|
17
|
+
import { ByteReader } from "./byte-reader.js";
|
|
18
|
+
import { parsePageIndex } from "./page-index.js";
|
|
19
|
+
import { unmarshalHeader, validateHeader } from "./header.js";
|
|
20
|
+
import { unmarshalPageHeader } from "./page.js";
|
|
21
|
+
import { unmarshalTrailer } from "./trailer.js";
|
|
22
|
+
import { decompressBlockInto } from "./lz4.js";
|
|
23
|
+
import { checksumPage } from "./checksum.js";
|
|
24
|
+
import { isSnapshot } from "./types.js";
|
|
25
|
+
import { LTXFormatError } from "../errors.js";
|
|
26
|
+
const stateHeader = 'header';
|
|
27
|
+
const statePage = 'page';
|
|
28
|
+
const stateClose = 'close';
|
|
29
|
+
const stateClosed = 'closed';
|
|
30
|
+
function noChecksum(h) {
|
|
31
|
+
return (h.flags & 0x00000002) !== 0;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Pulls bytes from an async iterable of chunks into an exact-bytes reader.
|
|
35
|
+
* Handles chunk boundaries that split logical records (page headers, LZ4
|
|
36
|
+
* blocks, varints) without buffering more than one chunk at a time.
|
|
37
|
+
*/
|
|
38
|
+
export class AsyncByteReader {
|
|
39
|
+
it;
|
|
40
|
+
buf = new Uint8Array(0);
|
|
41
|
+
bufPos = 0;
|
|
42
|
+
pos = 0;
|
|
43
|
+
constructor(source) {
|
|
44
|
+
this.it = source[Symbol.asyncIterator]();
|
|
45
|
+
}
|
|
46
|
+
/** Fills the internal buffer from the source. Returns false on EOF. */
|
|
47
|
+
async fill() {
|
|
48
|
+
if (this.bufPos < this.buf.length)
|
|
49
|
+
return true;
|
|
50
|
+
const next = await this.it.next();
|
|
51
|
+
if (next.done)
|
|
52
|
+
return false;
|
|
53
|
+
const v = next.value;
|
|
54
|
+
this.buf = v instanceof Uint8Array ? v : new Uint8Array(v);
|
|
55
|
+
this.bufPos = 0;
|
|
56
|
+
return true;
|
|
57
|
+
}
|
|
58
|
+
/** Reads exactly `n` bytes, advancing the position. Throws on short input. */
|
|
59
|
+
async readFull(n) {
|
|
60
|
+
const out = new Uint8Array(n);
|
|
61
|
+
let o = 0;
|
|
62
|
+
while (o < n) {
|
|
63
|
+
if (this.bufPos >= this.buf.length) {
|
|
64
|
+
if (!(await this.fill()))
|
|
65
|
+
throw new LTXFormatError('unexpected end of input');
|
|
66
|
+
}
|
|
67
|
+
const take = Math.min(n - o, this.buf.length - this.bufPos);
|
|
68
|
+
out.set(this.buf.subarray(this.bufPos, this.bufPos + take), o);
|
|
69
|
+
this.bufPos += take;
|
|
70
|
+
o += take;
|
|
71
|
+
}
|
|
72
|
+
this.pos += n;
|
|
73
|
+
return out;
|
|
74
|
+
}
|
|
75
|
+
/** Reads and returns every remaining byte in the source. */
|
|
76
|
+
async readAllRemaining() {
|
|
77
|
+
const parts = [];
|
|
78
|
+
let total = 0;
|
|
79
|
+
if (this.bufPos < this.buf.length) {
|
|
80
|
+
const rest = this.buf.subarray(this.bufPos);
|
|
81
|
+
parts.push(rest);
|
|
82
|
+
total += rest.length;
|
|
83
|
+
}
|
|
84
|
+
for (;;) {
|
|
85
|
+
const next = await this.it.next();
|
|
86
|
+
if (next.done)
|
|
87
|
+
break;
|
|
88
|
+
const v = next.value;
|
|
89
|
+
const c = v instanceof Uint8Array ? v : new Uint8Array(v);
|
|
90
|
+
parts.push(c);
|
|
91
|
+
total += c.length;
|
|
92
|
+
}
|
|
93
|
+
this.pos += total;
|
|
94
|
+
const out = new Uint8Array(total);
|
|
95
|
+
let o = 0;
|
|
96
|
+
for (const p of parts) {
|
|
97
|
+
out.set(p, o);
|
|
98
|
+
o += p.length;
|
|
99
|
+
}
|
|
100
|
+
return out;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Streams an LTX file from an async iterable of byte chunks, decoding the
|
|
105
|
+
* header and each page without holding the whole file in memory.
|
|
106
|
+
*/
|
|
107
|
+
export class StreamLtxReader {
|
|
108
|
+
r;
|
|
109
|
+
header;
|
|
110
|
+
trailer = { postApplyChecksum: 0n, fileChecksum: 0n };
|
|
111
|
+
state = stateHeader;
|
|
112
|
+
hash = new Crc64();
|
|
113
|
+
pageSeq = new Crc64();
|
|
114
|
+
chksum = 0n;
|
|
115
|
+
pageN = 0;
|
|
116
|
+
constructor(source) {
|
|
117
|
+
this.r = new AsyncByteReader(source);
|
|
118
|
+
}
|
|
119
|
+
/** Header returns the decoded header (after `readHeader`). */
|
|
120
|
+
Header() {
|
|
121
|
+
return this.header;
|
|
122
|
+
}
|
|
123
|
+
/** Trailer returns the trailer (valid after `verify`). */
|
|
124
|
+
Trailer() {
|
|
125
|
+
return this.trailer;
|
|
126
|
+
}
|
|
127
|
+
/** PageN returns the number of pages decoded. */
|
|
128
|
+
PageN() {
|
|
129
|
+
return this.pageN;
|
|
130
|
+
}
|
|
131
|
+
/** Reads and validates the LTX header frame. */
|
|
132
|
+
async readHeader() {
|
|
133
|
+
if (this.state === stateClosed)
|
|
134
|
+
throw new LTXFormatError('ltx reader closed');
|
|
135
|
+
if (this.state !== stateHeader) {
|
|
136
|
+
throw new LTXFormatError(`cannot read header, expected ${this.state}`);
|
|
137
|
+
}
|
|
138
|
+
const b = await this.r.readFull(HeaderSize);
|
|
139
|
+
this.header = unmarshalHeader(b);
|
|
140
|
+
this.hash.update(b);
|
|
141
|
+
this.state = statePage;
|
|
142
|
+
validateHeader(this.header);
|
|
143
|
+
if (!noChecksum(this.header))
|
|
144
|
+
this.chksum = ChecksumFlag;
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* Reads the next page frame into `data`. Returns the page header, or null
|
|
148
|
+
* when the end-of-pages marker is reached.
|
|
149
|
+
*/
|
|
150
|
+
async readPage(data) {
|
|
151
|
+
if (this.state === stateClosed)
|
|
152
|
+
throw new LTXFormatError('ltx reader closed');
|
|
153
|
+
if (this.state === stateClose)
|
|
154
|
+
return null;
|
|
155
|
+
if (this.state !== statePage) {
|
|
156
|
+
throw new LTXFormatError(`cannot read page header, expected ${this.state}`);
|
|
157
|
+
}
|
|
158
|
+
if (data.length !== this.header.pageSize) {
|
|
159
|
+
throw new LTXFormatError(`invalid page buffer size: ${data.length}, expecting ${this.header.pageSize}`);
|
|
160
|
+
}
|
|
161
|
+
const b = await this.r.readFull(PageHeaderSize);
|
|
162
|
+
const hdr = unmarshalPageHeader(b);
|
|
163
|
+
this.hash.update(b);
|
|
164
|
+
// An empty page header marks the end of the page block.
|
|
165
|
+
if (hdr.pgno === 0 && hdr.flags === 0) {
|
|
166
|
+
this.state = stateClose;
|
|
167
|
+
return null;
|
|
168
|
+
}
|
|
169
|
+
if (hdr.pgno > this.header.commit) {
|
|
170
|
+
throw new LTXFormatError(`page number ${hdr.pgno} out-of-bounds for commit size ${this.header.commit}`);
|
|
171
|
+
}
|
|
172
|
+
if (hdr.flags & PageHeaderFlagSize) {
|
|
173
|
+
const sizeBuf = await this.r.readFull(4);
|
|
174
|
+
this.hash.update(sizeBuf);
|
|
175
|
+
const dataSize = (sizeBuf[0] << 24 | sizeBuf[1] << 16 | sizeBuf[2] << 8 | sizeBuf[3]) >>> 0;
|
|
176
|
+
const compressed = await this.r.readFull(dataSize);
|
|
177
|
+
const n = decompressBlockInto(compressed, data);
|
|
178
|
+
if (n !== this.header.pageSize) {
|
|
179
|
+
throw new LTXFormatError(`lz4 block decompressed to ${n} bytes, expected ${this.header.pageSize}`);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
else {
|
|
183
|
+
throw new LTXFormatError('legacy LZ4 frame format not supported');
|
|
184
|
+
}
|
|
185
|
+
this.hash.update(data);
|
|
186
|
+
this.hashPgo(this.pageSeq, hdr.pgno);
|
|
187
|
+
this.pageN++;
|
|
188
|
+
// Track the rolling DB checksum for checksum-tracking snapshots.
|
|
189
|
+
if (isSnapshot(this.header) && !noChecksum(this.header)) {
|
|
190
|
+
if (hdr.pgno !== lockPgno(this.header.pageSize)) {
|
|
191
|
+
this.chksum = ChecksumFlag | (this.chksum ^ checksumPage(hdr.pgno, data));
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
return hdr;
|
|
195
|
+
}
|
|
196
|
+
/** Reads the page index + trailer and verifies the file checksum. */
|
|
197
|
+
async verify() {
|
|
198
|
+
if (this.state === stateClosed)
|
|
199
|
+
return;
|
|
200
|
+
if (this.state !== stateClose) {
|
|
201
|
+
throw new LTXFormatError(`cannot verify, expected ${this.state}`);
|
|
202
|
+
}
|
|
203
|
+
const remaining = await this.r.readAllRemaining();
|
|
204
|
+
const br = new ByteReader(remaining);
|
|
205
|
+
let n = 0;
|
|
206
|
+
let nextOffset = HeaderSize;
|
|
207
|
+
const seq = new Crc64();
|
|
208
|
+
parsePageIndex(br, (buf) => this.hash.update(buf), (pgno, offset, size) => {
|
|
209
|
+
if (pgno > this.header.commit) {
|
|
210
|
+
throw new LTXFormatError(`page index pgno ${pgno} exceeds commit ${this.header.commit}`);
|
|
211
|
+
}
|
|
212
|
+
if (offset < nextOffset) {
|
|
213
|
+
throw new LTXFormatError(`page index pgno ${pgno} offset ${offset} overlaps previous frame ending at ${nextOffset}`);
|
|
214
|
+
}
|
|
215
|
+
if (size <= PageHeaderSize) {
|
|
216
|
+
throw new LTXFormatError(`page index pgno ${pgno} has invalid frame size ${size}`);
|
|
217
|
+
}
|
|
218
|
+
nextOffset = offset + size;
|
|
219
|
+
n++;
|
|
220
|
+
this.hashPgo(seq, pgno);
|
|
221
|
+
});
|
|
222
|
+
if (n !== this.pageN) {
|
|
223
|
+
throw new LTXFormatError(`page index has ${n} entries but ${this.pageN} pages were decoded`);
|
|
224
|
+
}
|
|
225
|
+
if (seq.sum64() !== this.pageSeq.sum64()) {
|
|
226
|
+
throw new LTXFormatError('page index does not match the decoded page numbers');
|
|
227
|
+
}
|
|
228
|
+
const t = br.readFull(TrailerSize);
|
|
229
|
+
this.hash.update(t.subarray(0, TrailerChecksumOffset));
|
|
230
|
+
this.trailer = unmarshalTrailer(t);
|
|
231
|
+
if (br.remaining > 0) {
|
|
232
|
+
throw new LTXFormatError('unexpected data after trailer');
|
|
233
|
+
}
|
|
234
|
+
const computed = ChecksumFlag | this.hash.sum64();
|
|
235
|
+
if (computed !== this.trailer.fileChecksum) {
|
|
236
|
+
throw new LTXFormatError('file checksum mismatch');
|
|
237
|
+
}
|
|
238
|
+
if (isSnapshot(this.header) && !noChecksum(this.header)) {
|
|
239
|
+
if (this.trailer.postApplyChecksum !== this.chksum) {
|
|
240
|
+
throw new LTXFormatError(`post-apply checksum in trailer (${this.trailer.postApplyChecksum.toString(16)}) does not match calculated checksum (${this.chksum.toString(16)})`);
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
this.state = stateClosed;
|
|
244
|
+
}
|
|
245
|
+
hashPgo(h, pgno) {
|
|
246
|
+
h.update(u32BE(pgno));
|
|
247
|
+
}
|
|
248
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Streaming LTX Compactor.
|
|
3
|
+
*
|
|
4
|
+
* The in-memory {@link Compactor} requires each input LTX file as a single
|
|
5
|
+
* `Uint8Array`, which defeats bounded-memory compaction of large databases.
|
|
6
|
+
* This compactor instead merges {@link StreamLtxReader}s (which read pages
|
|
7
|
+
* incrementally) into a single output stream, so peak memory stays bounded by
|
|
8
|
+
* one page buffer per input plus the (small) page index.
|
|
9
|
+
*
|
|
10
|
+
* Semantics match Go's `ltx.Compactor`: later files override earlier files for
|
|
11
|
+
* the same page number, and the output is written with `HeaderFlagNoChecksum`.
|
|
12
|
+
*/
|
|
13
|
+
import type { ByteSink } from './sink.js';
|
|
14
|
+
import type { StreamLtxReader } from './stream.js';
|
|
15
|
+
import type { Header, Trailer } from './types.js';
|
|
16
|
+
/** StreamingCompactor merges LTX readers (page-by-page) into one writer. */
|
|
17
|
+
export declare class StreamingCompactor {
|
|
18
|
+
private enc;
|
|
19
|
+
private inputs;
|
|
20
|
+
private _headerFlags;
|
|
21
|
+
constructor(sink: ByteSink, readers: StreamLtxReader[], headerFlags?: number);
|
|
22
|
+
/** Header returns the output header (valid after `compact`). */
|
|
23
|
+
Header(): Header;
|
|
24
|
+
/** Trailer returns the output trailer (valid after `compact`). */
|
|
25
|
+
Trailer(): Trailer;
|
|
26
|
+
/** Compact merges all input readers into a single LTX stream. */
|
|
27
|
+
compact(): Promise<void>;
|
|
28
|
+
private writePageBlock;
|
|
29
|
+
/** Fills buffers as needed and returns the smallest available page number. */
|
|
30
|
+
private fillPageBuffers;
|
|
31
|
+
/** Writes the page with matching pgno from the latest input (highest index). */
|
|
32
|
+
private writePageBuffer;
|
|
33
|
+
}
|
|
34
|
+
//# sourceMappingURL=streaming-compactor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"streaming-compactor.d.ts","sourceRoot":"","sources":["../../src/ltx/streaming-compactor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAKH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAE1C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,KAAK,EAAE,MAAM,EAAc,OAAO,EAAE,MAAM,YAAY,CAAC;AAW9D,4EAA4E;AAC5E,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,GAAG,CAAU;IACrB,OAAO,CAAC,MAAM,CAAU;IACxB,OAAO,CAAC,YAAY,CAAS;gBAEjB,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,eAAe,EAAE,EAAE,WAAW,GAAE,MAA6B;IAWlG,gEAAgE;IAChE,MAAM,IAAI,MAAM;IAIhB,kEAAkE;IAClE,OAAO,IAAI,OAAO;IAIlB,iEAAiE;IAC3D,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;YAiEhB,cAAc;IAQ5B,8EAA8E;YAChE,eAAe;IAc7B,gFAAgF;YAClE,eAAe;CAqB9B"}
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Streaming LTX Compactor.
|
|
3
|
+
*
|
|
4
|
+
* The in-memory {@link Compactor} requires each input LTX file as a single
|
|
5
|
+
* `Uint8Array`, which defeats bounded-memory compaction of large databases.
|
|
6
|
+
* This compactor instead merges {@link StreamLtxReader}s (which read pages
|
|
7
|
+
* incrementally) into a single output stream, so peak memory stays bounded by
|
|
8
|
+
* one page buffer per input plus the (small) page index.
|
|
9
|
+
*
|
|
10
|
+
* Semantics match Go's `ltx.Compactor`: later files override earlier files for
|
|
11
|
+
* the same page number, and the output is written with `HeaderFlagNoChecksum`.
|
|
12
|
+
*/
|
|
13
|
+
import { HeaderFlagNoChecksum, } from "./constants.js";
|
|
14
|
+
import { Encoder } from "./encoder.js";
|
|
15
|
+
import { InvalidArgumentError, LTXFormatError, PageSizeMismatchError } from "../errors.js";
|
|
16
|
+
import { isContiguous } from "./types.js";
|
|
17
|
+
/** StreamingCompactor merges LTX readers (page-by-page) into one writer. */
|
|
18
|
+
export class StreamingCompactor {
|
|
19
|
+
enc;
|
|
20
|
+
inputs;
|
|
21
|
+
_headerFlags;
|
|
22
|
+
constructor(sink, readers, headerFlags = HeaderFlagNoChecksum) {
|
|
23
|
+
this.enc = new Encoder(sink);
|
|
24
|
+
this._headerFlags = headerFlags;
|
|
25
|
+
this.inputs = readers.map((reader) => ({
|
|
26
|
+
reader,
|
|
27
|
+
data: new Uint8Array(0),
|
|
28
|
+
hdr: null,
|
|
29
|
+
ended: false,
|
|
30
|
+
}));
|
|
31
|
+
}
|
|
32
|
+
/** Header returns the output header (valid after `compact`). */
|
|
33
|
+
Header() {
|
|
34
|
+
return this.enc.Header();
|
|
35
|
+
}
|
|
36
|
+
/** Trailer returns the output trailer (valid after `compact`). */
|
|
37
|
+
Trailer() {
|
|
38
|
+
return this.enc.Trailer();
|
|
39
|
+
}
|
|
40
|
+
/** Compact merges all input readers into a single LTX stream. */
|
|
41
|
+
async compact() {
|
|
42
|
+
if (this.inputs.length === 0) {
|
|
43
|
+
throw new InvalidArgumentError('at least one input reader required');
|
|
44
|
+
}
|
|
45
|
+
// Read headers from all inputs.
|
|
46
|
+
for (const input of this.inputs) {
|
|
47
|
+
await input.reader.readHeader();
|
|
48
|
+
}
|
|
49
|
+
// Validate page sizes & contiguous TXIDs.
|
|
50
|
+
for (let i = 1; i < this.inputs.length; i++) {
|
|
51
|
+
const prevHdr = this.inputs[i - 1].reader.Header();
|
|
52
|
+
const hdr = this.inputs[i].reader.Header();
|
|
53
|
+
if (prevHdr.pageSize !== hdr.pageSize) {
|
|
54
|
+
throw new PageSizeMismatchError(`input files have mismatched page sizes: ${prevHdr.pageSize} != ${hdr.pageSize}`);
|
|
55
|
+
}
|
|
56
|
+
if (!isContiguous(prevHdr.maxTXID, hdr.minTXID, hdr.maxTXID)) {
|
|
57
|
+
throw new LTXFormatError(`non-contiguous transaction ids in input files: (${prevHdr.minTXID},${prevHdr.maxTXID}) -> (${hdr.minTXID},${hdr.maxTXID})`);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
const minHdr = this.inputs[0].reader.Header();
|
|
61
|
+
const maxHdr = this.inputs[this.inputs.length - 1].reader.Header();
|
|
62
|
+
const pageSize = minHdr.pageSize;
|
|
63
|
+
for (const input of this.inputs) {
|
|
64
|
+
input.data = new Uint8Array(pageSize);
|
|
65
|
+
}
|
|
66
|
+
// Propagate the WAL position of the last (newest) input so that a later
|
|
67
|
+
// sync can resume reading the WAL from where the merged range ended,
|
|
68
|
+
// even after the level-0 sources have been pruned.
|
|
69
|
+
this.enc.encodeHeader({
|
|
70
|
+
version: 3,
|
|
71
|
+
flags: this._headerFlags,
|
|
72
|
+
pageSize,
|
|
73
|
+
commit: maxHdr.commit,
|
|
74
|
+
minTXID: minHdr.minTXID,
|
|
75
|
+
maxTXID: maxHdr.maxTXID,
|
|
76
|
+
timestamp: maxHdr.timestamp,
|
|
77
|
+
preApplyChecksum: minHdr.preApplyChecksum,
|
|
78
|
+
walOffset: maxHdr.walOffset,
|
|
79
|
+
walSize: maxHdr.walSize,
|
|
80
|
+
walSalt1: maxHdr.walSalt1,
|
|
81
|
+
walSalt2: maxHdr.walSalt2,
|
|
82
|
+
nodeID: maxHdr.nodeID,
|
|
83
|
+
});
|
|
84
|
+
await this.writePageBlock();
|
|
85
|
+
// Verify every input's index/trailer (catches a corrupt source) and
|
|
86
|
+
// propagate the final post-apply checksum.
|
|
87
|
+
const last = this.inputs[this.inputs.length - 1];
|
|
88
|
+
for (const input of this.inputs) {
|
|
89
|
+
await input.reader.verify();
|
|
90
|
+
}
|
|
91
|
+
this.enc.setPostApplyChecksum(last.reader.Trailer().postApplyChecksum);
|
|
92
|
+
this.enc.close();
|
|
93
|
+
}
|
|
94
|
+
async writePageBlock() {
|
|
95
|
+
for (;;) {
|
|
96
|
+
const pgno = await this.fillPageBuffers();
|
|
97
|
+
if (pgno === 0)
|
|
98
|
+
break;
|
|
99
|
+
await this.writePageBuffer(pgno);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
/** Fills buffers as needed and returns the smallest available page number. */
|
|
103
|
+
async fillPageBuffers() {
|
|
104
|
+
let pgno = 0;
|
|
105
|
+
for (const input of this.inputs) {
|
|
106
|
+
if (input.hdr === null && !input.ended) {
|
|
107
|
+
input.hdr = await input.reader.readPage(input.data);
|
|
108
|
+
if (input.hdr === null)
|
|
109
|
+
input.ended = true;
|
|
110
|
+
}
|
|
111
|
+
if (input.hdr !== null && (pgno === 0 || input.hdr.pgno < pgno)) {
|
|
112
|
+
pgno = input.hdr.pgno;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
return pgno;
|
|
116
|
+
}
|
|
117
|
+
/** Writes the page with matching pgno from the latest input (highest index). */
|
|
118
|
+
async writePageBuffer(pgno) {
|
|
119
|
+
let emitIndex = -1;
|
|
120
|
+
for (let i = 0; i < this.inputs.length; i++) {
|
|
121
|
+
const hdr = this.inputs[i].hdr;
|
|
122
|
+
if (hdr !== null && hdr.pgno === pgno)
|
|
123
|
+
emitIndex = i;
|
|
124
|
+
}
|
|
125
|
+
if (emitIndex < 0) {
|
|
126
|
+
throw new LTXFormatError(`internal: no input holds page ${pgno}`);
|
|
127
|
+
}
|
|
128
|
+
// Emit from the latest matching input, then consume every matching input
|
|
129
|
+
// so a page at the same pgno is only written once.
|
|
130
|
+
for (let i = 0; i < this.inputs.length; i++) {
|
|
131
|
+
const input = this.inputs[i];
|
|
132
|
+
if (input.hdr === null || input.hdr.pgno !== pgno)
|
|
133
|
+
continue;
|
|
134
|
+
if (i === emitIndex) {
|
|
135
|
+
this.enc.encodePage(input.hdr, input.data);
|
|
136
|
+
}
|
|
137
|
+
input.hdr = null;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LTX trailer frame marshalling.
|
|
3
|
+
*/
|
|
4
|
+
import type { Header, Trailer } from './types.js';
|
|
5
|
+
/** MarshalBinary encodes a Trailer to its 16-byte representation. */
|
|
6
|
+
export declare function marshalTrailer(t: Trailer): Uint8Array;
|
|
7
|
+
/** UnmarshalBinary decodes a Trailer from a byte slice. */
|
|
8
|
+
export declare function unmarshalTrailer(b: Uint8Array): Trailer;
|
|
9
|
+
/** Validate returns an error (thrown) if t is invalid for header h. */
|
|
10
|
+
export declare function validateTrailer(t: Trailer, h: Header): void;
|
|
11
|
+
export declare const TRAILER_CHECKSUM_OFFSET: number;
|
|
12
|
+
//# sourceMappingURL=trailer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"trailer.d.ts","sourceRoot":"","sources":["../../src/ltx/trailer.ts"],"names":[],"mappings":"AAAA;;GAEG;AAQH,OAAO,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAGlD,qEAAqE;AACrE,wBAAgB,cAAc,CAAC,CAAC,EAAE,OAAO,GAAG,UAAU,CAKrD;AAED,2DAA2D;AAC3D,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,UAAU,GAAG,OAAO,CAMvD;AAMD,uEAAuE;AACvE,wBAAgB,eAAe,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,GAAG,IAAI,CAoB3D;AAED,eAAO,MAAM,uBAAuB,QAAwB,CAAC"}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LTX trailer frame marshalling.
|
|
3
|
+
*/
|
|
4
|
+
import { ChecksumFlag, TrailerChecksumOffset, TrailerSize, } from "./constants.js";
|
|
5
|
+
import { assertFull, readU64BE, writeU64BE } from "./bytes.js";
|
|
6
|
+
import { LTXFormatError } from "../errors.js";
|
|
7
|
+
/** MarshalBinary encodes a Trailer to its 16-byte representation. */
|
|
8
|
+
export function marshalTrailer(t) {
|
|
9
|
+
const b = new Uint8Array(TrailerSize);
|
|
10
|
+
writeU64BE(b, 0, t.postApplyChecksum);
|
|
11
|
+
writeU64BE(b, 8, t.fileChecksum);
|
|
12
|
+
return b;
|
|
13
|
+
}
|
|
14
|
+
/** UnmarshalBinary decodes a Trailer from a byte slice. */
|
|
15
|
+
export function unmarshalTrailer(b) {
|
|
16
|
+
assertFull(b, 0, TrailerSize);
|
|
17
|
+
return {
|
|
18
|
+
postApplyChecksum: readU64BE(b, 0),
|
|
19
|
+
fileChecksum: readU64BE(b, 8),
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
function noChecksum(h) {
|
|
23
|
+
return (h.flags & 0x00000002) !== 0;
|
|
24
|
+
}
|
|
25
|
+
/** Validate returns an error (thrown) if t is invalid for header h. */
|
|
26
|
+
export function validateTrailer(t, h) {
|
|
27
|
+
if (noChecksum(h)) {
|
|
28
|
+
if (t.postApplyChecksum !== 0n) {
|
|
29
|
+
throw new LTXFormatError('post-apply checksum not allowed');
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
else {
|
|
33
|
+
if (t.postApplyChecksum === 0n) {
|
|
34
|
+
throw new LTXFormatError('post-apply checksum required');
|
|
35
|
+
}
|
|
36
|
+
if ((t.postApplyChecksum & ChecksumFlag) === 0n) {
|
|
37
|
+
throw new LTXFormatError('invalid post-apply checksum format');
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
if (t.fileChecksum === 0n) {
|
|
41
|
+
throw new LTXFormatError('file checksum required');
|
|
42
|
+
}
|
|
43
|
+
if ((t.fileChecksum & ChecksumFlag) === 0n) {
|
|
44
|
+
throw new LTXFormatError('invalid file checksum format');
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
export const TRAILER_CHECKSUM_OFFSET = TrailerChecksumOffset;
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LTX shared types and small value helpers.
|
|
3
|
+
*
|
|
4
|
+
* 64-bit unsigned values (TXID, Checksum) are represented as `bigint` to keep
|
|
5
|
+
* exact semantics. Page numbers and file offsets/sizes fit within JS safe
|
|
6
|
+
* integers for any realistic database, so they are `number`.
|
|
7
|
+
*/
|
|
8
|
+
export type TXID = bigint;
|
|
9
|
+
export type Checksum = bigint;
|
|
10
|
+
/** Pos is the transactional position of a database. */
|
|
11
|
+
export interface Pos {
|
|
12
|
+
txID: TXID;
|
|
13
|
+
postApplyChecksum: Checksum;
|
|
14
|
+
}
|
|
15
|
+
/** Formats a TXID as a fixed-width 16-char lowercase hex string. */
|
|
16
|
+
export declare function formatTXID(txID: bigint): string;
|
|
17
|
+
/** Parses a 16-character hex string into a TXID. */
|
|
18
|
+
export declare function parseTXID(s: string): bigint;
|
|
19
|
+
/** Formats a Checksum as a fixed-width 16-char lowercase hex string. */
|
|
20
|
+
export declare function formatChecksum(chk: bigint): string;
|
|
21
|
+
/** Parses a 16-character hex string into a Checksum. */
|
|
22
|
+
export declare function parseChecksum(s: string): bigint;
|
|
23
|
+
export declare function parsePos(s: string): Pos;
|
|
24
|
+
export declare function formatPos(p: Pos): string;
|
|
25
|
+
export declare function isZeroPos(p: Pos): boolean;
|
|
26
|
+
/** Header flags. */
|
|
27
|
+
export interface Header {
|
|
28
|
+
version: number;
|
|
29
|
+
flags: number;
|
|
30
|
+
pageSize: number;
|
|
31
|
+
commit: number;
|
|
32
|
+
minTXID: TXID;
|
|
33
|
+
maxTXID: TXID;
|
|
34
|
+
timestamp: bigint;
|
|
35
|
+
preApplyChecksum: Checksum;
|
|
36
|
+
walOffset: bigint;
|
|
37
|
+
walSize: bigint;
|
|
38
|
+
walSalt1: number;
|
|
39
|
+
walSalt2: number;
|
|
40
|
+
nodeID: bigint;
|
|
41
|
+
}
|
|
42
|
+
export declare function isSnapshot(h: Header): boolean;
|
|
43
|
+
export declare function noChecksum(h: Header): boolean;
|
|
44
|
+
/** PageHeader represents the header for a single page in an LTX file. */
|
|
45
|
+
export interface PageHeader {
|
|
46
|
+
pgno: number;
|
|
47
|
+
flags: number;
|
|
48
|
+
}
|
|
49
|
+
export declare function isZeroPageHeader(h: PageHeader): boolean;
|
|
50
|
+
/** Trailer represents the ending frame of an LTX file. */
|
|
51
|
+
export interface Trailer {
|
|
52
|
+
postApplyChecksum: Checksum;
|
|
53
|
+
fileChecksum: Checksum;
|
|
54
|
+
}
|
|
55
|
+
/** PageIndexElem describes a single page entry in the file's page index. */
|
|
56
|
+
export interface PageIndexElem {
|
|
57
|
+
level: number;
|
|
58
|
+
minTXID: TXID;
|
|
59
|
+
maxTXID: TXID;
|
|
60
|
+
offset: number;
|
|
61
|
+
size: number;
|
|
62
|
+
}
|
|
63
|
+
export type PageIndex = Map<number, PageIndexElem>;
|
|
64
|
+
/** FileInfo represents file information about an LTX file. */
|
|
65
|
+
export interface FileInfo {
|
|
66
|
+
level: number;
|
|
67
|
+
minTXID: TXID;
|
|
68
|
+
maxTXID: TXID;
|
|
69
|
+
preApplyChecksum?: Checksum;
|
|
70
|
+
postApplyChecksum?: Checksum;
|
|
71
|
+
size: number;
|
|
72
|
+
createdAt: number;
|
|
73
|
+
checksum?: Checksum;
|
|
74
|
+
}
|
|
75
|
+
export declare function preApplyPos(info: FileInfo): Pos;
|
|
76
|
+
export declare function postApplyPos(info: FileInfo): Pos;
|
|
77
|
+
/** IsContiguous reports whether a transaction range is contiguous with prevMaxTXID. */
|
|
78
|
+
export declare function isContiguous(prevMaxTXID: TXID, minTXID: TXID, maxTXID: TXID): boolean;
|
|
79
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/ltx/types.ts"],"names":[],"mappings":"AACA;;;;;;GAMG;AAEH,MAAM,MAAM,IAAI,GAAG,MAAM,CAAC;AAC1B,MAAM,MAAM,QAAQ,GAAG,MAAM,CAAC;AAE9B,uDAAuD;AACvD,MAAM,WAAW,GAAG;IAClB,IAAI,EAAE,IAAI,CAAC;IACX,iBAAiB,EAAE,QAAQ,CAAC;CAC7B;AAED,oEAAoE;AACpE,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAE/C;AAED,oDAAoD;AACpD,wBAAgB,SAAS,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAQ3C;AAED,wEAAwE;AACxE,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAElD;AAED,wDAAwD;AACxD,wBAAgB,aAAa,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAQ/C;AAED,wBAAgB,QAAQ,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAQvC;AAED,wBAAgB,SAAS,CAAC,CAAC,EAAE,GAAG,GAAG,MAAM,CAExC;AAED,wBAAgB,SAAS,CAAC,CAAC,EAAE,GAAG,GAAG,OAAO,CAEzC;AAED,oBAAoB;AACpB,MAAM,WAAW,MAAM;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,IAAI,CAAC;IACd,OAAO,EAAE,IAAI,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,QAAQ,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,wBAAgB,UAAU,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAE7C;AAED,wBAAgB,UAAU,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAE7C;AAID,yEAAyE;AACzE,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;CACf;AAED,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,UAAU,GAAG,OAAO,CAEvD;AAED,0DAA0D;AAC1D,MAAM,WAAW,OAAO;IACtB,iBAAiB,EAAE,QAAQ,CAAC;IAC5B,YAAY,EAAE,QAAQ,CAAC;CACxB;AAED,4EAA4E;AAC5E,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,IAAI,CAAC;IACd,OAAO,EAAE,IAAI,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,MAAM,SAAS,GAAG,GAAG,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;AAEnD,8DAA8D;AAC9D,MAAM,WAAW,QAAQ;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,IAAI,CAAC;IACd,OAAO,EAAE,IAAI,CAAC;IACd,gBAAgB,CAAC,EAAE,QAAQ,CAAC;IAC5B,iBAAiB,CAAC,EAAE,QAAQ,CAAC;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,QAAQ,CAAC;CACrB;AAED,wBAAgB,WAAW,CAAC,IAAI,EAAE,QAAQ,GAAG,GAAG,CAK/C;AAED,wBAAgB,YAAY,CAAC,IAAI,EAAE,QAAQ,GAAG,GAAG,CAKhD;AAED,uFAAuF;AACvF,wBAAgB,YAAY,CAAC,WAAW,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,GAAG,OAAO,CAErF"}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { LTXFormatError } from "../errors.js";
|
|
2
|
+
/** Formats a TXID as a fixed-width 16-char lowercase hex string. */
|
|
3
|
+
export function formatTXID(txID) {
|
|
4
|
+
return txID.toString(16).padStart(16, '0');
|
|
5
|
+
}
|
|
6
|
+
/** Parses a 16-character hex string into a TXID. */
|
|
7
|
+
export function parseTXID(s) {
|
|
8
|
+
if (s.length !== 16) {
|
|
9
|
+
throw new LTXFormatError(`invalid formatted transaction id length: ${JSON.stringify(s)}`);
|
|
10
|
+
}
|
|
11
|
+
if (!/^[0-9a-fA-F]{16}$/.test(s)) {
|
|
12
|
+
throw new LTXFormatError(`invalid transaction id format: ${JSON.stringify(s)}`);
|
|
13
|
+
}
|
|
14
|
+
return BigInt(`0x${s}`);
|
|
15
|
+
}
|
|
16
|
+
/** Formats a Checksum as a fixed-width 16-char lowercase hex string. */
|
|
17
|
+
export function formatChecksum(chk) {
|
|
18
|
+
return chk.toString(16).padStart(16, '0');
|
|
19
|
+
}
|
|
20
|
+
/** Parses a 16-character hex string into a Checksum. */
|
|
21
|
+
export function parseChecksum(s) {
|
|
22
|
+
if (s.length !== 16) {
|
|
23
|
+
throw new LTXFormatError(`invalid formatted checksum length: ${JSON.stringify(s)}`);
|
|
24
|
+
}
|
|
25
|
+
if (!/^[0-9a-fA-F]{16}$/.test(s)) {
|
|
26
|
+
throw new LTXFormatError(`invalid checksum format: ${JSON.stringify(s)}`);
|
|
27
|
+
}
|
|
28
|
+
return BigInt(`0x${s}`);
|
|
29
|
+
}
|
|
30
|
+
export function parsePos(s) {
|
|
31
|
+
if (s.length !== 33) {
|
|
32
|
+
throw new LTXFormatError(`invalid formatted position length: ${JSON.stringify(s)}`);
|
|
33
|
+
}
|
|
34
|
+
return {
|
|
35
|
+
txID: parseTXID(s.slice(0, 16)),
|
|
36
|
+
postApplyChecksum: parseChecksum(s.slice(17)),
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
export function formatPos(p) {
|
|
40
|
+
return `${formatTXID(p.txID)}/${formatChecksum(p.postApplyChecksum)}`;
|
|
41
|
+
}
|
|
42
|
+
export function isZeroPos(p) {
|
|
43
|
+
return p.txID === 0n && p.postApplyChecksum === 0n;
|
|
44
|
+
}
|
|
45
|
+
export function isSnapshot(h) {
|
|
46
|
+
return h.minTXID === 1n;
|
|
47
|
+
}
|
|
48
|
+
export function noChecksum(h) {
|
|
49
|
+
return (h.flags & HeaderFlagNoChecksumBit) !== 0;
|
|
50
|
+
}
|
|
51
|
+
const HeaderFlagNoChecksumBit = 0x00000002;
|
|
52
|
+
export function isZeroPageHeader(h) {
|
|
53
|
+
return h.pgno === 0 && h.flags === 0;
|
|
54
|
+
}
|
|
55
|
+
export function preApplyPos(info) {
|
|
56
|
+
return {
|
|
57
|
+
txID: info.minTXID - 1n,
|
|
58
|
+
postApplyChecksum: info.preApplyChecksum ?? 0n,
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
export function postApplyPos(info) {
|
|
62
|
+
return {
|
|
63
|
+
txID: info.maxTXID,
|
|
64
|
+
postApplyChecksum: info.postApplyChecksum ?? 0n,
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
/** IsContiguous reports whether a transaction range is contiguous with prevMaxTXID. */
|
|
68
|
+
export function isContiguous(prevMaxTXID, minTXID, maxTXID) {
|
|
69
|
+
return minTXID <= prevMaxTXID + 1n && maxTXID > prevMaxTXID;
|
|
70
|
+
}
|