@platformatic/sqlite-js-stream 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +40 -0
- package/LICENSE +202 -0
- package/NOTICE +22 -0
- package/README.md +482 -2
- package/dist/backends/file/index.d.ts +4 -0
- package/dist/backends/file/index.d.ts.map +1 -0
- package/dist/backends/file/index.js +3 -0
- package/dist/backends/file/leaser.d.ts +20 -0
- package/dist/backends/file/leaser.d.ts.map +1 -0
- package/dist/backends/file/leaser.js +123 -0
- package/dist/backends/file/replica-client.d.ts +38 -0
- package/dist/backends/file/replica-client.d.ts.map +1 -0
- package/dist/backends/file/replica-client.js +209 -0
- package/dist/db.d.ts +80 -0
- package/dist/db.d.ts.map +1 -0
- package/dist/db.js +484 -0
- package/dist/errors.d.ts +42 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +66 -0
- package/dist/index.d.ts +19 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +22 -0
- package/dist/leaser.d.ts +55 -0
- package/dist/leaser.d.ts.map +1 -0
- package/dist/leaser.js +60 -0
- package/dist/ltx/byte-reader.d.ts +18 -0
- package/dist/ltx/byte-reader.d.ts.map +1 -0
- package/dist/ltx/byte-reader.js +38 -0
- package/dist/ltx/bytes.d.ts +20 -0
- package/dist/ltx/bytes.d.ts.map +1 -0
- package/dist/ltx/bytes.js +69 -0
- package/dist/ltx/checksum.d.ts +20 -0
- package/dist/ltx/checksum.d.ts.map +1 -0
- package/dist/ltx/checksum.js +42 -0
- package/dist/ltx/compaction-level.d.ts +58 -0
- package/dist/ltx/compaction-level.d.ts.map +1 -0
- package/dist/ltx/compaction-level.js +115 -0
- package/dist/ltx/compactor.d.ts +33 -0
- package/dist/ltx/compactor.d.ts.map +1 -0
- package/dist/ltx/compactor.js +137 -0
- package/dist/ltx/constants.d.ts +49 -0
- package/dist/ltx/constants.d.ts.map +1 -0
- package/dist/ltx/constants.js +58 -0
- package/dist/ltx/crc64.d.ts +29 -0
- package/dist/ltx/crc64.d.ts.map +1 -0
- package/dist/ltx/crc64.js +57 -0
- package/dist/ltx/decoder.d.ts +55 -0
- package/dist/ltx/decoder.d.ts.map +1 -0
- package/dist/ltx/decoder.js +236 -0
- package/dist/ltx/encoder.d.ts +49 -0
- package/dist/ltx/encoder.d.ts.map +1 -0
- package/dist/ltx/encoder.js +159 -0
- package/dist/ltx/file-info.d.ts +24 -0
- package/dist/ltx/file-info.d.ts.map +1 -0
- package/dist/ltx/file-info.js +74 -0
- package/dist/ltx/filename.d.ts +13 -0
- package/dist/ltx/filename.d.ts.map +1 -0
- package/dist/ltx/filename.js +23 -0
- package/dist/ltx/header.d.ts +17 -0
- package/dist/ltx/header.d.ts.map +1 -0
- package/dist/ltx/header.js +115 -0
- package/dist/ltx/index.d.ts +24 -0
- package/dist/ltx/index.d.ts.map +1 -0
- package/dist/ltx/index.js +25 -0
- package/dist/ltx/lz4.d.ts +41 -0
- package/dist/ltx/lz4.d.ts.map +1 -0
- package/dist/ltx/lz4.js +264 -0
- package/dist/ltx/page-index.d.ts +26 -0
- package/dist/ltx/page-index.d.ts.map +1 -0
- package/dist/ltx/page-index.js +118 -0
- package/dist/ltx/page.d.ts +27 -0
- package/dist/ltx/page.d.ts.map +1 -0
- package/dist/ltx/page.js +66 -0
- package/dist/ltx/sink.d.ts +17 -0
- package/dist/ltx/sink.d.ts.map +1 -0
- package/dist/ltx/sink.js +21 -0
- package/dist/ltx/stream.d.ts +64 -0
- package/dist/ltx/stream.d.ts.map +1 -0
- package/dist/ltx/stream.js +248 -0
- package/dist/ltx/streaming-compactor.d.ts +34 -0
- package/dist/ltx/streaming-compactor.d.ts.map +1 -0
- package/dist/ltx/streaming-compactor.js +140 -0
- package/dist/ltx/trailer.d.ts +12 -0
- package/dist/ltx/trailer.d.ts.map +1 -0
- package/dist/ltx/trailer.js +47 -0
- package/dist/ltx/types.d.ts +79 -0
- package/dist/ltx/types.d.ts.map +1 -0
- package/dist/ltx/types.js +70 -0
- package/dist/replica-client.d.ts +92 -0
- package/dist/replica-client.d.ts.map +1 -0
- package/dist/replica-client.js +122 -0
- package/dist/replica-url.d.ts +20 -0
- package/dist/replica-url.d.ts.map +1 -0
- package/dist/replica-url.js +49 -0
- package/dist/replica.d.ts +379 -0
- package/dist/replica.d.ts.map +1 -0
- package/dist/replica.js +1253 -0
- package/dist/store.d.ts +34 -0
- package/dist/store.d.ts.map +1 -0
- package/dist/store.js +60 -0
- package/dist/util/fs.d.ts +28 -0
- package/dist/util/fs.d.ts.map +1 -0
- package/dist/util/fs.js +73 -0
- package/dist/util/logger.d.ts +10 -0
- package/dist/util/logger.d.ts.map +1 -0
- package/dist/util/logger.js +7 -0
- package/dist/util/stream.d.ts +11 -0
- package/dist/util/stream.d.ts.map +1 -0
- package/dist/util/stream.js +25 -0
- package/dist/wal/checksum.d.ts +17 -0
- package/dist/wal/checksum.d.ts.map +1 -0
- package/dist/wal/checksum.js +30 -0
- package/dist/wal/constants.d.ts +8 -0
- package/dist/wal/constants.d.ts.map +1 -0
- package/dist/wal/constants.js +7 -0
- package/dist/wal/index.d.ts +5 -0
- package/dist/wal/index.d.ts.map +1 -0
- package/dist/wal/index.js +4 -0
- package/dist/wal/reader.d.ts +61 -0
- package/dist/wal/reader.d.ts.map +1 -0
- package/dist/wal/reader.js +207 -0
- package/package.json +68 -3
- package/src/backends/file/index.ts +3 -0
- package/src/backends/file/leaser.ts +145 -0
- package/src/backends/file/replica-client.ts +246 -0
- package/src/db.ts +553 -0
- package/src/errors.ts +74 -0
- package/src/index.ts +26 -0
- package/src/leaser.ts +86 -0
- package/src/ltx/byte-reader.ts +46 -0
- package/src/ltx/bytes.ts +87 -0
- package/src/ltx/checksum.ts +46 -0
- package/src/ltx/compaction-level.ts +135 -0
- package/src/ltx/compactor.ts +167 -0
- package/src/ltx/constants.ts +71 -0
- package/src/ltx/crc64.ts +65 -0
- package/src/ltx/decoder.ts +282 -0
- package/src/ltx/encoder.ts +200 -0
- package/src/ltx/file-info.ts +81 -0
- package/src/ltx/filename.ts +27 -0
- package/src/ltx/header.ts +126 -0
- package/src/ltx/index.ts +27 -0
- package/src/ltx/lz4.ts +279 -0
- package/src/ltx/page-index.ts +140 -0
- package/src/ltx/page.ts +80 -0
- package/src/ltx/sink.ts +33 -0
- package/src/ltx/stream.ts +286 -0
- package/src/ltx/streaming-compactor.ts +169 -0
- package/src/ltx/trailer.ts +58 -0
- package/src/ltx/types.ts +152 -0
- package/src/lz4js.d.ts +27 -0
- package/src/replica-client.ts +222 -0
- package/src/replica-url.ts +72 -0
- package/src/replica.ts +1480 -0
- package/src/store.ts +71 -0
- package/src/util/fs.ts +82 -0
- package/src/util/logger.ts +15 -0
- package/src/util/stream.ts +28 -0
- package/src/wal/checksum.ts +39 -0
- package/src/wal/constants.ts +10 -0
- package/src/wal/index.ts +5 -0
- package/src/wal/reader.ts +264 -0
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LTX Compactor merges multiple LTX files (readers) into a single stream,
|
|
3
|
+
* where later files override earlier ones for the same page number.
|
|
4
|
+
*
|
|
5
|
+
* Mirrors Go's `ltx.Compactor`. Used by restore (snapshot + incremental L0
|
|
6
|
+
* files) and by multi-level compaction.
|
|
7
|
+
*/
|
|
8
|
+
import { isZeroPageHeader, isContiguous } from "./types.js";
|
|
9
|
+
import { Encoder } from "./encoder.js";
|
|
10
|
+
import { Decoder } from "./decoder.js";
|
|
11
|
+
import { InvalidArgumentError, LTXFormatError, PageSizeMismatchError } from "../errors.js";
|
|
12
|
+
/** Compactor merges LTX readers into a single writer. */
|
|
13
|
+
export class Compactor {
|
|
14
|
+
enc;
|
|
15
|
+
inputs = [];
|
|
16
|
+
_headerFlags = 0;
|
|
17
|
+
allowNonContiguousTXIDs = false;
|
|
18
|
+
constructor(sink, readers) {
|
|
19
|
+
this.enc = new Encoder(sink);
|
|
20
|
+
for (const r of readers) {
|
|
21
|
+
const dec = new Decoder(r);
|
|
22
|
+
dec.setRetainPageIndex(false);
|
|
23
|
+
this.inputs.push({ dec, hdr: { pgno: 0, flags: 0 }, data: new Uint8Array(0) });
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
get headerFlags() {
|
|
27
|
+
return this._headerFlags;
|
|
28
|
+
}
|
|
29
|
+
set headerFlags(flags) {
|
|
30
|
+
this._headerFlags = flags;
|
|
31
|
+
}
|
|
32
|
+
/** AllowNonContiguousTXIDs disables contiguous-TXID validation between inputs. */
|
|
33
|
+
setAllowNonContiguousTXIDs(v) {
|
|
34
|
+
this.allowNonContiguousTXIDs = v;
|
|
35
|
+
}
|
|
36
|
+
/** Header returns the output header (after Compact). */
|
|
37
|
+
Header() {
|
|
38
|
+
return this.enc.Header();
|
|
39
|
+
}
|
|
40
|
+
/** Trailer returns the output trailer (after Compact). */
|
|
41
|
+
Trailer() {
|
|
42
|
+
return this.enc.Trailer();
|
|
43
|
+
}
|
|
44
|
+
/** Compact merges all input readers into a single LTX stream. */
|
|
45
|
+
compact() {
|
|
46
|
+
if (this.inputs.length === 0) {
|
|
47
|
+
throw new InvalidArgumentError('at least one input reader required');
|
|
48
|
+
}
|
|
49
|
+
// Read headers from all inputs.
|
|
50
|
+
for (let i = 0; i < this.inputs.length; i++) {
|
|
51
|
+
this.inputs[i].dec.decodeHeader();
|
|
52
|
+
}
|
|
53
|
+
// Validate page sizes & contiguous TXIDs.
|
|
54
|
+
for (let i = 1; i < this.inputs.length; i++) {
|
|
55
|
+
const prevHdr = this.inputs[i - 1].dec.Header();
|
|
56
|
+
const hdr = this.inputs[i].dec.Header();
|
|
57
|
+
if (prevHdr.pageSize !== hdr.pageSize) {
|
|
58
|
+
throw new PageSizeMismatchError(`input files have mismatched page sizes: ${prevHdr.pageSize} != ${hdr.pageSize}`);
|
|
59
|
+
}
|
|
60
|
+
if (!this.allowNonContiguousTXIDs &&
|
|
61
|
+
!isContiguous(prevHdr.maxTXID, hdr.minTXID, hdr.maxTXID)) {
|
|
62
|
+
throw new LTXFormatError(`non-contiguous transaction ids in input files: (${prevHdr.minTXID},${prevHdr.maxTXID}) -> (${hdr.minTXID},${hdr.maxTXID})`);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
const minHdr = this.inputs[0].dec.Header();
|
|
66
|
+
const maxHdr = this.inputs[this.inputs.length - 1].dec.Header();
|
|
67
|
+
this.enc.encodeHeader({
|
|
68
|
+
version: 3,
|
|
69
|
+
flags: this.headerFlags,
|
|
70
|
+
pageSize: minHdr.pageSize,
|
|
71
|
+
commit: maxHdr.commit,
|
|
72
|
+
minTXID: minHdr.minTXID,
|
|
73
|
+
maxTXID: maxHdr.maxTXID,
|
|
74
|
+
timestamp: maxHdr.timestamp,
|
|
75
|
+
preApplyChecksum: minHdr.preApplyChecksum,
|
|
76
|
+
walOffset: 0n,
|
|
77
|
+
walSize: 0n,
|
|
78
|
+
walSalt1: 0,
|
|
79
|
+
walSalt2: 0,
|
|
80
|
+
nodeID: 0n,
|
|
81
|
+
});
|
|
82
|
+
this.writePageBlock();
|
|
83
|
+
for (const input of this.inputs) {
|
|
84
|
+
input.dec.close();
|
|
85
|
+
}
|
|
86
|
+
const last = this.inputs[this.inputs.length - 1];
|
|
87
|
+
this.enc.setPostApplyChecksum(last.dec.Trailer().postApplyChecksum);
|
|
88
|
+
this.enc.close();
|
|
89
|
+
}
|
|
90
|
+
writePageBlock() {
|
|
91
|
+
const pageSize = this.enc.Header().pageSize;
|
|
92
|
+
for (const input of this.inputs) {
|
|
93
|
+
input.data = new Uint8Array(pageSize);
|
|
94
|
+
}
|
|
95
|
+
for (;;) {
|
|
96
|
+
const pgno = this.fillPageBuffers();
|
|
97
|
+
if (pgno === 0)
|
|
98
|
+
break;
|
|
99
|
+
this.writePageBuffer(pgno);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
/** Fills input buffers and returns the smallest available page number. */
|
|
103
|
+
fillPageBuffers() {
|
|
104
|
+
let pgno = 0;
|
|
105
|
+
for (const input of this.inputs) {
|
|
106
|
+
if (isZeroPageHeader(input.hdr)) {
|
|
107
|
+
const hdr = input.dec.decodePage(input.data);
|
|
108
|
+
if (hdr === null)
|
|
109
|
+
continue; // end of page block for this input
|
|
110
|
+
input.hdr = hdr;
|
|
111
|
+
}
|
|
112
|
+
if (pgno === 0 || input.hdr.pgno < pgno) {
|
|
113
|
+
pgno = input.hdr.pgno;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
return pgno;
|
|
117
|
+
}
|
|
118
|
+
/** Writes the buffer with a matching pgno from the latest input. */
|
|
119
|
+
writePageBuffer(pgno) {
|
|
120
|
+
const commit = this.enc.Header().commit;
|
|
121
|
+
let pageWritten = false;
|
|
122
|
+
for (let i = this.inputs.length - 1; i >= 0; i--) {
|
|
123
|
+
const input = this.inputs[i];
|
|
124
|
+
if (input.hdr.pgno !== pgno)
|
|
125
|
+
continue;
|
|
126
|
+
const hdr = input.hdr;
|
|
127
|
+
const data = input.data;
|
|
128
|
+
input.hdr = { pgno: 0, flags: 0 };
|
|
129
|
+
if (pageWritten)
|
|
130
|
+
continue;
|
|
131
|
+
if (pgno > commit)
|
|
132
|
+
continue;
|
|
133
|
+
pageWritten = true;
|
|
134
|
+
this.enc.encodePage(hdr, data);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LTX format constants.
|
|
3
|
+
*
|
|
4
|
+
* Ground truth: superfly/ltx v0.5.2 (vendored by litestream). The byte layout
|
|
5
|
+
* here must match Go's `ltx` package exactly so files interoperate.
|
|
6
|
+
*/
|
|
7
|
+
/** Magic is the first 4 bytes of every LTX file. */
|
|
8
|
+
export declare const Magic = "LTX1";
|
|
9
|
+
/** Version is the current version of the LTX file format. */
|
|
10
|
+
export declare const Version = 3;
|
|
11
|
+
/** Size constants. */
|
|
12
|
+
export declare const HeaderSize = 100;
|
|
13
|
+
export declare const PageHeaderSize = 6;
|
|
14
|
+
export declare const TrailerSize = 16;
|
|
15
|
+
/** Checksum size & positions. */
|
|
16
|
+
export declare const ChecksumSize = 8;
|
|
17
|
+
export declare const TrailerChecksumOffset: number;
|
|
18
|
+
/** ChecksumFlag ensures a checksum is non-zero. */
|
|
19
|
+
export declare const ChecksumFlag: bigint;
|
|
20
|
+
/** Entry byte offset of the PENDING_BYTE within a database. */
|
|
21
|
+
export declare const PENDING_BYTE = 1073741824;
|
|
22
|
+
/** Header flags. */
|
|
23
|
+
export declare const HeaderFlagMask = 2;
|
|
24
|
+
export declare const HeaderFlagNoChecksum = 2;
|
|
25
|
+
/** Page header flags. */
|
|
26
|
+
/** PageHeaderFlagSize indicates a 4-byte size field follows the page header. */
|
|
27
|
+
export declare const PageHeaderFlagSize = 1;
|
|
28
|
+
/** MaxPageSize is the maximum allowed SQLite page size. */
|
|
29
|
+
export declare const MaxPageSize = 65536;
|
|
30
|
+
/** SnapshotLevel is the LTX level reserved for full-database snapshots. */
|
|
31
|
+
export declare const SnapshotLevel = 9;
|
|
32
|
+
/** Error messages / codes. */
|
|
33
|
+
export declare const ErrInvalidFile = "invalid LTX file";
|
|
34
|
+
export declare const ErrDecoderClosed = "ltx decoder closed";
|
|
35
|
+
export declare const ErrEncoderClosed = "ltx encoder closed";
|
|
36
|
+
export declare const ErrEncoderAborted = "ltx encoder aborted";
|
|
37
|
+
export declare const ErrNoChecksum = "no file checksum";
|
|
38
|
+
export declare const ErrInvalidChecksumFormat = "invalid file checksum format";
|
|
39
|
+
export declare const ErrChecksumMismatch = "file checksum mismatch";
|
|
40
|
+
/**
|
|
41
|
+
* LockPgno returns the page number at which the PENDING_BYTE exists.
|
|
42
|
+
* This page is never part of the LTX payload.
|
|
43
|
+
*/
|
|
44
|
+
export declare function lockPgno(pageSize: number): number;
|
|
45
|
+
/** IsValidPageSize returns true if sz is between 512 and 64K and a power of two. */
|
|
46
|
+
export declare function isValidPageSize(sz: number): boolean;
|
|
47
|
+
/** IsValidHeaderFlags returns true unless flags outside the valid mask are set. */
|
|
48
|
+
export declare function isValidHeaderFlags(flags: number): boolean;
|
|
49
|
+
//# sourceMappingURL=constants.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/ltx/constants.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,oDAAoD;AACpD,eAAO,MAAM,KAAK,SAAS,CAAC;AAE5B,6DAA6D;AAC7D,eAAO,MAAM,OAAO,IAAI,CAAC;AAEzB,sBAAsB;AACtB,eAAO,MAAM,UAAU,MAAM,CAAC;AAC9B,eAAO,MAAM,cAAc,IAAI,CAAC;AAChC,eAAO,MAAM,WAAW,KAAK,CAAC;AAE9B,iCAAiC;AACjC,eAAO,MAAM,YAAY,IAAI,CAAC;AAC9B,eAAO,MAAM,qBAAqB,QAA6B,CAAC;AAEhE,mDAAmD;AACnD,eAAO,MAAM,YAAY,QAAY,CAAC;AAEtC,+DAA+D;AAC/D,eAAO,MAAM,YAAY,aAAa,CAAC;AAEvC,oBAAoB;AACpB,eAAO,MAAM,cAAc,IAAa,CAAC;AACzC,eAAO,MAAM,oBAAoB,IAAa,CAAC;AAE/C,yBAAyB;AACzB,gFAAgF;AAChF,eAAO,MAAM,kBAAkB,IAAS,CAAC;AAEzC,2DAA2D;AAC3D,eAAO,MAAM,WAAW,QAAQ,CAAC;AAEjC,2EAA2E;AAC3E,eAAO,MAAM,aAAa,IAAI,CAAC;AAE/B,8BAA8B;AAC9B,eAAO,MAAM,cAAc,qBAAqB,CAAC;AACjD,eAAO,MAAM,gBAAgB,uBAAuB,CAAC;AACrD,eAAO,MAAM,gBAAgB,uBAAuB,CAAC;AACrD,eAAO,MAAM,iBAAiB,wBAAwB,CAAC;AACvD,eAAO,MAAM,aAAa,qBAAqB,CAAC;AAChD,eAAO,MAAM,wBAAwB,iCAAiC,CAAC;AACvE,eAAO,MAAM,mBAAmB,2BAA2B,CAAC;AAE5D;;;GAGG;AACH,wBAAgB,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAEjD;AAED,oFAAoF;AACpF,wBAAgB,eAAe,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAKnD;AAED,mFAAmF;AACnF,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAEzD"}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LTX format constants.
|
|
3
|
+
*
|
|
4
|
+
* Ground truth: superfly/ltx v0.5.2 (vendored by litestream). The byte layout
|
|
5
|
+
* here must match Go's `ltx` package exactly so files interoperate.
|
|
6
|
+
*/
|
|
7
|
+
/** Magic is the first 4 bytes of every LTX file. */
|
|
8
|
+
export const Magic = 'LTX1';
|
|
9
|
+
/** Version is the current version of the LTX file format. */
|
|
10
|
+
export const Version = 3;
|
|
11
|
+
/** Size constants. */
|
|
12
|
+
export const HeaderSize = 100;
|
|
13
|
+
export const PageHeaderSize = 6;
|
|
14
|
+
export const TrailerSize = 16;
|
|
15
|
+
/** Checksum size & positions. */
|
|
16
|
+
export const ChecksumSize = 8;
|
|
17
|
+
export const TrailerChecksumOffset = TrailerSize - ChecksumSize;
|
|
18
|
+
/** ChecksumFlag ensures a checksum is non-zero. */
|
|
19
|
+
export const ChecksumFlag = 1n << 63n;
|
|
20
|
+
/** Entry byte offset of the PENDING_BYTE within a database. */
|
|
21
|
+
export const PENDING_BYTE = 0x40000000;
|
|
22
|
+
/** Header flags. */
|
|
23
|
+
export const HeaderFlagMask = 0x00000002;
|
|
24
|
+
export const HeaderFlagNoChecksum = 0x00000002;
|
|
25
|
+
/** Page header flags. */
|
|
26
|
+
/** PageHeaderFlagSize indicates a 4-byte size field follows the page header. */
|
|
27
|
+
export const PageHeaderFlagSize = 0x0001;
|
|
28
|
+
/** MaxPageSize is the maximum allowed SQLite page size. */
|
|
29
|
+
export const MaxPageSize = 65536;
|
|
30
|
+
/** SnapshotLevel is the LTX level reserved for full-database snapshots. */
|
|
31
|
+
export const SnapshotLevel = 9;
|
|
32
|
+
/** Error messages / codes. */
|
|
33
|
+
export const ErrInvalidFile = 'invalid LTX file';
|
|
34
|
+
export const ErrDecoderClosed = 'ltx decoder closed';
|
|
35
|
+
export const ErrEncoderClosed = 'ltx encoder closed';
|
|
36
|
+
export const ErrEncoderAborted = 'ltx encoder aborted';
|
|
37
|
+
export const ErrNoChecksum = 'no file checksum';
|
|
38
|
+
export const ErrInvalidChecksumFormat = 'invalid file checksum format';
|
|
39
|
+
export const ErrChecksumMismatch = 'file checksum mismatch';
|
|
40
|
+
/**
|
|
41
|
+
* LockPgno returns the page number at which the PENDING_BYTE exists.
|
|
42
|
+
* This page is never part of the LTX payload.
|
|
43
|
+
*/
|
|
44
|
+
export function lockPgno(pageSize) {
|
|
45
|
+
return Math.floor(PENDING_BYTE / pageSize) + 1;
|
|
46
|
+
}
|
|
47
|
+
/** IsValidPageSize returns true if sz is between 512 and 64K and a power of two. */
|
|
48
|
+
export function isValidPageSize(sz) {
|
|
49
|
+
for (let i = 512; i <= MaxPageSize; i *= 2) {
|
|
50
|
+
if (sz === i)
|
|
51
|
+
return true;
|
|
52
|
+
}
|
|
53
|
+
return false;
|
|
54
|
+
}
|
|
55
|
+
/** IsValidHeaderFlags returns true unless flags outside the valid mask are set. */
|
|
56
|
+
export function isValidHeaderFlags(flags) {
|
|
57
|
+
return flags === (flags & HeaderFlagMask);
|
|
58
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CRC-64 ISO implementation, byte-compatible with Go's `hash/crc64` using
|
|
3
|
+
* `crc64.MakeTable(crc64.ISO)`. This is what superfly/ltx uses for all LTX
|
|
4
|
+
* checksums (page checksums, rolling DB checksum, file checksum).
|
|
5
|
+
*
|
|
6
|
+
* NOTE: LTX uses CRC-64 ISO, NOT ECMA. The litestream docs are misleading;
|
|
7
|
+
* the actual code (`crc64.New(crc64.MakeTable(crc64.ISO))`) uses ISO.
|
|
8
|
+
*
|
|
9
|
+
* Algorithm matches Go's reflected `crc64.Update`:
|
|
10
|
+
* - init / xorout = all ones (0xFFFFFFFFFFFFFFFF)
|
|
11
|
+
* - polynomial register value = 0xD800000000000000
|
|
12
|
+
* - table built by right-shifting (reflected), update as
|
|
13
|
+
* `tab[byte(crc) ^ v] ^ (crc >> 8)`
|
|
14
|
+
*/
|
|
15
|
+
/**
|
|
16
|
+
* Crc64 is a streaming CRC-64 hasher, equivalent to Go's `hash/crc64`
|
|
17
|
+
* (ISO table). Call `update()` with successive byte chunks and `sum64()` to
|
|
18
|
+
* obtain the running checksum. `reset()` re-initializes the hasher.
|
|
19
|
+
*/
|
|
20
|
+
export declare class Crc64 {
|
|
21
|
+
private crc;
|
|
22
|
+
update(buf: Uint8Array): void;
|
|
23
|
+
reset(): void;
|
|
24
|
+
/** Returns the digest of all data written so far. */
|
|
25
|
+
sum64(): bigint;
|
|
26
|
+
}
|
|
27
|
+
/** Convenience: computes the CRC-64 ISO of a single buffer. */
|
|
28
|
+
export declare function crc64ISO(buf: Uint8Array): bigint;
|
|
29
|
+
//# sourceMappingURL=crc64.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"crc64.d.ts","sourceRoot":"","sources":["../../src/ltx/crc64.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAoBH;;;;GAIG;AACH,qBAAa,KAAK;IAChB,OAAO,CAAC,GAAG,CAAkB;IAE7B,MAAM,CAAC,GAAG,EAAE,UAAU,GAAG,IAAI;IAQ7B,KAAK,IAAI,IAAI;IAIb,qDAAqD;IACrD,KAAK,IAAI,MAAM;CAGhB;AAED,+DAA+D;AAC/D,wBAAgB,QAAQ,CAAC,GAAG,EAAE,UAAU,GAAG,MAAM,CAIhD"}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CRC-64 ISO implementation, byte-compatible with Go's `hash/crc64` using
|
|
3
|
+
* `crc64.MakeTable(crc64.ISO)`. This is what superfly/ltx uses for all LTX
|
|
4
|
+
* checksums (page checksums, rolling DB checksum, file checksum).
|
|
5
|
+
*
|
|
6
|
+
* NOTE: LTX uses CRC-64 ISO, NOT ECMA. The litestream docs are misleading;
|
|
7
|
+
* the actual code (`crc64.New(crc64.MakeTable(crc64.ISO))`) uses ISO.
|
|
8
|
+
*
|
|
9
|
+
* Algorithm matches Go's reflected `crc64.Update`:
|
|
10
|
+
* - init / xorout = all ones (0xFFFFFFFFFFFFFFFF)
|
|
11
|
+
* - polynomial register value = 0xD800000000000000
|
|
12
|
+
* - table built by right-shifting (reflected), update as
|
|
13
|
+
* `tab[byte(crc) ^ v] ^ (crc >> 8)`
|
|
14
|
+
*/
|
|
15
|
+
const ISO_POLY = 0xd800000000000000n;
|
|
16
|
+
const MASK64 = 0xffffffffffffffffn;
|
|
17
|
+
/** Builds the 256-entry CRC table for a reflected polynomial. */
|
|
18
|
+
function makeTable(poly) {
|
|
19
|
+
const table = new Array(256);
|
|
20
|
+
for (let i = 0; i < 256; i++) {
|
|
21
|
+
let crc = BigInt(i);
|
|
22
|
+
for (let j = 0; j < 8; j++) {
|
|
23
|
+
crc = (crc & 1n) !== 0n ? (crc >> 1n) ^ poly : crc >> 1n;
|
|
24
|
+
}
|
|
25
|
+
table[i] = crc;
|
|
26
|
+
}
|
|
27
|
+
return table;
|
|
28
|
+
}
|
|
29
|
+
const ISO_TABLE = makeTable(ISO_POLY);
|
|
30
|
+
/**
|
|
31
|
+
* Crc64 is a streaming CRC-64 hasher, equivalent to Go's `hash/crc64`
|
|
32
|
+
* (ISO table). Call `update()` with successive byte chunks and `sum64()` to
|
|
33
|
+
* obtain the running checksum. `reset()` re-initializes the hasher.
|
|
34
|
+
*/
|
|
35
|
+
export class Crc64 {
|
|
36
|
+
crc = MASK64;
|
|
37
|
+
update(buf) {
|
|
38
|
+
let crc = this.crc;
|
|
39
|
+
for (let i = 0; i < buf.length; i++) {
|
|
40
|
+
crc = ISO_TABLE[Number((crc ^ BigInt(buf[i])) & 0xffn)] ^ (crc >> 8n);
|
|
41
|
+
}
|
|
42
|
+
this.crc = crc;
|
|
43
|
+
}
|
|
44
|
+
reset() {
|
|
45
|
+
this.crc = MASK64;
|
|
46
|
+
}
|
|
47
|
+
/** Returns the digest of all data written so far. */
|
|
48
|
+
sum64() {
|
|
49
|
+
return this.crc ^ MASK64;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
/** Convenience: computes the CRC-64 ISO of a single buffer. */
|
|
53
|
+
export function crc64ISO(buf) {
|
|
54
|
+
const h = new Crc64();
|
|
55
|
+
h.update(buf);
|
|
56
|
+
return h.sum64();
|
|
57
|
+
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LTX decoder.
|
|
3
|
+
*
|
|
4
|
+
* Mirrors Go's `ltx.Decoder`. Reads header, page frames, page index, and
|
|
5
|
+
* trailer from an in-memory buffer, verifying the file checksum and (for
|
|
6
|
+
* checksum-tracking snapshots) the post-apply rolling checksum.
|
|
7
|
+
*/
|
|
8
|
+
import type { ByteSink } from './sink.js';
|
|
9
|
+
import type { Header, PageHeader, PageIndex, Trailer } from './types.js';
|
|
10
|
+
/** Decoder reads an LTX file from an in-memory buffer. */
|
|
11
|
+
export declare class Decoder {
|
|
12
|
+
private r;
|
|
13
|
+
private header;
|
|
14
|
+
private trailer;
|
|
15
|
+
private pageIndex;
|
|
16
|
+
private state;
|
|
17
|
+
private hash;
|
|
18
|
+
private pageSeq;
|
|
19
|
+
private chksum;
|
|
20
|
+
private pageN;
|
|
21
|
+
private retainPageIndex;
|
|
22
|
+
constructor(buf: Uint8Array);
|
|
23
|
+
/** SetRetainPageIndex controls whether Close materializes the page-index map. */
|
|
24
|
+
setRetainPageIndex(retain: boolean): void;
|
|
25
|
+
/** N returns the number of source bytes read. */
|
|
26
|
+
N(): number;
|
|
27
|
+
/** PageN returns the number of pages decoded. */
|
|
28
|
+
PageN(): number;
|
|
29
|
+
/** Header returns a copy of the header. */
|
|
30
|
+
Header(): Header;
|
|
31
|
+
/** Trailer returns the trailer. File checksum valid after Close(). */
|
|
32
|
+
Trailer(): Trailer;
|
|
33
|
+
/** PageIndex returns the page index (after Close), or null if not retained. */
|
|
34
|
+
PageIndex(): PageIndex | null;
|
|
35
|
+
/** PostApplyPos returns the replication position after applying this file. */
|
|
36
|
+
PostApplyPos(): {
|
|
37
|
+
txID: bigint;
|
|
38
|
+
postApplyChecksum: bigint;
|
|
39
|
+
};
|
|
40
|
+
/** DecodeHeader reads and validates the header frame. */
|
|
41
|
+
decodeHeader(): void;
|
|
42
|
+
/**
|
|
43
|
+
* DecodePage reads the next page frame into `data`. Returns the page header,
|
|
44
|
+
* or null when the end-of-pages marker is reached.
|
|
45
|
+
*/
|
|
46
|
+
decodePage(data: Uint8Array): PageHeader | null;
|
|
47
|
+
/** Verify reads the entire file, discarding page data (for validation). */
|
|
48
|
+
verify(): void;
|
|
49
|
+
/** DecodeDatabaseTo writes the snapshot as a SQLite database to `sink`. */
|
|
50
|
+
decodeDatabaseTo(sink: ByteSink): void;
|
|
51
|
+
/** Close verifies the page index and trailer, then checks the file checksum. */
|
|
52
|
+
close(): void;
|
|
53
|
+
private hashPgo;
|
|
54
|
+
}
|
|
55
|
+
//# sourceMappingURL=decoder.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"decoder.d.ts","sourceRoot":"","sources":["../../src/ltx/decoder.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAcH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAmBzE,0DAA0D;AAC1D,qBAAa,OAAO;IAClB,OAAO,CAAC,CAAC,CAAa;IACtB,OAAO,CAAC,MAAM,CAAU;IACxB,OAAO,CAAC,OAAO,CAAwD;IACvE,OAAO,CAAC,SAAS,CAA0B;IAC3C,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,IAAI,CAAe;IAC3B,OAAO,CAAC,OAAO,CAAe;IAC9B,OAAO,CAAC,MAAM,CAAM;IACpB,OAAO,CAAC,KAAK,CAAK;IAClB,OAAO,CAAC,eAAe,CAAU;gBAErB,GAAG,EAAE,UAAU;IAK3B,iFAAiF;IACjF,kBAAkB,CAAC,MAAM,EAAE,OAAO,GAAG,IAAI;IAIzC,iDAAiD;IACjD,CAAC,IAAI,MAAM;IAIX,iDAAiD;IACjD,KAAK,IAAI,MAAM;IAIf,2CAA2C;IAC3C,MAAM,IAAI,MAAM;IAIhB,sEAAsE;IACtE,OAAO,IAAI,OAAO;IAIlB,+EAA+E;IAC/E,SAAS,IAAI,SAAS,GAAG,IAAI;IAI7B,8EAA8E;IAC9E,YAAY;;;;IAOZ,yDAAyD;IACzD,YAAY,IAAI,IAAI;IAWpB;;;OAGG;IACH,UAAU,CAAC,IAAI,EAAE,UAAU,GAAG,UAAU,GAAG,IAAI;IAkD/C,2EAA2E;IAC3E,MAAM,IAAI,IAAI;IASd,2EAA2E;IAC3E,gBAAgB,CAAC,IAAI,EAAE,QAAQ,GAAG,IAAI;IA8BtC,gFAAgF;IAChF,KAAK,IAAI,IAAI;IA0Eb,OAAO,CAAC,OAAO;CAGhB"}
|
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LTX decoder.
|
|
3
|
+
*
|
|
4
|
+
* Mirrors Go's `ltx.Decoder`. Reads header, page frames, page index, and
|
|
5
|
+
* trailer from an in-memory buffer, verifying the file checksum and (for
|
|
6
|
+
* checksum-tracking snapshots) the post-apply rolling checksum.
|
|
7
|
+
*/
|
|
8
|
+
import { ChecksumFlag, HeaderSize, PageHeaderFlagSize, PageHeaderSize, TrailerChecksumOffset, TrailerSize, lockPgno, } from "./constants.js";
|
|
9
|
+
import { Crc64 } from "./crc64.js";
|
|
10
|
+
import { u32BE } from "./bytes.js";
|
|
11
|
+
import { ByteReader } from "./byte-reader.js";
|
|
12
|
+
import { isSnapshot } from "./types.js";
|
|
13
|
+
import { unmarshalHeader, validateHeader } from "./header.js";
|
|
14
|
+
import { unmarshalPageHeader } from "./page.js";
|
|
15
|
+
import { unmarshalTrailer } from "./trailer.js";
|
|
16
|
+
import { parsePageIndex } from "./page-index.js";
|
|
17
|
+
import { decompressBlock } from "./lz4.js";
|
|
18
|
+
import { checksumPage } from "./checksum.js";
|
|
19
|
+
import { LTXFormatError } from "../errors.js";
|
|
20
|
+
const stateHeader = 'header';
|
|
21
|
+
const statePage = 'page';
|
|
22
|
+
const stateClose = 'close';
|
|
23
|
+
const stateClosed = 'closed';
|
|
24
|
+
function noChecksum(h) {
|
|
25
|
+
return (h.flags & 0x00000002) !== 0;
|
|
26
|
+
}
|
|
27
|
+
/** Decoder reads an LTX file from an in-memory buffer. */
|
|
28
|
+
export class Decoder {
|
|
29
|
+
r;
|
|
30
|
+
header;
|
|
31
|
+
trailer = { postApplyChecksum: 0n, fileChecksum: 0n };
|
|
32
|
+
pageIndex = null;
|
|
33
|
+
state = stateHeader;
|
|
34
|
+
hash = new Crc64();
|
|
35
|
+
pageSeq = new Crc64();
|
|
36
|
+
chksum = 0n;
|
|
37
|
+
pageN = 0;
|
|
38
|
+
retainPageIndex;
|
|
39
|
+
constructor(buf) {
|
|
40
|
+
this.r = new ByteReader(buf);
|
|
41
|
+
this.retainPageIndex = true;
|
|
42
|
+
}
|
|
43
|
+
/** SetRetainPageIndex controls whether Close materializes the page-index map. */
|
|
44
|
+
setRetainPageIndex(retain) {
|
|
45
|
+
this.retainPageIndex = retain;
|
|
46
|
+
}
|
|
47
|
+
/** N returns the number of source bytes read. */
|
|
48
|
+
N() {
|
|
49
|
+
return this.r.pos;
|
|
50
|
+
}
|
|
51
|
+
/** PageN returns the number of pages decoded. */
|
|
52
|
+
PageN() {
|
|
53
|
+
return this.pageN;
|
|
54
|
+
}
|
|
55
|
+
/** Header returns a copy of the header. */
|
|
56
|
+
Header() {
|
|
57
|
+
return this.header;
|
|
58
|
+
}
|
|
59
|
+
/** Trailer returns the trailer. File checksum valid after Close(). */
|
|
60
|
+
Trailer() {
|
|
61
|
+
return this.trailer;
|
|
62
|
+
}
|
|
63
|
+
/** PageIndex returns the page index (after Close), or null if not retained. */
|
|
64
|
+
PageIndex() {
|
|
65
|
+
return this.pageIndex;
|
|
66
|
+
}
|
|
67
|
+
/** PostApplyPos returns the replication position after applying this file. */
|
|
68
|
+
PostApplyPos() {
|
|
69
|
+
return {
|
|
70
|
+
txID: this.header.maxTXID,
|
|
71
|
+
postApplyChecksum: this.trailer.postApplyChecksum,
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
/** DecodeHeader reads and validates the header frame. */
|
|
75
|
+
decodeHeader() {
|
|
76
|
+
const b = this.r.readFull(HeaderSize);
|
|
77
|
+
this.header = unmarshalHeader(b);
|
|
78
|
+
this.hash.update(b);
|
|
79
|
+
this.state = statePage;
|
|
80
|
+
validateHeader(this.header);
|
|
81
|
+
if (!noChecksum(this.header)) {
|
|
82
|
+
this.chksum = ChecksumFlag;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* DecodePage reads the next page frame into `data`. Returns the page header,
|
|
87
|
+
* or null when the end-of-pages marker is reached.
|
|
88
|
+
*/
|
|
89
|
+
decodePage(data) {
|
|
90
|
+
if (this.state === stateClosed)
|
|
91
|
+
throw new LTXFormatError('ltx decoder closed');
|
|
92
|
+
if (this.state === stateClose)
|
|
93
|
+
return null;
|
|
94
|
+
if (this.state !== statePage) {
|
|
95
|
+
throw new LTXFormatError(`cannot read page header, expected ${this.state}`);
|
|
96
|
+
}
|
|
97
|
+
if (data.length !== this.header.pageSize) {
|
|
98
|
+
throw new LTXFormatError(`invalid page buffer size: ${data.length}, expecting ${this.header.pageSize}`);
|
|
99
|
+
}
|
|
100
|
+
const b = this.r.readFull(PageHeaderSize);
|
|
101
|
+
const hdr = unmarshalPageHeader(b);
|
|
102
|
+
this.hash.update(b);
|
|
103
|
+
// An empty page header marks the end of the page block.
|
|
104
|
+
if (hdr.pgno === 0 && hdr.flags === 0) {
|
|
105
|
+
this.state = stateClose;
|
|
106
|
+
return null;
|
|
107
|
+
}
|
|
108
|
+
if (hdr.pgno > this.header.commit) {
|
|
109
|
+
throw new LTXFormatError(`page number ${hdr.pgno} out-of-bounds for commit size ${this.header.commit}`);
|
|
110
|
+
}
|
|
111
|
+
if (hdr.flags & PageHeaderFlagSize) {
|
|
112
|
+
// New block format: read size prefix, then LZ4 block data.
|
|
113
|
+
const sizeBuf = this.r.readFull(4);
|
|
114
|
+
this.hash.update(sizeBuf);
|
|
115
|
+
const dataSize = (sizeBuf[0] << 24 | sizeBuf[1] << 16 | sizeBuf[2] << 8 | sizeBuf[3]) >>> 0;
|
|
116
|
+
const compressed = this.r.readFull(dataSize);
|
|
117
|
+
const decompressed = decompressBlock(compressed, this.header.pageSize);
|
|
118
|
+
data.set(decompressed);
|
|
119
|
+
}
|
|
120
|
+
else {
|
|
121
|
+
throw new LTXFormatError('legacy LZ4 frame format not supported');
|
|
122
|
+
}
|
|
123
|
+
this.hash.update(data);
|
|
124
|
+
this.hashPgo(this.pageSeq, hdr.pgno);
|
|
125
|
+
this.pageN++;
|
|
126
|
+
// Track the rolling DB checksum for checksum-tracking snapshots.
|
|
127
|
+
if (isSnapshot(this.header) && !noChecksum(this.header)) {
|
|
128
|
+
if (hdr.pgno !== lockPgno(this.header.pageSize)) {
|
|
129
|
+
this.chksum = ChecksumFlag | (this.chksum ^ checksumPage(hdr.pgno, data));
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
return hdr;
|
|
133
|
+
}
|
|
134
|
+
/** Verify reads the entire file, discarding page data (for validation). */
|
|
135
|
+
verify() {
|
|
136
|
+
this.decodeHeader();
|
|
137
|
+
const data = new Uint8Array(this.header.pageSize);
|
|
138
|
+
for (;;) {
|
|
139
|
+
if (this.decodePage(data) === null)
|
|
140
|
+
break;
|
|
141
|
+
}
|
|
142
|
+
this.close();
|
|
143
|
+
}
|
|
144
|
+
/** DecodeDatabaseTo writes the snapshot as a SQLite database to `sink`. */
|
|
145
|
+
decodeDatabaseTo(sink) {
|
|
146
|
+
this.decodeHeader();
|
|
147
|
+
const hdr = this.header;
|
|
148
|
+
if (!isSnapshot(hdr)) {
|
|
149
|
+
throw new LTXFormatError('cannot decode non-snapshot LTX file to SQLite database');
|
|
150
|
+
}
|
|
151
|
+
const lockPg = lockPgno(hdr.pageSize);
|
|
152
|
+
const data = new Uint8Array(hdr.pageSize);
|
|
153
|
+
for (let pgno = 1; pgno <= hdr.commit; pgno++) {
|
|
154
|
+
if (pgno === lockPg) {
|
|
155
|
+
sink.write(new Uint8Array(hdr.pageSize));
|
|
156
|
+
continue;
|
|
157
|
+
}
|
|
158
|
+
const ph = this.decodePage(data);
|
|
159
|
+
if (ph === null) {
|
|
160
|
+
throw new LTXFormatError(`unexpected end of pages at pgno ${pgno}`);
|
|
161
|
+
}
|
|
162
|
+
if (ph.pgno !== pgno) {
|
|
163
|
+
throw new LTXFormatError(`unexpected pgno while decoding page: read ${ph.pgno}, expected ${pgno}`);
|
|
164
|
+
}
|
|
165
|
+
sink.write(data);
|
|
166
|
+
}
|
|
167
|
+
// Expect one final read to be the end-of-pages marker.
|
|
168
|
+
if (this.decodePage(data) !== null) {
|
|
169
|
+
throw new LTXFormatError('unexpected page after end of database');
|
|
170
|
+
}
|
|
171
|
+
this.close();
|
|
172
|
+
}
|
|
173
|
+
/** Close verifies the page index and trailer, then checks the file checksum. */
|
|
174
|
+
close() {
|
|
175
|
+
if (this.state === stateClosed)
|
|
176
|
+
return;
|
|
177
|
+
if (this.state !== stateClose) {
|
|
178
|
+
throw new LTXFormatError(`cannot close, expected ${this.state}`);
|
|
179
|
+
}
|
|
180
|
+
const index = new Map();
|
|
181
|
+
let n = 0;
|
|
182
|
+
let nextOffset = HeaderSize;
|
|
183
|
+
const seq = new Crc64();
|
|
184
|
+
const self = this;
|
|
185
|
+
parsePageIndex(this.r, (buf) => self.hash.update(buf), (pgno, offset, size) => {
|
|
186
|
+
if (pgno > self.header.commit) {
|
|
187
|
+
throw new LTXFormatError(`page index pgno ${pgno} exceeds commit ${self.header.commit}`);
|
|
188
|
+
}
|
|
189
|
+
if (offset < nextOffset) {
|
|
190
|
+
throw new LTXFormatError(`page index pgno ${pgno} offset ${offset} overlaps previous frame ending at ${nextOffset}`);
|
|
191
|
+
}
|
|
192
|
+
if (size <= PageHeaderSize) {
|
|
193
|
+
throw new LTXFormatError(`page index pgno ${pgno} has invalid frame size ${size}`);
|
|
194
|
+
}
|
|
195
|
+
nextOffset = offset + size;
|
|
196
|
+
n++;
|
|
197
|
+
self.hashPgo(seq, pgno);
|
|
198
|
+
if (index) {
|
|
199
|
+
index.set(pgno, {
|
|
200
|
+
level: 0,
|
|
201
|
+
minTXID: self.header.minTXID,
|
|
202
|
+
maxTXID: self.header.maxTXID,
|
|
203
|
+
offset,
|
|
204
|
+
size,
|
|
205
|
+
});
|
|
206
|
+
}
|
|
207
|
+
});
|
|
208
|
+
if (n !== this.pageN) {
|
|
209
|
+
throw new LTXFormatError(`page index has ${n} entries but ${this.pageN} pages were decoded`);
|
|
210
|
+
}
|
|
211
|
+
if (seq.sum64() !== this.pageSeq.sum64()) {
|
|
212
|
+
throw new LTXFormatError('page index does not match the decoded page numbers');
|
|
213
|
+
}
|
|
214
|
+
this.pageIndex = this.retainPageIndex ? index : null;
|
|
215
|
+
// Read the trailer; everything except the file checksum field is hashed.
|
|
216
|
+
const t = this.r.readFull(TrailerSize);
|
|
217
|
+
this.hash.update(t.subarray(0, TrailerChecksumOffset));
|
|
218
|
+
this.trailer = unmarshalTrailer(t);
|
|
219
|
+
if (this.r.remaining > 0) {
|
|
220
|
+
throw new LTXFormatError('unexpected data after trailer');
|
|
221
|
+
}
|
|
222
|
+
const computed = ChecksumFlag | this.hash.sum64();
|
|
223
|
+
if (computed !== this.trailer.fileChecksum) {
|
|
224
|
+
throw new LTXFormatError('file checksum mismatch');
|
|
225
|
+
}
|
|
226
|
+
if (isSnapshot(this.header) && !noChecksum(this.header)) {
|
|
227
|
+
if (this.trailer.postApplyChecksum !== this.chksum) {
|
|
228
|
+
throw new LTXFormatError(`post-apply checksum in trailer (${this.trailer.postApplyChecksum.toString(16)}) does not match calculated checksum (${this.chksum.toString(16)})`);
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
this.state = stateClosed;
|
|
232
|
+
}
|
|
233
|
+
hashPgo(h, pgno) {
|
|
234
|
+
h.update(u32BE(pgno));
|
|
235
|
+
}
|
|
236
|
+
}
|