inibase 3.7.0 → 3.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,59 @@
1
+ import { type UpdateFrame, UpdateLog } from "./updatelog.js";
2
+ /**
3
+ * In-memory view of a table's update log: `internalId -> { cols, updatedAt }`.
4
+ *
5
+ * `cols` holds the *encoded* column values exactly as `joinPathesContents`
6
+ * produces them (one entry per changed top-level column), so a merged read can
7
+ * decode them with the same field config as the base columns. The overlay is
8
+ * keyed by the internal numeric id, which is stable across line shifts caused
9
+ * by `post`/`delete`, so it never has to be rebuilt when rows move.
10
+ *
11
+ * Freshness: `refresh()` compares the log's size and mtime with the last scan.
12
+ * A change means another writer appended (or truncated) and the overlay is
13
+ * rebuilt from `baseGen + 1`. `baseGen` is pinned by the caller from the
14
+ * table's `.format` marker; a compaction (3.9) bumps it and the caller creates
15
+ * a fresh overlay.
16
+ */
17
+ export declare class TableOverlay {
18
+ readonly log: UpdateLog;
19
+ readonly baseGen: number;
20
+ committedGen: number;
21
+ readonly byId: Map<string, {
22
+ cols: Record<string, unknown>;
23
+ updatedAt?: number;
24
+ }>;
25
+ private scannedSize;
26
+ private scannedMtimeMs;
27
+ constructor(tablePath: string, baseGen: number);
28
+ get size(): number;
29
+ get(id: string | number): {
30
+ cols: Record<string, unknown>;
31
+ updatedAt?: number;
32
+ };
33
+ has(id: string | number): boolean;
34
+ delete(id: string | number): boolean;
35
+ /** Fold one committed frame onto the in-memory view (last write wins). */
36
+ applyFrame(frame: UpdateFrame): void;
37
+ private currentStat;
38
+ /**
39
+ * Reader refresh: rebuild only when the log file changed since the last
40
+ * scan. Both size *and* mtime are compared: a repaired torn tail followed by
41
+ * a same-size append (truncate + regrow) changes mtime even when the length
42
+ * is unchanged, which closes the stale-size window flagged in the plan. A
43
+ * torn tail is ignored (only complete, checksum-valid frames are applied);
44
+ * mid-log corruption is surfaced by the scanner.
45
+ */
46
+ refresh(): Promise<void>;
47
+ /**
48
+ * Writer refresh: hold the table lock, repair any torn tail, then rebuild
49
+ * from the committed boundary. Never leaves unreadable bytes behind the
50
+ * next frame.
51
+ */
52
+ refreshForWrite(): Promise<void>;
53
+ /**
54
+ * Apply frames that were just durably appended by this process without a
55
+ * rescan, and realign the scanned stat with the file. Only valid right after
56
+ * `refreshForWrite()` and while still holding the writer lock.
57
+ */
58
+ applyLocal(frames: UpdateFrame[]): Promise<void>;
59
+ }
@@ -0,0 +1,106 @@
1
+ import { stat } from "node:fs/promises";
2
+ import { DEFAULT_MAX_FRAME_BYTES, UpdateLog, } from "./updatelog.js";
3
+ /**
4
+ * In-memory view of a table's update log: `internalId -> { cols, updatedAt }`.
5
+ *
6
+ * `cols` holds the *encoded* column values exactly as `joinPathesContents`
7
+ * produces them (one entry per changed top-level column), so a merged read can
8
+ * decode them with the same field config as the base columns. The overlay is
9
+ * keyed by the internal numeric id, which is stable across line shifts caused
10
+ * by `post`/`delete`, so it never has to be rebuilt when rows move.
11
+ *
12
+ * Freshness: `refresh()` compares the log's size and mtime with the last scan.
13
+ * A change means another writer appended (or truncated) and the overlay is
14
+ * rebuilt from `baseGen + 1`. `baseGen` is pinned by the caller from the
15
+ * table's `.format` marker; a compaction (3.9) bumps it and the caller creates
16
+ * a fresh overlay.
17
+ */
18
+ export class TableOverlay {
19
+ log;
20
+ baseGen;
21
+ committedGen;
22
+ byId = new Map();
23
+ scannedSize = -1;
24
+ scannedMtimeMs = -1;
25
+ constructor(tablePath, baseGen) {
26
+ this.log = new UpdateLog(tablePath);
27
+ this.baseGen = baseGen;
28
+ this.committedGen = baseGen;
29
+ }
30
+ get size() {
31
+ return this.byId.size;
32
+ }
33
+ get(id) {
34
+ return this.byId.get(String(id));
35
+ }
36
+ has(id) {
37
+ return this.byId.has(String(id));
38
+ }
39
+ delete(id) {
40
+ return this.byId.delete(String(id));
41
+ }
42
+ /** Fold one committed frame onto the in-memory view (last write wins). */
43
+ applyFrame(frame) {
44
+ for (const record of frame.records) {
45
+ const key = String(record.id);
46
+ const previous = this.byId.get(key);
47
+ this.byId.set(key, {
48
+ cols: { ...(previous?.cols ?? {}), ...record.cols },
49
+ updatedAt: record.updatedAt,
50
+ });
51
+ }
52
+ this.committedGen = frame.gen;
53
+ }
54
+ async currentStat() {
55
+ try {
56
+ const stats = await stat(this.log.path);
57
+ return { size: stats.size, mtimeMs: stats.mtimeMs };
58
+ }
59
+ catch {
60
+ return { size: 0, mtimeMs: 0 };
61
+ }
62
+ }
63
+ /**
64
+ * Reader refresh: rebuild only when the log file changed since the last
65
+ * scan. Both size *and* mtime are compared: a repaired torn tail followed by
66
+ * a same-size append (truncate + regrow) changes mtime even when the length
67
+ * is unchanged, which closes the stale-size window flagged in the plan. A
68
+ * torn tail is ignored (only complete, checksum-valid frames are applied);
69
+ * mid-log corruption is surfaced by the scanner.
70
+ */
71
+ async refresh() {
72
+ const { size, mtimeMs } = await this.currentStat();
73
+ if (size === this.scannedSize && mtimeMs === this.scannedMtimeMs)
74
+ return;
75
+ this.byId.clear();
76
+ this.committedGen = this.baseGen;
77
+ await this.log.scan(this.baseGen + 1, DEFAULT_MAX_FRAME_BYTES, (frame) => this.applyFrame(frame));
78
+ this.scannedSize = size;
79
+ this.scannedMtimeMs = mtimeMs;
80
+ }
81
+ /**
82
+ * Writer refresh: hold the table lock, repair any torn tail, then rebuild
83
+ * from the committed boundary. Never leaves unreadable bytes behind the
84
+ * next frame.
85
+ */
86
+ async refreshForWrite() {
87
+ this.byId.clear();
88
+ this.committedGen = this.baseGen;
89
+ await this.log.repairTail(this.baseGen + 1, DEFAULT_MAX_FRAME_BYTES, (frame) => this.applyFrame(frame));
90
+ const { size, mtimeMs } = await this.currentStat();
91
+ this.scannedSize = size;
92
+ this.scannedMtimeMs = mtimeMs;
93
+ }
94
+ /**
95
+ * Apply frames that were just durably appended by this process without a
96
+ * rescan, and realign the scanned stat with the file. Only valid right after
97
+ * `refreshForWrite()` and while still holding the writer lock.
98
+ */
99
+ async applyLocal(frames) {
100
+ for (const frame of frames)
101
+ this.applyFrame(frame);
102
+ const { size, mtimeMs } = await this.currentStat();
103
+ this.scannedSize = size;
104
+ this.scannedMtimeMs = mtimeMs;
105
+ }
106
+ }
@@ -0,0 +1,115 @@
1
+ /**
2
+ * Append-only update log for incremental `put` writes.
3
+ *
4
+ * A table with the format-2 update log stores base columns exactly as before
5
+ * (`.txt` / `.gz`) plus one `updates.log` file in the table directory. Every
6
+ * logged update batch is a self-committing frame; readers overlay frames on the
7
+ * base columns by stable record id.
8
+ *
9
+ * Frame layout (binary, little-endian):
10
+ *
11
+ * <u32 payloadLength><u32 crc32(payload)><payload>
12
+ * payload = JSON: { "gen": number, "records": UpdateRecord[] }
13
+ *
14
+ * The payload ends with a `\n` and is included in both the length and the
15
+ * checksum, so a frame is unambiguous and self-delimiting. `crc32` is computed
16
+ * over the exact bytes (Node's zlib.crc32 where available, with a portable
17
+ * fallback for older 22.x runtimes).
18
+ *
19
+ * Recovery contract (see plan §2):
20
+ * - the committed boundary is the contiguous run of valid frames starting at
21
+ * `baseGen + 1`; a frame is committed once its bytes are complete and its
22
+ * checksum validates;
23
+ * - a failing frame at end-of-file (no bytes after it) is a torn tail and is
24
+ * truncated before the next append;
25
+ * - a failing frame with valid-looking bytes after it is mid-log corruption and
26
+ * is surfaced as {@link UpdateLogCorruptionError} — never silently skipped;
27
+ * - `gen` must be strictly increasing by exactly one per frame.
28
+ */
29
+ export declare const UPDATES_LOG_NAME = "updates.log";
30
+ /** Default hard cap for one encoded frame. Oversized batches are split. */
31
+ export declare const DEFAULT_MAX_FRAME_BYTES: number;
32
+ export interface UpdateRecord {
33
+ /** Stable record id (as stored in the base `id` column). */
34
+ id: string | number;
35
+ /** Encoded column values, exactly as written by `formatData`. */
36
+ cols: Record<string, unknown>;
37
+ updatedAt: number;
38
+ }
39
+ export interface UpdateFrame {
40
+ /** Committed generation; strictly `baseGen + 1, + 2, ...` and never reused. */
41
+ gen: number;
42
+ records: UpdateRecord[];
43
+ }
44
+ export interface ScanResult {
45
+ /** Byte offset just past the last valid frame. */
46
+ committedBytes: number;
47
+ /** True when trailing bytes did not form a complete valid frame. */
48
+ tornTail: boolean;
49
+ frameCount: number;
50
+ /** Last committed generation, or `firstGen - 1` when no frame is valid. */
51
+ lastGen: number;
52
+ }
53
+ /** Raised for mid-log corruption (a reliable frame is followed by more bytes). */
54
+ export declare class UpdateLogCorruptionError extends Error {
55
+ readonly path: string;
56
+ readonly offset: number;
57
+ readonly reason: string;
58
+ constructor(path: string, offset: number, reason: string);
59
+ }
60
+ /** Raised when a single record cannot fit in one bounded frame. */
61
+ export declare class UpdateLogFrameTooLargeError extends Error {
62
+ readonly path: string;
63
+ readonly size: number;
64
+ readonly maxFrameBytes: number;
65
+ constructor(path: string, size: number, maxFrameBytes: number);
66
+ }
67
+ export declare const encodeFrame: (frame: UpdateFrame) => Buffer;
68
+ /**
69
+ * Scan a fully-buffered log. `firstGen` is `baseGen + 1`. Throws
70
+ * {@link UpdateLogCorruptionError} for mid-log corruption; reports a torn tail
71
+ * through {@link ScanResult} instead of throwing.
72
+ */
73
+ export declare const scanFrames: (buffer: Buffer, firstGen: number, maxFrameBytes?: number, path?: string) => ScanResult;
74
+ /**
75
+ * Split records into bounded frames starting at `firstGen`. A record larger
76
+ * than the cap on its own is rejected (it cannot be logged safely).
77
+ */
78
+ export declare const buildFrames: (records: UpdateRecord[], firstGen: number, maxFrameBytes?: number, path?: string) => UpdateFrame[];
79
+ /**
80
+ * Append-only handle. One instance per open table; callers must hold the table
81
+ * writer lock around writes and repairs.
82
+ */
83
+ export declare class UpdateLog {
84
+ readonly path: string;
85
+ constructor(tablePath: string);
86
+ /**
87
+ * Append one frame and make it durable. On a non-durable error the frame may
88
+ * or may not have reached disk (unknown outcome); callers must not ack.
89
+ *
90
+ * The handle is opened and closed per call: the append handle would
91
+ * otherwise live inside the process-wide overlay cache and be collected by
92
+ * GC of a long-idle table, closing the descriptor asynchronously.
93
+ */
94
+ append(frame: UpdateFrame): Promise<void>;
95
+ /** Append a whole batch, splitting into bounded frames. Returns the last gen. */
96
+ appendBatch(records: UpdateRecord[], firstGen: number, maxFrameBytes?: number): Promise<number>;
97
+ /**
98
+ * Stream every valid frame from `firstGen` in order, calling `onFrame` for
99
+ * each. The log is never loaded into one buffer, so replay is bounded by
100
+ * frame size rather than log size. Returns the committed boundary and
101
+ * whether a torn tail remains.
102
+ */
103
+ scan(firstGen: number, maxFrameBytes?: number, onFrame?: (frame: UpdateFrame) => void | Promise<void>): Promise<ScanResult>;
104
+ /**
105
+ * Writer-only: scan, and when a torn tail is present truncate to the last
106
+ * valid boundary so future frames are never written behind unreadable bytes.
107
+ * Returns the committed generation. Throws on mid-log corruption.
108
+ */
109
+ repairTail(firstGen: number, maxFrameBytes?: number, onFrame?: (frame: UpdateFrame) => void | Promise<void>): Promise<ScanResult>;
110
+ dispose(): Promise<void>;
111
+ /** Current on-disk size of the log (0 when absent); drives backpressure. */
112
+ bytes(): Promise<number>;
113
+ }
114
+ /** True when a table directory contains an update log. */
115
+ export declare const updateLogExists: (tablePath: string) => Promise<boolean>;
@@ -0,0 +1,377 @@
1
+ import { open, stat } from "node:fs/promises";
2
+ import { join } from "node:path";
3
+ import * as zlib from "node:zlib";
4
+ /**
5
+ * Append-only update log for incremental `put` writes.
6
+ *
7
+ * A table with the format-2 update log stores base columns exactly as before
8
+ * (`.txt` / `.gz`) plus one `updates.log` file in the table directory. Every
9
+ * logged update batch is a self-committing frame; readers overlay frames on the
10
+ * base columns by stable record id.
11
+ *
12
+ * Frame layout (binary, little-endian):
13
+ *
14
+ * <u32 payloadLength><u32 crc32(payload)><payload>
15
+ * payload = JSON: { "gen": number, "records": UpdateRecord[] }
16
+ *
17
+ * The payload ends with a `\n` and is included in both the length and the
18
+ * checksum, so a frame is unambiguous and self-delimiting. `crc32` is computed
19
+ * over the exact bytes (Node's zlib.crc32 where available, with a portable
20
+ * fallback for older 22.x runtimes).
21
+ *
22
+ * Recovery contract (see plan §2):
23
+ * - the committed boundary is the contiguous run of valid frames starting at
24
+ * `baseGen + 1`; a frame is committed once its bytes are complete and its
25
+ * checksum validates;
26
+ * - a failing frame at end-of-file (no bytes after it) is a torn tail and is
27
+ * truncated before the next append;
28
+ * - a failing frame with valid-looking bytes after it is mid-log corruption and
29
+ * is surfaced as {@link UpdateLogCorruptionError} — never silently skipped;
30
+ * - `gen` must be strictly increasing by exactly one per frame.
31
+ */
32
+ export const UPDATES_LOG_NAME = "updates.log";
33
+ /** Default hard cap for one encoded frame. Oversized batches are split. */
34
+ export const DEFAULT_MAX_FRAME_BYTES = 16 * 1024 * 1024;
35
+ /** u32 length + u32 crc32. */
36
+ const HEADER_BYTES = 8;
37
+ const DURABLE = (process.env.INIBASE_DURABILITY ?? "full") !== "none";
38
+ const maybeSync = async (handle) => {
39
+ if (DURABLE)
40
+ await handle.sync();
41
+ };
42
+ /**
43
+ * Portable CRC-32 (IEEE 802.3, reflected, poly 0xEDB88320). Used only when the
44
+ * runtime does not expose `zlib.crc32` (added in Node 22.2).
45
+ */
46
+ const crc32Fallback = (() => {
47
+ const table = new Uint32Array(256);
48
+ for (let i = 0; i < 256; i++) {
49
+ let c = i;
50
+ for (let k = 0; k < 8; k++)
51
+ c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
52
+ table[i] = c >>> 0;
53
+ }
54
+ return (buffer) => {
55
+ let crc = 0xffffffff;
56
+ for (let i = 0; i < buffer.length; i++)
57
+ crc = table[(crc ^ buffer[i]) & 0xff] ^ (crc >>> 8);
58
+ return (crc ^ 0xffffffff) >>> 0;
59
+ };
60
+ })();
61
+ const zlibCrc32 = zlib
62
+ .crc32;
63
+ const checksum = (buffer) => (typeof zlibCrc32 === "function"
64
+ ? zlibCrc32(buffer)
65
+ : crc32Fallback(buffer)) >>> 0;
66
+ /** Raised for mid-log corruption (a reliable frame is followed by more bytes). */
67
+ export class UpdateLogCorruptionError extends Error {
68
+ path;
69
+ offset;
70
+ reason;
71
+ constructor(path, offset, reason) {
72
+ super(`UPDATE_LOG_CORRUPT: ${path} @${offset} (${reason})`);
73
+ this.name = "UPDATE_LOG_CORRUPT";
74
+ this.path = path;
75
+ this.offset = offset;
76
+ this.reason = reason;
77
+ }
78
+ }
79
+ /** Raised when a single record cannot fit in one bounded frame. */
80
+ export class UpdateLogFrameTooLargeError extends Error {
81
+ path;
82
+ size;
83
+ maxFrameBytes;
84
+ constructor(path, size, maxFrameBytes) {
85
+ super(`UPDATE_LOG_FRAME_TOO_LARGE: ${path} needs ${size} bytes, cap ${maxFrameBytes}`);
86
+ this.name = "UPDATE_LOG_FRAME_TOO_LARGE";
87
+ this.path = path;
88
+ this.size = size;
89
+ this.maxFrameBytes = maxFrameBytes;
90
+ }
91
+ }
92
+ const framePayload = (frame) => Buffer.from(`${JSON.stringify(frame)}\n`, "utf8");
93
+ export const encodeFrame = (frame) => {
94
+ const payload = framePayload(frame);
95
+ const header = Buffer.allocUnsafe(HEADER_BYTES);
96
+ header.writeUInt32LE(payload.length, 0);
97
+ header.writeUInt32LE(checksum(payload), 4);
98
+ return Buffer.concat([header, payload]);
99
+ };
100
+ const isRecord = (value) => typeof value === "object" &&
101
+ value !== null &&
102
+ "id" in value &&
103
+ "cols" in value &&
104
+ typeof value.cols === "object" &&
105
+ value.cols !== null;
106
+ /**
107
+ * Validate one decoded frame. Returns the parsed frame or throws a corruption
108
+ * error when the shape is wrong (this is never a torn tail — the bytes were
109
+ * complete and checksum-valid, so the content is untrustworthy).
110
+ */
111
+ const parseFramePayload = (payload, expectedGen, path, offset) => {
112
+ let parsed;
113
+ try {
114
+ parsed = JSON.parse(payload.toString("utf8"));
115
+ }
116
+ catch {
117
+ throw new UpdateLogCorruptionError(path, offset, "payload is not JSON");
118
+ }
119
+ if (typeof parsed !== "object" || parsed === null)
120
+ throw new UpdateLogCorruptionError(path, offset, "payload is not an object");
121
+ const frame = parsed;
122
+ if (frame.gen !== expectedGen)
123
+ throw new UpdateLogCorruptionError(path, offset, `generation ${frame.gen} out of order (expected ${expectedGen})`);
124
+ if (!Array.isArray(frame.records) || !frame.records.every(isRecord))
125
+ throw new UpdateLogCorruptionError(path, offset, "records are malformed");
126
+ return frame;
127
+ };
128
+ /**
129
+ * Scan a fully-buffered log. `firstGen` is `baseGen + 1`. Throws
130
+ * {@link UpdateLogCorruptionError} for mid-log corruption; reports a torn tail
131
+ * through {@link ScanResult} instead of throwing.
132
+ */
133
+ export const scanFrames = (buffer, firstGen, maxFrameBytes = DEFAULT_MAX_FRAME_BYTES, path = UPDATES_LOG_NAME) => {
134
+ const frames = [];
135
+ let offset = 0;
136
+ let tornTail = false;
137
+ while (offset < buffer.length) {
138
+ const remaining = buffer.length - offset;
139
+ if (remaining < HEADER_BYTES) {
140
+ tornTail = true;
141
+ break;
142
+ }
143
+ const length = buffer.readUInt32LE(offset);
144
+ const storedCrc = buffer.readUInt32LE(offset + 4);
145
+ const frameEnd = offset + HEADER_BYTES + length;
146
+ if (length > maxFrameBytes)
147
+ throw new UpdateLogCorruptionError(path, offset, `frame length ${length} exceeds cap ${maxFrameBytes}`);
148
+ if (frameEnd > buffer.length) {
149
+ // Header complete, payload short: a write was cut off at EOF.
150
+ tornTail = true;
151
+ break;
152
+ }
153
+ const payload = buffer.subarray(offset + HEADER_BYTES, frameEnd);
154
+ if (checksum(payload) !== storedCrc) {
155
+ // A failing checksum is only a torn tail when nothing follows it.
156
+ if (frameEnd === buffer.length) {
157
+ tornTail = true;
158
+ break;
159
+ }
160
+ throw new UpdateLogCorruptionError(path, offset, "checksum mismatch");
161
+ }
162
+ frames.push(parseFramePayload(payload, firstGen + frames.length, path, offset));
163
+ offset = frameEnd;
164
+ }
165
+ return {
166
+ committedBytes: offset,
167
+ tornTail,
168
+ frameCount: frames.length,
169
+ lastGen: firstGen - 1 + frames.length,
170
+ };
171
+ };
172
+ /**
173
+ * Split records into bounded frames starting at `firstGen`. A record larger
174
+ * than the cap on its own is rejected (it cannot be logged safely).
175
+ */
176
+ export const buildFrames = (records, firstGen, maxFrameBytes = DEFAULT_MAX_FRAME_BYTES, path = UPDATES_LOG_NAME) => {
177
+ const frames = [];
178
+ let current = [];
179
+ const sizeOf = (gen, recs) => framePayload({ gen, records: recs }).length + HEADER_BYTES;
180
+ for (const record of records) {
181
+ const alone = sizeOf(firstGen + frames.length + (current.length ? 1 : 0), [
182
+ record,
183
+ ]);
184
+ if (alone > maxFrameBytes)
185
+ throw new UpdateLogFrameTooLargeError(path, alone, maxFrameBytes);
186
+ const candidate = sizeOf(firstGen + frames.length, [...current, record]);
187
+ if (current.length && candidate > maxFrameBytes) {
188
+ frames.push({ gen: firstGen + frames.length, records: current });
189
+ current = [];
190
+ }
191
+ current.push(record);
192
+ }
193
+ if (current.length)
194
+ frames.push({ gen: firstGen + frames.length, records: current });
195
+ return frames;
196
+ };
197
+ /**
198
+ * Append-only handle. One instance per open table; callers must hold the table
199
+ * writer lock around writes and repairs.
200
+ */
201
+ export class UpdateLog {
202
+ path;
203
+ constructor(tablePath) {
204
+ this.path = join(tablePath, UPDATES_LOG_NAME);
205
+ }
206
+ /**
207
+ * Append one frame and make it durable. On a non-durable error the frame may
208
+ * or may not have reached disk (unknown outcome); callers must not ack.
209
+ *
210
+ * The handle is opened and closed per call: the append handle would
211
+ * otherwise live inside the process-wide overlay cache and be collected by
212
+ * GC of a long-idle table, closing the descriptor asynchronously.
213
+ */
214
+ async append(frame) {
215
+ const handle = await open(this.path, "a");
216
+ try {
217
+ await handle.write(encodeFrame(frame));
218
+ await maybeSync(handle);
219
+ }
220
+ finally {
221
+ await handle.close().catch(() => { });
222
+ }
223
+ }
224
+ /** Append a whole batch, splitting into bounded frames. Returns the last gen. */
225
+ async appendBatch(records, firstGen, maxFrameBytes = DEFAULT_MAX_FRAME_BYTES) {
226
+ const frames = buildFrames(records, firstGen, maxFrameBytes, this.path);
227
+ for (const frame of frames)
228
+ await this.append(frame);
229
+ return firstGen - 1 + frames.length;
230
+ }
231
+ /**
232
+ * Stream every valid frame from `firstGen` in order, calling `onFrame` for
233
+ * each. The log is never loaded into one buffer, so replay is bounded by
234
+ * frame size rather than log size. Returns the committed boundary and
235
+ * whether a torn tail remains.
236
+ */
237
+ async scan(firstGen, maxFrameBytes = DEFAULT_MAX_FRAME_BYTES, onFrame) {
238
+ let handle;
239
+ try {
240
+ handle = await open(this.path, "r");
241
+ }
242
+ catch (error) {
243
+ if (error?.code === "ENOENT")
244
+ return {
245
+ committedBytes: 0,
246
+ tornTail: false,
247
+ frameCount: 0,
248
+ lastGen: firstGen - 1,
249
+ };
250
+ throw error;
251
+ }
252
+ const chunk = Buffer.allocUnsafe(64 * 1024);
253
+ let pending = Buffer.alloc(0);
254
+ let committedBytes = 0;
255
+ let frameCount = 0;
256
+ let tornTail = false;
257
+ try {
258
+ for (;;) {
259
+ const { bytesRead } = await handle.read(chunk, 0, chunk.length, null);
260
+ if (bytesRead === 0)
261
+ break;
262
+ pending =
263
+ pending.length === 0
264
+ ? Buffer.from(chunk.subarray(0, bytesRead))
265
+ : Buffer.concat([pending, chunk.subarray(0, bytesRead)]);
266
+ // Consume every complete, valid frame currently buffered.
267
+ let offset = 0;
268
+ while (pending.length - offset >= HEADER_BYTES) {
269
+ const length = pending.readUInt32LE(offset);
270
+ const storedCrc = pending.readUInt32LE(offset + 4);
271
+ if (length > maxFrameBytes)
272
+ throw new UpdateLogCorruptionError(this.path, offset, `frame length ${length} exceeds cap ${maxFrameBytes}`);
273
+ const frameEnd = offset + HEADER_BYTES + length;
274
+ if (frameEnd > pending.length)
275
+ break; // wait for more bytes
276
+ const payload = pending.subarray(offset + HEADER_BYTES, frameEnd);
277
+ if (checksum(payload) !== storedCrc)
278
+ break; // decide at EOF
279
+ const frame = parseFramePayload(payload, firstGen + frameCount, this.path, committedBytes);
280
+ if (onFrame)
281
+ await onFrame(frame);
282
+ frameCount++;
283
+ offset = frameEnd;
284
+ committedBytes += HEADER_BYTES + length;
285
+ }
286
+ pending = pending.subarray(offset);
287
+ }
288
+ // Leftover bytes: a partial header/payload, or a checksum failure.
289
+ // If a frame's bytes all fit but the checksum failed and nothing is
290
+ // left after it, it is a torn tail. Anything else is corruption.
291
+ if (pending.length > 0) {
292
+ if (pending.length < HEADER_BYTES)
293
+ tornTail = true;
294
+ else {
295
+ const length = pending.readUInt32LE(0);
296
+ const storedCrc = pending.readUInt32LE(4);
297
+ const frameEnd = HEADER_BYTES + length;
298
+ if (length > maxFrameBytes)
299
+ throw new UpdateLogCorruptionError(this.path, committedBytes, `frame length ${length} exceeds cap ${maxFrameBytes}`);
300
+ if (frameEnd > pending.length)
301
+ tornTail = true;
302
+ else {
303
+ const payload = pending.subarray(HEADER_BYTES, frameEnd);
304
+ if (checksum(payload) !== storedCrc) {
305
+ if (frameEnd === pending.length)
306
+ tornTail = true;
307
+ else
308
+ throw new UpdateLogCorruptionError(this.path, committedBytes, "checksum mismatch");
309
+ }
310
+ else {
311
+ // Valid frame that only now fit: it is not corruption.
312
+ const frame = parseFramePayload(payload, firstGen + frameCount, this.path, committedBytes);
313
+ if (onFrame)
314
+ await onFrame(frame);
315
+ frameCount++;
316
+ committedBytes += HEADER_BYTES + length;
317
+ if (pending.length > frameEnd)
318
+ tornTail = true;
319
+ }
320
+ }
321
+ }
322
+ }
323
+ }
324
+ finally {
325
+ await handle.close().catch(() => { });
326
+ }
327
+ return {
328
+ committedBytes,
329
+ tornTail,
330
+ frameCount,
331
+ lastGen: firstGen - 1 + frameCount,
332
+ };
333
+ }
334
+ /**
335
+ * Writer-only: scan, and when a torn tail is present truncate to the last
336
+ * valid boundary so future frames are never written behind unreadable bytes.
337
+ * Returns the committed generation. Throws on mid-log corruption.
338
+ */
339
+ async repairTail(firstGen, maxFrameBytes = DEFAULT_MAX_FRAME_BYTES, onFrame) {
340
+ const result = await this.scan(firstGen, maxFrameBytes, onFrame);
341
+ if (!result.tornTail)
342
+ return result;
343
+ const handle = await open(this.path, "r+");
344
+ try {
345
+ await handle.truncate(result.committedBytes);
346
+ await maybeSync(handle);
347
+ }
348
+ finally {
349
+ await handle.close().catch(() => { });
350
+ }
351
+ return result;
352
+ }
353
+ async dispose() {
354
+ // No persistent handle is held; kept for API symmetry so callers can
355
+ // release a log deterministically.
356
+ }
357
+ /** Current on-disk size of the log (0 when absent); drives backpressure. */
358
+ async bytes() {
359
+ try {
360
+ return (await stat(this.path)).size;
361
+ }
362
+ catch {
363
+ return 0;
364
+ }
365
+ }
366
+ }
367
+ /** True when a table directory contains an update log. */
368
+ export const updateLogExists = async (tablePath) => {
369
+ try {
370
+ const handle = await open(join(tablePath, UPDATES_LOG_NAME), "r");
371
+ await handle.close();
372
+ return true;
373
+ }
374
+ catch {
375
+ return false;
376
+ }
377
+ };