@effected/jsonl 0.18.1 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Envelope.js +1 -1
- package/Journal.js +1 -1
- package/JournalFile.js +320 -0
- package/JsonlError.js +1 -1
- package/Line.js +1 -1
- package/NodeJournalWatcher.js +1 -1
- package/README.md +31 -31
- package/index.d.ts +32 -0
- package/internal/engine.js +1 -1
- package/internal/tail.js +1 -1
- package/node.d.ts +177 -1
- package/node.js +2 -1
- package/package.json +1 -1
package/Envelope.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
+
import { DateTime, Option, Result, Schema } from "effect";
|
|
1
2
|
import { InvalidData, UnknownEvent, UnserializableData } from "./JsonlError.js";
|
|
2
3
|
import { Line, isBlank } from "./Line.js";
|
|
3
|
-
import { DateTime, Option, Result, Schema } from "effect";
|
|
4
4
|
|
|
5
5
|
// src/Envelope.ts
|
|
6
6
|
/**
|
package/Journal.js
CHANGED
package/JournalFile.js
ADDED
|
@@ -0,0 +1,320 @@
|
|
|
1
|
+
import { Buffer } from "node:buffer";
|
|
2
|
+
import { Option, PlatformError, Result, Schema } from "effect";
|
|
3
|
+
import * as NFS from "node:fs";
|
|
4
|
+
import { Envelope } from "./Envelope.js";
|
|
5
|
+
import { JournalNotFound } from "./JsonlError.js";
|
|
6
|
+
import { Line } from "./Line.js";
|
|
7
|
+
|
|
8
|
+
// src/JournalFile.ts
|
|
9
|
+
/** UTF-8 BOM, as bytes. */
|
|
10
|
+
const BOM = [
|
|
11
|
+
239,
|
|
12
|
+
187,
|
|
13
|
+
191
|
|
14
|
+
];
|
|
15
|
+
/** Byte value of `\n`. Cannot occur inside a UTF-8 multi-byte sequence. */
|
|
16
|
+
const LF = 10;
|
|
17
|
+
/** The first tail window, as the service's `latest` starts. */
|
|
18
|
+
const DEFAULT_WINDOW = 8192;
|
|
19
|
+
/** The largest tail window, as the service clamps. Only one line longer than it is read whole. */
|
|
20
|
+
const MAX_WINDOW = 1048576;
|
|
21
|
+
/** How many bytes before a cursor's `end` it keeps, to tell an append from a rewrite that kept the boundary. */
|
|
22
|
+
const TAIL_BYTES = 64;
|
|
23
|
+
const decoder = new TextDecoder("utf-8", { ignoreBOM: true });
|
|
24
|
+
/**
|
|
25
|
+
* Where an incremental read of a journal file resumes, and the identity of the
|
|
26
|
+
* file it was taken from.
|
|
27
|
+
*
|
|
28
|
+
* @remarks
|
|
29
|
+
* A plain JSON-round-trippable record: persisting it between runs is the
|
|
30
|
+
* caller's job. `end` is the logical byte offset just past the last complete
|
|
31
|
+
* line read, the same number as a service envelope's `position.end`. `size`
|
|
32
|
+
* is the logical size the read saw, `identity` the file's `dev:ino` and
|
|
33
|
+
* `mtime` its modification time in nanoseconds, both as decimal strings so a
|
|
34
|
+
* 64-bit inode survives JSON exactly. `tail` is the last 64 bytes before
|
|
35
|
+
* `end` (fewer when `end` is smaller), base64: a resume whose bytes there
|
|
36
|
+
* differ was rewritten, not appended to.
|
|
37
|
+
*
|
|
38
|
+
* @public
|
|
39
|
+
*/
|
|
40
|
+
const JournalCursor = Schema.Struct({
|
|
41
|
+
/** Logical byte offset just past the last complete line read: where the next read starts. */
|
|
42
|
+
end: Schema.Number,
|
|
43
|
+
/** The logical size of the file when the cursor was taken. */
|
|
44
|
+
size: Schema.Number,
|
|
45
|
+
/** The file's `dev:ino`, in decimal. */
|
|
46
|
+
identity: Schema.String,
|
|
47
|
+
/** The file's modification time in nanoseconds, in decimal. */
|
|
48
|
+
mtime: Schema.String,
|
|
49
|
+
/** The last bytes before `end`, base64. */
|
|
50
|
+
tail: Schema.String
|
|
51
|
+
});
|
|
52
|
+
/** The `SystemErrorTag` an errno code maps to, after core's Node filesystem mapping. */
|
|
53
|
+
const reasonOf = (code) => {
|
|
54
|
+
switch (code) {
|
|
55
|
+
case "ENOENT": return "NotFound";
|
|
56
|
+
case "EACCES":
|
|
57
|
+
case "EPERM": return "PermissionDenied";
|
|
58
|
+
case "EISDIR":
|
|
59
|
+
case "ENOTDIR":
|
|
60
|
+
case "ELOOP": return "BadResource";
|
|
61
|
+
default: return "Unknown";
|
|
62
|
+
}
|
|
63
|
+
};
|
|
64
|
+
/** A thrown `node:fs` error as the result it becomes: `JournalNotFound` for a missing path. */
|
|
65
|
+
const failure = (path, method, cause) => {
|
|
66
|
+
const code = cause?.code;
|
|
67
|
+
if (code === "ENOENT") return Result.fail(new JournalNotFound({ path }));
|
|
68
|
+
return Result.fail(PlatformError.systemError({
|
|
69
|
+
_tag: reasonOf(code),
|
|
70
|
+
module: "FileSystem",
|
|
71
|
+
method,
|
|
72
|
+
pathOrDescriptor: path,
|
|
73
|
+
syscall: method,
|
|
74
|
+
cause
|
|
75
|
+
}));
|
|
76
|
+
};
|
|
77
|
+
/** Physical bytes `[from, to)` of an open file, read whole. */
|
|
78
|
+
const readAt = (fd, from, to) => {
|
|
79
|
+
const bytes = new Uint8Array(Math.max(0, to - from));
|
|
80
|
+
let filled = 0;
|
|
81
|
+
while (filled < bytes.length) {
|
|
82
|
+
const read = NFS.readSync(fd, bytes, filled, bytes.length - filled, from + filled);
|
|
83
|
+
if (read === 0) return bytes.subarray(0, filled);
|
|
84
|
+
filled += read;
|
|
85
|
+
}
|
|
86
|
+
return bytes;
|
|
87
|
+
};
|
|
88
|
+
/** Bytes of leading BOM in a file of physical `size`. */
|
|
89
|
+
const bomBytes = (fd, size) => {
|
|
90
|
+
if (size < BOM.length) return 0;
|
|
91
|
+
const head = readAt(fd, 0, BOM.length);
|
|
92
|
+
return head[0] === BOM[0] && head[1] === BOM[1] && head[2] === BOM[2] ? BOM.length : 0;
|
|
93
|
+
};
|
|
94
|
+
/** Open `path`, run `body` over it, and close it, with every `node:fs` throw as a typed failure. */
|
|
95
|
+
const withFile = (path, method, body) => {
|
|
96
|
+
let fd;
|
|
97
|
+
try {
|
|
98
|
+
fd = NFS.openSync(path, "r");
|
|
99
|
+
} catch (cause) {
|
|
100
|
+
return failure(path, "open", cause);
|
|
101
|
+
}
|
|
102
|
+
try {
|
|
103
|
+
return Result.succeed(body(fd, NFS.fstatSync(fd, { bigint: true })));
|
|
104
|
+
} catch (cause) {
|
|
105
|
+
return failure(path, method, cause);
|
|
106
|
+
} finally {
|
|
107
|
+
NFS.closeSync(fd);
|
|
108
|
+
}
|
|
109
|
+
};
|
|
110
|
+
const toBase64 = (bytes) => Buffer.from(bytes).toString("base64");
|
|
111
|
+
/** Where a read with `cursor` starts in a file now at logical `size`, and why. */
|
|
112
|
+
const startOf = (size, identity, mtime, cursor) => {
|
|
113
|
+
if (cursor === void 0 || !Number.isSafeInteger(cursor.end) || cursor.end < 0) return {
|
|
114
|
+
from: 0,
|
|
115
|
+
reset: "initial"
|
|
116
|
+
};
|
|
117
|
+
if (cursor.identity !== identity) return {
|
|
118
|
+
from: 0,
|
|
119
|
+
reset: "replaced"
|
|
120
|
+
};
|
|
121
|
+
if (size < cursor.size || size < cursor.end) return {
|
|
122
|
+
from: 0,
|
|
123
|
+
reset: "truncated"
|
|
124
|
+
};
|
|
125
|
+
if (size === cursor.size && mtime !== cursor.mtime) return {
|
|
126
|
+
from: 0,
|
|
127
|
+
reset: "rewritten"
|
|
128
|
+
};
|
|
129
|
+
return {
|
|
130
|
+
from: cursor.end,
|
|
131
|
+
reset: "none"
|
|
132
|
+
};
|
|
133
|
+
};
|
|
134
|
+
/**
|
|
135
|
+
* The window of at most {@link MAX_WINDOW} bytes ending at physical `end`,
|
|
136
|
+
* decoded from a line boundary: the leading partial line is dropped through
|
|
137
|
+
* the first `\n`. A window with no boundary inside it holds part of one line
|
|
138
|
+
* longer than the window, which is read whole instead. The same reads as the
|
|
139
|
+
* service's `readWindow` and `readLineEnding`.
|
|
140
|
+
*/
|
|
141
|
+
const tailWindow = (fd, end, window, bom) => {
|
|
142
|
+
const from = Math.max(bom, end - Math.min(window, MAX_WINDOW));
|
|
143
|
+
const bytes = readAt(fd, from, end);
|
|
144
|
+
if (from === bom) return {
|
|
145
|
+
text: decoder.decode(bytes),
|
|
146
|
+
start: 0,
|
|
147
|
+
atFileStart: true
|
|
148
|
+
};
|
|
149
|
+
const newline = bytes.indexOf(LF);
|
|
150
|
+
const cut = newline === -1 ? bytes.length : newline + 1;
|
|
151
|
+
if (cut < bytes.length) return {
|
|
152
|
+
text: decoder.decode(bytes.subarray(cut)),
|
|
153
|
+
start: from + cut - bom,
|
|
154
|
+
atFileStart: false
|
|
155
|
+
};
|
|
156
|
+
return lineEnding(fd, end, bom);
|
|
157
|
+
};
|
|
158
|
+
/** The one line that ends at physical `end`, however long, read back in clamped steps to the `\n` before it. */
|
|
159
|
+
const lineEnding = (fd, end, bom) => {
|
|
160
|
+
const chunks = [];
|
|
161
|
+
let position = end;
|
|
162
|
+
for (;;) {
|
|
163
|
+
const from = Math.max(bom, position - MAX_WINDOW);
|
|
164
|
+
const bytes = readAt(fd, from, position);
|
|
165
|
+
const searchFrom = position === end ? bytes.length - 2 : bytes.length - 1;
|
|
166
|
+
const newline = searchFrom < 0 ? -1 : bytes.lastIndexOf(LF, searchFrom);
|
|
167
|
+
if (newline !== -1) {
|
|
168
|
+
chunks.unshift(bytes.subarray(newline + 1));
|
|
169
|
+
return {
|
|
170
|
+
text: decoder.decode(join(chunks)),
|
|
171
|
+
start: from + newline + 1 - bom,
|
|
172
|
+
atFileStart: false
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
chunks.unshift(bytes);
|
|
176
|
+
if (from === bom) return {
|
|
177
|
+
text: decoder.decode(join(chunks)),
|
|
178
|
+
start: 0,
|
|
179
|
+
atFileStart: true
|
|
180
|
+
};
|
|
181
|
+
position = from;
|
|
182
|
+
}
|
|
183
|
+
};
|
|
184
|
+
const join = (chunks) => {
|
|
185
|
+
const out = new Uint8Array(chunks.reduce((total, chunk) => total + chunk.length, 0));
|
|
186
|
+
let offset = 0;
|
|
187
|
+
for (const chunk of chunks) {
|
|
188
|
+
out.set(chunk, offset);
|
|
189
|
+
offset += chunk.length;
|
|
190
|
+
}
|
|
191
|
+
return out;
|
|
192
|
+
};
|
|
193
|
+
/**
|
|
194
|
+
* Synchronous reads of a journal file over `node:fs`, for a consumer with no
|
|
195
|
+
* Effect runtime.
|
|
196
|
+
*
|
|
197
|
+
* @remarks
|
|
198
|
+
* Nothing here builds an `Effect` or needs a layer, and nothing throws: a
|
|
199
|
+
* missing path fails `JournalNotFound`, any other filesystem refusal a core
|
|
200
|
+
* `PlatformError`, both in a `Result`. Offsets are logical bytes, past any
|
|
201
|
+
* BOM, exactly as the `Journal` service reports them, so a cursor from either
|
|
202
|
+
* side reads on in the other.
|
|
203
|
+
*
|
|
204
|
+
* @example
|
|
205
|
+
* ```ts
|
|
206
|
+
* import type { JournalCursor } from "@effected/jsonl/node";
|
|
207
|
+
* import { JournalFile } from "@effected/jsonl/node";
|
|
208
|
+
* import { Result } from "effect";
|
|
209
|
+
*
|
|
210
|
+
* declare const saved: JournalCursor | undefined;
|
|
211
|
+
*
|
|
212
|
+
* const read = JournalFile.readFromResult(".claude/dogfood/mailbox.jsonl", saved);
|
|
213
|
+
* if (Result.isSuccess(read)) {
|
|
214
|
+
* // Any reset reads from 0: drop what earlier reads built, or it is applied twice.
|
|
215
|
+
* if (read.success.reset !== "none") {
|
|
216
|
+
* // reset the state you derived from earlier lines
|
|
217
|
+
* }
|
|
218
|
+
* for (const line of read.success.lines) line.text; // each complete line, in order
|
|
219
|
+
* read.success.cursor; // persist it for the next run
|
|
220
|
+
* }
|
|
221
|
+
* ```
|
|
222
|
+
*
|
|
223
|
+
* @public
|
|
224
|
+
*/
|
|
225
|
+
var JournalFile = class {
|
|
226
|
+
constructor() {}
|
|
227
|
+
/**
|
|
228
|
+
* Read the lines appended to `path` since `cursor`.
|
|
229
|
+
*
|
|
230
|
+
* @remarks
|
|
231
|
+
* Reads the bytes from the cursor's `end` to the file's current size and
|
|
232
|
+
* returns every complete line among them; an unterminated final line is held
|
|
233
|
+
* back and read again, whole, once its `\n` lands. Without a cursor, or when
|
|
234
|
+
* the file was truncated, replaced or rewritten since the cursor was taken
|
|
235
|
+
* (see {@link JournalReset}), it reads from the start, and `reset` says why.
|
|
236
|
+
* The read is one buffer from the start offset to the end of the file, so a
|
|
237
|
+
* first read of a large journal costs its size; every read after that costs
|
|
238
|
+
* only what was appended.
|
|
239
|
+
*
|
|
240
|
+
* Line offsets are byte-exact for UTF-8 text. A byte sequence that is not
|
|
241
|
+
* UTF-8 decodes as U+FFFD, and the offsets of the lines after it in the same
|
|
242
|
+
* read drift; the cursor's `end` never does.
|
|
243
|
+
*
|
|
244
|
+
* @param path - The journal file.
|
|
245
|
+
* @param cursor - The cursor the previous read returned, if any.
|
|
246
|
+
* @returns What was read, or `JournalNotFound` when `path` names nothing,
|
|
247
|
+
* or the `PlatformError` the filesystem raised.
|
|
248
|
+
*/
|
|
249
|
+
static readFromResult(path, cursor) {
|
|
250
|
+
return withFile(path, "read", (fd, info) => {
|
|
251
|
+
const physical = Number(info.size);
|
|
252
|
+
const bom = bomBytes(fd, physical);
|
|
253
|
+
const size = physical - bom;
|
|
254
|
+
const identity = `${info.dev}:${info.ino}`;
|
|
255
|
+
const mtime = `${info.mtimeNs}`;
|
|
256
|
+
let { from, reset } = startOf(size, identity, mtime, cursor);
|
|
257
|
+
let back = Math.min(from, TAIL_BYTES);
|
|
258
|
+
let buffer = readAt(fd, bom + from - back, physical);
|
|
259
|
+
if (reset === "none" && cursor !== void 0 && toBase64(buffer.subarray(0, back)) !== cursor.tail) {
|
|
260
|
+
from = 0;
|
|
261
|
+
reset = "rewritten";
|
|
262
|
+
back = 0;
|
|
263
|
+
buffer = readAt(fd, bom, physical);
|
|
264
|
+
}
|
|
265
|
+
const bytes = buffer.subarray(back);
|
|
266
|
+
const complete = bytes.lastIndexOf(LF) + 1;
|
|
267
|
+
const lines = complete === 0 ? [] : Line.split(decoder.decode(bytes.subarray(0, complete)), from);
|
|
268
|
+
const end = from + complete;
|
|
269
|
+
const tailEnd = back + complete;
|
|
270
|
+
return {
|
|
271
|
+
lines,
|
|
272
|
+
cursor: {
|
|
273
|
+
end,
|
|
274
|
+
size,
|
|
275
|
+
identity,
|
|
276
|
+
mtime,
|
|
277
|
+
tail: toBase64(buffer.subarray(Math.max(0, tailEnd - TAIL_BYTES), tailEnd))
|
|
278
|
+
},
|
|
279
|
+
reset
|
|
280
|
+
};
|
|
281
|
+
});
|
|
282
|
+
}
|
|
283
|
+
/**
|
|
284
|
+
* The last valid envelope in `path`: the journal's current state, read the
|
|
285
|
+
* way the service's `latest` reads it.
|
|
286
|
+
*
|
|
287
|
+
* @remarks
|
|
288
|
+
* Steps back from the end one window at a time, each window covering only
|
|
289
|
+
* bytes no earlier one did, growing from `window` by fours up to 1 MiB, and
|
|
290
|
+
* stops at the first window holding a valid envelope. A line longer than
|
|
291
|
+
* the window is read whole. So the cost tracks the answer, not the age of
|
|
292
|
+
* the journal, and `Option.none()` means the whole file holds no valid
|
|
293
|
+
* envelope.
|
|
294
|
+
*
|
|
295
|
+
* @param events - The registry the writer declared.
|
|
296
|
+
* @param path - The journal file.
|
|
297
|
+
* @param window - The first window's size in bytes. Defaults to 8 KiB.
|
|
298
|
+
* @returns The last valid envelope, `Option.none()` if there is none, or
|
|
299
|
+
* `JournalNotFound` / the `PlatformError` the filesystem raised.
|
|
300
|
+
*/
|
|
301
|
+
static lastValidResult(events, path, window = DEFAULT_WINDOW) {
|
|
302
|
+
return withFile(path, "read", (fd, info) => {
|
|
303
|
+
const physical = Number(info.size);
|
|
304
|
+
const bom = bomBytes(fd, physical);
|
|
305
|
+
let end = physical;
|
|
306
|
+
let size = Number.isFinite(window) ? Math.max(1, Math.floor(window)) : DEFAULT_WINDOW;
|
|
307
|
+
while (end > bom) {
|
|
308
|
+
const tail = tailWindow(fd, end, size, bom);
|
|
309
|
+
const found = Envelope.lastValid(events, tail.text, tail.start);
|
|
310
|
+
if (Option.isSome(found) || tail.atFileStart) return found;
|
|
311
|
+
if (tail.start + bom >= end) return Option.none();
|
|
312
|
+
end = tail.start + bom;
|
|
313
|
+
size = Math.min(size * 4, MAX_WINDOW);
|
|
314
|
+
}
|
|
315
|
+
return Option.none();
|
|
316
|
+
});
|
|
317
|
+
}
|
|
318
|
+
};
|
|
319
|
+
|
|
320
|
+
export { JournalCursor, JournalFile };
|
package/JsonlError.js
CHANGED
package/Line.js
CHANGED
package/NodeJournalWatcher.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { JournalWatcher } from "./JournalWatcher.js";
|
|
2
1
|
import { Cause, Effect, Layer, PlatformError, Queue, Stream } from "effect";
|
|
3
2
|
import * as NFS from "node:fs";
|
|
3
|
+
import { JournalWatcher } from "./JournalWatcher.js";
|
|
4
4
|
|
|
5
5
|
// src/NodeJournalWatcher.ts
|
|
6
6
|
/** The `SystemErrorTag` an errno code maps to, after core's Node filesystem mapping. */
|
package/README.md
CHANGED
|
@@ -98,46 +98,46 @@ Effect.runPromise(program.pipe(Effect.provide(MailJournalLive), Effect.provide([
|
|
|
98
98
|
|
|
99
99
|
## Reading a journal with no runtime
|
|
100
100
|
|
|
101
|
-
A hook script wants one answer — what is the current state — and should not build an Effect runtime or read the history to get it.
|
|
101
|
+
A hook script wants one answer — what is the current state — and should not build an Effect runtime or read the history to get it. `JournalFile` on the `./node` entry reads the file synchronously over `node:fs`, with no layer and no runtime, and returns a `Result` instead of throwing:
|
|
102
102
|
|
|
103
103
|
```ts
|
|
104
|
-
import {
|
|
105
|
-
import { Option } from "effect";
|
|
106
|
-
import { closeSync, openSync, readSync, statSync } from "node:fs";
|
|
104
|
+
import { JournalFile } from "@effected/jsonl/node";
|
|
105
|
+
import { Option, Result } from "effect";
|
|
107
106
|
// The same registry the writer declared, exported from your own module.
|
|
108
107
|
import { events } from "./events.js";
|
|
109
108
|
|
|
110
|
-
const
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
109
|
+
const current = JournalFile.lastValidResult(events, ".claude/dogfood/mailbox.jsonl");
|
|
110
|
+
if (Result.isSuccess(current) && Option.isSome(current.success)) {
|
|
111
|
+
current.success.value.event; // the tag
|
|
112
|
+
current.success.value.position.end; // its logical byte offset, for a resumable cursor
|
|
113
|
+
}
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
`lastValidResult` reads the way the service's `latest` does: a bounded window from the end, then wider windows further back only while none holds a valid envelope. `Option.none()` means the whole file holds none. A missing file fails `JournalNotFound`.
|
|
117
|
+
|
|
118
|
+
A monitor that follows a journal across runs reads only what was appended since last time. `readFromResult` takes the cursor the previous read returned and returns the new lines with the next cursor, a plain JSON record to persist however you like:
|
|
119
|
+
|
|
120
|
+
```ts
|
|
121
|
+
import type { JournalCursor } from "@effected/jsonl/node";
|
|
122
|
+
import { JournalFile } from "@effected/jsonl/node";
|
|
123
|
+
import { Result } from "effect";
|
|
124
|
+
|
|
125
|
+
declare const saved: JournalCursor | undefined;
|
|
126
|
+
|
|
127
|
+
const read = JournalFile.readFromResult(".claude/dogfood/mailbox.jsonl", saved);
|
|
128
|
+
if (Result.isSuccess(read)) {
|
|
129
|
+
// "initial" | "none" | "truncated" | "replaced" | "rewritten". Every reset but
|
|
130
|
+
// "none" reads from offset 0: drop what earlier reads built, or it is applied twice.
|
|
131
|
+
if (read.success.reset !== "none") {
|
|
132
|
+
// reset whatever you derived from earlier lines
|
|
119
133
|
}
|
|
120
|
-
const
|
|
121
|
-
|
|
122
|
-
// A window that does not start at 0 opens mid-line — drop that fragment. With
|
|
123
|
-
// no newline anywhere in it, the whole window IS one fragment and there is
|
|
124
|
-
// nothing usable here: return nothing rather than a half a line.
|
|
125
|
-
const newline = text.indexOf("\n");
|
|
126
|
-
return newline === -1 ? "" : text.slice(newline + 1);
|
|
127
|
-
};
|
|
128
|
-
|
|
129
|
-
const current = Envelope.lastValidResult(events, readTail(".claude/dogfood/mailbox.jsonl", 8_192));
|
|
130
|
-
// Option.some(envelope) once a valid envelope is in the window.
|
|
131
|
-
// Option.none() means no valid envelope was found in the supplied text — the
|
|
132
|
-
// last line may be longer than the window, the window may hold only a torn
|
|
133
|
-
// fragment, or the journal may genuinely be empty. Widen and read again before
|
|
134
|
-
// concluding it is the last of those.
|
|
135
|
-
if (Option.isSome(current)) {
|
|
136
|
-
current.value.event; // the tag
|
|
137
|
-
current.value.line.end; // its logical byte offset, for a resumable cursor
|
|
134
|
+
for (const line of read.success.lines) line.text; // each complete line, in order
|
|
135
|
+
read.success.cursor; // persist it for the next run
|
|
138
136
|
}
|
|
139
137
|
```
|
|
140
138
|
|
|
139
|
+
A torn final line is held back until its newline lands. A file that shrank, was renamed over or was rewritten in place is read again from the start, and `reset` says which. A rewrite is seen when the size is unchanged but the modification time moved, or when the 64 bytes before the cursor differ from what the last read saw; a rewrite that grows the file and leaves those bytes alone is not.
|
|
140
|
+
|
|
141
141
|
"Last valid" always means the last valid **envelope**, never merely the last valid JSON. That distinction is what catches a torn tail: an envelope is always an object and every strict prefix of an object text is invalid JSON, whereas a torn scalar (`42` cut mid-write leaves `4`) parses cleanly as a different value that no walk-back can detect.
|
|
142
142
|
|
|
143
143
|
## Writers, watchers and what is actually guaranteed
|
package/index.d.ts
CHANGED
|
@@ -1,3 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Append-only, schema-validated JSONL journals as a definable Effect service.
|
|
3
|
+
*
|
|
4
|
+
* The subject is not the format — one JSON value per line needs no library.
|
|
5
|
+
* The subject is the file as a live object: a journal that only ever grows,
|
|
6
|
+
* whose current state is its last valid line, whose tail may be torn mid-append,
|
|
7
|
+
* and which several processes read while one writes.
|
|
8
|
+
*
|
|
9
|
+
* The pure core is synchronous and `Result`-based so a hook script can read the
|
|
10
|
+
* current state of a journal with no Effect runtime at all.
|
|
11
|
+
*
|
|
12
|
+
* @example
|
|
13
|
+
* ```ts
|
|
14
|
+
* import { Envelope, JsonlEvent } from "@effected/jsonl";
|
|
15
|
+
* import { Option, Schema } from "effect";
|
|
16
|
+
*
|
|
17
|
+
* const events = [JsonlEvent.make("state", { data: Schema.Struct({ round: Schema.Number }) })] as const;
|
|
18
|
+
*
|
|
19
|
+
* declare const sourceText: string;
|
|
20
|
+
*
|
|
21
|
+
* // The whole read path for a snapshot journal, with no runtime.
|
|
22
|
+
* const current = Envelope.lastValid(events, sourceText);
|
|
23
|
+
* if (Option.isSome(current)) {
|
|
24
|
+
* current.value.data.round; // the decoded payload of the last valid envelope
|
|
25
|
+
* current.value.position.end; // a resumable cursor
|
|
26
|
+
* }
|
|
27
|
+
* ```
|
|
28
|
+
*
|
|
29
|
+
* @see {@link https://effect.website | Effect}
|
|
30
|
+
*
|
|
31
|
+
* @packageDocumentation
|
|
32
|
+
*/
|
|
1
33
|
import { Context, DateTime, Duration, Effect, FileSystem, Layer, Option, PlatformError, Result, Schema, Scope, Stream } from "effect";
|
|
2
34
|
declare const MalformedLine_base: Schema.Class<MalformedLine, Schema.TaggedStruct<"MalformedLine", {
|
|
3
35
|
/** The offending line, with its byte offsets into the source. */
|
package/internal/engine.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { ByteSize, Channel, DateTime, Deferred, Duration, Effect, Exit, FileSystem, Option, PubSub, Result, Semaphore, Stream, SubscriptionRef } from "effect";
|
|
1
2
|
import { Envelope, completeResult, frameResult } from "../Envelope.js";
|
|
2
3
|
import { JournalWatcher } from "../JournalWatcher.js";
|
|
3
4
|
import { JournalClosed, JournalNotFound, JournalResync, TerminalViolation } from "../JsonlError.js";
|
|
@@ -5,7 +6,6 @@ import { Line, isBlank } from "../Line.js";
|
|
|
5
6
|
import { matchesFrame } from "../Slice.js";
|
|
6
7
|
import { canMerge, shallowMerge } from "./merge.js";
|
|
7
8
|
import { probeBomBytes, readLinePages, readTailUntil } from "./tail.js";
|
|
8
|
-
import { ByteSize, Channel, DateTime, Deferred, Duration, Effect, Exit, FileSystem, Option, PubSub, Result, Semaphore, Stream, SubscriptionRef } from "effect";
|
|
9
9
|
|
|
10
10
|
// src/internal/engine.ts
|
|
11
11
|
const positionOf = (item) => Result.isSuccess(item) ? item.success.position : item.failure.position;
|
package/internal/tail.js
CHANGED
|
@@ -33,7 +33,7 @@ const MAX_WINDOW = 1048576;
|
|
|
33
33
|
*/
|
|
34
34
|
const PAGE_SIZE = 65536;
|
|
35
35
|
const EMPTY = /* @__PURE__ */ new Uint8Array(0);
|
|
36
|
-
const decoder = new TextDecoder();
|
|
36
|
+
const decoder = new TextDecoder("utf-8", { ignoreBOM: true });
|
|
37
37
|
/** Join byte chunks with one allocation. */
|
|
38
38
|
const join = (chunks) => {
|
|
39
39
|
if (chunks.length === 1) return chunks[0];
|
package/node.d.ts
CHANGED
|
@@ -1,5 +1,180 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The Node backend for `@effected/jsonl`: {@link NodeJournalWatcher}, which
|
|
3
|
+
* provides the root's `JournalWatcher` over `node:fs`, and {@link JournalFile},
|
|
4
|
+
* synchronous cursor-resumed reads of a journal file for a consumer with no
|
|
5
|
+
* Effect runtime.
|
|
6
|
+
*
|
|
7
|
+
* The root never imports `node:*`; this entry is the one place the package
|
|
8
|
+
* touches a platform, and only a consumer that imports it pays for that.
|
|
9
|
+
*
|
|
10
|
+
* @packageDocumentation
|
|
11
|
+
*/
|
|
1
12
|
import * as Jsonl from "@effected/jsonl";
|
|
2
|
-
import { Layer } from "effect";
|
|
13
|
+
import { Layer, Option, PlatformError, Result, Schema } from "effect";
|
|
14
|
+
/**
|
|
15
|
+
* Where an incremental read of a journal file resumes, and the identity of the
|
|
16
|
+
* file it was taken from.
|
|
17
|
+
*
|
|
18
|
+
* @remarks
|
|
19
|
+
* A plain JSON-round-trippable record: persisting it between runs is the
|
|
20
|
+
* caller's job. `end` is the logical byte offset just past the last complete
|
|
21
|
+
* line read, the same number as a service envelope's `position.end`. `size`
|
|
22
|
+
* is the logical size the read saw, `identity` the file's `dev:ino` and
|
|
23
|
+
* `mtime` its modification time in nanoseconds, both as decimal strings so a
|
|
24
|
+
* 64-bit inode survives JSON exactly. `tail` is the last 64 bytes before
|
|
25
|
+
* `end` (fewer when `end` is smaller), base64: a resume whose bytes there
|
|
26
|
+
* differ was rewritten, not appended to.
|
|
27
|
+
*
|
|
28
|
+
* @public
|
|
29
|
+
*/
|
|
30
|
+
export declare const JournalCursor: Schema.Struct<{
|
|
31
|
+
/** Logical byte offset just past the last complete line read: where the next read starts. */
|
|
32
|
+
readonly end: Schema.Number;
|
|
33
|
+
/** The logical size of the file when the cursor was taken. */
|
|
34
|
+
readonly size: Schema.Number;
|
|
35
|
+
/** The file's `dev:ino`, in decimal. */
|
|
36
|
+
readonly identity: Schema.String;
|
|
37
|
+
/** The file's modification time in nanoseconds, in decimal. */
|
|
38
|
+
readonly mtime: Schema.String;
|
|
39
|
+
/** The last bytes before `end`, base64. */
|
|
40
|
+
readonly tail: Schema.String;
|
|
41
|
+
}>;
|
|
42
|
+
/**
|
|
43
|
+
* Where an incremental read of a journal file resumes.
|
|
44
|
+
*
|
|
45
|
+
* @public
|
|
46
|
+
*/
|
|
47
|
+
export type JournalCursor = typeof JournalCursor.Type;
|
|
48
|
+
/**
|
|
49
|
+
* Why a read started where it did.
|
|
50
|
+
*
|
|
51
|
+
* @remarks
|
|
52
|
+
* - `initial` — no cursor, or one that names no offset in this file (not a
|
|
53
|
+
* non-negative integer): read from the start.
|
|
54
|
+
* - `none` — read on from the cursor's `end`.
|
|
55
|
+
* - `truncated` — the file is smaller than when the cursor was taken: read
|
|
56
|
+
* from the start.
|
|
57
|
+
* - `replaced` — the path names a different file (a new inode, a rename over
|
|
58
|
+
* it): read from the start.
|
|
59
|
+
* - `rewritten` — the same file was rewritten in place: its size is unchanged
|
|
60
|
+
* but its modification time moved, or the last 64 bytes before `end` are not
|
|
61
|
+
* the ones the cursor saw. Read from the start. A `touch` of an unchanged
|
|
62
|
+
* file also reads as `rewritten`, so lines already seen are returned again.
|
|
63
|
+
* A rewrite that grows the file and leaves those 64 bytes as they were is
|
|
64
|
+
* not seen: the check reads only the bytes at the cursor, never the whole
|
|
65
|
+
* prefix.
|
|
66
|
+
*
|
|
67
|
+
* On any reset other than `none`, `lines` starts at offset 0: discard what
|
|
68
|
+
* earlier reads built before applying them, or they are applied twice.
|
|
69
|
+
*
|
|
70
|
+
* An append-only journal only ever reads `initial` once, then `none`. Every
|
|
71
|
+
* other reset means the writer broke the cooperative-writer contract, which
|
|
72
|
+
* the service reports as `JournalResync`; here the caller decides.
|
|
73
|
+
*
|
|
74
|
+
* @public
|
|
75
|
+
*/
|
|
76
|
+
type JournalReset = "initial" | "none" | "truncated" | "replaced" | "rewritten";
|
|
77
|
+
/**
|
|
78
|
+
* What {@link JournalFile.readFromResult} read.
|
|
79
|
+
*
|
|
80
|
+
* @public
|
|
81
|
+
*/
|
|
82
|
+
interface JournalRead {
|
|
83
|
+
/**
|
|
84
|
+
* Every complete line from where the read started up to the last `\n`, in
|
|
85
|
+
* file order, offsets logical. A torn final line is held back: it is read
|
|
86
|
+
* again, whole, once its terminator lands.
|
|
87
|
+
*/
|
|
88
|
+
readonly lines: ReadonlyArray<Jsonl.LineSlice>;
|
|
89
|
+
/** Where the next read resumes. */
|
|
90
|
+
readonly cursor: JournalCursor;
|
|
91
|
+
/** Why the read started where it did. */
|
|
92
|
+
readonly reset: JournalReset;
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Why a journal file could not be read: the path names nothing, or the
|
|
96
|
+
* filesystem refused, untranslated.
|
|
97
|
+
*
|
|
98
|
+
* @public
|
|
99
|
+
*/
|
|
100
|
+
type JournalFileError = Jsonl.JournalNotFound | PlatformError.PlatformError;
|
|
101
|
+
/**
|
|
102
|
+
* Synchronous reads of a journal file over `node:fs`, for a consumer with no
|
|
103
|
+
* Effect runtime.
|
|
104
|
+
*
|
|
105
|
+
* @remarks
|
|
106
|
+
* Nothing here builds an `Effect` or needs a layer, and nothing throws: a
|
|
107
|
+
* missing path fails `JournalNotFound`, any other filesystem refusal a core
|
|
108
|
+
* `PlatformError`, both in a `Result`. Offsets are logical bytes, past any
|
|
109
|
+
* BOM, exactly as the `Journal` service reports them, so a cursor from either
|
|
110
|
+
* side reads on in the other.
|
|
111
|
+
*
|
|
112
|
+
* @example
|
|
113
|
+
* ```ts
|
|
114
|
+
* import type { JournalCursor } from "@effected/jsonl/node";
|
|
115
|
+
* import { JournalFile } from "@effected/jsonl/node";
|
|
116
|
+
* import { Result } from "effect";
|
|
117
|
+
*
|
|
118
|
+
* declare const saved: JournalCursor | undefined;
|
|
119
|
+
*
|
|
120
|
+
* const read = JournalFile.readFromResult(".claude/dogfood/mailbox.jsonl", saved);
|
|
121
|
+
* if (Result.isSuccess(read)) {
|
|
122
|
+
* // Any reset reads from 0: drop what earlier reads built, or it is applied twice.
|
|
123
|
+
* if (read.success.reset !== "none") {
|
|
124
|
+
* // reset the state you derived from earlier lines
|
|
125
|
+
* }
|
|
126
|
+
* for (const line of read.success.lines) line.text; // each complete line, in order
|
|
127
|
+
* read.success.cursor; // persist it for the next run
|
|
128
|
+
* }
|
|
129
|
+
* ```
|
|
130
|
+
*
|
|
131
|
+
* @public
|
|
132
|
+
*/
|
|
133
|
+
export declare class JournalFile {
|
|
134
|
+
private constructor();
|
|
135
|
+
/**
|
|
136
|
+
* Read the lines appended to `path` since `cursor`.
|
|
137
|
+
*
|
|
138
|
+
* @remarks
|
|
139
|
+
* Reads the bytes from the cursor's `end` to the file's current size and
|
|
140
|
+
* returns every complete line among them; an unterminated final line is held
|
|
141
|
+
* back and read again, whole, once its `\n` lands. Without a cursor, or when
|
|
142
|
+
* the file was truncated, replaced or rewritten since the cursor was taken
|
|
143
|
+
* (see {@link JournalReset}), it reads from the start, and `reset` says why.
|
|
144
|
+
* The read is one buffer from the start offset to the end of the file, so a
|
|
145
|
+
* first read of a large journal costs its size; every read after that costs
|
|
146
|
+
* only what was appended.
|
|
147
|
+
*
|
|
148
|
+
* Line offsets are byte-exact for UTF-8 text. A byte sequence that is not
|
|
149
|
+
* UTF-8 decodes as U+FFFD, and the offsets of the lines after it in the same
|
|
150
|
+
* read drift; the cursor's `end` never does.
|
|
151
|
+
*
|
|
152
|
+
* @param path - The journal file.
|
|
153
|
+
* @param cursor - The cursor the previous read returned, if any.
|
|
154
|
+
* @returns What was read, or `JournalNotFound` when `path` names nothing,
|
|
155
|
+
* or the `PlatformError` the filesystem raised.
|
|
156
|
+
*/
|
|
157
|
+
static readFromResult(path: string, cursor?: JournalCursor): Result.Result<JournalRead, JournalFileError>;
|
|
158
|
+
/**
|
|
159
|
+
* The last valid envelope in `path`: the journal's current state, read the
|
|
160
|
+
* way the service's `latest` reads it.
|
|
161
|
+
*
|
|
162
|
+
* @remarks
|
|
163
|
+
* Steps back from the end one window at a time, each window covering only
|
|
164
|
+
* bytes no earlier one did, growing from `window` by fours up to 1 MiB, and
|
|
165
|
+
* stops at the first window holding a valid envelope. A line longer than
|
|
166
|
+
* the window is read whole. So the cost tracks the answer, not the age of
|
|
167
|
+
* the journal, and `Option.none()` means the whole file holds no valid
|
|
168
|
+
* envelope.
|
|
169
|
+
*
|
|
170
|
+
* @param events - The registry the writer declared.
|
|
171
|
+
* @param path - The journal file.
|
|
172
|
+
* @param window - The first window's size in bytes. Defaults to 8 KiB.
|
|
173
|
+
* @returns The last valid envelope, `Option.none()` if there is none, or
|
|
174
|
+
* `JournalNotFound` / the `PlatformError` the filesystem raised.
|
|
175
|
+
*/
|
|
176
|
+
static lastValidResult<const R extends Jsonl.JsonlEvent.Registry>(events: R, path: string, window?: number): Result.Result<Option.Option<Jsonl.EnvelopeUnion<R>>, JournalFileError>;
|
|
177
|
+
}
|
|
3
178
|
/**
|
|
4
179
|
* The Node implementation of `JournalWatcher` (from `@effected/jsonl`), over `node:fs`'s
|
|
5
180
|
* `watch`.
|
|
@@ -32,4 +207,5 @@ export declare class NodeJournalWatcher {
|
|
|
32
207
|
/** Provides `JournalWatcher` (from `@effected/jsonl`) over `node:fs`. */
|
|
33
208
|
static readonly layer: Layer.Layer<Jsonl.JournalWatcher>;
|
|
34
209
|
}
|
|
210
|
+
export type { JournalFileError, JournalRead, JournalReset };
|
|
35
211
|
//# sourceMappingURL=node.d.ts.map
|
package/node.js
CHANGED