@effected/jsonl 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Envelope.js +267 -0
- package/Journal.js +475 -0
- package/JsonlError.js +246 -0
- package/JsonlEvent.js +45 -0
- package/LICENSE +21 -0
- package/Line.js +219 -0
- package/LineSlice.js +62 -0
- package/README.md +181 -0
- package/Slice.js +23 -0
- package/index.d.ts +1032 -0
- package/index.js +8 -0
- package/internal/merge.js +112 -0
- package/internal/tail.js +143 -0
- package/internal/utf8.js +43 -0
- package/package.json +45 -0
- package/tsdoc-metadata.json +11 -0
package/index.d.ts
ADDED
|
@@ -0,0 +1,1032 @@
|
|
|
1
|
+
import { Context, DateTime, Duration, Effect, FileSystem, Layer, Option, PlatformError, PubSub, Result, Schema, Stream, SubscriptionRef, Take } from "effect";
|
|
2
|
+
//#region src/LineSlice.d.ts
|
|
3
|
+
declare const LineSlice_base: Schema.Class<LineSlice, Schema.Struct<{
|
|
4
|
+
/** UTF-8 byte offset of this line's first content byte. */
|
|
5
|
+
readonly offset: Schema.Number;
|
|
6
|
+
/**
|
|
7
|
+
* UTF-8 byte offset just past this line's terminator — the offset at which
|
|
8
|
+
* the next line begins, and the resume cursor for an incremental read.
|
|
9
|
+
*
|
|
10
|
+
* Equal to `offset + length` when the line is unterminated.
|
|
11
|
+
*/
|
|
12
|
+
readonly end: Schema.Number;
|
|
13
|
+
/** UTF-8 byte length of `LineSlice.text`, excluding any terminator. */
|
|
14
|
+
readonly length: Schema.Number;
|
|
15
|
+
/** The line's content, with its terminator and any paired `\r` removed. */
|
|
16
|
+
readonly text: Schema.String;
|
|
17
|
+
/**
|
|
18
|
+
* Whether a `\n` terminated this line in the source.
|
|
19
|
+
*
|
|
20
|
+
* `false` can only occur on the final line, and means the line **may be a
|
|
21
|
+
* torn tail** — a writer caught mid-append. A reader walks back over it and
|
|
22
|
+
* leaves its bytes unconsumed so the next read sees the completed line.
|
|
23
|
+
*/
|
|
24
|
+
readonly terminated: Schema.Boolean;
|
|
25
|
+
}>, {}>;
|
|
26
|
+
/**
|
|
27
|
+
* A single candidate line: its text, and where it lives in the source **in
|
|
28
|
+
* bytes**.
|
|
29
|
+
*
|
|
30
|
+
* Every offset on this class is a UTF-8 byte offset, never a UTF-16 code-unit
|
|
31
|
+
* index, because these values are cursors into a file: they are handed to
|
|
32
|
+
* `FileSystem.stream`'s `offset` option and persisted across process restarts.
|
|
33
|
+
* A `String.length`-derived offset is correct only for ASCII journals and is
|
|
34
|
+
* the single most likely bug in this module.
|
|
35
|
+
*
|
|
36
|
+
* The terminator is **not** part of the content: `text` and `length` exclude
|
|
37
|
+
* the trailing `\n`, and exclude the `\r` of a `\r\n` pair. `end` includes it,
|
|
38
|
+
* which is why `end - offset` is not always `length`.
|
|
39
|
+
*
|
|
40
|
+
* @example
|
|
41
|
+
* ```ts
|
|
42
|
+
* import { Line } from "@effected/jsonl";
|
|
43
|
+
*
|
|
44
|
+
* const [first] = Line.split('{"a":1}\r\n{"b":2}\n');
|
|
45
|
+
* first?.text; // '{"a":1}'
|
|
46
|
+
* first?.offset; // 0
|
|
47
|
+
* first?.length; // 7 — content bytes, no terminator
|
|
48
|
+
* first?.end; // 9 — past the CR and the LF
|
|
49
|
+
* first?.terminated; // true
|
|
50
|
+
* ```
|
|
51
|
+
*
|
|
52
|
+
* @public
|
|
53
|
+
*/
|
|
54
|
+
declare class LineSlice extends LineSlice_base {}
|
|
55
|
+
//#endregion
|
|
56
|
+
//#region src/JsonlError.d.ts
|
|
57
|
+
declare const MalformedLine_base: Schema.Class<MalformedLine, Schema.TaggedStruct<"MalformedLine", {
|
|
58
|
+
/** The offending line, with its byte offsets into the source. */
|
|
59
|
+
readonly line: typeof LineSlice;
|
|
60
|
+
}>, import("effect/Cause").YieldableError>;
|
|
61
|
+
/**
|
|
62
|
+
* A journal line that is not valid JSON.
|
|
63
|
+
*
|
|
64
|
+
* **This is the expected steady state at the tail of a live journal**, not
|
|
65
|
+
* necessarily corruption: a writer caught mid-`write` leaves a partial final
|
|
66
|
+
* line, and `LineSlice.terminated` is what distinguishes the two cases. An
|
|
67
|
+
* unterminated malformed line is a torn tail that the next append completes; a
|
|
68
|
+
* *terminated* malformed line is a hole in the history that will never heal.
|
|
69
|
+
*
|
|
70
|
+
* Malformed input always fails through this typed channel — never as a defect,
|
|
71
|
+
* and never by being silently dropped from a read.
|
|
72
|
+
*
|
|
73
|
+
* @example
|
|
74
|
+
* ```ts
|
|
75
|
+
* import { Line } from "@effected/jsonl";
|
|
76
|
+
* import { Result } from "effect";
|
|
77
|
+
*
|
|
78
|
+
* const [line] = Line.split('{"a":');
|
|
79
|
+
* const parsed = Line.parseResult(line!);
|
|
80
|
+
* if (Result.isFailure(parsed)) {
|
|
81
|
+
* parsed.failure.line.terminated; // false — a torn tail, not a hole
|
|
82
|
+
* }
|
|
83
|
+
* ```
|
|
84
|
+
*
|
|
85
|
+
* @public
|
|
86
|
+
*/
|
|
87
|
+
declare class MalformedLine extends MalformedLine_base {
|
|
88
|
+
get message(): string;
|
|
89
|
+
}
|
|
90
|
+
declare const UnknownEvent_base: Schema.Class<UnknownEvent, Schema.TaggedStruct<"UnknownEvent", {
|
|
91
|
+
/** The offending line, with its byte offsets into the source. */
|
|
92
|
+
readonly line: typeof LineSlice;
|
|
93
|
+
/** The unrecognized tag as it appeared on the envelope. */
|
|
94
|
+
readonly event: Schema.String;
|
|
95
|
+
/** The tags this journal's registry does define. */
|
|
96
|
+
readonly known: Schema.$Array<Schema.String>;
|
|
97
|
+
}>, import("effect/Cause").YieldableError>;
|
|
98
|
+
/**
|
|
99
|
+
* A line whose `event` tag is not in the registry.
|
|
100
|
+
*
|
|
101
|
+
* Typed rather than a defect on purpose: a journal written by an older or newer
|
|
102
|
+
* version of the same application is hostile input in the technical sense, and
|
|
103
|
+
* a reader that crashed on an unrecognized tag would be unable to skip forward
|
|
104
|
+
* past one. The known tags travel with the error so a caller can report the
|
|
105
|
+
* mismatch without reaching back for the registry.
|
|
106
|
+
*
|
|
107
|
+
* @public
|
|
108
|
+
*/
|
|
109
|
+
declare class UnknownEvent extends UnknownEvent_base {
|
|
110
|
+
get message(): string;
|
|
111
|
+
}
|
|
112
|
+
declare const InvalidData_base: Schema.Class<InvalidData, Schema.TaggedStruct<"InvalidData", {
|
|
113
|
+
/** The offending line, with its byte offsets into the source. */
|
|
114
|
+
readonly line: typeof LineSlice;
|
|
115
|
+
/**
|
|
116
|
+
* The event tag whose payload schema rejected the data, or `none` when it
|
|
117
|
+
* was the envelope frame itself that failed.
|
|
118
|
+
*/
|
|
119
|
+
readonly event: Schema.Option<Schema.String>;
|
|
120
|
+
/** The schema failure, carried structurally — `issue` is the full tree. */
|
|
121
|
+
readonly error: Schema.declare<Schema.SchemaError, Schema.SchemaError>;
|
|
122
|
+
}>, import("effect/Cause").YieldableError>;
|
|
123
|
+
/**
|
|
124
|
+
* A line whose envelope or payload failed schema validation.
|
|
125
|
+
*
|
|
126
|
+
* Covers both stages of the two-stage decode, distinguished by `event`: the
|
|
127
|
+
* frame itself (`Option.none()` — the line is JSON but not an envelope) and a
|
|
128
|
+
* registered payload (`Option.some(tag)` — the envelope is well-formed but its
|
|
129
|
+
* `data` does not match the schema registered for that tag).
|
|
130
|
+
*
|
|
131
|
+
* The `SchemaError` is carried **whole**, so `error.issue` is the full issue
|
|
132
|
+
* tree with its paths and expected types intact. Nothing here is stringified;
|
|
133
|
+
* `message` renders lazily and only when something asks for it.
|
|
134
|
+
*
|
|
135
|
+
* @public
|
|
136
|
+
*/
|
|
137
|
+
declare class InvalidData extends InvalidData_base {
|
|
138
|
+
get message(): string;
|
|
139
|
+
}
|
|
140
|
+
declare const TerminalViolation_base: Schema.Class<TerminalViolation, Schema.TaggedStruct<"TerminalViolation", {
|
|
141
|
+
/** The tag of the event whose append was refused. */
|
|
142
|
+
readonly event: Schema.String;
|
|
143
|
+
/** The terminal event currently at the tail of the journal. */
|
|
144
|
+
readonly terminal: Schema.String;
|
|
145
|
+
}>, import("effect/Cause").YieldableError>;
|
|
146
|
+
/**
|
|
147
|
+
* An append attempted after a terminal event, by an event not marked `reopen`.
|
|
148
|
+
*
|
|
149
|
+
* A journal whose tail is terminal is quiescent: it is finished, and appending
|
|
150
|
+
* to it would silently resurrect a closed loop. Reopening is legal but must be
|
|
151
|
+
* declared, which is what `reopen` marks.
|
|
152
|
+
*
|
|
153
|
+
* @public
|
|
154
|
+
*/
|
|
155
|
+
declare class TerminalViolation extends TerminalViolation_base {
|
|
156
|
+
get message(): string;
|
|
157
|
+
}
|
|
158
|
+
declare const JournalNotFound_base: Schema.Class<JournalNotFound, Schema.TaggedStruct<"JournalNotFound", {
|
|
159
|
+
/** The path that does not exist. */
|
|
160
|
+
readonly path: Schema.String;
|
|
161
|
+
}>, import("effect/Cause").YieldableError>;
|
|
162
|
+
/**
|
|
163
|
+
* An operation against a journal file that does not exist.
|
|
164
|
+
*
|
|
165
|
+
* A missing journal is a **legal state** — building the layer over a path that
|
|
166
|
+
* does not exist yet succeeds, and the watcher activates once the file appears.
|
|
167
|
+
* What is not legal is materializing it implicitly: `append`, `query` and
|
|
168
|
+
* `latest` fail with this rather than creating the file, so a typo in a path
|
|
169
|
+
* cannot quietly produce a second, empty journal that looks like a working
|
|
170
|
+
* system with no history. Creation is always explicit, via `create`.
|
|
171
|
+
*
|
|
172
|
+
* @public
|
|
173
|
+
*/
|
|
174
|
+
declare class JournalNotFound extends JournalNotFound_base {
|
|
175
|
+
get message(): string;
|
|
176
|
+
}
|
|
177
|
+
declare const UnserializableData_base: Schema.Class<UnserializableData, Schema.TaggedStruct<"UnserializableData", {
|
|
178
|
+
/** The event tag whose payload could not be serialized. */
|
|
179
|
+
readonly event: Schema.String;
|
|
180
|
+
/** The value `JSON.stringify` threw, carried structurally. */
|
|
181
|
+
readonly cause: Schema.Unknown;
|
|
182
|
+
}>, import("effect/Cause").YieldableError>;
|
|
183
|
+
/**
|
|
184
|
+
* A payload that validated against its schema but cannot be serialized to JSON.
|
|
185
|
+
*
|
|
186
|
+
* The two reachable causes are a `bigint` anywhere in the encoded value and a
|
|
187
|
+
* reference cycle; `JSON.stringify` throws a `TypeError` for both. A payload
|
|
188
|
+
* schema is free to permit either — `Schema.Unknown` permits everything — so
|
|
189
|
+
* schema validity does not imply serializability and the encode path has to
|
|
190
|
+
* treat them as separate questions.
|
|
191
|
+
*
|
|
192
|
+
* This is its own tag rather than a flavour of {@link InvalidData} because the
|
|
193
|
+
* recovery differs: `InvalidData` means the value is wrong and the caller
|
|
194
|
+
* should fix the value, while this means the value is fine but unrepresentable
|
|
195
|
+
* in this format and the caller must change the *shape* they are journaling.
|
|
196
|
+
*
|
|
197
|
+
* The thrown value is carried as-is under `cause` — never stringified, and
|
|
198
|
+
* never re-thrown. A raw `TypeError` escaping the encode path would be
|
|
199
|
+
* especially bad here: `Effect.fromResult` evaluates its argument eagerly, so
|
|
200
|
+
* an unguarded throw happens at **construction** of the `Effect`, before any
|
|
201
|
+
* fiber exists, where not even `Effect.runSyncExit` can catch it.
|
|
202
|
+
*
|
|
203
|
+
* @public
|
|
204
|
+
*/
|
|
205
|
+
declare class UnserializableData extends UnserializableData_base {
|
|
206
|
+
get message(): string;
|
|
207
|
+
}
|
|
208
|
+
declare const JournalClosed_base: Schema.Class<JournalClosed, Schema.TaggedStruct<"JournalClosed", {
|
|
209
|
+
/** The tag of the event whose append was refused. */
|
|
210
|
+
readonly event: Schema.String;
|
|
211
|
+
}>, import("effect/Cause").YieldableError>;
|
|
212
|
+
/**
|
|
213
|
+
* An append refused because the journal's scope has closed.
|
|
214
|
+
*
|
|
215
|
+
* Its own tag rather than a flavour of {@link TerminalViolation}, because the
|
|
216
|
+
* recoveries have nothing in common: a terminal journal is a *state* the
|
|
217
|
+
* consumer can reason about and reopen from with a `reopen` event, while a
|
|
218
|
+
* closed one is a *lifecycle* fact — this service is gone, and the only moves
|
|
219
|
+
* are to build a new layer or stop.
|
|
220
|
+
*
|
|
221
|
+
* Refusal is deliberately a typed failure rather than a wait. A `Latch` would
|
|
222
|
+
* suspend the late append with no failure channel, turning "the journal is
|
|
223
|
+
* closing" into a hang; failing fast is what lets a caller react.
|
|
224
|
+
*
|
|
225
|
+
* @public
|
|
226
|
+
*/
|
|
227
|
+
declare class JournalClosed extends JournalClosed_base {
|
|
228
|
+
get message(): string;
|
|
229
|
+
}
|
|
230
|
+
declare const JournalResync_base: Schema.Class<JournalResync, Schema.TaggedStruct<"JournalResync", {
|
|
231
|
+
/** The journal path whose file changed identity or shrank. */
|
|
232
|
+
readonly path: Schema.String;
|
|
233
|
+
/** Which contract breach was detected. Diagnostic; the recovery is the same. */
|
|
234
|
+
readonly reason: Schema.Literals<readonly ["truncated", "replaced"]>;
|
|
235
|
+
/** The logical offset the reader had consumed to. */
|
|
236
|
+
readonly expected: Schema.Number;
|
|
237
|
+
/** The file's logical size when the breach was noticed. */
|
|
238
|
+
readonly actual: Schema.Number;
|
|
239
|
+
}>, import("effect/Cause").YieldableError>;
|
|
240
|
+
/**
|
|
241
|
+
* The journal file was truncated or replaced beneath a reader.
|
|
242
|
+
*
|
|
243
|
+
* The cooperative-writer contract is append-only: a journal only ever grows,
|
|
244
|
+
* and every cursor this package hands out depends on that. When the file shrinks
|
|
245
|
+
* below a tracked offset, or the path comes to name a different file entirely,
|
|
246
|
+
* the contract has been broken by something outside the package and every
|
|
247
|
+
* offset-derived belief is now meaningless.
|
|
248
|
+
*
|
|
249
|
+
* Surfaced rather than repaired, deliberately. Silently re-reading from zero
|
|
250
|
+
* would paper over a real operational fault — a rotating log shipper, a
|
|
251
|
+
* `>` where `>>` was meant — and leave projections quietly inconsistent with
|
|
252
|
+
* the file. The recovery is the consumer's: discard cursor-derived state,
|
|
253
|
+
* re-read, and tell somebody.
|
|
254
|
+
*
|
|
255
|
+
* One tag rather than two, though `reason` distinguishes the causes: truncation
|
|
256
|
+
* and replacement have the **same** recovery, and a tag per cause would split
|
|
257
|
+
* one recovery across two tags.
|
|
258
|
+
*
|
|
259
|
+
* @remarks
|
|
260
|
+
* Detection is as complete as the platform allows and no more. Truncation is
|
|
261
|
+
* caught by size; replacement is caught by inode identity, which
|
|
262
|
+
* `FileSystem.File.Info` exposes as an **`Option`** — on a platform that does
|
|
263
|
+
* not report it, a replacement at equal or greater size is undetectable and
|
|
264
|
+
* only truncation is caught.
|
|
265
|
+
*
|
|
266
|
+
* @public
|
|
267
|
+
*/
|
|
268
|
+
declare class JournalResync extends JournalResync_base {
|
|
269
|
+
get message(): string;
|
|
270
|
+
}
|
|
271
|
+
/**
|
|
272
|
+
* Every error this package raises from the pure core and the journal service.
|
|
273
|
+
*
|
|
274
|
+
* Core's `PlatformError` is deliberately **not** a member: IO failures pass
|
|
275
|
+
* through untranslated rather than being wrapped in a taxonomy that would add
|
|
276
|
+
* no recovery information.
|
|
277
|
+
*
|
|
278
|
+
* @public
|
|
279
|
+
*/
|
|
280
|
+
type JsonlError = MalformedLine | UnknownEvent | InvalidData | UnserializableData | TerminalViolation | JournalClosed | JournalNotFound | JournalResync;
|
|
281
|
+
//#endregion
|
|
282
|
+
//#region src/JsonlEvent.d.ts
|
|
283
|
+
/**
|
|
284
|
+
* The bound every registered payload schema must satisfy: a codec requiring
|
|
285
|
+
* **no services** in either direction.
|
|
286
|
+
*
|
|
287
|
+
* This is a contract, not an implementation detail. The pure core decodes with
|
|
288
|
+
* `Schema.decodeUnknownResult` and encodes with `Schema.encodeUnknownResult`,
|
|
289
|
+
* both of which demand `never` in both service slots — so a payload schema that
|
|
290
|
+
* needed a service would make the synchronous, runtime-free read path
|
|
291
|
+
* impossible. Bounding it here fails such a schema at **registration**, where
|
|
292
|
+
* the mistake is, instead of at some consumer's call site later.
|
|
293
|
+
*
|
|
294
|
+
* `Schema.Top` is deliberately not the bound: its service parameters are
|
|
295
|
+
* `unknown`, so it does not satisfy the sync codecs and the constraint would
|
|
296
|
+
* silently fail to bind. The value slots are `unknown` rather than `any`
|
|
297
|
+
* because `Codec`'s type and encoded parameters are covariant, so `unknown`
|
|
298
|
+
* admits every concrete payload schema while still rejecting a
|
|
299
|
+
* service-requiring one — the precision costs nothing here.
|
|
300
|
+
*
|
|
301
|
+
* @public
|
|
302
|
+
*/
|
|
303
|
+
type DataSchema = Schema.Codec<unknown, unknown, never, never>;
|
|
304
|
+
/**
|
|
305
|
+
* Unique type identifier marking a JSONL event definition.
|
|
306
|
+
*
|
|
307
|
+
* @public
|
|
308
|
+
*/
|
|
309
|
+
type JsonlEventTypeId = "~effected/jsonl/JsonlEvent";
|
|
310
|
+
/**
|
|
311
|
+
* Runtime type identifier marking a JSONL event definition.
|
|
312
|
+
*
|
|
313
|
+
* @public
|
|
314
|
+
*/
|
|
315
|
+
declare const JsonlEventTypeId: JsonlEventTypeId;
|
|
316
|
+
/**
|
|
317
|
+
* One event definition: a tag, the schema its `data` must satisfy, and the two
|
|
318
|
+
* lifecycle markings.
|
|
319
|
+
*
|
|
320
|
+
* The type parameters are what make a registry more than a runtime list — the
|
|
321
|
+
* literal `Tag`, the payload schema and the `terminal`/`reopen` flags all
|
|
322
|
+
* survive into the derived envelope union, so `append` narrows on the tag and a
|
|
323
|
+
* projection over a slice is exhaustively checkable.
|
|
324
|
+
*
|
|
325
|
+
* @public
|
|
326
|
+
*/
|
|
327
|
+
interface JsonlEvent<out Tag extends string, in out Data extends DataSchema = Schema.Codec<void, void, never, never>, out Terminal extends boolean = false, out Reopen extends boolean = false> {
|
|
328
|
+
readonly [JsonlEventTypeId]: JsonlEventTypeId;
|
|
329
|
+
/** The string tag: the envelope discriminant and the primary filter key. */
|
|
330
|
+
readonly tag: Tag;
|
|
331
|
+
/** The schema the envelope's `data` is validated against. */
|
|
332
|
+
readonly data: Data;
|
|
333
|
+
/**
|
|
334
|
+
* Whether this event makes the journal quiescent.
|
|
335
|
+
*
|
|
336
|
+
* After a terminal event is the tail, appending fails with
|
|
337
|
+
* `TerminalViolation` unless the appended event is marked `reopen`.
|
|
338
|
+
*/
|
|
339
|
+
readonly terminal: Terminal;
|
|
340
|
+
/** Whether this event may follow a terminal one, reopening the journal. */
|
|
341
|
+
readonly reopen: Reopen;
|
|
342
|
+
}
|
|
343
|
+
/**
|
|
344
|
+
* Helper types for working with event definitions and registries.
|
|
345
|
+
*
|
|
346
|
+
* @public
|
|
347
|
+
*/
|
|
348
|
+
declare namespace JsonlEvent {
|
|
349
|
+
/**
|
|
350
|
+
* A type-erased event definition.
|
|
351
|
+
*
|
|
352
|
+
* Note `data` is bounded by {@link DataSchema} rather than widened to
|
|
353
|
+
* `Schema.Top`: erasing to `Top` would lose the no-services guarantee that
|
|
354
|
+
* the sync codecs depend on.
|
|
355
|
+
*
|
|
356
|
+
* @public
|
|
357
|
+
*/
|
|
358
|
+
interface Any {
|
|
359
|
+
readonly [JsonlEventTypeId]: JsonlEventTypeId;
|
|
360
|
+
readonly tag: string;
|
|
361
|
+
readonly data: DataSchema;
|
|
362
|
+
readonly terminal: boolean;
|
|
363
|
+
readonly reopen: boolean;
|
|
364
|
+
}
|
|
365
|
+
/** A registry: the set of events one journal may carry. */
|
|
366
|
+
type Registry = ReadonlyArray<Any>;
|
|
367
|
+
/** The union of every event definition in a registry. */
|
|
368
|
+
type Events<R extends Registry> = R[number];
|
|
369
|
+
/** The union of every tag in a registry — the envelope discriminant. */
|
|
370
|
+
type Tag<R extends Registry> = Events<R>["tag"];
|
|
371
|
+
/** The event definition in a registry carrying a given tag. */
|
|
372
|
+
type WithTag<R extends Registry, T extends string> = Extract<Events<R>, {
|
|
373
|
+
readonly tag: T;
|
|
374
|
+
}>;
|
|
375
|
+
/** The decoded payload type registered for a given tag. */
|
|
376
|
+
type Data<R extends Registry, T extends string> = WithTag<R, T>["data"]["Type"];
|
|
377
|
+
/** The tags marked `terminal` in a registry. */
|
|
378
|
+
type TerminalTags<R extends Registry> = Extract<Events<R>, {
|
|
379
|
+
readonly terminal: true;
|
|
380
|
+
}>["tag"];
|
|
381
|
+
/** The tags marked `reopen` in a registry. */
|
|
382
|
+
type ReopenTags<R extends Registry> = Extract<Events<R>, {
|
|
383
|
+
readonly reopen: true;
|
|
384
|
+
}>["tag"];
|
|
385
|
+
}
|
|
386
|
+
/**
|
|
387
|
+
* Defines an event.
|
|
388
|
+
*
|
|
389
|
+
* The `const` type parameters are load-bearing: they keep `"unlinked"` a
|
|
390
|
+
* literal rather than widening it to `string`, and keep `terminal: true` a
|
|
391
|
+
* literal `true`, so both survive into the derived envelope union where the
|
|
392
|
+
* narrowing and the terminal/reopen state machine depend on them.
|
|
393
|
+
*
|
|
394
|
+
* `data` is required rather than defaulted. A payload-less event says so
|
|
395
|
+
* explicitly with `Schema.Void`, because a silent default would make a typo in
|
|
396
|
+
* the options object look like a deliberate void payload.
|
|
397
|
+
*
|
|
398
|
+
* @example
|
|
399
|
+
* ```ts
|
|
400
|
+
* import { JsonlEvent } from "@effected/jsonl";
|
|
401
|
+
* import { Schema } from "effect";
|
|
402
|
+
*
|
|
403
|
+
* const MailReceived = JsonlEvent.make("mail-received", {
|
|
404
|
+
* data: Schema.Struct({ round: Schema.Number }),
|
|
405
|
+
* });
|
|
406
|
+
* const Unlinked = JsonlEvent.make("unlinked", { data: Schema.Void, terminal: true });
|
|
407
|
+
* const Relinked = JsonlEvent.make("relinked", { data: Schema.Void, reopen: true });
|
|
408
|
+
*
|
|
409
|
+
* const registry = [MailReceived, Unlinked, Relinked] as const;
|
|
410
|
+
* ```
|
|
411
|
+
*
|
|
412
|
+
* @public
|
|
413
|
+
*/
|
|
414
|
+
declare const JsonlEvent: {
|
|
415
|
+
readonly make: <const Tag extends string, Data extends DataSchema, const Terminal extends boolean = false, const Reopen extends boolean = false>(tag: Tag, options: {
|
|
416
|
+
readonly data: Data;
|
|
417
|
+
readonly terminal?: Terminal | undefined;
|
|
418
|
+
readonly reopen?: Reopen | undefined;
|
|
419
|
+
}) => JsonlEvent<Tag, Data, Terminal, Reopen>;
|
|
420
|
+
};
|
|
421
|
+
//#endregion
|
|
422
|
+
//#region src/Envelope.d.ts
|
|
423
|
+
/**
|
|
424
|
+
* Stage one of the read path: the envelope with its payload left undecoded.
|
|
425
|
+
*
|
|
426
|
+
* A `Schema.Struct` rather than a `Schema.Class` deliberately. This is the hot
|
|
427
|
+
* path — it runs for every line of every read, including the lines a `Slice` is
|
|
428
|
+
* about to discard — and it is a transient the caller never holds, so there is
|
|
429
|
+
* nothing to gain from wrapping each one in a class instance.
|
|
430
|
+
*
|
|
431
|
+
* `data` is `Schema.Unknown`, which passes the payload through without
|
|
432
|
+
* traversing it. That is what keeps frame decoding independent of payload
|
|
433
|
+
* depth, and what lets filtering happen before any payload schema runs.
|
|
434
|
+
*
|
|
435
|
+
* @public
|
|
436
|
+
*/
|
|
437
|
+
declare const EnvelopeFrame: Schema.Struct<{
|
|
438
|
+
readonly at: Schema.DateTimeUtcFromString;
|
|
439
|
+
readonly event: Schema.String;
|
|
440
|
+
readonly scope: Schema.optionalKey<Schema.String>;
|
|
441
|
+
readonly data: Schema.Unknown;
|
|
442
|
+
}>;
|
|
443
|
+
/**
|
|
444
|
+
* A decoded envelope: the frame, its validated payload, and where it sits in
|
|
445
|
+
* the source.
|
|
446
|
+
*
|
|
447
|
+
* This interface is what the "derived discriminated union" derives *into*.
|
|
448
|
+
* A registry of event definitions produces a union of these discriminated on
|
|
449
|
+
* `event`, so narrowing on the tag recovers the payload type registered for it.
|
|
450
|
+
*
|
|
451
|
+
* @public
|
|
452
|
+
*/
|
|
453
|
+
interface Envelope<out Tag extends string, out Data> {
|
|
454
|
+
/** UTC timestamp, assigned by the service at append time from the `Clock`. */
|
|
455
|
+
readonly at: DateTime.Utc;
|
|
456
|
+
/** The event tag: the union discriminant and the primary filter key. */
|
|
457
|
+
readonly event: Tag;
|
|
458
|
+
/** The partition key, with no further semantics. */
|
|
459
|
+
readonly scope?: string | undefined;
|
|
460
|
+
/** The payload, validated against the schema registered for `event`. */
|
|
461
|
+
readonly data: Data;
|
|
462
|
+
/** Where this envelope's line sits in the source, in bytes. */
|
|
463
|
+
readonly line: LineSlice;
|
|
464
|
+
}
|
|
465
|
+
/**
|
|
466
|
+
* The envelope type derived from one event definition. Distributes over a union
|
|
467
|
+
* of definitions, which is what turns a registry into a discriminated union.
|
|
468
|
+
*
|
|
469
|
+
* @public
|
|
470
|
+
*/
|
|
471
|
+
type EnvelopeOf<E extends JsonlEvent.Any> = E extends JsonlEvent<infer Tag, infer Data extends DataSchema, boolean, boolean> ? Envelope<Tag, Data["Type"]> : never;
|
|
472
|
+
/**
|
|
473
|
+
* The discriminated union of every envelope a registry can carry.
|
|
474
|
+
*
|
|
475
|
+
* @public
|
|
476
|
+
*/
|
|
477
|
+
type EnvelopeUnion<R extends JsonlEvent.Registry> = EnvelopeOf<JsonlEvent.Events<R>>;
|
|
478
|
+
/**
|
|
479
|
+
* The envelope variant carrying a given tag — the type a slice narrows to.
|
|
480
|
+
*
|
|
481
|
+
* @public
|
|
482
|
+
*/
|
|
483
|
+
type EnvelopeWithTag<R extends JsonlEvent.Registry, T extends string> = Extract<EnvelopeUnion<R>, {
|
|
484
|
+
readonly event: T;
|
|
485
|
+
}>;
|
|
486
|
+
/**
|
|
487
|
+
* Decoding, encoding and the envelope-level walk-back.
|
|
488
|
+
*
|
|
489
|
+
* Every operation here is synchronous and `Result`-based; the `Effect` forms are
|
|
490
|
+
* defined **in terms of** the sync ones via `Effect.fromResult`, so the two can
|
|
491
|
+
* never drift into two implementations of the same rule.
|
|
492
|
+
*
|
|
493
|
+
* @public
|
|
494
|
+
*/
|
|
495
|
+
declare const Envelope: {
|
|
496
|
+
/**
|
|
497
|
+
* Stage one only: decode the frame, leaving `data` untouched.
|
|
498
|
+
*
|
|
499
|
+
* This is what filtering reads. It never runs a payload schema, so its cost
|
|
500
|
+
* is independent of payload size and depth, and a line destined to be
|
|
501
|
+
* filtered out never pays for its payload at all.
|
|
502
|
+
*
|
|
503
|
+
* @param line - A slice from `Line.split`.
|
|
504
|
+
*/
|
|
505
|
+
readonly frameResult: (line: LineSlice) => Result.Result<typeof EnvelopeFrame.Type & {
|
|
506
|
+
readonly line: LineSlice;
|
|
507
|
+
}, MalformedLine | InvalidData>;
|
|
508
|
+
/**
|
|
509
|
+
* Decode one line into a fully validated envelope, running both stages.
|
|
510
|
+
*
|
|
511
|
+
* @param events - The registry defining which tags are legal and what their
|
|
512
|
+
* payloads must look like.
|
|
513
|
+
* @param line - A slice from `Line.split`.
|
|
514
|
+
* @returns The envelope, or the typed reason it is not one: not JSON
|
|
515
|
+
* (`MalformedLine`), not an envelope or a bad payload (`InvalidData`), or
|
|
516
|
+
* a tag this registry does not define (`UnknownEvent`).
|
|
517
|
+
*/
|
|
518
|
+
readonly decodeResult: <const R extends JsonlEvent.Registry>(events: R, line: LineSlice) => Result.Result<EnvelopeUnion<R>, MalformedLine | InvalidData | UnknownEvent>;
|
|
519
|
+
/**
|
|
520
|
+
* Decode a line only if its **frame** is selected, running the payload schema
|
|
521
|
+
* for selected lines only.
|
|
522
|
+
*
|
|
523
|
+
* This is the structural form of the filter-before-decode guarantee: when
|
|
524
|
+
* `select` rejects the frame, the registered `data` schema is never reached,
|
|
525
|
+
* so a consumer reading a journal it shares with a noisy neighbour pays
|
|
526
|
+
* nothing for the neighbour's payloads. Writing the filter *after* a full
|
|
527
|
+
* decode would produce identical output while charging for every line — which
|
|
528
|
+
* is why this exists as its own operation rather than as advice.
|
|
529
|
+
*
|
|
530
|
+
* @param events - The registry to validate against.
|
|
531
|
+
* @param line - A slice from `Line.split`.
|
|
532
|
+
* @param select - Predicate over the decoded frame — `at`, `event`, `scope`
|
|
533
|
+
* and the raw undecoded `data`.
|
|
534
|
+
* @returns `Option.none()` when the frame is not selected; otherwise the
|
|
535
|
+
* fully decoded envelope, or the typed reason it is not one.
|
|
536
|
+
*/
|
|
537
|
+
readonly decodeSelectedResult: <const R extends JsonlEvent.Registry>(events: R, line: LineSlice, select: (frame: typeof EnvelopeFrame.Type) => boolean) => Option.Option<Result.Result<EnvelopeUnion<R>, MalformedLine | InvalidData | UnknownEvent>>;
|
|
538
|
+
/**
|
|
539
|
+
* Decode every non-blank line, reporting failures rather than dropping them.
|
|
540
|
+
*
|
|
541
|
+
* Same contract as `Line.parseAll` one layer down: a line that is not a
|
|
542
|
+
* legal envelope is a `Result` in the array, never a gap in it.
|
|
543
|
+
*/
|
|
544
|
+
readonly decodeAllResult: <const R extends JsonlEvent.Registry>(events: R, text: string) => ReadonlyArray<Result.Result<EnvelopeUnion<R>, MalformedLine | InvalidData | UnknownEvent>>;
|
|
545
|
+
/**
|
|
546
|
+
* Walk back from the end to the last valid **envelope**.
|
|
547
|
+
*
|
|
548
|
+
* This is the binding definition of "the last valid line", and the envelope
|
|
549
|
+
* layer is where it has to live. `Line.lastValid` stops at JSON validity,
|
|
550
|
+
* which is not enough: a torn scalar tail (`42` cut mid-write leaves `4`)
|
|
551
|
+
* parses cleanly as a different value, so a JSON-level walk-back would hand
|
|
552
|
+
* back corruption as the current state. An envelope is always an object,
|
|
553
|
+
* every strict prefix of an object text is invalid JSON, and a bare `4` is
|
|
554
|
+
* not an envelope — so this walk-back detects the torn tail that the one
|
|
555
|
+
* below it structurally cannot.
|
|
556
|
+
*
|
|
557
|
+
* Decoding stops at the first success, so the cost tracks the damage at the
|
|
558
|
+
* tail rather than the age of the file.
|
|
559
|
+
*
|
|
560
|
+
* @param events - The registry to validate against.
|
|
561
|
+
* @param text - JSONL source text.
|
|
562
|
+
* @returns The last valid envelope, or `Option.none()` if there is none.
|
|
563
|
+
*/
|
|
564
|
+
readonly lastValidResult: <const R extends JsonlEvent.Registry>(events: R, text: string) => Option.Option<EnvelopeUnion<R>>;
|
|
565
|
+
/**
|
|
566
|
+
* Encode one envelope into a complete JSONL line, terminator included.
|
|
567
|
+
*
|
|
568
|
+
* The trailing `\n` is part of the returned string on purpose: the
|
|
569
|
+
* cooperative-writer contract is one `writeAll` of a *complete line*, and an
|
|
570
|
+
* API that returned the text without its terminator would invite a caller to
|
|
571
|
+
* write the two separately — which is exactly the interleaving the contract
|
|
572
|
+
* exists to prevent.
|
|
573
|
+
*
|
|
574
|
+
* No `Schema.Union` is involved. The tag is known, so the payload schema is
|
|
575
|
+
* a direct registry lookup; a union would buy nothing and decode eagerly.
|
|
576
|
+
*
|
|
577
|
+
* @param events - The registry to validate against.
|
|
578
|
+
* @param envelope - The event tag, its payload, the service-assigned
|
|
579
|
+
* timestamp, and an optional scope.
|
|
580
|
+
*/
|
|
581
|
+
readonly encodeResult: <const R extends JsonlEvent.Registry, const T extends JsonlEvent.Tag<R>>(events: R, envelope: {
|
|
582
|
+
readonly event: T;
|
|
583
|
+
readonly data: JsonlEvent.Data<R, T>;
|
|
584
|
+
readonly at: DateTime.Utc;
|
|
585
|
+
readonly scope?: string | undefined;
|
|
586
|
+
}) => Result.Result<string, UnknownEvent | InvalidData | UnserializableData>;
|
|
587
|
+
/**
|
|
588
|
+
* {@link (Envelope:variable).decodeResult} in the `Effect` channel.
|
|
589
|
+
*
|
|
590
|
+
* Prefer the sync form in a hook or any caller that has no runtime; this
|
|
591
|
+
* exists for programs that already are one.
|
|
592
|
+
*/
|
|
593
|
+
readonly decode: <const R extends JsonlEvent.Registry>(events: R, line: LineSlice) => Effect.Effect<EnvelopeUnion<R>, MalformedLine | InvalidData | UnknownEvent>;
|
|
594
|
+
/**
|
|
595
|
+
* {@link (Envelope:variable).encodeResult} in the `Effect` channel.
|
|
596
|
+
*/
|
|
597
|
+
readonly encode: <const R extends JsonlEvent.Registry, const T extends JsonlEvent.Tag<R>>(events: R, envelope: {
|
|
598
|
+
readonly event: T;
|
|
599
|
+
readonly data: JsonlEvent.Data<R, T>;
|
|
600
|
+
readonly at: DateTime.Utc;
|
|
601
|
+
readonly scope?: string | undefined;
|
|
602
|
+
}) => Effect.Effect<string, UnknownEvent | InvalidData | UnserializableData>;
|
|
603
|
+
};
|
|
604
|
+
//#endregion
|
|
605
|
+
//#region src/Slice.d.ts
|
|
606
|
+
/**
|
|
607
|
+
* A filter over envelope fields.
|
|
608
|
+
*
|
|
609
|
+
* Every field is optional and they combine with AND. An omitted field does not
|
|
610
|
+
* filter; an empty `events: []` or `scopes: []` matches nothing, which is the
|
|
611
|
+
* honest reading of "restrict to this empty set" and is tested.
|
|
612
|
+
*
|
|
613
|
+
* @public
|
|
614
|
+
*/
|
|
615
|
+
interface Slice<R extends JsonlEvent.Registry, T extends JsonlEvent.Tag<R> = JsonlEvent.Tag<R>> {
|
|
616
|
+
/**
|
|
617
|
+
* Restrict to these event tags.
|
|
618
|
+
*
|
|
619
|
+
* Passing a literal array narrows the element type of the surface's stream to
|
|
620
|
+
* exactly those variants, so a projection over a slice is exhaustively
|
|
621
|
+
* checkable.
|
|
622
|
+
*/
|
|
623
|
+
readonly events?: ReadonlyArray<T> | undefined;
|
|
624
|
+
/** Restrict to these partition keys. An envelope with no `scope` matches none of them. */
|
|
625
|
+
readonly scopes?: ReadonlyArray<string> | undefined;
|
|
626
|
+
/** Lower bound on `at`, **inclusive**. */
|
|
627
|
+
readonly from?: DateTime.Utc | undefined;
|
|
628
|
+
/**
|
|
629
|
+
* Upper bound on `at`, **exclusive**.
|
|
630
|
+
*
|
|
631
|
+
* Half-open so adjacent windows tile without double-delivering an envelope on
|
|
632
|
+
* the seam — which matters because `at` collisions are ordinary at
|
|
633
|
+
* millisecond resolution and universal under `TestClock`.
|
|
634
|
+
*/
|
|
635
|
+
readonly to?: DateTime.Utc | undefined;
|
|
636
|
+
}
|
|
637
|
+
/**
|
|
638
|
+
* A {@link Slice} plus a resume point.
|
|
639
|
+
*
|
|
640
|
+
* @public
|
|
641
|
+
*/
|
|
642
|
+
interface CursoredSlice<R extends JsonlEvent.Registry, T extends JsonlEvent.Tag<R> = JsonlEvent.Tag<R>> extends Slice<R, T> {
|
|
643
|
+
/**
|
|
644
|
+
* Resume from this logical byte offset, **inclusive**.
|
|
645
|
+
*
|
|
646
|
+
* Offsets are post-BOM, matching every offset this package emits. Persist a
|
|
647
|
+
* processed envelope's `line.end` and pass it back to replay exactly the
|
|
648
|
+
* remainder: `end` is the start of the next line, so nothing is redelivered
|
|
649
|
+
* and nothing is skipped.
|
|
650
|
+
*/
|
|
651
|
+
readonly cursor?: number | undefined;
|
|
652
|
+
}
|
|
653
|
+
//#endregion
|
|
654
|
+
//#region src/Journal.d.ts
|
|
655
|
+
/**
|
|
656
|
+
* The failure channel of a write operation.
|
|
657
|
+
*
|
|
658
|
+
* `PlatformError` rides through untranslated per the taxonomy's rule. **Any
|
|
659
|
+
* `PlatformError` out of an append must be treated as a possibly-torn tail**:
|
|
660
|
+
* `writeAll` reports no byte count, so a failure cannot be read as "nothing was
|
|
661
|
+
* written". The next reader walks back over whatever fragment survived.
|
|
662
|
+
*
|
|
663
|
+
* @public
|
|
664
|
+
*/
|
|
665
|
+
type JournalWriteError = JournalClosed | JournalNotFound | TerminalViolation | UnknownEvent | MalformedLine | InvalidData | UnserializableData | PlatformError.PlatformError;
|
|
666
|
+
/**
|
|
667
|
+
* The failure channel of a read operation.
|
|
668
|
+
*
|
|
669
|
+
* @public
|
|
670
|
+
*/
|
|
671
|
+
type JournalReadError = JournalNotFound | PlatformError.PlatformError;
|
|
672
|
+
/**
|
|
673
|
+
* Options for one append.
|
|
674
|
+
*
|
|
675
|
+
* @public
|
|
676
|
+
*/
|
|
677
|
+
interface AppendOptions {
|
|
678
|
+
/** The partition key to stamp on the envelope. */
|
|
679
|
+
readonly scope?: string | undefined;
|
|
680
|
+
}
|
|
681
|
+
/**
|
|
682
|
+
* The shape of a `Journal`, typed by its registry.
|
|
683
|
+
*
|
|
684
|
+
* @public
|
|
685
|
+
*/
|
|
686
|
+
interface JournalShape<R extends JsonlEvent.Registry> {
|
|
687
|
+
/**
|
|
688
|
+
* Validate, encode and append one envelope.
|
|
689
|
+
*
|
|
690
|
+
* `at` is stamped here from the Effect `Clock` — never by the caller — so
|
|
691
|
+
* ordering does not depend on two writers agreeing about the time, and
|
|
692
|
+
* `TestClock` controls it exactly in tests.
|
|
693
|
+
*/
|
|
694
|
+
readonly append: <T extends JsonlEvent.Tag<R>>(event: T, data: JsonlEvent.Data<R, T>, options?: AppendOptions | undefined) => Effect.Effect<EnvelopeWithTag<R, T>, JournalWriteError>;
|
|
695
|
+
/**
|
|
696
|
+
* Inherit-and-patch: read the last valid envelope, shallow-merge `patch`
|
|
697
|
+
* over its `data`, validate the result and append it.
|
|
698
|
+
*
|
|
699
|
+
* This is the snapshot-journal primitive — each line is a complete state and
|
|
700
|
+
* most transitions change one field. The merge is **shallow** by decision: a
|
|
701
|
+
* nested object in the patch replaces the one beneath it.
|
|
702
|
+
*/
|
|
703
|
+
readonly appendPatch: <T extends JsonlEvent.Tag<R>>(event: T, patch: Partial<JsonlEvent.Data<R, T>>, options?: AppendOptions | undefined) => Effect.Effect<EnvelopeWithTag<R, T>, JournalWriteError>;
|
|
704
|
+
/**
|
|
705
|
+
* The current last valid **envelope**, as an observable `Option`.
|
|
706
|
+
*
|
|
707
|
+
* "Last valid" always means the last valid envelope, never merely the last
|
|
708
|
+
* valid JSON — a torn scalar tail parses as a different value and only the
|
|
709
|
+
* envelope contract detects it.
|
|
710
|
+
*/
|
|
711
|
+
readonly latest: SubscriptionRef.SubscriptionRef<Option.Option<EnvelopeUnion<R>>>;
|
|
712
|
+
/**
|
|
713
|
+
* Whether the journal is quiescent — its tail is an event marked `terminal`.
|
|
714
|
+
*
|
|
715
|
+
* Derived from {@link JournalShape.latest} rather than tracked separately, so
|
|
716
|
+
* the two cannot disagree.
|
|
717
|
+
*/
|
|
718
|
+
readonly quiescent: Effect.Effect<boolean>;
|
|
719
|
+
/**
|
|
720
|
+
* Historical read: a finite `Stream` of the envelopes matching `slice`.
|
|
721
|
+
*
|
|
722
|
+
* Every element carries its logical byte offsets on `line`, so iteration is
|
|
723
|
+
* resumable across process restarts by persisting `line.end` and passing it
|
|
724
|
+
* back as `cursor`.
|
|
725
|
+
*
|
|
726
|
+
* Filtering happens on the envelope **frame**, strictly before the payload
|
|
727
|
+
* schema runs, so a non-matching line's `data` is never decoded.
|
|
728
|
+
*
|
|
729
|
+
* @remarks
|
|
730
|
+
* **Cost, stated rather than implied.** As built, the requested region —
|
|
731
|
+
* `cursor` to the end of the file — is read in ONE allocation bounded by the
|
|
732
|
+
* file's size, and the matching envelopes are buffered before the first is
|
|
733
|
+
* emitted. An unsliced `query()` over a large journal therefore does hold
|
|
734
|
+
* that journal in memory; a `cursor` is what bounds the read, which is
|
|
735
|
+
* precisely what a resuming consumer already persists. The window-bounded
|
|
736
|
+
* reads are `latest` and the `lastValid`-backed paths. Emitting per window,
|
|
737
|
+
* so an unsliced query costs a window rather than a file, is tracked as
|
|
738
|
+
* spencerbeggs/effected#233.
|
|
739
|
+
*/
|
|
740
|
+
readonly query: <T extends JsonlEvent.Tag<R> = JsonlEvent.Tag<R>>(slice?: CursoredSlice<R, T> | undefined) => Stream.Stream<EnvelopeWithTag<R, T>, JournalReadError | JsonlError>;
|
|
741
|
+
/**
|
|
742
|
+
* Live read: a `Stream` of matching envelopes as they are appended.
|
|
743
|
+
*
|
|
744
|
+
* With a `cursor`, replay-from-cursor and the live tail are **one seam**: the
|
|
745
|
+
* replayed history and the live tail are the same stream, filtered the same
|
|
746
|
+
* way, so a consumer cannot observe a gap or a duplicate at the join.
|
|
747
|
+
*
|
|
748
|
+
* The stream **ends** — it never hangs — when the journal becomes quiescent
|
|
749
|
+
* (a `terminal` event reaches the tail) or its scope closes. Subscribing to
|
|
750
|
+
* an already-quiescent journal ends immediately.
|
|
751
|
+
*
|
|
752
|
+
* @remarks
|
|
753
|
+
* Two properties are worth knowing before relying on them.
|
|
754
|
+
*
|
|
755
|
+
* The **delivered-before-end guarantee is scoped to consuming subscribers**:
|
|
756
|
+
* every append that completed is delivered before the stream ends, provided
|
|
757
|
+
* the subscriber keeps taking. One that stops taking cannot observe
|
|
758
|
+
* completion by definition, and shutdown will not wait on it past its bound.
|
|
759
|
+
*
|
|
760
|
+
* The **filter-before-decode guarantee is structural on the historical half
|
|
761
|
+
* only**. Replayed lines are filtered on the frame before their payload is
|
|
762
|
+
* decoded. Live envelopes arrive already decoded — the appender decoded its
|
|
763
|
+
* own payload in order to return it — so a filtered-out live envelope was
|
|
764
|
+
* decoded once, by the writer, not by each subscriber.
|
|
765
|
+
*/
|
|
766
|
+
readonly changes: <T extends JsonlEvent.Tag<R> = JsonlEvent.Tag<R>>(slice?: CursoredSlice<R, T> | undefined) => Stream.Stream<EnvelopeWithTag<R, T>, JournalReadError | JsonlError>;
|
|
767
|
+
/**
|
|
768
|
+
* A running fold over a slice, emitted as it advances.
|
|
769
|
+
*
|
|
770
|
+
* The fold only ever sees its own slice, so a per-scope state machine over a
|
|
771
|
+
* shared journal is exhaustively checkable against just that scope's events.
|
|
772
|
+
*/
|
|
773
|
+
readonly projection: <S, T extends JsonlEvent.Tag<R> = JsonlEvent.Tag<R>>(initial: S, fold: (state: S, envelope: EnvelopeWithTag<R, T>) => S, slice?: CursoredSlice<R, T> | undefined) => Stream.Stream<S, JournalReadError | JsonlError>;
|
|
774
|
+
/** Create the journal file if it does not exist. Always explicit. */
|
|
775
|
+
readonly create: Effect.Effect<void, PlatformError.PlatformError>;
|
|
776
|
+
/** Remove the journal file. Always explicit. */
|
|
777
|
+
readonly remove: Effect.Effect<void, PlatformError.PlatformError>;
|
|
778
|
+
/**
|
|
779
|
+
* The internal hub, exposed for the read surfaces built over it.
|
|
780
|
+
*
|
|
781
|
+
* Elements are `Take` chunks and end-of-stream is a published `Exit`, which
|
|
782
|
+
* is what lets quiescence and graceful shutdown arrive at a subscriber as a
|
|
783
|
+
* normal stream end. Never shut this down to signal completion —
|
|
784
|
+
* `PubSub.shutdown` *interrupts* subscribers, which is the tear the design
|
|
785
|
+
* forbids.
|
|
786
|
+
*
|
|
787
|
+
* @internal
|
|
788
|
+
*/
|
|
789
|
+
readonly hub: PubSub.PubSub<Take.Take<EnvelopeUnion<R>, JsonlError>>;
|
|
790
|
+
}
|
|
791
|
+
/**
|
|
792
|
+
* Configuration for one journal layer.
|
|
793
|
+
*
|
|
794
|
+
* @public
|
|
795
|
+
*/
|
|
796
|
+
interface JournalConfig {
|
|
797
|
+
/** Path to the journal file. It need not exist yet. */
|
|
798
|
+
readonly path: string;
|
|
799
|
+
/**
|
|
800
|
+
* The directory to watch while the journal does not exist yet.
|
|
801
|
+
*
|
|
802
|
+
* Defaults to `path`'s parent, derived by taking everything before its last
|
|
803
|
+
* `/` or `\`. That derivation is the one piece of path arithmetic this
|
|
804
|
+
* package performs, and it is here so the common case needs no second field;
|
|
805
|
+
* name the directory explicitly when your path convention does not survive
|
|
806
|
+
* it — a drive-relative Windows path, or a path whose parent is not a plain
|
|
807
|
+
* prefix of it.
|
|
808
|
+
*
|
|
809
|
+
* It is used for exactly one thing: `FileSystem.watch` on a journal that has
|
|
810
|
+
* not been created yet. Once the journal exists the watch moves to the file
|
|
811
|
+
* itself and this value is not consulted again.
|
|
812
|
+
*/
|
|
813
|
+
readonly directory?: string | undefined;
|
|
814
|
+
/**
|
|
815
|
+
* Capacity of the internal subscriber hub. Bounded with backpressure: the
|
|
816
|
+
* journal never drops an envelope for a slow subscriber, so slowness shows
|
|
817
|
+
* up at the producer where it is visible rather than as a silent gap in
|
|
818
|
+
* somebody's subscription.
|
|
819
|
+
*/
|
|
820
|
+
readonly capacity?: number | undefined;
|
|
821
|
+
/**
|
|
822
|
+
* How long scope close waits for the terminal `Exit` to be accepted.
|
|
823
|
+
*
|
|
824
|
+
* Defaults to five seconds. Bounded rather than indefinite because a
|
|
825
|
+
* subscriber that never consumes cannot observe completion by definition,
|
|
826
|
+
* and must not be able to hold shutdown hostage.
|
|
827
|
+
*/
|
|
828
|
+
readonly shutdownPublishTimeout?: Duration.Input | undefined;
|
|
829
|
+
}
|
|
830
|
+
/**
|
|
831
|
+
* A per-registry `Journal` service class.
|
|
832
|
+
*
|
|
833
|
+
* @public
|
|
834
|
+
*/
|
|
835
|
+
interface JournalClass<Self, Id extends string, R extends JsonlEvent.Registry> extends Context.ServiceClass<Self, Id, JournalShape<R>> {
|
|
836
|
+
/** The registry this journal was defined over. */
|
|
837
|
+
readonly events: R;
|
|
838
|
+
/**
|
|
839
|
+
* Build the layer for this journal.
|
|
840
|
+
*
|
|
841
|
+
* @remarks
|
|
842
|
+
* **Construction can fail with a `PlatformError`, and that is deliberate.** A
|
|
843
|
+
* journal file that does not exist yet is a legal state and constructs
|
|
844
|
+
* cleanly — but a file that exists and cannot be read (`EACCES`, a bad
|
|
845
|
+
* handle) is a real failure, and typing this channel `never` would have made
|
|
846
|
+
* it arrive as an untypeable defect that no caller could catch.
|
|
847
|
+
*
|
|
848
|
+
* **Bind the result to a const and provide that const.** Layers are memoized
|
|
849
|
+
* by reference, so calling this twice mints **two independent journals over
|
|
850
|
+
* one file** — two semaphores, two hubs, two `latest` refs — and their
|
|
851
|
+
* appends are not serialized against each other. That is the in-process form
|
|
852
|
+
* of exactly the interleaving the cooperative-writer rules exist to prevent,
|
|
853
|
+
* and it typechecks perfectly.
|
|
854
|
+
*
|
|
855
|
+
* ```ts
|
|
856
|
+
* // correct
|
|
857
|
+
* export const layer = MailJournal.layer({ path });
|
|
858
|
+
*
|
|
859
|
+
* // WRONG — a second, unserialized journal over the same file
|
|
860
|
+
* Effect.provide(program, MailJournal.layer({ path }));
|
|
861
|
+
* ```
|
|
862
|
+
*/
|
|
863
|
+
readonly layer: (config: JournalConfig) => Layer.Layer<Self, PlatformError.PlatformError, FileSystem.FileSystem>;
|
|
864
|
+
}
|
|
865
|
+
/**
|
|
866
|
+
* Define a `Journal` service class over a registry.
|
|
867
|
+
*
|
|
868
|
+
* A `Context.Service` cannot itself be generic over the registry — the shape
|
|
869
|
+
* binds at declaration and a Key cannot be parameterized at retrieval — so each
|
|
870
|
+
* registry gets its own uniquely-keyed class, and several journals coexist in
|
|
871
|
+
* one layer graph with each one's operations typed by *its* registry. This
|
|
872
|
+
* mirrors `ConfigFile.Service<Self, A>()(id)`.
|
|
873
|
+
*
|
|
874
|
+
* @example
|
|
875
|
+
* ```ts
|
|
876
|
+
* import { Journal, JsonlEvent } from "@effected/jsonl";
|
|
877
|
+
* import { Schema } from "effect";
|
|
878
|
+
*
|
|
879
|
+
* const events = [JsonlEvent.make("started", { data: Schema.Void })] as const;
|
|
880
|
+
*
|
|
881
|
+
* class MailJournal extends Journal.Service<MailJournal>()("dogfood/MailJournal", { events }) {}
|
|
882
|
+
*
|
|
883
|
+
* export const layer = MailJournal.layer({ path: ".claude/dogfood/silk.jsonl" });
|
|
884
|
+
* ```
|
|
885
|
+
*
|
|
886
|
+
* @public
|
|
887
|
+
*/
|
|
888
|
+
declare const Journal: {
|
|
889
|
+
readonly Service: <Self>() => <const Id extends string, const R extends JsonlEvent.Registry>(id: Id, options: {
|
|
890
|
+
readonly events: R;
|
|
891
|
+
}) => JournalClass<Self, Id, R>;
|
|
892
|
+
};
|
|
893
|
+
//#endregion
|
|
894
|
+
//#region src/Line.d.ts
|
|
895
|
+
declare const ParsedLine_base: Schema.Class<ParsedLine, Schema.Struct<{
|
|
896
|
+
/** Where this line lives in the source. */
|
|
897
|
+
readonly line: typeof LineSlice;
|
|
898
|
+
/** The parsed JSON value — any JSON value, not necessarily an object. */
|
|
899
|
+
readonly value: Schema.Unknown;
|
|
900
|
+
}>, {}>;
|
|
901
|
+
/**
|
|
902
|
+
* A line that parsed as JSON, paired with the slice it came from.
|
|
903
|
+
*
|
|
904
|
+
* `value` is deliberately `unknown`: this layer knows JSON, not envelopes.
|
|
905
|
+
* Validating `event`, `at`, `scope` and the registered payload schema is the
|
|
906
|
+
* envelope layer's job, and keeping the split means a malformed *envelope* and
|
|
907
|
+
* a malformed *line* stay distinguishable failures.
|
|
908
|
+
*
|
|
909
|
+
* @public
|
|
910
|
+
*/
|
|
911
|
+
declare class ParsedLine extends ParsedLine_base {}
|
|
912
|
+
/**
|
|
913
|
+
* Splitting, parsing and corrupt-tail walk-back over JSONL text.
|
|
914
|
+
*
|
|
915
|
+
* Every operation is total and synchronous: it returns a value for every
|
|
916
|
+
* input, including empty text, torn tails, and text that is not JSONL at all.
|
|
917
|
+
* Nothing here throws — a `JSON.parse` failure becomes a {@link MalformedLine}
|
|
918
|
+
* in the returned `Result`, because a journal is untrusted input and untrusted
|
|
919
|
+
* input fails typed.
|
|
920
|
+
*
|
|
921
|
+
* @public
|
|
922
|
+
*/
|
|
923
|
+
declare class Line {
|
|
924
|
+
/**
|
|
925
|
+
* The UTF-8 byte length of a string.
|
|
926
|
+
*
|
|
927
|
+
* Exposed because callers doing their own offset arithmetic must measure
|
|
928
|
+
* the same way this module does. `String.length` is a different number for
|
|
929
|
+
* any non-ASCII line.
|
|
930
|
+
*
|
|
931
|
+
* An unpaired surrogate counts as the 3 bytes of U+FFFD, matching what a
|
|
932
|
+
* UTF-8 write actually emits. The offsets therefore stay true to the file
|
|
933
|
+
* even though the character itself cannot survive the round trip — text
|
|
934
|
+
* read back from a UTF-8 journal never contains one.
|
|
935
|
+
*
|
|
936
|
+
* @example
|
|
937
|
+
* ```ts
|
|
938
|
+
* import { Line } from "@effected/jsonl";
|
|
939
|
+
*
|
|
940
|
+
* Line.byteLength("\u{1F600}"); // 4
|
|
941
|
+
* "\u{1F600}".length; // 2 — the trap
|
|
942
|
+
* ```
|
|
943
|
+
*/
|
|
944
|
+
static byteLength(text: string): number;
|
|
945
|
+
/**
|
|
946
|
+
* Split text into candidate lines with byte-exact offsets.
|
|
947
|
+
*
|
|
948
|
+
* Lines are separated by `\n`; a `\r` immediately preceding it is treated as
|
|
949
|
+
* part of the terminator, so CRLF journals split identically to LF ones. A
|
|
950
|
+
* bare `\r` is ordinary content, because only `\n` terminates.
|
|
951
|
+
*
|
|
952
|
+
* A trailing terminator does **not** produce a phantom empty final line, so
|
|
953
|
+
* a well-formed journal yields exactly as many slices as it has entries.
|
|
954
|
+
* Interior blank lines *are* returned — they are real bytes at real offsets,
|
|
955
|
+
* and hiding them would make the offsets lie.
|
|
956
|
+
*
|
|
957
|
+
* Only the final slice can have `terminated: false`.
|
|
958
|
+
*
|
|
959
|
+
* @param text - JSONL source text.
|
|
960
|
+
* @returns One {@link LineSlice} per candidate line, in source order.
|
|
961
|
+
*/
|
|
962
|
+
static split(text: string): ReadonlyArray<LineSlice>;
|
|
963
|
+
/**
|
|
964
|
+
* The byte offset up to which this text has been fully consumed.
|
|
965
|
+
*
|
|
966
|
+
* This is the offset past the last **terminated** line. An unterminated
|
|
967
|
+
* final line is left unconsumed on purpose: it may be a writer caught
|
|
968
|
+
* mid-append, and re-reading from this offset once the file grows sees the
|
|
969
|
+
* completed line rather than gluing a stale fragment to fresh bytes.
|
|
970
|
+
*
|
|
971
|
+
* @param text - JSONL source text.
|
|
972
|
+
* @returns The resume cursor, in UTF-8 bytes.
|
|
973
|
+
*/
|
|
974
|
+
static consumedOffset(text: string): number;
|
|
975
|
+
/**
|
|
976
|
+
* Parse one candidate line's JSON.
|
|
977
|
+
*
|
|
978
|
+
* Any JSON value succeeds — objects, arrays and scalars alike. This layer
|
|
979
|
+
* does not know what an envelope is.
|
|
980
|
+
*
|
|
981
|
+
* @param line - A slice from {@link Line.split}.
|
|
982
|
+
* @returns The {@link ParsedLine}, or a {@link MalformedLine} carrying the
|
|
983
|
+
* slice so the caller can locate the damage and decide whether an
|
|
984
|
+
* unterminated line is a torn tail worth waiting for.
|
|
985
|
+
*/
|
|
986
|
+
static parseResult(line: LineSlice): Result.Result<ParsedLine, MalformedLine>;
|
|
987
|
+
/**
|
|
988
|
+
* Parse every non-blank line, reporting failures rather than dropping them.
|
|
989
|
+
*
|
|
990
|
+
* There is deliberately no "valid lines only" variant. A hole in the middle
|
|
991
|
+
* of a journal is information — it means a line was written that no reader
|
|
992
|
+
* can interpret — and whether that is tolerable is the caller's decision,
|
|
993
|
+
* not this module's. Filter the `Result`s yourself and the choice stays
|
|
994
|
+
* visible at the call site.
|
|
995
|
+
*
|
|
996
|
+
* Whitespace-only lines are skipped rather than reported: they carry no
|
|
997
|
+
* information to lose, and reporting them would make a hand-edited journal
|
|
998
|
+
* look corrupt.
|
|
999
|
+
*
|
|
1000
|
+
* @param text - JSONL source text.
|
|
1001
|
+
* @returns One `Result` per non-blank line, in source order.
|
|
1002
|
+
*/
|
|
1003
|
+
static parseAll(text: string): ReadonlyArray<Result.Result<ParsedLine, MalformedLine>>;
|
|
1004
|
+
/**
|
|
1005
|
+
* Walk back from the end to the last line that parses.
|
|
1006
|
+
*
|
|
1007
|
+
* This is the whole read path for a snapshot-style journal, where the
|
|
1008
|
+
* current state *is* the last valid line: a session killed mid-append leaves
|
|
1009
|
+
* a partial final line, and the walk-back steps over it — and over any
|
|
1010
|
+
* number of malformed or blank trailing lines — to the last line that means
|
|
1011
|
+
* something.
|
|
1012
|
+
*
|
|
1013
|
+
* Parsing stops at the first success, so the cost is proportional to the
|
|
1014
|
+
* damage at the tail rather than to the age of the file.
|
|
1015
|
+
*
|
|
1016
|
+
* **A torn tail is only detectable when its fragment fails to parse.**
|
|
1017
|
+
* Every strict prefix of a JSON *object* is invalid JSON, so a truncated
|
|
1018
|
+
* `{"event":"unlinked"}` is always caught. A truncated *scalar* is not:
|
|
1019
|
+
* `42` cut mid-write leaves `4`, which parses cleanly as a different value.
|
|
1020
|
+
* The JSON layer cannot close that hole — the envelope layer can, because
|
|
1021
|
+
* `4` is not an envelope, which is one more reason the envelope is the
|
|
1022
|
+
* package's contract rather than an option. Check `LineSlice.terminated` on
|
|
1023
|
+
* the result when it matters.
|
|
1024
|
+
*
|
|
1025
|
+
* @param text - JSONL source text.
|
|
1026
|
+
* @returns The last parseable line, or `Option.none()` if none parses.
|
|
1027
|
+
*/
|
|
1028
|
+
static lastValid(text: string): Option.Option<ParsedLine>;
|
|
1029
|
+
}
|
|
1030
|
+
//#endregion
|
|
1031
|
+
export { type AppendOptions, type CursoredSlice, type DataSchema, Envelope, EnvelopeFrame, type EnvelopeOf, type EnvelopeUnion, type EnvelopeWithTag, InvalidData, Journal, type JournalClass, JournalClosed, type JournalConfig, JournalNotFound, type JournalReadError, JournalResync, type JournalShape, type JournalWriteError, type JsonlError, JsonlEvent, JsonlEventTypeId, Line, LineSlice, MalformedLine, ParsedLine, type Slice, TerminalViolation, UnknownEvent, UnserializableData };
|
|
1032
|
+
//# sourceMappingURL=index.d.ts.map
|