@effected/jsonl 0.9.0 → 0.10.0

package/index.js

@@ -1,8 +1,8 @@
 import { LineSlice } from "./LineSlice.js";
 import { InvalidData, JournalClosed, JournalNotFound, JournalResync, MalformedLine, TerminalViolation, UnknownEvent, UnserializableData } from "./JsonlError.js";
-import { Line, ParsedLine } from "./Line.js";
-import { Envelope, EnvelopeFrame } from "./Envelope.js";
+import { Line } from "./Line.js";
+import { Envelope } from "./Envelope.js";
 import { Journal } from "./Journal.js";
-import { JsonlEvent, JsonlEventTypeId } from "./JsonlEvent.js";
+import { JsonlEvent } from "./JsonlEvent.js";
 
-export { Envelope, EnvelopeFrame, InvalidData, Journal, JournalClosed, JournalNotFound, JournalResync, JsonlEvent, JsonlEventTypeId, Line, LineSlice, MalformedLine, ParsedLine, TerminalViolation, UnknownEvent, UnserializableData };
+export { Envelope, InvalidData, Journal, JournalClosed, JournalNotFound, JournalResync, JsonlEvent, Line, LineSlice, MalformedLine, TerminalViolation, UnknownEvent, UnserializableData };

package/internal/engine.js

@@ -0,0 +1,399 @@
+import { JournalClosed, JournalNotFound, JournalResync, TerminalViolation } from "../JsonlError.js";
+import { Line, isBlank } from "../Line.js";
+import { Envelope, completeResult, frameResult } from "../Envelope.js";
+import { matchesFrame } from "../Slice.js";
+import { canMerge, shallowMerge } from "./merge.js";
+import { probeBomBytes, readLinePages, readTailUntil } from "./tail.js";
+import { ByteSize, Channel, DateTime, Deferred, Duration, Effect, Exit, Fiber, FileSystem, Option, PubSub, Result, Semaphore, Stream, SubscriptionRef } from "effect";
+
+//#region src/internal/engine.ts
+const positionOf = (item) => Result.isSuccess(item) ? item.success.position : item.failure.position;
+const SHUTDOWN_PUBLISH_TIMEOUT = Duration.seconds(5);
+/**
+* How many times the watcher re-arms a watch that ends without observing
+* anything before it gives up. No timer is permitted here to back off a watch
+* that completes instantly, so giving up is the honest failure: local appends
+* keep working, live observation does not.
+*/
+const MAX_IMMEDIATE_REARMS = 8;
+/**
+* Scheduler turns yielded to a freshly-forked watch consumer before the
+* catch-up read. `fs.watch` exposes no "registered" signal, so arming is
+* ordered ahead of catch-up by scheduling, not synchronisation — empirically
+* sufficient, not a proof. The airtight primitive would be
+* `FileSystem.WatchBackend.register`, at the cost of `WatchBackend` in `R`.
+*/
+const ARM_YIELDS = 3;
+const encoder = new TextEncoder();
+/**
+* Index of the last path separator, on either convention — a Windows path
+* contains no `/` at all.
+*/
+const lastSeparator = (path) => Math.max(path.lastIndexOf("/"), path.lastIndexOf("\\"));
+/** The last segment of a path, for comparison only — never opened. */
+const basenameOf = (path) => path.slice(lastSeparator(path) + 1);
+/** {@link JournalConfig.directory}'s default. */
+const parentOf = (path) => {
+	const separator = lastSeparator(path);
+	if (separator < 0) return ".";
+	return separator === 0 ? path.slice(0, 1) : path.slice(0, separator);
+};
+const identityOf = (info) => Option.map(info.ino, (ino) => `${info.dev}:${ino}`);
+/**
+* Admit decoded items through a slice: matching envelopes pass, rejections end
+* the stream only when the slice asks (`onInvalid: "fail"`) and the line could
+* have mattered to it — its frame matched, or it had no frame to match.
+*/
+const admit = (items, slice) => {
+	const admitted = [];
+	for (const item of items) {
+		if (Result.isSuccess(item)) {
+			if (matchesFrame(item.success, slice)) admitted.push(item.success);
+			continue;
+		}
+		const rejected = item.failure;
+		const relevant = Option.isNone(rejected.frame) || matchesFrame(rejected.frame.value, slice);
+		if (slice?.onInvalid === "fail" && relevant) return Stream.concat(Stream.fromIterable(admitted), Stream.fail(rejected.error));
+	}
+	return Stream.fromIterable(admitted);
+};
+/**
+* Build the engine over one journal file.
+*
+* @internal
+*/
+const makeEngine = (id, events, config) => Effect.gen(function* () {
+	const fs = yield* FileSystem.FileSystem;
+	const { path } = config;
+	const terminalTags = new Set(events.filter((event) => event.terminal).map((event) => event.tag));
+	const reopenTags = new Set(events.filter((event) => event.reopen).map((event) => event.tag));
+	const isTerminal = (item) => Result.isSuccess(item) && terminalTags.has(item.success.event);
+	/** Guards the WRITE critical section: file write plus state updates. */
+	const writePermit = Semaphore.makeUnsafe(1);
+	/**
+	* Serializes ingests against each other — two overlapping runs would read
+	* the same `consumed` and publish the same lines twice. Its own lock so an
+	* append is not blocked behind a slow catch-up read.
+	*/
+	const ingestPermit = Semaphore.makeUnsafe(1);
+	/**
+	* The publish baton. Each publisher links a fresh Deferred onto this chain
+	* **under the write permit**, fixing publish order to write order, then
+	* awaits its predecessor and publishes OUTSIDE the permit — so a stalled
+	* subscriber blocks publishers without wedging writers or scope close.
+	*/
+	let publishBaton = Deferred.makeUnsafe();
+	Deferred.doneUnsafe(publishBaton, Exit.void);
+	/** Bytes of leading BOM, probed from the file — never inferred from a window. */
+	let bomBytes = 0;
+	/**
+	* The file's identity, for detecting replacement. `ino` is an `Option` in
+	* core's `File.Info`; where it is missing this degrades to truncation-only
+	* detection.
+	*/
+	let identity = Option.none();
+	/** Refusal state. Read before the permit so a late append fails fast. */
+	let closed = false;
+	/** Logical bytes consumed so far — where the next ingest resumes. */
+	let consumed = 0;
+	const hub = yield* PubSub.bounded(config.capacity ?? 64);
+	const latest = yield* SubscriptionRef.make(Option.none());
+	const logicalSize = (info) => ByteSize.toNumberUnsafe(info.size) - bomBytes;
+	/** `stat`, with a missing file as `none` rather than a failure. */
+	const statOrMissing = fs.stat(path).pipe(Effect.map(Option.some), Effect.catchReason("PlatformError", "NotFound", () => Effect.succeedNone));
+	const requireStat = Effect.flatMap(statOrMissing, (info) => Option.match(info, {
+		onNone: () => Effect.fail(new JournalNotFound({ path })),
+		onSome: Effect.succeed
+	}));
+	const isTerminalTail = Effect.map(SubscriptionRef.get(latest), (current) => Option.isSome(current) && terminalTags.has(current.value.event));
+	/**
+	* Decode one line into an item, filter before decode: a line whose frame
+	* does not match `slice` returns `undefined` and its payload schema never
+	* runs. With no slice, every line becomes an item.
+	*/
+	const itemOf = (line, slice) => {
+		const position = {
+			offset: line.offset,
+			end: line.end
+		};
+		const frame = frameResult(line);
+		if (Result.isFailure(frame)) return Result.fail({
+			error: frame.failure,
+			frame: Option.none(),
+			position
+		});
+		if (!matchesFrame(frame.success, slice)) return;
+		const envelope = completeResult(events, line, frame.success);
+		return Result.isSuccess(envelope) ? Result.succeed(envelope.success) : Result.fail({
+			error: envelope.failure,
+			frame: Option.some(frame.success),
+			position
+		});
+	};
+	const itemsOf = (page, slice) => {
+		const items = [];
+		for (const line of Line.split(page.text, page.start)) {
+			if (!line.terminated || isBlank(line)) continue;
+			const item = itemOf(line, slice);
+			if (item !== void 0) items.push(item);
+		}
+		return items;
+	};
+	/**
+	* The complete, non-blank lines in `[from, to)`, in file order. `advanced`
+	* is where the last COMPLETE line ended: a torn tail holds the offset
+	* until its writer finishes it.
+	*/
+	const scanRange = (from, to) => Effect.gen(function* () {
+		const lines = [];
+		let advanced = from;
+		yield* Stream.runForEach(readLinePages(fs, path, from, to, bomBytes), (page) => Effect.sync(() => {
+			for (const line of Line.split(page.text, page.start)) {
+				if (!line.terminated) return;
+				advanced = line.end;
+				if (!isBlank(line)) lines.push(line);
+			}
+		}));
+		return {
+			lines,
+			advanced
+		};
+	});
+	const toItems = (lines) => lines.map((line) => itemOf(line, void 0));
+	/** Decode the gap `[from, to)` a watcher catch-up has to publish. */
+	const decodeRange = (from, to) => Effect.map(scanRange(from, to), ({ lines, advanced }) => ({
+		items: toItems(lines),
+		advanced
+	}));
+	const lastEnvelope = (items) => {
+		for (let index = items.length - 1; index >= 0; index--) {
+			const item = items[index];
+			if (Result.isSuccess(item)) return Option.some(item.success);
+		}
+		return Option.none();
+	};
+	/** Link onto the publish chain. Call ONLY under the write permit. */
+	const linkBaton = () => {
+		const predecessor = publishBaton;
+		const baton = Deferred.makeUnsafe();
+		publishBaton = baton;
+		return {
+			predecessor,
+			baton
+		};
+	};
+	/**
+	* Publish in chain order, outside the write permit. `uninterruptibleMask`
+	* installs the baton hand-off before any interrupt can land, while the
+	* await-and-publish itself stays interruptible — otherwise an interrupt in
+	* between would leave every later publisher waiting on a baton nobody
+	* passes.
+	*/
+	const publishAfter = (link, items) => Effect.uninterruptibleMask((restore) => restore(Effect.gen(function* () {
+		yield* Deferred.await(link.predecessor);
+		for (const item of items) yield* PubSub.publish(hub, [item]);
+	})).pipe(Effect.ensuring(Deferred.done(link.baton, Exit.void))));
+	/**
+	* Adopt the file as it is now: its BOM, its identity, its current state,
+	* and the end of its last complete line as the resume point. Construction
+	* and recovery from a resync both start here, so there is one definition
+	* of "caught up".
+	*/
+	const seed = (info) => Effect.gen(function* () {
+		bomBytes = yield* probeBomBytes(fs, path);
+		identity = identityOf(info);
+		const resume = yield* readTailUntil(fs, path, bomBytes, (window) => {
+			const lines = Line.split(window.text, window.start);
+			for (let index = lines.length - 1; index >= 0; index--) {
+				const line = lines[index];
+				if (line.terminated) return Option.some(line.end);
+			}
+			return Option.none();
+		});
+		consumed = Option.getOrElse(resume, () => 0);
+		const found = yield* readTailUntil(fs, path, bomBytes, (window) => Envelope.lastValid(events, window.text, window.start));
+		yield* SubscriptionRef.set(latest, found);
+	});
+	/**
+	* The one write path. `build` runs **inside the write permit**, so an
+	* inherit-and-patch reads the state it patches under the same lock that
+	* serializes the write — reading it outside was a lost-update race.
+	*/
+	const appendWith = (event, build, scope) => Effect.gen(function* () {
+		if (closed) return yield* new JournalClosed({ event });
+		const { envelope, before, after, link } = yield* writePermit.withPermits(1)(Effect.gen(function* () {
+			if (closed) return yield* new JournalClosed({ event });
+			yield* requireStat;
+			const current = yield* SubscriptionRef.get(latest);
+			if (Option.isSome(current) && terminalTags.has(current.value.event) && !reopenTags.has(event)) return yield* new TerminalViolation({
+				event,
+				terminal: current.value.event
+			});
+			const encoded = Envelope.encodeResult(events, {
+				event,
+				data: build(current),
+				at: yield* DateTime.now,
+				...scope === void 0 ? {} : { scope }
+			});
+			if (Result.isFailure(encoded)) {
+				if (encoded.failure._tag === "UnknownEvent") return yield* Effect.die(encoded.failure);
+				return yield* encoded.failure;
+			}
+			const bytes = encoder.encode(encoded.success);
+			let end = yield* Effect.scoped(Effect.gen(function* () {
+				const file = yield* fs.open(path, { flag: "a" });
+				yield* file.writeAll(bytes);
+				return logicalSize(yield* file.stat);
+			}));
+			const text = encoded.success.slice(0, -1);
+			let own = {
+				offset: end - bytes.length,
+				end,
+				length: bytes.length - 1,
+				text,
+				terminated: true
+			};
+			let before = [];
+			let after = [];
+			if (own.offset !== consumed) {
+				const scan = yield* scanRange(consumed, end);
+				const index = scan.lines.findLastIndex((line) => line.text === text && line.offset <= own.offset);
+				if (index !== -1) {
+					own = scan.lines[index];
+					before = toItems(scan.lines.slice(0, index));
+					after = toItems(scan.lines.slice(index + 1));
+					end = scan.advanced;
+				} else before = toItems(scan.lines);
+			}
+			consumed = Math.max(consumed, end);
+			const decoded = Envelope.decodeResult(events, own);
+			if (Result.isFailure(decoded)) {
+				if (decoded.failure._tag === "InvalidData") return yield* decoded.failure;
+				return yield* Effect.die(decoded.failure);
+			}
+			yield* SubscriptionRef.set(latest, Option.orElse(lastEnvelope(after), () => Option.some(decoded.success)));
+			return {
+				envelope: decoded.success,
+				before,
+				after,
+				link: linkBaton()
+			};
+		}));
+		yield* publishAfter(link, [
+			...before,
+			Result.succeed(envelope),
+			...after
+		]);
+		return envelope;
+	}).pipe(Effect.withSpan(`${id}.append`, { attributes: { event } }));
+	/**
+	* The historical read, one array of items per page. The region is fixed
+	* by ONE stat, so an append landing while the pages are read stays out.
+	*/
+	const history = (slice) => Stream.unwrap(Effect.map(requireStat, (info) => readLinePages(fs, path, slice?.cursor ?? 0, logicalSize(info), bomBytes).pipe(Stream.map((page) => itemsOf(page, slice)))));
+	const changes = (slice) => Stream.unwrap(Effect.gen(function* () {
+		const replay = slice?.cursor === void 0 ? Stream.empty : history(slice);
+		if ((yield* isTerminalTail) || closed) return replay.pipe(Stream.flatMap((items) => admit(items, slice)));
+		const subscription = yield* PubSub.subscribe(hub);
+		let replayedThrough = slice?.cursor ?? 0;
+		const replayed = replay.pipe(Stream.tap((items) => Effect.sync(() => {
+			const last = items.at(-1);
+			if (last !== void 0) replayedThrough = Math.max(replayedThrough, positionOf(last).end);
+		})));
+		const live = Stream.fromChannel(Channel.fromEffectTake(PubSub.take(subscription))).pipe(Stream.takeUntil(isTerminal), Stream.filter((item) => positionOf(item).offset >= replayedThrough), Stream.map((item) => [item]));
+		return Stream.concat(replayed, live).pipe(Stream.flatMap((items) => admit(items, slice)));
+	}));
+	/**
+	* Ingest whatever has been appended since `consumed`, into the same hub
+	* and `latest` as a local append, in file order.
+	*/
+	const ingest = ingestPermit.withPermits(1)(Effect.gen(function* () {
+		const floor = consumed;
+		const stat = yield* statOrMissing;
+		if (Option.isNone(stat)) return;
+		const info = stat.value;
+		const currentIdentity = identityOf(info);
+		const size = logicalSize(info);
+		const replaced = Option.isSome(identity) && Option.isSome(currentIdentity) && identity.value !== currentIdentity.value;
+		if (replaced || size < floor) {
+			const failure = new JournalResync({
+				path,
+				reason: replaced ? "replaced" : "truncated",
+				expected: floor,
+				actual: size
+			});
+			yield* writePermit.withPermits(1)(Effect.flatMap(statOrMissing, (current) => Option.match(current, {
+				onSome: seed,
+				onNone: () => Effect.andThen(Effect.sync(() => {
+					identity = Option.none();
+					bomBytes = 0;
+					consumed = 0;
+				}), SubscriptionRef.set(latest, Option.none()))
+			})));
+			yield* PubSub.publish(hub, Exit.fail(failure));
+			return;
+		}
+		identity = currentIdentity;
+		if (size <= consumed) return;
+		const { items, link } = yield* writePermit.withPermits(1)(Effect.gen(function* () {
+			const { items, advanced } = yield* decodeRange(consumed, size);
+			consumed = advanced;
+			const last = lastEnvelope(items);
+			if (Option.isSome(last)) yield* SubscriptionRef.set(latest, last);
+			return {
+				items,
+				link: linkBaton()
+			};
+		}));
+		yield* publishAfter(link, items);
+	}));
+	const journal = {
+		append: (event, data, options) => appendWith(event, () => data, options?.scope),
+		appendPatch: (event, patch, options) => appendWith(event, (current) => {
+			const base = Option.isSome(current) ? current.value.data : void 0;
+			return canMerge(base, patch) ? shallowMerge(base, patch) : patch;
+		}, options?.scope),
+		latest: SubscriptionRef.get(latest),
+		latestChanges: SubscriptionRef.changes(latest),
+		quiescent: isTerminalTail,
+		query: (slice) => history(slice).pipe(Stream.flatMap((items) => admit(items, slice))),
+		changes,
+		projection: (initial, fold, slice) => changes(slice).pipe(Stream.scan(() => initial, fold)),
+		create: Effect.scoped(fs.open(path, { flag: "a" })).pipe(Effect.asVoid, Effect.withSpan(`${id}.create`)),
+		remove: fs.remove(path, { force: true }).pipe(Effect.withSpan(`${id}.remove`))
+	};
+	const initial = yield* statOrMissing;
+	if (Option.isSome(initial)) yield* seed(initial.value).pipe(Effect.catchReason("PlatformError", "NotFound", () => Effect.void));
+	const basename = basenameOf(path);
+	const directory = config.directory ?? parentOf(path);
+	const watchJournal = fs.watch(path).pipe(Stream.runForEach(() => ingest), Effect.ignore);
+	const watchForCreation = fs.watch(directory).pipe(Stream.filter((event) => basenameOf(event.path) === basename), Stream.takeUntilEffect(() => fs.exists(path)), Stream.runForEach(() => ingest), Effect.ignore);
+	const supervise = Effect.gen(function* () {
+		let immediateCompletions = 0;
+		for (;;) {
+			const startedAt = consumed;
+			if (yield* fs.exists(path)) {
+				const armed = yield* Effect.forkChild(watchJournal);
+				for (let turn = 0; turn < ARM_YIELDS; turn++) yield* Effect.yieldNow;
+				yield* ingest;
+				yield* Fiber.join(armed);
+				yield* ingest;
+			} else yield* watchForCreation;
+			immediateCompletions = consumed === startedAt ? immediateCompletions + 1 : 0;
+			if (immediateCompletions > MAX_IMMEDIATE_REARMS) return;
+		}
+	});
+	yield* Effect.forkScoped(supervise.pipe(Effect.ignore));
+	yield* Effect.addFinalizer(() => Effect.gen(function* () {
+		closed = true;
+		const pending = yield* writePermit.withPermits(1)(Effect.sync(() => publishBaton));
+		yield* Effect.interruptible(Effect.andThen(Deferred.await(pending), PubSub.publish(hub, Exit.void))).pipe(Effect.timeout(config.shutdownPublishTimeout ?? SHUTDOWN_PUBLISH_TIMEOUT), Effect.ignore);
+	}));
+	return {
+		journal,
+		hub
+	};
+});
+
+//#endregion
+export { makeEngine };