@effected/jsonl 0.10.0 → 0.11.1

package/Journal.js

@@ -20,6 +20,7 @@ var Journal = class {
 	* @example
 	* ```ts
 	* import { Journal, JsonlEvent } from "@effected/jsonl";
+	* import { NodeJournalWatcher } from "@effected/jsonl/node";
 	* import { NodeFileSystem } from "@effect/platform-node";
 	* import { Effect, Schema } from "effect";
 	*
@@ -33,7 +34,10 @@ var Journal = class {
 	* const program = Effect.gen(function* () {
 	*   const mail = yield* Mail;
 	*   yield* mail.append("mail", { round: 1 });
-	* }).pipe(Effect.provide(Mail.layer), Effect.provide(NodeFileSystem.layer));
+	* }).pipe(
+	*   Effect.provide(Mail.layer),
+	*   Effect.provide([NodeFileSystem.layer, NodeJournalWatcher.layer]),
+	* );
 	* ```
 	*/
 	static Service() {

package/JournalWatcher.js

@@ -0,0 +1,25 @@
+import { Context } from "effect";
+
+//#region src/JournalWatcher.ts
+/**
+* How a journal is told its file changed underneath it: a watch that reports
+* when it is armed.
+*
+* Core's `FileSystem.watch` cannot serve here. Its stream registers the
+* platform watch in a fiber it forks, after an asynchronous `stat`, and gives
+* no signal when that has happened — so a journal reading what it missed
+* before the watch was live cannot know whether a concurrent append landed
+* before registration (and is lost for good) or after (and will be reported).
+* Under load that window is real. This service closes it by construction:
+* {@link JournalWatcherShape.watch} succeeds only once registration is done.
+*
+* The library owns no backend. On Node, provide `NodeJournalWatcher.layer`
+* from `@effected/jsonl/node`; elsewhere, implement the shape over the
+* platform's own watch primitive, honouring the arm-before-success contract.
+*
+* @public
+*/
+var JournalWatcher = class extends Context.Service()("@effected/jsonl/JournalWatcher") {};
+
+//#endregion
+export { JournalWatcher };

package/NodeJournalWatcher.js

@@ -0,0 +1,104 @@
+import { JournalWatcher } from "./JournalWatcher.js";
+import { Cause, Effect, Layer, PlatformError, Queue, Stream } from "effect";
+import * as NFS from "node:fs";
+
+//#region src/NodeJournalWatcher.ts
+/** The `SystemErrorTag` an errno code maps to, after core's Node filesystem mapping. */
+const reasonOf = (code) => {
+	switch (code) {
+		case "ENOENT": return "NotFound";
+		case "EACCES":
+		case "EPERM": return "PermissionDenied";
+		case "ENOTDIR":
+		case "ELOOP": return "BadResource";
+		default: return "Unknown";
+	}
+};
+const watchError = (path, cause) => PlatformError.systemError({
+	_tag: reasonOf(cause?.code),
+	module: "FileSystem",
+	method: "watch",
+	pathOrDescriptor: path,
+	syscall: "watch",
+	cause
+});
+/** The identity a watched file is armed on, or `undefined` once the path names nothing. */
+const identityAt = (path) => {
+	try {
+		const info = NFS.statSync(path);
+		return `${info.dev}:${info.ino}`;
+	} catch {
+		return;
+	}
+};
+/**
+* Register `fs.watch` synchronously, inside the acquire — so when the effect
+* succeeds the platform watch is live, which is the contract
+* {@link Jsonl.JournalWatcherShape.watch} states and core's
+* `FileSystem.watch` cannot.
+*
+* A file watch follows the inode it was armed on, and Node emits neither
+* `close` nor `error` when that inode is unlinked or renamed over: left
+* alone, the stream would outlive the file and the journal would follow a
+* dead inode forever. So every event on a file re-checks the path's identity,
+* and the stream ends — after delivering the event — once the path is gone or
+* names a different file. A directory watch never ends that way: a sibling
+* appearing or vanishing is routine there.
+*/
+const watch = (path) => Effect.gen(function* () {
+	const queue = yield* Queue.unbounded();
+	yield* Effect.acquireRelease(Effect.try({
+		try: () => {
+			const armed = NFS.statSync(path);
+			const identity = armed.isDirectory() ? void 0 : `${armed.dev}:${armed.ino}`;
+			const watcher = NFS.watch(path, (_event, name) => {
+				Queue.offerUnsafe(queue, name ?? void 0);
+				if (identity !== void 0 && identityAt(path) !== identity) Queue.endUnsafe(queue);
+			});
+			watcher.on("error", (cause) => {
+				Queue.failCauseUnsafe(queue, Cause.fail(watchError(path, cause)));
+			});
+			watcher.on("close", () => {
+				Queue.endUnsafe(queue);
+			});
+			return watcher;
+		},
+		catch: (cause) => watchError(path, cause)
+	}), (watcher) => Effect.sync(() => watcher.close()));
+	return Stream.fromQueue(queue);
+});
+/**
+* The Node implementation of {@link Jsonl.JournalWatcher}, over `node:fs`'s
+* `watch`.
+*
+* Registration happens synchronously while the watch effect runs, so a
+* journal's catch-up read after arming can never miss an append that the
+* watch also fails to report.
+*
+* @example
+* ```ts
+* import { Journal, JsonlEvent } from "@effected/jsonl";
+* import { NodeJournalWatcher } from "@effected/jsonl/node";
+* import { NodeFileSystem } from "@effect/platform-node";
+* import { Effect, Schema } from "effect";
+*
+* const events = [JsonlEvent.make("mail", { data: Schema.Struct({ round: Schema.Number }) })] as const;
+*
+* class Mail extends Journal.Service<Mail>()("app/Mail", { events, config: { path: ".app/mail.jsonl" } }) {}
+*
+* const program = Effect.gen(function* () {
+*   const mail = yield* Mail;
+*   yield* mail.append("mail", { round: 1 });
+* }).pipe(Effect.provide(Mail.layer), Effect.provide([NodeFileSystem.layer, NodeJournalWatcher.layer]));
+* ```
+*
+* @public
+*/
+var NodeJournalWatcher = class {
+	constructor() {}
+	/** Provides {@link Jsonl.JournalWatcher} over `node:fs`. */
+	static layer = Layer.succeed(JournalWatcher, { watch });
+};
+
+//#endregion
+export { NodeJournalWatcher };

package/README.md

@@ -41,7 +41,7 @@ Requires Node.js >=24.11.0. `effect` v4 is the only peer dependency, and the onl
 
 All `@effected/*` packages are ESM-only: the exports maps publish only `import` conditions, so `require()` — including tools that resolve in CJS mode — fails with Node's `ERR_PACKAGE_PATH_NOT_EXPORTED` rather than loading a CJS build that does not exist. Import from an ES module.
 
-`FileSystem` comes from `effect` core, not from a platform package, so a consumer provides it once at the edge (`NodeFileSystem.layer` from `@effect/platform-node` on Node). `Path` is deliberately not required — every path is an opaque string handed straight to `FileSystem`, and this package never joins, resolves or splits one.
+`FileSystem` comes from `effect` core, not from a platform package, so a consumer provides it once at the edge (`NodeFileSystem.layer` from `@effect/platform-node` on Node). The journal also needs a `JournalWatcher` — a file watch that reports when it is armed, so an append cannot land unseen while the watch is being set up; on Node that is `NodeJournalWatcher.layer` from the `@effected/jsonl/node` subpath, the only part of the package that touches `node:*`. `Path` is deliberately not required — every path is an opaque string handed straight to `FileSystem`, and this package never joins, resolves or splits one.
 
 ## Quick start
 
@@ -49,6 +49,7 @@ Declare the events a journal may contain, mint a service class for that registry
 
 ```ts
 import { Journal, JsonlEvent } from "@effected/jsonl";
+import { NodeJournalWatcher } from "@effected/jsonl/node";
 import { NodeFileSystem } from "@effect/platform-node";
 import { Effect, Option, Schema, Stream, SubscriptionRef } from "effect";
 
@@ -90,7 +91,7 @@ const program = Effect.gen(function* () {
   );
 });
 
-Effect.runPromise(program.pipe(Effect.provide(MailJournalLive), Effect.provide(NodeFileSystem.layer)));
+Effect.runPromise(program.pipe(Effect.provide(MailJournalLive), Effect.provide([NodeFileSystem.layer, NodeJournalWatcher.layer])));
 ```
 
 `events: ["mail-received"]` narrows the stream's element type to that variant, so a projection over a slice is exhaustively checkable rather than a cast. An event marked `terminal` makes the journal quiescent once it reaches the tail: further appends fail typed unless the event is marked `reopen`, and every `changes` stream ends rather than hanging — including one whose slice excludes the terminal event, and including a subscriber that attaches after the fact.

package/index.d.ts

@@ -582,6 +582,58 @@ export declare const Envelope: {
   }) => Result.Result<string, UnknownEvent | InvalidData | UnserializableData>;
 };
 //#endregion
+//#region src/JournalWatcher.d.ts
+/**
+ * The service shape behind {@link JournalWatcher}.
+ *
+ * @public
+ */
+interface JournalWatcherShape {
+  /**
+   * Watch `path`, a journal file or the directory it will be created in.
+   *
+   * The effect succeeds only once the watch is **registered**: every change
+   * to `path` after it succeeds is delivered on the returned stream. That
+   * ordering is the whole contract — the journal arms a watch, then reads
+   * what it missed, then follows the stream, and a change landing between a
+   * watch that was merely requested and one that was registered would be
+   * lost. The watch lives until the enclosing scope closes. Watching a file,
+   * the stream ends once the path is removed or names a different file (a
+   * rename over it), because a platform watch follows the inode it was armed
+   * on and would otherwise go blind; the journal then re-arms on whatever the
+   * path names now. Watching a directory, it does not end for a child
+   * appearing or vanishing.
+   *
+   * Each element is the name the platform reports for the change, which may
+   * be a bare basename and may be absent. The journal treats every element as
+   * an untyped poke — "re-stat the file" — and never opens what it names.
+   *
+   * A path that does not exist fails with a `PlatformError` whose reason is
+   * `NotFound`.
+   */
+  readonly watch: (path: string) => Effect.Effect<Stream.Stream<string | undefined, PlatformError.PlatformError>, PlatformError.PlatformError, Scope.Scope>;
+}
+declare const JournalWatcher_base: Context.ServiceClass<JournalWatcher, "@effected/jsonl/JournalWatcher", JournalWatcherShape>;
+/**
+ * How a journal is told its file changed underneath it: a watch that reports
+ * when it is armed.
+ *
+ * Core's `FileSystem.watch` cannot serve here. Its stream registers the
+ * platform watch in a fiber it forks, after an asynchronous `stat`, and gives
+ * no signal when that has happened — so a journal reading what it missed
+ * before the watch was live cannot know whether a concurrent append landed
+ * before registration (and is lost for good) or after (and will be reported).
+ * Under load that window is real. This service closes it by construction:
+ * {@link JournalWatcherShape.watch} succeeds only once registration is done.
+ *
+ * The library owns no backend. On Node, provide `NodeJournalWatcher.layer`
+ * from `@effected/jsonl/node`; elsewhere, implement the shape over the
+ * platform's own watch primitive, honouring the arm-before-success contract.
+ *
+ * @public
+ */
+export declare class JournalWatcher extends JournalWatcher_base {}
+//#endregion
 //#region src/Slice.d.ts
 /**
  * A filter over envelope fields, plus where to resume and what to do with a
@@ -796,13 +848,13 @@ interface JournalClass<Self, Id extends string, R extends JsonlEvent.Registry, E
    * A journal file that does not exist yet is legal and constructs cleanly; one
    * that exists and cannot be read fails with `PlatformError`.
    */
-  readonly layer: Layer.Layer<Self, PlatformError.PlatformError | E, FileSystem.FileSystem | RC>;
+  readonly layer: Layer.Layer<Self, PlatformError.PlatformError | E, FileSystem.FileSystem | JournalWatcher | RC>;
   /**
    * Build a journal over an explicit config, for a path only known at run
    * time. Wrap it in `Layer.effect(Class, Class.make(config))` and bind that
    * layer once: every build is an independent journal.
    */
-  readonly make: (config: JournalConfig) => Effect.Effect<JournalShape<R>, PlatformError.PlatformError, FileSystem.FileSystem | Scope.Scope>;
+  readonly make: (config: JournalConfig) => Effect.Effect<JournalShape<R>, PlatformError.PlatformError, FileSystem.FileSystem | JournalWatcher | Scope.Scope>;
 }
 /**
  * Defines journal services.
@@ -822,6 +874,7 @@ export declare class Journal {
    * @example
    * ```ts
    * import { Journal, JsonlEvent } from "@effected/jsonl";
+   * import { NodeJournalWatcher } from "@effected/jsonl/node";
    * import { NodeFileSystem } from "@effect/platform-node";
    * import { Effect, Schema } from "effect";
    *
@@ -835,7 +888,10 @@ export declare class Journal {
    * const program = Effect.gen(function* () {
    *   const mail = yield* Mail;
    *   yield* mail.append("mail", { round: 1 });
-   * }).pipe(Effect.provide(Mail.layer), Effect.provide(NodeFileSystem.layer));
+   * }).pipe(
+   *   Effect.provide(Mail.layer),
+   *   Effect.provide([NodeFileSystem.layer, NodeJournalWatcher.layer]),
+   * );
    * ```
    */
   static Service<Self>(): <const Id extends string, const R extends JsonlEvent.Registry, E = never, RC = never>(id: Id, options: {
@@ -902,5 +958,5 @@ export declare class Line {
   static parseResult(line: LineSlice): Result.Result<unknown, MalformedLine>;
 }
 //#endregion
-export type { AppendError, AppendOptions, ChangesError, DataSchema, DecodeError, EnvelopeOf, EnvelopeUnion, EnvelopeWithTag, JournalClass, JournalConfig, JournalShape, JsonlError, LinePosition, QueryError, Slice };
+export type { AppendError, AppendOptions, ChangesError, DataSchema, DecodeError, EnvelopeOf, EnvelopeUnion, EnvelopeWithTag, JournalClass, JournalConfig, JournalShape, JournalWatcherShape, JsonlError, LinePosition, QueryError, Slice };
 //# sourceMappingURL=index.d.ts.map

package/index.js

@@ -2,7 +2,8 @@ import { LineSlice } from "./LineSlice.js";
 import { InvalidData, JournalClosed, JournalNotFound, JournalResync, MalformedLine, TerminalViolation, UnknownEvent, UnserializableData } from "./JsonlError.js";
 import { Line } from "./Line.js";
 import { Envelope } from "./Envelope.js";
+import { JournalWatcher } from "./JournalWatcher.js";
 import { Journal } from "./Journal.js";
 import { JsonlEvent } from "./JsonlEvent.js";
 
-export { Envelope, InvalidData, Journal, JournalClosed, JournalNotFound, JournalResync, JsonlEvent, Line, LineSlice, MalformedLine, TerminalViolation, UnknownEvent, UnserializableData };
+export { Envelope, InvalidData, Journal, JournalClosed, JournalNotFound, JournalResync, JournalWatcher, JsonlEvent, Line, LineSlice, MalformedLine, TerminalViolation, UnknownEvent, UnserializableData };

package/internal/engine.js

@@ -1,10 +1,11 @@
 import { JournalClosed, JournalNotFound, JournalResync, TerminalViolation } from "../JsonlError.js";
 import { Line, isBlank } from "../Line.js";
 import { Envelope, completeResult, frameResult } from "../Envelope.js";
+import { JournalWatcher } from "../JournalWatcher.js";
 import { matchesFrame } from "../Slice.js";
 import { canMerge, shallowMerge } from "./merge.js";
 import { probeBomBytes, readLinePages, readTailUntil } from "./tail.js";
-import { ByteSize, Channel, DateTime, Deferred, Duration, Effect, Exit, Fiber, FileSystem, Option, PubSub, Result, Semaphore, Stream, SubscriptionRef } from "effect";
+import { ByteSize, Channel, DateTime, Deferred, Duration, Effect, Exit, FileSystem, Option, PubSub, Result, Semaphore, Stream, SubscriptionRef } from "effect";
 
 //#region src/internal/engine.ts
 const positionOf = (item) => Result.isSuccess(item) ? item.success.position : item.failure.position;
@@ -16,14 +17,6 @@ const SHUTDOWN_PUBLISH_TIMEOUT = Duration.seconds(5);
 * keep working, live observation does not.
 */
 const MAX_IMMEDIATE_REARMS = 8;
-/**
-* Scheduler turns yielded to a freshly-forked watch consumer before the
-* catch-up read. `fs.watch` exposes no "registered" signal, so arming is
-* ordered ahead of catch-up by scheduling, not synchronisation — empirically
-* sufficient, not a proof. The airtight primitive would be
-* `FileSystem.WatchBackend.register`, at the cost of `WatchBackend` in `R`.
-*/
-const ARM_YIELDS = 3;
 const encoder = new TextEncoder();
 /**
 * Index of the last path separator, on either convention — a Windows path
@@ -64,6 +57,7 @@ const admit = (items, slice) => {
 */
 const makeEngine = (id, events, config) => Effect.gen(function* () {
 	const fs = yield* FileSystem.FileSystem;
+	const watcher = yield* JournalWatcher;
 	const { path } = config;
 	const terminalTags = new Set(events.filter((event) => event.terminal).map((event) => event.tag));
 	const reopenTags = new Set(events.filter((event) => event.reopen).map((event) => event.tag));
@@ -366,19 +360,26 @@ const makeEngine = (id, events, config) => Effect.gen(function* () {
 	if (Option.isSome(initial)) yield* seed(initial.value).pipe(Effect.catchReason("PlatformError", "NotFound", () => Effect.void));
 	const basename = basenameOf(path);
 	const directory = config.directory ?? parentOf(path);
-	const watchJournal = fs.watch(path).pipe(Stream.runForEach(() => ingest), Effect.ignore);
-	const watchForCreation = fs.watch(directory).pipe(Stream.filter((event) => basenameOf(event.path) === basename), Stream.takeUntilEffect(() => fs.exists(path)), Stream.runForEach(() => ingest), Effect.ignore);
+	/** A directory event names the journal, or names nothing and might. */
+	const namesJournal = (name) => name === void 0 || basenameOf(name) === basename;
+	const followJournal = Effect.scoped(Effect.gen(function* () {
+		const changes = yield* watcher.watch(path);
+		yield* ingest;
+		yield* Stream.runForEach(changes, () => ingest);
+	}));
+	const awaitCreation = Effect.scoped(Effect.gen(function* () {
+		const changes = yield* watcher.watch(directory);
+		if (yield* fs.exists(path)) return;
+		yield* changes.pipe(Stream.filter(namesJournal), Stream.takeUntilEffect(() => fs.exists(path)), Stream.runForEach(() => ingest));
+	}));
 	const supervise = Effect.gen(function* () {
 		let immediateCompletions = 0;
 		for (;;) {
 			const startedAt = consumed;
 			if (yield* fs.exists(path)) {
-				const armed = yield* Effect.forkChild(watchJournal);
-				for (let turn = 0; turn < ARM_YIELDS; turn++) yield* Effect.yieldNow;
-				yield* ingest;
-				yield* Fiber.join(armed);
+				yield* Effect.ignore(followJournal);
 				yield* ingest;
-			} else yield* watchForCreation;
+			} else yield* Effect.ignore(awaitCreation);
 			immediateCompletions = consumed === startedAt ? immediateCompletions + 1 : 0;
 			if (immediateCompletions > MAX_IMMEDIATE_REARMS) return;
 		}

package/node.d.ts

@@ -0,0 +1,37 @@
+import * as Jsonl from "@effected/jsonl";
+import { Layer } from "effect";
+//#region src/NodeJournalWatcher.d.ts
+/**
+ * The Node implementation of {@link Jsonl.JournalWatcher}, over `node:fs`'s
+ * `watch`.
+ *
+ * Registration happens synchronously while the watch effect runs, so a
+ * journal's catch-up read after arming can never miss an append that the
+ * watch also fails to report.
+ *
+ * @example
+ * ```ts
+ * import { Journal, JsonlEvent } from "@effected/jsonl";
+ * import { NodeJournalWatcher } from "@effected/jsonl/node";
+ * import { NodeFileSystem } from "@effect/platform-node";
+ * import { Effect, Schema } from "effect";
+ *
+ * const events = [JsonlEvent.make("mail", { data: Schema.Struct({ round: Schema.Number }) })] as const;
+ *
+ * class Mail extends Journal.Service<Mail>()("app/Mail", { events, config: { path: ".app/mail.jsonl" } }) {}
+ *
+ * const program = Effect.gen(function* () {
+ *   const mail = yield* Mail;
+ *   yield* mail.append("mail", { round: 1 });
+ * }).pipe(Effect.provide(Mail.layer), Effect.provide([NodeFileSystem.layer, NodeJournalWatcher.layer]));
+ * ```
+ *
+ * @public
+ */
+export declare class NodeJournalWatcher {
+  private constructor();
+  /** Provides {@link Jsonl.JournalWatcher} over `node:fs`. */
+  static readonly layer: Layer.Layer<Jsonl.JournalWatcher>;
+}
+//#endregion
+//# sourceMappingURL=node.d.ts.map

package/node.js

@@ -0,0 +1,3 @@
+import { NodeJournalWatcher } from "./NodeJournalWatcher.js";
+
+export { NodeJournalWatcher };

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@effected/jsonl",
-	"version": "0.10.0",
+	"version": "0.11.1",
 	"private": false,
 	"description": "Append-only, schema-validated JSONL journals as a definable Effect service.",
 	"keywords": [
@@ -34,10 +34,15 @@
 			"import": "./index.js",
 			"default": "./index.js"
 		},
+		"./node": {
+			"types": "./node.d.ts",
+			"import": "./node.js",
+			"default": "./node.js"
+		},
 		"./package.json": "./package.json"
 	},
 	"peerDependencies": {
-		"effect": "^4.0.0"
+		"effect": "4.0.2"
 	},
 	"engines": {
 		"node": ">=24.11.0"