@executablemd/durable-streams 0.7.0 → 0.8.0

package/esm/retained.js

@@ -0,0 +1,390 @@
+/**
+ * Retained events — what a run reads a journal as.
+ *
+ * A journal is data supplied by a backend, and every phase of a replay reads
+ * the same events: a private authority gate, the replay index, public guard
+ * policy, and the replay path itself. If those are separate reads of the
+ * backend's own objects, a source that answers differently between them decides
+ * one thing for validation and another for execution, and nothing downstream
+ * can detect the substitution.
+ *
+ * A retained event is therefore read once and detached. **Every** event that
+ * participates in admission, indexing, or terminal reuse is retained, Close as
+ * well as Yield: a Close decides whether a coroutine has a terminal result to
+ * reuse, so leaving it as the backend's own object lets it belong to a child
+ * coroutine while one phase asks and to the root while the next does.
+ *
+ * The discriminator is settled once, by the classification that chooses a
+ * retained event's kind, and never read from the source again. Identity — the
+ * coroutine an event belongs to, and a Yield's complete effect description — is
+ * settled once too, so no phase can be shown a different event than the phase
+ * before it.
+ *
+ * A Yield's *settlement* stays lazy and separate: the index is built before
+ * guards run, and a guard that would refuse an event has to get that chance
+ * before the stream is asked to produce its result. A Close keeps its own cell,
+ * memoized the same way, so every later read receives the same detached answer.
+ */
+function settle(read) {
+    try {
+        return { kind: "value", value: read() };
+    }
+    catch (refusal) {
+        return { kind: "refusal", refusal };
+    }
+}
+function resolve(settled) {
+    if (settled.kind === "refusal") {
+        throw settled.refusal;
+    }
+    return settled.value;
+}
+/**
+ * A detached copy of one retained JSON value.
+ *
+ * Every property is read once and rebuilt, so nothing the stream still owns
+ * remains reachable: a nested accessor cannot answer one thing to one phase and
+ * another to the next, and no later mutation of the source changes what replay
+ * used.
+ *
+ * The copy is ordinary JSON. Detaching is the claim against the *stream*;
+ * making the copy immutable would be a claim against the *consumer*, and
+ * replayed values are legitimately mutable — an eval binding restored from a
+ * journal is pushed to by the iteration that resumes on it. Members are
+ * therefore writable and configurable like any other JSON.
+ *
+ * Keys are defined rather than assigned all the same, because `__proto__`
+ * reaches an inherited setter on some runtimes and would rewrite the copy's
+ * prototype instead of becoming a member of it.
+ *
+ * A cycle is refused. `Json` has none, and a value that does is not something
+ * this can detach — refusing is remembered like any other refusal.
+ */
+export function detachJson(value, seen = new Set()) {
+    if (value === null || typeof value !== "object") {
+        return value;
+    }
+    if (seen.has(value)) {
+        throw new TypeError("a retained value cannot contain a cycle");
+    }
+    seen.add(value);
+    try {
+        if (Array.isArray(value)) {
+            const items = [];
+            for (let index = 0; index < value.length; index++) {
+                items.push(detachJson(value[index], seen));
+            }
+            return items;
+        }
+        const detached = {};
+        for (const [key, member] of Object.entries(value)) {
+            Object.defineProperty(detached, key, {
+                value: detachJson(member, seen),
+                enumerable: true,
+                writable: true,
+                configurable: true,
+            });
+        }
+        return detached;
+    }
+    finally {
+        seen.delete(value);
+    }
+}
+/**
+ * A detached copy, frozen through.
+ *
+ * The authoritative retained graph is what admission validated and what replay
+ * consumes, so nothing that reaches a caller may write to it. Freezing is the
+ * claim against *policy*, not against a workflow: what a document finally
+ * receives is a fresh mutable copy taken from this, never this.
+ */
+function sealJson(value) {
+    const detached = detachJson(value);
+    freezeDeep(detached);
+    return detached;
+}
+function freezeDeep(value) {
+    if (value === null || typeof value !== "object") {
+        return;
+    }
+    Object.freeze(value);
+    for (const member of Array.isArray(value) ? value : Object.values(value)) {
+        freezeDeep(member);
+    }
+}
+/** A detached copy of a retained failure's description. */
+function detachError(error) {
+    const name = error.name;
+    const stack = error.stack;
+    return Object.freeze({
+        message: error.message,
+        ...(name === undefined ? {} : { name }),
+        ...(stack === undefined ? {} : { stack }),
+    });
+}
+/**
+ * A retained result detached from everything the stream still owns.
+ *
+ * Each member is read exactly once, here, and the tree beneath it is rebuilt.
+ * Copying only the outer object would leave a `value` the journal can still
+ * rewrite, which is the substitution this exists to prevent.
+ */
+function detachResult(result) {
+    const status = result.status;
+    if (status === "ok") {
+        // Every successful shape, including the one that settled to nothing. A
+        // `Result<void>` carries no value to detach, but the envelope is still
+        // authority — left writable, a caller of a public observation could add one
+        // before replay reads it.
+        if (!("value" in result)) {
+            return Object.freeze({ status });
+        }
+        const value = result.value;
+        return Object.freeze(value === undefined ? { status } : { status, value: sealJson(value) });
+    }
+    if (status === "err") {
+        if (!("error" in result)) {
+            throw new TypeError("a retained failure carries the error it failed with");
+        }
+        return Object.freeze({ status, error: detachError(result.error) });
+    }
+    return Object.freeze({ status });
+}
+/**
+ * A detached copy of an effect description.
+ *
+ * `type` and `name` are the identity divergence detection compares; every other
+ * member is extra data a guard may read. All of it is rebuilt, so a description
+ * cannot name one effect while one phase looks and another while the next does.
+ */
+function detachDescription(description) {
+    // One enumeration, so every member — `type` and `name` included — is read
+    // exactly once. Reading them directly and then enumerating would read each of
+    // them twice, which is the second read this exists to remove.
+    const members = Object.entries(description);
+    let type;
+    let name;
+    const extra = [];
+    for (const [key, member] of members) {
+        if (key === "type") {
+            type = member;
+            continue;
+        }
+        if (key === "name") {
+            name = member;
+            continue;
+        }
+        extra.push([key, member]);
+    }
+    if (typeof type !== "string" || typeof name !== "string") {
+        throw new TypeError("a retained effect description carries a type and a name");
+    }
+    const detached = { type, name };
+    for (const [key, member] of extra) {
+        Object.defineProperty(detached, key, {
+            value: sealJson(member),
+            enumerable: true,
+            writable: false,
+            configurable: false,
+        });
+    }
+    return Object.freeze(detached);
+}
+function readCoroutineId(source) {
+    const coroutineId = source.coroutineId;
+    if (typeof coroutineId !== "string") {
+        throw new TypeError("a retained event belongs to a coroutine");
+    }
+    return coroutineId;
+}
+function readIdentity(source) {
+    return {
+        coroutineId: readCoroutineId(source),
+        description: detachDescription(source.description),
+    };
+}
+/**
+ * One retained Yield: one identity, and one cell for what it settled to.
+ *
+ * Identity is read together and once. Reading `type` here and `coroutineId`
+ * there would let a source present an unrelated event to one phase and the root
+ * import to the next, which is the whole reason identity is a single settled
+ * fact rather than three accessors.
+ *
+ * The settlement is separate and lazy on purpose: the index is built before
+ * guards run, so a guard that would refuse an event must get that chance before
+ * the stream is asked for its result. Both outcomes of both reads are kept — a
+ * refusal is remembered and re-raised rather than retried, so a source cannot
+ * refuse one phase and then answer the next.
+ */
+/**
+ * Present a retained member the way the event it stands for presents it.
+ *
+ * Own and enumerable, so a retained event spreads, serializes, and compares
+ * like the plain event a backend would have supplied. The settled cells stay
+ * genuinely private, which is what keeps them out of all of that.
+ */
+function present(target, name, read) {
+    Object.defineProperty(target, name, { enumerable: true, get: read });
+}
+class RetainedYield {
+    #source;
+    #identity;
+    #settled;
+    constructor(source) {
+        this.#source = source;
+        present(this, "type", () => "yield");
+        present(this, "coroutineId", () => this.#stable().coroutineId);
+        present(this, "description", () => this.#stable().description);
+        present(this, "result", () => {
+            this.#settled ??= settle(() => detachResult(this.#source.result));
+            return resolve(this.#settled);
+        });
+    }
+    #stable() {
+        this.#identity ??= settle(() => readIdentity(this.#source));
+        return resolve(this.#identity);
+    }
+}
+/**
+ * One retained Close: settled identity, and one cell for its terminal result.
+ *
+ * A Close decides whether a coroutine has a terminal result to reuse, so it
+ * participates in admission exactly as a Yield does. Left as the backend's own
+ * object it could belong to a child coroutine while one phase asks and to the
+ * root while the next does — a history nobody could admit, reused as a result
+ * nobody asked for.
+ */
+class RetainedClose {
+    #identity;
+    #settled;
+    constructor(source) {
+        // Settled here, while the history is being retained, rather than at a first
+        // later read. A Close carries the result a completed run hands back, and
+        // deferring that read leaves an interval — between the moment a consumer's
+        // private admission accepts the history and the moment terminal reuse
+        // consumes it — in which the backend still owns the answer and can replace
+        // it. Reading once at a later getter closes repeated reads and leaves that
+        // window open.
+        //
+        // Settling cannot throw: a refusal is captured and re-raised from the
+        // getter, so retaining a history is never the thing that fails.
+        this.#identity = settle(() => readCoroutineId(source));
+        this.#settled = settle(() => detachResult(source.result));
+        present(this, "type", () => "close");
+        present(this, "coroutineId", () => resolve(this.#identity));
+        present(this, "result", () => resolve(this.#settled));
+    }
+}
+/**
+ * An event that would not say what it is.
+ *
+ * Classification is the one thing every later phase depends on, so an event
+ * that refuses it is not an event this history can describe. The refusal is
+ * remembered and re-raised from every member, rather than retried — a source
+ * that refuses one phase must not answer the next.
+ */
+class RetainedRefusal {
+    #refusal;
+    constructor(refusal) {
+        this.#refusal = refusal;
+        for (const name of ["type", "coroutineId", "description", "result"]) {
+            present(this, name, () => {
+                throw this.#refusal;
+            });
+        }
+    }
+}
+/**
+ * The retained form of a journal's events.
+ *
+ * Idempotent: retaining an already-retained event returns it, so a caller that
+ * has produced the stable history hands the same objects onward rather than a
+ * second wrapping of them. That is what lets one snapshot serve every phase.
+ *
+ * Only the event type is read here, which is the least a caller can read and
+ * still tell a Yield from a Close. Everything else is the retained event's own.
+ */
+export function retainEvents(events) {
+    return events.map((event) => {
+        if (isRetained(event)) {
+            return event;
+        }
+        // The one read of the source's discriminator. Whatever it says here is what
+        // the retained event reports from now on, to every phase.
+        const classified = settle(() => event.type);
+        if (classified.kind === "refusal") {
+            return new RetainedRefusal(classified.refusal);
+        }
+        if (classified.value === "yield") {
+            return new RetainedYield(event);
+        }
+        if (classified.value === "close") {
+            return new RetainedClose(event);
+        }
+        return new RetainedRefusal(new TypeError("a retained event is a yield or a close"));
+    });
+}
+function isRetained(event) {
+    return (event instanceof RetainedYield ||
+        event instanceof RetainedClose ||
+        event instanceof RetainedRefusal);
+}
+/**
+ * An isolated observation of a retained event, for public policy to read.
+ *
+ * A replay guard is composable policy, and composition means handlers read,
+ * annotate, and pass along. What it must never mean is that a handler edits the
+ * history the execution already validated: the authoritative graph is what
+ * admission accepted and what replay consumes, and a guard that could rewrite a
+ * root selection or an effect description after admission would hold exactly
+ * the authority the private gate exists to keep out of public hands.
+ *
+ * So policy reads a copy. It is deep and mutable, so middleware may compose over
+ * it as freely as it likes, and nothing it does reaches replay.
+ */
+export function observeEvent(event) {
+    if (event.type === "close") {
+        return { type: "close", coroutineId: event.coroutineId, result: consumable(event.result) };
+    }
+    return {
+        type: "yield",
+        coroutineId: event.coroutineId,
+        description: observeDescription(event.description),
+        result: consumable(event.result),
+    };
+}
+function observeDescription(description) {
+    const copy = { type: description.type, name: description.name };
+    for (const [key, member] of Object.entries(description)) {
+        if (key === "type" || key === "name") {
+            continue;
+        }
+        Object.defineProperty(copy, key, {
+            value: detachJson(member),
+            enumerable: true,
+            writable: true,
+            configurable: true,
+        });
+    }
+    return copy;
+}
+/**
+ * A retained result as a consumer may hold it: ordinary mutable JSON.
+ *
+ * The authoritative copy is frozen so policy cannot rewrite it. A document
+ * that resumes on a restored binding writes to it, so what a workflow receives
+ * is a fresh copy taken from that authority rather than the authority itself.
+ */
+export function consumable(result) {
+    if (result.status === "ok") {
+        return "value" in result && result.value !== undefined
+            ? { status: "ok", value: detachJson(result.value) }
+            : { status: "ok" };
+    }
+    if (result.status === "err") {
+        return { status: "err", error: { ...result.error } };
+    }
+    return { status: "cancelled" };
+}

package/esm/run.js

@@ -13,11 +13,16 @@
  * See integration doc §10, protocol spec §4.
  */
 import { useScope } from "effection";
-import { DurableCtx } from "./context.js";
-import { EarlyReturnDivergenceError } from "./errors.js";
+import { DurableContext } from "./context.js";
+import { activeDurabilityFailure, appendDurableEvent } from "./durability.js";
+import { EarlyReturnDivergenceError, TerminalDivergenceError } from "./errors.js";
 import { ReplayGuard } from "./replay-guard.js";
+import { consumable, observeEvent } from "./retained.js";
 import { ReplayIndex } from "./replay-index.js";
 import { deserializeError, serializeError } from "./serialize.js";
+function unalignedReplay(replayIndex, coroutineId) {
+    return replayIndex.firstUnaligned(coroutineId);
+}
 /**
  * Run the ReplayGuard check phase over all Yield events.
  *
@@ -26,12 +31,22 @@ import { deserializeError, serializeError } from "./serialize.js";
  * phase to gather observations (hash files, check timestamps) and cache
  * results for the decide phase.
  *
+ * The events come from the index rather than from the stream, so a guard reads
+ * the same retained result the replay path will use. Handing over the stream's
+ * own events instead would make validation and consumption two separate reads,
+ * and a source that answered differently between them could have a guard
+ * approve one result while execution used another.
+ *
  * See replay-guard-spec.md §5.5.
  */
-function* runCheckPhase(events, scope) {
-    for (const event of events) {
-        if (event.type === "yield") {
-            yield* ReplayGuard.invoke(scope, "check", [event]);
+function* runCheckPhase(replayIndex, scope) {
+    for (const event of replayIndex.retainedYields()) {
+        // An isolated observation, not the retained event. Guards compose by
+        // reading and passing along; what composition must not become is the power
+        // to edit a history the execution already validated.
+        const observed = observeEvent(event);
+        if (observed.type === "yield") {
+            yield* ReplayGuard.invoke(scope, "check", [observed]);
         }
     }
 }
@@ -43,7 +58,8 @@ function* runCheckPhase(events, scope) {
  * 3. Runs the workflow — replayed effects resolve synchronously from
  *    the index; live effects execute and persist before resuming.
  * 4. On completion, appends a Close event to the stream.
- * 5. On error, appends a Close(err) event.
+ * 5. Before any Close, rejects durability failures and retained coroutine
+ *    history the current definition did not align with.
  *
  * Returns the workflow's result value.
  *
@@ -62,25 +78,47 @@ export function* durableRun(workflow, options) {
     // Inherit the caller's scope — middleware (e.g., Divergence, ReplayGuard)
     // is already installed by the caller before yield*-ing into durableRun.
     const scope = yield* useScope();
-    scope.set(DurableCtx, {
+    const ctx = {
         replayIndex,
         stream,
         coroutineId,
         childCounter: 0,
-    });
+        durability: {},
+    };
+    scope.set(DurableContext, ctx);
     // ── REPLAY GUARD: Check phase ──
     // Run before the workflow starts. Middleware can yield* for I/O (hash
     // files, make network requests) to gather observations for the decide
     // phase. The check loop iterates all Yield events in journal order.
     // See replay-guard-spec.md §5.5.
-    yield* runCheckPhase(events, scope);
+    yield* runCheckPhase(replayIndex, scope);
+    // ── REPLAY GUARD: Admit phase ──
+    // The retained history has been offered in full and nothing has been reused
+    // yet. A guard that requires something of the history as a whole — that an
+    // event it validates is present, and present once — refuses here, before the
+    // recorded terminal result below can answer for history nobody validated.
+    yield* ReplayGuard.invoke(scope, "admit", [
+        {
+            coroutineId,
+            yields: replayIndex.retainedYields().flatMap((event) => {
+                const observed = observeEvent(event);
+                return observed.type === "yield" ? [observed] : [];
+            }),
+            terminal: replayIndex.hasClose(coroutineId),
+        },
+    ]);
     // If the root coroutine already has a Close event in the journal,
     // the workflow completed in a previous run. Return the stored result
     // directly without re-running the workflow.
     if (replayIndex.hasClose(coroutineId)) {
         const closeEvent = replayIndex.getClose(coroutineId);
         if (closeEvent.result.status === "ok") {
-            return closeEvent.result.value;
+            // A fresh consumer copy, exactly as a replayed Yield's result is. The
+            // retained settlement is frozen so policy cannot rewrite it; what a
+            // caller receives from a completed run is ordinary data it may hold and
+            // change, and changing it cannot reach the next replay.
+            const settled = consumable(closeEvent.result);
+            return (settled.status === "ok" ? settled.value : undefined);
         }
         else if (closeEvent.result.status === "err") {
             throw deserializeError(closeEvent.result.error);
@@ -89,34 +127,37 @@ export function* durableRun(workflow, options) {
             throw new Error("Workflow was cancelled");
         }
     }
+    replayIndex.claim(coroutineId);
     try {
         // Workflow<T> is structurally assignable to Operation<T>, so
         // yield* accepts it directly — no cast needed.
         const result = yield* workflow();
-        // §6.3: Check for early return divergence.
-        // If the generator returned but the replay index has unconsumed yields,
-        // the workflow has diverged. Skip this check when replay has been
-        // disabled (run-live mode) — the workflow intentionally diverged and
-        // the Divergence API already approved it.
-        if (!replayIndex.isReplayDisabled(coroutineId)) {
-            const unconsumed = replayIndex.firstUnconsumed();
-            if (unconsumed) {
-                throw new EarlyReturnDivergenceError(unconsumed.coroutineId, unconsumed.cursor, unconsumed.totalYields);
-            }
+        const durabilityFailure = activeDurabilityFailure(ctx);
+        if (durabilityFailure) {
+            throw durabilityFailure;
+        }
+        const unconsumed = unalignedReplay(replayIndex, coroutineId);
+        if (unconsumed) {
+            throw new EarlyReturnDivergenceError(unconsumed.coroutineId, unconsumed.cursor, unconsumed.totalYields);
         }
         const closeEvent = {
             type: "close",
             coroutineId,
             result: { status: "ok", value: result },
         };
-        yield* stream.append(closeEvent);
+        yield* appendDurableEvent(ctx, closeEvent);
         return result;
     }
     catch (error) {
-        // Normalize the error once — use the same Error object for both the
-        // Close event and the rethrow so that live runs and replayed Close
-        // events carry identical error shapes.
         const primary = error instanceof Error ? error : new Error(String(error));
+        const durabilityFailure = activeDurabilityFailure(ctx, primary);
+        if (durabilityFailure) {
+            throw durabilityFailure;
+        }
+        const unconsumed = unalignedReplay(replayIndex, coroutineId);
+        if (unconsumed) {
+            throw new TerminalDivergenceError(unconsumed.coroutineId, unconsumed.cursor, unconsumed.totalYields, { cause: primary });
+        }
         const closeEvent = {
             type: "close",
             coroutineId,
@@ -126,11 +167,15 @@ export function* durableRun(workflow, options) {
             },
         };
         try {
-            yield* stream.append(closeEvent);
+            yield* appendDurableEvent(ctx, closeEvent);
         }
-        catch (appendError) {
-            const appendFailure = appendError instanceof Error ? appendError : new Error(String(appendError));
-            throw new AggregateError([primary, appendFailure], "Workflow failed and Close append also failed");
+        catch (closeError) {
+            const closeDurabilityFailure = activeDurabilityFailure(ctx, closeError);
+            if (closeDurabilityFailure) {
+                throw closeDurabilityFailure;
+            }
+            const closeFailure = closeError instanceof Error ? closeError : new Error(String(closeError));
+            throw new AggregateError([primary, closeFailure], "Workflow failed and Close append also failed");
         }
         throw primary;
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@executablemd/durable-streams",
-  "version": "0.7.0",
+  "version": "0.8.0",
   "description": "Durable, replayable event streams for executable.md.",
   "homepage": "https://executable.md",
   "repository": {
@@ -24,7 +24,7 @@
   "scripts": {},
   "dependencies": {
     "@durable-streams/client": "^0.2.2",
-    "effection": "4.1.0-alpha.10"
+    "effection": "4.1.0"
   },
   "_generatedBy": "dnt@dev"
 }

package/types/_dnt.polyfills.d.ts

@@ -0,0 +1,6 @@
+declare global {
+    interface Error {
+        cause?: unknown;
+    }
+}
+export {};

package/types/context.d.ts

@@ -5,10 +5,17 @@
  * inherit the shared replayIndex and stream, but get their own
  * coroutineId and childCounter.
  */
-import { type Context } from "effection";
+import { type Context, type Operation } from "effection";
 import type { ReplayIndex } from "./replay-index.js";
 import type { DurableStream } from "./stream.js";
 import type { CoroutineId } from "./types.js";
+export interface DurableAppendFence {
+    hold(): Operation<void>;
+}
+export interface DurabilityState {
+    failure?: Error;
+    appendFence?: DurableAppendFence;
+}
 export interface DurableContext {
     /** Shared replay index (built from stream on startup). */
     replayIndex: ReplayIndex;
@@ -18,9 +25,11 @@ export interface DurableContext {
     coroutineId: CoroutineId;
     /** Counter for assigning child IDs. */
     childCounter: number;
+    /** Protocol failure shared by the root and every durable child. */
+    durability?: DurabilityState;
 }
 /**
  * Effection Context for durable execution state.
  * Set on the root scope by durableRun(); inherited by child scopes.
  */
-export declare const DurableCtx: Context<DurableContext>;
+export declare const DurableContext: Context<DurableContext>;

package/types/durability.d.ts

@@ -0,0 +1,7 @@
+import type { Operation } from "effection";
+import type { DurableContext } from "./context.js";
+import type { DurableEvent } from "./types.js";
+export declare function findDurabilityFailure(error: unknown): Error | undefined;
+export declare function rememberDurabilityFailure(ctx: DurableContext, error: Error): Error;
+export declare function activeDurabilityFailure(ctx: DurableContext, error?: unknown): Error | undefined;
+export declare function appendDurableEvent(ctx: DurableContext, event: DurableEvent): Operation<void>;

package/types/effect.d.ts

@@ -20,6 +20,7 @@
  * See integration doc §5.1, protocol spec §4.2, §5, §6.
  */
 import type { Operation } from "effection";
+import { type LiveDurableOperationCoordinator } from "./live-coordinator.js";
 import type { DurableEffect, EffectDescription, Json, Result } from "./types.js";
 /**
  * Executor function signature for live execution (callback-based).
@@ -56,5 +57,9 @@ export declare function createDurableEffect<T>(desc: EffectDescription, execute:
  *
  * @param desc Structured description for the journal and divergence detection
  * @param execute Returns an Operation to run during live execution
+ * @param options.coordinator Selects the live execution/publication boundary;
+ * replay never invokes it
  */
-export declare function createDurableOperation<T extends Json>(desc: EffectDescription, execute: () => Operation<T>): DurableEffect<T>;
+export declare function createDurableOperation<T extends Json>(desc: EffectDescription, execute: () => Operation<T>, options?: {
+    coordinator?: LiveDurableOperationCoordinator;
+}): DurableEffect<T>;