@executablemd/durable-streams 0.2.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Taras Mankovski
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/esm/_dnt.shims.js

@@ -0,0 +1,57 @@
+const dntGlobals = {};
+export const dntGlobalThis = createMergeProxy(globalThis, dntGlobals);
+function createMergeProxy(baseObj, extObj) {
+    return new Proxy(baseObj, {
+        get(_target, prop, _receiver) {
+            if (prop in extObj) {
+                return extObj[prop];
+            }
+            else {
+                return baseObj[prop];
+            }
+        },
+        set(_target, prop, value) {
+            if (prop in extObj) {
+                delete extObj[prop];
+            }
+            baseObj[prop] = value;
+            return true;
+        },
+        deleteProperty(_target, prop) {
+            let success = false;
+            if (prop in extObj) {
+                delete extObj[prop];
+                success = true;
+            }
+            if (prop in baseObj) {
+                delete baseObj[prop];
+                success = true;
+            }
+            return success;
+        },
+        ownKeys(_target) {
+            const baseKeys = Reflect.ownKeys(baseObj);
+            const extKeys = Reflect.ownKeys(extObj);
+            const extKeysSet = new Set(extKeys);
+            return [...baseKeys.filter((k) => !extKeysSet.has(k)), ...extKeys];
+        },
+        defineProperty(_target, prop, desc) {
+            if (prop in extObj) {
+                delete extObj[prop];
+            }
+            Reflect.defineProperty(baseObj, prop, desc);
+            return true;
+        },
+        getOwnPropertyDescriptor(_target, prop) {
+            if (prop in extObj) {
+                return Reflect.getOwnPropertyDescriptor(extObj, prop);
+            }
+            else {
+                return Reflect.getOwnPropertyDescriptor(baseObj, prop);
+            }
+        },
+        has(_target, prop) {
+            return prop in extObj || prop in baseObj;
+        },
+    });
+}

package/esm/combinators.js

@@ -0,0 +1,227 @@
+/**
+ * Structured concurrency combinators for durable workflows.
+ *
+ * durableSpawn, durableAll, durableRace — each wraps child workflows
+ * with DurableContext (coroutine IDs, Close events) so that structured
+ * concurrency is fully journaled and replayable.
+ *
+ * Each combinator returns Workflow<T> (not Operation<T>) so it can be
+ * used directly inside a Workflow via yield*. Internally, the infrastructure
+ * effects (useScope, spawn, all, race) are wrapped with ephemeral() —
+ * these are durable-safe operations that set up scope/context and don't
+ * need journaling. See DEC-034.
+ *
+ * Child workflows must be Workflow<T> — bare Operations are rejected at
+ * compile time. Use ephemeral() to explicitly opt in to non-durable
+ * children.
+ *
+ * See protocol spec §7 (structured concurrency), §10 (race semantics).
+ */
+import { all as effectionAll, ensure, race as effectionRace, spawn, suspend, useScope, } from "effection";
+import { DurableCtx } from "./context.js";
+import { ephemeral } from "./ephemeral.js";
+import { deserializeError, serializeError } from "./serialize.js";
+// ---------------------------------------------------------------------------
+// Internal: wrap a child workflow with DurableContext + Close emission
+// ---------------------------------------------------------------------------
+/**
+ * Run a child workflow within a spawned scope, setting up its own
+ * DurableContext and emitting a Close event when it terminates.
+ *
+ * This is the core building block for all structured concurrency combinators.
+ *
+ * It:
+ *  1. Checks if the child already completed (has Close event) — short-circuits
+ *  2. Sets DurableCtx on the child's scope with the child's coroutineId
+ *  3. Runs the child workflow (its DurableEffects use the child's coroutineId)
+ *  4. Appends Close(ok|err) when the child terminates
+ *
+ * IMPORTANT: This must be called inside a spawn() so it gets its own scope.
+ * The caller is responsible for spawn().
+ */
+function* runDurableChild(childWorkflow, childId, parentCtx) {
+    const { replayIndex, stream } = parentCtx;
+    // Short-circuit: child already completed in a previous run.
+    // NOTE: Replay guard validation is not bypassed here — the check phase
+    // (runCheckPhase in durableRun) already iterated ALL Yield events
+    // (including this child's) before any workflow code runs. Guards have
+    // already had a chance to veto stale data. This fast-path only skips
+    // re-running the child's generator, not the guard validation.
+    if (replayIndex.hasClose(childId)) {
+        const closeEvent = replayIndex.getClose(childId);
+        if (closeEvent.result.status === "ok") {
+            return closeEvent.result.value;
+        }
+        else if (closeEvent.result.status === "err") {
+            throw deserializeError(closeEvent.result.error);
+        }
+        else {
+            // cancelled — this child was cancelled in a previous run (e.g.,
+            // a race loser). Instead of throwing, we suspend forever. The
+            // parent combinator (race/all) will cancel this child as part of
+            // normal structured concurrency teardown, just like the original
+            // run. The Close(cancelled) event already exists in the journal,
+            // so we skip re-emitting it (the ensure teardown checks for this).
+            //
+            // INVARIANT: This branch is only reachable when a parent combinator
+            // (durableRace or durableAll with a failed sibling) will cancel this
+            // child. Close(cancelled) in the journal means the child was
+            // previously cancelled by structured concurrency, so on replay the
+            // same combinator will cancel it again. This cannot deadlock.
+            yield* suspend();
+            // unreachable — suspend blocks until cancelled
+            return undefined;
+        }
+    }
+    // Set child's DurableContext on this scope
+    const scope = yield* useScope();
+    scope.set(DurableCtx, {
+        replayIndex,
+        stream,
+        coroutineId: childId,
+        childCounter: 0,
+    });
+    let closeEvent;
+    yield* ensure(function* () {
+        // closeEvent still undefined means the child was cancelled before the
+        // normal-return or catch path ran.
+        if (!closeEvent) {
+            closeEvent = {
+                type: "close",
+                coroutineId: childId,
+                result: { status: "cancelled" },
+            };
+        }
+        // Don't re-emit a Close event if one already exists in the journal
+        // (e.g., a cancelled child being replayed via suspend()).
+        if (!replayIndex.hasClose(childId)) {
+            yield* stream.append(closeEvent);
+        }
+    });
+    try {
+        // Run the child workflow. DurableEffects inside the child read
+        // DurableCtx from the scope, so they'll use childId.
+        const result = yield* childWorkflow();
+        closeEvent = {
+            type: "close",
+            coroutineId: childId,
+            result: { status: "ok", value: result },
+        };
+        return result;
+    }
+    catch (error) {
+        closeEvent = {
+            type: "close",
+            coroutineId: childId,
+            result: {
+                status: "err",
+                error: serializeError(error instanceof Error ? error : new Error(String(error))),
+            },
+        };
+        throw error;
+    }
+}
+// ---------------------------------------------------------------------------
+// durableSpawn — spawn a single durable child, returns Task<T>
+// ---------------------------------------------------------------------------
+/**
+ * Spawn a durable child workflow.
+ *
+ * Assigns a deterministic coroutine ID (parentId.N), sets up DurableContext
+ * on the child scope, and ensures Close events are emitted.
+ *
+ * Returns a Task<T> that can be yield*-ed to get the child's result.
+ *
+ * Returns Workflow<Task<T>> via ephemeral() — the infrastructure effects
+ * (useScope, spawn) are durable-safe scope setup that doesn't need
+ * journaling and re-runs correctly on replay.
+ */
+export function durableSpawn(childWorkflow) {
+    return ephemeral((function* () {
+        const scope = yield* useScope();
+        const ctx = scope.expect(DurableCtx);
+        // Assign deterministic child ID
+        const childIndex = ctx.childCounter++;
+        const childId = `${ctx.coroutineId}.${childIndex}`;
+        // Spawn the child with durable wrapping
+        return yield* spawn(() => runDurableChild(childWorkflow, childId, ctx));
+    })());
+}
+// ---------------------------------------------------------------------------
+// durableAll — fork/join, wait for all children
+// ---------------------------------------------------------------------------
+/**
+ * Run multiple durable workflows concurrently and wait for all to complete.
+ *
+ * Each child gets a deterministic coroutine ID (parentId.0, parentId.1, ...).
+ * Each child's effects are journaled under its own coroutineId.
+ * Each child emits a Close event on termination.
+ *
+ * If any child fails, remaining children are cancelled (fail-fast,
+ * Effection's default structured concurrency behavior via all()).
+ *
+ * Returns an array of results in the same order as the input workflows.
+ *
+ * See spec §7, §11.5.
+ */
+export function durableAll(workflows) {
+    return ephemeral((function* () {
+        const scope = yield* useScope();
+        const ctx = scope.expect(DurableCtx);
+        // Build child Operations, one per workflow. Each gets its own
+        // deterministic coroutineId and Close event handling.
+        const childOps = workflows.map((workflow) => {
+            const childIndex = ctx.childCounter++;
+            const childId = `${ctx.coroutineId}.${childIndex}`;
+            return {
+                *[Symbol.iterator]() {
+                    return yield* runDurableChild(workflow, childId, ctx);
+                },
+            };
+        });
+        // Delegate to Effection's native all() which uses trap() internally
+        // for proper error isolation. This means:
+        // - Child errors are catchable by the caller via try/catch
+        // - When any child fails, remaining siblings are cancelled
+        // - The error propagates with the original message intact
+        return yield* effectionAll(childOps);
+    })());
+}
+// ---------------------------------------------------------------------------
+// durableRace — first child to complete wins, others cancelled
+// ---------------------------------------------------------------------------
+/**
+ * Race multiple durable workflows. The first to complete wins;
+ * remaining children are cancelled.
+ *
+ * Each child gets a deterministic coroutine ID. When the winner
+ * completes, Effection cancels the remaining children via
+ * iterator.return(). The runDurableChild wrapper detects this
+ * (closeEvent is undefined in the finally block) and emits
+ * Close(cancelled) for each loser.
+ *
+ * On replay, children with Close(cancelled) in the journal suspend
+ * indefinitely (yield* suspend()), letting the parent race cancel
+ * them naturally — matching the original live behavior.
+ *
+ * See spec §10.
+ */
+export function durableRace(workflows) {
+    return ephemeral((function* () {
+        const scope = yield* useScope();
+        const ctx = scope.expect(DurableCtx);
+        // Build Operations for each child — each gets its own coroutineId
+        // and Close event handling via runDurableChild.
+        const childOps = workflows.map((workflow) => {
+            const childIndex = ctx.childCounter++;
+            const childId = `${ctx.coroutineId}.${childIndex}`;
+            return {
+                *[Symbol.iterator]() {
+                    return yield* runDurableChild(workflow, childId, ctx);
+                },
+            };
+        });
+        // Use Effection's native race() which handles cancellation properly
+        return yield* effectionRace(childOps);
+    })());
+}

package/esm/context.js

@@ -0,0 +1,13 @@
+/**
+ * DurableContext — the scope-local state for durable execution.
+ *
+ * Stored on each Effection scope via createContext(). Child scopes
+ * inherit the shared replayIndex and stream, but get their own
+ * coroutineId and childCounter.
+ */
+import { createContext } from "effection";
+/**
+ * Effection Context for durable execution state.
+ * Set on the root scope by durableRun(); inherited by child scopes.
+ */
+export const DurableCtx = createContext("@effection/durable");

package/esm/divergence.js

@@ -0,0 +1,56 @@
+/**
+ * Divergence API — pluggable policy for handling replay mismatches.
+ *
+ * When a durable effect's description doesn't match the replay index
+ * during replay, or when a generator continues to yield effects past
+ * a recorded Close event, a divergence is detected.
+ *
+ * By default, divergence is fatal (throws DivergenceError). Users can
+ * override this behavior per-scope via Effection's around() middleware
+ * to implement custom policies (e.g., switching to live execution).
+ *
+ * Uses createApi() from effection/experimental (not @effectionx/context-api)
+ * because it requires synchronous Api.invoke() dispatch, which the
+ * vendored context-api does not yet support. Will migrate once invoke()
+ * lands in @effectionx/context-api.
+ *
+ * The core decide() function is synchronous (not a generator) because
+ * it is called from inside Effect.enter(), which is a synchronous
+ * callback. createApi().invoke() dispatches synchronously, so this
+ * is safe. See DEC-031.
+ */
+import { createApi } from "effection/experimental";
+import { ContinuePastCloseDivergenceError, DivergenceError } from "./errors.js";
+// ---------------------------------------------------------------------------
+// Default policy (strict — all divergences are fatal)
+// ---------------------------------------------------------------------------
+/** The default (strict) decide function. */
+function defaultDecide(info) {
+    if (info.kind === "description-mismatch") {
+        return {
+            type: "throw",
+            error: new DivergenceError(info.coroutineId, info.cursor, info.expected, info.actual),
+        };
+    }
+    else {
+        return {
+            type: "throw",
+            error: new ContinuePastCloseDivergenceError(info.coroutineId, info.yieldCount),
+        };
+    }
+}
+// ---------------------------------------------------------------------------
+// The Divergence API instance
+// ---------------------------------------------------------------------------
+/**
+ * The Divergence API.
+ *
+ * Created via Effection's createApi() which provides proper middleware
+ * dispatch with WeakMap-based handle caching, automatic cache
+ * invalidation on Api.around(), and a fast-path that skips
+ * middleware dispatch entirely when no middleware is installed.
+ *
+ * Default behavior is strict: all divergences produce a throw decision
+ * with the appropriate error type.
+ */
+export const Divergence = createApi("DurableEffection.Divergence", { decide: defaultDecide });

package/esm/each.js

@@ -0,0 +1,193 @@
+/**
+ * durableEach — durable iteration primitive for Effection workflows.
+ *
+ * Mirrors Effection's `each()` / `each.next()` pattern but journals
+ * every fetch as a DurableEffect, so iteration survives crashes and
+ * replays from the journal.
+ *
+ * Usage:
+ *   for (let msg of yield* durableEach("queue", source)) {
+ *     yield* durableCall("process", () => process(msg));
+ *     yield* durableEach.next();
+ *   }
+ *
+ * See effection-integration.md §12.6 for the full design.
+ */
+import { ensure } from "effection";
+import { createDurableOperation } from "./effect.js";
+import { ephemeral } from "./ephemeral.js";
+// ---------------------------------------------------------------------------
+// Internal state
+// ---------------------------------------------------------------------------
+/** Sentinel for source exhaustion. Not exported — cannot collide with JSON. */
+const DONE = Symbol("durableEach.done");
+/** Type guard for the DONE sentinel. */
+function isDone(value) {
+    return value === DONE;
+}
+/**
+ * Module-level active state for sharing between durableEach() and durableEach.next().
+ *
+ * Safe because durable execution is single-threaded — only one coroutine
+ * runs at a time, so there's no concurrent access. durableEach() sets this
+ * before returning the iterable; durableEach.next() reads it directly.
+ *
+ * This avoids using Effection context, which doesn't work when both
+ * functions are individually wrapped in ephemeral() (each gets its own
+ * child scope, making context invisible across them).
+ *
+ * LIMITATION: Only one durableEach iteration can be active at a time across
+ * the entire process. Concurrent durableEach calls (even in separate
+ * durableRun instances) will collide. The nesting guard below throws a
+ * clear error if this happens. For concurrent iterations, use child scopes
+ * via durableSpawn so each finishes its iteration before another starts.
+ *
+ * TODO: Consider migrating to a scope-local mechanism if Effection adds
+ * support for context that spans across ephemeral() boundaries.
+ */
+let activeState = null;
+// ---------------------------------------------------------------------------
+// durableEachFetch — shared helper for fetching one item
+// ---------------------------------------------------------------------------
+/**
+ * Fetch a single item from the source (or replay it from the journal).
+ *
+ * Both the initial fetch (inside durableEach) and subsequent fetches
+ * (inside durableEach.next) go through this helper. Same effect
+ * description, same journal format, same replay path.
+ *
+ * Uses createDurableOperation to run the source's Operation-native next()
+ * with full structured concurrency — cancellation of the scope cancels
+ * the in-flight source.next() call.
+ *
+ * Journal shape: Yield event with description { type: "each", name }
+ * and result value { value: T } | { done: true }.
+ */
+function durableEachFetch(name, source) {
+    return (function* () {
+        const result = (yield createDurableOperation({ type: "each", name }, () => source.next()));
+        if ("done" in result)
+            return DONE;
+        return result.value;
+    })();
+}
+// ---------------------------------------------------------------------------
+// durableEach — initial fetch + returns synchronous iterable
+// ---------------------------------------------------------------------------
+/**
+ * Durable iteration over a DurableSource (internal implementation).
+ *
+ * Returns Operation<Iterable<T>> because it uses ensure() (an
+ * infrastructure Operation). The public API wraps this in ephemeral()
+ * to return Workflow<Iterable<T>>.
+ *
+ * durableEach and durableEach.next share state through a module-level
+ * variable (activeState). This is safe because durable execution is
+ * single-threaded — only one coroutine runs at a time. This avoids
+ * Effection context, which doesn't work when both functions are
+ * individually wrapped in ephemeral() (each would get its own child
+ * scope, making context invisible across them).
+ */
+function* _durableEachOp(name, source) {
+    // Guard against nested durableEach — the single module-level slot
+    // would clobber the outer iteration's state.
+    if (activeState !== null) {
+        throw new Error(`durableEach("${name}"): cannot nest durableEach calls in the same scope. Use a child scope (e.g., via spawn) for inner iterations.`);
+    }
+    // Register source teardown on scope exit — ensures cleanup even if
+    // the for...of loop breaks or the scope is cancelled without an
+    // active effect. Safe to call alongside effect-level teardown
+    // because DurableSource.close() must be idempotent.
+    yield* ensure(() => {
+        source.close?.();
+    });
+    // Durable fetch of first item — journaled as a Yield event.
+    // ensure() is already registered, so cancellation here is safe.
+    const first = yield* durableEachFetch(name, source);
+    // Store state in module-level slot for durableEach.next() to access.
+    // Cleared when the iteration completes (done or break).
+    const state = {
+        name,
+        source,
+        current: first,
+        advanced: true, // first item was just fetched
+    };
+    activeState = state;
+    // Return a synchronous iterable. The iterator generator checks
+    // the shared state on each re-entry. The try/finally ensures
+    // source.close() is called when the loop exits — whether by
+    // exhaustion (DONE), break, or throw. This provides immediate
+    // cleanup without waiting for scope teardown (ensure() is still
+    // registered as a safety net for cancellation during fetch).
+    return {
+        *[Symbol.iterator]() {
+            try {
+                while (!isDone(state.current)) {
+                    // Advance guard: detect missing yield* durableEach.next()
+                    if (!state.advanced) {
+                        throw new Error(`durableEach("${name}"): yield* durableEach.next() must be called before the next iteration. Each loop body must end with yield* durableEach.next() to checkpoint progress and fetch the next item.`);
+                    }
+                    state.advanced = false;
+                    yield state.current;
+                }
+            }
+            finally {
+                // Clear module-level state so a subsequent durableEach can run.
+                activeState = null;
+                source.close?.();
+            }
+        },
+    };
+}
+/**
+ * Durable iteration over a DurableSource.
+ *
+ * Returns Workflow<Iterable<T>> via ephemeral() — the infrastructure
+ * effect (ensure) is durable-safe and re-runs correctly on replay.
+ */
+function _durableEach(name, source) {
+    return ephemeral(_durableEachOp(name, source));
+}
+// ---------------------------------------------------------------------------
+// durableEach.next — static method to advance iteration
+// ---------------------------------------------------------------------------
+/**
+ * Advance the current durable iteration.
+ *
+ * Reads state from the module-level activeState slot (set by durableEach).
+ * This is a pure Workflow — no infrastructure effects, no ephemeral()
+ * needed. The only yielded effect is durableEachFetch, which is already
+ * a DurableEffect (journaled).
+ */
+function* _durableEachNext() {
+    if (activeState === null) {
+        throw new Error("durableEach.next(): no active durableEach iteration. " +
+            "durableEach.next() must be called inside a durableEach loop.");
+    }
+    const state = activeState;
+    // Fetch next item first, then mark advanced. If the fetch throws
+    // (source error), advanced stays false and re-entry triggers the
+    // advance guard — preventing stale current from being re-yielded.
+    state.current = yield* durableEachFetch(state.name, state.source);
+    state.advanced = true;
+}
+// ---------------------------------------------------------------------------
+// Public API — durableEach with static .next() method
+// ---------------------------------------------------------------------------
+/**
+ * Durable iteration over a DurableSource.
+ *
+ * Returns a Workflow that fetches the first item and yields a
+ * synchronous iterable. Use with `for...of` inside a Workflow:
+ *
+ * ```typescript
+ * for (let msg of yield* durableEach("queue", source)) {
+ *   yield* durableCall("process", () => process(msg));
+ *   yield* durableEach.next();
+ * }
+ * ```
+ *
+ * @param name Stable name for the iteration
+ * @param source Operation-native source of items
+ */
+export const durableEach = Object.assign(_durableEach, { next: _durableEachNext });