@intx/workflow-host 0.2.2

package/dist/supervisor/drain-timeout.js

@@ -0,0 +1,231 @@
+// Supervisor-side drainTimeout accumulator and escalation path.
+//
+// When the supervisor's control loop sends `drain` to the workflow
+// process, the child's `DrainController` flips its signal and the
+// runtime body picks up the change on the next tick at each of the
+// four observation points. Steps whose declared `drainBehavior` is
+// `"cancel"` abort their local controller; steps whose behavior is
+// `"wait"` continue running.
+//
+// The supervisor's accumulator tracks wall-clock time spent against
+// `"cancel"`-behavior work after drain has been issued. Time spent
+// while every in-flight step is `"wait"`-behavior does NOT tick the
+// accumulator -- waiting for a human-in-the-loop pause should not
+// burn the drain budget. On `drainTimeout` expiry, the accumulator
+// invokes the supervisor's injected `signAsPrincipal("supervisor",
+// ...)` callback to obtain a signed `CancelRequested{origin:
+// "supervisor-drain"}` and commits it via the injected substrate
+// handle. The runtime body's existing cancellation cascade handles
+// teardown from there.
+//
+// The accumulator is started by the supervisor on receipt of the
+// host's drain command; it is stopped when (a) drainTimeout expires
+// and the CancelRequested commit lands, or (b) the run reaches a
+// terminal phase before expiry. Pausing is driven by the supervisor's
+// view of which step kinds are in flight -- it consults the same
+// `behaviorFor` projection the workflow-host child uses.
+import { getLogger } from "@intx/log";
+import { commitCancelRequested } from "./cancel-signing.js";
+const logger = getLogger(["workflow-host", "supervisor", "drain-timeout"]);
+/**
+ * Default `drainTimeout` per deployment. Operators override this via
+ * a per-deployment policy; this value is the spec's locked default.
+ */
+export const DEFAULT_DRAIN_TIMEOUT_MS = 60_000;
+/**
+ * Construct the supervisor's drainTimeout accumulator. The returned
+ * handle is single-use: once `escalated` flips true the accumulator
+ * has committed `CancelRequested{origin: "supervisor-drain"}` and
+ * cannot tick again.
+ */
+export function createDrainTimeoutAccumulator(opts) {
+    const drainTimeoutMs = opts.drainTimeoutMs ?? DEFAULT_DRAIN_TIMEOUT_MS;
+    const reason = opts.reason ?? "drainTimeout expired";
+    let state = { phase: "idle" };
+    let escalated = false;
+    /**
+     * Active terminal-event iterator the accumulator is consuming for
+     * its tracked runId. Held so the dispose path can finalise it via
+     * `return()`. `null` when no source binding was supplied or the
+     * iterator has already been finalised.
+     */
+    let terminalIterator = null;
+    /** Promise the per-runId terminal-watcher coroutine resolves into. */
+    let terminalWatcherDone = null;
+    /** Latest pending escalation commit; awaited by `disposed()`. */
+    let escalationPending = null;
+    let disposedPromise = null;
+    function armTerminalWatcher() {
+        if (opts.terminalEventSource === undefined)
+            return;
+        if (terminalIterator !== null)
+            return;
+        const iterable = opts.terminalEventSource(opts.runId);
+        const iterator = iterable[Symbol.asyncIterator]();
+        terminalIterator = iterator;
+        terminalWatcherDone = (async () => {
+            try {
+                // The first terminal event the source yields is the signal to
+                // settle. The source pre-filters on runId, so any element it
+                // produces applies to the tracked run.
+                const next = await iterator.next();
+                if (next.done === true)
+                    return;
+                settleOnTerminal();
+            }
+            catch (cause) {
+                // The iterator's failure does not need an explicit
+                // escalation: the outer `setTimer`-based deadline keeps
+                // ticking against the same `state.phase === "running"` slot
+                // and fires `escalate()` if no terminal event arrives. The
+                // warn here surfaces the iterator failure to operator logs
+                // so a persistent broken event source is visible; the
+                // accumulator's contract is preserved by the timer's
+                // fall-through.
+                const message = cause instanceof Error ? cause.message : String(cause);
+                logger.warn `terminal-event watcher for run ${opts.runId} threw: ${message}`;
+            }
+        })();
+    }
+    function settleOnTerminal() {
+        if (escalated)
+            return;
+        if (state.phase === "stopped" || state.phase === "escalated")
+            return;
+        if (state.phase === "running") {
+            opts.clearTimer(state.timer);
+        }
+        state = { phase: "stopped" };
+    }
+    function remainingMs() {
+        if (state.phase === "running") {
+            const elapsed = opts.now() - state.startedAt + state.baseline;
+            return Math.max(0, drainTimeoutMs - elapsed);
+        }
+        if (state.phase === "paused") {
+            return Math.max(0, drainTimeoutMs - state.baseline);
+        }
+        return drainTimeoutMs;
+    }
+    function arm(baseline) {
+        const remaining = drainTimeoutMs - baseline;
+        if (remaining <= 0) {
+            void escalate();
+            return;
+        }
+        const timer = opts.setTimer(() => {
+            // The timer is the authority for escalation -- the accumulator
+            // does not poll. When the timer fires we transition to
+            // `escalated` and commit through the substrate.
+            void escalate();
+        }, remaining);
+        state = {
+            phase: "running",
+            startedAt: opts.now(),
+            baseline,
+            timer,
+        };
+    }
+    function escalate() {
+        if (escalated)
+            return;
+        escalated = true;
+        if (state.phase === "running") {
+            opts.clearTimer(state.timer);
+        }
+        state = { phase: "escalated" };
+        // The terminal watcher's role ends when the deadline fires: the
+        // commit below races the on-disk CancelRequested against any
+        // natural terminal arrival, and either way the accumulator settles
+        // here. Finalising the iterator lets the producer free its
+        // resources.
+        finaliseTerminalWatcher();
+        const origin = "supervisor-drain";
+        escalationPending = commitCancelRequested({
+            substrate: opts.substrate,
+            repoId: opts.repoId,
+            ref: opts.ref,
+            deploymentId: opts.deploymentId,
+            runId: opts.runId,
+            origin,
+            reason,
+            at: new Date().toISOString(),
+            signAsPrincipal: opts.signAsPrincipal,
+        })
+            .then(() => undefined)
+            .catch((cause) => {
+            const message = cause instanceof Error ? cause.message : String(cause);
+            logger.error `drainTimeout escalation commit failed for run ${opts.runId}: ${message}`;
+            throw cause instanceof Error ? cause : new Error(message);
+        });
+    }
+    function finaliseTerminalWatcher() {
+        const iterator = terminalIterator;
+        if (iterator === null)
+            return;
+        terminalIterator = null;
+        if (typeof iterator.return !== "function")
+            return;
+        // Fire the iterator's `return()` so the producer side observes the
+        // cancel. The producer (`subscribeKind`-backed) wires its own
+        // `AbortSignal`; finalising the iterator surfaces as an abort
+        // through the substrate's `subscribe` primitive, which is the
+        // contract `subscribeKind` documents.
+        void iterator.return(undefined).catch((cause) => {
+            const message = cause instanceof Error ? cause.message : String(cause);
+            logger.warn `terminal-event watcher return() for run ${opts.runId} threw: ${message}`;
+        });
+    }
+    return {
+        start() {
+            if (state.phase !== "idle")
+                return;
+            armTerminalWatcher();
+            arm(0);
+        },
+        pause() {
+            if (state.phase !== "running")
+                return;
+            const elapsed = opts.now() - state.startedAt + state.baseline;
+            opts.clearTimer(state.timer);
+            state = { phase: "paused", baseline: elapsed };
+        },
+        resume() {
+            if (state.phase !== "paused")
+                return;
+            arm(state.baseline);
+        },
+        stop() {
+            if (state.phase === "running") {
+                opts.clearTimer(state.timer);
+            }
+            finaliseTerminalWatcher();
+            if (state.phase === "escalated")
+                return;
+            state = { phase: "stopped" };
+        },
+        accumulatedMs() {
+            return drainTimeoutMs - remainingMs();
+        },
+        get escalated() {
+            return escalated;
+        },
+        disposed() {
+            if (disposedPromise !== null)
+                return disposedPromise;
+            disposedPromise = (async () => {
+                if (escalationPending !== null) {
+                    await escalationPending.catch(() => {
+                        /* error already logged in `escalate`. */
+                    });
+                }
+                if (terminalWatcherDone !== null) {
+                    await terminalWatcherDone.catch(() => {
+                        /* error already logged in the watcher coroutine. */
+                    });
+                }
+            })();
+            return disposedPromise;
+        },
+    };
+}

package/dist/supervisor/index.d.ts

@@ -0,0 +1,7 @@
+export { createWorkflowSupervisor, DEFAULT_TERMINAL_WRITE_WATCHDOG_MS, type CancelCommitInfo, type CancelRequestOpts, type DeliverSignalOpts, type DeliverSourcesOpts, type DrainOpts, type RecycleOpts, type SpawnOpts, type SpawnResult, type WorkflowSupervisor, } from "./supervisor.js";
+export { assembleCredentialsSnapshot, defaultStepRepoId, hashGrants, STEP_GRANTS_PATH, STEP_GRANTS_REF, type AssembleCredentialsSnapshotOpts, type CredentialsSnapshot, type CredentialsSnapshotStep, type DeriveStepAddress, type DeriveStepRepoId, } from "./credentials.js";
+export { commitCancelRequested, SUPERVISOR_PRINCIPAL_KIND, type CommitCancelRequestedOpts, type CommitCancelRequestedResult, } from "./cancel-signing.js";
+export { createDrainTimeoutAccumulator, DEFAULT_DRAIN_TIMEOUT_MS, type DrainTimeoutAccumulator, type DrainTimeoutAccumulatorFactory, type DrainTimeoutOpts, } from "./drain-timeout.js";
+export { DEFAULT_KILL_TIMEOUT_MS } from "./child-termination.js";
+export { createRecyclePolicy, triggerRecycle, DEFAULT_POLICY_INTERVAL_MS, MAX_BUFFERED_MAIL, type ChildWiring, type RecycleAttempt, type RecycleContext, type RecycleOrigin, type RecyclePolicy, type RecyclePolicyBounds, type RecyclePolicyOpts, type TriggerRecycleOpts, } from "./recycle.js";
+export type { DeriveMailAuditRef, DispatchStructuralCounters, DispatchSubstrateLeg, DispatchTimingMark, InboxPrimitives, MailAuditRef, MailBusBindings, PrincipalSigner, SignedPayload, SubprocessHandle, SubprocessSpawner, TerminalEventSource, TerminalRunEvent, WorkflowSupervisorBindings, WorkflowSupervisorPrincipalKind, } from "./types.js";

package/dist/supervisor/index.js

@@ -0,0 +1,6 @@
+export { createWorkflowSupervisor, DEFAULT_TERMINAL_WRITE_WATCHDOG_MS, } from "./supervisor.js";
+export { assembleCredentialsSnapshot, defaultStepRepoId, hashGrants, STEP_GRANTS_PATH, STEP_GRANTS_REF, } from "./credentials.js";
+export { commitCancelRequested, SUPERVISOR_PRINCIPAL_KIND, } from "./cancel-signing.js";
+export { createDrainTimeoutAccumulator, DEFAULT_DRAIN_TIMEOUT_MS, } from "./drain-timeout.js";
+export { DEFAULT_KILL_TIMEOUT_MS } from "./child-termination.js";
+export { createRecyclePolicy, triggerRecycle, DEFAULT_POLICY_INTERVAL_MS, MAX_BUFFERED_MAIL, } from "./recycle.js";

package/dist/supervisor/recycle.d.ts

@@ -0,0 +1,212 @@
+import { type ControlChannelSender, type ControlPayload, type EventPayload } from "../ipc/index.js";
+import { type CredentialsSnapshot } from "./credentials.js";
+import type { SubprocessHandle, WorkflowSupervisorBindings } from "./types.js";
+/**
+ * Bound on the supervisor's mail buffer across the kill/respawn gap.
+ * A real workflow's inbound rate is well below this; saturation
+ * indicates either an upstream stuck on the deployment or a recycle
+ * stuck partway through. Either case is one the operator must see.
+ */
+export declare const MAX_BUFFERED_MAIL = 256;
+/**
+ * Default supervisor-policy check interval. The policy thread wakes
+ * roughly every minute, evaluates the configured bounds against the
+ * live child, and triggers a recycle if any threshold has been
+ * crossed. Operator-overridable via the supervisor's policy bindings.
+ */
+export declare const DEFAULT_POLICY_INTERVAL_MS = 60000;
+/**
+ * Origin tag the recycle path stamps onto its log messages so an
+ * operator scanning logs can distinguish operator-initiated,
+ * policy-initiated, and self-initiated recycles at a glance.
+ */
+export type RecycleOrigin = "operator" | "policy" | "self";
+export interface RecycleAttempt {
+    /** Origin the recycle was initiated from. */
+    readonly origin: RecycleOrigin;
+    /** Human-readable reason carried with the recycle through to the audit log. */
+    readonly reason: string;
+    /** ChannelId the recycled child was minted with. */
+    readonly newChannelId: string;
+    /** ChannelId the previous child was running under. */
+    readonly previousChannelId: string;
+}
+/**
+ * Per-handle subprocess wiring the recycle path owns. The supervisor
+ * passes the live child's wiring on entry; the recycle path replaces
+ * it with the freshly-spawned child's wiring before returning.
+ */
+export interface ChildWiring {
+    handle: SubprocessHandle;
+    controlSender: ControlChannelSender;
+    channelId: string;
+    eventPump: Promise<void>;
+}
+/**
+ * Bindings the supervisor passes into `triggerRecycle`. The shape
+ * mirrors the subset of supervisor state the recycle sequence
+ * touches; it intentionally does NOT include the supervisor's mail-
+ * subscription disposer (the supervisor holds the registration across
+ * the recycle) nor the mail-bus binding itself (the recycle path does
+ * not re-register).
+ */
+export interface RecycleContext {
+    /** The supervisor's full bindings, reused on respawn for credentials and spawn. */
+    readonly bindings: WorkflowSupervisorBindings;
+    /** Step ids in this deployment's `stepOrder` for credentials re-assembly. */
+    readonly stepOrder: readonly string[];
+    /** Definition hash carried on respawn env (unchanged across recycle). */
+    readonly definitionHash: string;
+    /**
+     * Warm-keep flag carried on the respawn env (design §3b). Unchanged
+     * across recycle: the respawned child rebuilds its empty warm-agent
+     * cache lazily on the next message, so the deterministic warm-keep
+     * decision must survive the respawn rather than be re-derived.
+     */
+    readonly warmKeep: boolean;
+    /** Forward target for InferenceEvents the new child publishes. */
+    readonly onInferenceEvent: (event: EventPayload) => void;
+    /** Live child wiring on entry; replaced before return. */
+    readonly current: ChildWiring;
+    /** Supervisor-side drain primitive; sends the existing drain mail. */
+    readonly drain: (deadlineMs: number) => Promise<void>;
+    /**
+     * Replay any `processing/` claim-check entries for the deployment's
+     * mail address back to `inbox/` so the FIFO ordering survives the
+     * recycle. Invoked AFTER the drain step settles and BEFORE the kill
+     * step lands, which eliminates the race window where a processing
+     * entry would exist with no owner. The supervisor closes this
+     * callback over its `inboxPrimitives.replayProcessingToInbox` plus
+     * the deployment's substrate principal and repo identity.
+     */
+    readonly replayProcessingToInbox: () => Promise<void>;
+    /**
+     * Abort the prior cohort's terminal source and wake the dispatch
+     * loop so it exits before the kill step lands. Invoked AFTER drain
+     * and replay settle and BEFORE the kill -- earlier would starve the
+     * drain step's accumulators of live terminal events; later would
+     * race the kill against the dispatch loop's next iteration.
+     */
+    readonly abortPriorCohort: () => void;
+    /**
+     * Onward sink the supervisor uses to install the new child wiring
+     * once the freshly-spawned child has emitted `ready` and the
+     * credentialsSnapshot has been re-assembled.
+     */
+    readonly installNewChild: (next: {
+        wiring: ChildWiring;
+        credentialsSnapshot: CredentialsSnapshot;
+        /**
+         * Live upstream control iterator the new child's receiver
+         * yields. The supervisor's upstream-control pump consumes the
+         * iterator after `installNewChild` returns so child-initiated
+         * `recycle.request` frames on the new wiring continue to funnel
+         * through `triggerRecycle`.
+         */
+        controlIncoming: AsyncGenerator<ControlPayload, void, void>;
+    }) => void;
+    /**
+     * Crash hook the new child's IPC channels wire to. Identical shape
+     * to the supervisor's spawn-time onCrash so a frame violation on the
+     * recycled wiring tears the deployment down through the same path.
+     */
+    readonly onCrash: (reason: string) => void;
+    /**
+     * Optional kill-timeout override (ms). Defaults to
+     * `DEFAULT_KILL_TIMEOUT_MS`.
+     */
+    readonly killTimeoutMs?: number;
+    /**
+     * Deadline (ms) for the respawned child's `ready` handshake, matching
+     * the bound the spawn path applies. The supervisor resolves the
+     * effective value at its edge (`bindings.readyTimeoutMs ??
+     * DEFAULT_READY_TIMEOUT_MS`) and passes it through; the `??` fallback
+     * here only fires for a direct test caller.
+     */
+    readonly readyTimeoutMs?: number;
+    /**
+     * Optional drain deadline (ms) used in step 1. The supervisor's own
+     * drainTimeout accumulator escalates separately; this deadline is
+     * the wait the recycle path itself observes before proceeding to
+     * step 2. Defaults to the drain accumulator's default.
+     */
+    readonly drainDeadlineMs?: number;
+    /**
+     * Optional setTimer/clearTimer pair used by the SIGKILL escalation
+     * wait. Production wires `setTimeout`/`clearTimeout`; tests inject
+     * a deterministic timer so the SIGKILL window is observable.
+     */
+    readonly setTimer?: (cb: () => void, ms: number) => unknown;
+    readonly clearTimer?: (handle: unknown) => void;
+}
+export interface TriggerRecycleOpts {
+    origin: RecycleOrigin;
+    reason: string;
+}
+/**
+ * Run the six-step recycle sequence. The function returns once the
+ * new child has emitted `ready` and the supervisor has drained its
+ * buffered mail into it; the supervisor installs the new wiring via
+ * `ctx.installNewChild` before that point.
+ */
+export declare function triggerRecycle(ctx: RecycleContext, opts: TriggerRecycleOpts): Promise<RecycleAttempt>;
+export interface RecyclePolicyBounds {
+    /**
+     * Maximum uptime (ms) for the workflow-process child before a
+     * recycle is triggered. `undefined` disables the bound.
+     */
+    maxUptimeMs?: number;
+    /**
+     * Maximum resident-set size (bytes) for the workflow-process child
+     * before a recycle is triggered. `undefined` disables the bound.
+     * The supervisor's `readRssBytes` callback is consulted on every
+     * policy tick; an absent callback disables the bound regardless of
+     * the threshold.
+     */
+    maxRssBytes?: number;
+    /**
+     * Maximum age (ms) since grants were last refreshed before a
+     * recycle is triggered. The supervisor's `readGrantsAgeMs` callback
+     * is consulted on every policy tick; an absent callback disables
+     * the bound regardless of the threshold.
+     */
+    maxGrantsAgeMs?: number;
+}
+export interface RecyclePolicyOpts {
+    /** Bounds the policy evaluates each tick. */
+    bounds: RecyclePolicyBounds;
+    /** Tick interval in ms. Defaults to `DEFAULT_POLICY_INTERVAL_MS`. */
+    intervalMs?: number;
+    /** Wall-clock reader; production wires `() => Date.now()`. */
+    now: () => number;
+    /** Spawn-time wall-clock the policy compares against `now()`. */
+    spawnedAt: number;
+    /** Per-tick RSS reader; absent disables the `maxRssBytes` bound. */
+    readRssBytes?: () => number | undefined;
+    /** Per-tick grants-age reader; absent disables the staleness bound. */
+    readGrantsAgeMs?: () => number | undefined;
+    /** Timer setter; production wires `setInterval`-style via `setTimer`. */
+    setTimer: (cb: () => void, ms: number) => unknown;
+    /** Timer disposer; production wires the matching `clearTimer`. */
+    clearTimer: (handle: unknown) => void;
+    /**
+     * Recycle entry point the policy invokes on a threshold trip. The
+     * supervisor's `recycle()` method wraps `triggerRecycle` and is the
+     * production callback.
+     */
+    trigger: (reason: string) => Promise<void>;
+}
+export interface RecyclePolicy {
+    /** Stop the timer; idempotent. */
+    stop(): void;
+    /** Evaluate the bounds once and trigger if any are tripped. */
+    tick(): Promise<void>;
+}
+/**
+ * Start the supervisor-policy periodic recycle check. Returns a
+ * handle the supervisor calls `stop()` on at shutdown. The policy is
+ * single-trigger per tick: even if multiple bounds are tripped on the
+ * same tick, exactly one `trigger` invocation lands with a reason
+ * naming the first tripped bound.
+ */
+export declare function createRecyclePolicy(opts: RecyclePolicyOpts): RecyclePolicy;