@intx/workflow-host 0.2.2 → 0.4.0

package/dist/supervisor/recycle.js

@@ -4,8 +4,9 @@
 //
 // Recycle is the supervisor's "same deploy tree, fresh process" path.
 // It tears the existing workflow-process child down and stands a new
-// one up against the SAME deploy tree (same `workflow.json`, same
-// per-step credential repos). It is STRICTLY ORTHOGONAL TO REDEPLOY:
+// one up against the SAME deploy tree (same materialized source
+// closure, same per-step credential repos). It is STRICTLY ORTHOGONAL
+// TO REDEPLOY:
 //
 //   - Recycle  = same deploy tree, fresh process.
 //   - Redeploy = new deploy tree.
@@ -20,6 +21,15 @@
 //
 // Six-step sequence (locked):
 //
+// The `drain` step and the `SubprocessHandle` handed in as `current` are
+// caller-parameterized. For the operator/policy/self recycle origins the
+// child is live: `drain` sends the real drain control mail and `kill`
+// terminates a running process. For the `crash` origin the child has
+// already exited unexpectedly, so the caller supplies a no-op `drain`
+// (there is nothing to drain) and the `kill` in step 2 lands on an
+// already-dead handle as a cheap no-op. Steps 3-6 are identical for every
+// origin.
+//
 //   1. `drain` -- send the existing drain control mail. Wait for
 //      in-flight runs to drain per each step's `drainBehavior`.
 //      `drainTimeout` escalation applies normally; the drain-timeout
@@ -153,8 +163,8 @@ export async function triggerRecycle(ctx, opts) {
     // Step 3: respawn. Fresh channelId, fresh HMAC key, fresh Ed25519
     // IPC keypair. Per-step credentials are re-read so a grants update
     // that landed since the original spawn is reflected in the new
-    // child's snapshot. The deploy tree (`workflow.json`, agents,
-    // workflow-asset repo) is UNCHANGED.
+    // child's snapshot. The deploy tree (the materialized source closure,
+    // the workflow-asset repo, the agent-state repos) is UNCHANGED.
     const channelId = generateChannelId();
     const hmacKey = generateHmacKey();
     const ipcKeypair = await (ctx.bindings.ipcKeyPairFactory ?? generateKeyPair)();
@@ -164,7 +174,7 @@ export async function triggerRecycle(ctx, opts) {
         channelId,
         hmacKey,
         hostPublicKey: ipcKeypair.publicKey,
-        deploymentId: ctx.bindings.deploymentId,
+        anchorRunId: ctx.bindings.anchorRunId,
         deploymentMailAddress: ctx.bindings.deploymentMailAddress,
         stepCount: ctx.bindings.stepCount,
         definitionHash: ctx.definitionHash,
@@ -210,7 +220,8 @@ export async function triggerRecycle(ctx, opts) {
     // the previous child's lifetime is picked up here -- the recycle
     // doubles as the supervisor's grant-refresh path. The deploy tree
     // is not consulted; this read is against the `agent-state` repos
-    // alone, whose contents are independent of `workflow.json`.
+    // alone, whose contents are independent of the materialized source
+    // closure.
     //
     // This is a substrate read that can reject -- a grants file that
     // became malformed is precisely the recycle's grant-refresh path. The
@@ -227,7 +238,7 @@ export async function triggerRecycle(ctx, opts) {
             repoStore: ctx.bindings.repoStore,
             principal: ctx.bindings.readPrincipal,
             stepOrder: ctx.stepOrder,
-            deploymentId: ctx.bindings.deploymentId,
+            anchorRunId: ctx.bindings.anchorRunId,
             deriveStepAddress: ctx.bindings.deriveStepAddress,
             ...(ctx.bindings.deriveStepRepoId !== undefined
                 ? { deriveStepRepoId: ctx.bindings.deriveStepRepoId }

package/dist/supervisor/run-event-compaction.d.ts

@@ -6,8 +6,8 @@ export type CompactRunEventsOpts = {
     repoId: RepoId;
     /** Events ref the workflow-run repo writes to. */
     ref: string;
-    /** Deployment id used to construct the supervisor principal. */
-    deploymentId: string;
+    /** Anchor run id used to construct the supervisor principal. */
+    anchorRunId: string;
     /** Run to seal. */
     runId: string;
 };
@@ -19,9 +19,9 @@ export type CompactRunEventsOpts = {
  *
  * Idempotent and terminal-only: a run already sealed (no `events/` subtree)
  * or one whose latest event is not terminal is left untouched, so the call
- * is safe to repeat. The live caller fires it once per run, right after the
- * run terminates; a bounded recovery sweep that would re-fire it to seal a
- * run whose fold a crash interrupted is not yet implemented.
+ * is safe to repeat. The live caller invokes it once per run, right after the
+ * run terminates; `recoverInterruptedCompactions` re-runs it for a run whose
+ * fold a crash interrupted before it could seal.
  *
  * The combined file is the verbatim byte concatenation of the per-event
  * blobs in seq order (`encodeCombinedEventLog`), the exact shape the

package/dist/supervisor/run-event-compaction.js

@@ -5,18 +5,12 @@
 // and drops the per-event files. This shrinks the workflow-run repo's
 // file count -- and every per-commit cost that scales with it --
 // without losing any event. The fold writes under the substrate's
-// per-repo lock as the `supervisor` principal, whose `deploymentId`
+// per-repo lock as the `supervisor` principal, whose `anchorRunId`
 // the workflow-run kind handler checks against `repoId.id`.
-import { WORKFLOW_RUN_EVENTS_FILE, encodeCombinedEventLog, } from "@intx/hub-sessions/substrate";
+import { classifyTerminalEvent, parseEventSeq, WORKFLOW_RUN_EVENTS_FILE, encodeCombinedEventLog, } from "@intx/hub-sessions/substrate";
 import { SUPERVISOR_PRINCIPAL_KIND } from "./cancel-signing.js";
 const RUNS_PREFIX = "runs";
 const EVENTS_DIR = "events";
-const EVENT_FILENAME_RE = /^(0|[1-9][0-9]*)\.json$/;
-const TERMINAL_EVENT_TYPES = new Set([
-    "RunCompleted",
-    "RunFailed",
-    "RunCancelled",
-]);
 /**
  * Fold a terminated run's per-event `events/<seq>.json` blobs into one
  * combined `events.jsonl`, dropping the per-event files. This shrinks the
@@ -25,9 +19,9 @@ const TERMINAL_EVENT_TYPES = new Set([
  *
  * Idempotent and terminal-only: a run already sealed (no `events/` subtree)
  * or one whose latest event is not terminal is left untouched, so the call
- * is safe to repeat. The live caller fires it once per run, right after the
- * run terminates; a bounded recovery sweep that would re-fire it to seal a
- * run whose fold a crash interrupted is not yet implemented.
+ * is safe to repeat. The live caller invokes it once per run, right after the
+ * run terminates; `recoverInterruptedCompactions` re-runs it for a run whose
+ * fold a crash interrupted before it could seal.
  *
  * The combined file is the verbatim byte concatenation of the per-event
  * blobs in seq order (`encodeCombinedEventLog`), the exact shape the
@@ -55,10 +49,10 @@ export async function compactRunEvents(opts) {
     }
     const seqs = [];
     for (const name of filenames) {
-        const match = EVENT_FILENAME_RE.exec(name);
-        if (match === null || match[1] === undefined)
+        const seq = parseEventSeq(name);
+        if (seq === null)
             continue;
-        seqs.push(Number.parseInt(match[1], 10));
+        seqs.push(seq);
     }
     if (seqs.length === 0)
         return { compacted: false };
@@ -74,14 +68,15 @@ export async function compactRunEvents(opts) {
         return { compacted: false };
     }
     const lastType = parsed.type;
-    if (typeof lastType !== "string" || !TERMINAL_EVENT_TYPES.has(lastType)) {
+    if (typeof lastType !== "string" ||
+        !classifyTerminalEvent(lastType).terminal) {
         return { compacted: false };
     }
     const prefix = `${RUNS_PREFIX}/${opts.runId}/${EVENTS_DIR}/`;
     const combinedPath = `${RUNS_PREFIX}/${opts.runId}/${WORKFLOW_RUN_EVENTS_FILE}`;
     const principal = {
         kind: SUPERVISOR_PRINCIPAL_KIND,
-        deploymentId: opts.deploymentId,
+        anchorRunId: opts.anchorRunId,
     };
     let sealed = false;
     await opts.substrate.writeTreePreservingPrefix(principal, opts.repoId, opts.ref, {
@@ -90,11 +85,11 @@ export async function compactRunEvents(opts) {
             const entries = [];
             for (const [filepath, bytes] of existing) {
                 const name = filepath.slice(prefix.length);
-                const match = EVENT_FILENAME_RE.exec(name);
-                if (match === null || match[1] === undefined) {
+                const seq = parseEventSeq(name);
+                if (seq === null) {
                     throw new Error(`supervisor run-event-compaction: unexpected non-event file ${filepath} under run ${opts.runId}; refusing to compact`);
                 }
-                entries.push({ seq: Number.parseInt(match[1], 10), bytes });
+                entries.push({ seq, bytes });
             }
             if (entries.length === 0)
                 return {};

package/dist/supervisor/run-event-recovery.d.ts

@@ -0,0 +1,34 @@
+import type { RepoId, RepoStore as SubstrateRepoStore } from "@intx/hub-sessions/substrate";
+export type RecoverInterruptedCompactionsOpts = {
+    /** Substrate handle the supervisor writes through. */
+    substrate: SubstrateRepoStore;
+    /** Workflow-run repo for this deployment. */
+    repoId: RepoId;
+    /** Events ref the workflow-run repo writes to. */
+    ref: string;
+    /** Anchor run id used to construct the supervisor principal. */
+    anchorRunId: string;
+    /** Runs the boot scan proposes as terminal-but-per-event. */
+    pendingSealRunIds: readonly string[];
+};
+/** A run whose recovery fold threw, paired with the failure cause. */
+export type RecoveryFoldFailure = {
+    runId: string;
+    message: string;
+};
+/**
+ * Re-seal runs a crash left terminal but still in per-event form, by re-running
+ * the idempotent `compactRunEvents` for each proposed run. `compactRunEvents`
+ * is authoritative: it no-ops a run that is already sealed or whose latest
+ * event is not terminal, so a stale or mistaken proposal is a harmless no-op.
+ *
+ * Folds run serially. Every fold contends the same per-repo write lock that
+ * live dispatch also takes, so folding one run at a time drains the backlog
+ * without a thundering herd on that lock. One run's failure is caught so it
+ * cannot abort the rest; the failed run id and its cause are returned -- not
+ * logged here -- so the caller owns how to surface the aggregate.
+ */
+export declare function recoverInterruptedCompactions(opts: RecoverInterruptedCompactionsOpts): Promise<{
+    sealed: number;
+    failed: RecoveryFoldFailure[];
+}>;

package/dist/supervisor/run-event-recovery.js

@@ -0,0 +1,45 @@
+// Bounded recovery for run-event compaction folds a crash interrupted.
+//
+// When a run terminates, the supervisor fires `compactRunEvents` in the
+// background. A crash between the terminal commit and that fold leaves the
+// run terminal but still in per-event form, and the terminal signal never
+// fires again for it. At the next spawn the boot scan proposes those runs;
+// this sweep re-runs the idempotent fold for each so the leaked per-event
+// file count is reclaimed.
+import { compactRunEvents } from "./run-event-compaction.js";
+/**
+ * Re-seal runs a crash left terminal but still in per-event form, by re-running
+ * the idempotent `compactRunEvents` for each proposed run. `compactRunEvents`
+ * is authoritative: it no-ops a run that is already sealed or whose latest
+ * event is not terminal, so a stale or mistaken proposal is a harmless no-op.
+ *
+ * Folds run serially. Every fold contends the same per-repo write lock that
+ * live dispatch also takes, so folding one run at a time drains the backlog
+ * without a thundering herd on that lock. One run's failure is caught so it
+ * cannot abort the rest; the failed run id and its cause are returned -- not
+ * logged here -- so the caller owns how to surface the aggregate.
+ */
+export async function recoverInterruptedCompactions(opts) {
+    let sealed = 0;
+    const failed = [];
+    for (const runId of opts.pendingSealRunIds) {
+        try {
+            const { compacted } = await compactRunEvents({
+                substrate: opts.substrate,
+                repoId: opts.repoId,
+                ref: opts.ref,
+                anchorRunId: opts.anchorRunId,
+                runId,
+            });
+            if (compacted)
+                sealed += 1;
+        }
+        catch (cause) {
+            failed.push({
+                runId,
+                message: cause instanceof Error ? cause.message : String(cause),
+            });
+        }
+    }
+    return { sealed, failed };
+}

package/dist/supervisor/spawn-env.d.ts

@@ -19,8 +19,8 @@ export interface ChildSpawnEnvParts {
     hmacKey: Uint8Array;
     /** Supervisor's Ed25519 public key for this spawn's control channel. */
     hostPublicKey: Uint8Array;
-    /** Deployment identity the supervisor manages. */
-    deploymentId: string;
+    /** Anchor run id the supervisor manages. */
+    anchorRunId: string;
     /** Mail address the deployment registered on the bus. */
     deploymentMailAddress: string;
     /** Step count of the deployed workflow (`stepOrder.length`). */

package/dist/supervisor/spawn-env.js

@@ -20,7 +20,7 @@ export function buildChildSpawnEnv(parts) {
         IPC_CHANNEL_ID: parts.channelId,
         IPC_HMAC_KEY: hexEncode(parts.hmacKey),
         HOST_PUBKEY: hexEncode(parts.hostPublicKey),
-        DEPLOYMENT_ID: parts.deploymentId,
+        DEPLOYMENT_ID: parts.anchorRunId,
         DEFINITION_HASH: parts.definitionHash,
         MAILBOX_ADDRESS: parts.deploymentMailAddress,
         STEP_COUNT: String(parts.stepCount),

package/dist/supervisor/supervisor.d.ts

@@ -1,3 +1,4 @@
+import { type CredentialDelivery } from "@intx/types/sidecar";
 import type { InferenceSource } from "@intx/types/runtime";
 import type { CancelOrigin } from "@intx/workflow";
 import { type EventPayload } from "../ipc/index.js";
@@ -5,20 +6,57 @@ import { type CredentialsSnapshot } from "./credentials.js";
 import { type RecycleAttempt, type RecycleOrigin } from "./recycle.js";
 import type { WorkflowSupervisorBindings } from "./types.js";
 /**
- * Default watchdog timeout for the supervisor's
- * `synchronouslyDispatchTerminalWrite`. The handler holds the
- * `substrate.write.response` back to the child until the dispatch
- * loop's `markConsumed` settles for the matching terminal event; an
- * unbounded wait would chain into a child / runtime / dispatch loop
- * deadlock if `markConsumed` never armed (bug in the dispatch loop, a
- * torn-down cohort, a stalled inbox primitive). 30s sits between the
- * recycle path's `DEFAULT_KILL_TIMEOUT_MS` (5s, a hard process-level
- * kill cap) and `DEFAULT_DRAIN_TIMEOUT_MS` (60s, the per-deployment
- * drain budget) -- generous enough to absorb a slow legitimate
- * markConsumed, tight enough to surface a real deadlock long before
- * the drainTimeout would otherwise mask it.
+ * Default crash-loop bound: the supervisor stops respawning and latches
+ * the deployment once the workflow-process child exits unexpectedly this
+ * many times within `DEFAULT_CRASH_LOOP_WINDOW_MS`. Overridable via
+ * `WorkflowSupervisorBindings.crashLoopMaxCount`.
  */
-export declare const DEFAULT_TERMINAL_WRITE_WATCHDOG_MS = 30000;
+export declare const DEFAULT_CRASH_LOOP_MAX_COUNT = 3;
+/**
+ * Default sliding window (ms) over which `DEFAULT_CRASH_LOOP_MAX_COUNT`
+ * unexpected exits latch the deployment. Overridable via
+ * `WorkflowSupervisorBindings.crashLoopWindowMs`.
+ */
+export declare const DEFAULT_CRASH_LOOP_WINDOW_MS = 60000;
+/**
+ * Default stable-run duration (ms): once a respawned child stays up this
+ * long, the crash counter resets so flapping followed by stability does
+ * not permanently latch. Overridable via
+ * `WorkflowSupervisorBindings.crashLoopStableResetMs`.
+ */
+export declare const DEFAULT_CRASH_LOOP_STABLE_RESET_MS = 60000;
+/**
+ * Default initial respawn backoff (ms): the wait before the first respawn
+ * after an unexpected exit. Overridable via
+ * `WorkflowSupervisorBindings.respawnBackoffInitialMs`.
+ */
+export declare const DEFAULT_RESPAWN_BACKOFF_INITIAL_MS = 1000;
+/**
+ * Default cap (ms) on the exponential respawn backoff. Kept below
+ * `DEFAULT_CRASH_LOOP_WINDOW_MS` so a slow flapper's crashes still fall
+ * within the window and latch the guard. Overridable via
+ * `WorkflowSupervisorBindings.respawnBackoffMaxMs`.
+ */
+export declare const DEFAULT_RESPAWN_BACKOFF_MAX_MS = 30000;
+/**
+ * Default watchdog for `reEmitParkedCorrelations`' wait on the child's
+ * `parked-correlations.response`. 30s is generous enough for a healthy child
+ * to enumerate its in-flight runs and load each parked snapshot, tight
+ * enough that a wedged-but-alive child does not hang the reconnect caller
+ * until some coarser timeout intervenes.
+ */
+export declare const DEFAULT_PARKED_QUERY_WATCHDOG_MS = 30000;
+/**
+ * Backstop for `waitForRunTerminalOrPark`. A dispatch waits here for the child
+ * to park or terminate the run before releasing `markConsumed`; a lost park
+ * wake or a wedged child would otherwise hang the deployment's dispatch loop
+ * forever. Five minutes is far beyond any healthy per-message dispatch (which
+ * settles in well under a second), so this never fires on a legitimately long
+ * run without also being a genuine fault -- and when it does fire it is logged
+ * loudly and fails the dispatch (the mail is left reclaimable, never consumed
+ * on the assumption the run made progress), not silently swallowed.
+ */
+export declare const TERMINAL_OR_PARK_BACKSTOP_MS = 300000;
 /**
  * Public surface returned by `createWorkflowSupervisor`. Each method
  * advances the supervisor through one lifecycle transition; the
@@ -89,6 +127,42 @@ export interface WorkflowSupervisor {
      * closing pipe. Throws otherwise.
      */
     deliverSources(opts: DeliverSourcesOpts): Promise<void>;
+    /**
+     * Push refreshed credential material to the child's in-memory cell. Mirrors
+     * `deliverSources`: the supervisor is the single producer of
+     * `credentials-updated` control frames, phase-guarded to starting/running so
+     * a frame is never written into a recycling child's closing pipe. A revoked
+     * credential is delivered by omitting its material so the child evicts it.
+     */
+    deliverCredentials(opts: DeliverCredentialsOpts): Promise<void>;
+    /**
+     * Refresh a live run's grant floor mid-run by re-reading its durable
+     * `runs/<runId>/grants.json` and pushing it as a `grants-updated` frame. The
+     * enforcement path for a standing (`scope: "always"`) approval that lowers a
+     * tool's `ask` to `allow` in that file. Unlike `deliverSignal`/
+     * `deliverSources`, a refresh for a non-live child is normal, so this
+     * NO-OPS (`skipped`) instead of throwing, and a send failure to a live child
+     * is logged loudly but stays non-fatal -- the durable file governs the next
+     * barrier/respawn. It pushes only that file's contents, never caller-supplied
+     * grants, so it can only tighten or refresh a floor. Returns whether a live
+     * push happened.
+     */
+    deliverGrants(runId: string): Promise<"pushed" | "skipped">;
+    /**
+     * Re-register every correlation the child is currently parked on by
+     * querying it for its parked correlations and re-emitting each through
+     * `onSuspensionRegister`. Recovers a `park.notify` register the hub may have
+     * missed while it was down at suspend time.
+     *
+     * Best-effort and safe to call at any time, including concurrently. Unlike
+     * `deliverSignal`/`deliverSources`, which throw when the child is not
+     * addressable, this NO-OPS on a non-addressable phase (idle / stopping /
+     * stopped / recycling): a re-establishment landing mid-recycle must not
+     * crash, and the next spawn re-drives it. A query that fails or times out is
+     * logged and dropped; the hub co-write is idempotent, so the next
+     * re-establishment re-drives it. The caller need not guard the call site.
+     */
+    reEmitParkedCorrelations(): Promise<void>;
     /**
      * Current snapshot of the credentials pushed to the child. Surfaced
      * so the host can audit the per-step contentHash without
@@ -97,7 +171,11 @@ export interface WorkflowSupervisor {
     getCredentialsSnapshot(): CredentialsSnapshot | null;
 }
 export type SpawnOpts = {
-    /** Step ids in this deployment's `stepOrder` for credentials assembly. */
+    /**
+     * Every step id in this deployment's flat step-id namespace -- the
+     * definition's own `stepOrder` plus the step ids of every `loop` body it
+     * carries -- for credentials assembly.
+     */
     stepOrder: readonly string[];
     /** Content hash of the deployment's workflow definition. */
     definitionHash: string;
@@ -172,6 +250,20 @@ export type DeliverSourcesOpts = {
     /** The default source id; the wire boundary requires it to equal `sources[0].id`. */
     defaultSource: string;
 };
+export type DeliverCredentialsOpts = {
+    /**
+     * The refreshed credential material and per-handle descriptors. The child
+     * MERGES this into its cell (materials upsert by credentialId, bindings by
+     * consumer-and-handle); it does not evict by omission.
+     */
+    delivery: CredentialDelivery;
+    /**
+     * CredentialIds to drop from the child's cell (a deletion or a deliberate
+     * revocation). The child removes each id's material and any binding that
+     * references it. A pure revocation pairs an empty `delivery` with these ids.
+     */
+    revoke?: string[];
+};
 export type RecycleOpts = {
     reason: string;
     /**
@@ -182,18 +274,6 @@ export type RecycleOpts = {
      */
     origin?: RecycleOrigin;
 };
-/**
- * Raised when a `pendingMerges` entry or a
- * `markConsumedCompletionWaiters` waiter is rejected because the
- * cohort it was registered against has been aborted (cohort transition
- * during a recycle, or a supervisor shutdown). Callers awaiting the
- * resolved value receive an instance of this error so the failure mode
- * is recognisable from a generic substrate-merge or markConsumed
- * failure.
- */
-export declare class MergeAbortedError extends Error {
-    constructor(reason: string);
-}
 /**
  * Construct a per-deployment supervisor. All host-specific
  * dependencies are pulled in via `bindings`; nothing in the