@intx/workflow-host 0.3.0 → 0.4.0

package/dist/run-body-then-cleanup.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Run `body`, then always run `cleanup`, without letting a `cleanup`
+ * failure mask a `body` failure.
+ *
+ * When `body` throws, `cleanup` still runs and a `cleanup` failure is
+ * handed to `onCleanupErrorAfterBodyError` (to log) and then dropped, so
+ * the original `body` error is what propagates. When `body` succeeds, a
+ * `cleanup` failure propagates -- there is no primary error to protect, so
+ * a failing teardown is the error worth surfacing.
+ *
+ * This exists so teardown in a `finally` (agent close, warm-cache
+ * eviction) can surface its own failure without a `throw` inside a
+ * `finally` block, which `no-unsafe-finally` forbids precisely because it
+ * silently swallows the in-flight exception -- the masking bug this guards
+ * against.
+ */
+export declare function runBodyThenCleanup<T>(body: () => Promise<T>, cleanup: () => Promise<void>, onCleanupErrorAfterBodyError: (cause: unknown) => void): Promise<T>;

package/dist/run-body-then-cleanup.js

@@ -0,0 +1,38 @@
+/**
+ * Run `body`, then always run `cleanup`, without letting a `cleanup`
+ * failure mask a `body` failure.
+ *
+ * When `body` throws, `cleanup` still runs and a `cleanup` failure is
+ * handed to `onCleanupErrorAfterBodyError` (to log) and then dropped, so
+ * the original `body` error is what propagates. When `body` succeeds, a
+ * `cleanup` failure propagates -- there is no primary error to protect, so
+ * a failing teardown is the error worth surfacing.
+ *
+ * This exists so teardown in a `finally` (agent close, warm-cache
+ * eviction) can surface its own failure without a `throw` inside a
+ * `finally` block, which `no-unsafe-finally` forbids precisely because it
+ * silently swallows the in-flight exception -- the masking bug this guards
+ * against.
+ */
+export async function runBodyThenCleanup(body, cleanup, onCleanupErrorAfterBodyError) {
+    let outcome;
+    try {
+        outcome = { ok: true, value: await body() };
+    }
+    catch (error) {
+        outcome = { ok: false, error };
+    }
+    try {
+        await cleanup();
+    }
+    catch (cleanupError) {
+        if (outcome.ok) {
+            throw cleanupError;
+        }
+        onCleanupErrorAfterBodyError(cleanupError);
+    }
+    if (!outcome.ok) {
+        throw outcome.error;
+    }
+    return outcome.value;
+}

package/dist/seams/scheduler.d.ts

@@ -52,6 +52,18 @@ export type SchedulerOpts = {
      * callbacks and to skip past-due cron entries on recovery.
      */
     clock: () => Date;
+    /**
+     * Arms a one-shot timer and returns its canceller. Defaults to the global
+     * timer, which is what production wants.
+     *
+     * The clock above decides WHEN a timer should fire; this decides what
+     * actually fires it. Without both, a caller can compute a deterministic
+     * delay and still have to wait out the real interval to observe the
+     * firing -- which left this module's own tests waiting past a `fireAt`,
+     * and, in one case, waiting longer than a cancelled timer's delay to argue
+     * from silence that the cancel had worked.
+     */
+    scheduleTimeout?: (handler: () => void, ms: number) => () => void;
 };
 export type SchedulerHandle = {
     /**

package/dist/seams/scheduler.js

@@ -63,6 +63,15 @@ export const TimerEventEnvelope = type({
     "+": "ignore",
 });
 export function createWorkflowHostScheduler(opts) {
+    // Production arms the global timer; a caller supplying its own can fire a
+    // queued timer on demand instead of waiting out its delay.
+    const schedule = opts.scheduleTimeout ??
+        ((handler, ms) => {
+            const handle = setTimeout(handler, ms);
+            return () => {
+                clearTimeout(handle);
+            };
+        });
     const queues = new Map();
     const liveSubscriptions = [];
     let started = false;
@@ -87,7 +96,7 @@ export function createWorkflowHostScheduler(opts) {
         if (queues.has(key))
             return; // idempotent
         const delayMs = Math.max(0, fireAtMs - opts.clock().getTime());
-        const timeout = setTimeout(() => {
+        const cancelTimeout = schedule(() => {
             void fireTimer(runId, timerId).catch((cause) => {
                 // The scheduler's commit failed. Surface as unhandled so
                 // operators see it; the runtime body's awaiter will hang
@@ -97,7 +106,7 @@ export function createWorkflowHostScheduler(opts) {
                     : new Error(`scheduler ${String(repoId.id)}/${runId}/${timerId} commit failed: ${String(cause)}`);
             });
         }, delayMs);
-        queues.set(key, { runId, timerId, fireAtMs, timeout, cron });
+        queues.set(key, { runId, timerId, fireAtMs, cancelTimeout, cron });
     }
     function startLiveSubscription(repoId) {
         const abort = new AbortController();
@@ -202,7 +211,7 @@ export function createWorkflowHostScheduler(opts) {
                 return;
             stopped = true;
             for (const t of queues.values())
-                clearTimeout(t.timeout);
+                t.cancelTimeout();
             queues.clear();
             for (const sub of liveSubscriptions.splice(0)) {
                 sub.abort.abort();
@@ -216,7 +225,7 @@ export function createWorkflowHostScheduler(opts) {
             const entry = queues.get(key);
             if (entry === undefined)
                 return;
-            clearTimeout(entry.timeout);
+            entry.cancelTimeout();
             queues.delete(key);
         },
         queuedTimers() {

package/dist/supervisor/cancel-signing.js

@@ -24,6 +24,7 @@
 // `supervisor`-kind principal), which is the cross-check the
 // supervisor's runtime-side signing keeps coherent.
 import { type } from "arktype";
+import { parseEventSeq } from "@intx/hub-sessions/substrate";
 import { hexEncode } from "@intx/types";
 /**
  * Path inside the workflow-run repo each `CancelRequested` event
@@ -40,7 +41,6 @@ const EVENTS_DIR = "events";
  * push principal and enforces the principal-vs-origin map.
  */
 export const SUPERVISOR_PRINCIPAL_KIND = "supervisor";
-const EVENT_FILENAME_RE = /^(0|[1-9][0-9]*)\.json$/;
 const OnDiskEnvelope = type({
     seq: "number >= 0",
     type: "string",
@@ -85,13 +85,9 @@ export async function commitCancelRequested(opts) {
             let maxSeq = -1;
             for (const filepath of existing.keys()) {
                 const name = filepath.slice(prefix.length);
-                const match = EVENT_FILENAME_RE.exec(name);
-                if (match === null)
+                const seq = parseEventSeq(name);
+                if (seq === null)
                     continue;
-                const seqStr = match[1];
-                if (seqStr === undefined)
-                    continue;
-                const seq = Number.parseInt(seqStr, 10);
                 if (seq > maxSeq)
                     maxSeq = seq;
             }

package/dist/supervisor/credentials.d.ts

@@ -14,7 +14,11 @@ export declare const STEP_GRANTS_PATH = "state/grants.json";
  */
 export declare const STEP_GRANTS_REF = "refs/heads/main";
 export type CredentialsSnapshotStep = {
-    /** Workflow step id from `WorkflowDefinition.stepOrder`. */
+    /**
+     * Workflow step id: a `WorkflowDefinition.stepOrder` entry, or a step id
+     * from one of the definition's `loop` bodies (a loop body shares the
+     * enclosing definition's flat step-id namespace).
+     */
     stepId: string;
     /** Mail address the step's agent presents to the bus. */
     address: string;
@@ -24,7 +28,7 @@ export type CredentialsSnapshotStep = {
     contentHash: string;
 };
 export type CredentialsSnapshot = {
-    /** Step-id keyed entries in `stepOrder` traversal order. */
+    /** Step-id keyed entries in the caller's traversal order. */
     steps: readonly CredentialsSnapshotStep[];
 };
 /**
@@ -55,9 +59,17 @@ export type AssembleCredentialsSnapshotOpts = {
     /** Principal presented for each step's read. */
     principal: Principal;
     /**
-     * Step ids in the deployment's `stepOrder`. The trivial workflow
-     * passes a single entry; multi-step deployments pass every step in
-     * the order the workflow asset declared.
+     * Every step id the snapshot must carry an entry for. The trivial
+     * workflow passes a single entry; multi-step deployments pass every
+     * step in the order the workflow asset declared.
+     *
+     * This is the deployment's flat step-id namespace, which is WIDER than
+     * the definition's own `stepOrder`: a `loop` body runs in-process as a
+     * child run inheriting the parent's env, so a body step authorizes
+     * against this same snapshot under its own plain step id. The caller
+     * owns that widening -- it is the layer that holds the definition --
+     * and the snapshot is total over whatever it passes, because the
+     * child's authorize treats a missing entry as unrecoverable.
      */
     stepOrder: readonly string[];
     /** Anchor run id used in agent-state repo identity and address derivation. */

package/dist/supervisor/recycle.d.ts

@@ -56,7 +56,11 @@ export interface ChildWiring {
 export interface RecycleContext {
     /** The supervisor's full bindings, reused on respawn for credentials and spawn. */
     readonly bindings: WorkflowSupervisorBindings;
-    /** Step ids in this deployment's `stepOrder` for credentials re-assembly. */
+    /**
+     * Every step id in this deployment's flat step-id namespace -- the
+     * definition's own `stepOrder` plus the step ids of every `loop` body it
+     * carries -- for credentials re-assembly.
+     */
     readonly stepOrder: readonly string[];
     /** Definition hash carried on respawn env (unchanged across recycle). */
     readonly definitionHash: string;

package/dist/supervisor/run-event-compaction.d.ts

@@ -20,8 +20,8 @@ export type CompactRunEventsOpts = {
  * Idempotent and terminal-only: a run already sealed (no `events/` subtree)
  * or one whose latest event is not terminal is left untouched, so the call
  * is safe to repeat. The live caller invokes it once per run, right after the
- * run terminates; a bounded recovery sweep that would retry the fold for a run
- * whose compaction a crash interrupted is not yet implemented.
+ * run terminates; `recoverInterruptedCompactions` re-runs it for a run whose
+ * fold a crash interrupted before it could seal.
  *
  * The combined file is the verbatim byte concatenation of the per-event
  * blobs in seq order (`encodeCombinedEventLog`), the exact shape the

package/dist/supervisor/run-event-compaction.js

@@ -7,16 +7,10 @@
 // without losing any event. The fold writes under the substrate's
 // per-repo lock as the `supervisor` principal, whose `anchorRunId`
 // the workflow-run kind handler checks against `repoId.id`.
-import { WORKFLOW_RUN_EVENTS_FILE, encodeCombinedEventLog, } from "@intx/hub-sessions/substrate";
+import { classifyTerminalEvent, parseEventSeq, WORKFLOW_RUN_EVENTS_FILE, encodeCombinedEventLog, } from "@intx/hub-sessions/substrate";
 import { SUPERVISOR_PRINCIPAL_KIND } from "./cancel-signing.js";
 const RUNS_PREFIX = "runs";
 const EVENTS_DIR = "events";
-const EVENT_FILENAME_RE = /^(0|[1-9][0-9]*)\.json$/;
-const TERMINAL_EVENT_TYPES = new Set([
-    "RunCompleted",
-    "RunFailed",
-    "RunCancelled",
-]);
 /**
  * Fold a terminated run's per-event `events/<seq>.json` blobs into one
  * combined `events.jsonl`, dropping the per-event files. This shrinks the
@@ -26,8 +20,8 @@ const TERMINAL_EVENT_TYPES = new Set([
  * Idempotent and terminal-only: a run already sealed (no `events/` subtree)
  * or one whose latest event is not terminal is left untouched, so the call
  * is safe to repeat. The live caller invokes it once per run, right after the
- * run terminates; a bounded recovery sweep that would retry the fold for a run
- * whose compaction a crash interrupted is not yet implemented.
+ * run terminates; `recoverInterruptedCompactions` re-runs it for a run whose
+ * fold a crash interrupted before it could seal.
  *
  * The combined file is the verbatim byte concatenation of the per-event
  * blobs in seq order (`encodeCombinedEventLog`), the exact shape the
@@ -55,10 +49,10 @@ export async function compactRunEvents(opts) {
     }
     const seqs = [];
     for (const name of filenames) {
-        const match = EVENT_FILENAME_RE.exec(name);
-        if (match === null || match[1] === undefined)
+        const seq = parseEventSeq(name);
+        if (seq === null)
             continue;
-        seqs.push(Number.parseInt(match[1], 10));
+        seqs.push(seq);
     }
     if (seqs.length === 0)
         return { compacted: false };
@@ -74,7 +68,8 @@ export async function compactRunEvents(opts) {
         return { compacted: false };
     }
     const lastType = parsed.type;
-    if (typeof lastType !== "string" || !TERMINAL_EVENT_TYPES.has(lastType)) {
+    if (typeof lastType !== "string" ||
+        !classifyTerminalEvent(lastType).terminal) {
         return { compacted: false };
     }
     const prefix = `${RUNS_PREFIX}/${opts.runId}/${EVENTS_DIR}/`;
@@ -90,11 +85,11 @@ export async function compactRunEvents(opts) {
             const entries = [];
             for (const [filepath, bytes] of existing) {
                 const name = filepath.slice(prefix.length);
-                const match = EVENT_FILENAME_RE.exec(name);
-                if (match === null || match[1] === undefined) {
+                const seq = parseEventSeq(name);
+                if (seq === null) {
                     throw new Error(`supervisor run-event-compaction: unexpected non-event file ${filepath} under run ${opts.runId}; refusing to compact`);
                 }
-                entries.push({ seq: Number.parseInt(match[1], 10), bytes });
+                entries.push({ seq, bytes });
             }
             if (entries.length === 0)
                 return {};

package/dist/supervisor/run-event-recovery.d.ts

@@ -0,0 +1,34 @@
+import type { RepoId, RepoStore as SubstrateRepoStore } from "@intx/hub-sessions/substrate";
+export type RecoverInterruptedCompactionsOpts = {
+    /** Substrate handle the supervisor writes through. */
+    substrate: SubstrateRepoStore;
+    /** Workflow-run repo for this deployment. */
+    repoId: RepoId;
+    /** Events ref the workflow-run repo writes to. */
+    ref: string;
+    /** Anchor run id used to construct the supervisor principal. */
+    anchorRunId: string;
+    /** Runs the boot scan proposes as terminal-but-per-event. */
+    pendingSealRunIds: readonly string[];
+};
+/** A run whose recovery fold threw, paired with the failure cause. */
+export type RecoveryFoldFailure = {
+    runId: string;
+    message: string;
+};
+/**
+ * Re-seal runs a crash left terminal but still in per-event form, by re-running
+ * the idempotent `compactRunEvents` for each proposed run. `compactRunEvents`
+ * is authoritative: it no-ops a run that is already sealed or whose latest
+ * event is not terminal, so a stale or mistaken proposal is a harmless no-op.
+ *
+ * Folds run serially. Every fold contends the same per-repo write lock that
+ * live dispatch also takes, so folding one run at a time drains the backlog
+ * without a thundering herd on that lock. One run's failure is caught so it
+ * cannot abort the rest; the failed run id and its cause are returned -- not
+ * logged here -- so the caller owns how to surface the aggregate.
+ */
+export declare function recoverInterruptedCompactions(opts: RecoverInterruptedCompactionsOpts): Promise<{
+    sealed: number;
+    failed: RecoveryFoldFailure[];
+}>;

package/dist/supervisor/run-event-recovery.js

@@ -0,0 +1,45 @@
+// Bounded recovery for run-event compaction folds a crash interrupted.
+//
+// When a run terminates, the supervisor fires `compactRunEvents` in the
+// background. A crash between the terminal commit and that fold leaves the
+// run terminal but still in per-event form, and the terminal signal never
+// fires again for it. At the next spawn the boot scan proposes those runs;
+// this sweep re-runs the idempotent fold for each so the leaked per-event
+// file count is reclaimed.
+import { compactRunEvents } from "./run-event-compaction.js";
+/**
+ * Re-seal runs a crash left terminal but still in per-event form, by re-running
+ * the idempotent `compactRunEvents` for each proposed run. `compactRunEvents`
+ * is authoritative: it no-ops a run that is already sealed or whose latest
+ * event is not terminal, so a stale or mistaken proposal is a harmless no-op.
+ *
+ * Folds run serially. Every fold contends the same per-repo write lock that
+ * live dispatch also takes, so folding one run at a time drains the backlog
+ * without a thundering herd on that lock. One run's failure is caught so it
+ * cannot abort the rest; the failed run id and its cause are returned -- not
+ * logged here -- so the caller owns how to surface the aggregate.
+ */
+export async function recoverInterruptedCompactions(opts) {
+    let sealed = 0;
+    const failed = [];
+    for (const runId of opts.pendingSealRunIds) {
+        try {
+            const { compacted } = await compactRunEvents({
+                substrate: opts.substrate,
+                repoId: opts.repoId,
+                ref: opts.ref,
+                anchorRunId: opts.anchorRunId,
+                runId,
+            });
+            if (compacted)
+                sealed += 1;
+        }
+        catch (cause) {
+            failed.push({
+                runId,
+                message: cause instanceof Error ? cause.message : String(cause),
+            });
+        }
+    }
+    return { sealed, failed };
+}

package/dist/supervisor/supervisor.d.ts

@@ -135,6 +135,19 @@ export interface WorkflowSupervisor {
      * credential is delivered by omitting its material so the child evicts it.
      */
     deliverCredentials(opts: DeliverCredentialsOpts): Promise<void>;
+    /**
+     * Refresh a live run's grant floor mid-run by re-reading its durable
+     * `runs/<runId>/grants.json` and pushing it as a `grants-updated` frame. The
+     * enforcement path for a standing (`scope: "always"`) approval that lowers a
+     * tool's `ask` to `allow` in that file. Unlike `deliverSignal`/
+     * `deliverSources`, a refresh for a non-live child is normal, so this
+     * NO-OPS (`skipped`) instead of throwing, and a send failure to a live child
+     * is logged loudly but stays non-fatal -- the durable file governs the next
+     * barrier/respawn. It pushes only that file's contents, never caller-supplied
+     * grants, so it can only tighten or refresh a floor. Returns whether a live
+     * push happened.
+     */
+    deliverGrants(runId: string): Promise<"pushed" | "skipped">;
     /**
      * Re-register every correlation the child is currently parked on by
      * querying it for its parked correlations and re-emitting each through
@@ -158,7 +171,11 @@ export interface WorkflowSupervisor {
     getCredentialsSnapshot(): CredentialsSnapshot | null;
 }
 export type SpawnOpts = {
-    /** Step ids in this deployment's `stepOrder` for credentials assembly. */
+    /**
+     * Every step id in this deployment's flat step-id namespace -- the
+     * definition's own `stepOrder` plus the step ids of every `loop` body it
+     * carries -- for credentials assembly.
+     */
     stepOrder: readonly string[];
     /** Content hash of the deployment's workflow definition. */
     definitionHash: string;
@@ -235,11 +252,17 @@ export type DeliverSourcesOpts = {
 };
 export type DeliverCredentialsOpts = {
     /**
-     * The refreshed credential material and per-handle descriptors. A revoked
-     * credential is delivered by omitting its material entry, so the child's
-     * wholesale swap evicts it.
+     * The refreshed credential material and per-handle descriptors. The child
+     * MERGES this into its cell (materials upsert by credentialId, bindings by
+     * consumer-and-handle); it does not evict by omission.
      */
     delivery: CredentialDelivery;
+    /**
+     * CredentialIds to drop from the child's cell (a deletion or a deliberate
+     * revocation). The child removes each id's material and any binding that
+     * references it. A pure revocation pairs an empty `delivery` with these ids.
+     */
+    revoke?: string[];
 };
 export type RecycleOpts = {
     reason: string;