@bevel-software/ael 0.0.0-stage → 0.2.0

package/src/interpreter/interpret.workflow.ts

@@ -0,0 +1,907 @@
+import {
+  ActivityCancellationType,
+  ActivityFailure,
+  ApplicationFailure,
+  CancellationScope,
+  CancelledFailure,
+  ContinueAsNew,
+  condition,
+  continueAsNew,
+  defineSignal,
+  TimeoutFailure,
+  getExternalWorkflowHandle,
+  isCancellation,
+  log,
+  proxyActivities,
+  setHandler,
+  uuid4,
+  workflowInfo,
+} from '@temporalio/workflow';
+import type { Verdict } from '../run/contract.js';
+import { ATTEMPTS_KEY, ENTRIES_KEY, RETRIES_KEY, decide, errorForVerdict, isRetryKeyOf } from '../workflow/failure-policy.js';
+import { evaluate, interpolate, nextAfter, stepByName, stripReservedKeys, type ErrorName, type StepDef, type WorkflowDefinition } from '../workflow/definition.js';
+import {
+  EXECUTOR_TASK_QUEUE,
+  INTERPRETER_STATE_KEYS,
+  MAX_RUN_STEPS,
+  MAX_STEP_RESTARTS,
+  PLATFORM_RESTART,
+  PLATFORM_TASK_QUEUE,
+  RUN_AT_GATE_SIGNAL,
+  RUN_NEEDS_SLOT_SIGNAL,
+  RUN_SETTLED_SIGNAL,
+  SLOT_GRANTED_SIGNAL,
+  CONTAINER_STEP_HEARTBEAT_MS,
+  namedExecutorQueue,
+  type Actor,
+  type AgentStepInput,
+  type AgentStepOutcome,
+  type ExecutorActivities,
+  type InterpretInput,
+  type InterpretResult,
+  type InterpreterActivities,
+  type RunStatus,
+  type ToolChainOutcome,
+} from './contract.js';
+
+/**
+ * The interpreter: one Temporal workflow per run of a workflow. It encodes no
+ * step behaviour. Everything a step does comes from the `.workflow` file,
+ * re-read through an activity before every step (edits take effect at the
+ * next step; a change to the step structure blocks the run). The failure
+ * policy is the file's; this code only applies it. Temporal gives the loop
+ * what the old runner's worker and state machine were for: durable timers for
+ * gates and backoff, signals for gate outcomes, recovery when a
+ * worker or executor dies, and a history of every decision.
+ *
+ * Ported from the execution layer's state machine and step handlers; the
+ * decisions and their reasons are theirs.
+ */
+
+export const gateOutcomeSignal = defineSignal<[string]>('gateOutcome');
+export const cancelRunSignal = defineSignal<[string | undefined]>('cancelRun');
+
+const PROBE_ERROR_CAP = 3;
+const DEFAULT_GATE_POLL_SECONDS = 60;
+const DEFAULT_GATE_TIMEOUT_SECONDS = 30 * 24 * 3600;
+const ACCOUNT_LIMIT_DEFAULT_PAUSE_MS = 15 * 60_000;
+/** How long a run waits for a slot before its thread says that is what it waits for. */
+const SLOT_NOTICE_MS = 30_000;
+/**
+ * How long a run waits for a slot before it asks its dispatcher again. The
+ * grant is the fast path; asking again is the net under it, for a grant or
+ * an ask that was lost, and for a dispatcher that has gone meanwhile (the
+ * ask then fails, and the run goes on: no cap is left to wait under).
+ */
+const SLOT_ASK_AGAIN_MS = 10 * 60_000;
+
+/** The platform's activities: short, retried on infrastructure errors. */
+const platform = proxyActivities<Pick<InterpreterActivities, 'loadWorkflow' | 'resolveRunParameters' | 'openStepThread' | 'recordTransition' | 'notifyRunFailed' | 'prepareContainerStep'>>({
+  taskQueue: PLATFORM_TASK_QUEUE,
+  startToCloseTimeout: '10 minutes',
+  retry: { maximumAttempts: 5, initialInterval: '2 seconds', backoffCoefficient: 2 },
+});
+
+/**
+ * The end of a step: its thread closed and its keys revoked. A key left
+ * live is the one thing an outage must not leave behind, so these are
+ * retried for as long as a key store can plausibly be away — a day, with
+ * the interval capped — rather than the handful of tries a read gets.
+ */
+const stepEnd = proxyActivities<Pick<InterpreterActivities, 'closeStepThread' | 'finishAgentStep' | 'finishContainerStep'>>({
+  taskQueue: PLATFORM_TASK_QUEUE,
+  startToCloseTimeout: '10 minutes',
+  retry: { maximumAttempts: 300, initialInterval: '2 seconds', backoffCoefficient: 2, maximumInterval: '5 minutes' },
+});
+
+/** A tool chain or a gate's probe: a step like any other, so a Cancel reaches it — the workflow moves on at once and the activity is told. */
+const chainStep = proxyActivities<Pick<InterpreterActivities, 'runToolChain' | 'runProbe'>>({
+  taskQueue: PLATFORM_TASK_QUEUE,
+  startToCloseTimeout: '10 minutes',
+  retry: { maximumAttempts: 5, initialInterval: '2 seconds', backoffCoefficient: 2 },
+  cancellationType: ActivityCancellationType.TRY_CANCEL,
+});
+
+/** An in-process agent step on the platform's own worker. Retries are the file's policy, not Temporal's. */
+const inProcessStep = proxyActivities<Pick<InterpreterActivities, 'runAgentStep'>>({
+  taskQueue: PLATFORM_TASK_QUEUE,
+  startToCloseTimeout: '6 hours',
+  heartbeatTimeout: '2 minutes',
+  retry: { maximumAttempts: 1 },
+});
+
+/**
+ * A container step on an executor queue: the shared one, the executor this
+ * run is pinned to, or the actor's own executor (`waitForIt`).
+ */
+function containerStep(queue: string, waitForIt = false) {
+  return proxyActivities<ExecutorActivities>({
+    taskQueue: queue,
+    startToCloseTimeout: '12 hours',
+    // A pinned executor that is gone never picks the task up; the schedule-
+    // to-start timeout is how the run notices and falls back to the shared
+    // queue with a fresh workspace. A person's own executor is waited for
+    // instead, however long it is away: nothing else reaches what it reaches.
+    scheduleToStartTimeout: queue === EXECUTOR_TASK_QUEUE || waitForIt ? undefined : '5 minutes',
+    // Above the platform's limit for a silent personal executor, so its reason is the one recorded.
+    heartbeatTimeout: CONTAINER_STEP_HEARTBEAT_MS,
+    retry: { maximumAttempts: 1 },
+  });
+}
+
+/**
+ * The end of a run on the executor it was pinned to: its workspace volume
+ * goes. Short and never retried: an executor that is gone misses it, and
+ * its own sweep removes the volume once it is back.
+ */
+function pinnedRelease(queue: string) {
+  return proxyActivities<Pick<ExecutorActivities, 'releaseRun'>>({
+    taskQueue: queue,
+    scheduleToStartTimeout: '1 minute',
+    startToCloseTimeout: '2 minutes',
+    retry: { maximumAttempts: 1 },
+  });
+}
+
+type Counters = Record<string, number>;
+
+function counters(state: Record<string, unknown>, key: string): Counters {
+  const raw = state[key];
+  return raw && typeof raw === 'object' && !Array.isArray(raw) ? { ...(raw as Counters) } : {};
+}
+
+function errMessage(err: unknown): string {
+  const text = err instanceof Error ? err.message : String(err);
+  return text.length > 300 ? `${text.slice(0, 300)}…` : text;
+}
+
+/** A wait, in the unit that reads naturally: `7 days`, `3 h`, `45 min`, `5 s`. */
+function durationText(ms: number): string {
+  if (ms >= 86_400_000) return `${Math.round(ms / 86_400_000)} days`;
+  if (ms >= 3_600_000) return `${Math.round(ms / 3_600_000)} h`;
+  if (ms >= 60_000) return `${Math.round(ms / 60_000)} min`;
+  return `${Math.round(ms / 1000)} s`;
+}
+
+/**
+ * What an activity actually failed with. Temporal hands the workflow an
+ * `ActivityFailure` whose `cause` is the failure the activity raised (an
+ * `ApplicationFailure`, a timeout); a check on the wrapper alone sees none
+ * of it.
+ */
+function activityCause(err: unknown): unknown {
+  return err instanceof ActivityFailure ? (err.cause ?? err) : err;
+}
+
+/**
+ * Whether a step failed because the platform stopped under it rather than on its own: the worker said so as it
+ * stopped (`PLATFORM_RESTART`), or it went without a word and missed its heartbeat.
+ */
+function cutOffByRestart(err: unknown): boolean {
+  const cause = activityCause(err);
+  if (cause instanceof ApplicationFailure) return cause.type === PLATFORM_RESTART;
+  return cause instanceof TimeoutFailure && cause.timeoutType === 'HEARTBEAT';
+}
+
+/** The failure's own words, through the wrapper; a cancellation says so, whatever its message. */
+function failureReason(err: unknown): string {
+  if (isCancellation(err) || err instanceof CancelledFailure) return 'cancelled';
+  const cause = activityCause(err);
+  const text = (cause instanceof Error ? cause.message || cause.name : String(cause)) || (err instanceof Error ? err.message : String(err));
+  return errMessage(err instanceof ActivityFailure && err.message ? `${text} (${err.message})` : text);
+}
+
+export async function interpret(input: InterpretInput): Promise<InterpretResult> {
+  const actor: Actor = input.actor;
+  const state: Record<string, unknown> = { ...(input.initialState ?? {}) };
+  const threadIds: Record<string, string> = { ...(counters(state, 'sessionIds') as unknown as Record<string, string>) };
+  let pendingGate: string | null = null;
+  let cancelled: string | null = null;
+  let current: string | null = null;
+  let executorQueue: string | null = input.executorQueue ?? null;
+  let finalReason: string | null = null;
+  // The deadline of the gate a continued-as-new execution resumes in, and its probe's errors in a row so far; each used once.
+  let carriedGateDeadline: number | null = input.gateDeadline ?? null;
+  let carriedProbeErrors: number | null = input.probeErrors ?? null;
+  /** A step paused on an account limit: the thread it goes on on when the limit lifts. */
+  let pausedThread: { step: string; threadId: string } | null = input.pausedThread ?? null;
+  /** Steps taken by this RUN, every execution counted: the cap a cycling file hits, however many times the history was cut. */
+  let stepsTaken = input.stepsTaken ?? 0;
+
+  /**
+   * The run goes on as a new execution from its recorded step, the state
+   * carried, when Temporal says the history is getting long — a run of many
+   * steps, or a gate polled for weeks, would otherwise be terminated with its
+   * history at the limit and nothing recorded. Only between steps or polls,
+   * with no signal pending: a signal waiting to be read would be lost.
+   */
+  let stepsThisExecution = 0;
+  const continueIfSuggested = async (gate: { deadline: number; probeErrors: number } | null): Promise<void> => {
+    const suggested = workflowInfo().continueAsNewSuggested || (input.continueAfterSteps !== undefined && stepsThisExecution >= input.continueAfterSteps);
+    if (!suggested || pendingGate !== null || cancelled !== null) return;
+    state.sessionIds = { ...threadIds };
+    // The position the next execution resumes at: the step about to run (or the gate being polled), not the one just finished.
+    if (current) state.step = current;
+    await continueAsNew<typeof interpret>({ ...input, initialState: state, workflowVersion: version, executorQueue: executorQueue ?? undefined, gateDeadline: gate?.deadline, probeErrors: gate?.probeErrors, pausedThread: pausedThread ?? undefined, stepsTaken, slotGivenBack: !holdsSlot });
+  };
+
+  // A capped run holds a slot under its trigger's cap while it has work to
+  // do and none while it waits at a gate: a gate can wait days for a person,
+  // and a slot held that long is a trigger that starts nothing. The key (the
+  // work item) stays the run's throughout.
+  let holdsSlot = !input.slotGivenBack;
+  let slotGranted = false;
+  setHandler(defineSignal(SLOT_GRANTED_SIGNAL), () => {
+    slotGranted = true;
+  });
+  /** Whether the dispatcher was told; false when it is gone, or there is none. */
+  const tellDispatcher = async (signal: string): Promise<boolean> => {
+    const parent = workflowInfo().parent;
+    if (!parent) return false;
+    try {
+      await getExternalWorkflowHandle(parent.workflowId).signal(signal, input.runId);
+      return true;
+    } catch (err) {
+      log.warn(`run ${input.runId}: its dispatcher ${parent.workflowId} did not take "${signal}": ${errMessage(err)}`);
+      return false;
+    }
+  };
+  const giveSlotBack = async (): Promise<void> => {
+    if (!input.capped || !holdsSlot) return;
+    holdsSlot = false;
+    await tellDispatcher(RUN_AT_GATE_SIGNAL);
+  };
+  /** Returns once the run holds a slot, its dispatcher is gone, or the run was cancelled (the caller looks). */
+  const takeSlot = async (stepName: string): Promise<void> => {
+    if (!input.capped || holdsSlot) return;
+    let said = false;
+    while (cancelled === null) {
+      slotGranted = false;
+      if (!(await tellDispatcher(RUN_NEEDS_SLOT_SIGNAL))) break;
+      if (await condition(() => slotGranted || cancelled !== null, said ? SLOT_ASK_AGAIN_MS : SLOT_NOTICE_MS)) break;
+      if (!said) {
+        said = true;
+        await record(stepName, `⏳ ${stepName} waits for a free slot: its trigger has as many runs at work as it allows`, true);
+      }
+    }
+    holdsSlot = true;
+  };
+
+  setHandler(gateOutcomeSignal, (outcome) => {
+    pendingGate = outcome;
+  });
+  // The step in flight, if one: a cancel reaches it through its scope, so the
+  // activity is cancelled where it is (the session aborted, the container
+  // stopped) instead of running on to its own end.
+  let stepScope: CancellationScope | null = null;
+  setHandler(cancelRunSignal, (reason) => {
+    cancelled = reason ?? 'cancelled';
+    stepScope?.cancel();
+  });
+
+  const gatePollMs = (input.gatePollSeconds ?? DEFAULT_GATE_POLL_SECONDS) * 1000;
+  const gateTimeoutMs = (input.gateTimeoutSeconds ?? DEFAULT_GATE_TIMEOUT_SECONDS) * 1000;
+
+  let def: WorkflowDefinition;
+  // The happy path as the thread's record needs it: empty until the file is read.
+  let happyPath: string[] = [];
+  try {
+    def = await platform.loadWorkflow({ workflowPath: input.workflowPath, actor });
+  } catch (err) {
+    // The thread still learns how the run ended: a run that never read its file is blocked, not running for ever.
+    return finish('blocked', `the workflow could not be read: ${failureReason(err)}`);
+  }
+  // The structure a continued execution started under is the one it keeps checking against: an edit between executions blocks like one between steps.
+  const version = input.workflowVersion ?? def.version;
+  if (def.version !== version) return finish('blocked', `the workflow's steps changed while this run was in flight (${version} → ${def.version}); release the run to start it over`);
+  happyPath = def.happyPath;
+  state.workflowId = def.id;
+  state.workflowVersion = version;
+  const reserved = new Set<string>([...INTERPRETER_STATE_KEYS, ...def.reservedStateKeys]);
+
+  let params: Record<string, string>;
+  let missing: string[];
+  try {
+    ({ values: params, missing } = await platform.resolveRunParameters({ definition: def, nodePath: input.subjectPath ?? input.workItemPath, actor }));
+  } catch (err) {
+    if (isCancellation(err) || err instanceof CancelledFailure) return finish('cancelled', cancelled ?? 'cancelled');
+    return finish('blocked', `the run's parameters could not be resolved: ${failureReason(err)}`);
+  }
+  if (missing.length > 0) {
+    return finish('blocked', `missing required parameter${missing.length > 1 ? 's' : ''}: ${missing.join(', ')}`);
+  }
+
+  // A released or resumed run continues from its recorded step when that
+  // step still exists; anything else starts at the top — through the same
+  // door every step is entered by, so the first entry counts against a budget.
+  // `step` is where a continued execution of this run was: its counters came
+  // with it, so it goes straight on. `resumeStep` is where an earlier run of
+  // this work item stopped (see `finish`): this run's counters are new, so it
+  // enters that step through the door, like the top.
+  const released = typeof state.resumeStep === 'string' && stepByName(def, state.resumeStep) ? state.resumeStep : null;
+  delete state.resumeStep;
+  if (!released && typeof state.step === 'string' && stepByName(def, state.step)) {
+    current = state.step;
+  } else {
+    const entered = await enter('start', released ?? def.happyPath[0]);
+    if (entered !== 'moved') return { status: entered, state, ...(finalReason ? { reason: finalReason } : {}) };
+  }
+
+  /** `waiting`: the run is parked in the step until something outside it answers (a gate, a limit); the thread says so. */
+  async function record(status: string, logLine: string | null, waiting = false): Promise<void> {
+    state.sessionIds = { ...threadIds };
+    await platform.recordTransition({
+      runId: input.runId,
+      workflowPath: input.workflowPath,
+      workItemPath: input.workItemPath,
+      workItemFields: input.workItemFields,
+      actor,
+      status,
+      stepName: current,
+      waiting,
+      log: logLine,
+      state: { ...state },
+      threadIds: { ...threadIds },
+      happyPath,
+    });
+  }
+
+  /**
+   * The end of the run, whatever ended it — a run whose file was never read
+   * included. Its activities run in a scope cancellation cannot reach: a
+   * cancelled run's record, its notice and its workspace release are
+   * exactly what a cancel must still write.
+   */
+  async function finish(status: RunStatus, reason: string | null): Promise<InterpretResult> {
+    finalReason = reason;
+    if (reason) state.lastError = reason;
+    if (status === 'blocked' && reason) state.blocker = reason;
+    // Where a run that did not finish stopped: a work item released back to its trigger's status goes on from there.
+    if (status !== 'done' && current) state.resumeStep = current;
+    state.step = status;
+    state.assignee = null;
+    await CancellationScope.nonCancellable(async () => {
+      // A step still parked on an account limit when the run ends: its thread
+      // is closed here, whatever ended the run, before the end is recorded.
+      if (pausedThread) {
+        const parked = pausedThread;
+        pausedThread = null;
+        await stepEnd.closeStepThread({ threadId: parked.threadId, reason: `the run ended ${status}${reason ? `: ${reason}` : ''}` });
+      }
+      await record(status, reason ? `${status}: ${reason}` : status);
+      if (status === 'failed' || status === 'blocked') {
+        await platform.notifyRunFailed({ runId: input.runId, workflowPath: input.workflowPath, actor, status, reason: reason ?? status });
+      }
+      if (executorQueue) {
+        try {
+          await pinnedRelease(executorQueue).releaseRun({ runId: input.runId });
+        } catch (err) {
+          log.warn(`executor ${executorQueue} did not release the workspace of run ${input.runId}: ${errMessage(err)}`);
+        }
+      }
+      // The dispatcher that started this run, if one did, learns it is over
+      // here — the way that survives the dispatcher continuing as new.
+      const parent = workflowInfo().parent;
+      if (parent) {
+        try {
+          await getExternalWorkflowHandle(parent.workflowId).signal(RUN_SETTLED_SIGNAL, input.runId);
+        } catch (err) {
+          log.warn(`run ${input.runId}: its dispatcher ${parent.workflowId} could not be told it ended: ${errMessage(err)}`);
+        }
+      }
+    });
+    return { status, state, ...(reason ? { reason } : {}) };
+  }
+
+  /**
+   * Route into `target`, charging its entry budget. Returns the terminal
+   * status when the run settled instead. `exhausted` names the steps whose
+   * spent budget redirected this routing so far: budgets that point at each
+   * other (A's `then` is B, B's is A) would otherwise chase each other for
+   * ever without a step running — a loop the step cap never sees.
+   */
+  async function enter(fromStep: string, target: string, exhausted: Set<string> = new Set()): Promise<RunStatus | 'moved'> {
+    const step = stepByName(def, target);
+    if (!step?.entries) {
+      current = target;
+      return 'moved';
+    }
+    const entries = counters(state, ENTRIES_KEY);
+    const count = (entries[target] ?? 0) + 1;
+    state[ENTRIES_KEY] = { ...entries, [target]: count };
+    if (count <= step.entries.max) {
+      current = target;
+      return 'moved';
+    }
+    const reason = `"${target}" entered ${count} times, budget ${step.entries.max} — needs a human`;
+    const then = step.entries.then;
+    if (then === 'blocked') return (await finish('blocked', `${fromStep}: ${reason}`)).status;
+    if (then === 'failed') return (await finish('failed', `${fromStep}: ${reason}`)).status;
+    if (then === 'done') return (await finish('done', null)).status;
+    if (exhausted.has(target)) {
+      return (await finish('blocked', `${fromStep}: the entry budgets of ${[...exhausted, target].join(' → ')} route in a circle — needs a human`)).status;
+    }
+    await record(fromStep, `⛔ ${fromStep}: ${reason}. Routing to ${then}.`);
+    // The budget's own target is entered like any other, and its budget, if it has one, counts this too. `continue` is the step after
+    // the BUDGETED step — the file says "when this step is spent, go on past it" — not the step after whichever step was routing into it.
+    const spent = new Set([...exhausted, target]);
+    return then === 'continue' ? continueFrom(target, spent) : enter(target, then, spent);
+  }
+
+  /**
+   * The step after `fromStep` on the happy path, entered; `done` when there is
+   * none. An off-path step has no place on it, and the file may not say
+   * `continue` in its own routing (the parser refuses); a `defaults` catcher
+   * that says it applies to every step, so from an off-path one it is a
+   * routing the file left undefined — the run is blocked by name, not failed
+   * by an exception.
+   */
+  async function continueFrom(fromStep: string, exhausted?: Set<string>): Promise<RunStatus | 'moved'> {
+    if (!def.happyPath.includes(fromStep)) {
+      return (await finish('blocked', `"${fromStep}" is off the happy path, and its catcher says \`continue\`, which names no step there — name the step`)).status;
+    }
+    const next = nextAfter(def, fromStep);
+    if (next === null) return (await finish('done', null)).status;
+    return enter(fromStep, next, exhausted);
+  }
+
+  /** `continue`, a terminal target or a step name, as the file spells them. Routing to `failed` or `blocked` goes through the policy. */
+  async function route(fromStep: string, target: string, exhausted?: Set<string>): Promise<RunStatus | 'moved' | 'retry'> {
+    if (target === 'continue') return continueFrom(fromStep, exhausted);
+    if (target === 'done') return (await finish('done', null)).status;
+    if (target === 'failed') return fail(fromStep, 'Fail', 'routed to failed');
+    if (target === 'blocked') return fail(fromStep, 'Blocked', 'routed to blocked');
+    return enter(fromStep, target, exhausted);
+  }
+
+  /** Apply the file's failure policy to a named error at `stepName`. */
+  async function fail(stepName: string, error: ErrorName, reason: string): Promise<RunStatus | 'moved' | 'retry'> {
+    const step = stepByName(def, stepName)!;
+    const attempts = counters(state, ATTEMPTS_KEY);
+    const retries = counters(state, RETRIES_KEY);
+    const decision = decide(def, { step, error, attempts, retries });
+
+    if (decision.action === 'unhandled') return (await finish('blocked', `${stepName}: unhandled ${error}: ${reason}`)).status;
+
+    if (decision.action === 'retry') {
+      state[RETRIES_KEY] = { ...retries, [decision.counterKey]: decision.attempt };
+      state.lastError = reason;
+      await record(stepName, `🔁 ${stepName} hit ${error} (retry ${decision.attempt}/${decision.max}): ${reason}`);
+      // A wait a Cancel wakes: the loop's next turn sees the cancellation.
+      if (decision.delayMs > 0) await condition(() => cancelled !== null, decision.delayMs);
+      return 'retry';
+    }
+
+    if (decision.counterKey) state[ATTEMPTS_KEY] = { ...attempts, [decision.counterKey]: decision.count };
+    state.lastError = reason;
+    state[RETRIES_KEY] = {};
+    const target = decision.target;
+    if (target === 'blocked') return (await finish('blocked', `${stepName}: ${reason}`)).status;
+    if (target === 'failed') {
+      const exhausted = decision.max !== undefined && decision.count !== undefined && decision.count > decision.max;
+      return (await finish('failed', `${stepName} ${error}: ${reason}${exhausted ? ` (sent back ${decision.max} times already)` : ''}`)).status;
+    }
+    if (target === 'done') return (await finish('done', null)).status;
+    await record(stepName, `❌ ${stepName} ${error}: ${reason}. Routing to ${target}${decision.max !== undefined ? ` (${decision.count}/${decision.max})` : ''}.`);
+    // A catcher's target is routed as the file spells it: a step name, or `continue` for the step after the one that failed.
+    return route(stepName, target);
+  }
+
+  function clearRetries(stepName: string): void {
+    const retries = counters(state, RETRIES_KEY);
+    const rest = Object.fromEntries(Object.entries(retries).filter(([key]) => !isRetryKeyOf(stepName, key)));
+    state[RETRIES_KEY] = rest;
+    delete state.lastError;
+  }
+
+  function thenFor(step: StepDef, token: string): string | null {
+    if (step.then === undefined) return token === 'PASS' ? 'continue' : null;
+    if (typeof step.then === 'string') return token === 'PASS' ? step.then : null;
+    return step.then[token] ?? null;
+  }
+
+  function mergeOutputs(stepName: string, outputs: Record<string, unknown> | null): void {
+    if (!outputs) return;
+    const kept = stripReservedKeys(outputs, [...reserved], (key) => log.warn(`step "${stepName}" tried to write reserved STATE key "${key}" — dropped`));
+    Object.assign(state, kept);
+  }
+
+  const vars = () => ({ ...params, ...state });
+
+  try {
+    while (current) {
+      if (cancelled) return finish('cancelled', cancelled);
+      // The cap is the run's, not the execution's: a file whose routing loops is stopped at the same count whether or not the history was cut meanwhile.
+      if (stepsTaken >= MAX_RUN_STEPS) return finish('blocked', `the run exceeded ${MAX_RUN_STEPS} steps`);
+      // The slot follows the step about to run: given back at a gate, held
+      // for a step that does work. A choice does none and changes nothing.
+      const kind = stepByName(def, current)?.kind;
+      if (kind === 'wait') await giveSlotBack();
+      else if (kind === 'agent' || kind === 'tool_chain') {
+        await takeSlot(current);
+        if (cancelled) return finish('cancelled', cancelled);
+      }
+      await continueIfSuggested(null);
+      stepsTaken += 1;
+      stepsThisExecution += 1;
+
+      // The file is re-read before every step: prose edits apply at the next
+      // step, a structural change blocks rather than restarting silently.
+      let fresh: WorkflowDefinition;
+      try {
+        fresh = await platform.loadWorkflow({ workflowPath: input.workflowPath, actor });
+      } catch (err) {
+        if (isCancellation(err) || err instanceof CancelledFailure) throw err;
+        // The file went away or became unreadable mid-run: the thread says so, like the first read.
+        return finish('blocked', `the workflow could not be read: ${failureReason(err)}`);
+      }
+      if (fresh.version !== version) return finish('blocked', `the workflow's steps changed while this run was in flight (${version} → ${fresh.version}); release the run to start it over`);
+      def = fresh;
+      happyPath = fresh.happyPath;
+      const step = stepByName(def, current);
+      if (!step) return finish('blocked', `step "${current}" is not in the workflow`);
+
+      state.step = current;
+      // In-place retries of this step so far, under each retrier's own counter (`retryKey`): the attempt number the thread and the transcript links carry.
+      const stepName = current;
+      const retried = Object.entries(counters(state, RETRIES_KEY))
+        .filter(([key]) => isRetryKeyOf(stepName, key))
+        .reduce((n, [, count]) => n + count, 0);
+      const attempt = retried + 1;
+      state.attempt = attempt;
+      // A retry goes on from the attempt before it: its session state is saved after every turn, so the step resumes where it stopped.
+      // A step paused on an account limit goes on on the thread it paused on, the same way: a new thread beside it would leave it `waiting` for ever.
+      // The memo stays until that thread's step has ended: a transition write that fails before it ends the run through `finish`, which closes a thread still remembered.
+      const paused = pausedThread?.step === stepName ? pausedThread.threadId : null;
+      const resumeThreadId = paused ?? (attempt > 1 ? (threadIds[`${stepName}#${attempt - 1}`] ?? null) : null);
+
+      let moved: RunStatus | 'moved' | 'retry';
+      switch (step.kind) {
+        case 'agent': {
+          await record(current, `→ ${current}`);
+          // The step's thread exists, and the run knows it, before the step runs: an attempt that dies mid-way leaves no thread nobody can find.
+          // Its id is chosen here (replay-safe) and remembered BEFORE the activity is awaited, so the activity's retries open one thread, not one per
+          // attempt — and an open that fails for good, or is cancelled once its insert landed, still names the thread this run has to close.
+          const threadId = resumeThreadId ?? uuid4();
+          threadIds[`${stepName}#${attempt}`] = threadId;
+          if (!resumeThreadId) {
+            try {
+              await platform.openStepThread({ runId: input.runId, workflowPath: input.workflowPath, actor, stepName, attempt, workItemPath: input.workItemPath, threadId });
+            } catch (err) {
+              // The row may be there (its completion lost) or not (`closeStepThread` leaves a thread it cannot find alone): closed from outside
+              // the scope a cancel just ended, and the failure then goes on as it would have — a cancel to the run's end, anything else as the run's failure.
+              const cancelledOpen = isCancellation(err) || err instanceof CancelledFailure;
+              await CancellationScope.nonCancellable(() =>
+                stepEnd.closeStepThread({ threadId, reason: cancelledOpen ? 'the run was cancelled' : `the step's thread could not be opened: ${failureReason(err)}` }),
+              );
+              throw err;
+            }
+          }
+          const outcome = await runAgent(step, threadId, resumeThreadId);
+          // The step's end, for good whatever the outcome: its keys revoked
+          // (the activity's own revocation as it ended was its first try)
+          // and its thread's status written from the outcome, by the one
+          // activity Temporal retries until the stores answer.
+          await CancellationScope.nonCancellable(() => stepEnd.finishAgentStep({ threadId, outcome }));
+          // The thread this step ran on has its status now, paused or not: a memo of an earlier pause on it is spent (a new pause sets its own below).
+          if (pausedThread?.threadId === threadId) pausedThread = null;
+          moved = await settleAgent(step, outcome, threadId);
+          break;
+        }
+        case 'tool_chain': {
+          await record(current, `→ ${current}`);
+          const result = await inStepScope(() =>
+            chainStep.runToolChain({
+              runId: input.runId,
+              workflowPath: input.workflowPath,
+              stepName: step.name,
+              actor,
+              // As written, like a probe: the chain reads `state` and `params` as objects, not through `${…}`.
+              chain: step.instructions,
+              state: { ...state },
+              params,
+              workItemPath: input.workItemPath,
+            }),
+          );
+          moved = await settleChain(step, result);
+          break;
+        }
+        case 'wait': {
+          await record(current, `⏸ ${current}`, true);
+          moved = await gate(step);
+          break;
+        }
+        case 'choice': {
+          const chosen = step.choices?.find((rule) => evaluate(rule.when, vars()))?.then ?? step.default;
+          if (!chosen) return finish('blocked', `choice "${step.name}" matched nothing and has no default`);
+          moved = await route(step.name, chosen);
+          break;
+        }
+      }
+      if (moved === 'moved' || moved === 'retry') continue;
+      return { status: moved, state, ...(finalReason ? { reason: finalReason } : {}) };
+    }
+    // Every step enters the next by name, so the loop ends only through a terminal status above; this is the honest answer should it not.
+    return finish('blocked', 'the run has no step to go on with');
+  } catch (err) {
+    if (isCancellation(err) || err instanceof CancelledFailure) return finish('cancelled', cancelled ?? 'cancelled');
+    // Continue-as-new is how the run goes on, not a failure: Temporal's control exception passes through.
+    if (err instanceof ContinueAsNew) throw err;
+    // Anything else is the run's failure, and the thread learns it: a thread
+    // left `running` by a workflow that died would be polled for ever.
+    log.error(`run ${input.runId} failed: ${failureReason(err)}`);
+    return finish('failed', `the run failed: ${failureReason(err)}`);
+  }
+
+  /**
+   * One agent step. An activity that fails as an activity — a worker that
+   * died mid-step, a heartbeat that stopped, no executor polling the queue,
+   * a refusal the activity raised — is a harness error of the step, which
+   * the file's failure policy then handles; it never fails the run as a
+   * whole. Cancellation alone passes through.
+   */
+  /**
+   * The scope a Cancel reaches: every step's activity — an agent session, a
+   * tool chain, a gate's probe — runs in it, so the cancel signal stops the
+   * step where it is instead of waiting for it to end on its own.
+   */
+  async function inStepScope<T>(work: () => Promise<T>): Promise<T> {
+    return CancellationScope.cancellable(async () => {
+      stepScope = CancellationScope.current();
+      if (cancelled !== null) stepScope.cancel();
+      try {
+        return await work();
+      } finally {
+        stepScope = null;
+      }
+    });
+  }
+
+  async function runAgent(step: StepDef, threadId: string, resumeThreadId: string | null): Promise<AgentStepOutcome> {
+    try {
+      return await inStepScope(() => runAgentActivity(step, threadId, resumeThreadId));
+    } catch (err) {
+      if (isCancellation(err) || err instanceof CancelledFailure) {
+        // A cancelled step closes its own thread when its activity is alive
+        // to hear the cancel; one whose worker died cannot, so the thread is
+        // closed from here too, outside the scope the cancel just ended. A
+        // thread the activity closed already is left as it is.
+        await CancellationScope.nonCancellable(() => stepEnd.closeStepThread({ threadId, reason: 'the run was cancelled' }));
+        throw err;
+      }
+      const reason = `the step could not be run: ${failureReason(err)}`;
+      log.warn(`step "${step.name}" of run ${input.runId}: ${failureReason(err)}`);
+      // The activity may have died before it could close the thread it was
+      // given (a worker gone mid-step): closed from here, so the harness error
+      // the file's policy handles leaves no step thread running.
+      await stepEnd.closeStepThread({ threadId, reason });
+      return { kind: 'harness-error', threadId, reason };
+    }
+  }
+
+  async function runAgentActivity(step: StepDef, threadId: string, resumeThreadId: string | null): Promise<AgentStepOutcome> {
+    const stepInput: AgentStepInput = {
+      runId: input.runId,
+      workflowPath: input.workflowPath,
+      stepName: step.name,
+      stepIndex: step.index,
+      attempt: typeof state.attempt === 'number' ? state.attempt : 1,
+      threadId,
+      actor,
+      agentPath: step.agent ?? null,
+      systemPrompt: step.systemPrompt ?? null,
+      instructions: interpolate(step.instructions, vars()),
+      verdict: step.verdict ?? null,
+      expectedOutputs: step.outputs ?? [],
+      workItemPath: input.workItemPath,
+      input: input.input,
+      state: { ...state },
+      params,
+      resumeThreadId,
+      // A continued conversation: only the very first session of the run starts from the earlier thread's state.
+      continueFromThreadId: step.index === 1 && !resumeThreadId && (typeof state.attempt !== 'number' || state.attempt === 1) ? (input.continueFromThreadId ?? null) : null,
+      executorQueue,
+      environment: def.environment,
+    };
+    if (def.environment.kind === 'none') {
+      // The platform stopped under the step (a deploy): it goes on on the worker that comes up, on its own thread,
+      // from its saved session. A person's run on 9 Nov 2026 was blocked by a deploy instead, and nobody answered them.
+      for (let restarts = 0; ; restarts++) {
+        try {
+          return await inProcessStep.runAgentStep(restarts === 0 ? stepInput : { ...stepInput, resumeThreadId: threadId });
+        } catch (err) {
+          if (isCancellation(err) || err instanceof CancelledFailure || restarts >= MAX_STEP_RESTARTS || !cutOffByRestart(err)) throw err;
+          await record(step.name, `↻ ${step.name} goes on: the platform restarted under it (${failureReason(err)})`);
+        }
+      }
+    }
+
+    // A container step: the platform prepares it (the thread, the run key,
+    // the configuration, the Dockerfile's content), an executor runs it, and
+    // the platform closes it whatever happened — the key must not outlive
+    // the step.
+    const prepared = await platform.prepareContainerStep(stepInput);
+    if (prepared.kind === 'blocked') return { kind: 'blocked', threadId: prepared.threadId, reason: prepared.reason };
+    // A run that names an executor runs there and nowhere else, and while it
+    // is not connected the step waits for it — never a fallback to another,
+    // which cannot reach what the named one reaches (a person's VPN, a
+    // server's network). A personal name is resolved against the actor, so
+    // no run reaches another person's machine.
+    const named = namedExecutorQueue({ ...input, actor });
+    const namedLabel = input.executorKind === 'deployment' ? `executor "${input.executor}"` : `your executor "${input.executor}"`;
+    /** What the executor is handed: the prepared step, or the same in a fresh workspace once its pinned executor is gone. */
+    let containerInput = prepared.input;
+    let outcome: AgentStepOutcome | null = null;
+    let error: string | null = null;
+    try {
+      for (;;) {
+        try {
+          outcome = await containerStep(named ?? executorQueue ?? EXECUTOR_TASK_QUEUE, named !== null).runContainerStep(containerInput);
+          break;
+        } catch (err) {
+          if (isCancellation(err)) throw err;
+          const cause = activityCause(err);
+          const notTaken = cause instanceof ApplicationFailure && cause.type === 'ExecutorNotConnected';
+          if (cause instanceof ApplicationFailure && cause.type === 'ExecutorPaused') {
+            // The executor is paused until the provider's reset time: the run
+            // waits, as it does when the limit is hit mid-step, and keeps its
+            // workspace; nothing is charged to the item.
+            const resetAt = typeof cause.details?.[0] === 'number' ? cause.details[0] : null;
+            outcome = { kind: 'account-limit', threadId: prepared.input.threadId, resetAt, text: cause.message };
+          } else if (named && notTaken) {
+            // It went away before it took the step: nothing ran, so the step is sent again and waits for it.
+            log.info(`${namedLabel} left before taking step "${step.name}" (${failureReason(err)}) — waiting for it again`);
+            continue;
+          } else if (named) {
+            // It took the step and failed it: a harness error, so the file's own retriers and catchers decide.
+            outcome = { kind: 'harness-error', threadId: prepared.input.threadId, reason: `${namedLabel} did not finish the step: ${failureReason(err)}` };
+          } else if (executorQueue && (notTaken || !(cause instanceof ApplicationFailure && cause.nonRetryable))) {
+            // The pinned executor did not answer, or left before taking the
+            // step: it is gone. On the shared queue from now on, with a fresh
+            // workspace (the session is told so).
+            log.warn(`executor ${executorQueue} did not pick up step "${step.name}" (${failureReason(err)}) — falling back to the shared queue`);
+            executorQueue = null;
+            containerInput = { ...prepared.input, step: { ...prepared.input.step, executorQueue: null }, freshWorkspace: true };
+            continue;
+          } else if (notTaken) {
+            // Every deployment executor left before one took it: nothing ran; it waits on the shared queue again.
+            log.info(`the deployment's executors left before taking step "${step.name}" — waiting for one again`);
+            continue;
+          } else {
+            throw err;
+          }
+          break;
+        }
+      }
+      // A run on a named executor stays pinned to the queue that reaches it, whatever name the executor gives itself.
+      if (named && 'executorQueue' in outcome && outcome.executorQueue) outcome = { ...outcome, executorQueue: named };
+      return outcome;
+    } catch (err) {
+      error = failureReason(err);
+      throw err;
+    } finally {
+      // Closed whatever happened, a cancel included: the key must not outlive the step.
+      await CancellationScope.nonCancellable(() => stepEnd.finishContainerStep({ threadId: prepared.input.threadId, keyId: prepared.input.actorKey.keyId ?? null, outcome, error }));
+    }
+  }
+
+  async function settleAgent(step: StepDef, outcome: AgentStepOutcome, threadId: string): Promise<RunStatus | 'moved' | 'retry'> {
+    if ('threadId' in outcome && outcome.threadId) threadIds[`${step.name}#${state.attempt ?? 1}`] = outcome.threadId;
+    if ('executorQueue' in outcome && outcome.executorQueue) executorQueue = outcome.executorQueue;
+    switch (outcome.kind) {
+      case 'verdict': {
+        if (outcome.verdict === 'PASS') {
+          mergeOutputs(step.name, outcome.outputs);
+          clearRetries(step.name);
+          state.verdict = 'PASS';
+          await record(step.name, `✅ ${step.name} passed`);
+          return route(step.name, thenFor(step, 'PASS') ?? 'continue');
+        }
+        state.verdict = outcome.verdict;
+        const explicit = thenFor(step, outcome.verdict);
+        if (explicit) {
+          clearRetries(step.name);
+          return route(step.name, explicit);
+        }
+        return fail(step.name, errorForVerdict(outcome.verdict as Verdict), outcome.reason ?? `verdict ${outcome.verdict}`);
+      }
+      case 'no-verdict':
+        return fail(step.name, 'NoVerdict', outcome.reason);
+      case 'timeout':
+        return fail(step.name, 'Timeout', outcome.reason);
+      case 'harness-error':
+        return fail(step.name, 'HarnessError', outcome.reason);
+      case 'blocked':
+        return fail(step.name, 'Blocked', outcome.reason);
+      case 'account-limit': {
+        // Nothing about the item changes: no attempt, no retry, no error. The
+        // step runs again when the limit lifts.
+        const until = outcome.resetAt ? Math.max(0, outcome.resetAt - Date.now()) : ACCOUNT_LIMIT_DEFAULT_PAUSE_MS;
+        // The thread the step goes on on when the limit lifts, from the state
+        // it saved — remembered BEFORE anything that can fail: a run that
+        // ends while it is parked here (the pause's own record failing, a
+        // Cancel waking the pause, Temporal's cancellation, a file gone or
+        // changed at the next loop) closes it in `finish`, the one terminal
+        // path, so no thread is left `waiting` for good.
+        pausedThread = { step: step.name, threadId };
+        await record(step.name, `⏳ ${step.name} paused: the account is refusing sessions (${outcome.text}); retrying in ${Math.round(until / 60_000)} min`, true);
+        await condition(() => cancelled !== null, until); // a Cancel signal wakes the pause
+        return 'retry';
+      }
+    }
+  }
+
+  async function settleChain(step: StepDef, result: ToolChainOutcome): Promise<RunStatus | 'moved' | 'retry'> {
+    if (result.threadId) threadIds[`${step.name}#${state.attempt ?? 1}`] = result.threadId;
+    if (result.kind === 'error') return fail(step.name, 'HarnessError', result.reason);
+    clearRetries(step.name);
+    const target = typeof step.then === 'string' || step.then === undefined ? (step.then ?? 'continue') : step.then[result.outcome];
+    if (!target) return fail(step.name, 'Fail', `chain answered "${result.outcome}", which the step does not route`);
+    await record(step.name, `✅ ${step.name}: ${result.outcome}`);
+    return route(step.name, target);
+  }
+
+  async function gate(step: StepDef): Promise<RunStatus | 'moved' | 'retry'> {
+    const outcomes = step.outcomes ?? {};
+    // A probed gate has the same deadline as a signalled one, kept across a
+    // continue-as-new; every poll past it is history for nothing.
+    const deadline = carriedGateDeadline ?? Date.now() + gateTimeoutMs;
+    carriedGateDeadline = null;
+    // Probe errors in a row: a cap on consecutive failures, not on failures over a gate's whole life (a gate polls for weeks). Carried across a continue-as-new like the deadline.
+    let probeErrors = carriedProbeErrors ?? 0;
+    carriedProbeErrors = null;
+    for (;;) {
+      if (cancelled) return (await finish('cancelled', cancelled)).status;
+      if (step.probe) {
+        await continueIfSuggested({ deadline, probeErrors });
+        if (Date.now() >= deadline) return fail(step.name, 'Timeout', `no answer at "${step.name}" within ${durationText(gateTimeoutMs)}`);
+        const probe = step.probe;
+        const result = await inStepScope(() =>
+          chainStep.runProbe({
+            runId: input.runId,
+            workflowPath: input.workflowPath,
+            stepName: step.name,
+            actor,
+            // The source as written: a chain reads `state` and `params` as
+            // objects the activity hands it, never through `${…}` text,
+            // which is JavaScript's own template syntax inside it.
+            chain: probe,
+            state: { ...state },
+            params,
+            workItemPath: input.workItemPath,
+          }),
+        );
+        if (result.kind === 'error') {
+          probeErrors += 1;
+          if (probeErrors >= PROBE_ERROR_CAP) return fail(step.name, 'ProbeError', result.reason);
+        } else if (result.outcome === 'wait') {
+          // A probe that answered, even "not yet", is working: the errors in a row start over.
+          probeErrors = 0;
+        } else {
+          const target = outcomes[result.outcome];
+          if (!target) return fail(step.name, 'ProbeError', `probe answered "${result.outcome}", which is not one of ${Object.keys(outcomes).join(', ')}`);
+          clearRetries(step.name);
+          await record(step.name, `✅ ${step.name}: ${result.outcome}`);
+          return route(step.name, target);
+        }
+        await condition(() => pendingGate !== null || cancelled !== null, gatePollMs);
+      } else {
+        // What is left of the gate's deadline, not the whole timeout again: an answer the gate does not declare leaves it waiting, never waiting longer.
+        const signalled = await condition(() => pendingGate !== null || cancelled !== null, Math.max(0, deadline - Date.now()));
+        if (!signalled) return fail(step.name, 'Timeout', `no answer at "${step.name}" within ${durationText(gateTimeoutMs)}`);
+      }
+      if (pendingGate !== null) {
+        const outcome: string = pendingGate;
+        pendingGate = null;
+        const target = outcomes[outcome];
+        if (!target) {
+          log.warn(`gate "${step.name}" was signalled "${outcome}", which it does not declare — still waiting`);
+          continue;
+        }
+        clearRetries(step.name);
+        await record(step.name, `✅ ${step.name}: ${outcome}`);
+        return route(step.name, target);
+      }
+    }
+  }
+}