@humwick/server 0.130.1 → 0.130.3

package/src/services/workflow-effects.ts

@@ -1,6 +1,5 @@
-import { planCiNotice } from "./plan-ci-recovery";
+import { planCiNotice } from "./plan-ci-notice";
 import { performWorkflowDecision, performWithdrawWorkflowDecision } from "./workflow-decisions";
-import { getCommandAsync } from "../commands-write";
 // #3045 — the src half of a workflow run: the kernel writes effect intents and waits; this module
 // performs each effect through the existing services and reads the durable facts that end a wait.
 // Every effect is keyed by its intent id, so a re-drive after a crash repeats nothing.
@@ -23,9 +22,10 @@ import { spawnTaskDispatch } from "./task-dispatch";
 import type { DispatchPin } from "../task-dispatcher";
 import { createTaskEntityAsync } from "./task-entity";
 import { workflowNotice } from "./workflow-notice";
+import { workflowPauseNotice } from "./workflow-pause-text";
 import { planQuietingRunNotice } from "./plan-wake-policy";
 import { workflowOverdueLead } from "./workflow-overdue-notice";
-import { workflowLandNotice } from "./workflow-land-notice";
+import { workflowLandNotice, workflowLandVerification } from "./workflow-land-notice";
 import { ciRedLead, ciResultLine } from "./workflow-ci-main";
 import { stepTaskDispatch } from "./workflow-pins";
 import { workflowDispatchResult } from "./workflow-dispatch-result";
@@ -36,12 +36,15 @@ import { isStewardHandbackPendingError } from "../db/withdraw-land-error";
 import { ShutdownAuthError, ShutdownTargetError, shutdownAgent, UnlandedWorkRefusal } from "./shutdown-agent";
 import { OfflineControlHostError, resolveControlOrchestrator } from "./agent-control-host";
 import { workflowCoordinatorTarget } from "./workflow-coordinator";
+import { supervisorNow } from "./workflow-supervisor-state";
+import { isTerminalCommandStatus } from "../db/command-terminal-status";
+import { isCoordinatorSeat } from "./workflow-caller";
 import { deliveredWorkspace, ensureWorkflowLanding, landedSignal, WORKFLOW_CTX } from "./workflow-land";
 import type { AuthContext } from "./auth-context";
 import type { Command, TaskDetail } from "../types";
 
 type Json = Record<string, unknown>;
-type WorkflowOp = (op: string, input: Json) => Promise<unknown>;
+export type WorkflowOp = (op: string, input: Json) => Promise<unknown>;
 
 function short(sha: string | undefined): string {
   return sha ? sha.slice(0, 12) : "an unknown head";
@@ -112,7 +115,7 @@ async function dispatchRole(run: WorkflowRunRecord, effect: WorkflowEffect, agen
   const spawnRequestId = spawnRequestIdFor(effect);
   await call("prepareDispatch", { id: effect.id, taskId, spawnRequestId });
   const spawn = await spawnTaskDispatch({
-    taskId, role, roleSha: run.sha, ...(run.callerAgentId ? { workflowCaller: { agentId: run.callerAgentId, maxLiveAgents: run.maxLiveAgents ?? 2, runId: run.rootRunId ?? run.id } } : {}), ...(run.snapshot?.[run.workflow]?.roles[role] ? { roleSnapshot: run.snapshot[run.workflow]!.roles[role] as unknown as import("../project-roles").ProjectRole } : {}), workflowRunId: run.id, pin: stepDispatchPin(pin, step), pinSources: pin?.from, createReview: false, ...(pin?.orchestratorId ? { orchestratorId: pin.orchestratorId } : {}),
+    taskId, role, roleSha: run.sha, ...(run.callerAgentId ? { workflowCaller: { agentId: run.callerAgentId, maxLiveAgents: run.maxLiveAgents ?? 2, runId: run.rootRunId ?? run.id } } : {}), ...(run.snapshot?.[run.workflow]?.roles[role] ? { roleSnapshot: run.snapshot[run.workflow]!.roles[role] as unknown as import("../project-roles").ProjectRole } : {}), workflowRunId: run.id, workflowNames: [...new Set([run.workflow, ...Object.keys(run.snapshot ?? {})])], pin: stepDispatchPin(pin, step), pinSources: pin?.from, createReview: false, ...(pin?.orchestratorId ? { orchestratorId: pin.orchestratorId } : {}),
     taskRepoCwd: run.cwd, roleHost: run.orchestratorId, ...(pin?.profile ? { roleProfile: pin.profile } : {}),
     spawnRequestId, waitForRegistrationMs: 0, verbose: false, ...(steer ? { steer } : {}), ...avoid,
   }, WORKFLOW_CTX);
@@ -168,16 +171,21 @@ async function createStepTask(run: WorkflowRunRecord, effect: WorkflowEffect): P
   return task.id;
 }
 
-/** The builder of record if it still runs: the agent a loop's findings go to (warm, never respawned). */
-export async function liveBuilder(taskId: number): Promise<{ id: string | undefined; live: boolean }> {
-  const id = (await requireTask(taskId)).ownerAgentId ?? undefined;
+/** The builder of record if it still runs: the agent a loop's findings go to (warm, never respawned). It is the run's
+ *  newest registered agent on the input Task, else the Task's assigned agent unless that is a coordinator seat (#3769):
+ *  a sub-coordinator assigned the Task owns its runs and never builds them. */
+export async function liveBuilder(run: Pick<WorkflowRunRecord, "id" | "taskId">, call: WorkflowOp): Promise<{ id: string | undefined; live: boolean }> {
+  const assigned = (await requireTask(run.taskId)).ownerAgentId ?? undefined;
+  const { items } = await call("runAgents", { runId: run.id }) as { items: WorkflowAgent[] };
+  const id = items.filter((entry) => entry.taskId === run.taskId && entry.agentId).at(-1)?.agentId
+    ?? (assigned && !isCoordinatorSeat(getAgent(assigned)) ? assigned : undefined);
   const agent = id ? getAgent(id) : null;
   return { id, live: Boolean(agent && agent.status !== "offline") };
 }
 
-async function warmBuilder(taskId: number): Promise<string> {
-  const builder = await liveBuilder(taskId);
-  if (!builder.live) throw new ValidationError(`the builder of T${taskId} (${builder.id ?? "none bound"}) is gone, so the findings cannot reach it`);
+async function warmBuilder(run: WorkflowRunRecord, call: WorkflowOp): Promise<string> {
+  const builder = await liveBuilder(run, call);
+  if (!builder.live) throw new ValidationError(`the builder of T${run.taskId} (${builder.id ?? "none bound"}) is gone, so the findings cannot reach it`);
   return builder.id!;
 }
 
@@ -251,14 +259,19 @@ function reapTargets(run: WorkflowRunRecord, effect: WorkflowEffect, agents: rea
     && (agentIds ? agentIds.has(agent.id) : workflowReapIncludesAgent(run, effect.payload, agent)));
 }
 
+/** A stop the host found no target for is retried this rarely: a missing target does not reappear each pass. */
+const TARGET_NOT_FOUND_RETRY_MS = 15 * 60_000;
+
 /** The run's latest stop command for an agent. A failed or timed-out one is terminal and never runs again,
- *  so the next attempt gets its own id; a pump therefore retries the stop instead of rereading the failure. */
-export async function currentStop(baseId: string, retryFailed: (command: Command) => boolean = () => true): Promise<{ id: string; prior?: Command }> {
-  for (let attempt = 1; ; attempt++) {
-    const id = attempt === 1 ? baseId : `${baseId}:${attempt}`;
-    const prior = await getCommandAsync(id);
-    if (!prior || (prior.status !== "failed" && prior.status !== "timed_out") || !retryFailed(prior)) return { id, ...(prior ? { prior } : {}) };
-  }
+ *  so the next attempt gets its own id; a pump therefore retries the stop instead of rereading the failure.
+ *  One read of the newest attempt, whatever the history (#3827). */
+export async function currentStop(baseId: string, now = supervisorNow()): Promise<{ id: string; prior?: Command }> {
+  const latest = await getGateway().read("commands.latestAttempt", { baseId });
+  if (!latest) return { id: baseId };
+  const { attempt, command } = latest;
+  const retry = (command.status === "failed" || command.status === "timed_out")
+    && (command.result?.targetNotFound !== true || now - command.updatedAt >= TARGET_NOT_FOUND_RETRY_MS);
+  return retry ? { id: `${baseId}:${attempt + 1}` } : { id: command.id, prior: command };
 }
 
 /** Shut each target's live generation down through the shutdown service. The unlanded-work guard's
@@ -268,6 +281,7 @@ async function reapAgents(run: WorkflowRunRecord, effect: WorkflowEffect, target
   const kept: Array<{ spawnRequestId: string; reason: string }> = [];
   const retry: Array<{ spawnRequestId: string; reason: string }> = [];
   const deferred: Array<{ spawnRequestId: string; orchestratorId: string }> = [];
+  let inFlight = false;
   for (const agent of targets) {
     const current = agent.agentId ? getAgent(agent.agentId) : null;
     const agentId = current?.meta?.spawnRequestId === agent.spawnRequestId ? agent.agentId : undefined;
@@ -288,6 +302,7 @@ async function reapAgents(run: WorkflowRunRecord, effect: WorkflowEffect, target
       if (stopId && command.type === "agent.spawn" && command.status === "canceled") command = (await shutdownAgent(stop, WORKFLOW_CTX)).command;
       if (stopId && (command.status !== "succeeded" || command.result?.processTreeStopped !== true)) {
         retry.push({ spawnRequestId: agent.spawnRequestId, reason: `shutdown ${command.id}: ${command.status}` });
+        if (!isTerminalCommandStatus(command.status)) inFlight = true;
         continue;
       }
       reaped.push({ spawnRequestId: agent.spawnRequestId, ...(agent.agentId ? { agentId: agent.agentId } : {}) });
@@ -298,7 +313,7 @@ async function reapAgents(run: WorkflowRunRecord, effect: WorkflowEffect, target
       else retry.push(entry);
     }
   }
-  return { reaped, ...(kept.length ? { kept } : {}), ...(retry.length ? { retry } : {}), ...(deferred.length ? { deferred } : {}) };
+  return { reaped, ...(kept.length ? { kept } : {}), ...(retry.length ? { retry } : {}), ...(deferred.length ? { deferred } : {}), ...(inFlight ? { inFlight } : {}) };
 }
 
 /** What a timed-out wait was waiting for, in words. */
@@ -359,7 +374,7 @@ export async function performWorkflowEffect(run: WorkflowRunRecord, effect: Work
   if (run.cancel && ["dispatch", "findings", "clear-hold", "hold", "close"].includes(effect.kind)) return { stale: true };
   switch (effect.kind) {
     case "decision": return performWorkflowDecision(run, effect, call);
-    case "withdraw-decision": return performWithdrawWorkflowDecision(effect);
+    case "withdraw-decision": return (await performWithdrawWorkflowDecision(effect));
     case "cancel-children": throw new ValidationError("tree cancellation is owned by the supervisor");
     case "child": throw new ValidationError("child effects are owned by the workflow supervisor");
     case "close": return closeWorkflowTask(run, effect);
@@ -367,7 +382,7 @@ export async function performWorkflowEffect(run: WorkflowRunRecord, effect: Work
     case "started": {
       const quietFor = await planQuietingRunNotice(run.taskId, "started");
       if (quietFor) return { quiet: "plan-dispatched", planId: quietFor };
-      const message = notifySystemMessage(await workflowCoordinatorTarget(run), {
+      const message = await notifySystemMessage(await workflowCoordinatorTarget(run), {
         subject: `${runLabel(run)} started`,
         body: `${runLabel(run)} started at step ${run.definition.start}. You hear back only on a spawn failure, an escalation or the result; relay_workflow_get shows the run and its agents.`,
         idempotencyKey: key,
@@ -377,6 +392,10 @@ export async function performWorkflowEffect(run: WorkflowRunRecord, effect: Work
       return { messageId: message.id };
     }
     case "landed": return { superseded: "the CI verdict owns the coordinator notice" };
+    case "pause-notice": {
+      const message = await notifySystemMessage(await workflowCoordinatorTarget(run), { ...workflowPauseNotice(runLabel(run), run, effect), idempotencyKey: key, replyExpected: false });
+      return { messageId: message.id };
+    }
     case "reap":
       return reapAgents(run, effect, reapTargets(run, effect, agents));
     case "dispatch": {
@@ -390,9 +409,9 @@ export async function performWorkflowEffect(run: WorkflowRunRecord, effect: Work
       } else if (typeof effect.payload.taskId === "number") await holdTask(run, "routes its review");
       if (typeof effect.payload.taskId === "number") {
         const taskId = effect.payload.taskId;
-        const builder = resume ? await liveBuilder(taskId) : undefined;
+        const builder = resume ? await liveBuilder({ id: run.id, taskId }, call) : undefined;
         if (builder?.live) {
-          const message = notifySystemMessage(builder.id!, {
+          const message = await notifySystemMessage(builder.id!, {
             subject: `${runLabel(run)} resumed at ${String(effect.payload.step)}`,
             body: brief!,
             idempotencyKey: key,
@@ -408,9 +427,9 @@ export async function performWorkflowEffect(run: WorkflowRunRecord, effect: Work
       return dispatchRole(run, effect, agents, await createStepTask(run, effect), role, brief, call);
     }
     case "findings": {
-      const builder = await warmBuilder(run.taskId);
+      const builder = await warmBuilder(run, call);
       const head = str(effect.payload.head);
-      const message = notifySystemMessage(builder, {
+      const message = await notifySystemMessage(builder, {
         subject: `${runLabel(run)}: ${String(effect.payload.step)} returned NO-GO on ${short(head)}`,
         body: [
           `${String(effect.payload.step)} (T${String(effect.payload.taskId)}) returned NO-GO on head ${head ?? "(unknown)"}. Round ${String(effect.payload.round)} starts now.`,
@@ -433,7 +452,7 @@ export async function performWorkflowEffect(run: WorkflowRunRecord, effect: Work
       const head = str(effect.payload.head);
       const ensured = await ensureWorkflowLanding(run, effect);
       if (ensured.outcome === "REFUSED" && ensured.held === true) {
-        const message = notifySystemMessage(await workflowCoordinatorTarget(run), {
+        const message = await notifySystemMessage(await workflowCoordinatorTarget(run), {
           subject: `${runLabel(run)}: GO, awaiting your review hold release`,
           body: `${runLabel(run)}: review returned GO on ${head ?? "an unknown head"}. Your review hold stays set; nothing is submitted for landing. After your review, clear it with relay_task_set_review_hold (taskId: ${run.taskId}, held: false, expectedHeadSha: ${head ?? "the held head"}). The run waits for the land.\n${workflowNotice(run, [])}`,
           idempotencyKey: key,
@@ -451,12 +470,13 @@ export async function performWorkflowEffect(run: WorkflowRunRecord, effect: Work
       }
       if (effect.payload.reason === "ci-red" || effect.payload.reason === "ci-late") {
         const sha = str(effect.payload.landedSha) ?? run.landed?.sha ?? "";
-        const message = notifySystemMessage(await workflowCoordinatorTarget(run), {
-          subject: `${runLabel(run)} landed, CI on main ${effect.payload.reason === "ci-red" ? "red" : "late"}`,
-          body: workflowNotice(run, [`T${run.taskId} landed as ${sha}.`, ...(effect.payload.reason === "ci-red" ? ciRedLead(runLabel(run), effect.payload) : [ciResultLine(effect.payload.ci as WorkflowCiVerdict)!])]),
+        const proof = await workflowLandVerification(run, sha);
+        const message = await notifySystemMessage(await workflowCoordinatorTarget(run), {
+          subject: `${runLabel(run)} ${proof.verified ? "landed" : "land verification FAILED"}, CI on main ${effect.payload.reason === "ci-red" ? "red" : "late"}`,
+          body: workflowNotice(run, [proof.verified ? `T${run.taskId} landed as ${sha}. ${proof.line}` : proof.line, ...(effect.payload.reason === "ci-red" ? ciRedLead(runLabel(run), effect.payload) : [ciResultLine(effect.payload.ci as WorkflowCiVerdict)!])]),
           idempotencyKey: `workflow:landed:${run.id}:${sha}:${effect.waitKey ?? effect.id}`,
           replyExpected: false,
-          payload: { kind: "workflow.landed", runId: run.id, taskId: run.taskId, landedSha: sha, reason: effect.payload.reason, ci: effect.payload.ci },
+          payload: { verified: proof.verified, kind: "workflow.landed", runId: run.id, taskId: run.taskId, landedSha: sha, reason: effect.payload.reason, ci: effect.payload.ci },
         });
         return { messageId: message.id };
       }
@@ -465,7 +485,7 @@ export async function performWorkflowEffect(run: WorkflowRunRecord, effect: Work
       const lead = effect.payload.reason === "run-overdue" ? await workflowOverdueLead(run, effect.payload) : escalationLead(run, effect.payload);
       if ((effect.payload.reason === "step-timeout" || effect.payload.reason === "run-overdue")
         && isRecord(effect.payload.waitingFor) && effect.payload.waitingFor.kind === "landed") lead.push(await workflowLandNotice(run));
-      const message = notifySystemMessage(await workflowCoordinatorTarget(run), {
+      const message = await notifySystemMessage(await workflowCoordinatorTarget(run), {
         subject: `${runLabel(run)} ${run.status === "failed" ? "failed" : "escalated"}`,
         body: workflowNotice(run, [...(run.status === "failed" ? [`Script failed: ${String(effect.payload.reason)}`] : lead), `Outcome: ${effect.payload.alert ? "still waiting" : run.status === "failed" ? "failed" : "escalated"}. Landed: ${run.landed?.sha ?? "none"}. Reason: ${String(effect.payload.reason)}.`, ciResultLine(isRecord(effect.payload.ci) ? effect.payload.ci as WorkflowCiVerdict : undefined) ?? "CI on main: pending or not configured."]),
         idempotencyKey: key,
@@ -494,18 +514,19 @@ export async function performWorkflowEffect(run: WorkflowRunRecord, effect: Work
         const receipt = await planCiNotice(run, effect.payload, true);
         if (receipt) return receipt;
       }
+      const proof = sha ? await workflowLandVerification(run, sha) : undefined;
       const ciLine = ciResultLine(ci);
       const quietFor = await planQuietingRunNotice(run.taskId, "result", ci);
-      if (quietFor) return { quiet: "plan-dispatched", planId: quietFor };
+      if (quietFor && proof?.verified !== false) return { quiet: "plan-dispatched", planId: quietFor };
       const nothingToLand = effect.payload.nothingToLand === true || child?.output?.nothingToLand === true;
       const rounds = typeof effect.payload.rounds === "number" ? effect.payload.rounds : child?.output?.rounds ?? child?.round;
-      const outcome = !sha ? "finished its script steps" : nothingToLand ? `had nothing to land: its reviewed head is already on main as ${sha}` : `landed as ${sha}`;
-      const message = notifySystemMessage(await workflowCoordinatorTarget(run), {
-        subject: `${runLabel(run)} done${nothingToLand ? ", nothing to land" : ""}${ci?.status === "fail" ? ", CI on main red" : ""}`,
-        body: workflowNotice(run, [`${runLabel(run)} is done: T${run.taskId} ${outcome}${rounds === undefined ? "" : ` after ${rounds} round(s)`}.`, ciLine ?? (child || !run.rootRunId ? "CI on main: not configured." : "CI on main: unknown.")], children.items, !!children.cursor),
+      const outcome = !sha ? "finished its script steps" : !proof?.verified ? `has an unverified land receipt for ${sha}` : nothingToLand ? `had nothing to land: its reviewed head is already on main as ${sha}` : `landed as ${sha}`;
+      const message = await notifySystemMessage(await workflowCoordinatorTarget(run), {
+        subject: `${runLabel(run)} ${proof?.verified === false ? "land verification FAILED" : "done"}${nothingToLand ? ", nothing to land" : ""}${ci?.status === "fail" ? ", CI on main red" : ""}`,
+        body: workflowNotice(run, [`${runLabel(run)} is done: T${run.taskId} ${outcome}${rounds === undefined ? "" : ` after ${rounds} round(s)`}.${proof ? ` ${proof.line}` : ""}`, ciLine ?? (child || !run.rootRunId ? "CI on main: not configured." : "CI on main: unknown.")], children.items, !!children.cursor),
         idempotencyKey: key,
         replyExpected: false,
-        payload: { kind: "workflow.result", runId: run.id, taskId: run.taskId, landedSha: sha, ...(rounds === undefined ? {} : { rounds }), ...(nothingToLand ? { nothingToLand } : {}), ...(ci ? { ci } : {}) },
+        payload: { ...(proof ? { verified: proof.verified } : {}), kind: "workflow.result", runId: run.id, taskId: run.taskId, landedSha: sha, ...(rounds === undefined ? {} : { rounds }), ...(nothingToLand ? { nothingToLand } : {}), ...(ci ? { ci } : {}) },
       });
       return { messageId: message.id };
     }

package/src/services/workflow-land-notice.ts

@@ -1,3 +1,5 @@
+import { isRecord } from "@humwick/sdk";
+import { verifyPublishedLand } from "./landed-ancestry";
 import type { WorkflowRunRecord } from "@humwick/kernel/workflow.pilot";
 import { landLaneSnapshot } from "../db/commands-land-wait";
 import { formatRelativeTime } from "../mcp/presentation";
@@ -16,3 +18,11 @@ export async function workflowLandNotice(run: WorkflowRunRecord): Promise<string
   const progress = workspace.landProgressAt == null ? "unknown" : formatRelativeTime(workspace.landProgressAt).relative;
   return `Land workspace ${workspace.id}: landState ${state} (${age}); last reported progress: ${progress}; lane position ${position} among waiting lands; merging now: ${active.length ? active.join(", ") : lane ? "none" : "unknown"}; gate host: ${gateHost}.`;
 }
+
+export async function workflowLandVerification(run: WorkflowRunRecord, sha: string): Promise<{ verified: boolean; line: string }> {
+  const workspace = await deliveredWorkspace(run);
+  const hostProof = workspace?.metadata.mergeResult;
+  const verdict = isRecord(hostProof) && hostProof.landVerifiedSha === sha ? "confirmed" : await verifyPublishedLand(workspace?.repoRoot ?? run.cwd, sha, workspace?.baseRef);
+  return verdict === "confirmed" ? { verified: true, line: "verified: true" }
+    : { verified: false, line: `Land NOT verified: ${sha} is ${verdict === "not-ancestor" ? "absent from" : "unresolvable on"} fetched origin. Coordinator action required.` };
+}

package/src/services/workflow-pause-text.ts

@@ -0,0 +1,43 @@
+// #3757 — what a paused run tells its coordinator: a notice on pause and unpause, and one reminder line per paused run
+// of its team in each new context generation (the self-care section of the self-resume continuation).
+import { isRecord, parseJson } from "@humwick/sdk";
+import type { WorkflowEffect, WorkflowRunRecord } from "@humwick/kernel/workflow.pilot";
+import { coordinatedTeamIds } from "../db/coordinator-seat";
+import { taskOwnerRoleTeamId } from "../db/task-owner";
+import { pausedWorkflowRuns } from "../db/workflow-task-read-ops";
+import { workflowText } from "./workflow-notice";
+
+const UNPAUSE = "relay_workflow_resume {runId, pause: false}";
+type Pause = NonNullable<WorkflowRunRecord["pause"]>;
+
+function pauseOf(value: unknown): Pause | undefined {
+  return isRecord(value) && typeof value.reason === "string" && typeof value.by === "string" && typeof value.at === "number"
+    ? { reason: value.reason, by: value.by, at: value.at } : undefined;
+}
+
+function pausedBy(pause: Pause): string {
+  return `paused since ${new Date(pause.at).toISOString()} by ${pause.by}: ${workflowText(pause.reason, 240)}`;
+}
+
+/** The coordinator notice a pause or an unpause sends. */
+export function workflowPauseNotice(label: string, run: WorkflowRunRecord, effect: WorkflowEffect): { subject: string; body: string; payload: Record<string, unknown> } {
+  const pause = pauseOf(effect.payload) ?? { reason: "", by: "unknown", at: effect.createdAt };
+  const kind = effect.payload.action === "unpaused" ? "workflow.unpaused" : "workflow.paused";
+  const body = kind === "workflow.paused"
+    ? `${label} is ${pausedBy(pause)}. A working step agent finishes; the run does not go past its next step, delivery, review or land, and its timers stand still. Continue it: ${UNPAUSE}.`
+    : `${label} was unpaused by ${String(effect.payload.unpausedBy)} after ${Math.round(Number(effect.payload.pausedMs) / 60_000)}m. It was ${pausedBy(pause)}. It goes on from step ${run.step}; its deadlines moved by the paused time.`;
+  return { subject: `${label} ${kind === "workflow.paused" ? "paused" : "unpaused"}`, body, payload: { kind, runId: run.id, taskId: run.taskId, ...pause, ...(kind === "workflow.unpaused" ? { unpausedBy: effect.payload.unpausedBy } : {}) } };
+}
+
+/** The paused runs of the teams this agent coordinates, and the ones it started; "" when none are paused. */
+export function renderPausedRunsSection(agentId: string): string {
+  const teams = new Set(coordinatedTeamIds(agentId));
+  const lines = pausedWorkflowRuns().flatMap((row) => {
+    const teamId = taskOwnerRoleTeamId(row.owner) ?? row.teamId;
+    if (row.coordinator !== agentId && row.callerAgentId !== agentId && !(teamId && teams.has(teamId))) return [];
+    const pause = pauseOf(parseJson<unknown>(row.pause, undefined));
+    if (!pause) return [];
+    return [`- ${row.runId} on T${row.taskId}${row.taskTitle ? ` (${workflowText(row.taskTitle, 80)})` : ""}: ${pausedBy(pause)}`];
+  });
+  return lines.length ? [`Paused workflow runs of your team: they keep their agents and wait. Continue one with ${UNPAUSE}.`, ...lines].join("\n") : "";
+}

package/src/services/workflow-pins.ts

@@ -1,7 +1,7 @@
 // #3121 — per-step pins on a workflow run: the wire parser every surface shares, and the start check that
 // resolves each step's launch (pin, role file and dispatch policy) through the spawn's own seam and refuses a
 // launch that cannot run before anything spawns. Precedence (call pin > YAML step) is the kernel's workflowStepPin.
-import { cleanString, isRecord, isSpawnProvider, SPAWN_PROVIDERS, stringValue, type SpawnProvider, type TaskDispatch } from "@humwick/sdk";
+import { cleanString, isRecord, isSpawnProvider, SPAWN_PROVIDERS, stringValue, type SpawnProvider, type TaskDispatch, type WorkflowRunPins } from "@humwick/sdk";
 import { findProviderCatalogModel, type ProviderCatalogEntry } from "@humwick/sdk/provider-catalog";
 import {
   stampWorkflowPins, WORKFLOW_PIN_FIELDS, WorkflowDefinitionError, workflowStepPin,
@@ -15,11 +15,12 @@ import { sameHostIdentity } from "../utils";
 import { roleSpawnableProviders } from "./spawn-project-role";
 import { resolveSpawnLaunch, roleLaunchDefaults } from "./spawn-launch";
 import { ServiceAuthError } from "./errors";
+import { PinnedHostCapacityError } from "../spawn-admission";
 import type { ProjectRole } from "../project-roles";
 import type { AgentCard } from "../types";
 import type { AuthContext } from "./auth-context";
 
-export type WorkflowPins = Record<string, WorkflowPin>;
+export type WorkflowPins = WorkflowRunPins;
 
 export const PIN_SOURCE_LABEL = { start: "start pin", resume: "resume pin", team: "team default", global: "global default", workflow: "workflow file" } as const;
 
@@ -118,12 +119,14 @@ export async function assertStepLaunches(check: StepLaunchCheck, ctx: AuthContex
     const from = declared.crossProvider;
     const avoid = from && !pin?.provider ? check.avoided?.(from) ?? launched.get(from) : undefined;
     const defaults = roleLaunchDefaults(role);
-    const launch = await resolveSpawnLaunch({
+    const input = {
       profile: defaults.profile, spawnCap: { callerAgentId: ctx.callerAgentId },
       provider: (pin?.provider as SpawnProvider | undefined) ?? defaults.provider, dispatchPin: { model: pin?.model, effort: pin?.effort },
       dispatchShape: stepTaskDispatch(check.inputTask, declared) ?? defaults.dispatchShape, orchestratorId: pin?.orchestratorId, ...(avoid ? { avoidProviders: [avoid] } : {}), projectRole: role,
       pinSources: pin?.from,
-    }).catch((error: Error) => {
+    };
+    const launch = await resolveSpawnLaunch(input).catch((error: Error) => {
+      if (error instanceof PinnedHostCapacityError) return resolveSpawnLaunch({ ...input, spawnCap: { ...input.spawnCap, attach: true } });
       throw new ValidationError(`${where}: ${error.message}`);
     });
     const runnable = roleSpawnableProviders(pin?.orchestratorId, role, launch.catalog);

package/src/services/workflow-reconcile.ts

@@ -8,7 +8,7 @@ import { workflowDecisionSignal } from "./workflow-decisions";
 import { errMessage, isRecord, stringValue } from "@humwick/sdk";
 
 import { waitTimeoutMs } from "@humwick/kernel/workflow-definition";
-import { workflowEffectId, type WorkflowAgent, type WorkflowEffect, type WorkflowRunRecord, type WorkflowSignal } from "@humwick/kernel/workflow.pilot";
+import { workflowEffectId, workflowRunClock, type WorkflowAgent, type WorkflowEffect, type WorkflowRunRecord, type WorkflowSignal } from "@humwick/kernel/workflow.pilot";
 import { listActivityEventsAsync } from "../db";
 
 import { getGateway } from "../db/gateway";
@@ -21,7 +21,7 @@ import { workflowCleanupDeferred, notifyDeferredWorkflowCleanup } from "./workfl
 
 import { deliveredWorkspace, landedSignal, landStopSignal } from "./workflow-land";
 
-import { invoke, SUPERVISOR, testOptions, type Supervisor } from "./workflow-supervisor-state";
+import { invoke, SUPERVISOR, supervisorNow, testOptions, type Supervisor } from "./workflow-supervisor-state";
 import { workflowWaitAgent } from "./workflow-agent-obligation";
 import { landingProgress } from "../workspace-phase";
 type Json = Record<string, unknown>;
@@ -48,8 +48,9 @@ async function alertStalled(sup: Supervisor, run: WorkflowRunRecord, wait: Await
 
 /** A wait past its deadline alerts the coordinator once and keeps waiting, like a stopped land (#3045 H1). */
 async function alertTimeout(sup: Supervisor, run: WorkflowRunRecord, wait: Awaiting, taskId: number | undefined): Promise<boolean> {
+  if (wait.slot) return false;
   const timeoutMs = wait.deadline !== undefined ? wait.deadline - wait.since : waitTimeoutMs(run.definition, run.step);
-  const now = (testOptions.now ?? Date.now)();
+  const now = workflowRunClock(run, supervisorNow());
   if (wait.timedOutAt !== undefined || now < wait.since + timeoutMs) return false;
   if (wait.kind === "landed") {
     const workspace = await deliveredWorkspace(run);
@@ -65,13 +66,15 @@ export async function reconcileRun(sup: Supervisor, run: WorkflowRunRecord, land
   const wait = run.awaiting;
   if (!wait || (await invoke<{ present: boolean }>(sup, "hasSignal", { key: wait.key })).present) return false;
   if (wait.kind === "for-each") return false;
+  // #3757 — a paused run's deadlines stand still: they are read at its pause.
+  const now = workflowRunClock(run, supervisorNow());
   let payload: Json | undefined;
   let kind: WorkflowSignal["kind"] = wait.kind;
   const taskId = wait.taskId ?? await effectTaskId(sup, wait.effectId);
   if (wait.kind === "decision") {
     payload = workflowDecisionSignal(wait.effectId!);
     if (!payload) {
-      if (wait.deadline === undefined || (testOptions.now ?? Date.now)() < wait.deadline) return false;
+      if (wait.deadline === undefined || now < wait.deadline) return false;
       kind = "effect-failed";
       payload = { effect: "decision", error: "DECISION_DEADLINE: operator decision reached its explicit deadline" };
     }
@@ -80,7 +83,7 @@ export async function reconcileRun(sup: Supervisor, run: WorkflowRunRecord, land
     if (!effect?.result?.childRunId) return false;
     const child = await invoke<WorkflowRunRecord>(sup, "get", { runId: effect.result.childRunId });
     if (child.status === "running") {
-      if (wait.deadline !== undefined && (testOptions.now ?? Date.now)() >= wait.deadline) return (await invoke<{ created: boolean }>(sup, "parkChild", { runId: run.id, childRunId: child.id, reason: `CHILD_TIMEOUT: ${child.id} exceeded ${wait.deadline}` })).created;
+      if (wait.deadline !== undefined && now >= wait.deadline) return (await invoke<{ created: boolean }>(sup, "parkChild", { runId: run.id, childRunId: child.id, reason: `CHILD_TIMEOUT: ${child.id} exceeded ${wait.deadline}` })).created;
       return false;
     }
     if (child.status === "escalated") {
@@ -112,19 +115,19 @@ export async function reconcileRun(sup: Supervisor, run: WorkflowRunRecord, land
     payload = taskId === undefined ? undefined : await deliverySignal(run, taskId, wait.afterEventId, wait.since);
     if (!payload) {
       const { effect } = await invoke<{ effect?: WorkflowEffect }>(sup, "getEffect", { id: wait.effectId ?? workflowEffectId(run.id, run.waitSeq, "dispatch") });
-      payload = await spawnFailureSignal(effect, (testOptions.now ?? Date.now)());
+      payload = await spawnFailureSignal(effect, now);
       if (payload) kind = "effect-failed";
     }
     if (!payload) {
-      payload = await exitSignal(sup, run);
+      payload = await exitSignal(sup, run, now);
       if (payload) kind = payload.effect === "dispatch" ? "effect-failed" : "agent-exited";
     }
   } else if (wait.kind === "ci") {
     // A CI wait ends on its verdict, never on a timeout alert: past the deadline it reports CI red or "no CI".
-    const outcome = await ciMainOutcome(run, wait, (testOptions.now ?? Date.now)(), (await deliveredWorkspace(run))?.baseRef);
+    const outcome = await ciMainOutcome(run, wait, now, (await deliveredWorkspace(run))?.baseRef);
     if (outcome.red && (await invoke<{ created: boolean }>(sup, "alert", { runId: run.id, waitKey: wait.key, ...outcome.red })).created) return true;
     if (!outcome.verdict) return false;
-    if (outcome.verdict.status === "none" && wait.deadline !== undefined && (testOptions.now ?? Date.now)() >= wait.deadline
+    if (outcome.verdict.status === "none" && wait.deadline !== undefined && now >= wait.deadline
       && (await invoke<{ created: boolean }>(sup, "alert", { runId: run.id, waitKey: wait.key, key: "ci-late", reason: "ci-late", detail: { ci: outcome.verdict, landedSha: wait.sha } })).created) return true;
     payload = outcome.verdict;
   } else {
@@ -153,10 +156,10 @@ export async function reconcileRun(sup: Supervisor, run: WorkflowRunRecord, land
 }
 
 /** #3166 — the wait's step agent exited without a delivery, has stayed gone past the grace, and is not live again. */
-async function exitSignal(sup: Supervisor, run: WorkflowRunRecord): Promise<Json | undefined> {
+async function exitSignal(sup: Supervisor, run: WorkflowRunRecord, now: number): Promise<Json | undefined> {
   const { agent } = await invoke<{ agent?: WorkflowAgent }>(sup, "exitedWaitAgent", { runId: run.id });
   if (agent?.spawnFailedAt && !agent.readyAt) return { effect: "dispatch", error: `spawn ${agent.spawnRequestId}: ${agent.spawnError ?? "spawn failed"}` };
-  if (!agent?.exitedAt || (testOptions.now ?? Date.now)() - agent.exitedAt < AGENT_EXIT_GRACE_MS) return undefined;
+  if (!agent?.exitedAt || now - agent.exitedAt < AGENT_EXIT_GRACE_MS) return undefined;
   return agentExitSignal(agent);
 }
 

package/src/services/workflow-resume-pins.ts

@@ -4,16 +4,16 @@ import type { WorkflowAgent, WorkflowRunRecord } from "@humwick/kernel/workflow.
 import { getAgent, getTaskDetailAsync, ValidationError } from "../db";
 import type { ProjectRole } from "../project-roles";
 import { resolveSpawnProjectRole } from "./spawn-project-role";
-import { liveBuilder, stepAgentProvider } from "./workflow-effects";
+import { liveBuilder, stepAgentProvider, type WorkflowOp } from "./workflow-effects";
 import { assertStepLaunches, unmetPinFields } from "./workflow-pins";
 import type { AuthContext } from "./auth-context";
 
 /** A resume's new pins: a live warm builder must already meet its step's pin, since the run hands it findings
  *  instead of spawning and replacing it would take its workspace; and each named step's launch must resolve and run. */
-export async function assertResumePins(run: WorkflowRunRecord, pins: Record<string, WorkflowCallPin>, task: NonNullable<Awaited<ReturnType<typeof getTaskDetailAsync>>>, ctx: AuthContext, agents: readonly WorkflowAgent[]): Promise<void> {
+export async function assertResumePins(run: WorkflowRunRecord, pins: Record<string, WorkflowCallPin>, task: NonNullable<Awaited<ReturnType<typeof getTaskDetailAsync>>>, ctx: AuthContext, agents: readonly WorkflowAgent[], call: WorkflowOp): Promise<void> {
   const merged = { ...run.pins, ...pins };
   const steps = Object.keys(run.definition.steps).filter((step) => pins[step]);
-  const builder = await liveBuilder(run.taskId);
+  const builder = await liveBuilder(run, call);
   const agent = builder.live ? getAgent(builder.id!) : null;
   for (const step of steps.filter((name) => run.definition.steps[name]!.task === "input")) {
     const pin = workflowStepPin(run.definition, merged, step, run.routingDefaults);

package/src/services/workflow-routing-defaults.ts

@@ -0,0 +1,18 @@
+import type { StepRoutingDefaults, TaskDetail } from "@humwick/sdk";
+import type { WorkflowRoutingDefaults } from "@humwick/kernel/workflow-definition";
+import { getAgent, getTeam } from "../db";
+import { taskOwnerRoleTeamId } from "../db/task-owner";
+
+/** #3829 — defaults follow the Task's durable owner, then its nearest configured ancestor. */
+export function resolveWorkflowRoutingDefaults(defaults: StepRoutingDefaults | undefined, task: Pick<TaskDetail, "owner" | "teamId" | "createdBy">, callerAgentId?: string): WorkflowRoutingDefaults | undefined {
+  if (!defaults) return undefined;
+  const coordinatorId = callerAgentId ?? task.createdBy;
+  const teamId = taskOwnerRoleTeamId(task.owner) ?? task.teamId ?? (coordinatorId ? getAgent(coordinatorId)?.teamId : undefined);
+  const visited = new Set<string>();
+  for (let current = teamId; current && !visited.has(current); current = getTeam(current)?.parentTeamId) {
+    visited.add(current);
+    const team = defaults.teams?.[current];
+    if (team) return { global: defaults.global, team, teamId: current };
+  }
+  return { global: defaults.global, ...(teamId ? { teamId } : {}) };
+}

package/src/services/workflow-run-view.ts

@@ -6,9 +6,9 @@ import { workflowText } from "./workflow-notice";
 
 type Awaiting = NonNullable<WorkflowRunRecord["awaiting"]>;
 
-function awaitingView({ kind, taskId, sha, deadline, timedOutAt }: Awaiting): NonNullable<WorkflowRunView["awaiting"]> {
+function awaitingView({ kind, taskId, sha, deadline, timedOutAt, slot }: Awaiting): NonNullable<WorkflowRunView["awaiting"]> {
   return {
-    kind, ...(taskId !== undefined ? { taskId } : {}), ...(sha ? { sha } : {}), ...(deadline !== undefined ? { deadline } : {}), ...(timedOutAt !== undefined ? { timedOutAt } : {}),
+    kind, ...(taskId !== undefined ? { taskId } : {}), ...(sha ? { sha } : {}), ...(deadline !== undefined ? { deadline } : {}), ...(timedOutAt !== undefined ? { timedOutAt } : {}), ...(slot ? { slot } : {}),
   };
 }
 
@@ -23,7 +23,7 @@ export function runView(run: WorkflowRunRecord): WorkflowRunView {
     /** The declared steps in order; `land` follows the last. */
     steps: Object.entries(run.definition.steps).map(([name, step]) => {
       const pin = workflowStepPin(run.definition, run.pins, name, run.routingDefaults);
-      return { name, role: step.role, ...(step.gate ? { gate: step.gate } : {}), ...(step.crossProvider ? { crossProvider: step.crossProvider } : {}), ...(pin ? { pin } : {}) };
+      return { name, role: step.role, started: run.entered.includes(name), ...(step.gate ? { gate: step.gate } : {}), ...(step.crossProvider ? { crossProvider: step.crossProvider } : {}), ...(pin ? { pin } : {}) };
     }),
     ...(run.definition.ciMain ? { ciMain: { ...(run.definition.ciMain.workflow ? { workflow: run.definition.ciMain.workflow } : {}) } } : {}),
     ref: run.ref,
@@ -40,8 +40,10 @@ export function runView(run: WorkflowRunRecord): WorkflowRunView {
     ...(run.awaiting ? { awaiting: awaitingView(run.awaiting) } : {}),
     ...(run.overdueAt !== undefined ? { overdueAt: run.overdueAt } : {}),
     ...(run.overdueNotifiedAt !== undefined ? { overdueNotifiedAt: run.overdueNotifiedAt } : {}),
+    ...(run.pause ? { paused: run.pause } : {}),
     ...(run.reason ? { reason: run.reason } : {}),
     ...(run.resumes?.length ? { resumes: run.resumes } : {}),
+    ...(run.pinChanges?.length ? { pinChanges: run.pinChanges } : {}),
     ...(run.output ? { output: run.output } : {}),
     createdAt: run.createdAt,
     updatedAt: run.updatedAt,
@@ -57,6 +59,7 @@ export function leanRunView(run: WorkflowRunRecord, agents: WorkflowRunAgent[]):
     ...(run.overdueAt !== undefined ? { overdueAt: run.overdueAt } : {}),
     ...(run.overdueNotifiedAt !== undefined ? { overdueNotifiedAt: run.overdueNotifiedAt } : {}),
     ...(run.awaiting ? { awaiting: awaitingView(run.awaiting) } : {}),
+    ...(run.pause ? { paused: run.pause } : {}),
     ...(run.reason ? { reason: workflowText(run.reason) } : {}),
     ...(run.output ? { output: { ...run.output, ...(run.output.ci ? { ci: { status: run.output.ci.status, ...(run.output.ci.sha ? { sha: run.output.ci.sha } : {}) } } : {}) } } : {}),
     agents: agents.map((agent) => ({

package/src/services/workflow-run.test-fixture.ts

@@ -90,7 +90,7 @@ export function installWorkflowHost(options: { confirmStops?: boolean } = {}): (
 export async function initWorkflowDb(): Promise<void> {
   await stopWorkflowServingForTests();
   await settleLandLifecycleDrives();
-  flushTaskStatusPulses();
+  await flushTaskStatusPulses();
   initDb(":memory:");
 }