@humwick/server 0.130.2 → 0.130.4

package/src/services/workflow-caller.ts

@@ -1,8 +1,9 @@
 import { stringValue } from "@humwick/sdk";
 import { childWorkflowsEnabled } from "../config";
 import { getAgentProfile } from "../config-store";
-import { getAgent, ValidationError } from "../db";
-import type { TaskDetail } from "../types";
+import { activeTeamCoordinatorId, getAgent, ValidationError } from "../db";
+import { taskOwnerRoleTeamId } from "../db/task-owner";
+import type { AgentCard, TaskDetail } from "../types";
 import { ServiceAuthError } from "./errors";
 import type { AuthContext } from "./auth-context";
 import { canSpawn } from "./spawn-agent";
@@ -15,14 +16,70 @@ export function assertWorkflowCallerLive(agentId: string): void {
   if (workflowCallerGone(agentId)) throw new ValidationError("WORKFLOW_CALLER_GONE: caller is canceled or reaped");
 }
 
+type WorkflowTaskSubject = Pick<TaskDetail, "createdBy" | "ownerAgentId" | "teamId" | "owner">;
+
+/** An agent seated as its team's coordinator: it owns work and runs, it never builds a run's Task. */
+export function isCoordinatorSeat(agent: Pick<AgentCard, "id" | "teamId"> | null | undefined): boolean {
+  return Boolean(agent?.teamId && activeTeamCoordinatorId(agent.teamId) === agent.id);
+}
+
+/** A spawned coordinator seat: a delegated sub-coordinator, never a worker it runs. */
+function delegatedSeat(agentId: string | null | undefined): AgentCard | null {
+  const agent = agentId ? getAgent(agentId) : null;
+  return agent?.spawnedBy && isCoordinatorSeat(agent) ? agent : null;
+}
+
+/** Who may own a Task's runs (#3769): its assigned agent, its team's seat, its owner role's seat, its creator. */
+function taskOwnerCandidates(task: WorkflowTaskSubject): Array<string | null | undefined> {
+  return [task.ownerAgentId, activeTeamCoordinatorId(task.teamId), activeTeamCoordinatorId(taskOwnerRoleTeamId(task.owner)), task.createdBy];
+}
+
+/** #3769 — the delegated sub-coordinator that owns a Task: the first candidate holding its private team's seat. */
+export function delegatedTaskOwner(task: WorkflowTaskSubject): AgentCard | null {
+  for (const id of taskOwnerCandidates(task)) {
+    const seat = delegatedSeat(id);
+    if (seat) return seat;
+  }
+  return null;
+}
+
+/** #3818 — a Task its parent coordinator created (or holds the owner seat of) is the sub-coordinator's queue, no hand-over needed. */
+function parentQueuedTask(seat: AgentCard, task: WorkflowTaskSubject): boolean {
+  const parent = seat.spawnedBy;
+  return Boolean(parent && (task.createdBy === parent || activeTeamCoordinatorId(taskOwnerRoleTeamId(task.owner)) === parent));
+}
+
+/** THE workflow-control predicate (start, read, resume, pause, cancel, budget): the Task's coordinators and admins, and
+ *  a delegated sub-coordinator that created it, is assigned it, holds its team's seat or was spawned by its creator.
+ *  A worker is never authorized. */
+export function canControlWorkflowTask(ctx: AuthContext, task: WorkflowTaskSubject): boolean {
+  if (canControlTaskReviewHold(ctx, task)) return true;
+  const caller = ctx.callerAgentId;
+  const seat = delegatedSeat(caller);
+  return Boolean(seat && (taskOwnerCandidates(task).includes(seat.id) || parentQueuedTask(seat, task)));
+}
+
 /** Workflow authority delegates execution to Relay, never the caller's direct-spawn capability. */
 export function admitWorkflowCaller(ctx: AuthContext, task: TaskDetail, workflow?: { name: string; source: string }) {
   const agent = ctx.callerAgentId ? getAgent(ctx.callerAgentId) : null;
   if (!agent || (!agent.spawnedBy && canSpawn(ctx))) return undefined;
   if (!childWorkflowsEnabled()) throw new ServiceAuthError("WORKFLOW_DISABLED: child workflows are disabled");
   assertWorkflowCallerLive(agent.id);
-  if (task.ownerAgentId !== agent.id && task.createdBy !== agent.id && !canControlTaskReviewHold(ctx, task)) throw new ServiceAuthError("WORKFLOW_TASK_FORBIDDEN: caller must own or create the Task");
-  const profile = getAgentProfile(ctx.constraints?.profile ?? stringValue(agent.meta?.profile) ?? "worker")?.value;
+  if (task.ownerAgentId !== agent.id && task.createdBy !== agent.id && !canControlWorkflowTask(ctx, task)) throw new ServiceAuthError("WORKFLOW_TASK_FORBIDDEN: caller must own or create the Task, or be a sub-coordinator of the coordinator that created it (hand one over with relay_task_assign_agent)");
+  return workflowCallerPolicy(agent, ctx.constraints?.profile, workflow);
+}
+
+/** #3769 — a run a Task coordinator starts on a Task a live delegated sub-coordinator owns is that owner's run:
+ *  its spawn lineage, its quota and its notices, exactly as if the owner had started it. */
+export function admitWorkflowTaskOwner(task: TaskDetail, workflow?: { name: string; source: string }) {
+  const owner = delegatedTaskOwner(task);
+  if (!owner || workflowCallerGone(owner.id)) return undefined;
+  if (!childWorkflowsEnabled()) throw new ServiceAuthError(`WORKFLOW_DISABLED: T${task.id} belongs to sub-coordinator ${owner.id}, and child workflows are disabled`);
+  return workflowCallerPolicy(owner, undefined, workflow);
+}
+
+function workflowCallerPolicy(agent: AgentCard, profileName: string | undefined, workflow?: { name: string; source: string }) {
+  const profile = getAgentProfile(profileName ?? stringValue(agent.meta?.profile) ?? "worker")?.value;
   const policy = profile?.workflows;
   const maxConcurrentRuns = policy?.maxConcurrentRuns ?? 1;
   const maxLiveAgents = policy?.maxLiveAgents ?? 2;

package/src/services/workflow-child-proof-worker.ts

@@ -43,7 +43,7 @@ if (phase === "crash") {
   setWorkflowSupervisorTestOptions({ fault(label) {
     if (label === "after-child-complete") process.kill(process.pid, "SIGKILL");
   } });
-  announceBranchLanded({ workspace: { id: WF.workspace, repoRoot: repo.cwd, branch: WF.branch, baseRef: "origin/main", ownerAgentId: WF.builder, orchestratorId: WF.orchestrator }, sourceHeadSha: sha, mergedSha: sha, pushed: true, taskIds: [task.id] });
+  await announceBranchLanded({ workspace: { id: WF.workspace, repoRoot: repo.cwd, branch: WF.branch, baseRef: "origin/main", ownerAgentId: WF.builder, orchestratorId: WF.orchestrator }, sourceHeadSha: sha, mergedSha: sha, pushed: true, taskIds: [task.id] });
   await workflowTaskDelivered();
   await settleChildProofStops();
   throw new Error("restart seam was not reached");

package/src/services/workflow-control-proof-worker.ts

@@ -76,7 +76,7 @@ async function build(node: Node) {
   git(node.cwd!, "push", "-q", "origin", "HEAD:main");
   node.sha = sha; save();
   upsertWorkspace({ ...getWorkspace(node.workspace!)!, landedAt: Date.now(), landedSha: sha });
-  announceBranchLanded({ workspace: { id: node.workspace!, repoRoot: node.cwd!, branch: node.branch!, baseRef: "main", ownerAgentId: node.agent!, orchestratorId: WF.orchestrator }, sourceHeadSha: sha, mergedSha: sha, pushed: true, taskIds: [node.taskId] });
+  await announceBranchLanded({ workspace: { id: node.workspace!, repoRoot: node.cwd!, branch: node.branch!, baseRef: "main", ownerAgentId: node.agent!, orchestratorId: WF.orchestrator }, sourceHeadSha: sha, mergedSha: sha, pushed: true, taskIds: [node.taskId] });
   await workflowTaskDelivered(); await settleChildProofStops(); await settleLandLifecycleDrives();
   meta.events.push({ kind: "land", id: node.id }); save();
 }

package/src/services/workflow-coordinator.ts

@@ -1,8 +1,14 @@
 import type { WorkflowRunRecord } from "@humwick/kernel/workflow.pilot";
 import { getTaskDetailAsync } from "../db";
 import { taskOwnerTarget } from "../task-owner";
+import { delegatedTaskOwner, workflowCallerGone } from "./workflow-caller";
 
-/** Notices and decisions share the starter, then durable Task owner, then run coordinator. */
+/** Notices and decisions share the starter, then the Task's live owning sub-coordinator (#3769), then its durable
+ *  owner, then the run coordinator: the root hears of a sub-coordinator's Task only once that owner is gone. */
 export async function workflowCoordinatorTarget(run: WorkflowRunRecord): Promise<string> {
-  return run.callerAgentId ?? taskOwnerTarget(await getTaskDetailAsync(run.taskId)) ?? run.coordinator;
+  if (run.callerAgentId) return run.callerAgentId;
+  const task = await getTaskDetailAsync(run.taskId);
+  const owner = task ? delegatedTaskOwner(task) : null;
+  if (owner && !workflowCallerGone(owner.id)) return owner.id;
+  return taskOwnerTarget(task) ?? run.coordinator;
 }

package/src/services/workflow-decisions.ts

@@ -42,7 +42,7 @@ export async function performWorkflowDecision(run: WorkflowRunRecord, effect: Wo
     workflow: { runId: run.id, step: run.step, effectId: effect.id, waitKey: effect.waitKey!, ...(multi ? { items } : {}), ...(step.outputSchema ? { outputSchema: step.outputSchema } : {}) },
   };
   if (spec.answeredBy === "coordinator") {
-    const message = notifySystemMessage(await workflowCoordinatorTarget(run), {
+    const message = await notifySystemMessage(await workflowCoordinatorTarget(run), {
       subject: `Workflow ${run.id}: ${request.title}`,
       body: [request.title, request.context, `Choices (JSON data): ${JSON.stringify(items)}`,
         `Answer with relay_workflow_resume({runId: "${run.id}", decision: {action: "answer", waitKey: "${effect.waitKey}", answers: {${multi ? 'approved: ["item-id"]' : 'choice: "option-id"'}}}, note: "reason"}).`,
@@ -63,16 +63,16 @@ export function workflowDecisionSignal(effectId: string): Record<string, unknown
   return { outcome: "complete", decisionId: decision.id, outputs: workflowDecisionAnswer(decision, decision.answers!, decision.note) };
 }
 
-export function performWithdrawWorkflowDecision(effect: WorkflowEffect): Record<string, unknown> {
-  withdrawWorkflowDecision(String(effect.payload.effectId));
+export async function performWithdrawWorkflowDecision(effect: WorkflowEffect): Promise<Record<string, unknown>> {
+  (await withdrawWorkflowDecision(String(effect.payload.effectId)));
   return { withdrawn: true };
 }
 
 async function fileOperatorDecision(run: WorkflowRunRecord, request: WorkflowDecisionRequest, note?: string): Promise<Record<string, unknown>> {
   const task = await getTaskDetailAsync(run.taskId);
-  const decision = fileWorkflowDecision({ actor: { id: run.coordinator, kind: "agent" }, callerAgentId: run.coordinator, scopes: ["*", "command:spawn"] }, {
+  const decision = (await fileWorkflowDecision({ actor: { id: run.coordinator, kind: "agent" }, callerAgentId: run.coordinator, scopes: ["*", "command:spawn"] }, {
     ...request,
-  }, request.workflow, task?.projectId, note);
+  }, request.workflow, task?.projectId, note));
   return { decisionId: decision.id };
 }
 

package/src/services/workflow-deferred-cleanup.ts

@@ -31,7 +31,7 @@ async function resumeHosts(sup: Supervisor, orchestratorId?: string): Promise<vo
 
 export async function notifyDeferredWorkflowCleanup(run: WorkflowRunRecord, effect: WorkflowEffect): Promise<void> {
   const proof = run.landed?.sha ? await workflowLandVerification(run, run.landed.sha) : undefined;
-  notifySystemMessage(await workflowCoordinatorTarget(run), {
+  await notifySystemMessage(await workflowCoordinatorTarget(run), {
     subject: `Workflow ${run.id}: cleanup deferred`,
     body: `Workflow ${run.id}: ${proof?.verified ? `landed as ${run.landed?.sha}. verified: true.` : proof?.line ?? "No land receipt."} Shutdowns on offline hosts are recorded durably and will run when those hosts re-register. The run can finish with its CI result.`,
     payload: { ...(proof ? { verified: proof.verified } : {}), kind: "workflow.cleanup-deferred", runId: run.id, effectId: effect.id, deferred: effect.result?.deferred },

package/src/services/workflow-effect-driver.ts

@@ -19,12 +19,14 @@ import { ServiceAuthError } from "./errors";
 
 import { performWorkflowEffect } from "./workflow-effects";
 import { workflowCleanupDeferred } from "./workflow-deferred-cleanup";
+import { PinnedHostCapacityError } from "../spawn-admission";
 
-import { invoke, live, runAgents, noteAgent, testOptions, type Supervisor } from "./workflow-supervisor-state";
+import { effectDue, invoke, retryResult, live, runAgents, noteAgent, announceChange, supervisorNow, testOptions, type Supervisor } from "./workflow-supervisor-state";
 type Json = Record<string, unknown>;
 const EFFECT_BATCH = 50;
 const MAX_EFFECT_ATTEMPTS = 3;
 const REAP_RETRY_MS = 60 * 60_000;
+const SLOT_WAIT_MS = 30 * 60_000;
 function isPermanentEffectError(error: unknown): boolean {
   return error instanceof WorkflowValueError || error instanceof ValidationError || error instanceof McpNotFoundError || error instanceof McpAuthError
     || error instanceof ServiceAuthError || error instanceof ReviewHoldConflictError;
@@ -39,21 +41,31 @@ async function settleEffect(sup: Supervisor, run: WorkflowRunRecord, effect: Wor
     run = { ...run, snapshot: root.snapshot };
   }
   try {
+    if (effect.kind === "dispatch" && !run.cancel && run.status === "running" && run.awaiting?.slot && run.awaiting.key === effect.waitKey && supervisorNow() >= run.awaiting.slot.deadline) {
+      throw new ValidationError(`Timed out waiting for a slot on ${run.awaiting.slot.host} after 30 minutes`);
+    }
     if (effect.kind === "dispatch" && run.rootRunId && !run.cancel) await invoke(sup, "reserveAgent", { runId: run.id, effectId: effect.id });
     if (effect.kind === "script" && !(await invoke<{ reserved: boolean }>(sup, "reserveScript", { runId: run.id, effectId: effect.id })).reserved) return false;
     const performed = effect.kind === "cancel-children" ? await cancelChildren(sup, run, effect) : effect.kind === "child" ? await createChild(sup, run, effect) : await performWorkflowEffect(run, effect, effect.kind === "reap" || effect.kind === "dispatch" ? await runAgents(sup, run.id) : [], (op, input) => invoke(sup, op, input));
     if (!performed && effect.kind !== "script") return false;
     result = performed ?? { pending: true };
   } catch (error) {
+    if (effect.kind === "dispatch" && error instanceof PinnedHostCapacityError) {
+      if (!run.awaiting?.slot) {
+        await invoke(sup, "waitForSlot", { id: effect.id, host: error.orchestratorId, now: supervisorNow(), timeoutMs: SLOT_WAIT_MS });
+        announceChange();
+      }
+      return false;
+    }
     const message = errMessage(error);
     if (workflowOwnsAgentCleanup(run) && ["cancel-children", "reap"].includes(effect.kind)) {
       if (run.status !== "done") await invoke(sup, "cleanup", { runId: run.id, complete: false, error: message });
-      await invoke(sup, "settleEffect", { id: effect.id, status: "pending", error: message });
+      await invoke(sup, "settleEffect", { id: effect.id, status: "pending", error: message, result: retryResult(effect.result, effect.result ?? {}, supervisorNow()) });
       return false;
     }
     const failed = isPermanentEffectError(error) || effect.attempts + 1 >= MAX_EFFECT_ATTEMPTS;
     console.warn(`workflow effect ${effect.id} ${failed ? "failed" : "will retry"}: ${message}`);
-    await invoke(sup, "settleEffect", { id: effect.id, status: failed ? "failed" : "pending", error: message });
+    await invoke(sup, "settleEffect", { id: effect.id, status: failed ? "failed" : "pending", error: message, now: supervisorNow() });
     if (failed && effect.kind === "dispatch") await releaseUnusedDispatch(sup, effect);
     if (failed && effect.kind === "script") await invoke(sup, "releaseReservation", { effectId: effect.id });
     const refusal = failed && effect.kind === "dispatch" ? sup.refusable.get(run.id) : undefined;
@@ -81,14 +93,15 @@ async function settleEffect(sup: Supervisor, run: WorkflowRunRecord, effect: Wor
   // The reap becomes a dormant deferred-cleanup record until its host returns (#3658).
   const deferred = Array.isArray(result.deferred) && result.deferred.length > 0;
   if (deferred || Array.isArray(result.retry) && result.retry.length && (workflowOwnsAgentCleanup(run) || Date.now() - effect.createdAt < REAP_RETRY_MS)) {
-    const pending = await invoke<WorkflowEffect>(sup, "settleEffect", { id: effect.id, status: deferred && !(Array.isArray(result.retry) && result.retry.length) ? "deferred" : "pending", result, error: JSON.stringify(result.retry ?? result.deferred).slice(0, 2000) });
+    const status = deferred && !(Array.isArray(result.retry) && result.retry.length) ? "deferred" : "pending";
+    const pending = await invoke<WorkflowEffect>(sup, "settleEffect", { id: effect.id, status, result: status === "pending" ? retryResult(effect.result, result, supervisorNow()) : result, error: JSON.stringify(result.retry ?? result.deferred).slice(0, 2000) });
     if (!workflowCleanupDeferred(pending) && workflowOwnsAgentCleanup(run) && run.status !== "done") await invoke(sup, "cleanup", { runId: run.id, complete: false, error: JSON.stringify(result.retry).slice(0, 2000) });
     return workflowCleanupDeferred(pending);
   }
   if (effect.kind === "reap" && run.status === "cancelled" && !run.rootRunId) await invoke(sup, "cleanup", { runId: run.id, complete: true });
   if (effect.kind === "dispatch" && !result.spawnRequestId) await releaseUnusedDispatch(sup, effect);
   if (effect.kind === "script" && result.pending !== true) await invoke(sup, "releaseReservation", { effectId: effect.id });
-  await invoke(sup, "settleEffect", { id: effect.id, status: "done", result });
+  await invoke(sup, "settleEffect", { id: effect.id, status: "done", result, now: supervisorNow() });
   return true;
 }
 
@@ -145,23 +158,33 @@ async function createChild(sup: Supervisor, run: WorkflowRunRecord, effect: Work
   return { childRunId: child.id };
 }
 
+/** Perform up to a batch of due pending effects. A backing-off effect is skipped before any read and does not
+ *  count toward the batch, so effects that never converge cannot starve newer ones (#3827). */
 export async function drainEffects(sup: Supervisor): Promise<number> {
-  const { items } = await invoke<{ items: WorkflowEffect[] }>(sup, "pendingEffects", { limit: EFFECT_BATCH });
+  const now = supervisorNow();
   let settled = 0;
-  for (const effect of items) {
-    if (live() !== sup) break;
-    try {
-      if (effect.kind === "cancel-agent-tree") {
-        if (await performSpawnTreeCleanup(sup, effect)) settled++;
-        continue;
+  let attempted = 0;
+  let cursor: string | undefined;
+  do {
+    const page = await invoke<{ items: WorkflowEffect[]; cursor?: string }>(sup, "pendingEffects", { limit: EFFECT_BATCH, ...(cursor ? { cursor } : {}) });
+    for (const effect of page.items) {
+      if (live() !== sup) return settled;
+      if (!effectDue(effect, now)) continue;
+      if (attempted++ === EFFECT_BATCH) return settled;
+      try {
+        if (effect.kind === "cancel-agent-tree") {
+          if (await performSpawnTreeCleanup(sup, effect)) settled++;
+          continue;
+        }
+        const run = await invoke<WorkflowRunRecord>(sup, "get", { runId: effect.runId });
+        // #3757 — a paused run spawns, messages, arms and reaps nothing until its unpause; only its pause notices go out.
+        if (run.pause && effect.kind !== "pause-notice") continue;
+        if (await settleEffect(sup, run, effect)) settled += 1;
+      } catch (error) {
+        console.warn(`workflow effect ${effect.id} could not be settled: ${errMessage(error)}`);
       }
-      const run = await invoke<WorkflowRunRecord>(sup, "get", { runId: effect.runId });
-      // #3757 — a paused run spawns, messages, arms and reaps nothing until its unpause; only its pause notices go out.
-      if (run.pause && effect.kind !== "pause-notice") continue;
-      if (await settleEffect(sup, run, effect)) settled += 1;
-    } catch (error) {
-      console.warn(`workflow effect ${effect.id} could not be settled: ${errMessage(error)}`);
     }
-  }
+    cursor = page.cursor;
+  } while (cursor);
   return settled;
 }

package/src/services/workflow-effects.ts

@@ -1,6 +1,5 @@
 import { planCiNotice } from "./plan-ci-notice";
 import { performWorkflowDecision, performWithdrawWorkflowDecision } from "./workflow-decisions";
-import { getCommandAsync } from "../commands-write";
 // #3045 — the src half of a workflow run: the kernel writes effect intents and waits; this module
 // performs each effect through the existing services and reads the durable facts that end a wait.
 // Every effect is keyed by its intent id, so a re-drive after a crash repeats nothing.
@@ -37,12 +36,15 @@ import { isStewardHandbackPendingError } from "../db/withdraw-land-error";
 import { ShutdownAuthError, ShutdownTargetError, shutdownAgent, UnlandedWorkRefusal } from "./shutdown-agent";
 import { OfflineControlHostError, resolveControlOrchestrator } from "./agent-control-host";
 import { workflowCoordinatorTarget } from "./workflow-coordinator";
+import { supervisorNow } from "./workflow-supervisor-state";
+import { isTerminalCommandStatus } from "../db/command-terminal-status";
+import { isCoordinatorSeat } from "./workflow-caller";
 import { deliveredWorkspace, ensureWorkflowLanding, landedSignal, WORKFLOW_CTX } from "./workflow-land";
 import type { AuthContext } from "./auth-context";
 import type { Command, TaskDetail } from "../types";
 
 type Json = Record<string, unknown>;
-type WorkflowOp = (op: string, input: Json) => Promise<unknown>;
+export type WorkflowOp = (op: string, input: Json) => Promise<unknown>;
 
 function short(sha: string | undefined): string {
   return sha ? sha.slice(0, 12) : "an unknown head";
@@ -169,16 +171,21 @@ async function createStepTask(run: WorkflowRunRecord, effect: WorkflowEffect): P
   return task.id;
 }
 
-/** The builder of record if it still runs: the agent a loop's findings go to (warm, never respawned). */
-export async function liveBuilder(taskId: number): Promise<{ id: string | undefined; live: boolean }> {
-  const id = (await requireTask(taskId)).ownerAgentId ?? undefined;
+/** The builder of record if it still runs: the agent a loop's findings go to (warm, never respawned). It is the run's
+ *  newest registered agent on the input Task, else the Task's assigned agent unless that is a coordinator seat (#3769):
+ *  a sub-coordinator assigned the Task owns its runs and never builds them. */
+export async function liveBuilder(run: Pick<WorkflowRunRecord, "id" | "taskId">, call: WorkflowOp): Promise<{ id: string | undefined; live: boolean }> {
+  const assigned = (await requireTask(run.taskId)).ownerAgentId ?? undefined;
+  const { items } = await call("runAgents", { runId: run.id }) as { items: WorkflowAgent[] };
+  const id = items.filter((entry) => entry.taskId === run.taskId && entry.agentId).at(-1)?.agentId
+    ?? (assigned && !isCoordinatorSeat(getAgent(assigned)) ? assigned : undefined);
   const agent = id ? getAgent(id) : null;
   return { id, live: Boolean(agent && agent.status !== "offline") };
 }
 
-async function warmBuilder(taskId: number): Promise<string> {
-  const builder = await liveBuilder(taskId);
-  if (!builder.live) throw new ValidationError(`the builder of T${taskId} (${builder.id ?? "none bound"}) is gone, so the findings cannot reach it`);
+async function warmBuilder(run: WorkflowRunRecord, call: WorkflowOp): Promise<string> {
+  const builder = await liveBuilder(run, call);
+  if (!builder.live) throw new ValidationError(`the builder of T${run.taskId} (${builder.id ?? "none bound"}) is gone, so the findings cannot reach it`);
   return builder.id!;
 }
 
@@ -252,14 +259,19 @@ function reapTargets(run: WorkflowRunRecord, effect: WorkflowEffect, agents: rea
     && (agentIds ? agentIds.has(agent.id) : workflowReapIncludesAgent(run, effect.payload, agent)));
 }
 
+/** A stop the host found no target for is retried this rarely: a missing target does not reappear each pass. */
+const TARGET_NOT_FOUND_RETRY_MS = 15 * 60_000;
+
 /** The run's latest stop command for an agent. A failed or timed-out one is terminal and never runs again,
- *  so the next attempt gets its own id; a pump therefore retries the stop instead of rereading the failure. */
-export async function currentStop(baseId: string, retryFailed: (command: Command) => boolean = () => true): Promise<{ id: string; prior?: Command }> {
-  for (let attempt = 1; ; attempt++) {
-    const id = attempt === 1 ? baseId : `${baseId}:${attempt}`;
-    const prior = await getCommandAsync(id);
-    if (!prior || (prior.status !== "failed" && prior.status !== "timed_out") || !retryFailed(prior)) return { id, ...(prior ? { prior } : {}) };
-  }
+ *  so the next attempt gets its own id; a pump therefore retries the stop instead of rereading the failure.
+ *  One read of the newest attempt, whatever the history (#3827). */
+export async function currentStop(baseId: string, now = supervisorNow()): Promise<{ id: string; prior?: Command }> {
+  const latest = await getGateway().read("commands.latestAttempt", { baseId });
+  if (!latest) return { id: baseId };
+  const { attempt, command } = latest;
+  const retry = (command.status === "failed" || command.status === "timed_out")
+    && (command.result?.targetNotFound !== true || now - command.updatedAt >= TARGET_NOT_FOUND_RETRY_MS);
+  return retry ? { id: `${baseId}:${attempt + 1}` } : { id: command.id, prior: command };
 }
 
 /** Shut each target's live generation down through the shutdown service. The unlanded-work guard's
@@ -269,6 +281,7 @@ async function reapAgents(run: WorkflowRunRecord, effect: WorkflowEffect, target
   const kept: Array<{ spawnRequestId: string; reason: string }> = [];
   const retry: Array<{ spawnRequestId: string; reason: string }> = [];
   const deferred: Array<{ spawnRequestId: string; orchestratorId: string }> = [];
+  let inFlight = false;
   for (const agent of targets) {
     const current = agent.agentId ? getAgent(agent.agentId) : null;
     const agentId = current?.meta?.spawnRequestId === agent.spawnRequestId ? agent.agentId : undefined;
@@ -289,6 +302,7 @@ async function reapAgents(run: WorkflowRunRecord, effect: WorkflowEffect, target
       if (stopId && command.type === "agent.spawn" && command.status === "canceled") command = (await shutdownAgent(stop, WORKFLOW_CTX)).command;
       if (stopId && (command.status !== "succeeded" || command.result?.processTreeStopped !== true)) {
         retry.push({ spawnRequestId: agent.spawnRequestId, reason: `shutdown ${command.id}: ${command.status}` });
+        if (!isTerminalCommandStatus(command.status)) inFlight = true;
         continue;
       }
       reaped.push({ spawnRequestId: agent.spawnRequestId, ...(agent.agentId ? { agentId: agent.agentId } : {}) });
@@ -299,7 +313,7 @@ async function reapAgents(run: WorkflowRunRecord, effect: WorkflowEffect, target
       else retry.push(entry);
     }
   }
-  return { reaped, ...(kept.length ? { kept } : {}), ...(retry.length ? { retry } : {}), ...(deferred.length ? { deferred } : {}) };
+  return { reaped, ...(kept.length ? { kept } : {}), ...(retry.length ? { retry } : {}), ...(deferred.length ? { deferred } : {}), ...(inFlight ? { inFlight } : {}) };
 }
 
 /** What a timed-out wait was waiting for, in words. */
@@ -360,7 +374,7 @@ export async function performWorkflowEffect(run: WorkflowRunRecord, effect: Work
   if (run.cancel && ["dispatch", "findings", "clear-hold", "hold", "close"].includes(effect.kind)) return { stale: true };
   switch (effect.kind) {
     case "decision": return performWorkflowDecision(run, effect, call);
-    case "withdraw-decision": return performWithdrawWorkflowDecision(effect);
+    case "withdraw-decision": return (await performWithdrawWorkflowDecision(effect));
     case "cancel-children": throw new ValidationError("tree cancellation is owned by the supervisor");
     case "child": throw new ValidationError("child effects are owned by the workflow supervisor");
     case "close": return closeWorkflowTask(run, effect);
@@ -368,7 +382,7 @@ export async function performWorkflowEffect(run: WorkflowRunRecord, effect: Work
     case "started": {
       const quietFor = await planQuietingRunNotice(run.taskId, "started");
       if (quietFor) return { quiet: "plan-dispatched", planId: quietFor };
-      const message = notifySystemMessage(await workflowCoordinatorTarget(run), {
+      const message = await notifySystemMessage(await workflowCoordinatorTarget(run), {
         subject: `${runLabel(run)} started`,
         body: `${runLabel(run)} started at step ${run.definition.start}. You hear back only on a spawn failure, an escalation or the result; relay_workflow_get shows the run and its agents.`,
         idempotencyKey: key,
@@ -379,7 +393,7 @@ export async function performWorkflowEffect(run: WorkflowRunRecord, effect: Work
     }
     case "landed": return { superseded: "the CI verdict owns the coordinator notice" };
     case "pause-notice": {
-      const message = notifySystemMessage(await workflowCoordinatorTarget(run), { ...workflowPauseNotice(runLabel(run), run, effect), idempotencyKey: key, replyExpected: false });
+      const message = await notifySystemMessage(await workflowCoordinatorTarget(run), { ...workflowPauseNotice(runLabel(run), run, effect), idempotencyKey: key, replyExpected: false });
       return { messageId: message.id };
     }
     case "reap":
@@ -395,9 +409,9 @@ export async function performWorkflowEffect(run: WorkflowRunRecord, effect: Work
       } else if (typeof effect.payload.taskId === "number") await holdTask(run, "routes its review");
       if (typeof effect.payload.taskId === "number") {
         const taskId = effect.payload.taskId;
-        const builder = resume ? await liveBuilder(taskId) : undefined;
+        const builder = resume ? await liveBuilder({ id: run.id, taskId }, call) : undefined;
         if (builder?.live) {
-          const message = notifySystemMessage(builder.id!, {
+          const message = await notifySystemMessage(builder.id!, {
             subject: `${runLabel(run)} resumed at ${String(effect.payload.step)}`,
             body: brief!,
             idempotencyKey: key,
@@ -413,9 +427,9 @@ export async function performWorkflowEffect(run: WorkflowRunRecord, effect: Work
       return dispatchRole(run, effect, agents, await createStepTask(run, effect), role, brief, call);
     }
     case "findings": {
-      const builder = await warmBuilder(run.taskId);
+      const builder = await warmBuilder(run, call);
       const head = str(effect.payload.head);
-      const message = notifySystemMessage(builder, {
+      const message = await notifySystemMessage(builder, {
         subject: `${runLabel(run)}: ${String(effect.payload.step)} returned NO-GO on ${short(head)}`,
         body: [
           `${String(effect.payload.step)} (T${String(effect.payload.taskId)}) returned NO-GO on head ${head ?? "(unknown)"}. Round ${String(effect.payload.round)} starts now.`,
@@ -438,7 +452,7 @@ export async function performWorkflowEffect(run: WorkflowRunRecord, effect: Work
       const head = str(effect.payload.head);
       const ensured = await ensureWorkflowLanding(run, effect);
       if (ensured.outcome === "REFUSED" && ensured.held === true) {
-        const message = notifySystemMessage(await workflowCoordinatorTarget(run), {
+        const message = await notifySystemMessage(await workflowCoordinatorTarget(run), {
           subject: `${runLabel(run)}: GO, awaiting your review hold release`,
           body: `${runLabel(run)}: review returned GO on ${head ?? "an unknown head"}. Your review hold stays set; nothing is submitted for landing. After your review, clear it with relay_task_set_review_hold (taskId: ${run.taskId}, held: false, expectedHeadSha: ${head ?? "the held head"}). The run waits for the land.\n${workflowNotice(run, [])}`,
           idempotencyKey: key,
@@ -457,7 +471,7 @@ export async function performWorkflowEffect(run: WorkflowRunRecord, effect: Work
       if (effect.payload.reason === "ci-red" || effect.payload.reason === "ci-late") {
         const sha = str(effect.payload.landedSha) ?? run.landed?.sha ?? "";
         const proof = await workflowLandVerification(run, sha);
-        const message = notifySystemMessage(await workflowCoordinatorTarget(run), {
+        const message = await notifySystemMessage(await workflowCoordinatorTarget(run), {
           subject: `${runLabel(run)} ${proof.verified ? "landed" : "land verification FAILED"}, CI on main ${effect.payload.reason === "ci-red" ? "red" : "late"}`,
           body: workflowNotice(run, [proof.verified ? `T${run.taskId} landed as ${sha}. ${proof.line}` : proof.line, ...(effect.payload.reason === "ci-red" ? ciRedLead(runLabel(run), effect.payload) : [ciResultLine(effect.payload.ci as WorkflowCiVerdict)!])]),
           idempotencyKey: `workflow:landed:${run.id}:${sha}:${effect.waitKey ?? effect.id}`,
@@ -471,7 +485,7 @@ export async function performWorkflowEffect(run: WorkflowRunRecord, effect: Work
       const lead = effect.payload.reason === "run-overdue" ? await workflowOverdueLead(run, effect.payload) : escalationLead(run, effect.payload);
       if ((effect.payload.reason === "step-timeout" || effect.payload.reason === "run-overdue")
         && isRecord(effect.payload.waitingFor) && effect.payload.waitingFor.kind === "landed") lead.push(await workflowLandNotice(run));
-      const message = notifySystemMessage(await workflowCoordinatorTarget(run), {
+      const message = await notifySystemMessage(await workflowCoordinatorTarget(run), {
         subject: `${runLabel(run)} ${run.status === "failed" ? "failed" : "escalated"}`,
         body: workflowNotice(run, [...(run.status === "failed" ? [`Script failed: ${String(effect.payload.reason)}`] : lead), `Outcome: ${effect.payload.alert ? "still waiting" : run.status === "failed" ? "failed" : "escalated"}. Landed: ${run.landed?.sha ?? "none"}. Reason: ${String(effect.payload.reason)}.`, ciResultLine(isRecord(effect.payload.ci) ? effect.payload.ci as WorkflowCiVerdict : undefined) ?? "CI on main: pending or not configured."]),
         idempotencyKey: key,
@@ -507,7 +521,7 @@ export async function performWorkflowEffect(run: WorkflowRunRecord, effect: Work
       const nothingToLand = effect.payload.nothingToLand === true || child?.output?.nothingToLand === true;
       const rounds = typeof effect.payload.rounds === "number" ? effect.payload.rounds : child?.output?.rounds ?? child?.round;
       const outcome = !sha ? "finished its script steps" : !proof?.verified ? `has an unverified land receipt for ${sha}` : nothingToLand ? `had nothing to land: its reviewed head is already on main as ${sha}` : `landed as ${sha}`;
-      const message = notifySystemMessage(await workflowCoordinatorTarget(run), {
+      const message = await notifySystemMessage(await workflowCoordinatorTarget(run), {
         subject: `${runLabel(run)} ${proof?.verified === false ? "land verification FAILED" : "done"}${nothingToLand ? ", nothing to land" : ""}${ci?.status === "fail" ? ", CI on main red" : ""}`,
         body: workflowNotice(run, [`${runLabel(run)} is done: T${run.taskId} ${outcome}${rounds === undefined ? "" : ` after ${rounds} round(s)`}.${proof ? ` ${proof.line}` : ""}`, ciLine ?? (child || !run.rootRunId ? "CI on main: not configured." : "CI on main: unknown.")], children.items, !!children.cursor),
         idempotencyKey: key,

package/src/services/workflow-pins.ts

@@ -1,7 +1,7 @@
 // #3121 — per-step pins on a workflow run: the wire parser every surface shares, and the start check that
 // resolves each step's launch (pin, role file and dispatch policy) through the spawn's own seam and refuses a
 // launch that cannot run before anything spawns. Precedence (call pin > YAML step) is the kernel's workflowStepPin.
-import { cleanString, isRecord, isSpawnProvider, SPAWN_PROVIDERS, stringValue, type SpawnProvider, type TaskDispatch } from "@humwick/sdk";
+import { cleanString, isRecord, isSpawnProvider, SPAWN_PROVIDERS, stringValue, type SpawnProvider, type TaskDispatch, type WorkflowRunPins } from "@humwick/sdk";
 import { findProviderCatalogModel, type ProviderCatalogEntry } from "@humwick/sdk/provider-catalog";
 import {
   stampWorkflowPins, WORKFLOW_PIN_FIELDS, WorkflowDefinitionError, workflowStepPin,
@@ -15,11 +15,12 @@ import { sameHostIdentity } from "../utils";
 import { roleSpawnableProviders } from "./spawn-project-role";
 import { resolveSpawnLaunch, roleLaunchDefaults } from "./spawn-launch";
 import { ServiceAuthError } from "./errors";
+import { PinnedHostCapacityError } from "../spawn-admission";
 import type { ProjectRole } from "../project-roles";
 import type { AgentCard } from "../types";
 import type { AuthContext } from "./auth-context";
 
-export type WorkflowPins = Record<string, WorkflowPin>;
+export type WorkflowPins = WorkflowRunPins;
 
 export const PIN_SOURCE_LABEL = { start: "start pin", resume: "resume pin", team: "team default", global: "global default", workflow: "workflow file" } as const;
 
@@ -118,12 +119,14 @@ export async function assertStepLaunches(check: StepLaunchCheck, ctx: AuthContex
     const from = declared.crossProvider;
     const avoid = from && !pin?.provider ? check.avoided?.(from) ?? launched.get(from) : undefined;
     const defaults = roleLaunchDefaults(role);
-    const launch = await resolveSpawnLaunch({
+    const input = {
       profile: defaults.profile, spawnCap: { callerAgentId: ctx.callerAgentId },
       provider: (pin?.provider as SpawnProvider | undefined) ?? defaults.provider, dispatchPin: { model: pin?.model, effort: pin?.effort },
       dispatchShape: stepTaskDispatch(check.inputTask, declared) ?? defaults.dispatchShape, orchestratorId: pin?.orchestratorId, ...(avoid ? { avoidProviders: [avoid] } : {}), projectRole: role,
       pinSources: pin?.from,
-    }).catch((error: Error) => {
+    };
+    const launch = await resolveSpawnLaunch(input).catch((error: Error) => {
+      if (error instanceof PinnedHostCapacityError) return resolveSpawnLaunch({ ...input, spawnCap: { ...input.spawnCap, attach: true } });
       throw new ValidationError(`${where}: ${error.message}`);
     });
     const runnable = roleSpawnableProviders(pin?.orchestratorId, role, launch.catalog);

package/src/services/workflow-reconcile.ts

@@ -48,6 +48,7 @@ async function alertStalled(sup: Supervisor, run: WorkflowRunRecord, wait: Await
 
 /** A wait past its deadline alerts the coordinator once and keeps waiting, like a stopped land (#3045 H1). */
 async function alertTimeout(sup: Supervisor, run: WorkflowRunRecord, wait: Awaiting, taskId: number | undefined): Promise<boolean> {
+  if (wait.slot) return false;
   const timeoutMs = wait.deadline !== undefined ? wait.deadline - wait.since : waitTimeoutMs(run.definition, run.step);
   const now = workflowRunClock(run, supervisorNow());
   if (wait.timedOutAt !== undefined || now < wait.since + timeoutMs) return false;

package/src/services/workflow-resume-pins.ts

@@ -4,16 +4,16 @@ import type { WorkflowAgent, WorkflowRunRecord } from "@humwick/kernel/workflow.
 import { getAgent, getTaskDetailAsync, ValidationError } from "../db";
 import type { ProjectRole } from "../project-roles";
 import { resolveSpawnProjectRole } from "./spawn-project-role";
-import { liveBuilder, stepAgentProvider } from "./workflow-effects";
+import { liveBuilder, stepAgentProvider, type WorkflowOp } from "./workflow-effects";
 import { assertStepLaunches, unmetPinFields } from "./workflow-pins";
 import type { AuthContext } from "./auth-context";
 
 /** A resume's new pins: a live warm builder must already meet its step's pin, since the run hands it findings
  *  instead of spawning and replacing it would take its workspace; and each named step's launch must resolve and run. */
-export async function assertResumePins(run: WorkflowRunRecord, pins: Record<string, WorkflowCallPin>, task: NonNullable<Awaited<ReturnType<typeof getTaskDetailAsync>>>, ctx: AuthContext, agents: readonly WorkflowAgent[]): Promise<void> {
+export async function assertResumePins(run: WorkflowRunRecord, pins: Record<string, WorkflowCallPin>, task: NonNullable<Awaited<ReturnType<typeof getTaskDetailAsync>>>, ctx: AuthContext, agents: readonly WorkflowAgent[], call: WorkflowOp): Promise<void> {
   const merged = { ...run.pins, ...pins };
   const steps = Object.keys(run.definition.steps).filter((step) => pins[step]);
-  const builder = await liveBuilder(run.taskId);
+  const builder = await liveBuilder(run, call);
   const agent = builder.live ? getAgent(builder.id!) : null;
   for (const step of steps.filter((name) => run.definition.steps[name]!.task === "input")) {
     const pin = workflowStepPin(run.definition, merged, step, run.routingDefaults);

package/src/services/workflow-routing-defaults.ts

@@ -0,0 +1,18 @@
+import type { StepRoutingDefaults, TaskDetail } from "@humwick/sdk";
+import type { WorkflowRoutingDefaults } from "@humwick/kernel/workflow-definition";
+import { getAgent, getTeam } from "../db";
+import { taskOwnerRoleTeamId } from "../db/task-owner";
+
+/** #3829 — defaults follow the Task's durable owner, then its nearest configured ancestor. */
+export function resolveWorkflowRoutingDefaults(defaults: StepRoutingDefaults | undefined, task: Pick<TaskDetail, "owner" | "teamId" | "createdBy">, callerAgentId?: string): WorkflowRoutingDefaults | undefined {
+  if (!defaults) return undefined;
+  const coordinatorId = callerAgentId ?? task.createdBy;
+  const teamId = taskOwnerRoleTeamId(task.owner) ?? task.teamId ?? (coordinatorId ? getAgent(coordinatorId)?.teamId : undefined);
+  const visited = new Set<string>();
+  for (let current = teamId; current && !visited.has(current); current = getTeam(current)?.parentTeamId) {
+    visited.add(current);
+    const team = defaults.teams?.[current];
+    if (team) return { global: defaults.global, team, teamId: current };
+  }
+  return { global: defaults.global, ...(teamId ? { teamId } : {}) };
+}

package/src/services/workflow-run-view.ts

@@ -6,9 +6,9 @@ import { workflowText } from "./workflow-notice";
 
 type Awaiting = NonNullable<WorkflowRunRecord["awaiting"]>;
 
-function awaitingView({ kind, taskId, sha, deadline, timedOutAt }: Awaiting): NonNullable<WorkflowRunView["awaiting"]> {
+function awaitingView({ kind, taskId, sha, deadline, timedOutAt, slot }: Awaiting): NonNullable<WorkflowRunView["awaiting"]> {
   return {
-    kind, ...(taskId !== undefined ? { taskId } : {}), ...(sha ? { sha } : {}), ...(deadline !== undefined ? { deadline } : {}), ...(timedOutAt !== undefined ? { timedOutAt } : {}),
+    kind, ...(taskId !== undefined ? { taskId } : {}), ...(sha ? { sha } : {}), ...(deadline !== undefined ? { deadline } : {}), ...(timedOutAt !== undefined ? { timedOutAt } : {}), ...(slot ? { slot } : {}),
   };
 }
 
@@ -23,7 +23,7 @@ export function runView(run: WorkflowRunRecord): WorkflowRunView {
     /** The declared steps in order; `land` follows the last. */
     steps: Object.entries(run.definition.steps).map(([name, step]) => {
       const pin = workflowStepPin(run.definition, run.pins, name, run.routingDefaults);
-      return { name, role: step.role, ...(step.gate ? { gate: step.gate } : {}), ...(step.crossProvider ? { crossProvider: step.crossProvider } : {}), ...(pin ? { pin } : {}) };
+      return { name, role: step.role, started: run.entered.includes(name), ...(step.gate ? { gate: step.gate } : {}), ...(step.crossProvider ? { crossProvider: step.crossProvider } : {}), ...(pin ? { pin } : {}) };
     }),
     ...(run.definition.ciMain ? { ciMain: { ...(run.definition.ciMain.workflow ? { workflow: run.definition.ciMain.workflow } : {}) } } : {}),
     ref: run.ref,
@@ -43,6 +43,7 @@ export function runView(run: WorkflowRunRecord): WorkflowRunView {
     ...(run.pause ? { paused: run.pause } : {}),
     ...(run.reason ? { reason: run.reason } : {}),
     ...(run.resumes?.length ? { resumes: run.resumes } : {}),
+    ...(run.pinChanges?.length ? { pinChanges: run.pinChanges } : {}),
     ...(run.output ? { output: run.output } : {}),
     createdAt: run.createdAt,
     updatedAt: run.updatedAt,

package/src/services/workflow-run.test-fixture.ts

@@ -90,7 +90,7 @@ export function installWorkflowHost(options: { confirmStops?: boolean } = {}): (
 export async function initWorkflowDb(): Promise<void> {
   await stopWorkflowServingForTests();
   await settleLandLifecycleDrives();
-  flushTaskStatusPulses();
+  await flushTaskStatusPulses();
   initDb(":memory:");
 }