@humwick/server 0.130.1 → 0.130.3

package/src/services/workflow-auto-start.ts

@@ -1,14 +1,17 @@
 // #3091 — workflows by default: a builder spawn for a Task that lands code starts the project's build-review
 // workflow instead, and its run spawns the builder. One core behind relay_spawn_agent, relay_task_dispatch and
 // POST /api/agents/spawn; `workflow: false` keeps the manual path. review, research and ops Tasks never start one.
+// #3736 — a project spawn-policy rule matching the Task names the workflow instead, whatever the Task's type.
 import { WORKFLOW_AUTO_START_NAME, workflowAutoStarts, type WorkflowRunAgent, type WorkflowRunView } from "@humwick/sdk";
-import { getTaskDetailAsync, ValidationError } from "../db";
+import { SPAWN_POLICY_FILE } from "@humwick/sdk/spawn-policy";
+import { getAgent, getTaskDetailAsync, resolveProjectForCwd, ValidationError } from "../db";
 import { getAgentProfile } from "../config-store";
 import { McpAuthError, McpNotFoundError } from "../mcp-errors";
 import { ServiceAuthError } from "./errors";
 import { getWorkflowRun, launchWorkflowRun, prepareWorkflowRun, type PreparedWorkflowRun } from "./workflow-runs";
 import type { WorkflowPins } from "./workflow-pins";
 import type { AuthContext } from "./auth-context";
+import { spawnerKind, spawnPolicyTask, spawnPolicyWorkflowFor } from "./spawn-policy";
 
 /** A builder spawn as its transport received it: the Task, the opt-out, and every other field the caller set (wire names). */
 export interface BuilderSpawn {
@@ -58,18 +61,27 @@ function refused(fields: Record<string, unknown>): string[] {
 export async function workflowForBuilderSpawn(spawn: BuilderSpawn, ctx: AuthContext): Promise<BuilderSpawnOutcome> {
   if (spawn.workflow === false || spawn.taskId === undefined) return { started: false };
   const task = await getTaskDetailAsync(spawn.taskId);
-  const type = task?.kind === "deliverable" ? task.dispatch?.type : undefined;
-  if (!task || !workflowAutoStarts(type)) return { started: false };
-  const subject = `T${task.id} is a ${type} Task, so this spawn starts workflow ${WORKFLOW_AUTO_START_NAME}, whose run spawns the builder`;
-  const optOut = "pass workflow:false to spawn by hand";
+  if (!task || task.kind !== "deliverable") return { started: false };
+  const type = task.dispatch?.type;
+  const { cwd, orchestratorId } = spawn.fields;
+  const caller = ctx.callerAgentId ? getAgent(ctx.callerAgentId) ?? undefined : undefined;
+  const checkout = typeof cwd === "string" && cwd ? { cwd, ...(typeof orchestratorId === "string" && orchestratorId ? { orchestratorId } : {}) } : undefined;
+  const policy = await spawnPolicyWorkflowFor(
+    { projectId: task.projectId ?? (checkout ? resolveProjectForCwd(checkout.cwd)?.id : undefined), ...(checkout ? { checkout } : {}) },
+    { spawner: spawnerKind(ctx, {}, caller), task: spawnPolicyTask(task)! },
+  );
+  if (!policy && !workflowAutoStarts(type)) return { started: false };
+  const workflow = policy?.workflow ?? WORKFLOW_AUTO_START_NAME;
+  const why = policy ? `T${task.id} matches ${SPAWN_POLICY_FILE} rule "${policy.rule}"` : `T${task.id} is a ${type} Task`;
+  const subject = `${why}, so this spawn starts workflow ${workflow}, whose run spawns the builder`;
+  const optOut = policy ? "the project's spawn policy runs this Task only through that workflow" : "pass workflow:false to spawn by hand";
   if (!taskRepos(task).length) return { started: false, reason: `T${task.id} names no repo, so no workflow can run on it` };
   const conflicts = refused(spawn.fields);
-  if (conflicts.length) throw new ValidationError(`${subject}; the run cannot honor ${conflicts.map((key) => `${key}=${JSON.stringify(spawn.fields[key])}`).join(", ")}. Drop ${conflicts.length > 1 ? "them" : "it"}, or ${optOut}`);
+  if (conflicts.length) throw new ValidationError(`${subject}; the run cannot honor ${conflicts.map((key) => `${key}=${JSON.stringify(spawn.fields[key])}`).join(", ")}. Drop ${conflicts.length > 1 ? "them" : "it"}${policy ? "" : `, or ${optOut}`}`);
   try {
     // Located as the spawn itself would be: its cwd and host, else the Task's repo root.
-    const { cwd, orchestratorId } = spawn.fields;
     const prepared = await prepareWorkflowRun({
-      workflow: WORKFLOW_AUTO_START_NAME, taskId: task.id, ...(typeof cwd === "string" && cwd ? { cwd } : {}), ...(typeof orchestratorId === "string" && orchestratorId ? { orchestratorId } : {}),
+      workflow, taskId: task.id, ...(typeof cwd === "string" && cwd ? { cwd } : {}), ...(typeof orchestratorId === "string" && orchestratorId ? { orchestratorId } : {}),
     }, ctx);
     const inputSteps = Object.entries(prepared.read.workflow.definition.steps).filter(([, step]) => step.task === "input");
     const role = spawn.fields.role;
@@ -80,7 +92,7 @@ export async function workflowForBuilderSpawn(spawn: BuilderSpawn, ctx: AuthCont
     const steer = BRIEF_FIELDS.map((key) => spawn.fields[key]).filter((value): value is string => typeof value === "string" && Boolean(value.trim())).join("\n\n");
     // The call answers for the builder's spawn: one the run cannot dispatch fails it, as a spawn by hand would (#3160).
     const started = await launchWorkflowRun(prepared, { ...(pins ? { pins } : {}), ...(steer ? { steer } : {}), refuseUndispatched: true }, ctx) as WorkflowRunView & { existing?: true; warnings?: string[] };
-    return { started: true, receipt: await receiptFor(prepared, started, set(spawn.fields).filter((key) => UNUSED_FIELDS.includes(key)), ctx) };
+    return { started: true, receipt: await receiptFor(prepared, started, set(spawn.fields).filter((key) => UNUSED_FIELDS.includes(key)), { why, policyRule: policy?.rule }, ctx) };
   } catch (error) {
     if (error instanceof ServiceAuthError) throw new McpAuthError(`${subject}: ${error.message}; ${optOut}`);
     if (error instanceof ValidationError || error instanceof McpAuthError || error instanceof McpNotFoundError) error.message = `${subject}: ${error.message}; ${optOut}`;
@@ -99,16 +111,17 @@ function builderPin(fields: Record<string, unknown>): Record<string, string> {
 }
 
 /** The spawn's receipt: the run, and its builder as a spawn receipt names one (the newest agent of the build step). */
-async function receiptFor(prepared: PreparedWorkflowRun, started: WorkflowRunView & { existing?: true; warnings?: string[] }, ignored: string[], ctx: AuthContext): Promise<Record<string, unknown>> {
+async function receiptFor(prepared: PreparedWorkflowRun, started: WorkflowRunView & { existing?: true; warnings?: string[] }, ignored: string[], cause: { why: string; policyRule?: string }, ctx: AuthContext): Promise<Record<string, unknown>> {
   const { task, read } = prepared;
   const run = await getWorkflowRun(started.runId, ctx, { verbose: true }) as WorkflowRunView & { agents: WorkflowRunAgent[] };
   const step = run.steps.find((entry) => read.workflow.definition.steps[entry.name]?.task === "input")!;
   const agent = run.agents.filter((entry) => entry.step === step.name).at(-1);
   const profile = step.pin?.profile ?? read.workflow.roles.get(step.role)?.extends;
   const label = `workflow ${run.workflow} (${run.source ?? "project"}) run ${run.runId}`;
+  const progress = run.awaiting?.slot ? `is waiting for a slot on ${run.awaiting.slot.host} before spawning` : agent ? "spawned" : "is spawning";
   const note = started.existing
-    ? `T${task.id} already runs ${label}, which owns its builder: no builder was spawned. relay_workflow_get shows its agents; pass workflow:false to spawn beside it.`
-    : `T${task.id} is a ${task.dispatch?.type} Task, so Relay started ${label}; its run ${agent ? "spawned" : "is spawning"} the builder (role ${step.role} on profile ${profile}). You hear back on a step spawn failure, an escalation or the result. Pass workflow:false to spawn by hand.`;
+    ? `T${task.id} already runs ${label}, which owns its builder: no builder was spawned. relay_workflow_get shows its agents${cause.policyRule ? "" : "; pass workflow:false to spawn beside it"}.`
+    : `${cause.why}, so Relay started ${label}; its run ${progress} the builder (role ${step.role} on profile ${profile}). You hear back on a step spawn failure, an escalation or the result.${cause.policyRule ? "" : " Pass workflow:false to spawn by hand."}`;
   const warnings = [...(started.warnings ?? []), ...(ignored.length ? [`the run sets these for its own agents, so ${ignored.join(", ")} went unused`] : [])];
   return {
     ok: true,
@@ -116,7 +129,7 @@ async function receiptFor(prepared: PreparedWorkflowRun, started: WorkflowRunVie
     taskId: task.id,
     workflowRun: {
       runId: run.runId, workflow: run.workflow, ...(run.source ? { source: run.source } : {}), sha: run.sha, status: run.status, step: run.step,
-      taskType: task.dispatch?.type, ...(started.existing ? { existing: true } : {}),
+      taskType: task.dispatch?.type, ...(cause.policyRule ? { spawnPolicyRule: cause.policyRule } : {}), ...(started.existing ? { existing: true } : {}),
       builder: { step: step.name, role: step.role, ...(profile ? { profile } : {}), ...(step.pin ? { pin: step.pin } : {}) },
     },
     ...(agent ? { spawnRequestId: agent.spawnRequestId, agentId: agent.agentId ?? null, registered: Boolean(agent.agentId), ...(agent.provider ? { provider: agent.provider } : {}) } : {}),

package/src/services/workflow-caller.ts

@@ -1,8 +1,9 @@
 import { stringValue } from "@humwick/sdk";
 import { childWorkflowsEnabled } from "../config";
 import { getAgentProfile } from "../config-store";
-import { getAgent, ValidationError } from "../db";
-import type { TaskDetail } from "../types";
+import { activeTeamCoordinatorId, getAgent, ValidationError } from "../db";
+import { taskOwnerRoleTeamId } from "../db/task-owner";
+import type { AgentCard, TaskDetail } from "../types";
 import { ServiceAuthError } from "./errors";
 import type { AuthContext } from "./auth-context";
 import { canSpawn } from "./spawn-agent";
@@ -15,14 +16,70 @@ export function assertWorkflowCallerLive(agentId: string): void {
   if (workflowCallerGone(agentId)) throw new ValidationError("WORKFLOW_CALLER_GONE: caller is canceled or reaped");
 }
 
+type WorkflowTaskSubject = Pick<TaskDetail, "createdBy" | "ownerAgentId" | "teamId" | "owner">;
+
+/** An agent seated as its team's coordinator: it owns work and runs, it never builds a run's Task. */
+export function isCoordinatorSeat(agent: Pick<AgentCard, "id" | "teamId"> | null | undefined): boolean {
+  return Boolean(agent?.teamId && activeTeamCoordinatorId(agent.teamId) === agent.id);
+}
+
+/** A spawned coordinator seat: a delegated sub-coordinator, never a worker it runs. */
+function delegatedSeat(agentId: string | null | undefined): AgentCard | null {
+  const agent = agentId ? getAgent(agentId) : null;
+  return agent?.spawnedBy && isCoordinatorSeat(agent) ? agent : null;
+}
+
+/** Who may own a Task's runs (#3769): its assigned agent, its team's seat, its owner role's seat, its creator. */
+function taskOwnerCandidates(task: WorkflowTaskSubject): Array<string | null | undefined> {
+  return [task.ownerAgentId, activeTeamCoordinatorId(task.teamId), activeTeamCoordinatorId(taskOwnerRoleTeamId(task.owner)), task.createdBy];
+}
+
+/** #3769 — the delegated sub-coordinator that owns a Task: the first candidate holding its private team's seat. */
+export function delegatedTaskOwner(task: WorkflowTaskSubject): AgentCard | null {
+  for (const id of taskOwnerCandidates(task)) {
+    const seat = delegatedSeat(id);
+    if (seat) return seat;
+  }
+  return null;
+}
+
+/** #3818 — a Task its parent coordinator created (or holds the owner seat of) is the sub-coordinator's queue, no hand-over needed. */
+function parentQueuedTask(seat: AgentCard, task: WorkflowTaskSubject): boolean {
+  const parent = seat.spawnedBy;
+  return Boolean(parent && (task.createdBy === parent || activeTeamCoordinatorId(taskOwnerRoleTeamId(task.owner)) === parent));
+}
+
+/** THE workflow-control predicate (start, read, resume, pause, cancel, budget): the Task's coordinators and admins, and
+ *  a delegated sub-coordinator that created it, is assigned it, holds its team's seat or was spawned by its creator.
+ *  A worker is never authorized. */
+export function canControlWorkflowTask(ctx: AuthContext, task: WorkflowTaskSubject): boolean {
+  if (canControlTaskReviewHold(ctx, task)) return true;
+  const caller = ctx.callerAgentId;
+  const seat = delegatedSeat(caller);
+  return Boolean(seat && (taskOwnerCandidates(task).includes(seat.id) || parentQueuedTask(seat, task)));
+}
+
 /** Workflow authority delegates execution to Relay, never the caller's direct-spawn capability. */
 export function admitWorkflowCaller(ctx: AuthContext, task: TaskDetail, workflow?: { name: string; source: string }) {
   const agent = ctx.callerAgentId ? getAgent(ctx.callerAgentId) : null;
   if (!agent || (!agent.spawnedBy && canSpawn(ctx))) return undefined;
   if (!childWorkflowsEnabled()) throw new ServiceAuthError("WORKFLOW_DISABLED: child workflows are disabled");
   assertWorkflowCallerLive(agent.id);
-  if (task.ownerAgentId !== agent.id && task.createdBy !== agent.id && !canControlTaskReviewHold(ctx, task)) throw new ServiceAuthError("WORKFLOW_TASK_FORBIDDEN: caller must own or create the Task");
-  const profile = getAgentProfile(ctx.constraints?.profile ?? stringValue(agent.meta?.profile) ?? "worker")?.value;
+  if (task.ownerAgentId !== agent.id && task.createdBy !== agent.id && !canControlWorkflowTask(ctx, task)) throw new ServiceAuthError("WORKFLOW_TASK_FORBIDDEN: caller must own or create the Task, or be a sub-coordinator of the coordinator that created it (hand one over with relay_task_assign_agent)");
+  return workflowCallerPolicy(agent, ctx.constraints?.profile, workflow);
+}
+
+/** #3769 — a run a Task coordinator starts on a Task a live delegated sub-coordinator owns is that owner's run:
+ *  its spawn lineage, its quota and its notices, exactly as if the owner had started it. */
+export function admitWorkflowTaskOwner(task: TaskDetail, workflow?: { name: string; source: string }) {
+  const owner = delegatedTaskOwner(task);
+  if (!owner || workflowCallerGone(owner.id)) return undefined;
+  if (!childWorkflowsEnabled()) throw new ServiceAuthError(`WORKFLOW_DISABLED: T${task.id} belongs to sub-coordinator ${owner.id}, and child workflows are disabled`);
+  return workflowCallerPolicy(owner, undefined, workflow);
+}
+
+function workflowCallerPolicy(agent: AgentCard, profileName: string | undefined, workflow?: { name: string; source: string }) {
+  const profile = getAgentProfile(profileName ?? stringValue(agent.meta?.profile) ?? "worker")?.value;
   const policy = profile?.workflows;
   const maxConcurrentRuns = policy?.maxConcurrentRuns ?? 1;
   const maxLiveAgents = policy?.maxLiveAgents ?? 2;

package/src/services/workflow-child-proof-worker.ts

@@ -43,7 +43,7 @@ if (phase === "crash") {
   setWorkflowSupervisorTestOptions({ fault(label) {
     if (label === "after-child-complete") process.kill(process.pid, "SIGKILL");
   } });
-  announceBranchLanded({ workspace: { id: WF.workspace, repoRoot: repo.cwd, branch: WF.branch, baseRef: "origin/main", ownerAgentId: WF.builder, orchestratorId: WF.orchestrator }, sourceHeadSha: sha, mergedSha: sha, pushed: true, taskIds: [task.id] });
+  await announceBranchLanded({ workspace: { id: WF.workspace, repoRoot: repo.cwd, branch: WF.branch, baseRef: "origin/main", ownerAgentId: WF.builder, orchestratorId: WF.orchestrator }, sourceHeadSha: sha, mergedSha: sha, pushed: true, taskIds: [task.id] });
   await workflowTaskDelivered();
   await settleChildProofStops();
   throw new Error("restart seam was not reached");

package/src/services/workflow-ci-main.ts

@@ -1,4 +1,4 @@
-import { correlatedCiIncidents } from "./plan-ci-recovery";
+import { correlatedCiIncidents } from "./plan-ci-notice";
 // #3045 H5 — the `ci-main` step after the land: CI on main for the exact landed commit. The CI monitor's
 // existing tick observes every commit a run waits on (ci_sha_observations); this module only reads that
 // evidence. Red alerts the coordinator once while the run keeps waiting; green on every watch ends the run

package/src/services/workflow-control-proof-worker.ts

@@ -76,7 +76,7 @@ async function build(node: Node) {
   git(node.cwd!, "push", "-q", "origin", "HEAD:main");
   node.sha = sha; save();
   upsertWorkspace({ ...getWorkspace(node.workspace!)!, landedAt: Date.now(), landedSha: sha });
-  announceBranchLanded({ workspace: { id: node.workspace!, repoRoot: node.cwd!, branch: node.branch!, baseRef: "main", ownerAgentId: node.agent!, orchestratorId: WF.orchestrator }, sourceHeadSha: sha, mergedSha: sha, pushed: true, taskIds: [node.taskId] });
+  await announceBranchLanded({ workspace: { id: node.workspace!, repoRoot: node.cwd!, branch: node.branch!, baseRef: "main", ownerAgentId: node.agent!, orchestratorId: WF.orchestrator }, sourceHeadSha: sha, mergedSha: sha, pushed: true, taskIds: [node.taskId] });
   await workflowTaskDelivered(); await settleChildProofStops(); await settleLandLifecycleDrives();
   meta.events.push({ kind: "land", id: node.id }); save();
 }

package/src/services/workflow-coordinator.ts

@@ -1,8 +1,14 @@
 import type { WorkflowRunRecord } from "@humwick/kernel/workflow.pilot";
 import { getTaskDetailAsync } from "../db";
 import { taskOwnerTarget } from "../task-owner";
+import { delegatedTaskOwner, workflowCallerGone } from "./workflow-caller";
 
-/** Notices and decisions share the starter, then durable Task owner, then run coordinator. */
+/** Notices and decisions share the starter, then the Task's live owning sub-coordinator (#3769), then its durable
+ *  owner, then the run coordinator: the root hears of a sub-coordinator's Task only once that owner is gone. */
 export async function workflowCoordinatorTarget(run: WorkflowRunRecord): Promise<string> {
-  return run.callerAgentId ?? taskOwnerTarget(await getTaskDetailAsync(run.taskId)) ?? run.coordinator;
+  if (run.callerAgentId) return run.callerAgentId;
+  const task = await getTaskDetailAsync(run.taskId);
+  const owner = task ? delegatedTaskOwner(task) : null;
+  if (owner && !workflowCallerGone(owner.id)) return owner.id;
+  return taskOwnerTarget(task) ?? run.coordinator;
 }

package/src/services/workflow-decisions.ts

@@ -42,7 +42,7 @@ export async function performWorkflowDecision(run: WorkflowRunRecord, effect: Wo
     workflow: { runId: run.id, step: run.step, effectId: effect.id, waitKey: effect.waitKey!, ...(multi ? { items } : {}), ...(step.outputSchema ? { outputSchema: step.outputSchema } : {}) },
   };
   if (spec.answeredBy === "coordinator") {
-    const message = notifySystemMessage(await workflowCoordinatorTarget(run), {
+    const message = await notifySystemMessage(await workflowCoordinatorTarget(run), {
       subject: `Workflow ${run.id}: ${request.title}`,
       body: [request.title, request.context, `Choices (JSON data): ${JSON.stringify(items)}`,
         `Answer with relay_workflow_resume({runId: "${run.id}", decision: {action: "answer", waitKey: "${effect.waitKey}", answers: {${multi ? 'approved: ["item-id"]' : 'choice: "option-id"'}}}, note: "reason"}).`,
@@ -63,16 +63,16 @@ export function workflowDecisionSignal(effectId: string): Record<string, unknown
   return { outcome: "complete", decisionId: decision.id, outputs: workflowDecisionAnswer(decision, decision.answers!, decision.note) };
 }
 
-export function performWithdrawWorkflowDecision(effect: WorkflowEffect): Record<string, unknown> {
-  withdrawWorkflowDecision(String(effect.payload.effectId));
+export async function performWithdrawWorkflowDecision(effect: WorkflowEffect): Promise<Record<string, unknown>> {
+  (await withdrawWorkflowDecision(String(effect.payload.effectId)));
   return { withdrawn: true };
 }
 
 async function fileOperatorDecision(run: WorkflowRunRecord, request: WorkflowDecisionRequest, note?: string): Promise<Record<string, unknown>> {
   const task = await getTaskDetailAsync(run.taskId);
-  const decision = fileWorkflowDecision({ actor: { id: run.coordinator, kind: "agent" }, callerAgentId: run.coordinator, scopes: ["*", "command:spawn"] }, {
+  const decision = (await fileWorkflowDecision({ actor: { id: run.coordinator, kind: "agent" }, callerAgentId: run.coordinator, scopes: ["*", "command:spawn"] }, {
     ...request,
-  }, request.workflow, task?.projectId, note);
+  }, request.workflow, task?.projectId, note));
   return { decisionId: decision.id };
 }
 

package/src/services/workflow-deferred-cleanup.ts

@@ -1,3 +1,4 @@
+import { workflowLandVerification } from "./workflow-land-notice";
 import type { WorkflowEffect, WorkflowRunRecord } from "@humwick/kernel/workflow.pilot";
 import { notifySystemMessage } from "../notify";
 import { workflowCoordinatorTarget } from "./workflow-coordinator";
@@ -29,10 +30,11 @@ async function resumeHosts(sup: Supervisor, orchestratorId?: string): Promise<vo
 }
 
 export async function notifyDeferredWorkflowCleanup(run: WorkflowRunRecord, effect: WorkflowEffect): Promise<void> {
-  notifySystemMessage(await workflowCoordinatorTarget(run), {
+  const proof = run.landed?.sha ? await workflowLandVerification(run, run.landed.sha) : undefined;
+  await notifySystemMessage(await workflowCoordinatorTarget(run), {
     subject: `Workflow ${run.id}: cleanup deferred`,
-    body: `Workflow ${run.id} landed as ${run.landed?.sha}. Shutdowns on offline hosts are recorded durably and will run when those hosts re-register. The run can finish with its CI result.`,
-    payload: { kind: "workflow.cleanup-deferred", runId: run.id, effectId: effect.id, deferred: effect.result?.deferred },
+    body: `Workflow ${run.id}: ${proof?.verified ? `landed as ${run.landed?.sha}. verified: true.` : proof?.line ?? "No land receipt."} Shutdowns on offline hosts are recorded durably and will run when those hosts re-register. The run can finish with its CI result.`,
+    payload: { ...(proof ? { verified: proof.verified } : {}), kind: "workflow.cleanup-deferred", runId: run.id, effectId: effect.id, deferred: effect.result?.deferred },
     idempotencyKey: `workflow:${run.id}:cleanup-deferred`,
     replyExpected: false,
     maxAgeSeconds: null,

package/src/services/workflow-effect-driver.ts

@@ -19,12 +19,14 @@ import { ServiceAuthError } from "./errors";
 
 import { performWorkflowEffect } from "./workflow-effects";
 import { workflowCleanupDeferred } from "./workflow-deferred-cleanup";
+import { PinnedHostCapacityError } from "../spawn-admission";
 
-import { invoke, live, runAgents, noteAgent, testOptions, type Supervisor } from "./workflow-supervisor-state";
+import { effectDue, invoke, retryResult, live, runAgents, noteAgent, announceChange, supervisorNow, testOptions, type Supervisor } from "./workflow-supervisor-state";
 type Json = Record<string, unknown>;
 const EFFECT_BATCH = 50;
 const MAX_EFFECT_ATTEMPTS = 3;
 const REAP_RETRY_MS = 60 * 60_000;
+const SLOT_WAIT_MS = 30 * 60_000;
 function isPermanentEffectError(error: unknown): boolean {
   return error instanceof WorkflowValueError || error instanceof ValidationError || error instanceof McpNotFoundError || error instanceof McpAuthError
     || error instanceof ServiceAuthError || error instanceof ReviewHoldConflictError;
@@ -39,21 +41,31 @@ async function settleEffect(sup: Supervisor, run: WorkflowRunRecord, effect: Wor
     run = { ...run, snapshot: root.snapshot };
   }
   try {
+    if (effect.kind === "dispatch" && !run.cancel && run.status === "running" && run.awaiting?.slot && run.awaiting.key === effect.waitKey && supervisorNow() >= run.awaiting.slot.deadline) {
+      throw new ValidationError(`Timed out waiting for a slot on ${run.awaiting.slot.host} after 30 minutes`);
+    }
     if (effect.kind === "dispatch" && run.rootRunId && !run.cancel) await invoke(sup, "reserveAgent", { runId: run.id, effectId: effect.id });
     if (effect.kind === "script" && !(await invoke<{ reserved: boolean }>(sup, "reserveScript", { runId: run.id, effectId: effect.id })).reserved) return false;
     const performed = effect.kind === "cancel-children" ? await cancelChildren(sup, run, effect) : effect.kind === "child" ? await createChild(sup, run, effect) : await performWorkflowEffect(run, effect, effect.kind === "reap" || effect.kind === "dispatch" ? await runAgents(sup, run.id) : [], (op, input) => invoke(sup, op, input));
     if (!performed && effect.kind !== "script") return false;
     result = performed ?? { pending: true };
   } catch (error) {
+    if (effect.kind === "dispatch" && error instanceof PinnedHostCapacityError) {
+      if (!run.awaiting?.slot) {
+        await invoke(sup, "waitForSlot", { id: effect.id, host: error.orchestratorId, now: supervisorNow(), timeoutMs: SLOT_WAIT_MS });
+        announceChange();
+      }
+      return false;
+    }
     const message = errMessage(error);
     if (workflowOwnsAgentCleanup(run) && ["cancel-children", "reap"].includes(effect.kind)) {
       if (run.status !== "done") await invoke(sup, "cleanup", { runId: run.id, complete: false, error: message });
-      await invoke(sup, "settleEffect", { id: effect.id, status: "pending", error: message });
+      await invoke(sup, "settleEffect", { id: effect.id, status: "pending", error: message, result: retryResult(effect.result, effect.result ?? {}, supervisorNow()) });
       return false;
     }
     const failed = isPermanentEffectError(error) || effect.attempts + 1 >= MAX_EFFECT_ATTEMPTS;
     console.warn(`workflow effect ${effect.id} ${failed ? "failed" : "will retry"}: ${message}`);
-    await invoke(sup, "settleEffect", { id: effect.id, status: failed ? "failed" : "pending", error: message });
+    await invoke(sup, "settleEffect", { id: effect.id, status: failed ? "failed" : "pending", error: message, now: supervisorNow() });
     if (failed && effect.kind === "dispatch") await releaseUnusedDispatch(sup, effect);
     if (failed && effect.kind === "script") await invoke(sup, "releaseReservation", { effectId: effect.id });
     const refusal = failed && effect.kind === "dispatch" ? sup.refusable.get(run.id) : undefined;
@@ -81,14 +93,15 @@ async function settleEffect(sup: Supervisor, run: WorkflowRunRecord, effect: Wor
   // The reap becomes a dormant deferred-cleanup record until its host returns (#3658).
   const deferred = Array.isArray(result.deferred) && result.deferred.length > 0;
   if (deferred || Array.isArray(result.retry) && result.retry.length && (workflowOwnsAgentCleanup(run) || Date.now() - effect.createdAt < REAP_RETRY_MS)) {
-    const pending = await invoke<WorkflowEffect>(sup, "settleEffect", { id: effect.id, status: deferred && !(Array.isArray(result.retry) && result.retry.length) ? "deferred" : "pending", result, error: JSON.stringify(result.retry ?? result.deferred).slice(0, 2000) });
+    const status = deferred && !(Array.isArray(result.retry) && result.retry.length) ? "deferred" : "pending";
+    const pending = await invoke<WorkflowEffect>(sup, "settleEffect", { id: effect.id, status, result: status === "pending" ? retryResult(effect.result, result, supervisorNow()) : result, error: JSON.stringify(result.retry ?? result.deferred).slice(0, 2000) });
     if (!workflowCleanupDeferred(pending) && workflowOwnsAgentCleanup(run) && run.status !== "done") await invoke(sup, "cleanup", { runId: run.id, complete: false, error: JSON.stringify(result.retry).slice(0, 2000) });
     return workflowCleanupDeferred(pending);
   }
   if (effect.kind === "reap" && run.status === "cancelled" && !run.rootRunId) await invoke(sup, "cleanup", { runId: run.id, complete: true });
   if (effect.kind === "dispatch" && !result.spawnRequestId) await releaseUnusedDispatch(sup, effect);
   if (effect.kind === "script" && result.pending !== true) await invoke(sup, "releaseReservation", { effectId: effect.id });
-  await invoke(sup, "settleEffect", { id: effect.id, status: "done", result });
+  await invoke(sup, "settleEffect", { id: effect.id, status: "done", result, now: supervisorNow() });
   return true;
 }
 
@@ -145,21 +158,33 @@ async function createChild(sup: Supervisor, run: WorkflowRunRecord, effect: Work
   return { childRunId: child.id };
 }
 
+/** Perform up to a batch of due pending effects. A backing-off effect is skipped before any read and does not
+ *  count toward the batch, so effects that never converge cannot starve newer ones (#3827). */
 export async function drainEffects(sup: Supervisor): Promise<number> {
-  const { items } = await invoke<{ items: WorkflowEffect[] }>(sup, "pendingEffects", { limit: EFFECT_BATCH });
+  const now = supervisorNow();
   let settled = 0;
-  for (const effect of items) {
-    if (live() !== sup) break;
-    try {
-      if (effect.kind === "cancel-agent-tree") {
-        if (await performSpawnTreeCleanup(sup, effect)) settled++;
-        continue;
+  let attempted = 0;
+  let cursor: string | undefined;
+  do {
+    const page = await invoke<{ items: WorkflowEffect[]; cursor?: string }>(sup, "pendingEffects", { limit: EFFECT_BATCH, ...(cursor ? { cursor } : {}) });
+    for (const effect of page.items) {
+      if (live() !== sup) return settled;
+      if (!effectDue(effect, now)) continue;
+      if (attempted++ === EFFECT_BATCH) return settled;
+      try {
+        if (effect.kind === "cancel-agent-tree") {
+          if (await performSpawnTreeCleanup(sup, effect)) settled++;
+          continue;
+        }
+        const run = await invoke<WorkflowRunRecord>(sup, "get", { runId: effect.runId });
+        // #3757 — a paused run spawns, messages, arms and reaps nothing until its unpause; only its pause notices go out.
+        if (run.pause && effect.kind !== "pause-notice") continue;
+        if (await settleEffect(sup, run, effect)) settled += 1;
+      } catch (error) {
+        console.warn(`workflow effect ${effect.id} could not be settled: ${errMessage(error)}`);
       }
-      const run = await invoke<WorkflowRunRecord>(sup, "get", { runId: effect.runId });
-      if (await settleEffect(sup, run, effect)) settled += 1;
-    } catch (error) {
-      console.warn(`workflow effect ${effect.id} could not be settled: ${errMessage(error)}`);
     }
-  }
+    cursor = page.cursor;
+  } while (cursor);
   return settled;
 }