@humwick/server 0.130.1 → 0.130.3

package/src/services/workflow-runs.ts

@@ -4,6 +4,7 @@ import { spawnTreeContains } from "../db/spawn-tree";
 import { cleanSpawnBudget } from "../spawn-budget";
 import { readWorkflowRunBudgetAsync, reconcileRunBudgetAsync } from "../token-usage-async";
 import { getRoutingPolicyEnvelope } from "../routing-policy-service";
+import { resolveWorkflowRoutingDefaults } from "./workflow-routing-defaults";
 
 // #3045 — the workflow supervisor: mounts the kernel `workflow` module, starts runs, and pumps them.
 // A pass recovers crashed drivers, signals each waiting run from durable src facts (level-triggered,
@@ -14,10 +15,10 @@ import { errMessage, fireAndForget, stringValue, type WorkflowRunAgent, type Wor
 import { validateWorkflowValue } from "@humwick/kernel/workflow-values";
 import { OpError } from "@humwick/kernel";
 
-import { admitWorkflowCaller, assertWorkflowCallerLive } from "./workflow-caller";
+import { admitWorkflowCaller, admitWorkflowTaskOwner, assertWorkflowCallerLive, canControlWorkflowTask } from "./workflow-caller";
 import { taskVerificationCheck } from "../task-verification";
 
-import { workflow, workflowEffectId, type WorkflowEffect, type WorkflowRunRecord } from "@humwick/kernel/workflow.pilot";
+import { assertWorkflowFuturePins, workflow, workflowEffectId, type WorkflowEffect, type WorkflowRunRecord } from "@humwick/kernel/workflow.pilot";
 import { getAgent, getTaskDetailAsync, ValidationError } from "../db";
 import type { WorkflowRunCursor, WorkflowRunTaskRow } from "../db/workflow-task-read-ops";
 import { getGateway } from "../db/gateway";
@@ -31,7 +32,6 @@ import { McpNotFoundError } from "../mcp-errors";
 import { ServiceAuthError } from "./errors";
 import { canSpawn, spawnLineageAgentId, taskLinkedRepoCwd } from "./spawn-agent";
 import { resolveSpawnProjectWorkflow } from "./spawn-project-role";
-import { canControlTaskReviewHold } from "./task-review-hold";
 import { updateTaskEntityAsync } from "./task-entity";
 
 import { releaseRunHold } from "./workflow-effects";
@@ -43,6 +43,7 @@ import { leanRunView, runView } from "./workflow-run-view";
 import type { AuthContext } from "./auth-context";
 
 import { invoke, live, runAgents, requestPump, serialized, ensureSupervisor, type Supervisor } from "./workflow-supervisor";
+import { supervisorNow } from "./workflow-supervisor-state";
 export { workflowAgentObligation } from "./workflow-agent-obligation";
 import { reconcileRun } from "./workflow-reconcile";
 import { holdWorkflowVerification } from "./workflow-verification";
@@ -76,19 +77,19 @@ export async function prepareWorkflowRun(input: { workflow: string; taskId: numb
   if (!task || task.kind !== "deliverable") throw new ValidationError("task not found");
   if (task.status === "canceled") throw new ValidationError(`task ${task.id} is canceled; a workflow runs on a Task that is still wanted`);
   const caller = admitWorkflowCaller(ctx, task);
-  if (!caller && !canControlTaskReviewHold(ctx, task)) {
+  if (!caller && !canControlWorkflowTask(ctx, task)) {
     throw new ServiceAuthError(`starting a workflow on task ${task.id} is coordinator-only: the run sets the review hold and clears it on GO — ${COORDINATOR_AUTHORITY_HINT}`);
   }
   if (!caller && !canSpawn(ctx)) throw new ServiceAuthError("starting a workflow dispatches its roles, so it requires the command:spawn scope");
   const lineageId = spawnLineageAgentId(ctx, task);
   const lineage = lineageId ? getAgent(lineageId) : null;
-  const cwd = input.cwd ?? await taskLinkedRepoCwd(task.id, stringValue(lineage?.meta?.cwd));
+  const cwd = input.cwd ?? await taskLinkedRepoCwd(task.id, stringValue(lineage?.meta?.cwd), lineage ?? undefined);
   const read = await resolveSpawnProjectWorkflow({ workflow: input.workflow, cwd, orchestratorId: input.orchestratorId, preferHost: lineage?.machine, allowedOrchestratorIds: ctx.constraints?.orchestrators });
   return { task, cwd, read };
 }
 
 /** relay_workflow_start: read the landed workflow and its roles, check every step's pin, start one run on the Task. */
-export async function startWorkflowRun(input: { workflow: string; taskId: number; pins?: WorkflowPins; inputs?: Json; requestKey?: string; budget?: unknown }, ctx: AuthContext): Promise<Json> {
+export async function startWorkflowRun(input: { workflow: string; taskId: number; cwd?: string; pins?: WorkflowPins; inputs?: Json; requestKey?: string; budget?: unknown }, ctx: AuthContext): Promise<Json> {
   return launchWorkflowRun(await prepareWorkflowRun(input, ctx), input, ctx);
 }
 
@@ -98,18 +99,17 @@ export async function launchWorkflowRun(prepared: PreparedWorkflowRun, input: {
   const { task, cwd, read } = prepared;
   const budget = cleanSpawnBudget(input.budget);
   const { definition } = read.workflow;
+  // #3769 — a Task a delegated sub-coordinator owns runs as its run, whoever starts it: the step spawns' lineage and the run's origin agree.
+  const caller = admitWorkflowCaller(ctx, task, read.workflow) ?? admitWorkflowTaskOwner(task, read.workflow);
   await getGateway().read("spawnTree.validateTarget", { type: "agent.spawn", source: ctx.actor.id, target: read.orchestratorId,
-    params: { spawnedBy: ctx.callerAgentId, taskId: task.id, cwd, orchestratorId: read.orchestratorId, profile: "worker", workflowRunId: "preflight" } });
-  const caller = admitWorkflowCaller(ctx, task, read.workflow);
+    params: { spawnedBy: caller?.callerAgentId ?? ctx.callerAgentId, taskId: task.id, cwd, orchestratorId: read.orchestratorId, profile: "worker", workflowRunId: "preflight" } });
   const capabilityError = workflowCapabilityError(definition);
   if (capabilityError) throw new ValidationError(capabilityError);
   validateWorkflowValue(definition.inputs, input.inputs ?? {}, "INPUT_INVALID");
   if (Object.values(definition.steps).some(step => step.verify) && !taskVerificationCheck(task.doneWhen)) throw new ValidationError(`T${task.id} needs a nonempty verify: section in doneWhen`);
   const pins = input.pins ? stampPins(definition, input.pins, "start") : undefined;
   const defaults = (await getRoutingPolicyEnvelope()).policy.stepDefaults;
-  const coordinatorId = ctx.callerAgentId ?? task.createdBy;
-  const teamId = task.teamId ?? (coordinatorId ? getAgent(coordinatorId)?.teamId : undefined);
-  const routingDefaults = defaults ? { global: defaults.global, ...(teamId ? { team: defaults.teams?.[teamId], teamId } : {}) } : undefined;
+  const routingDefaults = resolveWorkflowRoutingDefaults(defaults, task, ctx.callerAgentId);
   await assertStepLaunches({ definition, pins, routingDefaults, steps: Object.keys(definition.steps), roles: read.workflow.roles, inputTask: task }, ctx);
   for (const [name, snapshot] of Object.entries(read.workflow.closure ?? {})) {
     if (name === read.workflow.name) continue;
@@ -146,7 +146,7 @@ export async function launchWorkflowRun(prepared: PreparedWorkflowRun, input: {
   });
   if (!created && run.workflow !== read.workflow.name) throw new ValidationError(`task ${task.id} already runs workflow "${run.workflow}" (${run.id}); a Task has one open run`);
   if (!created && pins && !sameCallPins(run.pins, pins)) {
-    throw new ValidationError(`task ${task.id} already runs ${run.id} with other pins; a running run keeps its pins, and relay_workflow_resume changes them after an escalation`);
+    throw new ValidationError(`task ${task.id} already runs ${run.id} with other pins; use relay_workflow_resume with pins to change not-yet-started steps`);
   }
   if (!created && budget && budget.tokenCap !== run.tokenBudget?.tokenCap) {
     throw new ValidationError(`task ${task.id} already runs ${run.id} with another budget; a run keeps its original token cap`);
@@ -180,9 +180,9 @@ async function settleRefusable(sup: Supervisor, run: WorkflowRunRecord): Promise
   throw error;
 }
 
-/** A run is visible to its coordinator, the input Task's coordinators and admins. */
-export function canViewRun(ctx: AuthContext, coordinator: string, task: Parameters<typeof canControlTaskReviewHold>[1] | null): boolean {
-  return ctx.callerAgentId === coordinator || Boolean(ctx.callerAgentId && spawnTreeContains(ctx.callerAgentId, coordinator)) || Boolean(task && canControlTaskReviewHold(ctx, task));
+/** A run is visible to its coordinator, the input Task's coordinators, its owning sub-coordinator and admins. */
+export function canViewRun(ctx: AuthContext, coordinator: string, task: Parameters<typeof canControlWorkflowTask>[1] | null): boolean {
+  return ctx.callerAgentId === coordinator || Boolean(ctx.callerAgentId && spawnTreeContains(ctx.callerAgentId, coordinator)) || Boolean(task && canControlWorkflowTask(ctx, task));
 }
 
 /** #3352 — the run's cap and observed spend, as admission counts them. */
@@ -190,7 +190,7 @@ export async function readWorkflowRunBudget(runId: string, ctx: AuthContext): Pr
   const sup = await ensureSupervisor();
   const run = await readRun(sup, runId);
   if (!canViewRun(ctx, run.coordinator, await getTaskDetailAsync(run.taskId))) {
-    throw new ServiceAuthError(`workflow run ${runId} is visible to its coordinator, the Task's coordinators and admins — ${COORDINATOR_AUTHORITY_HINT}`);
+    throw new ServiceAuthError(`workflow run ${runId} is visible to its coordinator, the Task's coordinators, its owning sub-coordinator and admins — ${COORDINATOR_AUTHORITY_HINT}`);
   }
   if (!run.tokenBudget) throw new McpNotFoundError(`workflow run ${runId} has no token budget`);
   return await readWorkflowRunBudgetAsync(run.id, run.tokenBudget.tokenCap) as unknown as Json;
@@ -198,12 +198,7 @@ export async function readWorkflowRunBudget(runId: string, ctx: AuthContext): Pr
 
 /** #3352 — a coordinator amends a live run's cap; new spawns see it at their next admission. */
 export async function amendWorkflowRunBudget(input: { runId: string; tokenCap: number }, ctx: AuthContext): Promise<Json> {
-  const sup = await ensureSupervisor();
-  const run = await readRun(sup, input.runId);
-  const task = await getTaskDetailAsync(run.taskId);
-  if (!task || !(ctx.callerAgentId === run.callerAgentId || canControlTaskReviewHold(ctx, task))) {
-    throw new ServiceAuthError(`amending the budget of workflow run ${run.id} is coordinator-only — ${COORDINATOR_AUTHORITY_HINT}`);
-  }
+  const { sup, run } = await controlledRun(input.runId, ctx, "amending the budget of");
   const amended = await serialized(sup, () => invoke<WorkflowRunRecord>(sup, "amendTokenBudget", { runId: run.id, tokenCap: input.tokenCap })).catch((error: unknown) => {
     if (error instanceof OpError && error.code === "conflict") throw new ValidationError(error.message);
     throw error;
@@ -222,7 +217,7 @@ async function readRun(sup: Supervisor, runId: string): Promise<WorkflowRunRecor
 }
 
 /** relay_workflow_resume: answer the run's escalation by re-entering it at its escalated step or a named one (#3045 H2),
- *  replacing the pins of each step `pins` names (#3121). */
+ *  replacing the pins of each step `pins` names (#3121); pins alone on a running or paused run update future steps (#3806). */
 export async function resumeWorkflowRun(input: { runId: string; activationId?: string; step?: string; note?: string; pins?: WorkflowPins; decision?: WorkflowDecisionResponse }, ctx: AuthContext): Promise<Json> {
   const sup = await ensureSupervisor();
   const run = await readRun(sup, input.runId);
@@ -236,14 +231,27 @@ export async function resumeWorkflowRun(input: { runId: string; activationId?: s
     return { ...runView(await readRun(sup, run.id)), resumed: input.decision.action === "answer" };
   }
   const task = await getTaskDetailAsync(run.taskId);
-  if (!task || !(ctx.callerAgentId === run.callerAgentId || canControlTaskReviewHold(ctx, task))) {
+  if (!task || !(ctx.callerAgentId === run.callerAgentId || canControlWorkflowTask(ctx, task))) {
     throw new ServiceAuthError(`resuming workflow run ${run.id} is coordinator-only: the run sets the review hold and clears it on GO — ${COORDINATOR_AUTHORITY_HINT}`);
   }
   if (run.callerAgentId && ctx.actor.kind !== "system") assertWorkflowCallerLive(run.callerAgentId);
   if (!run.callerAgentId && !canSpawn(ctx)) throw new ServiceAuthError("resuming a workflow can dispatch its roles, so it requires the command:spawn scope");
   if (task.status === "canceled") throw new ValidationError(`T${task.id} is canceled, so its run ${run.id} cannot resume`);
   const pins = input.pins ? stampPins(run.definition, input.pins, "resume") : undefined;
-  if (pins) await assertResumePins(run, pins, task, ctx, await runAgents(sup, run.id));
+  if (pins && !input.step && !input.activationId && run.status === "running") {
+    await serialized(sup, async () => {
+      const latest = await readRun(sup, run.id);
+      assertWorkflowFuturePins(latest, pins);
+      await assertResumePins(latest, pins, task, ctx, await runAgents(sup, run.id), (op, args) => invoke(sup, op, args));
+      await invoke(sup, "amendPins", { runId: run.id, pins, by: ctx.callerAgentId ?? ctx.actor.id, note: input.note });
+    }).catch((error: unknown) => {
+      if (error instanceof OpError && (error.code === "conflict" || error.code === "invalid_input")) throw new ValidationError(error.message);
+      throw error;
+    });
+    await requestPump(sup);
+    return { ...runView(await readRun(sup, run.id)), resumed: false, pinsUpdated: true };
+  }
+  if (pins) await assertResumePins(run, pins, task, ctx, await runAgents(sup, run.id), (op, args) => invoke(sup, op, args));
   // Between passes, so no pass is performing the end-of-run reap the resume voids. A fact that already ends the
   // current wait (a delivery, a land) is signaled first and wins: the resume then refuses.
   const resume = serialized(sup, async () => {
@@ -270,13 +278,7 @@ export async function resumeWorkflowRun(input: { runId: string; activationId?: s
 /** relay_workflow_cancel (#3166): end a running or escalated run cancelled. A running run ends at once, its step
  *  agents reaped per their `reap`; the Task, its review hold, any delivery and an armed land stay as they are. */
 export async function cancelWorkflowRun(input: { runId: string; reason?: string }, ctx: AuthContext): Promise<Json> {
-  const sup = await ensureSupervisor();
-  const run = await readRun(sup, input.runId);
-  const task = await getTaskDetailAsync(run.taskId);
-  if (!task || !(ctx.callerAgentId === run.callerAgentId || canControlTaskReviewHold(ctx, task))) {
-    throw new ServiceAuthError(`cancelling workflow run ${run.id} is coordinator-only — ${COORDINATOR_AUTHORITY_HINT}`);
-  }
-  const by = ctx.callerAgentId ?? ctx.actor.id;
+  const { sup, run, by } = await controlledRun(input.runId, ctx, "cancelling");
   const reason = `cancelled by ${by}${input.reason ? `: ${input.reason}` : ""}`;
   // Between passes, so the cancel lands before the next routing.
   const { cancelled } = await serialized(sup, () => invoke<{ cancelled: boolean }>(sup, "cancel", { runId: run.id, reason, by })).catch((error: unknown) => {
@@ -287,6 +289,32 @@ export async function cancelWorkflowRun(input: { runId: string; reason?: string
   return { ...runView(await readRun(sup, run.id)), cancelled };
 }
 
+/** The run a coordinator-only control acts on: its caller, the input Task's coordinators, its owning sub-coordinator and admins may. */
+async function controlledRun(runId: string, ctx: AuthContext, verb: string): Promise<{ sup: Supervisor; run: WorkflowRunRecord; by: string }> {
+  const sup = await ensureSupervisor();
+  const run = await readRun(sup, runId);
+  const task = await getTaskDetailAsync(run.taskId);
+  if (!task || !(ctx.callerAgentId === run.callerAgentId || canControlWorkflowTask(ctx, task))) {
+    throw new ServiceAuthError(`${verb} workflow run ${run.id} is coordinator-only — ${COORDINATOR_AUTHORITY_HINT}`);
+  }
+  return { sup, run, by: ctx.callerAgentId ?? ctx.actor.id };
+}
+
+/** #3757 — pause a running run at its next seam (reason required) or unpause it; its agents keep working meanwhile. */
+export async function pauseWorkflowRun(input: { runId: string; pause: boolean; reason?: string }, ctx: AuthContext): Promise<Json> {
+  const reason = input.reason?.trim();
+  if (input.pause && !reason) throw new ValidationError("pausing a workflow run needs a reason (MCP: note): say why it waits and for what");
+  const { sup, run, by } = await controlledRun(input.runId, ctx, input.pause ? "pausing" : "unpausing");
+  // Between passes, so no pass routes or performs with a stale view of the pause.
+  await serialized(sup, () => invoke(sup, input.pause ? "pause" : "unpause", { runId: run.id, by, now: supervisorNow(), ...(input.pause ? { reason } : {}) }))
+    .catch((error: unknown) => {
+      if (error instanceof OpError && (error.code === "conflict" || error.code === "invalid_input")) throw new ValidationError(error.message);
+      throw error;
+    });
+  await requestPump(sup);
+  return runView(await readRun(sup, run.id));
+}
+
 /** relay_workflow_get: the run's visible record, after every pass already scheduled. */
 export async function getWorkflowRun(runId: string, ctx: AuthContext, options: WorkflowRunReadOptions = {}): Promise<Json> {
   const sup = await ensureSupervisor();
@@ -294,7 +322,7 @@ export async function getWorkflowRun(runId: string, ctx: AuthContext, options: W
   await sup.notes;
   const run = await readRun(sup, runId);
   if (!canViewRun(ctx, run.coordinator, await getTaskDetailAsync(run.taskId))) {
-    throw new ServiceAuthError(`workflow run ${runId} is visible to its coordinator, the Task's coordinators and admins — ${COORDINATOR_AUTHORITY_HINT}`);
+    throw new ServiceAuthError(`workflow run ${runId} is visible to its coordinator, the Task's coordinators, its owning sub-coordinator and admins — ${COORDINATOR_AUTHORITY_HINT}`);
   }
   const scripts = options.verbose && Object.values(run.definition.steps).some(step => step.script)
     ? await invoke<{ items: Array<WorkflowScriptRun & { id: string; runId: string }>; cursor?: string }>(sup, "scriptAttempts", { runId, ...(options.scriptCursor ? { cursor: options.scriptCursor } : {}) }) : undefined;

package/src/services/workflow-supervisor-state.ts

@@ -1,7 +1,7 @@
 import { errMessage } from "@humwick/sdk";
 import type { ModuleHost, ModuleHostOptions } from "@humwick/kernel";
 import type { WorkflowRuntime } from "@humwick/kernel/workflow-runtime";
-import { workflow, type WorkflowAgent } from "@humwick/kernel/workflow.pilot";
+import { workflow, type WorkflowAgent, type WorkflowEffect } from "@humwick/kernel/workflow.pilot";
 import { getDbGeneration } from "../db";
 import { emitRelayEvent } from "../events";
 
@@ -45,6 +45,31 @@ let stopped = Promise.resolve();
 let changeTimer: Timer | undefined;
 export let testOptions: SupervisorTestOptions = {};
 
+/** The supervisor's clock: deadlines are read and a pause is stamped against it. */
+export function supervisorNow(): number {
+  return (testOptions.now ?? Date.now)();
+}
+
+const QUICK_RETRIES = 3;
+const RETRY_BASE_MS = 1000;
+const RETRY_MAX_MS = 60_000;
+
+/** #3827 — a retried effect's result with its next attempt. A stop still in flight (`inFlight`) is re-read next
+ *  pass; consecutive terminal misses (a failed stop, a gone target, a stop without proof) get a few quick retries,
+ *  then exponential backoff to a minute, so an effect that never converges costs one attempt a minute, not one a pass. */
+export function retryResult(previous: Json | undefined, result: Json, now: number): Json {
+  if (result.inFlight === true) return result;
+  const failures = (typeof previous?.failures === "number" ? previous.failures : 0) + 1;
+  const delay = failures <= QUICK_RETRIES ? 0 : Math.min(RETRY_MAX_MS, RETRY_BASE_MS * 2 ** Math.min(failures - QUICK_RETRIES - 1, 6));
+  return { ...result, failures, nextAttemptAt: now + delay };
+}
+
+/** Whether a pending effect's backoff has run out. */
+export function effectDue(effect: WorkflowEffect, now: number): boolean {
+  const at = effect.result?.nextAttemptAt;
+  return typeof at !== "number" || at <= now;
+}
+
 /** The supervisor bound to the current database handle (a test that swaps the handle rebuilds it). */
 export function live(): Supervisor | undefined {
   return current && current.generation === getDbGeneration() ? current : undefined;

package/src/services/workflow-supervisor.ts

@@ -1,4 +1,5 @@
 // #3045 — drives durable workflow transitions, reconciliation and external effects.
+import { framed } from "../hub-metrics";
 import { errMessage, fireAndForget, isRecord, stringValue } from "@humwick/sdk";
 import { createModuleHost, type ModuleHost } from "@humwick/kernel";
 import { workflow } from "@humwick/kernel/workflow.pilot";
@@ -25,9 +26,13 @@ function startSupervisor(): Supervisor {
     persistence: createGatewayKernelPersistence(),
     leaseTtlMs: testOptions.leaseTtlMs ?? LEASE_TTL_MS,
     opCommitFaults: testOptions.opCommitFaults,
-    // A wait is satisfied by its signal row: one keyed read instead of the default event-log scan.
-    workflowWaitEvaluator: async (waitFor) => waitFor.kind === "event" && waitFor.correlationId !== undefined
-      && ((await host.invoke(workflow, "hasSignal", SUPERVISOR, { key: waitFor.correlationId })) as { present: boolean }).present,
+    // A wait is satisfied by its signal row: one keyed read instead of the default event-log scan. A paused run's
+    // signal waits for its unpause (#3757).
+    workflowWaitEvaluator: async (waitFor) => {
+      if (waitFor.kind !== "event" || waitFor.correlationId === undefined) return false;
+      const wait = await host.invoke(workflow, "hasSignal", SUPERVISOR, { key: waitFor.correlationId }) as { present: boolean; paused?: boolean };
+      return wait.present && !wait.paused;
+    },
   });
   host.register(workflow);
   const sup: Supervisor = {
@@ -53,10 +58,10 @@ function startSupervisor(): Supervisor {
     }),
     landed: new Map(), tail: predecessor, queued: undefined, notes: predecessor, refusable: new Map(),
   };
-  sup.ticker = setInterval(() => {
-    if (live() === sup) void fireAndForget("[workflow] supervisor pump", () => requestPump(sup));
+  sup.ticker = setInterval(framed("timer workflow-supervisor", () => {
+    if (live() === sup) return fireAndForget("[workflow] supervisor pump", () => requestPump(sup));
     else if (sup.ticker) clearInterval(sup.ticker);
-  }, workflowScriptsEnabled() ? 1000 : SCHEDULER_TICK_MS);
+  }), workflowScriptsEnabled() ? 1000 : SCHEDULER_TICK_MS);
   sup.ticker.unref?.();
   setSupervisor(sup);
   return sup;

package/src/services/workspace-recovery.ts

@@ -1,3 +1,4 @@
+import { insertRecoveredWorkspaceAsync } from "../db/runtime-admin-async";
 // #1502 — server-authoritative self-heal of a dropped/never-written isolated-workspace registration.
 //
 // A live worker holds its own workspace descriptor in AGENT_RELAY_WORKSPACE_JSON, but the workspace
@@ -18,7 +19,7 @@
 // 404), never a best-effort guess, because the output is a landing authorization.
 
 import { resolve } from "node:path";
-import { getDb, getOrchestrator, insertRecoveredWorkspace, workspaceLandMetadataFrom, type RecoveredWorkspaceInput } from "../db";
+import { getDb, getOrchestrator, workspaceLandMetadataFrom, type RecoveredWorkspaceInput } from "../db";
 import { parseJson, isPathWithinBase } from "../utils";
 import { sanitizeFsName } from "@humwick/sdk/fs-name";
 import { hasScope } from "@humwick/sdk/types";
@@ -69,7 +70,7 @@ function resolveOwningOrchestrator(ctx: AuthContext, params: Record<string, unkn
   return null;
 }
 
-export function recoverIsolatedWorkspace(ctx: AuthContext, hint: WorkspaceRecoveryHint | undefined, pathId: string | undefined): WorkspaceRecovery {
+export async function recoverIsolatedWorkspace(ctx: AuthContext, hint: WorkspaceRecoveryHint | undefined, pathId: string | undefined): Promise<WorkspaceRecovery> {
   const caller = ctx.callerAgentId;
   if (!caller) return REFUSE;
   // Minting a land/merge target is a privileged write — require command:write, not baseline identity
@@ -163,6 +164,6 @@ export function recoverIsolatedWorkspace(ctx: AuthContext, hint: WorkspaceRecove
     ownerAgentId: caller,
     landMetadata: workspaceLandMetadataFrom(params),
   };
-  const result = insertRecoveredWorkspace(input);
+  const result = (await insertRecoveredWorkspaceAsync(input));
   return { ok: true, workspace: result.workspace, created: result.created };
 }

package/src/settings/registry.ts

@@ -32,7 +32,6 @@ import {
   SPAWN_PROVIDERS,
   TEAM_IDLE_THRESHOLD_MINUTES_MAX,
   TEAM_IDLE_THRESHOLD_MINUTES_MIN,
-  VALID_EFFORTS,
   type JSONSchema,
   type SettingDescriptor,
   type SettingScope,
@@ -166,7 +165,7 @@ const STEWARD_SCHEMA: JSONSchema = {
     enabled: { type: "boolean" },
     provider: { type: "string", enum: [...SPAWN_PROVIDERS] },
     model: { type: "string", maxLength: 120 },
-    effort: { type: "string", enum: [...VALID_EFFORTS] },
+    effort: { type: "string", maxLength: 120 },
     permissionMode: { type: "string", enum: [...APPROVAL_MODES] },
     keepaliveSeconds: { type: "integer", minimum: 0, maximum: 2_592_000 },
   },

package/src/spawn-admission.ts

@@ -1,11 +1,18 @@
 import type { Orchestrator, SpawnProvider } from "./types";
+import { ValidationError } from "@humwick/sdk";
+
+export class PinnedHostCapacityError extends ValidationError {
+  constructor(message: string, readonly orchestratorId: string) {
+    super(message);
+  }
+}
 
 export interface AdmissionReason {
   check: "host" | "provider" | "auth" | "capacity";
   code: string;
   message: string;
 }
-export interface Admission { eligible: boolean; refusals: AdmissionReason[]; uncertainties: AdmissionReason[] }
+export interface Admission { eligible: boolean; refusals: AdmissionReason[]; uncertainties: AdmissionReason[]; agentCapOnly?: boolean }
 
 /** Interpret all probe legs independently: a failure on one never hides another leg's timeout. */
 export function providerProbeAdmission(item: Orchestrator, provider: SpawnProvider, pinned = false): Admission {
@@ -40,7 +47,8 @@ export function providerProbeAdmission(item: Orchestrator, provider: SpawnProvid
 
 export function assessSpawnAdmission(item: Orchestrator, provider: SpawnProvider, checks: AdmissionReason[] = [], pinned = false): Admission {
   const probe = providerProbeAdmission(item, provider, pinned);
-  return { eligible: probe.eligible && checks.length === 0, refusals: [...checks, ...probe.refusals], uncertainties: probe.uncertainties };
+  return { eligible: probe.eligible && checks.length === 0, refusals: [...checks, ...probe.refusals], uncertainties: probe.uncertainties,
+    agentCapOnly: probe.eligible && checks.length > 0 && checks.every(reason => reason.code === "agent_cap") };
 }
 
 export function admissionMessages(admission: Admission): string {

package/src/spawn-command.ts

@@ -112,11 +112,12 @@ export function resolveSpawnModelParams(
     // #2599 — the full record is server-computed once from the effective-catalog selection:
     // qualified runtime name + both windows + the explicit threshold. Same positive-int gates
     // as the legacy window (never 0/NaN/negative), via the shared builder.
-    const record = runnerSupportsModelRecord(opts.runnerProtocol)
+    const record = runnerSupportsModelRecord(opts.runnerProtocol ?? MODEL_RECORD_PROTOCOL_VERSION)
       ? resolvedModelRecordFromSelection({
         alias: selection.modelAlias,
         providerModel: selection.providerModel,
         effort: selection.effort,
+        efforts: selection.model?.efforts,
         qualifiedModel: qualifyServerModelName(provider, selection.providerModel, effectiveProviderCatalogList()),
         limits: selection.model?.limits,
         ...(selection.model?.contextThreshold ? { contextThreshold: selection.model.contextThreshold } : {}),
@@ -135,7 +136,6 @@ export function resolveSpawnModelParams(
       ...(record ? { modelRecord: record } : {}),
     };
   } catch (error) {
-    if (opts.modelSource === "runtime") return { ...(model ? { model } : {}), ...(effort ? { effort } : {}) };
     throw new ValidationError(errMessage(error));
   }
 }
@@ -157,11 +157,12 @@ export async function resolveSpawnModelParamsAsync(
     );
     const windowTokens = selection.model?.limits?.contextWindowTokens?.value;
     // #2599 — same full-record twin as the sync path (see its comment).
-    const record = runnerSupportsModelRecord(opts.runnerProtocol)
+    const record = runnerSupportsModelRecord(opts.runnerProtocol ?? MODEL_RECORD_PROTOCOL_VERSION)
       ? resolvedModelRecordFromSelection({
         alias: selection.modelAlias,
         providerModel: selection.providerModel,
         effort: selection.effort,
+        efforts: selection.model?.efforts,
         qualifiedModel: qualifyServerModelName(provider, selection.providerModel, liveCatalog),
         limits: selection.model?.limits,
         ...(selection.model?.contextThreshold ? { contextThreshold: selection.model.contextThreshold } : {}),
@@ -176,7 +177,6 @@ export async function resolveSpawnModelParamsAsync(
       ...(record ? { modelRecord: record } : {}),
     };
   } catch (error) {
-    if (opts.modelSource === "runtime") return { ...(model ? { model } : {}), ...(effort ? { effort } : {}) };
     throw new ValidationError(errMessage(error));
   }
 }

package/src/spawn-models.ts

@@ -2,21 +2,19 @@ import { isOpenAiApiModel } from "@humwick/sdk";
 import type { ProviderCatalogEntry } from "@humwick/sdk/provider-catalog";
 import type { Orchestrator, SpawnProvider } from "./types";
 
-/** A missing report preserves compatibility with older orchestrators; a reported catalog is authoritative. */
-export function hostSupportsModel(host: Orchestrator, provider: SpawnProvider, model?: string, effort?: string): boolean {
-  const catalog = host.providerCatalog?.find((entry) => entry.provider === provider);
-  if (!model || !catalog) return !isOpenAiApiModel(provider, model);
-  const status = catalog.models.find((entry) => entry.alias === model || entry.providerModel === model);
-  return status !== undefined && !status.disabled
-    && (!effort || status.efforts.includes(effort));
+/** Hosts report local capabilities; model definitions and efforts belong to the hub catalog. */
+export function hostSupportsModel(host: Orchestrator, provider: SpawnProvider, model?: string): boolean {
+  if (!isOpenAiApiModel(provider, model)) return true;
+  const ready = host.providerStatus?.find((entry) => entry.name === provider)?.features?.openaiApi;
+  if (ready !== undefined) return ready;
+  // Older hosts encoded credential readiness by including the paid model in their catalog.
+  return host.providerCatalog?.find((entry) => entry.provider === provider)?.models
+    .some((entry) => isOpenAiApiModel(provider, entry.alias) && !entry.disabled) ?? false;
 }
 
-/** Keep Relay's model definitions and settings, intersected with the host's reported capabilities. */
+/** Project the hub catalog onto the host's genuinely local capabilities. */
 export function hostRunnableCatalog(entry: ProviderCatalogEntry, host: Orchestrator): ProviderCatalogEntry {
   const models = entry.models.filter((model) => !model.disabled && !model.unavailable
-    && hostSupportsModel(host, entry.provider, model.alias)).map((model) => {
-    const efforts = model.efforts.filter((effort) => hostSupportsModel(host, entry.provider, model.alias, effort));
-    return { ...model, efforts, defaultEffort: model.defaultEffort && efforts.includes(model.defaultEffort) ? model.defaultEffort : undefined };
-  });
+    && hostSupportsModel(host, entry.provider, model.alias));
   return { ...entry, models };
 }

package/src/spawn-placement.ts

@@ -13,7 +13,7 @@ import { HOST_BAND_CONFIG } from "./config";
 import { hostProviderRefusal, liveHostDisabledProviders } from "./host-provider-store";
 
 import { hostSupportsModel } from "./spawn-models";
-import { admissionMessages, assessSpawnAdmission, type AdmissionReason, type Admission } from "./spawn-admission";
+import { admissionMessages, assessSpawnAdmission, PinnedHostCapacityError, type AdmissionReason, type Admission } from "./spawn-admission";
 
 const MATCHED_HOST_IDENTITY: HostIdentity = Object.freeze({ matched: true });
 
@@ -180,10 +180,14 @@ export function selectSpawnPlacement(
     const orchestrator = getOrchestrator(orchestratorId);
     if (!orchestrator) throw new McpNotFoundError(`orchestrator ${orchestratorId} not found`);
     const view = orchestrator.hostId ? byId.get(orchestrator.hostId) : undefined;
-    const admission = hostSpawnAdmission(orchestrator, view, identities, provider, model, effort, liveHostDisabledProviders(), true, spawnHostCapAdmission(capOpts)(view), repoPlatformAdmission(view, placementPlatforms));
+    const admission = hostSpawnAdmission(orchestrator, view, identities, provider, model, effort, liveHostDisabledProviders(), true, spawnHostCapAdmission(capOpts)(view), [
+      ...repoPlatformAdmission(view, placementPlatforms), ...workspaceHostAdmission(orchestrator, view, workspaceMode, canLand, capOpts, true),
+    ]);
     const detail = admissionMessages(admission);
     if (!admission.eligible) {
-      throw new ValidationError(spawnPlacementRefusal(listOrchestrators(), views, provider, model, effort, orchestratorId, capOpts, admission));
+      const message = spawnPlacementRefusal(listOrchestrators(), views, provider, model, effort, orchestratorId, capOpts, admission);
+      if (admission.agentCapOnly) throw new PinnedHostCapacityError(message, orchestratorId);
+      throw new ValidationError(message);
     }
     return { orchestrator, reason: `Explicit host pin: ${orchestrator.hostname} (${orchestrator.id})${detail ? `; ${detail}` : ""}.` };
   }
@@ -241,7 +245,25 @@ function headroomSummary(view: HostView | undefined): string {
   return `(${round(cores)} free cores, ${round(freeThreadsOf(view)!)} free threads, band ${view.band})`;
 }
 
-export interface SpawnCapOptions { callerAgentId?: string; attach?: boolean }
+export interface SpawnCapOptions {
+  callerAgentId?: string;
+  attach?: boolean;
+  buildWork?: boolean;
+  workspaceDepsDisabledHosts?: readonly string[];
+}
+
+/** Pins and landBlocked cannot turn isolated dependency/build work into a light hub probe (#3805). */
+function workspaceHostAdmission(item: Orchestrator, view: HostView | undefined, mode: WorkspaceMode | undefined, canLand: boolean, opts: SpawnCapOptions, pinned = false): AdmissionReason[] {
+  if (mode !== "isolated") return [];
+  const hubWork = (opts.buildWork ?? canLand) || (!opts.attach && !opts.workspaceDepsDisabledHosts?.includes(item.id));
+  if (hubWork && view?.policy.noLandingWorkspaces !== false && isRelayHost(item, view)) {
+    return [{ check: "host", code: "hub_workspace", message: `no builders on the hub: Relay server host ${item.hostname} (${item.id}) refuses isolated build or dependency-install workspaces, including pinned spawns and workflow steps; choose a non-hub host or set Fleet > Edit host > Isolated workspaces to Include this host (noLandingWorkspaces=false)` }];
+  }
+  if (pinned || !canLand) return [];
+  const landing = view?.policy.noLandingWorkspaces === true ? "host excludes landing workspaces"
+    : view?.policy.noLandingWorkspaces !== false && isRelayHost(item, view) ? "Relay server host excluded by default" : undefined;
+  return landing ? [{ check: "host", code: "landing", message: landing }] : [];
+}
 
 // Repository tools must run on the host even when the agent cannot land or pins a host (#3563).
 function repoPlatformAdmission(view: HostView | undefined, platforms?: readonly string[] | null): AdmissionReason[] {
@@ -282,11 +304,7 @@ export function spawnPlacementCandidates(
       const extra = repoPlatformAdmission(view, placementPlatforms);
       const capacity = capBlock(view);
       if (capacity) extra.push({ check: "capacity", code: "agent_cap", message: capacity });
-      if (isolatedAutoPlacement) {
-        const landing = view?.policy.noLandingWorkspaces === true ? "host excludes landing workspaces"
-          : view?.policy.noLandingWorkspaces !== false && isRelayHost(item, view) ? "Relay server host excluded by default" : undefined;
-        if (landing) extra.push({ check: "host", code: "landing", message: landing });
-      }
+      extra.push(...workspaceHostAdmission(item, view, workspaceMode, canLand, capOpts));
       const admission = hostSpawnAdmission(item, view, identities, provider, model, effort, hostDisabled, false, undefined, extra);
       if (!admission.eligible) exclusions.push(`${item.hostname}: ${admissionMessages(admission)}`);
       else if (admission.uncertainties.length) warnings.push(`${item.hostname}: ${admissionMessages(admission)}`);
@@ -321,7 +339,7 @@ export function hostSpawnAdmission(
   if (view?.policy.spawnEnabled === false) add("host", "draining", `host ${item.hostname} (${item.id}) is draining: new spawns disabled`);
   if (!pinned && view && (!view.enabled || view.state === "unavailable" || view.state === "retired" || view.band === "blocked")) add("host", "unavailable", "host unavailable or blocked");
   if (!pinned && view && view.aggregates.agentSlotsMax !== 0 && view.aggregates.agentSlotsUsed >= view.aggregates.agentSlotsMax) add("capacity", "slots", "host spawn slots full");
-  if (!pinned && !hostSupportsModel(item, provider, model, effort)) add("provider", "model", `provider '${provider}' does not support requested model or effort`);
+  if (!pinned && !hostSupportsModel(item, provider, model)) add("provider", "model", `provider '${provider}' does not support requested model or effort`);
   if (capacity) add("capacity", "agent_cap", `host ${item.hostname} (${item.id}) is ${capacity}`);
   return assessSpawnAdmission(item, provider, checks, pinnedAuth);
 }

package/src/spawn-targets.ts

@@ -1,6 +1,6 @@
 import { apiBudgetStatus } from "./db/claude-api-budget";
 import { apiBudgetStatusAsync } from "./claude-api-budget-async";
-import { isOpenAiApiModel, type ApiBudgetStatus } from "@humwick/sdk";
+import { API_BUDGET_IDS, isOpenAiApiModel, type ApiBudgetStatus } from "@humwick/sdk";
 import { collapseProviderQuotaRows } from "@humwick/sdk/provider-quota-order";
 import { resolveQuotaPacing, subscriptionExpiryForWindow, subscriptionExpiryPreference, type ResolvedQuotaPacing } from "./quota-pacing";
 import { countLiveSpawnedAgents, countLiveSpawnedAgentsAsync, getAgent, getAgentAsync, listOrchestrators, listOrchestratorsAsync, listProviderQuotas, providerQuotaRoutingSnapshotAsync } from "./db";
@@ -53,7 +53,7 @@ export function buildSpawnTargets(input: { callerId?: string; quota: number }):
   const me = input.callerId ? getAgent(input.callerId) : undefined;
   return buildSpawnTargetsFrom(input, {
     me,
-    openaiBudget: apiBudgetStatus(now, "openai-api"),
+    apiBudgets: API_BUDGET_IDS.map((id) => apiBudgetStatus(now, id)),
     orchestrators: listOrchestrators(),
     hostViews: listHostViews(now),
     liveChildren: input.callerId ? countLiveSpawnedAgents(input.callerId) : 0,
@@ -67,17 +67,17 @@ export function buildSpawnTargets(input: { callerId?: string; quota: number }):
 /** Worker-safe discovery path: every durable input is read through the reader gateway. */
 export async function buildSpawnTargetsAsync(input: { callerId?: string; quota: number }): Promise<Record<string, unknown>> {
   const now = Date.now();
-  const [me, orchestrators, hostViews, liveChildren, snapshot, openaiBudget] = await Promise.all([
+  const [me, orchestrators, hostViews, liveChildren, snapshot, apiBudgets] = await Promise.all([
     input.callerId ? getAgentAsync(input.callerId) : Promise.resolve(null),
     listOrchestratorsAsync(),
     listHostViewsAsync(),
     input.callerId ? countLiveSpawnedAgentsAsync(input.callerId) : Promise.resolve(0),
     providerQuotaRoutingSnapshotAsync(now),
-    apiBudgetStatusAsync("openai-api"),
+    Promise.all(API_BUDGET_IDS.map((id) => apiBudgetStatusAsync(id))),
   ]);
   return buildSpawnTargetsFrom(input, {
     me: me ?? undefined,
-    openaiBudget,
+    apiBudgets,
     orchestrators,
     hostViews,
     liveChildren,
@@ -90,7 +90,7 @@ export async function buildSpawnTargetsAsync(input: { callerId?: string; quota:
 
 function buildSpawnTargetsFrom(input: { callerId?: string; quota: number }, data: {
   me?: ReturnType<typeof getAgent>;
-  openaiBudget: ApiBudgetStatus;
+  apiBudgets: ApiBudgetStatus[];
   orchestrators: Orchestrator[];
   hostViews: readonly HostView[];
   liveChildren: number;
@@ -130,10 +130,10 @@ function buildSpawnTargetsFrom(input: { callerId?: string; quota: number }, data
           ...(o.utilization && Number.isFinite(o.utilization.agentSlotsUsedThisDaemon) && Number.isFinite(o.utilization.agentSlotsMaxThisDaemon) && o.utilization.agentSlotsMaxThisDaemon > 0
             ? { spawn_slots: { used: o.utilization.agentSlotsUsedThisDaemon, max: o.utilization.agentSlotsMaxThisDaemon } }
             : {}),
-          providers: providerMatrixFor(o, data.quotaAdvisories, data.catalog, data.profiles, hostDisabled, data.openaiBudget),
+          providers: providerMatrixFor(o, data.quotaAdvisories, data.catalog, data.profiles, hostDisabled, data.apiBudgets.find((budget) => budget.budgetId === "openai-api")),
         };
       }),
-    apiBudgets: [data.openaiBudget],
+    apiBudgets: data.apiBudgets.map((budget) => ({ ...budget, source: "relay-accounting" })),
     ...operatorDisabledProviders(data.catalog),
     profiles: spawnProfilesSummary(data.profiles),
     // #3327 — preferences steer automatic spawns inside the dispatcher; never copy them into explicit pins.
@@ -290,7 +290,7 @@ function providerMatrixFor(
   const providers = [...new Set([...o.providers, ...(o.providerStatus ?? []).filter((s) => s.degraded || s.cli?.transient || s.runner?.transient || s.auth?.transient).map((s) => s.name)])];
   return providers.filter((provider) => !disabledProviders.has(provider) && !hostDisabledFor?.has(provider)).map((provider) => {
     const globalEntry = global.find((e) => e.provider === provider);
-    // Relay owns model settings; the heartbeat limits what this host can run (#3202).
+    // The hub owns models and efforts; hosts contribute only local capabilities.
     const cat = globalEntry ? hostRunnableCatalog(globalEntry, o) : undefined;
     const quotaAdvisory = quotaAdvisories.get(provider) ?? defaultQuotaAdvisory(provider);
     // #932 — advertise the SAME default a bare spawn would actually resolve to. resolveDefaultModel