@humwick/server 0.130.2 → 0.130.4

package/src/services/workflow-runs.ts

@@ -4,6 +4,7 @@ import { spawnTreeContains } from "../db/spawn-tree";
 import { cleanSpawnBudget } from "../spawn-budget";
 import { readWorkflowRunBudgetAsync, reconcileRunBudgetAsync } from "../token-usage-async";
 import { getRoutingPolicyEnvelope } from "../routing-policy-service";
+import { resolveWorkflowRoutingDefaults } from "./workflow-routing-defaults";
 
 // #3045 — the workflow supervisor: mounts the kernel `workflow` module, starts runs, and pumps them.
 // A pass recovers crashed drivers, signals each waiting run from durable src facts (level-triggered,
@@ -14,10 +15,10 @@ import { errMessage, fireAndForget, stringValue, type WorkflowRunAgent, type Wor
 import { validateWorkflowValue } from "@humwick/kernel/workflow-values";
 import { OpError } from "@humwick/kernel";
 
-import { admitWorkflowCaller, assertWorkflowCallerLive } from "./workflow-caller";
+import { admitWorkflowCaller, admitWorkflowTaskOwner, assertWorkflowCallerLive, canControlWorkflowTask } from "./workflow-caller";
 import { taskVerificationCheck } from "../task-verification";
 
-import { workflow, workflowEffectId, type WorkflowEffect, type WorkflowRunRecord } from "@humwick/kernel/workflow.pilot";
+import { assertWorkflowFuturePins, workflow, workflowEffectId, type WorkflowEffect, type WorkflowRunRecord } from "@humwick/kernel/workflow.pilot";
 import { getAgent, getTaskDetailAsync, ValidationError } from "../db";
 import type { WorkflowRunCursor, WorkflowRunTaskRow } from "../db/workflow-task-read-ops";
 import { getGateway } from "../db/gateway";
@@ -31,7 +32,6 @@ import { McpNotFoundError } from "../mcp-errors";
 import { ServiceAuthError } from "./errors";
 import { canSpawn, spawnLineageAgentId, taskLinkedRepoCwd } from "./spawn-agent";
 import { resolveSpawnProjectWorkflow } from "./spawn-project-role";
-import { canControlTaskReviewHold } from "./task-review-hold";
 import { updateTaskEntityAsync } from "./task-entity";
 
 import { releaseRunHold } from "./workflow-effects";
@@ -77,19 +77,19 @@ export async function prepareWorkflowRun(input: { workflow: string; taskId: numb
   if (!task || task.kind !== "deliverable") throw new ValidationError("task not found");
   if (task.status === "canceled") throw new ValidationError(`task ${task.id} is canceled; a workflow runs on a Task that is still wanted`);
   const caller = admitWorkflowCaller(ctx, task);
-  if (!caller && !canControlTaskReviewHold(ctx, task)) {
+  if (!caller && !canControlWorkflowTask(ctx, task)) {
     throw new ServiceAuthError(`starting a workflow on task ${task.id} is coordinator-only: the run sets the review hold and clears it on GO — ${COORDINATOR_AUTHORITY_HINT}`);
   }
   if (!caller && !canSpawn(ctx)) throw new ServiceAuthError("starting a workflow dispatches its roles, so it requires the command:spawn scope");
   const lineageId = spawnLineageAgentId(ctx, task);
   const lineage = lineageId ? getAgent(lineageId) : null;
-  const cwd = input.cwd ?? await taskLinkedRepoCwd(task.id, stringValue(lineage?.meta?.cwd));
+  const cwd = input.cwd ?? await taskLinkedRepoCwd(task.id, stringValue(lineage?.meta?.cwd), lineage ?? undefined);
   const read = await resolveSpawnProjectWorkflow({ workflow: input.workflow, cwd, orchestratorId: input.orchestratorId, preferHost: lineage?.machine, allowedOrchestratorIds: ctx.constraints?.orchestrators });
   return { task, cwd, read };
 }
 
 /** relay_workflow_start: read the landed workflow and its roles, check every step's pin, start one run on the Task. */
-export async function startWorkflowRun(input: { workflow: string; taskId: number; pins?: WorkflowPins; inputs?: Json; requestKey?: string; budget?: unknown }, ctx: AuthContext): Promise<Json> {
+export async function startWorkflowRun(input: { workflow: string; taskId: number; cwd?: string; pins?: WorkflowPins; inputs?: Json; requestKey?: string; budget?: unknown }, ctx: AuthContext): Promise<Json> {
   return launchWorkflowRun(await prepareWorkflowRun(input, ctx), input, ctx);
 }
 
@@ -99,18 +99,17 @@ export async function launchWorkflowRun(prepared: PreparedWorkflowRun, input: {
   const { task, cwd, read } = prepared;
   const budget = cleanSpawnBudget(input.budget);
   const { definition } = read.workflow;
+  // #3769 — a Task a delegated sub-coordinator owns runs as its run, whoever starts it: the step spawns' lineage and the run's origin agree.
+  const caller = admitWorkflowCaller(ctx, task, read.workflow) ?? admitWorkflowTaskOwner(task, read.workflow);
   await getGateway().read("spawnTree.validateTarget", { type: "agent.spawn", source: ctx.actor.id, target: read.orchestratorId,
-    params: { spawnedBy: ctx.callerAgentId, taskId: task.id, cwd, orchestratorId: read.orchestratorId, profile: "worker", workflowRunId: "preflight" } });
-  const caller = admitWorkflowCaller(ctx, task, read.workflow);
+    params: { spawnedBy: caller?.callerAgentId ?? ctx.callerAgentId, taskId: task.id, cwd, orchestratorId: read.orchestratorId, profile: "worker", workflowRunId: "preflight" } });
   const capabilityError = workflowCapabilityError(definition);
   if (capabilityError) throw new ValidationError(capabilityError);
   validateWorkflowValue(definition.inputs, input.inputs ?? {}, "INPUT_INVALID");
   if (Object.values(definition.steps).some(step => step.verify) && !taskVerificationCheck(task.doneWhen)) throw new ValidationError(`T${task.id} needs a nonempty verify: section in doneWhen`);
   const pins = input.pins ? stampPins(definition, input.pins, "start") : undefined;
   const defaults = (await getRoutingPolicyEnvelope()).policy.stepDefaults;
-  const coordinatorId = ctx.callerAgentId ?? task.createdBy;
-  const teamId = task.teamId ?? (coordinatorId ? getAgent(coordinatorId)?.teamId : undefined);
-  const routingDefaults = defaults ? { global: defaults.global, ...(teamId ? { team: defaults.teams?.[teamId], teamId } : {}) } : undefined;
+  const routingDefaults = resolveWorkflowRoutingDefaults(defaults, task, ctx.callerAgentId);
   await assertStepLaunches({ definition, pins, routingDefaults, steps: Object.keys(definition.steps), roles: read.workflow.roles, inputTask: task }, ctx);
   for (const [name, snapshot] of Object.entries(read.workflow.closure ?? {})) {
     if (name === read.workflow.name) continue;
@@ -147,7 +146,7 @@ export async function launchWorkflowRun(prepared: PreparedWorkflowRun, input: {
   });
   if (!created && run.workflow !== read.workflow.name) throw new ValidationError(`task ${task.id} already runs workflow "${run.workflow}" (${run.id}); a Task has one open run`);
   if (!created && pins && !sameCallPins(run.pins, pins)) {
-    throw new ValidationError(`task ${task.id} already runs ${run.id} with other pins; a running run keeps its pins, and relay_workflow_resume changes them after an escalation`);
+    throw new ValidationError(`task ${task.id} already runs ${run.id} with other pins; use relay_workflow_resume with pins to change not-yet-started steps`);
   }
   if (!created && budget && budget.tokenCap !== run.tokenBudget?.tokenCap) {
     throw new ValidationError(`task ${task.id} already runs ${run.id} with another budget; a run keeps its original token cap`);
@@ -181,9 +180,9 @@ async function settleRefusable(sup: Supervisor, run: WorkflowRunRecord): Promise
   throw error;
 }
 
-/** A run is visible to its coordinator, the input Task's coordinators and admins. */
-export function canViewRun(ctx: AuthContext, coordinator: string, task: Parameters<typeof canControlTaskReviewHold>[1] | null): boolean {
-  return ctx.callerAgentId === coordinator || Boolean(ctx.callerAgentId && spawnTreeContains(ctx.callerAgentId, coordinator)) || Boolean(task && canControlTaskReviewHold(ctx, task));
+/** A run is visible to its coordinator, the input Task's coordinators, its owning sub-coordinator and admins. */
+export function canViewRun(ctx: AuthContext, coordinator: string, task: Parameters<typeof canControlWorkflowTask>[1] | null): boolean {
+  return ctx.callerAgentId === coordinator || Boolean(ctx.callerAgentId && spawnTreeContains(ctx.callerAgentId, coordinator)) || Boolean(task && canControlWorkflowTask(ctx, task));
 }
 
 /** #3352 — the run's cap and observed spend, as admission counts them. */
@@ -191,7 +190,7 @@ export async function readWorkflowRunBudget(runId: string, ctx: AuthContext): Pr
   const sup = await ensureSupervisor();
   const run = await readRun(sup, runId);
   if (!canViewRun(ctx, run.coordinator, await getTaskDetailAsync(run.taskId))) {
-    throw new ServiceAuthError(`workflow run ${runId} is visible to its coordinator, the Task's coordinators and admins — ${COORDINATOR_AUTHORITY_HINT}`);
+    throw new ServiceAuthError(`workflow run ${runId} is visible to its coordinator, the Task's coordinators, its owning sub-coordinator and admins — ${COORDINATOR_AUTHORITY_HINT}`);
   }
   if (!run.tokenBudget) throw new McpNotFoundError(`workflow run ${runId} has no token budget`);
   return await readWorkflowRunBudgetAsync(run.id, run.tokenBudget.tokenCap) as unknown as Json;
@@ -218,7 +217,7 @@ async function readRun(sup: Supervisor, runId: string): Promise<WorkflowRunRecor
 }
 
 /** relay_workflow_resume: answer the run's escalation by re-entering it at its escalated step or a named one (#3045 H2),
- *  replacing the pins of each step `pins` names (#3121). */
+ *  replacing the pins of each step `pins` names (#3121); pins alone on a running or paused run update future steps (#3806). */
 export async function resumeWorkflowRun(input: { runId: string; activationId?: string; step?: string; note?: string; pins?: WorkflowPins; decision?: WorkflowDecisionResponse }, ctx: AuthContext): Promise<Json> {
   const sup = await ensureSupervisor();
   const run = await readRun(sup, input.runId);
@@ -232,14 +231,27 @@ export async function resumeWorkflowRun(input: { runId: string; activationId?: s
     return { ...runView(await readRun(sup, run.id)), resumed: input.decision.action === "answer" };
   }
   const task = await getTaskDetailAsync(run.taskId);
-  if (!task || !(ctx.callerAgentId === run.callerAgentId || canControlTaskReviewHold(ctx, task))) {
+  if (!task || !(ctx.callerAgentId === run.callerAgentId || canControlWorkflowTask(ctx, task))) {
     throw new ServiceAuthError(`resuming workflow run ${run.id} is coordinator-only: the run sets the review hold and clears it on GO — ${COORDINATOR_AUTHORITY_HINT}`);
   }
   if (run.callerAgentId && ctx.actor.kind !== "system") assertWorkflowCallerLive(run.callerAgentId);
   if (!run.callerAgentId && !canSpawn(ctx)) throw new ServiceAuthError("resuming a workflow can dispatch its roles, so it requires the command:spawn scope");
   if (task.status === "canceled") throw new ValidationError(`T${task.id} is canceled, so its run ${run.id} cannot resume`);
   const pins = input.pins ? stampPins(run.definition, input.pins, "resume") : undefined;
-  if (pins) await assertResumePins(run, pins, task, ctx, await runAgents(sup, run.id));
+  if (pins && !input.step && !input.activationId && run.status === "running") {
+    await serialized(sup, async () => {
+      const latest = await readRun(sup, run.id);
+      assertWorkflowFuturePins(latest, pins);
+      await assertResumePins(latest, pins, task, ctx, await runAgents(sup, run.id), (op, args) => invoke(sup, op, args));
+      await invoke(sup, "amendPins", { runId: run.id, pins, by: ctx.callerAgentId ?? ctx.actor.id, note: input.note });
+    }).catch((error: unknown) => {
+      if (error instanceof OpError && (error.code === "conflict" || error.code === "invalid_input")) throw new ValidationError(error.message);
+      throw error;
+    });
+    await requestPump(sup);
+    return { ...runView(await readRun(sup, run.id)), resumed: false, pinsUpdated: true };
+  }
+  if (pins) await assertResumePins(run, pins, task, ctx, await runAgents(sup, run.id), (op, args) => invoke(sup, op, args));
   // Between passes, so no pass is performing the end-of-run reap the resume voids. A fact that already ends the
   // current wait (a delivery, a land) is signaled first and wins: the resume then refuses.
   const resume = serialized(sup, async () => {
@@ -277,12 +289,12 @@ export async function cancelWorkflowRun(input: { runId: string; reason?: string
   return { ...runView(await readRun(sup, run.id)), cancelled };
 }
 
-/** The run a coordinator-only control acts on: its caller, the input Task's coordinators and admins may. */
+/** The run a coordinator-only control acts on: its caller, the input Task's coordinators, its owning sub-coordinator and admins may. */
 async function controlledRun(runId: string, ctx: AuthContext, verb: string): Promise<{ sup: Supervisor; run: WorkflowRunRecord; by: string }> {
   const sup = await ensureSupervisor();
   const run = await readRun(sup, runId);
   const task = await getTaskDetailAsync(run.taskId);
-  if (!task || !(ctx.callerAgentId === run.callerAgentId || canControlTaskReviewHold(ctx, task))) {
+  if (!task || !(ctx.callerAgentId === run.callerAgentId || canControlWorkflowTask(ctx, task))) {
     throw new ServiceAuthError(`${verb} workflow run ${run.id} is coordinator-only — ${COORDINATOR_AUTHORITY_HINT}`);
   }
   return { sup, run, by: ctx.callerAgentId ?? ctx.actor.id };
@@ -310,7 +322,7 @@ export async function getWorkflowRun(runId: string, ctx: AuthContext, options: W
   await sup.notes;
   const run = await readRun(sup, runId);
   if (!canViewRun(ctx, run.coordinator, await getTaskDetailAsync(run.taskId))) {
-    throw new ServiceAuthError(`workflow run ${runId} is visible to its coordinator, the Task's coordinators and admins — ${COORDINATOR_AUTHORITY_HINT}`);
+    throw new ServiceAuthError(`workflow run ${runId} is visible to its coordinator, the Task's coordinators, its owning sub-coordinator and admins — ${COORDINATOR_AUTHORITY_HINT}`);
   }
   const scripts = options.verbose && Object.values(run.definition.steps).some(step => step.script)
     ? await invoke<{ items: Array<WorkflowScriptRun & { id: string; runId: string }>; cursor?: string }>(sup, "scriptAttempts", { runId, ...(options.scriptCursor ? { cursor: options.scriptCursor } : {}) }) : undefined;

package/src/services/workflow-supervisor-state.ts

@@ -1,7 +1,7 @@
 import { errMessage } from "@humwick/sdk";
 import type { ModuleHost, ModuleHostOptions } from "@humwick/kernel";
 import type { WorkflowRuntime } from "@humwick/kernel/workflow-runtime";
-import { workflow, type WorkflowAgent } from "@humwick/kernel/workflow.pilot";
+import { workflow, type WorkflowAgent, type WorkflowEffect } from "@humwick/kernel/workflow.pilot";
 import { getDbGeneration } from "../db";
 import { emitRelayEvent } from "../events";
 
@@ -50,6 +50,26 @@ export function supervisorNow(): number {
   return (testOptions.now ?? Date.now)();
 }
 
+const QUICK_RETRIES = 3;
+const RETRY_BASE_MS = 1000;
+const RETRY_MAX_MS = 60_000;
+
+/** #3827 — a retried effect's result with its next attempt. A stop still in flight (`inFlight`) is re-read next
+ *  pass; consecutive terminal misses (a failed stop, a gone target, a stop without proof) get a few quick retries,
+ *  then exponential backoff to a minute, so an effect that never converges costs one attempt a minute, not one a pass. */
+export function retryResult(previous: Json | undefined, result: Json, now: number): Json {
+  if (result.inFlight === true) return result;
+  const failures = (typeof previous?.failures === "number" ? previous.failures : 0) + 1;
+  const delay = failures <= QUICK_RETRIES ? 0 : Math.min(RETRY_MAX_MS, RETRY_BASE_MS * 2 ** Math.min(failures - QUICK_RETRIES - 1, 6));
+  return { ...result, failures, nextAttemptAt: now + delay };
+}
+
+/** Whether a pending effect's backoff has run out. */
+export function effectDue(effect: WorkflowEffect, now: number): boolean {
+  const at = effect.result?.nextAttemptAt;
+  return typeof at !== "number" || at <= now;
+}
+
 /** The supervisor bound to the current database handle (a test that swaps the handle rebuilds it). */
 export function live(): Supervisor | undefined {
   return current && current.generation === getDbGeneration() ? current : undefined;

package/src/services/workflow-supervisor.ts

@@ -1,4 +1,5 @@
 // #3045 — drives durable workflow transitions, reconciliation and external effects.
+import { framed } from "../hub-metrics";
 import { errMessage, fireAndForget, isRecord, stringValue } from "@humwick/sdk";
 import { createModuleHost, type ModuleHost } from "@humwick/kernel";
 import { workflow } from "@humwick/kernel/workflow.pilot";
@@ -57,10 +58,10 @@ function startSupervisor(): Supervisor {
     }),
     landed: new Map(), tail: predecessor, queued: undefined, notes: predecessor, refusable: new Map(),
   };
-  sup.ticker = setInterval(() => {
-    if (live() === sup) void fireAndForget("[workflow] supervisor pump", () => requestPump(sup));
+  sup.ticker = setInterval(framed("timer workflow-supervisor", () => {
+    if (live() === sup) return fireAndForget("[workflow] supervisor pump", () => requestPump(sup));
     else if (sup.ticker) clearInterval(sup.ticker);
-  }, workflowScriptsEnabled() ? 1000 : SCHEDULER_TICK_MS);
+  }), workflowScriptsEnabled() ? 1000 : SCHEDULER_TICK_MS);
   sup.ticker.unref?.();
   setSupervisor(sup);
   return sup;

package/src/services/workspace-recovery.ts

@@ -1,3 +1,4 @@
+import { insertRecoveredWorkspaceAsync } from "../db/runtime-admin-async";
 // #1502 — server-authoritative self-heal of a dropped/never-written isolated-workspace registration.
 //
 // A live worker holds its own workspace descriptor in AGENT_RELAY_WORKSPACE_JSON, but the workspace
@@ -18,7 +19,7 @@
 // 404), never a best-effort guess, because the output is a landing authorization.
 
 import { resolve } from "node:path";
-import { getDb, getOrchestrator, insertRecoveredWorkspace, workspaceLandMetadataFrom, type RecoveredWorkspaceInput } from "../db";
+import { getDb, getOrchestrator, workspaceLandMetadataFrom, type RecoveredWorkspaceInput } from "../db";
 import { parseJson, isPathWithinBase } from "../utils";
 import { sanitizeFsName } from "@humwick/sdk/fs-name";
 import { hasScope } from "@humwick/sdk/types";
@@ -69,7 +70,7 @@ function resolveOwningOrchestrator(ctx: AuthContext, params: Record<string, unkn
   return null;
 }
 
-export function recoverIsolatedWorkspace(ctx: AuthContext, hint: WorkspaceRecoveryHint | undefined, pathId: string | undefined): WorkspaceRecovery {
+export async function recoverIsolatedWorkspace(ctx: AuthContext, hint: WorkspaceRecoveryHint | undefined, pathId: string | undefined): Promise<WorkspaceRecovery> {
   const caller = ctx.callerAgentId;
   if (!caller) return REFUSE;
   // Minting a land/merge target is a privileged write — require command:write, not baseline identity
@@ -163,6 +164,6 @@ export function recoverIsolatedWorkspace(ctx: AuthContext, hint: WorkspaceRecove
     ownerAgentId: caller,
     landMetadata: workspaceLandMetadataFrom(params),
   };
-  const result = insertRecoveredWorkspace(input);
+  const result = (await insertRecoveredWorkspaceAsync(input));
   return { ok: true, workspace: result.workspace, created: result.created };
 }

package/src/setup.ts

@@ -1,4 +1,4 @@
-import { writeChildEnv } from "@humwick/sdk/env";
+import { relayEnv, writeChildEnv } from "@humwick/sdk/env";
 import { randomBytes } from "node:crypto";
 import { shellQuote } from "@humwick/sdk/shell-utils";
 import { access, mkdir, readFile, writeFile } from "node:fs/promises";
@@ -70,18 +70,24 @@ export function generateToken(): string {
   return randomBytes(24).toString("hex");
 }
 
+/**
+ * The setup plan. `existing` holds the values of the managed env file already on disk: a re-run
+ * keeps them (above all the admin token) unless an option overrides them (#2680).
+ */
 export function createSetupPlan(
   options: SetupOptions = {},
   env: SetupEnvironment = {},
+  existing: Record<string, string> = {},
 ): SetupPlan {
   const configDir = defaultConfigDir(env);
   const stateDir = defaultStateDir(env);
   const logDir = defaultLogDir(env);
   const envFile = resolve(options.envFile ?? join(configDir, "env"));
-  const host = options.host ?? DEFAULT_HOST;
-  const port = normalizePort(options.port);
-  const token = options.token ?? (options.generateToken === false ? "" : generateToken());
-  const dbPath = resolve(options.dbPath ?? join(stateDir, "agent-relay.db"));
+  const host = options.host ?? existing.HOST ?? DEFAULT_HOST;
+  const port = normalizePort(options.port ?? (existing.PORT ? Number(existing.PORT) : undefined));
+  const token = options.token
+    ?? (options.generateToken === false ? "" : relayEnv(existing, "AGENT_RELAY_TOKEN") || generateToken());
+  const dbPath = resolve(options.dbPath ?? existing.DB_PATH ?? join(stateDir, "agent-relay.db"));
   const runtimePrefix = resolve(options.runtimePrefix ?? defaultRuntimePrefix(env));
   const tarballDir = runtimeTarballDir(runtimePrefix);
   const warnings: string[] = [];
@@ -91,12 +97,13 @@ export function createSetupPlan(
   }
 
   const values: Record<string, string> = {
+    RETENTION_DAYS: "30",
+    ...existing,
     PORT: String(port),
     HOST: host,
     DB_PATH: dbPath,
-    RETENTION_DAYS: "30",
   };
-  if (token) writeChildEnv(values, { "AGENT_RELAY_TOKEN": token });
+  writeChildEnv(values, token ? { "AGENT_RELAY_TOKEN": token } : {}, token ? [] : ["AGENT_RELAY_TOKEN"]);
 
   return {
     envFile,
@@ -111,6 +118,38 @@ export function createSetupPlan(
   };
 }
 
+/** The admin token a plan writes, or "" for a token-less relay. */
+export function setupPlanToken(plan: Pick<SetupPlan, "values">): string {
+  return relayEnv(plan.values, "AGENT_RELAY_TOKEN") ?? "";
+}
+
+/** `createSetupPlan` over the managed env file already at the target path, if any. */
+export async function resolveSetupPlan(options: SetupOptions = {}, env: SetupEnvironment = {}): Promise<SetupPlan & { reused: boolean }> {
+  const envFile = resolve(options.envFile ?? join(defaultConfigDir(env), "env"));
+  const content = await readIfExists(envFile);
+  const existing = content?.includes(SETUP_MARKER) ? parseEnvFile(content) : undefined;
+  return { ...createSetupPlan(options, env, existing), reused: existing !== undefined };
+}
+
+/** The values of an env file in the format `renderEnvFile` writes. */
+export function parseEnvFile(content: string): Record<string, string> {
+  const values: Record<string, string> = {};
+  for (const line of content.split("\n")) {
+    const match = /^([A-Za-z_][A-Za-z0-9_]*)=(.*)$/.exec(line.trim());
+    if (!match) continue;
+    const raw = match[2]!;
+    values[match[1]!] = raw.startsWith("'") && raw.endsWith("'") && raw.length >= 2 ? raw.slice(1, -1).replaceAll("'\\''", "'") : raw;
+  }
+  return values;
+}
+
+/** The plan without the admin token, for any output a person or a log may see. */
+export function redactSetupPlan<T extends SetupPlan>(plan: T): T {
+  const values = { ...plan.values };
+  for (const key of ["AGENT_RELAY_TOKEN", "HUMWICK_TOKEN"]) if (values[key]) values[key] = "<redacted>";
+  return { ...plan, values, content: redactEnv(plan.content) };
+}
+
 export async function executeSetupPlan(
   plan: SetupPlan,
   options: { dryRun?: boolean; force?: boolean } = {},
@@ -195,5 +234,5 @@ function normalizePort(port: number | undefined): number {
 }
 
 function redactEnv(content: string): string {
-  return content.replace(/^(AGENT_RELAY_TOKEN=).+$/m, "$1'<generated-token>'");
+  return content.replace(/^((?:AGENT_RELAY|HUMWICK)_TOKEN=).+$/gm, "$1'<generated-token>'");
 }

package/src/spawn-admission.ts

@@ -1,11 +1,18 @@
 import type { Orchestrator, SpawnProvider } from "./types";
+import { ValidationError } from "@humwick/sdk";
+
+export class PinnedHostCapacityError extends ValidationError {
+  constructor(message: string, readonly orchestratorId: string) {
+    super(message);
+  }
+}
 
 export interface AdmissionReason {
   check: "host" | "provider" | "auth" | "capacity";
   code: string;
   message: string;
 }
-export interface Admission { eligible: boolean; refusals: AdmissionReason[]; uncertainties: AdmissionReason[] }
+export interface Admission { eligible: boolean; refusals: AdmissionReason[]; uncertainties: AdmissionReason[]; agentCapOnly?: boolean }
 
 /** Interpret all probe legs independently: a failure on one never hides another leg's timeout. */
 export function providerProbeAdmission(item: Orchestrator, provider: SpawnProvider, pinned = false): Admission {
@@ -40,7 +47,8 @@ export function providerProbeAdmission(item: Orchestrator, provider: SpawnProvid
 
 export function assessSpawnAdmission(item: Orchestrator, provider: SpawnProvider, checks: AdmissionReason[] = [], pinned = false): Admission {
   const probe = providerProbeAdmission(item, provider, pinned);
-  return { eligible: probe.eligible && checks.length === 0, refusals: [...checks, ...probe.refusals], uncertainties: probe.uncertainties };
+  return { eligible: probe.eligible && checks.length === 0, refusals: [...checks, ...probe.refusals], uncertainties: probe.uncertainties,
+    agentCapOnly: probe.eligible && checks.length > 0 && checks.every(reason => reason.code === "agent_cap") };
 }
 
 export function admissionMessages(admission: Admission): string {

package/src/spawn-placement.ts

@@ -13,7 +13,7 @@ import { HOST_BAND_CONFIG } from "./config";
 import { hostProviderRefusal, liveHostDisabledProviders } from "./host-provider-store";
 
 import { hostSupportsModel } from "./spawn-models";
-import { admissionMessages, assessSpawnAdmission, type AdmissionReason, type Admission } from "./spawn-admission";
+import { admissionMessages, assessSpawnAdmission, PinnedHostCapacityError, type AdmissionReason, type Admission } from "./spawn-admission";
 
 const MATCHED_HOST_IDENTITY: HostIdentity = Object.freeze({ matched: true });
 
@@ -180,10 +180,14 @@ export function selectSpawnPlacement(
     const orchestrator = getOrchestrator(orchestratorId);
     if (!orchestrator) throw new McpNotFoundError(`orchestrator ${orchestratorId} not found`);
     const view = orchestrator.hostId ? byId.get(orchestrator.hostId) : undefined;
-    const admission = hostSpawnAdmission(orchestrator, view, identities, provider, model, effort, liveHostDisabledProviders(), true, spawnHostCapAdmission(capOpts)(view), repoPlatformAdmission(view, placementPlatforms));
+    const admission = hostSpawnAdmission(orchestrator, view, identities, provider, model, effort, liveHostDisabledProviders(), true, spawnHostCapAdmission(capOpts)(view), [
+      ...repoPlatformAdmission(view, placementPlatforms), ...workspaceHostAdmission(orchestrator, view, workspaceMode, canLand, capOpts, true),
+    ]);
     const detail = admissionMessages(admission);
     if (!admission.eligible) {
-      throw new ValidationError(spawnPlacementRefusal(listOrchestrators(), views, provider, model, effort, orchestratorId, capOpts, admission));
+      const message = spawnPlacementRefusal(listOrchestrators(), views, provider, model, effort, orchestratorId, capOpts, admission);
+      if (admission.agentCapOnly) throw new PinnedHostCapacityError(message, orchestratorId);
+      throw new ValidationError(message);
     }
     return { orchestrator, reason: `Explicit host pin: ${orchestrator.hostname} (${orchestrator.id})${detail ? `; ${detail}` : ""}.` };
   }
@@ -241,7 +245,25 @@ function headroomSummary(view: HostView | undefined): string {
   return `(${round(cores)} free cores, ${round(freeThreadsOf(view)!)} free threads, band ${view.band})`;
 }
 
-export interface SpawnCapOptions { callerAgentId?: string; attach?: boolean }
+export interface SpawnCapOptions {
+  callerAgentId?: string;
+  attach?: boolean;
+  buildWork?: boolean;
+  workspaceDepsDisabledHosts?: readonly string[];
+}
+
+/** Pins and landBlocked cannot turn isolated dependency/build work into a light hub probe (#3805). */
+function workspaceHostAdmission(item: Orchestrator, view: HostView | undefined, mode: WorkspaceMode | undefined, canLand: boolean, opts: SpawnCapOptions, pinned = false): AdmissionReason[] {
+  if (mode !== "isolated") return [];
+  const hubWork = (opts.buildWork ?? canLand) || (!opts.attach && !opts.workspaceDepsDisabledHosts?.includes(item.id));
+  if (hubWork && view?.policy.noLandingWorkspaces !== false && isRelayHost(item, view)) {
+    return [{ check: "host", code: "hub_workspace", message: `no builders on the hub: Relay server host ${item.hostname} (${item.id}) refuses isolated build or dependency-install workspaces, including pinned spawns and workflow steps; choose a non-hub host or set Fleet > Edit host > Isolated workspaces to Include this host (noLandingWorkspaces=false)` }];
+  }
+  if (pinned || !canLand) return [];
+  const landing = view?.policy.noLandingWorkspaces === true ? "host excludes landing workspaces"
+    : view?.policy.noLandingWorkspaces !== false && isRelayHost(item, view) ? "Relay server host excluded by default" : undefined;
+  return landing ? [{ check: "host", code: "landing", message: landing }] : [];
+}
 
 // Repository tools must run on the host even when the agent cannot land or pins a host (#3563).
 function repoPlatformAdmission(view: HostView | undefined, platforms?: readonly string[] | null): AdmissionReason[] {
@@ -282,11 +304,7 @@ export function spawnPlacementCandidates(
       const extra = repoPlatformAdmission(view, placementPlatforms);
       const capacity = capBlock(view);
       if (capacity) extra.push({ check: "capacity", code: "agent_cap", message: capacity });
-      if (isolatedAutoPlacement) {
-        const landing = view?.policy.noLandingWorkspaces === true ? "host excludes landing workspaces"
-          : view?.policy.noLandingWorkspaces !== false && isRelayHost(item, view) ? "Relay server host excluded by default" : undefined;
-        if (landing) extra.push({ check: "host", code: "landing", message: landing });
-      }
+      extra.push(...workspaceHostAdmission(item, view, workspaceMode, canLand, capOpts));
       const admission = hostSpawnAdmission(item, view, identities, provider, model, effort, hostDisabled, false, undefined, extra);
       if (!admission.eligible) exclusions.push(`${item.hostname}: ${admissionMessages(admission)}`);
       else if (admission.uncertainties.length) warnings.push(`${item.hostname}: ${admissionMessages(admission)}`);

package/src/spawn-targets.ts

@@ -1,6 +1,6 @@
 import { apiBudgetStatus } from "./db/claude-api-budget";
 import { apiBudgetStatusAsync } from "./claude-api-budget-async";
-import { isOpenAiApiModel, type ApiBudgetStatus } from "@humwick/sdk";
+import { API_BUDGET_IDS, isOpenAiApiModel, type ApiBudgetStatus } from "@humwick/sdk";
 import { collapseProviderQuotaRows } from "@humwick/sdk/provider-quota-order";
 import { resolveQuotaPacing, subscriptionExpiryForWindow, subscriptionExpiryPreference, type ResolvedQuotaPacing } from "./quota-pacing";
 import { countLiveSpawnedAgents, countLiveSpawnedAgentsAsync, getAgent, getAgentAsync, listOrchestrators, listOrchestratorsAsync, listProviderQuotas, providerQuotaRoutingSnapshotAsync } from "./db";
@@ -53,7 +53,7 @@ export function buildSpawnTargets(input: { callerId?: string; quota: number }):
   const me = input.callerId ? getAgent(input.callerId) : undefined;
   return buildSpawnTargetsFrom(input, {
     me,
-    openaiBudget: apiBudgetStatus(now, "openai-api"),
+    apiBudgets: API_BUDGET_IDS.map((id) => apiBudgetStatus(now, id)),
     orchestrators: listOrchestrators(),
     hostViews: listHostViews(now),
     liveChildren: input.callerId ? countLiveSpawnedAgents(input.callerId) : 0,
@@ -67,17 +67,17 @@ export function buildSpawnTargets(input: { callerId?: string; quota: number }):
 /** Worker-safe discovery path: every durable input is read through the reader gateway. */
 export async function buildSpawnTargetsAsync(input: { callerId?: string; quota: number }): Promise<Record<string, unknown>> {
   const now = Date.now();
-  const [me, orchestrators, hostViews, liveChildren, snapshot, openaiBudget] = await Promise.all([
+  const [me, orchestrators, hostViews, liveChildren, snapshot, apiBudgets] = await Promise.all([
     input.callerId ? getAgentAsync(input.callerId) : Promise.resolve(null),
     listOrchestratorsAsync(),
     listHostViewsAsync(),
     input.callerId ? countLiveSpawnedAgentsAsync(input.callerId) : Promise.resolve(0),
     providerQuotaRoutingSnapshotAsync(now),
-    apiBudgetStatusAsync("openai-api"),
+    Promise.all(API_BUDGET_IDS.map((id) => apiBudgetStatusAsync(id))),
   ]);
   return buildSpawnTargetsFrom(input, {
     me: me ?? undefined,
-    openaiBudget,
+    apiBudgets,
     orchestrators,
     hostViews,
     liveChildren,
@@ -90,7 +90,7 @@ export async function buildSpawnTargetsAsync(input: { callerId?: string; quota:
 
 function buildSpawnTargetsFrom(input: { callerId?: string; quota: number }, data: {
   me?: ReturnType<typeof getAgent>;
-  openaiBudget: ApiBudgetStatus;
+  apiBudgets: ApiBudgetStatus[];
   orchestrators: Orchestrator[];
   hostViews: readonly HostView[];
   liveChildren: number;
@@ -130,10 +130,10 @@ function buildSpawnTargetsFrom(input: { callerId?: string; quota: number }, data
           ...(o.utilization && Number.isFinite(o.utilization.agentSlotsUsedThisDaemon) && Number.isFinite(o.utilization.agentSlotsMaxThisDaemon) && o.utilization.agentSlotsMaxThisDaemon > 0
             ? { spawn_slots: { used: o.utilization.agentSlotsUsedThisDaemon, max: o.utilization.agentSlotsMaxThisDaemon } }
             : {}),
-          providers: providerMatrixFor(o, data.quotaAdvisories, data.catalog, data.profiles, hostDisabled, data.openaiBudget),
+          providers: providerMatrixFor(o, data.quotaAdvisories, data.catalog, data.profiles, hostDisabled, data.apiBudgets.find((budget) => budget.budgetId === "openai-api")),
         };
       }),
-    apiBudgets: [data.openaiBudget],
+    apiBudgets: data.apiBudgets.map((budget) => ({ ...budget, source: "relay-accounting" })),
     ...operatorDisabledProviders(data.catalog),
     profiles: spawnProfilesSummary(data.profiles),
     // #3327 — preferences steer automatic spawns inside the dispatcher; never copy them into explicit pins.

package/src/sse-agent-status.ts

@@ -2,15 +2,23 @@ import { isRecord } from "@humwick/sdk";
 
 const FULL_CARD_KEYS = ["name", "kind", "label", "tags", "systemTags", "instanceId", "epoch", "capabilities", "providerCapabilities"];
 
-/** Per-connection baselines: reconnect/replay starts with full cards, never another client's delta. */
+export interface AgentStatusChange {
+  previous?: number;
+  seq: number;
+  full: Record<string, unknown>;
+  delta: Record<string, unknown>;
+}
+
+/** Build once in source order; consumers need only the last sequence they received. */
 export class AgentStatusDeltas {
-  private readonly agents = new Map<string, Map<string, string | undefined>>();
+  private readonly agents = new Map<string, { seq: number; fields: Map<string, string | undefined> }>();
 
   remove(id: string): void { this.agents.delete(id); }
 
-  project(data: Record<string, unknown>): Record<string, unknown> {
+  project(data: Record<string, unknown>, seq: number): AgentStatusChange {
     const id = data.id as string;
-    const previous = this.agents.get(id);
+    const baseline = this.agents.get(id);
+    const previous = baseline?.fields;
     const fields = new Map<string, string | undefined>();
     const delta: Record<string, unknown> = { id, status: data.status, statusSeq: data.statusSeq, epoch: data.epoch };
     for (const [key, value] of Object.entries(data)) {
@@ -29,7 +37,8 @@ export class AgentStatusDeltas {
       }
     }
     if (data.status === "offline") this.remove(id);
-    else this.agents.set(id, fields);
-    return !previous || FULL_CARD_KEYS.some((key) => previous.get(key) !== fields.get(key)) ? data : delta;
+    else this.agents.set(id, { seq, fields });
+    return { previous: baseline?.seq, seq, full: data,
+      delta: !previous || FULL_CARD_KEYS.some((key) => previous.get(key) !== fields.get(key)) ? data : delta };
   }
 }