@humwick/server 0.130.2 → 0.130.3

package/src/services/workflow-runs.ts

@@ -4,6 +4,7 @@ import { spawnTreeContains } from "../db/spawn-tree";
 import { cleanSpawnBudget } from "../spawn-budget";
 import { readWorkflowRunBudgetAsync, reconcileRunBudgetAsync } from "../token-usage-async";
 import { getRoutingPolicyEnvelope } from "../routing-policy-service";
+import { resolveWorkflowRoutingDefaults } from "./workflow-routing-defaults";
 
 // #3045 — the workflow supervisor: mounts the kernel `workflow` module, starts runs, and pumps them.
 // A pass recovers crashed drivers, signals each waiting run from durable src facts (level-triggered,
@@ -14,10 +15,10 @@ import { errMessage, fireAndForget, stringValue, type WorkflowRunAgent, type Wor
 import { validateWorkflowValue } from "@humwick/kernel/workflow-values";
 import { OpError } from "@humwick/kernel";
 
-import { admitWorkflowCaller, assertWorkflowCallerLive } from "./workflow-caller";
+import { admitWorkflowCaller, admitWorkflowTaskOwner, assertWorkflowCallerLive, canControlWorkflowTask } from "./workflow-caller";
 import { taskVerificationCheck } from "../task-verification";
 
-import { workflow, workflowEffectId, type WorkflowEffect, type WorkflowRunRecord } from "@humwick/kernel/workflow.pilot";
+import { assertWorkflowFuturePins, workflow, workflowEffectId, type WorkflowEffect, type WorkflowRunRecord } from "@humwick/kernel/workflow.pilot";
 import { getAgent, getTaskDetailAsync, ValidationError } from "../db";
 import type { WorkflowRunCursor, WorkflowRunTaskRow } from "../db/workflow-task-read-ops";
 import { getGateway } from "../db/gateway";
@@ -31,7 +32,6 @@ import { McpNotFoundError } from "../mcp-errors";
 import { ServiceAuthError } from "./errors";
 import { canSpawn, spawnLineageAgentId, taskLinkedRepoCwd } from "./spawn-agent";
 import { resolveSpawnProjectWorkflow } from "./spawn-project-role";
-import { canControlTaskReviewHold } from "./task-review-hold";
 import { updateTaskEntityAsync } from "./task-entity";
 
 import { releaseRunHold } from "./workflow-effects";
@@ -77,19 +77,19 @@ export async function prepareWorkflowRun(input: { workflow: string; taskId: numb
   if (!task || task.kind !== "deliverable") throw new ValidationError("task not found");
   if (task.status === "canceled") throw new ValidationError(`task ${task.id} is canceled; a workflow runs on a Task that is still wanted`);
   const caller = admitWorkflowCaller(ctx, task);
-  if (!caller && !canControlTaskReviewHold(ctx, task)) {
+  if (!caller && !canControlWorkflowTask(ctx, task)) {
     throw new ServiceAuthError(`starting a workflow on task ${task.id} is coordinator-only: the run sets the review hold and clears it on GO — ${COORDINATOR_AUTHORITY_HINT}`);
   }
   if (!caller && !canSpawn(ctx)) throw new ServiceAuthError("starting a workflow dispatches its roles, so it requires the command:spawn scope");
   const lineageId = spawnLineageAgentId(ctx, task);
   const lineage = lineageId ? getAgent(lineageId) : null;
-  const cwd = input.cwd ?? await taskLinkedRepoCwd(task.id, stringValue(lineage?.meta?.cwd));
+  const cwd = input.cwd ?? await taskLinkedRepoCwd(task.id, stringValue(lineage?.meta?.cwd), lineage ?? undefined);
   const read = await resolveSpawnProjectWorkflow({ workflow: input.workflow, cwd, orchestratorId: input.orchestratorId, preferHost: lineage?.machine, allowedOrchestratorIds: ctx.constraints?.orchestrators });
   return { task, cwd, read };
 }
 
 /** relay_workflow_start: read the landed workflow and its roles, check every step's pin, start one run on the Task. */
-export async function startWorkflowRun(input: { workflow: string; taskId: number; pins?: WorkflowPins; inputs?: Json; requestKey?: string; budget?: unknown }, ctx: AuthContext): Promise<Json> {
+export async function startWorkflowRun(input: { workflow: string; taskId: number; cwd?: string; pins?: WorkflowPins; inputs?: Json; requestKey?: string; budget?: unknown }, ctx: AuthContext): Promise<Json> {
   return launchWorkflowRun(await prepareWorkflowRun(input, ctx), input, ctx);
 }
 
@@ -99,18 +99,17 @@ export async function launchWorkflowRun(prepared: PreparedWorkflowRun, input: {
   const { task, cwd, read } = prepared;
   const budget = cleanSpawnBudget(input.budget);
   const { definition } = read.workflow;
+  // #3769 — a Task a delegated sub-coordinator owns runs as its run, whoever starts it: the step spawns' lineage and the run's origin agree.
+  const caller = admitWorkflowCaller(ctx, task, read.workflow) ?? admitWorkflowTaskOwner(task, read.workflow);
   await getGateway().read("spawnTree.validateTarget", { type: "agent.spawn", source: ctx.actor.id, target: read.orchestratorId,
-    params: { spawnedBy: ctx.callerAgentId, taskId: task.id, cwd, orchestratorId: read.orchestratorId, profile: "worker", workflowRunId: "preflight" } });
-  const caller = admitWorkflowCaller(ctx, task, read.workflow);
+    params: { spawnedBy: caller?.callerAgentId ?? ctx.callerAgentId, taskId: task.id, cwd, orchestratorId: read.orchestratorId, profile: "worker", workflowRunId: "preflight" } });
   const capabilityError = workflowCapabilityError(definition);
   if (capabilityError) throw new ValidationError(capabilityError);
   validateWorkflowValue(definition.inputs, input.inputs ?? {}, "INPUT_INVALID");
   if (Object.values(definition.steps).some(step => step.verify) && !taskVerificationCheck(task.doneWhen)) throw new ValidationError(`T${task.id} needs a nonempty verify: section in doneWhen`);
   const pins = input.pins ? stampPins(definition, input.pins, "start") : undefined;
   const defaults = (await getRoutingPolicyEnvelope()).policy.stepDefaults;
-  const coordinatorId = ctx.callerAgentId ?? task.createdBy;
-  const teamId = task.teamId ?? (coordinatorId ? getAgent(coordinatorId)?.teamId : undefined);
-  const routingDefaults = defaults ? { global: defaults.global, ...(teamId ? { team: defaults.teams?.[teamId], teamId } : {}) } : undefined;
+  const routingDefaults = resolveWorkflowRoutingDefaults(defaults, task, ctx.callerAgentId);
   await assertStepLaunches({ definition, pins, routingDefaults, steps: Object.keys(definition.steps), roles: read.workflow.roles, inputTask: task }, ctx);
   for (const [name, snapshot] of Object.entries(read.workflow.closure ?? {})) {
     if (name === read.workflow.name) continue;
@@ -147,7 +146,7 @@ export async function launchWorkflowRun(prepared: PreparedWorkflowRun, input: {
   });
   if (!created && run.workflow !== read.workflow.name) throw new ValidationError(`task ${task.id} already runs workflow "${run.workflow}" (${run.id}); a Task has one open run`);
   if (!created && pins && !sameCallPins(run.pins, pins)) {
-    throw new ValidationError(`task ${task.id} already runs ${run.id} with other pins; a running run keeps its pins, and relay_workflow_resume changes them after an escalation`);
+    throw new ValidationError(`task ${task.id} already runs ${run.id} with other pins; use relay_workflow_resume with pins to change not-yet-started steps`);
   }
   if (!created && budget && budget.tokenCap !== run.tokenBudget?.tokenCap) {
     throw new ValidationError(`task ${task.id} already runs ${run.id} with another budget; a run keeps its original token cap`);
@@ -181,9 +180,9 @@ async function settleRefusable(sup: Supervisor, run: WorkflowRunRecord): Promise
   throw error;
 }
 
-/** A run is visible to its coordinator, the input Task's coordinators and admins. */
-export function canViewRun(ctx: AuthContext, coordinator: string, task: Parameters<typeof canControlTaskReviewHold>[1] | null): boolean {
-  return ctx.callerAgentId === coordinator || Boolean(ctx.callerAgentId && spawnTreeContains(ctx.callerAgentId, coordinator)) || Boolean(task && canControlTaskReviewHold(ctx, task));
+/** A run is visible to its coordinator, the input Task's coordinators, its owning sub-coordinator and admins. */
+export function canViewRun(ctx: AuthContext, coordinator: string, task: Parameters<typeof canControlWorkflowTask>[1] | null): boolean {
+  return ctx.callerAgentId === coordinator || Boolean(ctx.callerAgentId && spawnTreeContains(ctx.callerAgentId, coordinator)) || Boolean(task && canControlWorkflowTask(ctx, task));
 }
 
 /** #3352 — the run's cap and observed spend, as admission counts them. */
@@ -191,7 +190,7 @@ export async function readWorkflowRunBudget(runId: string, ctx: AuthContext): Pr
   const sup = await ensureSupervisor();
   const run = await readRun(sup, runId);
   if (!canViewRun(ctx, run.coordinator, await getTaskDetailAsync(run.taskId))) {
-    throw new ServiceAuthError(`workflow run ${runId} is visible to its coordinator, the Task's coordinators and admins — ${COORDINATOR_AUTHORITY_HINT}`);
+    throw new ServiceAuthError(`workflow run ${runId} is visible to its coordinator, the Task's coordinators, its owning sub-coordinator and admins — ${COORDINATOR_AUTHORITY_HINT}`);
   }
   if (!run.tokenBudget) throw new McpNotFoundError(`workflow run ${runId} has no token budget`);
   return await readWorkflowRunBudgetAsync(run.id, run.tokenBudget.tokenCap) as unknown as Json;
@@ -218,7 +217,7 @@ async function readRun(sup: Supervisor, runId: string): Promise<WorkflowRunRecor
 }
 
 /** relay_workflow_resume: answer the run's escalation by re-entering it at its escalated step or a named one (#3045 H2),
- *  replacing the pins of each step `pins` names (#3121). */
+ *  replacing the pins of each step `pins` names (#3121); pins alone on a running or paused run update future steps (#3806). */
 export async function resumeWorkflowRun(input: { runId: string; activationId?: string; step?: string; note?: string; pins?: WorkflowPins; decision?: WorkflowDecisionResponse }, ctx: AuthContext): Promise<Json> {
   const sup = await ensureSupervisor();
   const run = await readRun(sup, input.runId);
@@ -232,14 +231,27 @@ export async function resumeWorkflowRun(input: { runId: string; activationId?: s
     return { ...runView(await readRun(sup, run.id)), resumed: input.decision.action === "answer" };
   }
   const task = await getTaskDetailAsync(run.taskId);
-  if (!task || !(ctx.callerAgentId === run.callerAgentId || canControlTaskReviewHold(ctx, task))) {
+  if (!task || !(ctx.callerAgentId === run.callerAgentId || canControlWorkflowTask(ctx, task))) {
     throw new ServiceAuthError(`resuming workflow run ${run.id} is coordinator-only: the run sets the review hold and clears it on GO — ${COORDINATOR_AUTHORITY_HINT}`);
   }
   if (run.callerAgentId && ctx.actor.kind !== "system") assertWorkflowCallerLive(run.callerAgentId);
   if (!run.callerAgentId && !canSpawn(ctx)) throw new ServiceAuthError("resuming a workflow can dispatch its roles, so it requires the command:spawn scope");
   if (task.status === "canceled") throw new ValidationError(`T${task.id} is canceled, so its run ${run.id} cannot resume`);
   const pins = input.pins ? stampPins(run.definition, input.pins, "resume") : undefined;
-  if (pins) await assertResumePins(run, pins, task, ctx, await runAgents(sup, run.id));
+  if (pins && !input.step && !input.activationId && run.status === "running") {
+    await serialized(sup, async () => {
+      const latest = await readRun(sup, run.id);
+      assertWorkflowFuturePins(latest, pins);
+      await assertResumePins(latest, pins, task, ctx, await runAgents(sup, run.id), (op, args) => invoke(sup, op, args));
+      await invoke(sup, "amendPins", { runId: run.id, pins, by: ctx.callerAgentId ?? ctx.actor.id, note: input.note });
+    }).catch((error: unknown) => {
+      if (error instanceof OpError && (error.code === "conflict" || error.code === "invalid_input")) throw new ValidationError(error.message);
+      throw error;
+    });
+    await requestPump(sup);
+    return { ...runView(await readRun(sup, run.id)), resumed: false, pinsUpdated: true };
+  }
+  if (pins) await assertResumePins(run, pins, task, ctx, await runAgents(sup, run.id), (op, args) => invoke(sup, op, args));
   // Between passes, so no pass is performing the end-of-run reap the resume voids. A fact that already ends the
   // current wait (a delivery, a land) is signaled first and wins: the resume then refuses.
   const resume = serialized(sup, async () => {
@@ -277,12 +289,12 @@ export async function cancelWorkflowRun(input: { runId: string; reason?: string
   return { ...runView(await readRun(sup, run.id)), cancelled };
 }
 
-/** The run a coordinator-only control acts on: its caller, the input Task's coordinators and admins may. */
+/** The run a coordinator-only control acts on: its caller, the input Task's coordinators, its owning sub-coordinator and admins may. */
 async function controlledRun(runId: string, ctx: AuthContext, verb: string): Promise<{ sup: Supervisor; run: WorkflowRunRecord; by: string }> {
   const sup = await ensureSupervisor();
   const run = await readRun(sup, runId);
   const task = await getTaskDetailAsync(run.taskId);
-  if (!task || !(ctx.callerAgentId === run.callerAgentId || canControlTaskReviewHold(ctx, task))) {
+  if (!task || !(ctx.callerAgentId === run.callerAgentId || canControlWorkflowTask(ctx, task))) {
     throw new ServiceAuthError(`${verb} workflow run ${run.id} is coordinator-only — ${COORDINATOR_AUTHORITY_HINT}`);
   }
   return { sup, run, by: ctx.callerAgentId ?? ctx.actor.id };
@@ -310,7 +322,7 @@ export async function getWorkflowRun(runId: string, ctx: AuthContext, options: W
   await sup.notes;
   const run = await readRun(sup, runId);
   if (!canViewRun(ctx, run.coordinator, await getTaskDetailAsync(run.taskId))) {
-    throw new ServiceAuthError(`workflow run ${runId} is visible to its coordinator, the Task's coordinators and admins — ${COORDINATOR_AUTHORITY_HINT}`);
+    throw new ServiceAuthError(`workflow run ${runId} is visible to its coordinator, the Task's coordinators, its owning sub-coordinator and admins — ${COORDINATOR_AUTHORITY_HINT}`);
   }
   const scripts = options.verbose && Object.values(run.definition.steps).some(step => step.script)
     ? await invoke<{ items: Array<WorkflowScriptRun & { id: string; runId: string }>; cursor?: string }>(sup, "scriptAttempts", { runId, ...(options.scriptCursor ? { cursor: options.scriptCursor } : {}) }) : undefined;

package/src/services/workflow-supervisor-state.ts

@@ -1,7 +1,7 @@
 import { errMessage } from "@humwick/sdk";
 import type { ModuleHost, ModuleHostOptions } from "@humwick/kernel";
 import type { WorkflowRuntime } from "@humwick/kernel/workflow-runtime";
-import { workflow, type WorkflowAgent } from "@humwick/kernel/workflow.pilot";
+import { workflow, type WorkflowAgent, type WorkflowEffect } from "@humwick/kernel/workflow.pilot";
 import { getDbGeneration } from "../db";
 import { emitRelayEvent } from "../events";
 
@@ -50,6 +50,26 @@ export function supervisorNow(): number {
   return (testOptions.now ?? Date.now)();
 }
 
+const QUICK_RETRIES = 3;
+const RETRY_BASE_MS = 1000;
+const RETRY_MAX_MS = 60_000;
+
+/** #3827 — a retried effect's result with its next attempt. A stop still in flight (`inFlight`) is re-read next
+ *  pass; consecutive terminal misses (a failed stop, a gone target, a stop without proof) get a few quick retries,
+ *  then exponential backoff to a minute, so an effect that never converges costs one attempt a minute, not one a pass. */
+export function retryResult(previous: Json | undefined, result: Json, now: number): Json {
+  if (result.inFlight === true) return result;
+  const failures = (typeof previous?.failures === "number" ? previous.failures : 0) + 1;
+  const delay = failures <= QUICK_RETRIES ? 0 : Math.min(RETRY_MAX_MS, RETRY_BASE_MS * 2 ** Math.min(failures - QUICK_RETRIES - 1, 6));
+  return { ...result, failures, nextAttemptAt: now + delay };
+}
+
+/** Whether a pending effect's backoff has run out. */
+export function effectDue(effect: WorkflowEffect, now: number): boolean {
+  const at = effect.result?.nextAttemptAt;
+  return typeof at !== "number" || at <= now;
+}
+
 /** The supervisor bound to the current database handle (a test that swaps the handle rebuilds it). */
 export function live(): Supervisor | undefined {
   return current && current.generation === getDbGeneration() ? current : undefined;

package/src/services/workflow-supervisor.ts

@@ -1,4 +1,5 @@
 // #3045 — drives durable workflow transitions, reconciliation and external effects.
+import { framed } from "../hub-metrics";
 import { errMessage, fireAndForget, isRecord, stringValue } from "@humwick/sdk";
 import { createModuleHost, type ModuleHost } from "@humwick/kernel";
 import { workflow } from "@humwick/kernel/workflow.pilot";
@@ -57,10 +58,10 @@ function startSupervisor(): Supervisor {
     }),
     landed: new Map(), tail: predecessor, queued: undefined, notes: predecessor, refusable: new Map(),
   };
-  sup.ticker = setInterval(() => {
-    if (live() === sup) void fireAndForget("[workflow] supervisor pump", () => requestPump(sup));
+  sup.ticker = setInterval(framed("timer workflow-supervisor", () => {
+    if (live() === sup) return fireAndForget("[workflow] supervisor pump", () => requestPump(sup));
     else if (sup.ticker) clearInterval(sup.ticker);
-  }, workflowScriptsEnabled() ? 1000 : SCHEDULER_TICK_MS);
+  }), workflowScriptsEnabled() ? 1000 : SCHEDULER_TICK_MS);
   sup.ticker.unref?.();
   setSupervisor(sup);
   return sup;

package/src/services/workspace-recovery.ts

@@ -1,3 +1,4 @@
+import { insertRecoveredWorkspaceAsync } from "../db/runtime-admin-async";
 // #1502 — server-authoritative self-heal of a dropped/never-written isolated-workspace registration.
 //
 // A live worker holds its own workspace descriptor in AGENT_RELAY_WORKSPACE_JSON, but the workspace
@@ -18,7 +19,7 @@
 // 404), never a best-effort guess, because the output is a landing authorization.
 
 import { resolve } from "node:path";
-import { getDb, getOrchestrator, insertRecoveredWorkspace, workspaceLandMetadataFrom, type RecoveredWorkspaceInput } from "../db";
+import { getDb, getOrchestrator, workspaceLandMetadataFrom, type RecoveredWorkspaceInput } from "../db";
 import { parseJson, isPathWithinBase } from "../utils";
 import { sanitizeFsName } from "@humwick/sdk/fs-name";
 import { hasScope } from "@humwick/sdk/types";
@@ -69,7 +70,7 @@ function resolveOwningOrchestrator(ctx: AuthContext, params: Record<string, unkn
   return null;
 }
 
-export function recoverIsolatedWorkspace(ctx: AuthContext, hint: WorkspaceRecoveryHint | undefined, pathId: string | undefined): WorkspaceRecovery {
+export async function recoverIsolatedWorkspace(ctx: AuthContext, hint: WorkspaceRecoveryHint | undefined, pathId: string | undefined): Promise<WorkspaceRecovery> {
   const caller = ctx.callerAgentId;
   if (!caller) return REFUSE;
   // Minting a land/merge target is a privileged write — require command:write, not baseline identity
@@ -163,6 +164,6 @@ export function recoverIsolatedWorkspace(ctx: AuthContext, hint: WorkspaceRecove
     ownerAgentId: caller,
     landMetadata: workspaceLandMetadataFrom(params),
   };
-  const result = insertRecoveredWorkspace(input);
+  const result = (await insertRecoveredWorkspaceAsync(input));
   return { ok: true, workspace: result.workspace, created: result.created };
 }

package/src/spawn-admission.ts

@@ -1,11 +1,18 @@
 import type { Orchestrator, SpawnProvider } from "./types";
+import { ValidationError } from "@humwick/sdk";
+
+export class PinnedHostCapacityError extends ValidationError {
+  constructor(message: string, readonly orchestratorId: string) {
+    super(message);
+  }
+}
 
 export interface AdmissionReason {
   check: "host" | "provider" | "auth" | "capacity";
   code: string;
   message: string;
 }
-export interface Admission { eligible: boolean; refusals: AdmissionReason[]; uncertainties: AdmissionReason[] }
+export interface Admission { eligible: boolean; refusals: AdmissionReason[]; uncertainties: AdmissionReason[]; agentCapOnly?: boolean }
 
 /** Interpret all probe legs independently: a failure on one never hides another leg's timeout. */
 export function providerProbeAdmission(item: Orchestrator, provider: SpawnProvider, pinned = false): Admission {
@@ -40,7 +47,8 @@ export function providerProbeAdmission(item: Orchestrator, provider: SpawnProvid
 
 export function assessSpawnAdmission(item: Orchestrator, provider: SpawnProvider, checks: AdmissionReason[] = [], pinned = false): Admission {
   const probe = providerProbeAdmission(item, provider, pinned);
-  return { eligible: probe.eligible && checks.length === 0, refusals: [...checks, ...probe.refusals], uncertainties: probe.uncertainties };
+  return { eligible: probe.eligible && checks.length === 0, refusals: [...checks, ...probe.refusals], uncertainties: probe.uncertainties,
+    agentCapOnly: probe.eligible && checks.length > 0 && checks.every(reason => reason.code === "agent_cap") };
 }
 
 export function admissionMessages(admission: Admission): string {

package/src/spawn-placement.ts

@@ -13,7 +13,7 @@ import { HOST_BAND_CONFIG } from "./config";
 import { hostProviderRefusal, liveHostDisabledProviders } from "./host-provider-store";
 
 import { hostSupportsModel } from "./spawn-models";
-import { admissionMessages, assessSpawnAdmission, type AdmissionReason, type Admission } from "./spawn-admission";
+import { admissionMessages, assessSpawnAdmission, PinnedHostCapacityError, type AdmissionReason, type Admission } from "./spawn-admission";
 
 const MATCHED_HOST_IDENTITY: HostIdentity = Object.freeze({ matched: true });
 
@@ -180,10 +180,14 @@ export function selectSpawnPlacement(
     const orchestrator = getOrchestrator(orchestratorId);
     if (!orchestrator) throw new McpNotFoundError(`orchestrator ${orchestratorId} not found`);
     const view = orchestrator.hostId ? byId.get(orchestrator.hostId) : undefined;
-    const admission = hostSpawnAdmission(orchestrator, view, identities, provider, model, effort, liveHostDisabledProviders(), true, spawnHostCapAdmission(capOpts)(view), repoPlatformAdmission(view, placementPlatforms));
+    const admission = hostSpawnAdmission(orchestrator, view, identities, provider, model, effort, liveHostDisabledProviders(), true, spawnHostCapAdmission(capOpts)(view), [
+      ...repoPlatformAdmission(view, placementPlatforms), ...workspaceHostAdmission(orchestrator, view, workspaceMode, canLand, capOpts, true),
+    ]);
     const detail = admissionMessages(admission);
     if (!admission.eligible) {
-      throw new ValidationError(spawnPlacementRefusal(listOrchestrators(), views, provider, model, effort, orchestratorId, capOpts, admission));
+      const message = spawnPlacementRefusal(listOrchestrators(), views, provider, model, effort, orchestratorId, capOpts, admission);
+      if (admission.agentCapOnly) throw new PinnedHostCapacityError(message, orchestratorId);
+      throw new ValidationError(message);
     }
     return { orchestrator, reason: `Explicit host pin: ${orchestrator.hostname} (${orchestrator.id})${detail ? `; ${detail}` : ""}.` };
   }
@@ -241,7 +245,25 @@ function headroomSummary(view: HostView | undefined): string {
   return `(${round(cores)} free cores, ${round(freeThreadsOf(view)!)} free threads, band ${view.band})`;
 }
 
-export interface SpawnCapOptions { callerAgentId?: string; attach?: boolean }
+export interface SpawnCapOptions {
+  callerAgentId?: string;
+  attach?: boolean;
+  buildWork?: boolean;
+  workspaceDepsDisabledHosts?: readonly string[];
+}
+
+/** Pins and landBlocked cannot turn isolated dependency/build work into a light hub probe (#3805). */
+function workspaceHostAdmission(item: Orchestrator, view: HostView | undefined, mode: WorkspaceMode | undefined, canLand: boolean, opts: SpawnCapOptions, pinned = false): AdmissionReason[] {
+  if (mode !== "isolated") return [];
+  const hubWork = (opts.buildWork ?? canLand) || (!opts.attach && !opts.workspaceDepsDisabledHosts?.includes(item.id));
+  if (hubWork && view?.policy.noLandingWorkspaces !== false && isRelayHost(item, view)) {
+    return [{ check: "host", code: "hub_workspace", message: `no builders on the hub: Relay server host ${item.hostname} (${item.id}) refuses isolated build or dependency-install workspaces, including pinned spawns and workflow steps; choose a non-hub host or set Fleet > Edit host > Isolated workspaces to Include this host (noLandingWorkspaces=false)` }];
+  }
+  if (pinned || !canLand) return [];
+  const landing = view?.policy.noLandingWorkspaces === true ? "host excludes landing workspaces"
+    : view?.policy.noLandingWorkspaces !== false && isRelayHost(item, view) ? "Relay server host excluded by default" : undefined;
+  return landing ? [{ check: "host", code: "landing", message: landing }] : [];
+}
 
 // Repository tools must run on the host even when the agent cannot land or pins a host (#3563).
 function repoPlatformAdmission(view: HostView | undefined, platforms?: readonly string[] | null): AdmissionReason[] {
@@ -282,11 +304,7 @@ export function spawnPlacementCandidates(
       const extra = repoPlatformAdmission(view, placementPlatforms);
       const capacity = capBlock(view);
       if (capacity) extra.push({ check: "capacity", code: "agent_cap", message: capacity });
-      if (isolatedAutoPlacement) {
-        const landing = view?.policy.noLandingWorkspaces === true ? "host excludes landing workspaces"
-          : view?.policy.noLandingWorkspaces !== false && isRelayHost(item, view) ? "Relay server host excluded by default" : undefined;
-        if (landing) extra.push({ check: "host", code: "landing", message: landing });
-      }
+      extra.push(...workspaceHostAdmission(item, view, workspaceMode, canLand, capOpts));
       const admission = hostSpawnAdmission(item, view, identities, provider, model, effort, hostDisabled, false, undefined, extra);
       if (!admission.eligible) exclusions.push(`${item.hostname}: ${admissionMessages(admission)}`);
       else if (admission.uncertainties.length) warnings.push(`${item.hostname}: ${admissionMessages(admission)}`);

package/src/spawn-targets.ts

@@ -1,6 +1,6 @@
 import { apiBudgetStatus } from "./db/claude-api-budget";
 import { apiBudgetStatusAsync } from "./claude-api-budget-async";
-import { isOpenAiApiModel, type ApiBudgetStatus } from "@humwick/sdk";
+import { API_BUDGET_IDS, isOpenAiApiModel, type ApiBudgetStatus } from "@humwick/sdk";
 import { collapseProviderQuotaRows } from "@humwick/sdk/provider-quota-order";
 import { resolveQuotaPacing, subscriptionExpiryForWindow, subscriptionExpiryPreference, type ResolvedQuotaPacing } from "./quota-pacing";
 import { countLiveSpawnedAgents, countLiveSpawnedAgentsAsync, getAgent, getAgentAsync, listOrchestrators, listOrchestratorsAsync, listProviderQuotas, providerQuotaRoutingSnapshotAsync } from "./db";
@@ -53,7 +53,7 @@ export function buildSpawnTargets(input: { callerId?: string; quota: number }):
   const me = input.callerId ? getAgent(input.callerId) : undefined;
   return buildSpawnTargetsFrom(input, {
     me,
-    openaiBudget: apiBudgetStatus(now, "openai-api"),
+    apiBudgets: API_BUDGET_IDS.map((id) => apiBudgetStatus(now, id)),
     orchestrators: listOrchestrators(),
     hostViews: listHostViews(now),
     liveChildren: input.callerId ? countLiveSpawnedAgents(input.callerId) : 0,
@@ -67,17 +67,17 @@ export function buildSpawnTargets(input: { callerId?: string; quota: number }):
 /** Worker-safe discovery path: every durable input is read through the reader gateway. */
 export async function buildSpawnTargetsAsync(input: { callerId?: string; quota: number }): Promise<Record<string, unknown>> {
   const now = Date.now();
-  const [me, orchestrators, hostViews, liveChildren, snapshot, openaiBudget] = await Promise.all([
+  const [me, orchestrators, hostViews, liveChildren, snapshot, apiBudgets] = await Promise.all([
     input.callerId ? getAgentAsync(input.callerId) : Promise.resolve(null),
     listOrchestratorsAsync(),
     listHostViewsAsync(),
     input.callerId ? countLiveSpawnedAgentsAsync(input.callerId) : Promise.resolve(0),
     providerQuotaRoutingSnapshotAsync(now),
-    apiBudgetStatusAsync("openai-api"),
+    Promise.all(API_BUDGET_IDS.map((id) => apiBudgetStatusAsync(id))),
   ]);
   return buildSpawnTargetsFrom(input, {
     me: me ?? undefined,
-    openaiBudget,
+    apiBudgets,
     orchestrators,
     hostViews,
     liveChildren,
@@ -90,7 +90,7 @@ export async function buildSpawnTargetsAsync(input: { callerId?: string; quota:
 
 function buildSpawnTargetsFrom(input: { callerId?: string; quota: number }, data: {
   me?: ReturnType<typeof getAgent>;
-  openaiBudget: ApiBudgetStatus;
+  apiBudgets: ApiBudgetStatus[];
   orchestrators: Orchestrator[];
   hostViews: readonly HostView[];
   liveChildren: number;
@@ -130,10 +130,10 @@ function buildSpawnTargetsFrom(input: { callerId?: string; quota: number }, data
           ...(o.utilization && Number.isFinite(o.utilization.agentSlotsUsedThisDaemon) && Number.isFinite(o.utilization.agentSlotsMaxThisDaemon) && o.utilization.agentSlotsMaxThisDaemon > 0
             ? { spawn_slots: { used: o.utilization.agentSlotsUsedThisDaemon, max: o.utilization.agentSlotsMaxThisDaemon } }
             : {}),
-          providers: providerMatrixFor(o, data.quotaAdvisories, data.catalog, data.profiles, hostDisabled, data.openaiBudget),
+          providers: providerMatrixFor(o, data.quotaAdvisories, data.catalog, data.profiles, hostDisabled, data.apiBudgets.find((budget) => budget.budgetId === "openai-api")),
         };
       }),
-    apiBudgets: [data.openaiBudget],
+    apiBudgets: data.apiBudgets.map((budget) => ({ ...budget, source: "relay-accounting" })),
     ...operatorDisabledProviders(data.catalog),
     profiles: spawnProfilesSummary(data.profiles),
     // #3327 — preferences steer automatic spawns inside the dispatcher; never copy them into explicit pins.

package/src/sse-agent-status.ts

@@ -2,15 +2,23 @@ import { isRecord } from "@humwick/sdk";
 
 const FULL_CARD_KEYS = ["name", "kind", "label", "tags", "systemTags", "instanceId", "epoch", "capabilities", "providerCapabilities"];
 
-/** Per-connection baselines: reconnect/replay starts with full cards, never another client's delta. */
+export interface AgentStatusChange {
+  previous?: number;
+  seq: number;
+  full: Record<string, unknown>;
+  delta: Record<string, unknown>;
+}
+
+/** Build once in source order; consumers need only the last sequence they received. */
 export class AgentStatusDeltas {
-  private readonly agents = new Map<string, Map<string, string | undefined>>();
+  private readonly agents = new Map<string, { seq: number; fields: Map<string, string | undefined> }>();
 
   remove(id: string): void { this.agents.delete(id); }
 
-  project(data: Record<string, unknown>): Record<string, unknown> {
+  project(data: Record<string, unknown>, seq: number): AgentStatusChange {
     const id = data.id as string;
-    const previous = this.agents.get(id);
+    const baseline = this.agents.get(id);
+    const previous = baseline?.fields;
     const fields = new Map<string, string | undefined>();
     const delta: Record<string, unknown> = { id, status: data.status, statusSeq: data.statusSeq, epoch: data.epoch };
     for (const [key, value] of Object.entries(data)) {
@@ -29,7 +37,8 @@ export class AgentStatusDeltas {
       }
     }
     if (data.status === "offline") this.remove(id);
-    else this.agents.set(id, fields);
-    return !previous || FULL_CARD_KEYS.some((key) => previous.get(key) !== fields.get(key)) ? data : delta;
+    else this.agents.set(id, { seq, fields });
+    return { previous: baseline?.seq, seq, full: data,
+      delta: !previous || FULL_CARD_KEYS.some((key) => previous.get(key) !== fields.get(key)) ? data : delta };
   }
 }

package/src/sse.ts

@@ -1,7 +1,8 @@
+import { sendMessageWithResultAsync } from "./db/message-send";
 import { eventVisibleTo } from "./sse-visibility";
 import { taskVisibleTo, memoizeViewerScope } from "./db/viewer-scope";
 import { agentPrincipalSink } from "./db/agent-ownership";
-import { getAgent, getMessage, getOperatorDecision, getOrchestrator, matchingDeliveryAgents, onWorkspaceChange, sendMessageWithResult, setChannelSubscriberProbe } from "./db";
+import { getAgent, getMessage, getOperatorDecision, getOrchestrator, matchingDeliveryAgents, onWorkspaceChange, setChannelSubscriberProbe } from "./db";
 import { hasReplayGap, type BusEvent } from "./bus-outbox";
 import { replayEventsSliceAsync } from "./bus-outbox-async";
 import { getFannedSeq } from "./sse-outbox-cursor";
@@ -14,10 +15,11 @@ import type { ActivityEvent, AgentCard, Message, RelayInjectionCategory, RelayNo
 import { isHumanAgentId } from "@humwick/sdk";
 import { messageVisibleTo, visibleMessages, type MessageViewer } from "./message-visibility";
 import { emitNotificationCreated, notificationForMessage, notificationForDelivery } from "./dashboard-notifications";
-import { AgentStatusDeltas } from "./sse-agent-status";
+import { AgentStatusDeltas, type AgentStatusChange } from "./sse-agent-status";
+import { getDbGeneration } from "./db/connection";
 import { AgentStatusSubscribers, wantsAgentStatus } from "./transport-subscriptions";
 import { withLoopFrameCapture } from "./loop-block-attribution";
-import { recordEmitBytes } from "./hub-metrics";
+import { framed, recordEmitBytes } from "./hub-metrics";
 
 interface Connection {
   id: string;
@@ -27,7 +29,8 @@ interface Connection {
   queuedBytes: number;
   /** Bytes ever enqueued on the controller; the replay pump measures its byte budget against it. */
   writtenBytes: number;
-  statusDeltas?: AgentStatusDeltas;
+  statusDeltas?: Map<string, number>;
+  replayDeltas?: AgentStatusDeltas;
   // Pending backpressure re-check. Set when a stream crosses the byte floor; cleared
   // once it drains or is closed after making no progress during the grace interval.
   drainCheck?: Timer;
@@ -46,6 +49,8 @@ interface Connection {
 const connections = new Map<string, Connection>();
 const statusSubscribers = new AgentStatusSubscribers<Connection>("sse");
 const encoder = new TextEncoder();
+let sourceDeltas = new AgentStatusDeltas();
+let statusGeneration = getDbGeneration();
 
 const SSE_KEEPALIVE_MS = 15_000;
 const SSE_RETRY_MS = 5_000;
@@ -65,7 +70,7 @@ subscribeRelayEvents((event) => {
   fanoutBytes = 0;
   withLoopFrameCapture(FANOUT_FRAME, () => fanout(event));
   recordEmitBytes(event.type, fanoutBytes);
-});
+}, "sse.fanout");
 
 function fanoutAgent(agentId: string, cache: Map<string, AgentCard | null>): AgentCard | null {
   if (!cache.has(agentId)) cache.set(agentId, getAgent(agentId));
@@ -90,14 +95,14 @@ export function createSSEStream(agentId: string | null, lastEventId = 0, statusD
     {
       start(controller) {
         connId = crypto.randomUUID();
-        const keepalive = setInterval(() => {
+        const keepalive = setInterval(framed("timer sse-keepalive", () => {
           const conn = connections.get(connId);
           if (!conn) return;
           sendEncoded(conn, encoder.encode(": keepalive\n\n"));
-        }, SSE_KEEPALIVE_MS);
+        }), SSE_KEEPALIVE_MS);
 
         connections.set(connId, { id: connId, agentId, controller, keepalive, queuedBytes: 0, writtenBytes: 0,
-          statusDeltas: statusDeltas ? new AgentStatusDeltas() : undefined, viewer });
+          statusDeltas: statusDeltas ? new Map() : undefined, viewer });
         statusSubscribers.add(connections.get(connId)!);
 
         const replay = replayPlan(lastEventId, agentId === null);
@@ -216,18 +221,34 @@ function sendDirect(conn: Connection, event: string, data: unknown, id?: number)
   writeFrame(conn, sseFrame(event, data, id));
 }
 
-// #2873/#3640 — one agent.status projection for live, buffered-live and replayed frames. Every frame goes
-// through it in the order the client receives it, so the per-connection delta baseline matches the client.
+// #3608 — share live deltas and encoded frames; select full/delta in client order even during replay.
 function tracksStatus(conn: Connection, event: RelayEvent): boolean {
   return conn.statusDeltas !== undefined && (event.type === "agent.status" || event.type === "agent.removed");
 }
 
+// An ephemeral frame carries no `id:` line: it must not move the client's Last-Event-ID resume cursor.
+function frameId(event: RelayEvent): number | undefined {
+  return event.ephemeral ? undefined : event.seq;
+}
+
+// Ephemeral events all carry seq 0, so they take a unique negative baseline token: two live-only
+// frames must never look like the same client-visible state.
+let ephemeralStatusToken = 0;
+
 // Reads `conn.statusDeltas` when called, not when queued: a replay.gap swaps in a fresh baseline.
-function statusFrame(conn: Connection, event: RelayEvent, full?: Uint8Array): Uint8Array {
-  const deltas = conn.statusDeltas!;
-  if (event.type === "agent.status") return sseFrame(event.type, deltas.project(event.data), event.seq);
-  deltas.remove(event.data.id as string);
-  return full ?? sseFrame(event.type, event.data, event.seq);
+function prepareStatusFrame(event: RelayEvent, deltas: AgentStatusDeltas, full = sseFrame(event.type, event.data, frameId(event))) {
+  const id = event.data.id as string;
+  const token = event.ephemeral ? --ephemeralStatusToken : event.seq;
+  const change: AgentStatusChange | undefined = event.type === "agent.status" ? deltas.project(event.data, token) : undefined;
+  if (!change) deltas.remove(id);
+  let delta: Uint8Array | undefined;
+  return (conn: Connection): Uint8Array => {
+    const previous = conn.statusDeltas!.get(id);
+    if (event.data.status === "offline" || !change) conn.statusDeltas!.delete(id);
+    else conn.statusDeltas!.set(id, token);
+    return change && previous !== undefined && previous === change.previous
+      ? (delta ??= change.delta === change.full ? full : sseFrame(event.type, change.delta, frameId(event))) : full;
+  };
 }
 
 function flushPendingLive(conn: Connection): void {
@@ -282,14 +303,17 @@ function fanout(event: RelayEvent): void {
     sendTaskChanged(event.data as unknown as Task, event.type, event.seq);
     return;
   }
-  // An ephemeral frame carries no `id:` line: it must not move the client's Last-Event-ID resume cursor.
-  const id = event.ephemeral ? undefined : event.seq;
-  const chunk = sseFrame(event.type, event.data, id);
+  const chunk = sseFrame(event.type, event.data, frameId(event));
+  if (statusGeneration !== getDbGeneration()) {
+    sourceDeltas = new AgentStatusDeltas();
+    statusGeneration = getDbGeneration();
+  }
+  const status = event.type === "agent.status" || event.type === "agent.removed" ? prepareStatusFrame(event, sourceDeltas, chunk) : undefined;
   const recipients = event.type === "agent.status" ? statusSubscribers.forEvent(event.subject, event.data.id) : connections.values();
   const visible = memoizeViewerScope((viewer) => eventVisibleTo(viewer, event));
   for (const conn of recipients) {
     if (!visible(conn.viewer)) continue;
-    sendEncoded(conn, chunk, tracksStatus(conn, event) ? () => statusFrame(conn, event, chunk) : undefined);
+    sendEncoded(conn, chunk, conn.statusDeltas && status ? () => status(conn) : undefined);
   }
 }
 
@@ -394,9 +418,9 @@ interface RelayInjectionTimelineInput {
   idempotencyKey?: string;
 }
 
-export function emitRelayInjectionEvent(input: RelayInjectionTimelineInput): Message {
+export async function emitRelayInjectionEvent(input: RelayInjectionTimelineInput): Promise<Message> {
   const timestamp = input.timestamp ?? Date.now();
-  const result = sendMessageWithResult({
+  const result = (await sendMessageWithResultAsync({
     from: input.source ?? input.agentId,
     to: input.source ? input.agentId : agentPrincipalSink(input.agentId),
     kind: "session",
@@ -416,7 +440,7 @@ export function emitRelayInjectionEvent(input: RelayInjectionTimelineInput): Mes
         },
       },
     },
-  });
+  }));
   if (result.created || result.updated) emitNewMessage(result.message);
   return result.message;
 }
@@ -561,7 +585,7 @@ async function pumpReplay(connId: string, plan: ReplayPlan): Promise<void> {
         // The client's REST reload covers everything up to the ceiling; the buffered live tail follows.
         // That reload brings full cards, so restart the delta baseline as a fresh connection would.
         sendDirect(conn, "replay.gap", { since: plan.since }, ceiling);
-        if (conn.statusDeltas) conn.statusDeltas = new AgentStatusDeltas();
+        conn.statusDeltas?.clear();
         break;
       }
       if (!slice.truncated) break;
@@ -570,6 +594,7 @@ async function pumpReplay(connId: string, plan: ReplayPlan): Promise<void> {
     drained = true;
   } finally {
     conn.replaying = false;
+    conn.replayDeltas = undefined;
     // Only flush into a still-open controller — a cancelled connection was already removed from
     // `connections` (and stopped receiving anything into `pendingLive` the moment it was removed,
     // since fanout only iterates live map entries), so there is nothing left to deliver.
@@ -615,7 +640,7 @@ function sendReplayEvent(conn: Connection, event: RelayEvent, agentCache: Map<st
     return;
   }
   if (tracksStatus(conn, event)) {
-    writeFrame(conn, statusFrame(conn, event));
+    writeFrame(conn, prepareStatusFrame(event, conn.replayDeltas ??= new AgentStatusDeltas())(conn));
     return;
   }
   sendDirect(conn, event.type, event.data, event.seq);