@humwick/server 0.130.1 → 0.130.3

package/src/services/restart-spawn.ts

@@ -1,3 +1,5 @@
+import { runnerRuntimeTokenEnvAsync, SUCCESSOR_LAUNCH, type RunnerRuntimeTokenInput } from "../runtime-tokens";
+import { updateManagedAgentStateAsync } from "../db/lifecycle-tail-async";
 import { createCommandAsync } from "../commands-write";
 import { restartHarnessSubagents } from "./harness-subagents";
 import { RELAY_TOKEN_HEADER, isRecord, isSpawnProvider, stringValue } from "@humwick/sdk";
@@ -5,7 +7,7 @@ import { recordedLaunchMode } from "../launch-mode";
 import type { ProviderEffort } from "@humwick/sdk/provider-catalog";
 import { listCommands } from "../commands-db";
 import { isTrustedOrchestratorApiUrl, relayToken } from "../config";
-import { getAgentProfile, getManagedAgentState, getSpawnPolicy, updateManagedAgentState } from "../config-store";
+import { getAgentProfile, getManagedAgentState, getSpawnPolicy } from "../config-store";
 import { getMergeLease, getOrchestrator, listWorkspaces, ownedIsolatedWorkspace, ValidationError, withResolvedProvisioning, withResolvedProvisioningAsync, workflowRunIdOf } from "../db";
 import { resolveProjectForCanonicalCwd } from "./project";
 import { isProvenStewardAgent, stewardWorktreeCwdPrefixes } from "../steward-spawn-guard";
@@ -16,7 +18,6 @@ import { resolveWhoamiRole } from "../mcp/identity-projection";
 import { listManagedOrchestratorsForAgent } from "../orchestrator-lookup";
 import { applyDefaultSharedMcp } from "../shared-mcp";
 import { buildSpawnCommand, generateSpawnRequestId, resolveSpawnModelParams, resolveSpawnModelParamsAsync } from "../spawn-command";
-import { SUCCESSOR_LAUNCH, runnerRuntimeTokenEnv } from "../runtime-tokens";
 import { restartEffortState } from "./self-effort";
 // #2033 — from isolated-workspace-placement.ts, NOT ./spawn-identity: that module also exports
 // buildProspectiveSpawnIdentity, which imports mcp/identity.ts → self-resume.ts → restart-spawn.ts,
@@ -27,7 +28,7 @@ import { restartInstructionParamsForAgent, restartInstructionParamsForAgentAsync
 import { launchedProjectRole, projectRoleRestartError, refuseSyncProjectRoleRestart, restartProjectRole } from "./restart-project-role";
 import { authContextFromSystem } from "./auth-context";
 import { markManagedAgentRunning } from "./managed-running";
-import type { AgentCard, AgentProfile, Command, Orchestrator, SpawnApprovalMode } from "../types";
+import type { AgentCard, AgentProfile, SpawnPolicy, Command, Orchestrator, SpawnApprovalMode } from "../types";
 
 export type SelfRestartReason = "upgrade" | "token-remint" | "manual" | "reset-unconfirmed";
 type RestartOptions = { resumeId?: string; continuation?: string; generation?: number; reason?: string };
@@ -65,13 +66,13 @@ function resumeWorkspaceForRestart(agent: AgentCard): { branch: string; mode: "a
  * self-restart. The token is minted here from the CURRENT profile, never cloned from the old
  * runtime token, so changed scopes/tool groups take effect on the successor.
  */
-function buildRestartSpawnParamsForAgent(
+async function buildRestartSpawnParamsForAgent(
   agent: AgentCard,
   orchestrator: Orchestrator | null,
   policyName?: string,
   predecessorSpawnRequestId?: string,
   opts: RestartOptions = {},
-): Record<string, unknown> | undefined {
+): Promise<Record<string, unknown> | undefined> {
   if (!orchestrator) return undefined;
   refuseSyncProjectRoleRestart(agent);
   const requestId = generateSpawnRequestId();
@@ -110,36 +111,15 @@ function buildRestartSpawnParamsForAgent(
       resumeWorktreePath: restartResumeWorkspace?.worktreePath,
     });
     const params = {
-      ...buildManagedSpawnParams(relaunchPolicy(policy.value, relaunch.effort), requestId, { createdBy: requestedBy, authority: SUCCESSOR_LAUNCH, identityAgentId: agent.id }),
+      ...(await buildManagedSpawnParams(relaunchPolicy(policy.value, relaunch.effort), requestId, { createdBy: requestedBy, authority: SUCCESSOR_LAUNCH, identityAgentId: agent.id })),
       ...relaunch.carried,
       agentId: agent.id,
       requestedBy,
       // Override the policy builder's normal mint so stable team/project bindings are also
       // carried by the fresh token, not merely preserved incidentally on the agent row.
-      env: runnerRuntimeTokenEnv({ authority: SUCCESSOR_LAUNCH,
-        agentId: agent.id,
-        orchestratorId: policy.value.orchestratorId,
-        cwd: policy.value.cwd,
-        provider: policy.value.provider,
-        label: policy.value.label,
-        policyName: policy.value.name,
-        spawnRequestId: requestId,
-        createdBy: requestedBy,
-        profile: policy.value.profile || undefined,
-        // #2245 — the successor's posture comes from the POLICY (server config, unforgeable), the
-        // same value `buildManagedSpawnParams` launches it with; stamp it into the signed constraints.
-        approvalMode: policy.value.permissionMode,
-        ...(agent.teamId ? { teamId: agent.teamId } : {}),
-        ...(projectId ? { projectId } : {}),
-        ...([...stewardExtraCwdPrefixes, ...isolatedExtraCwdPrefixes].length
-          ? { extraCwdPrefixes: [...stewardExtraCwdPrefixes, ...isolatedExtraCwdPrefixes] }
-          : {}),
-      }),
-      ...(opts.continuation ? { prompt: opts.continuation } : {}),
-      ...(restartResumeWorkspace ? { resumeWorkspace: restartResumeWorkspace } : {}),
-      // F5 — carry the self-restart generation marker on BOTH successor spawns (policy and
-      // dashboard/non-policy), so the audit lineage is symmetric regardless of branch.
-      ...(opts.generation ? { extra: { selfRestart: { generation: opts.generation, reason: opts.reason } } } : {}),
+      env: (await runnerRuntimeTokenEnvAsync(managedRestartTokenInput(agent, policy.value, requestId, requestedBy, projectId,
+        [...stewardExtraCwdPrefixes, ...isolatedExtraCwdPrefixes]))),
+      ...managedRestartOptions(opts, restartResumeWorkspace),
     };
     return opts.resumeId ? withResumeParams(params, opts.resumeId, agent.id) : params;
   }
@@ -147,7 +127,7 @@ function buildRestartSpawnParamsForAgent(
   const provider = stringValue(agent.meta?.provider);
   if (!isSpawnProvider(provider)) return undefined;
   // #2033 review — fold in #2041: for an ISOLATED agent, `agent.meta.cwd` is caller-writable
-  // (not an AUTHZ_META_KEY) and this `cwd` feeds `runnerRuntimeTokenEnv({cwd})` below as the
+  // (not an AUTHZ_META_KEY) and this `cwd` feeds `runnerRuntimeTokenEnvAsync({cwd})` below as the
   // successor token's cwdPrefixes[0] — a forged value would mint a token authorizing an arbitrary
   // path. Prefer the same unforgeable orchestrator-written DB row (`ownedIsolatedWorkspace`) G1
   // sources the resume worktree from; it's undefined for a non-isolated (shared/coordinator) agent
@@ -160,7 +140,7 @@ function buildRestartSpawnParamsForAgent(
   const resolvedProfile = withResolvedProvisioning(profileName ? getAgentProfile(profileName)?.value : undefined, provider);
   const agentProfile = resolvedProfile ? applyDefaultSharedMcp(resolvedProfile) : undefined;
   const workspaceMode = stringValue(agent.meta?.workspaceMode) ?? "inherit";
-  const resolvedModel = resolveSpawnModelParams(provider, model, effort, { modelSource: "runtime", skipDefaultWhenEmpty: true });
+  const resolvedModel = resolveSpawnModelParams(provider, model, effort, { modelSource: "runtime" });
   const projectedInstructions = restartInstructionParamsForAgent(agent, { spawnRequestId: predecessorSpawnRequestId, cwd }) ?? {};
   // #1598 — canonicalize so a worktree/subdir cwd binds to its owning checkout's project.
   const projectId = resolveProjectForCanonicalCwd(cwd).project?.id;
@@ -175,7 +155,7 @@ function buildRestartSpawnParamsForAgent(
     cwd,
     orchestrator.baseDir,
   );
-  return buildRestartSpawnCommand(agent, predecessorSpawnRequestId, agentProfile, projectedInstructions, opts, {
+  const options = {
     provider,
     modelParams: resolvedModel,
     cwd,
@@ -185,7 +165,10 @@ function buildRestartSpawnParamsForAgent(
     policyName: policyName || undefined,
     spawnRequestId: requestId,
     requestedBy,
-  }, { orchestratorId: orchestrator.id, projectId, extraCwdPrefixes: stewardExtraCwdPrefixes });
+  };
+  const tokenContext = { orchestratorId: orchestrator.id, projectId, extraCwdPrefixes: stewardExtraCwdPrefixes };
+  return buildRestartSpawnCommand(agent, predecessorSpawnRequestId, agentProfile, projectedInstructions, opts, options, tokenContext,
+    (await runnerRuntimeTokenEnvAsync(restartTokenInput(agent, options, tokenContext))));
 }
 
 /** Worker-safe restart command construction used by HTTP/orchestrator/self-restart flows. */
@@ -233,14 +216,8 @@ async function buildRestartSpawnParamsForAgentAsync(
     const params = {
       ...await buildManagedSpawnParamsAsync(relaunchPolicy(policy.value, relaunch.effort), requestId, { createdBy: requestedBy, authority: SUCCESSOR_LAUNCH, identityAgentId: agent.id, gitRepoRoot }),
       ...relaunch.carried, agentId: agent.id, requestedBy,
-      env: runnerRuntimeTokenEnv({ authority: SUCCESSOR_LAUNCH, agentId: agent.id, orchestratorId: policy.value.orchestratorId, cwd: policy.value.cwd,
-        provider: policy.value.provider, label: policy.value.label, policyName: policy.value.name, spawnRequestId: requestId,
-        createdBy: requestedBy, profile: policy.value.profile || undefined, ...(agent.teamId ? { teamId: agent.teamId } : {}),
-        approvalMode: policy.value.permissionMode,
-        ...(projectId ? { projectId } : {}), ...(combinedExtraCwdPrefixes.length ? { extraCwdPrefixes: combinedExtraCwdPrefixes } : {}), }),
-      ...(opts.continuation ? { prompt: opts.continuation } : {}),
-      ...(restartResumeWorkspace ? { resumeWorkspace: restartResumeWorkspace } : {}),
-      ...(opts.generation ? { extra: { selfRestart: { generation: opts.generation, reason: opts.reason } } } : {}),
+      env: await runnerRuntimeTokenEnvAsync(managedRestartTokenInput(agent, policy.value, requestId, requestedBy, projectId, combinedExtraCwdPrefixes)),
+      ...managedRestartOptions(opts, restartResumeWorkspace),
     };
     return opts.resumeId ? withResumeParams(params, opts.resumeId, agent.id) : params;
   }
@@ -257,15 +234,55 @@ async function buildRestartSpawnParamsForAgentAsync(
   const profileName = role ? role.profileName : stringValue(agent.meta?.profile);
   const resolvedProfile = role ? undefined : await withResolvedProvisioningAsync(profileName ? getAgentProfile(profileName)?.value : undefined, provider);
   const agentProfile = role?.agentProfile ?? (resolvedProfile ? applyDefaultSharedMcp(resolvedProfile) : undefined);
-  const resolvedModel = await resolveSpawnModelParamsAsync(provider, model, effort, { modelSource: "runtime", skipDefaultWhenEmpty: true });
+  const resolvedModel = await resolveSpawnModelParamsAsync(provider, model, effort, { modelSource: "runtime" });
   const projectedInstructions = await restartInstructionParamsForAgentAsync(agent, { spawnRequestId: predecessorSpawnRequestId, cwd, ...(role ? { profile: role.agentProfile } : {}) }) ?? {};
   const projectId = resolveProjectForCanonicalCwd(cwd).project?.id;
   const stewardExtraCwdPrefixes = stewardWorktreeCwdPrefixes(isProvenStewardAgent(agent.id, policyName), cwd, orchestrator.baseDir);
-  return buildRestartSpawnCommand(agent, predecessorSpawnRequestId, agentProfile, projectedInstructions, opts, {
+  const options = {
     provider, modelParams: resolvedModel, cwd, workspaceMode, profile: profileName || undefined,
     ...relaunch.carried, ...role?.spawn, policyName: policyName || undefined, spawnRequestId: requestId,
     requestedBy,
-  }, { orchestratorId: orchestrator.id, projectId, extraCwdPrefixes: stewardExtraCwdPrefixes });
+  };
+  const tokenContext = { orchestratorId: orchestrator.id, projectId, extraCwdPrefixes: stewardExtraCwdPrefixes };
+  return buildRestartSpawnCommand(agent, predecessorSpawnRequestId, agentProfile, projectedInstructions, opts, options, tokenContext,
+    await runnerRuntimeTokenEnvAsync(restartTokenInput(agent, options, tokenContext)));
+}
+
+function managedRestartOptions(opts: RestartOptions, restartResumeWorkspace: ReturnType<typeof resumeWorkspaceForRestart>) {
+  return {
+    ...(opts.continuation ? { prompt: opts.continuation } : {}),
+    ...(restartResumeWorkspace ? { resumeWorkspace: restartResumeWorkspace } : {}),
+    // F5 — carry the self-restart generation marker on BOTH successor spawns (policy and
+    // dashboard/non-policy), so the audit lineage is symmetric regardless of branch.
+    ...(opts.generation ? { extra: { selfRestart: { generation: opts.generation, reason: opts.reason } } } : {}),
+  };
+}
+
+function managedRestartTokenInput(agent: AgentCard, policy: SpawnPolicy, requestId: string, requestedBy: string, projectId: string | undefined, extraCwdPrefixes: string[]): RunnerRuntimeTokenInput {
+  return {
+    authority: SUCCESSOR_LAUNCH, agentId: agent.id, orchestratorId: policy.orchestratorId,
+    cwd: policy.cwd, provider: policy.provider, label: policy.label, policyName: policy.name,
+    spawnRequestId: requestId, createdBy: requestedBy, profile: policy.profile || undefined,
+    // #2245: both successor paths stamp the policy's current posture into signed constraints.
+    approvalMode: policy.permissionMode,
+    ...(agent.teamId ? { teamId: agent.teamId } : {}),
+    ...(projectId ? { projectId } : {}),
+    ...(extraCwdPrefixes.length ? { extraCwdPrefixes } : {}),
+  };
+}
+
+type RestartTokenContext = Pick<RunnerRuntimeTokenInput, "orchestratorId" | "projectId" | "extraCwdPrefixes">;
+const restartApprovalMode = (agent: AgentCard): SpawnApprovalMode => (stringValue(agent.meta?.approvalMode) as SpawnApprovalMode | undefined) ?? "guarded";
+const restartLabel = (agent: AgentCard): string | undefined => (stringValue(agent.meta?.label) ?? agent.label) || undefined;
+
+/** The successor token a non-policy restart carries; the sync twin and the writer-op path mint the same input. */
+function restartTokenInput(agent: AgentCard, options: Omit<Parameters<typeof buildSpawnCommand>[0], "env">, tokenContext: RestartTokenContext): RunnerRuntimeTokenInput {
+  return { ...tokenContext, authority: SUCCESSOR_LAUNCH,
+    agentId: agent.id, cwd: options.cwd, provider: options.provider, label: restartLabel(agent),
+    policyName: options.policyName, spawnRequestId: options.spawnRequestId,
+    createdBy: options.requestedBy, profile: options.profile, approvalMode: restartApprovalMode(agent),
+    ...(agent.teamId ? { teamId: agent.teamId } : {}),
+  };
 }
 
 function buildRestartSpawnCommand(
@@ -275,25 +292,21 @@ function buildRestartSpawnCommand(
   projectedInstructions: Record<string, unknown>,
   opts: RestartOptions,
   options: Omit<Parameters<typeof buildSpawnCommand>[0], "env">,
-  tokenContext: Pick<Parameters<typeof runnerRuntimeTokenEnv>[0], "orchestratorId" | "projectId" | "extraCwdPrefixes">,
+  tokenContext: RestartTokenContext,
+  env: Record<string, string>,
 ): Record<string, unknown> {
   const memorySeam = parseMemorySeam(projectedInstructions.memorySeam);
   const resumeWorkspace = resumeWorkspaceForRestart(agent);
   const providerArgs = metaStringArray(agent.meta, "providerArgs");
-  const approvalMode = (stringValue(agent.meta?.approvalMode) as SpawnApprovalMode | undefined) ?? "guarded";
-  const label = (stringValue(agent.meta?.label) ?? agent.label) || undefined;
+  const approvalMode = restartApprovalMode(agent);
+  const label = restartLabel(agent);
   const params = buildSpawnCommand({
     ...options,
     orchestratorId: tokenContext.orchestratorId,
     headless: true, requestedAt: Date.now(),
     ...(opts.continuation ? { prompt: opts.continuation } : {}),
     ...(opts.generation ? { extra: { selfRestart: { generation: opts.generation, reason: opts.reason } } } : {}),
-    env: runnerRuntimeTokenEnv({ ...tokenContext, authority: SUCCESSOR_LAUNCH,
-      agentId: agent.id, cwd: options.cwd, provider: options.provider, label,
-      policyName: options.policyName, spawnRequestId: options.spawnRequestId,
-      createdBy: options.requestedBy, profile: options.profile, approvalMode,
-      ...(agent.teamId ? { teamId: agent.teamId } : {}),
-    }),
+    env,
     agentId: agent.id, tags: agent.tags, systemTags: agent.systemTags, capabilities: agent.capabilities,
     label,
     approvalMode, permissionMode: approvalMode,
@@ -445,11 +458,11 @@ export async function createSelfRestartCommandAsync(
   const predecessorSpawnRequestId = stringValue(agent.meta?.spawnRequestId);
   const restartSpawn = await restartSpawnParamsForAgentAsync(agent, orchestrator, policyName, predecessorSpawnRequestId, { continuation, generation, reason });
   if (!restartSpawn) throw new ValidationError("self-restart could not build a fresh successor spawn");
-  return createCommandAsync(selfRestartInput(agent, orchestrator, restartSpawn, generation, continuation, fullContextMessageId, reason, now));
+  return (await createCommandAsync(selfRestartInput(agent, orchestrator, restartSpawn, generation, continuation, fullContextMessageId, reason, now)));
 }
 
 /** Settle managed runtime correlation only after the transactional orchestrator swap succeeds. */
-export function settleSelfRestartManagedState(command: Command): void {
+export async function settleSelfRestartManagedState(command: Command): Promise<void> {
   if (command.type !== "agent.restart" || !isRecord(command.params.selfRestart)) return;
   const policyName = stringValue(command.params.policyName);
   if (!policyName || command.status !== "succeeded") return;
@@ -460,7 +473,7 @@ export function settleSelfRestartManagedState(command: Command): void {
   const spawnRequestId = stringValue(restartSpawn?.spawnRequestId) ?? state?.spawnRequestId;
   const tmuxSession = stringValue(report?.tmuxSession) ?? stringValue(report?.sessionName) ?? state?.tmuxSession;
   if (!agentId || !spawnRequestId) return;
-  updateManagedAgentState(policyName, {
+  (await updateManagedAgentStateAsync(policyName, {
     status: "starting",
     agentId,
     spawnRequestId,
@@ -468,12 +481,12 @@ export function settleSelfRestartManagedState(command: Command): void {
     lastSpawnAt: command.updatedAt,
     lastStopAt: command.updatedAt,
     restartCount: (state?.restartCount ?? 0) + 1,
-  });
-  markManagedAgentRunning({ policyName, spawnRequestId, agentId, tmuxSession }, authContextFromSystem(), { now: () => command.updatedAt });
+  }));
+  (await markManagedAgentRunning({ policyName, spawnRequestId, agentId, tmuxSession }, authContextFromSystem(), { now: () => command.updatedAt }));
 }
 
-export function restartSpawnParamsForAgent(...args: Parameters<typeof buildRestartSpawnParamsForAgent>) {
-  return buildRestartSpawnParamsForAgent(...args);
+export async function restartSpawnParamsForAgent(...args: Parameters<typeof buildRestartSpawnParamsForAgent>) {
+  return (await buildRestartSpawnParamsForAgent(...args));
 }
 export async function restartSpawnParamsForAgentAsync(...args: Parameters<typeof buildRestartSpawnParamsForAgentAsync>) {
   return buildRestartSpawnParamsForAgentAsync(...args);

package/src/services/review-publish.ts

@@ -17,10 +17,10 @@ export interface ReviewPublishDispatch {
   error?: string;
 }
 
-export function dispatchReviewPublish(
+export async function dispatchReviewPublish(
   workspace: WorkspaceRecord | undefined,
   opts: { requestedBy: string; taskId: number; sha?: string; emit: (command: Command) => void },
-): ReviewPublishDispatch | undefined {
+): Promise<ReviewPublishDispatch | undefined> {
   if (!workspace || workspace.mode !== "isolated" || !workspace.worktreePath) return undefined;
   // #1452 round-6 H2b / round-7 H3 — publishing REQUIRES an immutable delivered COMMIT OID (the host
   // must pin exactly it, and settlement matches result.sha === requested sha). It must be a concrete
@@ -46,28 +46,11 @@ export function dispatchReviewPublish(
     // failing ledger insert was swallowed, leaving a POLLABLE publish command with NO cleanup ledger
     // (a crash/insert-failure orphan). The transaction makes them all-or-nothing: a ledger failure
     // rolls back the command row too, so a reviewer never sees a command with no durable cleanup path.
-    let command: Command | undefined;
-    let buildError: string | undefined;
-    writeTransaction(() => {
-      const built = buildWorkspacePublishReviewCommand(workspace, opts.requestedBy, { taskId: opts.taskId, sha, reviewRef });
-      if (!built.ok) { buildError = built.error; return; } // nothing was persisted (owner/worktree check)
-      // Round-9: PIN the exact ref (relay-derived, host-asserted-equal before push) so cleanup can
-      // delete it even if the host's success-result is lost. The row also holds the branch/host/
-      // requested-sha so cleanup survives a purge/recycle. On settlement the reconciler re-writes the
-      // host-confirmed ref (identical to this one, since the host refuses to push a divergent ref).
-      upsertReviewRefCleanup({
-        id: built.command.id,
-        ref: reviewRef,
-        workspaceId: workspace.id,
-        repoRoot: workspace.repoRoot,
-        worktreePath: workspace.worktreePath,
-        branch: workspace.branch,
-        orchestratorId: workspace.orchestratorId,
-        requestedSha: sha,
-      });
-      command = built.command;
-    });
-    if (!command) return { workspaceId: workspace.id, dispatched: false, error: buildError ?? "review publish not dispatched" };
+    const built = await buildWorkspacePublishReviewCommand(workspace, opts.requestedBy, { taskId: opts.taskId, sha, reviewRef,
+      cleanup: { ref: reviewRef, workspaceId: workspace.id, repoRoot: workspace.repoRoot, worktreePath: workspace.worktreePath,
+        branch: workspace.branch, orchestratorId: workspace.orchestratorId, requestedSha: sha } });
+    if (!built.ok) return { workspaceId: workspace.id, dispatched: false, error: built.error };
+    const command = built.command;
     opts.emit(command); // durable state (command + ledger) is committed before anyone is notified
     return { workspaceId: workspace.id, commandId: command.id, dispatched: true };
   } catch (err) {

package/src/services/review-ref-reconcile.ts

@@ -1,3 +1,4 @@
+import { cancelPendingCommandAsync, recordReviewRefCleanupDispatchAsync, markReviewRefPublishedAsync, markReviewRefFailedAsync, markReviewRefClosedAsync, markReviewRefAnnouncedAsync, deleteReviewRefCleanupAsync } from "../db/runtime-tail-async";
 // #1452 round-4 — the durable, OFF-critical-path engine that settles the review-ref (refs/review/*)
 // lifecycle and cleans up leaked refs. It reads the review_ref_cleanups ledger (one row per EXACT
 // server-derived ref) — NOT a per-workspace marker — so it fixes every round-4 finding at once:
@@ -23,18 +24,8 @@
 // was never enabled, since rows are only written when a publish is dispatched.
 
 import { getWorkspace } from "../db";
-import {
-  deleteReviewRefCleanup,
-  getReviewRefFenceToken,
-  listReviewRefCleanups,
-  markReviewRefAnnounced,
-  markReviewRefClosed,
-  markReviewRefFailed,
-  markReviewRefPublished,
-  recordReviewRefCleanupDispatch,
-  type ReviewRefCleanup,
-} from "../db/review-ref-cleanups";
-import { cancelPendingCommand, getCommand } from "../commands-db";
+import { getReviewRefFenceToken, listReviewRefCleanups, type ReviewRefCleanup } from "../db/review-ref-cleanups";
+import { getCommand } from "../commands-db";
 import { emitCommand } from "../maintenance/events";
 import { emitRelayEvent } from "../events";
 import { buildReviewRefUnpublishCommandFromFields } from "../workspace-actions";
@@ -81,16 +72,16 @@ interface ReconcileDeps {
    *  pending publish can still be externally spoofed to succeeded{pushed:true}. Requiring the token here
    *  refuses that phantom before it can announce a ref that was never pushed. Returns null when absent. */
   getFenceToken: (id: string) => string | null;
-  markPublished: typeof markReviewRefPublished;
-  markFailed: typeof markReviewRefFailed;
-  markAnnounced: typeof markReviewRefAnnounced;
-  markClosed: typeof markReviewRefClosed;
-  recordDispatch: typeof recordReviewRefCleanupDispatch;
-  deleteCleanup: typeof deleteReviewRefCleanup;
+  markPublished: typeof markReviewRefPublishedAsync;
+  markFailed: typeof markReviewRefFailedAsync;
+  markAnnounced: typeof markReviewRefAnnouncedAsync;
+  markClosed: typeof markReviewRefClosedAsync;
+  recordDispatch: typeof recordReviewRefCleanupDispatchAsync;
+  deleteCleanup: typeof deleteReviewRefCleanupAsync;
   buildUnpublish: typeof buildReviewRefUnpublishCommandFromFields;
   /** #1452 round-10 HIGH #1 — atomically cancel the publish command IFF still pending; true means it
    *  was never dispatched (proven no push can occur), so a closed round can retire it without racing. */
-  cancelPublish: (id: string) => boolean;
+  cancelPublish: (id: string) => boolean | Promise<boolean>;
   emit: (command: Command) => void;
   /** Durable publish-settled announcement: emitted for a published-but-unannounced row, then marked
    *  announced only on success — so a crash/outbox failure re-emits next pass (round-6 H-atomicity). */
@@ -164,21 +155,21 @@ const realDeps: ReconcileDeps = {
   },
   getCommand,
   getFenceToken: getReviewRefFenceToken,
-  markPublished: markReviewRefPublished,
-  markFailed: markReviewRefFailed,
-  markAnnounced: markReviewRefAnnounced,
-  markClosed: markReviewRefClosed,
-  recordDispatch: recordReviewRefCleanupDispatch,
-  deleteCleanup: deleteReviewRefCleanup,
+  markPublished: markReviewRefPublishedAsync,
+  markFailed: markReviewRefFailedAsync,
+  markAnnounced: markReviewRefAnnouncedAsync,
+  markClosed: markReviewRefClosedAsync,
+  recordDispatch: recordReviewRefCleanupDispatchAsync,
+  deleteCleanup: deleteReviewRefCleanupAsync,
   buildUnpublish: buildReviewRefUnpublishCommandFromFields,
-  cancelPublish: (id) => cancelPendingCommand(id, "workspace.publish-review", "review publish canceled: round closed before dispatch"),
+  cancelPublish: async (id) => (await cancelPendingCommandAsync(id, "workspace.publish-review", "review publish canceled: round closed before dispatch")),
   emit: emitCommand,
   emitPublished: emitPublishedEvent,
   now: () => Date.now(),
 };
 
 /** One reconcile pass over the durable ledger. Pure over its (injectable) deps for testing. */
-export function reconcileReviewRefs(deps: ReconcileDeps = realDeps, signal?: AbortSignal): ReviewRefReconcileResult {
+export async function reconcileReviewRefs(deps: ReconcileDeps = realDeps, signal?: AbortSignal): Promise<ReviewRefReconcileResult> {
   const rows = deps.listCleanups();
   const published: string[] = [];
   const failed: string[] = [];
@@ -208,7 +199,7 @@ export function reconcileReviewRefs(deps: ReconcileDeps = realDeps, signal?: Abo
       const roundClosed = !ws
         || TERMINAL_WORKSPACE_STATUSES.has(ws.status as any)
         || Boolean(ws.branch && row.branch && ws.branch !== row.branch);
-      if (roundClosed) { deps.markClosed(row.id, now); closed = true; }
+      if (roundClosed) { await deps.markClosed(row.id, now); closed = true; }
     }
 
     // 1) Settle the publish outcome (H2). A ref is only real once its push command TERMINALLY
@@ -229,8 +220,8 @@ export function reconcileReviewRefs(deps: ReconcileDeps = realDeps, signal?: Abo
         // push can occur) → retire with nothing on origin. If it was already claimed (accepted/running),
         // the host may push, so WAIT for its terminal settlement — cleanup then runs safely AFTER it.
         if (!closed) { inFlight.push(label()); continue; }
-        if (!deps.cancelPublish(row.id)) { inFlight.push(label()); continue; } // already claimed → wait
-        deps.markFailed(row.id, now, false); // canceled before dispatch → nothing on origin, drop the ref
+        if (!await deps.cancelPublish(row.id)) { inFlight.push(label()); continue; } // already claimed → wait
+        await deps.markFailed(row.id, now, false); // canceled before dispatch → nothing on origin, drop the ref
         confirmedRef = undefined;
         publishState = "failed";
         failed.push(label());
@@ -242,7 +233,7 @@ export function reconcileReviewRefs(deps: ReconcileDeps = realDeps, signal?: Abo
         // never-claimed pending publish spoofed to succeeded has none, so it never advances to published.
         const ref = confirmedPublishRef(cmd, row.requestedSha, row.ref, deps.getFenceToken(row.id));
         if (ref) {
-          deps.markPublished(row.id, ref, now); // sets announced=0 → announced below (H-atomicity)
+          await deps.markPublished(row.id, ref, now); // sets announced=0 → announced below (H-atomicity)
           confirmedRef = ref;
           publishState = "published";
           announced = false;
@@ -253,7 +244,7 @@ export function reconcileReviewRefs(deps: ReconcileDeps = realDeps, signal?: Abo
           // timed-out/canceled, a missing/pruned result, or a malformed success) may have pushed before
           // its result was lost ⇒ RETAIN the pinned ref as a tombstone and unpublish before deletion.
           const notPushed = definitivelyNotPushed(cmd);
-          deps.markFailed(row.id, now, !notPushed);
+          await deps.markFailed(row.id, now, !notPushed);
           if (notPushed) confirmedRef = undefined;
           publishState = "failed";
           failed.push(label());
@@ -267,7 +258,7 @@ export function reconcileReviewRefs(deps: ReconcileDeps = realDeps, signal?: Abo
     if (publishState === "published" && !announced && confirmedRef) {
       try {
         deps.emitPublished({ workspaceId: row.workspaceId, ref: confirmedRef, ...(row.requestedSha ? { sha: row.requestedSha } : {}), ...(row.branch ? { branch: row.branch } : {}) });
-        deps.markAnnounced(row.id, now);
+        await deps.markAnnounced(row.id, now);
         announced = true;
       } catch { /* emit failed → leave announced=0 so the next pass re-emits (at-least-once) */ }
     }
@@ -285,7 +276,7 @@ export function reconcileReviewRefs(deps: ReconcileDeps = realDeps, signal?: Abo
 
     // A closed round with NOTHING on origin — a DEFINITIVELY-not-pushed outcome, or a legacy row that
     // never pinned a ref — has nothing to delete. Retire it directly, no host round-trip, no leak.
-    if (!confirmedRef) { deps.deleteCleanup(row.id); cleared.push(label()); continue; }
+    if (!confirmedRef) { await deps.deleteCleanup(row.id); cleared.push(label()); continue; }
 
     // 3) Drive deletion (H4/M7) of the EXACT host-confirmed ref. CONFIRM a prior attempt first — a
     //    succeeded delete clears the row regardless of the backoff window; the backoff gates only a
@@ -294,20 +285,20 @@ export function reconcileReviewRefs(deps: ReconcileDeps = realDeps, signal?: Abo
       const cmd = deps.getCommand(row.cleanupCommandId);
       if (cmd && !COMMAND_TERMINAL.has(cmd.status)) { inFlight.push(confirmedRef); continue; }
       if (cmd && cmd.status === "succeeded" && !cmd.error) {
-        deps.deleteCleanup(row.id); // origin confirmed the ref is gone (deleted or already absent)
+        await deps.deleteCleanup(row.id); // origin confirmed the ref is gone (deleted or already absent)
         cleared.push(confirmedRef);
         continue;
       }
       // failed / timed_out / missing → re-dispatch below, subject to backoff.
     }
     if (row.nextAttemptAt != null && row.nextAttemptAt > now) { waiting.push(confirmedRef); continue; }
-    const built = deps.buildUnpublish(
+    const built = await deps.buildUnpublish(
       { workspaceId: row.workspaceId, repoRoot: row.repoRoot, worktreePath: row.worktreePath, branch: row.branch, orchestratorId: row.orchestratorId, ref: confirmedRef },
       "review-ref-reconciler",
     );
     if (!built.ok) continue; // no owning host reachable yet — retry next tick (no backoff burned)
     deps.emit(built.command);
-    deps.recordDispatch(row.id, built.command.id, now + cleanupBackoffMs(row.cleanupAttempts), now);
+    await deps.recordDispatch(row.id, built.command.id, now + cleanupBackoffMs(row.cleanupAttempts), now);
     dispatched.push(confirmedRef);
   }
 
@@ -326,7 +317,7 @@ export const REVIEW_REF_RECONCILE_JOB = {
   intervalMs: WORKSPACE_ORPHAN_REAPER_INTERVAL_MS,
   runOnStart: false,
   deadlineMs: 60 * 1000,
-  handler: (context: { signal?: AbortSignal }) => reconcileReviewRefs(undefined, context.signal) as unknown as Record<string, unknown>,
+  handler: async (context: { signal?: AbortSignal }) => (await reconcileReviewRefs(undefined, context.signal)) as unknown as Record<string, unknown>,
 };
 
 // Test seam: backoff parameters so tests assert scheduling without duplicating literals.

package/src/services/runtime-dependency-alert.ts

@@ -0,0 +1,27 @@
+import { createHash } from "node:crypto";
+import { isRecord } from "@humwick/sdk";
+import { createActivityEventByClientIdAsync } from "../db/activity-async";
+import { notifyWorkspaceHumanEscalation } from "../workspace-human-escalation";
+import { stewardFallbackTarget } from "../config";
+import { notifyTarget } from "../maintenance/events";
+
+/** Heartbeats already persist provider failures for Fleet; page once per broken checkout (#3791). */
+export async function reportRuntimeDependencyFailure(hostId: string, hostname: string, body: unknown): Promise<void> {
+  if (!isRecord(body) || !Array.isArray(body.providerStatus)) return;
+  const failure = body.providerStatus.find((status) => isRecord(status) && status.available === false
+    && typeof status.reason === "string" && status.reason.startsWith("Runtime dependencies unavailable in "));
+  if (!failure) return;
+  const reason = failure.reason as string;
+  const checkout = reason.split(": ")[0]!;
+  const key = `runtime-dependencies:${hostId}:${createHash("sha256").update(checkout).digest("hex")}`;
+  const subject = `Runtime dependencies broken on ${hostname}`;
+  const message = `${hostname}: ${reason}. Providers are unavailable until this checkout is repaired.`;
+  const event = await createActivityEventByClientIdAsync({ clientId: key, kind: "state", title: subject, body: message,
+    meta: hostId, icon: "ti-alert-triangle", view: "orchestrators", metadata: { hostId, reason } });
+  if (!event.created) return;
+  const payload = { hostId, reason, kind: "runtime-dependencies.failed" };
+  const human = await notifyWorkspaceHumanEscalation({ subject, body: message, priority: "urgent", source: "runtime-dependencies", payload, idempotencyKey: key });
+  const target = stewardFallbackTarget();
+  const routed = target ? await notifyTarget("maintenance.degraded", {}, target, subject, message, payload, key) : null;
+  console.warn(`[relay] ${subject}: ${reason}${human.accepted || routed ? "" : " — no notification target configured"}`);
+}

package/src/services/scheduler-command.ts

@@ -1,9 +1,11 @@
+import { updateCommandAsync } from "../commands-write";
 import { writeChildEnv } from "@humwick/sdk/env";
 import { basename } from "node:path";
 import { SCHEDULER_RETRY_MS } from "../config";
 import { errMessage, stringValue } from "@humwick/sdk";
 import { hasScope } from "@humwick/sdk/types";
-import { findLatestCommandByCorrelation, getCommand, updateCommand } from "../commands-db";
+import { getCommand } from "../commands-db";
+import { findLatestCommandByCorrelationAsync } from "../commands-write";
 import { RELAY_RUN_DEADLINE_GRACE_MS } from "../relay-run-lease";
 import { storeTextArtifact } from "../artifact-text";
 import { gateRunIdentity, recordAgentGateRun } from "../gate-run-capture";
@@ -119,7 +121,7 @@ export async function dispatchScheduledCommand(timer: ScheduledTimer, ctx = pers
   // Freeze the agent generation that owns this command. Managed restarts reuse the
   // durable agent id, so agentId alone would let predecessor teardown kill a successor's command.
   const executionOwnerSpawnRequestId = spawnRequestIds?.length === 1 ? spawnRequestIds[0] : undefined;
-  const command = findLatestCommandByCorrelation("scheduler.command", timer.correlationId) ?? await dispatchCommandAsync({
+  const command = await findLatestCommandByCorrelationAsync("scheduler.command", timer.correlationId) ?? await dispatchCommandAsync({
     type: "scheduler.command",
     source: timer.createdBy,
     target: `orchestrator-${timer.command.orchestratorId}`,
@@ -186,7 +188,7 @@ export async function fireScheduledCommandTimeout(timer: ScheduledTimer, now: nu
     return { status: "deferred" };
   }
   const neverStarted = !!command && (command.status === "pending" || command.status === "accepted") && commandResult(command).phase !== "running";
-  if (command) updateCommand(command.id, { status: "timed_out", error: "background command timed out" });
+  if (command) (await updateCommandAsync(command.id, { status: "timed_out", error: "background command timed out" }));
   const result = command ? commandResult(command) : {};
   const admittedAt = numberValue(result.admittedAt) ?? timer.command.admittedAt;
   const durationMs = admittedAt === undefined ? 0 : Math.max(0, now - admittedAt);
@@ -202,7 +204,7 @@ export async function fireScheduledCommandTimeout(timer: ScheduledTimer, now: nu
     stdoutTail: stringValue(result.stdoutTail) ?? timer.command.stdoutTail ?? "",
     stderrTail: stringValue(result.stderrTail) ?? timer.command.stderrTail ?? (neverStarted ? queuedTimeoutReason(timer.command.orchestratorId) : "background command timed out"),
   };
-  return fireScheduledCommandWake(timer, {
+  return (await fireScheduledCommandWake(timer, {
     status: "timed_out",
     finishedAt: timer.command.finishedAt ?? now,
     durationMs,
@@ -213,7 +215,7 @@ export async function fireScheduledCommandTimeout(timer: ScheduledTimer, now: nu
     exitCode: null,
     stdoutTail: timeoutResult.stdoutTail,
     stderrTail: timeoutResult.stderrTail,
-  }, timeoutResult, now, inlineWakeSuppressed(timer.command, now));
+  }, timeoutResult, now, inlineWakeSuppressed(timer.command, now)));
 }
 
 /** #2776 — admission report: the fallback timer now guards the RUN (end + grace), not the queue ceiling. */
@@ -243,7 +245,7 @@ export async function settleScheduledCommandTimer(command: Command, now = Date.n
       : "failed";
   const stdout = clipTail(stringValue(result.stdoutTail) ?? stringValue(result.stdout) ?? "");
   const stderr = clipTail(stringValue(result.stderrTail) ?? stringValue(result.stderr) ?? command.error ?? "");
-  return fireScheduledCommandWake(timer, {
+  return (await fireScheduledCommandWake(timer, {
     status,
     finishedAt: timer.command.finishedAt ?? now,
     startedAt: numberValue(result.startedAt),
@@ -262,7 +264,7 @@ export async function settleScheduledCommandTimer(command: Command, now = Date.n
     // Strictly stronger than truncation: the command was KILLED for flooding, it did not finish.
     outputLimitExceeded: result.outputLimitExceeded === true,
     logArtifactId: stringValue(result.logArtifactId),
-  }, result, now, inlineWakeSuppressed(timer.command, now));
+  }, result, now, inlineWakeSuppressed(timer.command, now)));
 }
 
 /**
@@ -358,7 +360,7 @@ async function fireScheduledCommandWake(
   try {
     const runNumber = timerForSend.runCount + 1;
     const payloadResult = commandPayload(timerForSend.command!, rawResult, logArtifactId);
-    const send = sendMessageService({
+    const send = (await sendMessageService({
       from: timerForSend.deliveryFrom,
       to: timerForSend.target,
       kind: timerForSend.messageKind as SendMessageInput["kind"] | undefined,
@@ -387,7 +389,7 @@ async function fireScheduledCommandWake(
       },
       // One stable key per command completion (not per run count): overlapping settlements dedupe.
       idempotencyKey: `${timerForSend.idempotencyKey ?? `scheduler:${timerForSend.id}`}:completion`,
-    }, persistedCommandAuthContext(timerForSend));
+    }, persistedCommandAuthContext(timerForSend)));
     await completeTimerRunAsync(timerForSend, send.message.id, now);
     return { status: "fired", messageId: send.message.id };
   } catch (error) {

package/src/services/scheduler.ts

@@ -1,3 +1,4 @@
+import { framed } from "../hub-metrics";
 import { createScheduledTimer } from "./scheduler-create";
 import { planScheduledTimerCreate, cleanOptionalString, canAdministerTimers, type ScheduleTimerInput } from "./scheduler-plan";
 export { type ScheduleTimerInput } from "./scheduler-plan";
@@ -84,9 +85,8 @@ export function startRelayScheduler(): void {
   if (isEphemeralMode()) return;
   if (schedulerTimer) return;
   void timePostBindPhase("runDueScheduledTimers", runDueScheduledTimers).catch((error) => console.warn(`relay scheduler tick failed: ${errMessage(error)}`));
-  schedulerTimer = setInterval(() => {
-    void timePostBindPhase("runDueScheduledTimers", runDueScheduledTimers).catch((error) => console.warn(`relay scheduler tick failed: ${errMessage(error)}`));
-  }, SCHEDULER_TICK_MS);
+  schedulerTimer = setInterval(framed("timer scheduled-timers", () =>
+    timePostBindPhase("runDueScheduledTimers", runDueScheduledTimers).catch((error) => console.warn(`relay scheduler tick failed: ${errMessage(error)}`))), SCHEDULER_TICK_MS);
   // #1408 K7 Phase-2: additively light up the kernel-served timers supervisor iff the reversible
   // switch is on (default OFF ⇒ pure no-op). Legacy above stays authoritative; the kernel path is
   // not yet a functional replacement for message/command dispatch (see kernel-timers.ts).
@@ -292,7 +292,7 @@ async function fireClaimedTimer(
       await disableTimerAsync(timer.id, "timer expired before delivery");
       return { status: "disabled" };
     }
-    return fireScheduledCommandTimeout(timer, now);
+    return (await fireScheduledCommandTimeout(timer, now));
   }
 
   try {
@@ -323,7 +323,7 @@ async function fireClaimedTimer(
       await disableTimerAsync(timer.id, "timer expired before delivery");
       return { status: "disabled" };
     }
-    const result = sendMessageService(sendInput, persistedAuthContext(timer));
+    const result = (await sendMessageService(sendInput, persistedAuthContext(timer)));
     await completeTimerRunAsync(timer, result.message.id, now);
     return { status: "fired", messageId: result.message.id };
   } catch (error) {