@humwick/server 0.130.0 → 0.130.2

package/runner/src/agent-runner/credentials.ts

@@ -14,13 +14,26 @@ interface CredentialsDeps {
   publishRenewalFailure(): void;
   logRunnerDiagnostic(message: string): void;
   reportBackgroundRejection(scope: string, status: string, title: string, error: unknown): void;
+  /** Recovery gave up: the runner must exit so Relay sees a terminal failure, not a mint loop (#3775). */
+  onTerminalAuthFailure(reason: string): void;
 }
 
 const TOKEN_RENEW_LOG_INTERVAL_MS = 5 * 60 * 1000;
 const TOKEN_RENEW_RETRY_MS = 60_000;
-// Debounce reactive token recovery so a burst of 401-ing calls in the same window
-// triggers a single re-mint attempt, not one per failing request.
-const REACTIVE_TOKEN_RECOVERY_DEBOUNCE_MS = 10_000;
+// #3775 — a successful mint does not prove the hub accepts the runner: an orphaned runner kept
+// getting 401 on its fresh token and re-minted ~2/s. Auth recoveries back off (1 s doubling to
+// 5 min, jittered) and stop after a bounded count; only a quiet spell resets the count.
+export const AUTH_RECOVERY_MAX_ATTEMPTS = 10;
+const AUTH_RECOVERY_BASE_MS = 1_000;
+const AUTH_RECOVERY_MAX_DELAY_MS = 5 * 60_000;
+const AUTH_RECOVERY_QUIET_RESET_MS = 30 * 60_000;
+
+/** Delay before auth recovery `attempt` (0-based): the first heals at once, later ones back off with equal jitter. */
+export function authRecoveryDelayMs(attempt: number, random: () => number = Math.random): number {
+  if (attempt <= 0) return 0;
+  const ceiling = Math.min(AUTH_RECOVERY_MAX_DELAY_MS, AUTH_RECOVERY_BASE_MS * 2 ** (attempt - 1));
+  return Math.round(ceiling / 2 + random() * (ceiling / 2));
+}
 
 export class Credentials {
   private currentToken?: string;
@@ -30,7 +43,10 @@ export class Credentials {
   private tokenRenewTimer?: Timer;
   private readonly tokenRotation = new TokenRotation();
   private tokenRenewLastLog?: { key: string; at: number };
-  private reactiveTokenRecoveryAt?: number;
+  private authRecoveryAttempts = 0;
+  private authRecoveryLastAt?: number;
+  private authRecoveryTimer?: Timer;
+  private authRecoveryExhausted = false;
 
   constructor(options: { token?: string; tokenJti?: string; tokenProfileId?: string; tokenExpiresAt?: number }, private readonly deps: CredentialsDeps) {
     this.currentToken = options.token;
@@ -47,6 +63,8 @@ export class Credentials {
   stop(): void {
     if (this.tokenRenewTimer) clearTimeout(this.tokenRenewTimer);
     this.tokenRenewTimer = undefined;
+    if (this.authRecoveryTimer) clearTimeout(this.authRecoveryTimer);
+    this.authRecoveryTimer = undefined;
   }
 
   // Timer/edge callers cannot await renewal; report an escaping rejection so it cannot
@@ -55,22 +73,35 @@ export class Credentials {
     void this.renewRuntimeToken().catch((error) => this.deps.reportBackgroundRejection("runtime token renewal", "runtime-token-renewal-failed", "Runtime token renewal aborted", error));
   }
 
-  // The TTL timer cannot detect revocation: recover on auth failure, coalescing a burst.
+  // The TTL timer cannot detect revocation: recover on auth failure, coalescing a burst into the
+  // one recovery in flight or scheduled.
   // Orchestrator re-mint heals stale/expired tokens but preserves hard revocation (#484).
   recoverRuntimeTokenAfterAuthFailure(source: string): void {
-    if (this.deps.isStopped() || this.tokenRotation.inFlight) return;
+    if (this.deps.isStopped() || this.authRecoveryExhausted || this.tokenRotation.inFlight || this.authRecoveryTimer) return;
     if (!this.isRuntimeTokenRenewable() && !this.canRemintViaOrchestrator()) return;
     const now = Date.now();
-    if (this.reactiveTokenRecoveryAt && now - this.reactiveTokenRecoveryAt < REACTIVE_TOKEN_RECOVERY_DEBOUNCE_MS) return;
-    this.reactiveTokenRecoveryAt = now;
-    this.deps.logRunnerDiagnostic(`[runner] relay auth failure on ${source}; recovering runtime token`);
-    this.renewRuntimeTokenInBackground();
+    if (this.authRecoveryLastAt !== undefined && now - this.authRecoveryLastAt >= AUTH_RECOVERY_QUIET_RESET_MS) this.authRecoveryAttempts = 0;
+    if (this.authRecoveryAttempts >= AUTH_RECOVERY_MAX_ATTEMPTS) {
+      this.authRecoveryExhausted = true;
+      this.stop();
+      this.deps.onTerminalAuthFailure(`runtime-token-auth-failed: relay still rejects the runner (last on ${source}) after ${AUTH_RECOVERY_MAX_ATTEMPTS} token recoveries`);
+      return;
+    }
+    const delayMs = authRecoveryDelayMs(this.authRecoveryAttempts);
+    this.authRecoveryAttempts += 1;
+    this.authRecoveryLastAt = now;
+    this.deps.logRunnerDiagnostic(`[runner] relay auth failure on ${source}; recovering runtime token (attempt ${this.authRecoveryAttempts}/${AUTH_RECOVERY_MAX_ATTEMPTS}${delayMs ? ` in ${Math.ceil(delayMs / 1000)}s` : ""})`);
+    if (!delayMs) return this.renewRuntimeTokenInBackground();
+    this.authRecoveryTimer = setTimeout(() => {
+      this.authRecoveryTimer = undefined;
+      this.renewRuntimeTokenInBackground();
+    }, delayMs);
   }
 
   scheduleRuntimeTokenRenewal(delayMs?: number): void {
     if (this.tokenRenewTimer) clearTimeout(this.tokenRenewTimer);
     this.tokenRenewTimer = undefined;
-    if (this.deps.isStopped()) return;
+    if (this.deps.isStopped() || this.authRecoveryExhausted) return;
     const canSelfRenew = this.isRuntimeTokenRenewable();
     const canRemint = this.canRemintViaOrchestrator();
     // Keep the renewal clock ticking as long as the session can recover its token
@@ -152,7 +183,6 @@ export class Credentials {
     this.currentTokenProfileId = record.profileId ?? this.currentTokenProfileId;
     this.currentTokenExpiresAt = record.expiresAt;
     this.deps.onTokenChanged(token, record);
-    this.reactiveTokenRecoveryAt = undefined;
     this.deps.resumeLivenessAndPublishRotation(status, record.jti);
     this.scheduleRuntimeTokenRenewal();
   }

package/runner/src/agent-runner/inbox.ts

@@ -0,0 +1,27 @@
+import type { Message } from "@humwick/sdk";
+import type { HumwickHttpClient } from "@humwick/sdk/http-client";
+import { isNotificationOnlyInput } from "../inbox-carrier";
+import { logger } from "../logger";
+
+export async function bootstrapUnreadMessages(http: Pick<HumwickHttpClient, "pollMessages">, agentId: string, enqueue: (message: Message) => void): Promise<void> {
+  try {
+    const messages = await http.pollMessages({ for: agentId, unread: true, limit: 100, turnDelivery: true });
+    for (const message of messages) enqueue(message);
+  } catch (error) { logger.error("runner", `inbox bootstrap failed: ${error}`); }
+}
+
+/** Keep startup failure visible to each sender, including mail queued before the check (#3751). */
+export async function reportUnavailableMessages(
+  http: Pick<HumwickHttpClient, "sendMessage" | "markRead">, agentId: string,
+  pending: Map<number, Message>, settled: Set<number>, reason: string,
+): Promise<void> {
+  for (const [id, message] of pending) {
+    try {
+      if (!isNotificationOnlyInput(message)) await http.sendMessage({ from: agentId, to: message.from, replyTo: id,
+        body: reason, idempotencyKey: `provider-startup-${agentId}-${id}`, payload: { kind: "notification" } });
+      await http.markRead(id, agentId);
+      pending.delete(id);
+      settled.add(id);
+    } catch (error) { logger.error("runner", `startup failure reply failed: ${error}`); }
+  }
+}

package/runner/src/agent-runner/lifecycle-milestones.ts

@@ -15,7 +15,6 @@ interface LifecycleMilestoneDeps {
   process(): ManagedProcess | undefined;
   stopAttachmentCacheSweep(): void;
   stopTestRunDedupSweep(): void;
-  stopGateRunLeaseProbe(): void;
   disposeEmptyTurns(): void;
   resetEmptyTurns(): void;
   resetLateClearRecovery(): void;
@@ -92,7 +91,6 @@ export class LifecycleMilestones {
   private backgroundStopped(): void {
     this.deps.stopAttachmentCacheSweep();
     this.deps.stopTestRunDedupSweep();
-    this.deps.stopGateRunLeaseProbe();
     // #2389 — before the shutdown await, not after: an escalation or settle timer firing during a
     // graceful provider shutdown would publish a "produced nothing" verdict about a turn the
     // shutdown itself ended.

package/runner/src/bin/claude.ts

@@ -0,0 +1,4 @@
+#!/usr/bin/env -S bun --no-env-file
+import { runCli } from "../index";
+
+runCli("claude");

package/runner/src/bin/codex.ts

@@ -0,0 +1,4 @@
+#!/usr/bin/env -S bun --no-env-file
+import { runCli } from "../index";
+
+runCli("codex");

package/runner/src/bin/opencode.ts

@@ -0,0 +1,4 @@
+#!/usr/bin/env -S bun --no-env-file
+import { runCli } from "../index";
+
+runCli("opencode");

package/runner/src/index.ts

@@ -375,8 +375,9 @@ export function detectApprovalModeFromArgs(provider: string, defaultArgs: string
   return detector ? detector([...defaultArgs, ...providerArgs]) : undefined;
 }
 
-if (import.meta.main) {
-  main().catch((error) => {
+export function runCli(provider?: string): void {
+  const argv = provider ? [...process.argv.slice(0, 2), provider, ...process.argv.slice(2)] : process.argv;
+  main(argv).catch((error) => {
     // A fatal startup failure (e.g. the runner could not register with the relay within the
     // deadline) must exit non-zero, not hang or swallow — the orchestrator's exit detection
     // then reports it and the spawn-parent is notified instead of waiting on a silent zombie.
@@ -384,3 +385,5 @@ if (import.meta.main) {
     exitRunner(1, `runner fatal: ${errMessage(error)}`);
   });
 }
+
+if (import.meta.main) runCli();

package/runner/src/provider-setup.ts

@@ -0,0 +1,34 @@
+import { errMessage } from "@humwick/sdk";
+import { checkProviderAuth } from "@humwick/sdk/provider-probe";
+import { selectProviderConfigEnv } from "@humwick/sdk/child-env";
+import type { ProviderAdapter, ProviderStatusEvent } from "./adapter";
+import { logger } from "./logger";
+import { hostSourcesFor } from "./host-asset-plan";
+
+/** All real adapters share the managed-spawn login check, including replacements (#3751). */
+export function withProviderSetup(adapter: ProviderAdapter): ProviderAdapter {
+  const spawn = adapter.spawn.bind(adapter);
+  adapter.spawn = async (config) => {
+    const env = selectProviderConfigEnv(config.providerConfig.env, hostSourcesFor(config).settings);
+    const auth = await checkProviderAuth(config.provider, undefined, env, config.providerConfig.command);
+    if (!auth.ok) throw new Error(auth.reason);
+    return spawn(config);
+  };
+  return adapter;
+}
+
+export function providerStartupFailure(error: unknown): ProviderStatusEvent {
+  const message = `Provider startup failed: ${errMessage(error)}`;
+  return { status: "error", reason: "provider-turn", providerState: {
+    state: "blocked", terminal: true, reason: "providerSetupRequired", label: "Provider setup required", message,
+    recommendedAction: "Complete the provider login and first-run setup on this host, then restart the runner.",
+  } };
+}
+
+export async function attemptProviderStartup(start: () => Promise<void>, failed: (status: ProviderStatusEvent) => void): Promise<void> {
+  try { await start(); }
+  catch (error) {
+    logger.error("startup", errMessage(error));
+    failed(providerStartupFailure(error));
+  }
+}

package/runner/src/providers.ts

@@ -1,3 +1,4 @@
+import { withProviderSetup } from "./provider-setup";
 import { ClaudeApiAdapter } from "./adapters/claude-api";
 import type { LaunchMode, SpawnProvider } from "@humwick/sdk";
 import { getAllManifests, supportedLaunchModesForProvider } from "@humwick/providers";
@@ -31,5 +32,5 @@ export function createProviderAdapter(provider: SpawnProvider, providerArgs: str
   if (launchMode && !supportedLaunchModesForProvider(provider).includes(launchMode)) {
     throw new Error(`${provider} does not support launchMode=${launchMode}`);
   }
-  return registration.createAdapter(providerArgs, launchMode);
+  return withProviderSetup(registration.createAdapter(providerArgs, launchMode));
 }

package/runner/src/runner-core.ts

@@ -1,3 +1,5 @@
+import { attemptProviderStartup } from "./provider-setup";
+import { bootstrapUnreadMessages, reportUnavailableMessages } from "./agent-runner/inbox";
 import { writeChildEnv } from "@humwick/sdk/env";
 import { normalizeDeliverResult } from "./delivery-result";
 import { TokenUsageReportCoverage } from "./agent-runner/token-usage-coverage";
@@ -34,7 +36,6 @@ import { AttachmentCacheSweep } from "./attachment-cache-sweep";
 import { resolveTestRunRootPid, TEST_RUN_DEDUP_SWEEP_MS, TestRunDedupSweep } from "./test-run-dedup";
 import { cachedGateRunCommandsForCwd } from "./gate-run-config";
 import { AreaMemoryTracker } from "./area-memory";
-import { GateRunLeaseProbe } from "./gate-run-lease-probe";
 import { ClaimTracker } from "./claim-tracker";
 import { ProviderQuotaHarvest } from "./adapters/provider-quota-harvest";
 import { agentShellLaunchEnv } from "./adapters/project-shell";
@@ -224,7 +225,6 @@ export class AgentRunner {
   private livenessReconcileTimer?: Timer;
   private readonly attachmentCacheSweep: AttachmentCacheSweep;
   private readonly testRunDedupSweep: TestRunDedupSweep;
-  private readonly gateRunLeaseProbe: GateRunLeaseProbe;
   private processStartedAt = 0;
   private providerSessionId = crypto.randomUUID();
   // Last transcript path seen this session — used by end-of-session Insights (#184)
@@ -363,7 +363,7 @@ export class AgentRunner {
     });
     this.lifecycleMilestones = new LifecycleMilestones({
       options: this.options, agentId: this.agentId, process: () => this.process,
-      stopAttachmentCacheSweep: () => this.attachmentCacheSweep.stop(), stopTestRunDedupSweep: () => this.testRunDedupSweep.stop(), stopGateRunLeaseProbe: () => this.gateRunLeaseProbe.stop(),
+      stopAttachmentCacheSweep: () => this.attachmentCacheSweep.stop(), stopTestRunDedupSweep: () => this.testRunDedupSweep.stop(),
       disposeEmptyTurns: () => this.emptyTurns.dispose(), resetEmptyTurns: () => this.emptyTurns.reset(), resetLateClearRecovery: () => { this.lateClearRecovery.reset(); this.continuationFence.reset(); },
       clearLifecycleAction: () => this.lifecycle.setLifecycleAction(undefined), setOffline: () => this.claims.setTerminalStatus("offline"), publishStatus: () => this.publishStatus(),
       stopDrainTimer: () => { if (this.drainTimer) clearTimeout(this.drainTimer); this.drainTimer = undefined; }, stopSubmitRecoveryPoll: () => this.stopSubmitRecoveryPoll(),
@@ -409,14 +409,12 @@ export class AgentRunner {
     const sessionProviderPid = () => resolveTestRunRootPid(this.process, (managedProcess) => this.options.adapter.resolveBusyPid?.(managedProcess));
     // #2461 — ancestry alone made this sweep a no-op for `relay_run` gate runs (a disjoint tree
     // under the orchestrator). It now also owns anything env-tagged with this agent's id, classifies
-    // against the repo's CONFIGURED gate commands, and defers to the launch-time lease holder. The
+    // against the repo's CONFIGURED gate commands, and reads live leases during each sweep. The
     // config read is cached for the session — it is a repo file, so re-reading it every 15s for the
-    // agent's whole life buys nothing. All three helpers live in their own leaves.
-    this.gateRunLeaseProbe = new GateRunLeaseProbe(this.agentId, TEST_RUN_DEDUP_SWEEP_MS);
+    // agent's whole life buys nothing.
     this.testRunDedupSweep = new TestRunDedupSweep({
       rootPid: sessionProviderPid, agentId: () => this.agentId,
       gateCommands: () => cachedGateRunCommandsForCwd(this.options.cwd),
-      leaseHolderPid: () => this.gateRunLeaseProbe.pid(),
       protectedPids: () => [process.pid], log: (message) => this.logRunnerDiagnostic(message),
     });
     this.context.configureLogger();
@@ -450,6 +448,7 @@ export class AgentRunner {
       },
       logRunnerDiagnostic: (message) => this.logRunnerDiagnostic(message),
       reportBackgroundRejection: (scope, status, title, error) => this.reportBackgroundRejection(scope, status, title, error),
+      onTerminalAuthFailure: (reason) => { logger.error("runner", `exiting: ${reason}`); this.options.onProviderExit?.(1, reason); },
     });
     const token = this.credentials.current();
     if (token) publishRuntimeTokenForCli(token, this.credentials.expiresAt, (message) => this.logRunnerDiagnostic(message));
@@ -733,16 +732,17 @@ export class AgentRunner {
     this.ensureScratch();
     void sweepRunnerStaleScratch({ listAgents: () => this.http.listAgents(), agentId: this.agentId, cwd: this.options.cwd, fallbackBaseDir: orchestratorBaseDirFromEnv(), log: (message) => this.logRunnerDiagnostic(message) });
     this.scheduleAttachmentCacheSweep();
-    this.adoptProcess(await this.spawnProvider());
-    this.registration.writeRunnerInfoFile();
-    await this.promptCommands.startLaunchSeededTurn();
-    this.publishStatus();
-    await this.promptCommands.deliverInitialPrompt();
+    await attemptProviderStartup(async () => {
+      this.adoptProcess(await this.spawnProvider());
+      this.registration.writeRunnerInfoFile();
+      await this.promptCommands.startLaunchSeededTurn();
+      this.publishStatus();
+      await this.promptCommands.deliverInitialPrompt();
+    }, (status) => this.setProviderStatus(status));
     await this.bootstrapUnreadMessages();
     this.taskClaims.start();
     this.startHttpLiveness();
     this.testRunDedupSweep.start();
-    this.gateRunLeaseProbe.start();
     this.credentials.scheduleRuntimeTokenRenewal();
     this.scheduleDrain();
   }
@@ -790,6 +790,7 @@ export class AgentRunner {
   }
 
   private handleBusRegistered(): void {
+    if (this.process) this.publishStatus();
     void this.nativeSelfResume.reconcile((id) => this.http.request("GET", `/api/commands/${encodeURIComponent(id)}`)).catch((error) => this.sessionLog(`compact settlement reconcile: ${errMessage(error)}`));
     // #2973 — a re-register follows a relay restart (a deploy): re-list so changed tools reach the agent.
     void this.proxy?.refreshTools().catch(() => {}); this.effort.trigger();
@@ -941,14 +942,7 @@ export class AgentRunner {
     this.scheduleDrain();
   }
 
-  private async bootstrapUnreadMessages(): Promise<void> {
-    try {
-      const messages = await this.http.pollMessages({ for: this.agentId, unread: true, limit: 100, turnDelivery: true });
-      for (const message of messages) this.enqueueMessage(message);
-    } catch (error) {
-      logger.error("runner", `inbox bootstrap failed: ${error}`);
-    }
-  }
+  private bootstrapUnreadMessages = () => bootstrapUnreadMessages(this.http, this.agentId, (message) => this.enqueueMessage(message));
 
   private scheduleDrain(delayMs = 0): void {
     if (this.lifecycle.stopped || this.drainTimer || this.deliverySubmitHold || this.submitResendInFlight) return;
@@ -1363,7 +1357,8 @@ export class AgentRunner {
   private async drainMessages(): Promise<void> {
     if (this.lifecycle.stopped || this.delivering || this.deliverySubmitHold || this.submitResendInFlight || this.pendingMessages.size === 0) return;
     if (!this.process) {
-      this.scheduleDrain(500);
+      if (this.terminalFailure) await reportUnavailableMessages(this.http, this.agentId, this.pendingMessages, this.settledMessageIds, this.terminalFailure.message);
+      if (this.pendingMessages.size) this.scheduleDrain(500);
       return;
     }
     this.delivering = true;
@@ -2822,7 +2817,7 @@ export class AgentRunner {
       ?? (activeWork.length === 0 ? this.emptyTurns.providerState() ?? null : null);
     const published = this.publishRunnerStatus({
       agentStatus,
-      ready: agentStatus !== "offline" && !this.lifecycle.stopped && !teardownLifecycle && !this.startupPaneDialog && !this.deliverySubmitHold,
+      ready: Boolean(this.process) && agentStatus !== "offline" && !this.lifecycle.stopped && !teardownLifecycle && !this.startupPaneDialog && !this.deliverySubmitHold,
       meta: {
         ...runnerIdentityMeta(this.options, { tmuxSession: providerTerminalSession(this.process), tmuxSocket: providerTerminalSocket(this.process), auth: { profileId: this.credentials.profileId, jti: this.credentials.jti, expiresAt: this.credentials.expiresAt } }, "status"),
         lifecycleAction: this.lifecycle.lifecycleAction ?? null,

package/runner/src/spawn-input.ts

@@ -105,6 +105,7 @@ export function parseRunnerCliArgs(
     else if (arg === "--model-record-provider-model" && ownArgs[i + 1]) modelRecordRaw.providerModel = ownArgs[++i];
     else if (arg === "--model-record-qualified" && ownArgs[i + 1]) modelRecordRaw.qualifiedModel = ownArgs[++i];
     else if (arg === "--model-record-effort" && ownArgs[i + 1]) modelRecordRaw.effort = ownArgs[++i];
+    else if (arg === "--model-record-efforts" && ownArgs[i + 1]) { try { modelRecordRaw.efforts = JSON.parse(ownArgs[++i]!); } catch {} }
     else if (arg === "--model-record-window" && ownArgs[i + 1]) {
       const parsed = Number(ownArgs[++i]);
       if (Number.isSafeInteger(parsed) && parsed > 0) modelRecordRaw.contextWindowTokens = parsed;
@@ -143,5 +144,5 @@ export function parseRunnerCliArgs(
 }
 
 export function runnerHelp(provider: string): string {
-  return `${provider}-relay [--headless] [--launch-mode interactive|headless] [--model ALIAS] [--effort LEVEL] [--context-window-tokens N] [--model-record-alias A --model-record-provider-model M --model-record-qualified Q --model-record-effort E --model-record-window N --model-record-max-output N --model-record-threshold JSON] [--rig NAME] [--profile NAME] [--cwd PATH] [--relay-url URL] [--approval MODE] [--label NAME] [--agent-id ID] [--prompt TEXT] [--system-prompt-append TEXT] [--tags a,b] [--caps x,y] [-- provider-args...]`;
+  return `${provider}-humwick [--headless] [--launch-mode interactive|headless] [--model ALIAS] [--effort LEVEL] [--context-window-tokens N] [--model-record-alias A --model-record-provider-model M --model-record-qualified Q --model-record-effort E --model-record-window N --model-record-max-output N --model-record-threshold JSON] [--rig NAME] [--profile NAME] [--cwd PATH] [--relay-url URL] [--approval MODE] [--label NAME] [--agent-id ID] [--prompt TEXT] [--system-prompt-append TEXT] [--tags a,b] [--caps x,y] [-- provider-args...]`;
 }

package/runner/src/test-run-dedup.ts

@@ -1,41 +1,14 @@
 import { relayEnv } from "@humwick/sdk/env";
-// #2142 — per-agent test-run dedup ("latest wins").
-//
-// Workers run `bun run test:affected` after every file change. When a change touches something
-// broad, a single run takes 10-15 minutes — and the agent starts the NEXT run long before the
-// previous one finishes. The stacked runs saturate the host: each one holds a gate slot, forks
-// its own shard processes, and the oldest run's result is thrown away the moment it lands.
-// Instructions don't fix this (an agent rationalizes around them), so the bound has to be
-// structural and live BELOW the agent: the runner watches its own session's process tree and,
-// whenever it sees more than one independent test run, kills every run except the newest.
-//
-// Scoping is per-agent BY CONSTRUCTION: the sweep only ever looks at processes it can PROVE belong
-// to this agent — descendants of this session's provider process (the tmux pane pid / direct child
-// pid the adapter already resolves for busy-liveness), plus processes carrying this agent's
-// AGENT_RELAY_ID in their environment. A sibling agent's runs satisfy neither test, so this can
-// never fratricide the fleet the way a host-wide `pkill -f` would (#753). It is also
-// provider-agnostic — nothing here knows whether the tree belongs to Claude or Codex.
-//
-// #2461 — the env-tag half of that ownership test is new, and it is why this file exists at all
-// after the lease landed. Ancestry ALONE made this a no-op for the case the issue reports: a
-// `relay_run` gate run executes under the ORCHESTRATOR daemon, a tree that shares no ancestor with
-// the provider pane, so a descendant-scoped sweep could never see it. The orchestrator now stamps
-// AGENT_RELAY_ID onto those children, which is what makes them attributable here.
-//
-// This sweep is now the BACKSTOP, not the primary mechanism. Kill-replace happens at LAUNCH, in
-// sdk/src/gate-run-lease.ts, entered from `scripts/gate-slot` and the orchestrator. What is left
-// for the sweep is the residual shape neither front door sees: a gate run started straight from the
-// provider shell in a repo that does not route through `gate.sh`. It is 15s-late by construction —
-// that is the documented coverage gap, tracked for a launch-exact PATH shim in the follow-up issue.
-// When a live lease holder exists it is AUTHORITATIVE: the sweep keeps it and kills the rest, so
-// the two mechanisms can never disagree about which run survives.
+// #3754 — the lease owns launch-time replacement. The sweep only deduplicates
+// unleased runs with the same captured checkout/revision; ancestry proves ownership,
+// but cannot prove two commands tested the same tree. Live leases protect whole runs.
 import { execFile } from "node:child_process";
 import { readFileSync, readdirSync } from "node:fs";
 import { promisify } from "node:util";
 import { errMessage } from "@humwick/sdk";
 import { killPid, waitForPidsExit } from "@humwick/sdk/process-utils";
 import { classifyGateRun, isTestRunCommand, type GateRunCommands } from "@humwick/sdk/gate-run-classify";
-import { GATE_RUN_AGENT_ENV } from "@humwick/sdk/gate-run-lease";
+import { GATE_RUN_AGENT_ENV, GATE_RUN_CONTEXT_ENV, GATE_RUN_HELD_ENV, readLiveGateRunHolderPids } from "@humwick/sdk/gate-run-lease";
 import type { ManagedProcess } from "./adapter";
 
 // Re-exported so the sweep and both lease front doors share ONE definition of "is this a test run".
@@ -215,6 +188,9 @@ export interface TestRunDedupScope {
   /** Classifier override. Defaults to the argv-shape heuristic; the sweep passes a classifier bound
    *  to the repo's CONFIGURED gate commands so a foreign runner (`pnpm tier:run …`) is matched too. */
   isGateRun?: (args: string) => boolean;
+  /** Launch-time checkout/revision identity. Missing identity cannot prove a duplicate. */
+  contextByPid?: ReadonlyMap<number, string>;
+  leasedPids?: readonly number[];
 }
 
 export function planTestRunDedup(
@@ -278,29 +254,34 @@ export function planTestRunDedup(
   }
   if (runRoots.length < 2) return EMPTY_PLAN;
 
-  // Lower elapsed = started later = the run the agent actually wants. `ps etime` only has
-  // second precision, though, and a Codex agent can launch two commands within that second.
-  // Pids are allocated in start order within the host pid namespace, so the higher pid is the
-  // newer root for this otherwise-unresolvable tie. Without this deterministic tie-breaker, two
-  // same-second delivery runs remain tied forever and #2142 fails to bound the exact burst it
-  // was added for. The scope is still this session's process tree only.
-  // A live lease holder wins outright — it is the run the launch-time mechanism already chose. Only
-  // when there is none (an ungated run in a repo without gate.sh) does newest-wins decide.
-  const leaseKeep = scope.keepPid !== undefined && runRoots.includes(scope.keepPid) ? scope.keepPid : undefined;
-  const keptPid = leaseKeep ?? runRoots.reduce((newest, pid) => {
-    const candidate = byPid.get(pid)!;
-    const current = byPid.get(newest)!;
-    return candidate.elapsedSeconds < current.elapsedSeconds
-      || (candidate.elapsedSeconds === current.elapsedSeconds && pid > newest)
-      ? pid
-      : newest;
-  });
-  const newestElapsed = byPid.get(keptPid)!.elapsedSeconds;
-  const staleRoots = runRoots
-    // With a lease holder the age comparison is meaningless — the holder may be the OLDER run, and
-    // it still wins. Without one, keep the original guard so a mis-ranked row is never killed.
-    .filter((pid) => pid !== keptPid && (leaseKeep !== undefined || byPid.get(pid)!.elapsedSeconds >= newestElapsed))
-    .sort((a, b) => a - b);
+  // A lease may belong to a child of the outer shell. Spare the WHOLE run, including
+  // ancestors and descendants, rather than signalling its owner indirectly (#3754).
+  const leased = new Set(scope.leasedPids ?? []);
+  const groups = new Map<string, number[]>();
+  const leasedRoots = new Set<number>();
+  for (const pid of runRoots) {
+    if (subtree(pid).some((child) => leased.has(child))) leasedRoots.add(pid);
+    const context = scope.contextByPid ? scope.contextByPid.get(pid) : "legacy";
+    if (!context) continue;
+    const group = groups.get(context) ?? [];
+    group.push(pid);
+    groups.set(context, group);
+  }
+  const staleRoots: number[] = [];
+  let keptPid: number | undefined;
+  for (const group of groups.values()) {
+    if (group.length < 2) continue;
+    const leaseKeep = scope.keepPid !== undefined && group.includes(scope.keepPid) ? scope.keepPid : undefined;
+    const newest = group.find((pid) => leasedRoots.has(pid)) ?? leaseKeep ?? group.reduce((current, pid) => {
+      const candidate = byPid.get(pid)!;
+      const prior = byPid.get(current)!;
+      return candidate.elapsedSeconds < prior.elapsedSeconds
+        || (candidate.elapsedSeconds === prior.elapsedSeconds && pid > current) ? pid : current;
+    });
+    keptPid = newest;
+    staleRoots.push(...group.filter((pid) => pid !== newest && !leasedRoots.has(pid)));
+  }
+  staleRoots.sort((a, b) => a - b);
   if (staleRoots.length === 0) return EMPTY_PLAN;
 
   const protectedSet = new Set([rootPid, ...protectedPids]);
@@ -334,6 +315,17 @@ export async function readProcessTable(timeoutMs: number = PROCESS_TABLE_TIMEOUT
   return stdout;
 }
 
+/** Only launch-time metadata can distinguish revisions; today's HEAD cannot (#3754). */
+export function readTestRunContext(pid: number): { context?: string; leased?: boolean } {
+  try {
+    const env = Object.fromEntries(readFileSync(`/proc/${pid}/environ`, "utf8").split("\0").map((entry) => {
+      const equals = entry.indexOf("=");
+      return [entry.slice(0, equals), entry.slice(equals + 1)];
+    }));
+    return { context: relayEnv(env, GATE_RUN_CONTEXT_ENV), leased: !!relayEnv(env, GATE_RUN_HELD_ENV) };
+  } catch { return {}; }
+}
+
 export interface TestRunDedupDeps {
   /** Resolve this session's provider pid. undefined ⇒ ancestry proves nothing; the env tag may. */
   rootPid: () => number | undefined;
@@ -344,6 +336,8 @@ export interface TestRunDedupDeps {
   gateCommands?: () => GateRunCommands;
   /** The live lease holder pid, when one exists — authoritative over newest-wins. */
   leaseHolderPid?: () => number | undefined;
+  readRunContext?: (pid: number) => { context?: string; leased?: boolean };
+  liveLeasePids?: () => Promise<number[]>;
   collectOwnedPids?: (agentId: string) => Promise<number[]>;
   readProcessTable?: () => Promise<string>;
   kill?: (pid: number, signal: "SIGTERM" | "SIGKILL") => void;
@@ -374,8 +368,17 @@ export async function runTestRunDedupSweep(deps: TestRunDedupDeps): Promise<numb
   }
 
   const gateCommands = deps.gateCommands?.() ?? {};
-  const plan = planTestRunDedup(parseProcessTable(table), rootPid, deps.protectedPids?.() ?? [], {
-    ownedPids,
+  const rows = parseProcessTable(table);
+  const contextByPid = new Map<number, string>();
+  const leasedPids = await (deps.liveLeasePids ?? (() => agentId ? readLiveGateRunHolderPids(agentId) : Promise.resolve([])))();
+  for (const row of rows) {
+    if (classifyGateRun(row.args, gateCommands) === null) continue;
+    const context = (deps.readRunContext ?? readTestRunContext)(row.pid);
+    if (context.context) contextByPid.set(row.pid, context.context);
+    if (context.leased) leasedPids.push(row.pid);
+  }
+  const plan = planTestRunDedup(rows, rootPid, deps.protectedPids?.() ?? [], {
+    ownedPids, contextByPid, leasedPids,
     keepPid: deps.leaseHolderPid?.(),
     isGateRun: (args) => classifyGateRun(args, gateCommands) !== null,
   });

package/scripts/cd-config.ts

@@ -27,6 +27,19 @@ export function resolveCanonicalRelayDbPath(env: NodeJS.ProcessEnv = process.env
   return resolve(env.DB_PATH || join(defaultStateDir(), "agent-relay.db"));
 }
 
+/** #3301 — the deploy ledger lives only on the hub. Off the hub a release publishes and stops. */
+export function hasCanonicalDeployLedger(env: NodeJS.ProcessEnv = process.env): boolean {
+  return existsSync(resolveCanonicalRelayDbPath(env));
+}
+
+function openCanonicalDeployLedger(): void {
+  const path = resolveCanonicalRelayDbPath();
+  if (!existsSync(path)) {
+    throw new Error(`no deploy ledger at ${path}: deploys run on the hub. CD deploys a published release there, or run \`bun run release --deploy-only\` on the hub.`);
+  }
+  openExistingDb(path);
+}
+
 /**
  * #1763 — the git repository cd-watcher's detached child `git fetch`es against every
  * tick. cd-watcher provisions and owns a DEDICATED clone here, deliberately separate
@@ -146,7 +159,7 @@ export function withDeployLedgerDb<T>(dbPath: string, fn: () => T): T {
 
 /** Claim the one live-ledger lease; both the watcher and release.ts use this path. */
 export function claimCanonicalDeployLease(input: { commitSha: string; version: string; hosts: DeployHost[]; initiatedBy: string; attachedEventId?: string; previousSha?: string }): DeployEvent | null {
-  openExistingDb(resolveCanonicalRelayDbPath());
+  openCanonicalDeployLedger();
   const attached = input.attachedEventId ? getDeployEvent(input.attachedEventId) : undefined;
   if (attached && (attached.status !== "pending" || attached.commitSha !== input.commitSha)) {
     throw new Error(`deploy event ${input.attachedEventId} is not a pending lease for ${input.commitSha}`);
@@ -159,6 +172,14 @@ export function claimCanonicalDeployLease(input: { commitSha: string; version: s
   return tryBeginDeployEvent(input);
 }
 
+/** A scoped release step borrows its parent's lease, including when deploying a rollback SHA (#3470). */
+export function attachCutoverDeployLease(id: string): DeployEvent {
+  openCanonicalDeployLedger();
+  const event = getDeployEvent(id);
+  if (!event || event.status !== "pending" || !event.initiatedBy.startsWith("humwick-cutover:")) throw new Error(`Not a pending cutover lease: ${id}`);
+  return event;
+}
+
 /**
  * #2105 — stamp THIS driver process onto the lease it just claimed, so cd-watcher's reconcile can
  * reclaim a stranded row on executor liveness (minutes) instead of the 2h lease TTL. Best-effort by
@@ -193,7 +214,7 @@ export function startDeployLeaseHeartbeat(id: string): ReturnType<typeof setInte
 
 /** Explicit emergency recovery; normal crash recovery remains the bounded TTL. */
 export function clearCanonicalDeployLease(id: string): boolean {
-  openExistingDb(resolveCanonicalRelayDbPath());
+  openCanonicalDeployLedger();
   return !!forceFailDeployEvent({ id, reason: "force-cleared by operator via release.ts" });
 }
 
@@ -206,7 +227,7 @@ export function clearCanonicalDeployLease(id: string): boolean {
  * the clock, is the authority here).
  */
 export function markCanonicalDeploySucceeded(id: string): boolean {
-  openExistingDb(resolveCanonicalRelayDbPath());
+  openCanonicalDeployLedger();
   return !!reconcilePendingDeployEvent({
     id, status: "succeeded", requireLeaseExpired: false,
     reason: "marked succeeded by operator via release.ts (#1532)",