@humwick/runner 0.0.0-stage → 0.130.1

package/src/agent-runner/commands/prompt.ts

@@ -0,0 +1,284 @@
+import { errMessage, type Message } from "@humwick/sdk";
+import { providerAttachmentLocalPaths, providerAttachmentText, PromptAcceptedError, type ManagedProcess, type PromptOfferOptions, type ProviderInputObserver, type ProviderPermissionDecision, type ProviderPermissionDecisionInput } from "../../adapter";
+import { ProviderSubmitFailure } from "../../provider-submit-failure";
+import type { EmptyTurnArm } from "../../empty-turn";
+import type { ProviderInputTicket } from "../../provider-input-queue";
+import { isTurnInProgressDeliveryError } from "../../runner-helpers";
+import { logger } from "../../logger";
+import type { RunnerTimelineEvent } from "../../turn-reconcile-event";
+import type { RunnerLaunchOptions } from "../runner-context";
+
+interface PromptCommandsDeps {
+  options: RunnerLaunchOptions;
+  agentId: string;
+  process(): ManagedProcess | undefined;
+  isStopped(): boolean;
+  armInput(now: number, mode?: "ladder", initial?: boolean): EmptyTurnArm;
+  disarmInput(arm: EmptyTurnArm): void;
+  inputObserver(arm: EmptyTurnArm): ProviderInputObserver;
+  recordInjectedPrompt(prompt: string, imagePaths?: string[]): void;
+  offerPromptOrPark(prompt: string, options: PromptOfferOptions, ticket?: ProviderInputTicket, onSubmitHeld?: () => void, heldFor?: string): Promise<"accepted" | "parked" | "not-durable" | "submit-held">;
+  offerProviderInput<T>(run: (observer?: ProviderInputObserver) => Promise<T>, offeredNothing?: (result: T) => boolean): Promise<T>;
+  terminalFailure(): Readonly<{ message: string }> | undefined;
+  sessionLog(message: string): void;
+  publishRunnerTimelineEvent(event: RunnerTimelineEvent): void;
+  holdFailedProviderSubmit(detail: string, held: { resend: () => Promise<unknown> }): void;
+  resolvePermissionDecision(input: ProviderPermissionDecisionInput): boolean | undefined;
+  reservePark(): ProviderInputTicket;
+  notePendingPrompt(messageId: number): void;
+  cacheAttachments(messages: Message[]): Promise<Message[]>;
+  parkPromptBehindSubmitHold(params: Record<string, unknown>, key: string, turn: ProviderInputTicket): Promise<Record<string, unknown>>;
+  waitForProviderTurnClear(): Promise<void>;
+  pendingQuestionApprovalId(): string | undefined;
+  clearPendingQuestionHold(): void;
+  publishStatus(): void;
+}
+
+// #329: a cold isolated profile (first-run trust/onboarding/install gates) can take far longer
+// than the warm-restart window to render an input-ready TUI, so the first prompt delivery gets a
+// generous timeout. If it still misses, the prompt is re-attempted on the next provider ready
+// signal (short timeout, since it just went idle), up to a bounded number of attempts.
+const INITIAL_PROMPT_FIRST_READY_TIMEOUT_MS = 30_000;
+const INITIAL_PROMPT_RETRY_READY_TIMEOUT_MS = 10_000;
+const MAX_INITIAL_PROMPT_ATTEMPTS = 6;
+const PROMPT_NOT_DURABLE = "prompt refused under a provider hold and could not be parked durably; not delivered";
+/** Who a prompt.inject came from (every producer stamps its session message's sender); parked copies keep it (#2664). */
+export const promptOrigin = (params: Record<string, unknown>): string => [params.from, params.requestedBy].find((id): id is string => typeof id === "string" && id !== "") ?? "user";
+
+export class PromptCommands {
+  // #329: a spawn-time initial prompt whose first delivery timed out before the TUI was
+  // ready. Held (not dropped) for a ready-signal-driven re-attempt; cleared on success.
+  private pendingInitialPrompt?: string;
+  private deliveringInitialPrompt = false;
+  private initialPromptAttempts = 0;
+
+  constructor(private readonly deps: PromptCommandsDeps) {}
+
+  noteLaunchSeededPrompt(opts: { resumeId?: string; suppressPrompt?: boolean }): string | undefined {
+    const launchSeededPrompt = this.deps.options.adapter.seedsInitialPromptAtLaunch && !opts.resumeId && !opts.suppressPrompt ? this.deps.options.prompt?.trim() : ""; if (launchSeededPrompt) { this.deps.recordInjectedPrompt(launchSeededPrompt); this.deps.armInput(Date.now()); }
+    return launchSeededPrompt;
+  }
+
+  async startLaunchSeededTurn(): Promise<void> {
+    if (this.deps.process()?.meta?.launchSeededTurn !== true) return;
+    try {
+      await this.deps.options.adapter.startLaunchSeededTurn?.(this.deps.process()!);
+    } catch (error) {
+      this.deps.sessionLog(`launch-seeded turn watch failed: ${errMessage(error)}`);
+    }
+  }
+
+  async deliverInitialPrompt(): Promise<void> {
+    const prompt = this.deps.options.prompt?.trim();
+    if (!prompt || this.deps.options.adapter.seedsInitialPromptAtLaunch) return;
+    await this.attemptInitialPromptDelivery(prompt, INITIAL_PROMPT_FIRST_READY_TIMEOUT_MS);
+  }
+
+  // Deliver the spawn-time first prompt, surviving a cold-start TUI that isn't input-ready yet
+  // (#329). Timeout stashes it for ready-signal retry and surfaces the stall on the timeline.
+  private async attemptInitialPromptDelivery(prompt: string, readyTimeoutMs: number): Promise<void> {
+    if (this.deliveringInitialPrompt || this.deps.isStopped() || !this.deps.process() || !this.deps.options.adapter.deliverInitialPrompt) return;
+    this.deliveringInitialPrompt = true;
+    const attempt = (this.initialPromptAttempts += 1);
+    let arm: EmptyTurnArm | undefined;
+    try {
+      const startedAt = Date.now();
+      // #2389 — armed BEFORE the delivery is awaited. On OpenCode `deliverInitialPrompt` awaits the
+      // synchronous /message POST, which returns only when the assistant message is complete; an arm
+      // taken on resolution lands at or after that turn's own idle edge and misses it entirely.
+      arm = this.deps.armInput(Date.now(), "ladder", true);
+      this.deps.recordInjectedPrompt(prompt);
+      const outcome = await this.deps.offerPromptOrPark(prompt, { readyTimeoutMs, isInitial: true, inputObserver: this.deps.inputObserver(arm) });
+      // #2619 — a refused launch prompt that could not be made durable stays pending (re-offered
+      // on the next ready edge); a durably parked one redelivers from the inbox.
+      this.pendingInitialPrompt = outcome === "not-durable" ? prompt : undefined;
+      if (outcome !== "accepted") {
+        this.deps.sessionLog(`initial prompt attempt ${attempt} refused under a provider hold (${prompt.length} chars); ${outcome === "parked" ? "parked for redelivery" : "not durable, kept pending"}`);
+        return;
+      }
+      // #2397 round 3 review (LOW-5r) — a synchronous-endpoint adapter (OpenCode) can resolve this
+      // call NORMALLY after running its own bounded retry+reroll loop into a TERMINAL provider
+      // failure (see failTurn) rather than throwing — throwing here would route into this method's
+      // own retry-and-give-up path, re-sending the same prompt on top of the budget the adapter
+      // already exhausted. `this.deps.terminalFailure()` is ground truth for that outcome: it is set by
+      // the adapter's own `error` status and cleared by the `busy` edge THIS delivery started with
+      // (setProviderStatus, ~line 1841), so a set value here can only have been raised during this
+      // very call, never stale. Without this check, a freshly spawned agent's terminally-failed
+      // first turn — the exact incident #2397 was filed on — was still logged "delivered".
+      if (this.deps.terminalFailure()) {
+        this.deps.sessionLog(`initial prompt attempt ${attempt} ended in a terminal provider failure (${prompt.length} chars, ${Date.now() - startedAt}ms): ${this.deps.terminalFailure()!.message}`);
+      } else {
+        // #2389 — delivery used to be logged ONLY on exception, so a spawn that idled forever left no
+        // evidence of whether its first prompt ever reached the provider; establishing that took
+        // SQLite archaeology on retained provider state. Say so on success too.
+        this.deps.sessionLog(`initial prompt delivered on attempt ${attempt} (${prompt.length} chars, ${Date.now() - startedAt}ms)`);
+      }
+    } catch (error) {
+      // Nothing reached the provider, so nothing is owed — unless a turn started meanwhile, which
+      // disarmInput checks: a synchronous-endpoint transport can throw after the model already ran.
+      if (arm) this.deps.disarmInput(arm);
+      // #2534 — the adapter proved the provider ACCEPTED and persisted this prompt (matched to the
+      // outbound request identity) before its completion await failed (e.g. the #2425 whole-turn
+      // abort on a long first turn). The provider is already working on it, so requeueing would
+      // run the same assignment a second time — and with the acceptance-keyed markers retained,
+      // the retry would also no longer carry the launch briefing. Drop it like a normal success.
+      if (error instanceof PromptAcceptedError) {
+        this.pendingInitialPrompt = undefined;
+        this.deps.sessionLog(`initial prompt attempt ${attempt} was accepted by the provider before its completion await failed (${prompt.length} chars); kept as delivered, not requeued: ${errMessage(error)}`);
+        return;
+      }
+      const giveUp = (error instanceof ProviderSubmitFailure && !error.retrySafe) || attempt >= MAX_INITIAL_PROMPT_ATTEMPTS;
+      this.pendingInitialPrompt = giveUp ? undefined : prompt;
+      logger.error("runner", `initial prompt attempt ${attempt} failed${giveUp ? " (giving up)" : "; will retry on next ready signal"}: ${errMessage(error)}`);
+      this.deps.publishRunnerTimelineEvent({ status: "prompt_failed", timestamp: Date.now(), title: "Initial prompt not delivered", body: errMessage(error), metadata: { attempts: attempt, terminal: giveUp } });
+      if (error instanceof ProviderSubmitFailure && giveUp) this.deps.holdFailedProviderSubmit(`Initial provider prompt submission unconfirmed after ${attempt} attempt(s): ${error.message}`, { resend: () => this.attemptInitialPromptDelivery(prompt, INITIAL_PROMPT_RETRY_READY_TIMEOUT_MS) });
+    } finally {
+      this.deliveringInitialPrompt = false;
+    }
+  }
+
+  async respondToPermissionDecision(params: Record<string, unknown>): Promise<Record<string, unknown>> {
+    const approvalId = typeof params.approvalId === "string" ? params.approvalId : "";
+    const decision = typeof params.decision === "string" ? params.decision : "";
+    if (!approvalId) throw new Error("approvalId required");
+    if (decision !== "approve" && decision !== "approve-session" && decision !== "deny" && decision !== "abort" && decision !== "answer") {
+      throw new Error("decision must be approve, approve-session, deny, abort, or answer");
+    }
+    const answersRaw = params.answers;
+    const answers = answersRaw && typeof answersRaw === "object" && !Array.isArray(answersRaw)
+      ? Object.fromEntries(
+          Object.entries(answersRaw as Record<string, unknown>).filter(([, v]) => typeof v === "string") as [string, string][],
+        )
+      : undefined;
+    if (decision === "answer" && (!answers || Object.keys(answers).length === 0)) {
+      throw new Error("answers required for answer decision");
+    }
+    const input: ProviderPermissionDecisionInput = {
+      approvalId,
+      decision: decision as ProviderPermissionDecision,
+      ...(typeof params.reason === "string" ? { reason: params.reason } : {}),
+      ...(answers ? { answers } : {}),
+    };
+    if (this.deps.resolvePermissionDecision(input)) {
+      return { approvalId, decision, provider: this.deps.options.provider };
+    }
+    if (!this.deps.process()) throw new Error("provider process is unavailable");
+    if (!this.deps.options.adapter.respondToPermissionDecision) throw new Error("provider does not support permission decisions");
+    const result = await this.deps.options.adapter.respondToPermissionDecision(this.deps.process()!, input);
+    return { approvalId, decision, ...(result ? { providerResult: result } : {}) };
+  }
+
+  async injectPrompt(params: Record<string, unknown>, ticket?: ProviderInputTicket, commandId?: string): Promise<Record<string, unknown>> {
+    const body = typeof params.body === "string" ? params.body : "";
+    if (!body) throw new Error("body required");
+    if (!this.deps.process()) throw new Error("provider process is unavailable");
+    if (!this.deps.options.adapter.deliverInitialPrompt) throw new Error("provider does not support prompt injection");
+    const messageId = typeof params.messageId === "number" ? params.messageId : undefined;
+    // A prompt that raced the hold's publication (the server 409s once it sees the hold) parks behind the held item,
+    // decided in its FIFO slot only, so parking follows the input order.
+    const parkKey = `submit-hold-prompt:${messageId ?? commandId ?? crypto.randomUUID()}`;
+    let parkTurn: ProviderInputTicket | undefined;
+    const reservePark = () => { parkTurn = this.deps.reservePark(); };
+    const attachments = Array.isArray(params.attachments)
+      ? params.attachments.filter((item): item is Record<string, unknown> => item !== null && typeof item === "object" && !Array.isArray(item))
+      : [];
+    if (messageId) this.deps.notePendingPrompt(messageId);
+    const { prompt, imagePaths } = await this.promptInjectionText(body, messageId, attachments);
+    // Mark so the matching UserPromptSubmit echo isn't double-posted: a chat-box
+    // prompt already created its own session message shown in the dashboard.
+    this.deps.recordInjectedPrompt(prompt.trim(), imagePaths);
+    // #1228: prompt injection is a manual dashboard "continue"/chat-box send, not the
+    // agent's genuine first turn — never label it as the initial prompt.
+    // #2389/#2539 — offer through the ONE provider-input seam: arm the empty-turn watch at
+    // hand-off, serialize through the FIFO behind any reserved ticket, disarm on rejection.
+    // #2543 — a busy recipient (the adapter answered with the shared turn-in-progress signal
+    // instead of an unguarded start) DEFERS: bounded wait for the provider turn to clear, then
+    // one re-offer on the idle path through a fresh FIFO slot — the same shape as the #2539
+    // context-injection defer. A timed-out wait (or any other failure) throws so the command
+    // fails visibly — never a silent drop.
+    const submitHeld = (outcome: string) => outcome === "submit-held";
+    try {
+      const outcome = await this.deps.offerProviderInput((inputObserver) => this.deps.offerPromptOrPark(prompt, { isInitial: false, imagePaths, inputObserver }, ticket, reservePark, promptOrigin(params)), submitHeld);
+      if (outcome === "not-durable") throw new Error(PROMPT_NOT_DURABLE);
+      if (outcome === "submit-held") return this.deps.parkPromptBehindSubmitHold(params, parkKey, parkTurn!);
+    } catch (error) {
+      if (!isTurnInProgressDeliveryError(error)) {
+        if (error instanceof ProviderSubmitFailure && !error.retrySafe) this.deps.holdFailedProviderSubmit(`Dashboard prompt submission unconfirmed: ${error.message}`, { resend: () => this.injectPrompt(params) });
+        throw error;
+      }
+      this.deps.sessionLog("prompt injection deferred behind the active provider turn; delivering when it ends");
+      await this.deps.waitForProviderTurnClear();
+      this.deps.recordInjectedPrompt(prompt.trim(), imagePaths); // refresh the echo-dedup window past the wait
+      const outcome = await this.deps.offerProviderInput((inputObserver) => this.deps.offerPromptOrPark(prompt, { isInitial: false, imagePaths, inputObserver }, undefined, reservePark, promptOrigin(params)), submitHeld);
+      if (outcome === "not-durable") throw new Error(PROMPT_NOT_DURABLE);
+      if (outcome === "submit-held") return this.deps.parkPromptBehindSubmitHold(params, parkKey, parkTurn!);
+    }
+    return { injected: true, messageId, attachmentCount: attachments.length };
+  }
+
+  // #1173: the resolution side of the deny-then-hold AskUserQuestion flow (see control-server.ts's
+  // handlePermissionRequest). By the time this fires, the original PreToolUse hook has long since
+  // been answered with `deny` — there's no live HTTP request to respond to — so the human's answer
+  // (or dismissal) is delivered by clearing the pending-question hold and injecting a normal
+  // user prompt via the same path the dashboard chat box uses.
+  async injectAnswerPrompt(input: { approvalId: string; text?: string }): Promise<void> {
+    const heldApprovalId = this.deps.pendingQuestionApprovalId();
+    if (heldApprovalId && heldApprovalId !== input.approvalId) return;
+    if (heldApprovalId === input.approvalId) {
+      this.deps.clearPendingQuestionHold();
+      this.deps.publishStatus();
+    }
+    const text = input.text ?? "The question was dismissed or timed out before the user answered. Proceed on your best judgment, or re-ask if you still need a decision.";
+    if (!this.deps.process() || !this.deps.options.adapter.deliverInitialPrompt) {
+      this.deps.sessionLog(`could not inject AskUserQuestion answer for ${input.approvalId}: provider process/prompt-injection unavailable`);
+      return;
+    }
+    this.deps.recordInjectedPrompt(text.trim());
+    // #1228: this resolves a held AskUserQuestion mid-session — never the initial prompt.
+    // #2389/#2539 — the answer unblocks a turn that is waiting on it, so it arms.
+    // #2543 — offered through the same seam with the same busy policy: the adapter's shared
+    // turn-in-progress refusal defers (bounded turn-clear wait + one re-offer) instead of an
+    // unguarded start; any other failure stays best-effort logged, exactly as before.
+    const deliver = async (inputObserver?: ProviderInputObserver) => {
+      if (await this.deps.offerPromptOrPark(text, { isInitial: false, inputObserver }, undefined, undefined, input.text !== undefined ? "user" : undefined) === "not-durable") throw new Error(PROMPT_NOT_DURABLE);
+    };
+    try {
+      await this.deps.offerProviderInput(deliver);
+    } catch (error) {
+      if (!isTurnInProgressDeliveryError(error)) {
+        this.deps.sessionLog(`failed to inject AskUserQuestion answer for ${input.approvalId}: ${errMessage(error)}`);
+        return;
+      }
+      try {
+        this.deps.sessionLog(`AskUserQuestion answer for ${input.approvalId} deferred behind the active provider turn; delivering when it ends`);
+        await this.deps.waitForProviderTurnClear();
+        this.deps.recordInjectedPrompt(text.trim()); // refresh the echo-dedup window past the wait
+        await this.deps.offerProviderInput(deliver);
+      } catch (retryError) {
+        this.deps.sessionLog(`failed to inject AskUserQuestion answer for ${input.approvalId}: ${errMessage(retryError)}`);
+      }
+    }
+  }
+
+  private async promptInjectionText(body: string, messageId: number | undefined, attachments: Record<string, unknown>[]): Promise<{ prompt: string; imagePaths: string[] }> {
+    if (!attachments.length) return { prompt: body, imagePaths: [] };
+    const messages = await this.deps.cacheAttachments([{
+      id: messageId ?? 0,
+      from: "user",
+      to: this.deps.agentId,
+      kind: "session",
+      body,
+      payload: { attachments },
+      readBy: [],
+      createdAt: Date.now(),
+    }]);
+    const attachmentText = providerAttachmentText(messages[0]!);
+    return { prompt: attachmentText ? `${body}\n\n${attachmentText}` : body, imagePaths: providerAttachmentLocalPaths(messages) };
+  }
+
+  retryInitialPrompt(): void {
+    if (this.pendingInitialPrompt && !this.deliveringInitialPrompt) {
+      void this.attemptInitialPromptDelivery(this.pendingInitialPrompt, INITIAL_PROMPT_RETRY_READY_TIMEOUT_MS);
+    }
+  }
+}

package/src/agent-runner/credentials.ts

@@ -0,0 +1,208 @@
+import type { HumwickHttpClient } from "@humwick/sdk";
+import { orchestratorUrlFromEnv } from "../config";
+import { httpErrorKey, runtimeTokenRenewDelayMs, runtimeTokenRenewTimerSchedule } from "../runner-helpers";
+import { rederiveRuntimeToken, TokenRotation } from "../runtime-token-remint";
+
+type RenewedTokenRecord = Awaited<ReturnType<HumwickHttpClient["renewRuntimeToken"]>>["record"];
+type RotationStatus = "runtime-token-renewed" | "runtime-token-reminted";
+interface CredentialsDeps {
+  isStopped(): boolean;
+  renewRuntimeToken: HumwickHttpClient["renewRuntimeToken"];
+  remintRuntimeToken: HumwickHttpClient["remintRuntimeToken"];
+  onTokenChanged(token: string, record: RenewedTokenRecord): void;
+  resumeLivenessAndPublishRotation(status: RotationStatus, jti: string): void;
+  publishRenewalFailure(): void;
+  logRunnerDiagnostic(message: string): void;
+  reportBackgroundRejection(scope: string, status: string, title: string, error: unknown): void;
+}
+
+const TOKEN_RENEW_LOG_INTERVAL_MS = 5 * 60 * 1000;
+const TOKEN_RENEW_RETRY_MS = 60_000;
+// Debounce reactive token recovery so a burst of 401-ing calls in the same window
+// triggers a single re-mint attempt, not one per failing request.
+const REACTIVE_TOKEN_RECOVERY_DEBOUNCE_MS = 10_000;
+
+export class Credentials {
+  private currentToken?: string;
+  private currentTokenJti?: string;
+  private currentTokenProfileId?: string;
+  private currentTokenExpiresAt?: number;
+  private tokenRenewTimer?: Timer;
+  private readonly tokenRotation = new TokenRotation();
+  private tokenRenewLastLog?: { key: string; at: number };
+  private reactiveTokenRecoveryAt?: number;
+
+  constructor(options: { token?: string; tokenJti?: string; tokenProfileId?: string; tokenExpiresAt?: number }, private readonly deps: CredentialsDeps) {
+    this.currentToken = options.token;
+    this.currentTokenJti = options.tokenJti;
+    this.currentTokenProfileId = options.tokenProfileId;
+    this.currentTokenExpiresAt = options.tokenExpiresAt;
+  }
+
+  current(): string | undefined { return this.currentToken; }
+  get jti(): string | undefined { return this.currentTokenJti; }
+  get profileId(): string | undefined { return this.currentTokenProfileId; }
+  get expiresAt(): number | undefined { return this.currentTokenExpiresAt; }
+
+  stop(): void {
+    if (this.tokenRenewTimer) clearTimeout(this.tokenRenewTimer);
+    this.tokenRenewTimer = undefined;
+  }
+
+  // Timer/edge callers cannot await renewal; report an escaping rejection so it cannot
+  // silently strand the runner off the bus (#1617).
+  private renewRuntimeTokenInBackground(): void {
+    void this.renewRuntimeToken().catch((error) => this.deps.reportBackgroundRejection("runtime token renewal", "runtime-token-renewal-failed", "Runtime token renewal aborted", error));
+  }
+
+  // The TTL timer cannot detect revocation: recover on auth failure, coalescing a burst.
+  // Orchestrator re-mint heals stale/expired tokens but preserves hard revocation (#484).
+  recoverRuntimeTokenAfterAuthFailure(source: string): void {
+    if (this.deps.isStopped() || this.tokenRotation.inFlight) return;
+    if (!this.isRuntimeTokenRenewable() && !this.canRemintViaOrchestrator()) return;
+    const now = Date.now();
+    if (this.reactiveTokenRecoveryAt && now - this.reactiveTokenRecoveryAt < REACTIVE_TOKEN_RECOVERY_DEBOUNCE_MS) return;
+    this.reactiveTokenRecoveryAt = now;
+    this.deps.logRunnerDiagnostic(`[runner] relay auth failure on ${source}; recovering runtime token`);
+    this.renewRuntimeTokenInBackground();
+  }
+
+  scheduleRuntimeTokenRenewal(delayMs?: number): void {
+    if (this.tokenRenewTimer) clearTimeout(this.tokenRenewTimer);
+    this.tokenRenewTimer = undefined;
+    if (this.deps.isStopped()) return;
+    const canSelfRenew = this.isRuntimeTokenRenewable();
+    const canRemint = this.canRemintViaOrchestrator();
+    // Keep the renewal clock ticking as long as the session can recover its token
+    // by EITHER path. Without the re-mint fallback an expired token would stop the
+    // timer forever (the old deadlock that stranded live agents off the bus).
+    if (!canSelfRenew && !canRemint) return;
+    let computedDelay = delayMs;
+    if (computedDelay === undefined) {
+      computedDelay = canSelfRenew
+        ? runtimeTokenRenewDelayMs(this.currentTokenExpiresAt!, Date.now())
+        : TOKEN_RENEW_RETRY_MS; // expired but re-mintable → retry via orchestrator soon
+      if (computedDelay === undefined) computedDelay = TOKEN_RENEW_RETRY_MS;
+    }
+    const schedule = runtimeTokenRenewTimerSchedule(computedDelay);
+    if (!schedule) return;
+    this.tokenRenewTimer = setTimeout(() => {
+      this.tokenRenewTimer = undefined;
+      if (!schedule.renew) {
+        this.scheduleRuntimeTokenRenewal();
+        return;
+      }
+      this.renewRuntimeTokenInBackground();
+    }, schedule.delayMs);
+  }
+
+  // Can the runner self-renew right now? Requires a non-expired runner-profile token
+  // (the relay rejects renewal of an expired token).
+  private isRuntimeTokenRenewable(): boolean {
+    return Boolean(
+      this.currentToken &&
+      this.currentTokenExpiresAt &&
+      this.currentTokenExpiresAt * 1000 > Date.now() &&
+      (this.currentTokenProfileId === "provider-agent" || this.currentTokenProfileId === "provider-interactive"),
+    );
+  }
+
+  // Can the runner recover its token via the orchestrator? Works even when the token
+  // is already expired — the orchestrator's standing credential is the authority.
+  private canRemintViaOrchestrator(): boolean {
+    return Boolean(
+      orchestratorUrlFromEnv() &&
+      this.currentToken &&
+      (this.currentTokenProfileId === "provider-agent" || this.currentTokenProfileId === "provider-interactive"),
+    );
+  }
+
+  private async renewRuntimeToken(): Promise<void> {
+    if (this.deps.isStopped() || this.tokenRotation.inFlight || !this.currentToken) return;
+    await this.tokenRotation.run(async () => {
+      // Preferred path: self-renew directly against the relay while the token is
+      // still valid. Cheapest and needs no orchestrator round-trip.
+      if (this.isRuntimeTokenRenewable()) {
+        try {
+          const renewed = await this.deps.renewRuntimeToken();
+          this.applyRenewedToken(renewed.token, renewed.record, "runtime-token-renewed");
+          return;
+        } catch (error) {
+          this.logRuntimeTokenRenewalFailure(error);
+          // Relay unreachable or token rejected — fall through to orchestrator re-mint.
+        }
+      }
+      // Recovery path: token expired, or self-renew failed. Ask the orchestrator —
+      // it holds a long-lived credential and can mint a fresh runner token, so a
+      // live session heals instead of being stranded off the bus.
+      if (this.canRemintViaOrchestrator() && await this.remintViaOrchestrator()) return;
+      this.deps.publishRenewalFailure();
+      this.scheduleRuntimeTokenRenewal(TOKEN_RENEW_RETRY_MS);
+    });
+  }
+
+  // Install state before core fan-out, then resume liveness and renew on the same clock.
+  private applyRenewedToken(
+    token: string,
+    record: RenewedTokenRecord,
+    status: RotationStatus,
+  ): void {
+    this.currentToken = token;
+    this.currentTokenJti = record.jti;
+    this.currentTokenProfileId = record.profileId ?? this.currentTokenProfileId;
+    this.currentTokenExpiresAt = record.expiresAt;
+    this.deps.onTokenChanged(token, record);
+    this.reactiveTokenRecoveryAt = undefined;
+    this.deps.resumeLivenessAndPublishRotation(status, record.jti);
+    this.scheduleRuntimeTokenRenewal();
+  }
+
+  // Recover the runtime token through the orchestrator. The runner proxies its own
+  // (possibly expired) token; the orchestrator re-mints it via the relay using its
+  // standing credential. Returns true on success.
+  private async remintViaOrchestrator(): Promise<boolean> {
+    const orchUrl = orchestratorUrlFromEnv();
+    if (!orchUrl || !this.currentToken) return false;
+    try {
+      const res = await fetch(`${orchUrl.replace(/\/+$/, "")}/api/runtime-tokens/runner-renew`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ token: this.currentToken }),
+        signal: AbortSignal.timeout(10_000),
+      });
+      if (!res.ok) return false;
+      const renewed = await res.json() as { token?: string; record?: { jti: string; profileId?: string; expiresAt?: number } };
+      if (!renewed?.token || !renewed.record) return false;
+      this.applyRenewedToken(renewed.token, renewed.record, "runtime-token-reminted");
+      this.deps.logRunnerDiagnostic(`[runner] runtime token re-minted via orchestrator (jti ${renewed.record.jti})`);
+      return true;
+    } catch (error) {
+      this.logRuntimeTokenRenewalFailure(error);
+      return false;
+    }
+  }
+
+  // #1245 — a newly allowed tool group or scope reaches the next generation without a restart.
+  async remintRuntimeTokenAtBoundary(reason: "clear" | "compact"): Promise<Record<string, unknown>> {
+    try {
+      // Queued behind a renewal in flight, so it re-derives from the token that renewal installs.
+      return await this.tokenRotation.runNext(async () => {
+        if (this.deps.isStopped() || !this.currentToken || this.currentTokenProfileId !== "provider-agent") return { status: "skipped", reason: "no spawned runner token" };
+        return rederiveRuntimeToken({ remintRuntimeToken: this.deps.remintRuntimeToken }, reason, (token, record) => this.applyRenewedToken(token, record, "runtime-token-reminted"), (message) => this.deps.logRunnerDiagnostic(message));
+      });
+    } finally {
+      this.scheduleRuntimeTokenRenewal(); // a renewal timer that fired meanwhile returned without re-arming
+    }
+  }
+
+  private logRuntimeTokenRenewalFailure(error: unknown): void {
+    const key = httpErrorKey(error);
+    const now = Date.now();
+    if (
+      this.tokenRenewLastLog?.key === key &&
+      now - this.tokenRenewLastLog.at < TOKEN_RENEW_LOG_INTERVAL_MS
+    ) return;
+    this.tokenRenewLastLog = { key, at: now };
+    this.deps.logRunnerDiagnostic(`[runner] runtime token renewal failed: ${String(error)}`);
+  }
+}

package/src/agent-runner/http-liveness.ts

@@ -0,0 +1,85 @@
+import { errMessage, type HumwickHttpClient } from "@humwick/sdk";
+import { httpErrorKey, isHttpAuthError, runnerAgentStatus } from "../runner-helpers";
+import type { ClaimTracker } from "../claim-tracker";
+
+const HTTP_LIVENESS_INTERVAL_MS = 20_000;
+
+interface HttpLivenessDeps {
+  http: Pick<HumwickHttpClient, "heartbeat">;
+  agentId: string;
+  instanceId: string;
+  logIntervalMs: number;
+  isStopped(): boolean;
+  isTerminalStatusLatched(): boolean;
+  currentStatus: ClaimTracker["currentStatus"];
+  publishQuota(): Promise<void>;
+  bootstrapUnreadMessages(): Promise<void>;
+  triggerEffort(): void;
+  recoverRuntimeTokenAfterAuthFailure(source: string): void;
+  logRunnerDiagnostic(message: string): void;
+}
+
+export class HttpLiveness {
+  private httpLivenessTimer?: Timer;
+  private httpLivenessInFlight = false;
+  private httpLivenessAuthFailed = false;
+  private httpLivenessLastLog?: { key: string; at: number };
+
+  constructor(private readonly deps: HttpLivenessDeps) {}
+
+  stop(): void {
+    if (this.httpLivenessTimer) clearInterval(this.httpLivenessTimer);
+    this.httpLivenessTimer = undefined;
+  }
+
+  resetAuthFailure(): void { this.httpLivenessAuthFailed = false; }
+
+  start(): void {
+    if (this.httpLivenessTimer) clearInterval(this.httpLivenessTimer);
+    void this.publishHttpLiveness();
+    this.httpLivenessTimer = setInterval(() => {
+      void this.publishHttpLiveness();
+    }, HTTP_LIVENESS_INTERVAL_MS);
+  }
+
+  private async publishHttpLiveness(): Promise<void> {
+    if (this.deps.isStopped() || this.deps.isTerminalStatusLatched() || this.httpLivenessInFlight || this.httpLivenessAuthFailed) return;
+    this.httpLivenessInFlight = true;
+    const status = this.deps.currentStatus();
+    const agentStatus = runnerAgentStatus(status);
+    try {
+      // #3606 — one call; the hub writes and emits only when status or readiness actually changed.
+      await this.deps.http.heartbeat(this.deps.agentId, this.deps.instanceId, { status: agentStatus, ready: agentStatus !== "offline" });
+      await this.deps.publishQuota().catch((error) => this.deps.logRunnerDiagnostic(`passive provider quota publish failed: ${errMessage(error)}`));
+      await this.deps.bootstrapUnreadMessages(); this.deps.triggerEffort();
+    } catch (error) {
+      if (!this.deps.isStopped()) this.handleHttpLivenessFailure(error);
+    } finally {
+      this.httpLivenessInFlight = false;
+    }
+  }
+
+  private handleHttpLivenessFailure(error: unknown): void {
+    const authFailed = isHttpAuthError(error);
+    this.logHttpLivenessFailure(error, authFailed);
+    if (!authFailed) return;
+    this.httpLivenessAuthFailed = true;
+    if (this.httpLivenessTimer) clearInterval(this.httpLivenessTimer);
+    this.httpLivenessTimer = undefined;
+    // A 401/403 here is the only timely signal that the token died — stopping the
+    // liveness timer means there is no second chance, so recover from THIS failure.
+    this.deps.recoverRuntimeTokenAfterAuthFailure("http-liveness");
+  }
+
+  private logHttpLivenessFailure(error: unknown, authFailed: boolean): void {
+    const key = httpErrorKey(error);
+    const now = Date.now();
+    if (
+      this.httpLivenessLastLog?.key === key &&
+      now - this.httpLivenessLastLog.at < this.deps.logIntervalMs
+    ) return;
+    this.httpLivenessLastLog = { key, at: now };
+    const suffix = authFailed ? "auth failed; stopping HTTP liveness retries until restart" : String(error);
+    this.deps.logRunnerDiagnostic(`[runner] HTTP liveness update failed: ${suffix}`);
+  }
+}