@humwick/server 0.0.0-stage → 0.130.0

package/runner/src/control-server.ts

@@ -0,0 +1,730 @@
+import type { Server, ServerWebSocket } from "bun";
+import type { InteractivePrompt, Message, OperatorDecisionFileInput, ReplyObligation } from "@humwick/sdk";
+import { errMessage, isRecord } from "@humwick/sdk";
+import type { ProviderPermissionDecisionInput, ProviderPermissionDecisionReason, ProviderPermissionPromptHandler, ProviderPermissionResolvedPrompt, ProviderStatusEvent, SemanticStatus, TerminalAttachSpec } from "./adapter";
+import { obligationNagEligible } from "./reply-obligation-cache";
+import { relayReplyActionText } from "./relay-instructions";
+import { logger, parseLogLevel, LOG_LEVELS } from "./logger";
+import { buildRateLimitProviderState } from "./rate-limit";
+import { bestEffortDeletePendingPermissionRequest, bestEffortListPendingPermissionRequests, bestEffortUpsertPendingPermissionRequest, type PendingPermissionRequestRecord, type PendingPermissionStore } from "./pending-permission-store";
+
+// Provider permission hooks can wait a long time. Resolve the server-side wait with
+// a margin under the provider hook timeout so the server returns a deterministic
+// decision before the provider abandons the hook.
+const PERMISSION_WAIT_MS = 880_000;
+// Distinct from a human dismissal ("Dismissed/Denied from Humwick dashboard") so the
+// agent can tell "nobody answered in time" (retriable) from "the human said no" (#693).
+function permissionTimeoutReason(view?: Pick<InteractivePrompt, "toolName" | "title">): string {
+  const tool = view?.toolName?.trim() || view?.title?.replace(/^Approve\s+/i, "").trim() || "provider tool";
+  return `Permission request for ${tool} timed out after ~15 minutes without a coordinator decision; the tool was denied (timeout, not a dismissal). Re-ask if you still need it.`;
+}
+const PERMISSION_STOP_REASON = "The runner restarted while this permission request was pending; the original hook is no longer connected.";
+const RESOLVED_APPROVAL_LIMIT = 1024;
+// #1173: the immediate PreToolUse response for a "questions" prompt (see the deny-then-hold
+// branch in handlePermissionRequest below). Bridges the hand-off so the model doesn't read a
+// bare denial and re-ask — the real answer follows shortly as an injected user prompt.
+const PENDING_QUESTION_DENY_REASON =
+  "The user is answering this question directly via Agent Relay, not through this tool call — do not re-ask it. Their answer will follow as your next message; wait for it and continue from there.";
+
+// A hook that failed in a way it could not handle itself reports here so the
+// failure is never silent (#198 item 5). Phase 1 logs it FATAL to the per-agent
+// log; Phase 2 (#196) will additionally route it through the runner outbox to the
+// server.
+export interface HookFatalReport {
+  hook: string;
+  error: string;
+}
+
+interface MonitorSocketData {
+  kind: "monitor";
+}
+
+type MonitorSocket = ServerWebSocket<MonitorSocketData>;
+
+interface PendingPermissionWaiter {
+  resolve(input: ProviderPermissionDecisionInput): void;
+  timer: Timer;
+  suppressResolvedStatus?: boolean;
+}
+
+export interface ControlServer {
+  port: number;
+  url: string;
+  stop(): void;
+  deliverToMonitor(messages: Message[], timeoutMs?: number): Promise<number[]>;
+  resolvePermissionDecision(input: ProviderPermissionDecisionInput): boolean;
+}
+
+export interface ControlServerOptions {
+  onStatus(status: ProviderStatusEvent): void;
+  onTerminalAttachSpec?(): Promise<TerminalAttachSpec>;
+  // #2459 — the CURRENT non-secret attachment generation (derived from the provider's active
+  // session id), read by the orchestrator before terminal input and on stream connect/
+  // reconnect to detect a session rotation. Returns ONLY that digest — never the session id,
+  // never a credential. Absent handler ⇒ endpoint reports no generation, which the
+  // orchestrator treats as "nothing to check" (e.g. codex's attach target cannot go stale).
+  onTerminalGeneration?(): Promise<string | undefined>;
+  onReplyObligations?(): Promise<ReplyObligation[]>;
+  // Provider session-final and pre-flush hooks hand over their capture source so
+  // the runner can surface the provider turn in dashboard chat without the agent
+  // re-emitting it via /reply.
+  onSubagentUsage?(transcriptPath: string): Promise<void>;
+  onSessionTurn?(input: { transcriptPath: string; lastAssistantMessage?: unknown; promptId?: string; isPreFlush?: boolean; toolName?: string; toolInput?: unknown }): Promise<{ reasoningSettled: boolean; toolUseOccurredAt?: number } | void>;
+  onPermissionResolved?(input: ResolvedPermissionPrompt): Promise<void> | void;
+  onPermissionAbandoned?(input: PendingPermissionRequestRecord & { reason: string; decisionReason: ProviderPermissionDecisionReason }): Promise<void> | void;
+  // #1173: a "questions" (AskUserQuestion) prompt's own PreToolUse hook is answered with `deny`
+  // immediately (see the branch in handlePermissionRequest) instead of held open for the human,
+  // so there is no live HTTP request left to answer once the decision resolves. This delivers
+  // that resolution instead: `text` is the prompt to inject as a normal user turn.
+  onAnswerInject?(input: { approvalId: string; text?: string }): Promise<void> | void;
+  // #2914: files a "questions" prompt as a durable operator decision; undefined = not filed, so the
+  // question falls back to the live deny-then-hold relay above.
+  onQuestionDecision?(input: OperatorDecisionFileInput): Promise<{ id: number } | undefined>;
+  // A provider user-prompt hook hands over text typed directly into the session
+  // so the runner can mirror it into dashboard chat.
+  onUserPrompt?(input: { prompt: string; transcriptPath?: string; promptId?: string; firedAt?: number }): Promise<void>;
+  // A session-start hook checks in with its fire time, proving the hook path works (#2664).
+  onHookCheck?(input: { firedAt?: number }): void;
+  onToolUse?(input: unknown, signal: AbortSignal): Promise<string>;
+  // A provider session-boundary hook signals an imminent context reset or
+  // termination so the runner can run end-of-session work before the operation.
+  onSessionBoundary?(input: { reason?: string; transcriptPath?: string }): Promise<void>;
+  onSessionStart?(input: { source?: string; providerSessionId?: string }): string | undefined;
+  // Phase 1 observability (#198): a hook reporting an unhandled failure. The
+  // control server already logs it FATAL; this is the seam for Phase 2 to also
+  // surface it to the server via the runner outbox.
+  onHookFatal?(report: HookFatalReport): void;
+  // Test seam: how long the permission hook waits for a dashboard decision before
+  // resolving as a timeout. Defaults to PERMISSION_WAIT_MS (margin under the 900s hook).
+  permissionWaitMs?: number;
+  instanceId?: string;
+  pendingPermissionStore?: PendingPermissionStore;
+  permissionPromptHandler?: ProviderPermissionPromptHandler;
+  // #3648 — the agent runs in `open` approval mode: answer every permission request at once.
+  autoApprovePermissions?: boolean;
+  providerSourceId?: string;
+  stopObligationPath?: string;
+  monitorUnavailableMessage?: string;
+}
+
+export type ResolvedPermissionPrompt = ProviderPermissionResolvedPrompt;
+
+export function startControlServer(options: ControlServerOptions): ControlServer {
+  const monitors = new Set<MonitorSocket>();
+  const pendingDeliveries = new Map<string, { resolve(ids: number[]): void; timer: Timer }>();
+  const pendingPermissionRequests = new Map<string, PendingPermissionWaiter>();
+  // approvalIds already resolved, so a duplicate decision (e.g. bus replay after a
+  // reconnect, or both the PreToolUse and PermissionRequest hooks racing one answer)
+  // is an idempotent no-op instead of a silent command failure (#693).
+  const resolvedApprovals = new Set<string>();
+  let server!: Server<MonitorSocketData>;
+  recoverStoredPermissionRequests(options);
+
+  server = Bun.serve<MonitorSocketData>({
+    hostname: "127.0.0.1",
+    port: 0,
+    fetch(req, srv) {
+      const url = new URL(req.url);
+      if (url.pathname === "/health") return Response.json({ ok: true });
+      if (url.pathname === "/status" && req.method === "GET") return Response.json({ ok: true });
+      if (url.pathname === "/status" && req.method === "POST") {
+        return handleStatus(req, options);
+      }
+      if (url.pathname === "/terminal/attach-spec" && req.method === "GET") {
+        if (!options.onTerminalAttachSpec) return Response.json({ error: "terminal attach is unavailable" }, { status: 404 });
+        return options.onTerminalAttachSpec()
+          // #2459 round 1 (review A) — strip any credential-bearing env record from the body:
+          // credentials travel ONLY by runner-staged env file (spec.envFile), so nothing
+          // secret ever serializes into this HTTP response.
+          .then((spec) => {
+            const safeSpec: Record<string, unknown> = { ...spec };
+            delete safeSpec.env;
+            return Response.json(safeSpec);
+          })
+          .catch((error) => Response.json({ error: errMessage(error) }, { status: 409 }));
+      }
+      if (url.pathname === "/terminal/generation" && req.method === "GET") {
+        // #2459 — deliberately returns ONLY the non-secret generation digest. No session id,
+        // no password, no attach detail: this endpoint exists so the orchestrator can detect
+        // staleness, and anything more would widen the secret surface for no benefit.
+        if (!options.onTerminalGeneration) return Response.json({});
+        return options.onTerminalGeneration()
+          .then((generation) => Response.json(generation ? { generation } : {}))
+          .catch((error) => Response.json({ error: errMessage(error) }, { status: 503 }));
+      }
+      if (url.pathname === "/reply-obligations" && req.method === "GET") {
+        if (!options.onReplyObligations) return Response.json({ pending: false, count: 0 });
+        return options.onReplyObligations()
+          .then((obligations) => Response.json(replyObligationSummary(obligations)))
+          .catch((error) => Response.json({ error: errMessage(error) }, { status: 503 }));
+      }
+      if (url.pathname === `/reply-obligations/${options.stopObligationPath ?? "turn-stop"}` && req.method === "GET") {
+        if (!options.onReplyObligations) return Response.json({});
+        return options.onReplyObligations()
+          .then((obligations) => Response.json(replyObligationStopDecision(obligations)))
+          .catch((error) => Response.json({ error: errMessage(error) }, { status: 503 }));
+      }
+      if (url.pathname === "/permissions/request" && req.method === "POST") {
+        return handlePermissionRequest(req, options, pendingPermissionRequests);
+      }
+      if (url.pathname === "/subagent-usage" && req.method === "POST") {
+        return req.json().then(async (body) => {
+          if (!isRecord(body) || typeof body.agent_transcript_path !== "string" || !body.agent_transcript_path) return Response.json({ error: "agent_transcript_path required" }, { status: 400 });
+          if (!options.onSubagentUsage) return Response.json({ error: "sub-agent usage capture unavailable" }, { status: 503 });
+          await options.onSubagentUsage(body.agent_transcript_path);
+          return Response.json({ ok: true });
+        }).catch((error) => Response.json({ error: errMessage(error) }, { status: 500 }));
+      }
+      if (url.pathname === "/session-turn" && req.method === "POST") {
+        return handleSessionTurn(req, options);
+      }
+      if (url.pathname === "/user-prompt" && req.method === "POST") {
+        return handleUserPrompt(req, url, options);
+      }
+      if (url.pathname === "/hook-check" && req.method === "POST") {
+        options.onHookCheck?.(hookFiredAt(url));
+        return Response.json({ ok: Boolean(options.onHookCheck) });
+      }
+      if (url.pathname === "/tool-use" && req.method === "POST") {
+        return req.json().then(async (body) => {
+          const content = await options.onToolUse?.(body, AbortSignal.any([req.signal, AbortSignal.timeout(3_500)]));
+          return new Response(content ?? "");
+        }).catch((error) => Response.json({ error: errMessage(error) }, { status: 400 }));
+      }
+      if (url.pathname === "/session-boundary" && req.method === "POST") {
+        return handleSessionBoundary(req, options);
+      }
+      if (url.pathname === "/session-start" && req.method === "POST") {
+        return req.json().then((body) => new Response(isRecord(body) ? options.onSessionStart?.({
+          source: typeof body.source === "string" ? body.source : undefined,
+          providerSessionId: typeof body.providerSessionId === "string" ? body.providerSessionId : undefined,
+        }) ?? "" : "")).catch(() => new Response("", { status: 400 }));
+      }
+      if (url.pathname === "/log-level" && req.method === "GET") {
+        return Response.json({ level: logger.getLevel(), levels: LOG_LEVELS });
+      }
+      if (url.pathname === "/log-level" && req.method === "POST") {
+        return handleLogLevel(req);
+      }
+      if (url.pathname === "/hook-fatal" && req.method === "POST") {
+        return handleHookFatal(req, options);
+      }
+      if (url.pathname === "/rate-limit" && req.method === "POST") {
+        return handleRateLimit(req, options);
+      }
+      if (url.pathname === "/monitor") {
+        const upgraded = srv.upgrade(req, { data: { kind: "monitor" } });
+        return upgraded ? undefined : new Response("WebSocket upgrade failed", { status: 400 });
+      }
+      return new Response("not found", { status: 404 });
+    },
+    websocket: {
+      open(ws) {
+        monitors.add(ws);
+      },
+      message(_ws, raw) {
+        const parsed = parseJson(raw);
+        if (parsed?.type === "message.delivered" && Array.isArray(parsed.messageIds)) {
+          const deliveryId = typeof parsed.deliveryId === "string" ? parsed.deliveryId : "";
+          const pending = pendingDeliveries.get(deliveryId);
+          if (pending) {
+            clearTimeout(pending.timer);
+            pendingDeliveries.delete(deliveryId);
+            pending.resolve(parsed.messageIds.filter((id): id is number => Number.isSafeInteger(id)));
+          }
+        }
+      },
+      close(ws) {
+        monitors.delete(ws);
+      },
+    },
+  });
+
+  const port = server.port;
+  if (port === undefined) throw new Error("runner control server did not bind a port");
+
+  return {
+    port,
+    url: `http://127.0.0.1:${port}`,
+    stop() {
+      for (const pending of pendingDeliveries.values()) clearTimeout(pending.timer);
+      pendingDeliveries.clear();
+      for (const pending of pendingPermissionRequests.values()) clearTimeout(pending.timer);
+      for (const [approvalId, pending] of pendingPermissionRequests.entries()) {
+        const decision = withDecisionReason({
+          approvalId,
+          decision: "deny",
+          reason: PERMISSION_STOP_REASON,
+        }, "dismissed");
+        bestEffortDeletePendingPermissionRequest(options.pendingPermissionStore, approvalId);
+        pending.resolve(decision);
+      }
+      pendingPermissionRequests.clear();
+      server.stop(true);
+    },
+    async deliverToMonitor(messages: Message[], timeoutMs = 30_000): Promise<number[]> {
+        if (monitors.size === 0) throw new Error(options.monitorUnavailableMessage ?? "no provider monitor connected");
+      const deliveryId = crypto.randomUUID();
+      const payload = JSON.stringify({ type: "message.deliver", deliveryId, messages });
+      return await new Promise<number[]>((resolve, reject) => {
+        const timer = setTimeout(() => {
+          pendingDeliveries.delete(deliveryId);
+          reject(new Error("monitor delivery timed out"));
+        }, timeoutMs);
+        pendingDeliveries.set(deliveryId, { resolve, timer });
+        for (const monitor of monitors) monitor.send(payload);
+      });
+    },
+    resolvePermissionDecision(input: ProviderPermissionDecisionInput): boolean {
+      const pending = pendingPermissionRequests.get(input.approvalId);
+      if (!pending) {
+        // Idempotent: a duplicate decision for an already-resolved approval is a
+        // benign no-op, not a dropped answer that fails the command (#693).
+        return resolvedApprovals.has(input.approvalId);
+      }
+      clearTimeout(pending.timer);
+      pendingPermissionRequests.delete(input.approvalId);
+      bestEffortDeletePendingPermissionRequest(options.pendingPermissionStore, input.approvalId);
+      recordResolvedApproval(resolvedApprovals, input.approvalId);
+      pending.resolve(withDecisionReason(input, input.decision === "deny" || input.decision === "abort" ? "dismissed" : "answered"));
+      if (!pending.suppressResolvedStatus) {
+        options.onStatus({
+          status: "busy",
+          reason: "provider-turn",
+          providerState: {
+            state: "active",
+            reason: "permissionResolved",
+            label: options.permissionPromptHandler?.resolvedLabel ?? "Provider approval resolved",
+            source: options.permissionPromptHandler?.source ?? options.providerSourceId ?? "provider",
+            updatedAt: Date.now(),
+          },
+        });
+      }
+      return true;
+    },
+  };
+}
+
+export function recordResolvedApproval(resolvedApprovals: Set<string>, approvalId: string, limit = RESOLVED_APPROVAL_LIMIT): void {
+  resolvedApprovals.delete(approvalId);
+  resolvedApprovals.add(approvalId);
+  while (resolvedApprovals.size > limit) {
+    const oldest = resolvedApprovals.values().next().value;
+    if (!oldest) break;
+    resolvedApprovals.delete(oldest);
+  }
+}
+
+function replyObligationSummary(obligations: ReplyObligation[]): Record<string, unknown> {
+  const obligation = obligations[0];
+  if (!obligation) return { pending: false, count: 0 };
+  const subject = obligation.subject ? ` (${obligation.subject})` : "";
+  return {
+    pending: true,
+    count: obligations.length,
+    obligation,
+    reason: `Pending Agent Relay response for message #${obligation.messageId} from ${obligation.from}${subject}. If it still needs an answer, use ${relayReplyActionText(obligation.messageId)}. If multiple delivered messages are part of one current request, answer once to the latest relevant message. If the useful response was already delivered through Relay, do not send a separate status-only confirmation.`,
+  };
+}
+
+function replyObligationStopDecision(obligations: ReplyObligation[]): Record<string, unknown> {
+  // #418 — never block the turn to nag for a `user` obligation: the operator already receives
+  // this turn via the session mirror, so an explicit /reply double-delivers. Only agent↔agent /
+  // spawner obligations (#413) gate the turn here; the mirror's replyTo-linking clears the user
+  // one. (The /reply-obligations summary endpoint is left unfiltered — the obligation IS pending
+  // until the mirror lands, so observability stays accurate.)
+  const enforceable = obligations.filter(obligationNagEligible);
+  const summary = replyObligationSummary(enforceable);
+  if (summary.pending !== true) return {};
+  return {
+    decision: "block",
+    reason: summary.reason,
+  };
+}
+
+function recoverStoredPermissionRequests(options: ControlServerOptions): void {
+  const store = options.pendingPermissionStore;
+  if (!store) return;
+  for (const record of bestEffortListPendingPermissionRequests(store)) {
+    bestEffortDeletePendingPermissionRequest(store, record.approvalId);
+    const decisionReason: ProviderPermissionDecisionReason = {
+      kind: Date.now() >= record.hookDeadlineAt ? "timeout" : "dismissed",
+      message: Date.now() >= record.hookDeadlineAt ? permissionTimeoutReason(record.view) : PERMISSION_STOP_REASON,
+    };
+    options.onPermissionAbandoned?.({ ...record, reason: decisionReason.message ?? PERMISSION_STOP_REASON, decisionReason });
+  }
+}
+
+function withDecisionReason(input: ProviderPermissionDecisionInput, kind: ProviderPermissionDecisionReason["kind"]): ProviderPermissionDecisionInput {
+  if (input.decisionReason) return input;
+  return {
+    ...input,
+    decisionReason: {
+      kind,
+      ...(input.reason ? { message: input.reason } : {}),
+    },
+  };
+}
+
+async function handlePermissionRequest(
+  req: Request,
+  options: ControlServerOptions,
+  pendingPermissionRequests: Map<string, PendingPermissionWaiter>,
+): Promise<Response> {
+  const handler = options.permissionPromptHandler;
+  if (!handler) return Response.json({ ok: false, reason: "permission prompt handler unavailable" }, { status: 501 });
+  const occurredAt = Date.now();
+  const body = await req.json().catch(() => null);
+  if (!isRecord(body)) return Response.json({});
+  const transcriptPath = typeof body.transcript_path === "string" ? body.transcript_path : "";
+  const toolName = typeof body.tool_name === "string" ? body.tool_name : undefined;
+  const toolInput = isRecord(body.tool_input) ? body.tool_input : undefined;
+  const approvalId = crypto.randomUUID();
+
+  const probeView = handler.buildView(approvalId, body, true);
+  const answerInput = permissionRequestQuestionAnswers(body);
+  if (body.hook_event_name === "PermissionRequest" && probeView.kind === "questions" && answerInput) {
+    return Response.json(handler.buildHookResponse({ approvalId, decision: "answer", answers: answerInput }, body));
+  }
+  if (
+    (body.hook_event_name === "PreToolUse" || body.hook_event_name === "PermissionRequest")
+    && probeView.kind === "questions"
+    && handler.buildAnswerInjectionPrompt
+  ) {
+    const decision = await fileQuestionDecision(options, handler, body);
+    if (decision) return Response.json(handler.buildHookResponse({ approvalId, decision: "deny", reason: decisionFiledReason(decision.id) }, body));
+    return holdQuestionWithImmediateDeny({
+      options, pendingPermissionRequests, handler, approvalId, body, occurredAt, transcriptPath, toolName, toolInput,
+    });
+  }
+
+  if (options.autoApprovePermissions) return autoApprovePermission(options, handler, approvalId, body, occurredAt);
+  const { decision } = await awaitPermissionDecision({
+    options, pendingPermissionRequests, handler, approvalId, body, occurredAt, transcriptPath, toolName, toolInput,
+    blockedReason: "permissionRequest",
+    recommendedAction: "Review the permission request in Agent Relay.",
+  });
+  return Response.json(handler.buildHookResponse(decision, body));
+}
+
+// #3648 — bypass mode does not cover every provider check (Claude's dangerous-rm check on a variable
+// target still asks). An open agent has no one to answer, so the request is approved, not held.
+const OPEN_MODE_APPROVAL_REASON = "Auto-approved: the agent runs in open approval mode.";
+
+async function autoApprovePermission(
+  options: ControlServerOptions,
+  handler: ProviderPermissionPromptHandler,
+  approvalId: string,
+  body: Record<string, unknown>,
+  occurredAt: number,
+): Promise<Response> {
+  const decision: ProviderPermissionDecisionInput = { approvalId, decision: "approve", decisionReason: { kind: "answered", message: OPEN_MODE_APPROVAL_REASON } };
+  if (options.onPermissionResolved) {
+    await Promise.resolve(options.onPermissionResolved(handler.buildResolvedPrompt(handler.buildView(approvalId, body, true), decision, body, occurredAt))).catch(() => {});
+  }
+  return Response.json(handler.buildHookResponse(decision, body));
+}
+
+function permissionRequestQuestionAnswers(body: Record<string, unknown>): Record<string, string> | undefined {
+  if (body.hook_event_name !== "PermissionRequest") return undefined;
+  const toolInput = isRecord(body.tool_input) ? body.tool_input : {};
+  if (!isRecord(toolInput.answers)) return undefined;
+  const answers = Object.fromEntries(
+    Object.entries(toolInput.answers)
+      .filter((entry): entry is [string, string] => typeof entry[1] === "string" && entry[1].trim().length > 0),
+  );
+  return Object.keys(answers).length ? answers : undefined;
+}
+
+async function fileQuestionDecision(options: ControlServerOptions, handler: ProviderPermissionPromptHandler, body: Record<string, unknown>): Promise<{ id: number } | undefined> {
+  const request = options.onQuestionDecision ? handler.buildDecisionRequest?.(body) : undefined;
+  return request ? options.onQuestionDecision!(request).catch(() => undefined) : undefined;
+}
+
+function decisionFiledReason(id: number): string {
+  return `Agent Relay filed this as operator decision #${id}; the user answers it on the Relay dashboard. Do not re-ask it and do not wait in this turn: the answer arrives later as a Relay message ("Decision #${id} answered"). Continue other work or end your turn. Next time, file questions with relay_decision directly.`;
+}
+
+function holdQuestionWithImmediateDeny(params: {
+  options: ControlServerOptions;
+  pendingPermissionRequests: Map<string, PendingPermissionWaiter>;
+  handler: ProviderPermissionPromptHandler;
+  approvalId: string;
+  body: Record<string, unknown>;
+  occurredAt: number;
+  transcriptPath: string;
+  toolName?: string;
+  toolInput?: unknown;
+}): Response {
+  const { options, pendingPermissionRequests, handler, approvalId, body, occurredAt, transcriptPath, toolName, toolInput } = params;
+  // #1173/#1194: a fresh "questions" prompt (the provider's AskUserQuestion) must not block
+  // its own hook response while the human answers. Denying immediately makes the provider persist
+  // the buffered turn plus rejected tool_result, so the pre-flush harvest below can publish the
+  // reasoning before the dashboard form appears. The eventual answer is injected as a normal prompt.
+  const response = Response.json(handler.buildHookResponse(
+    { approvalId, decision: "deny", reason: PENDING_QUESTION_DENY_REASON },
+    body,
+  ));
+  void awaitPermissionDecision({
+    options, pendingPermissionRequests, handler, approvalId, body, occurredAt, transcriptPath, toolName, toolInput,
+    blockedReason: "pendingQuestion",
+    recommendedAction: "Answer the question in Agent Relay.",
+    suppressResolvedStatus: true,
+  }).then(({ decision, view }) => decision.decision === "answer" && decision.answers && Object.keys(decision.answers).length
+    ? handler.buildAnswerInjectionPrompt!(view, decision, body)
+    : pendingQuestionDismissalPrompt(decision),
+  ).then(
+    (text) => options.onAnswerInject?.({ approvalId, text }),
+    // #2913: the inject is the only release of the runner's question hold — never skip it.
+    () => options.onAnswerInject?.({ approvalId }),
+  ).catch(() => {});
+  return response;
+}
+
+// Shared by both the traditional block-until-decided flow and the #1173 deny-then-hold flow:
+// pre-flushes narrative (#435/#1116), builds+persists the interactive view, publishes the
+// blocked providerState, and resolves once a decision arrives (or the wait times out). Callers
+// differ only in what happens with the resolved decision (build a hook response vs. inject an
+// answer prompt), so that part is left to them.
+async function awaitPermissionDecision(params: {
+  options: ControlServerOptions;
+  pendingPermissionRequests: Map<string, PendingPermissionWaiter>;
+  handler: ProviderPermissionPromptHandler;
+  approvalId: string;
+  body: Record<string, unknown>;
+  occurredAt: number;
+  transcriptPath: string;
+  toolName?: string;
+  toolInput?: unknown;
+  blockedReason: string;
+  recommendedAction: string;
+  suppressResolvedStatus?: boolean;
+}): Promise<{ decision: ProviderPermissionDecisionInput; view: InteractivePrompt }> {
+  const { options, pendingPermissionRequests, handler, approvalId, body, occurredAt, transcriptPath, toolName, toolInput, blockedReason, recommendedAction, suppressResolvedStatus } = params;
+  // #435/#1116: flush any un-mirrored narrative before the blocking form appears in
+  // chat. The adapter owns how to settle its capture source; the transport only
+  // forwards the hook payload as an anchor and stamps the returned truth value.
+  let reasoningSettled = true;
+  // Defaults to the hook's own wall-clock receipt time; refined below to the transcript's own
+  // tool_use event time when the pre-flush harvest can resolve it (#1185) — the hook can receive
+  // before the provider flushes/timestamps that entry, which otherwise sorts the resolved prompt
+  // ahead of the narration that led up to it.
+  let resolvedOccurredAt = occurredAt;
+  if (transcriptPath && options.onSessionTurn) {
+    const result = await options.onSessionTurn({ transcriptPath, isPreFlush: true, toolName, toolInput }).catch(() => undefined);
+    if (result && typeof result.reasoningSettled === "boolean") reasoningSettled = result.reasoningSettled;
+    if (result && typeof result.toolUseOccurredAt === "number") resolvedOccurredAt = result.toolUseOccurredAt;
+  }
+  const view = handler.buildView(approvalId, body, reasoningSettled);
+  const waitMs = options.permissionWaitMs ?? PERMISSION_WAIT_MS;
+  const hookDeadlineAt = occurredAt + waitMs;
+  bestEffortUpsertPendingPermissionRequest(options.pendingPermissionStore, {
+    approvalId,
+    view,
+    hookDeadlineAt,
+    ownerInstanceId: options.instanceId ?? "",
+    occurredAt,
+  });
+  options.onStatus({
+    status: "busy",
+    reason: "provider-turn",
+    providerState: {
+      state: "blocked",
+      reason: blockedReason,
+      label: view.title,
+      recommendedAction,
+      source: handler.source,
+      pendingApproval: view,
+      updatedAt: Date.now(),
+    },
+  });
+
+  const decision = await new Promise<ProviderPermissionDecisionInput>((resolve) => {
+    const timer = setTimeout(() => {
+      pendingPermissionRequests.delete(approvalId);
+      bestEffortDeletePendingPermissionRequest(options.pendingPermissionStore, approvalId);
+      if (!suppressResolvedStatus) {
+        options.onStatus({
+          status: "busy",
+          reason: "provider-turn",
+          providerState: {
+            state: "active",
+            reason: "permissionTimedOut",
+            label: "Provider approval timed out",
+            source: handler.source,
+            updatedAt: Date.now(),
+          },
+        });
+      }
+      resolve(withDecisionReason({ approvalId, decision: "deny", reason: permissionTimeoutReason(view) }, "timeout"));
+    }, waitMs);
+    pendingPermissionRequests.set(approvalId, { resolve, timer, suppressResolvedStatus });
+  });
+
+  // A timeout is a resolved prompt too. Persist it with its distinct timeout reason so
+  // the coordinator can see how the hold ended instead of only watching it disappear.
+  if (options.onPermissionResolved) {
+    await Promise.resolve(options.onPermissionResolved(handler.buildResolvedPrompt(view, decision, body, resolvedOccurredAt))).catch(() => {});
+  }
+
+  return { decision, view };
+}
+
+function pendingQuestionDismissalPrompt(decision: ProviderPermissionDecisionInput): string {
+  const timedOut = decision.decisionReason?.kind === "timeout";
+  const event = timedOut ? "timed out before the user answered" : "was dismissed before the user answered";
+  return `Your previous AskUserQuestion ${event}. Proceed on your best judgment, or re-ask if you still need a decision.`;
+}
+
+async function handleSessionTurn(req: Request, options: ControlServerOptions): Promise<Response> {
+  if (!options.onSessionTurn) return Response.json({ ok: false, reason: "session capture unavailable" });
+  const body = await req.json().catch(() => null);
+  const transcriptPath = isRecord(body) && typeof body.transcriptPath === "string" ? body.transcriptPath : "";
+  const lastAssistantMessage = isRecord(body) ? body.lastAssistantMessage : undefined;
+  const promptId = isRecord(body) && typeof body.promptId === "string" && body.promptId ? body.promptId : undefined;
+  // transcriptPath is the primary source, but it can be absent (known CC Stop edge case).
+  // lastAssistantMessage is the fallback for exactly that — accept either; reject only when
+  // both are missing so the turn still reaches the dashboard chat (#264).
+  if (!transcriptPath && lastAssistantMessage === undefined) {
+    return Response.json({ ok: false, reason: "transcriptPath or lastAssistantMessage required" }, { status: 400 });
+  }
+  // Awaited on purpose: the Stop hook posts this synchronously before its
+  // reply-obligation check, so the captured turn must be persisted (and the
+  // obligation cleared) before this response returns.
+  try {
+    await options.onSessionTurn({ transcriptPath, lastAssistantMessage, ...(promptId ? { promptId } : {}) });
+    return Response.json({ ok: true });
+  } catch (error) {
+    return Response.json({ ok: false, reason: errMessage(error) }, { status: 500 });
+  }
+}
+
+/** `?firedAt=<epoch ms>`: when the hook ran, stamped by the hook itself (#2664). */
+function hookFiredAt(url: URL): { firedAt?: number } {
+  const raw = url.searchParams.get("firedAt") ?? "";
+  const firedAt = /^\d{1,16}$/.test(raw) ? Number(raw) : NaN;
+  return Number.isSafeInteger(firedAt) ? { firedAt } : {};
+}
+
+// The body is the provider hook's raw stdin payload (Claude: prompt/transcript_path/prompt_id).
+async function handleUserPrompt(req: Request, url: URL, options: ControlServerOptions): Promise<Response> {
+  if (!options.onUserPrompt) return Response.json({ ok: false, reason: "prompt echo unavailable" });
+  const body = await req.json().catch(() => null);
+  const field = (key: string) => isRecord(body) && typeof body[key] === "string" && body[key] ? body[key] as string : undefined;
+  const prompt = field("prompt") ?? "";
+  if (!prompt.trim()) return Response.json({ ok: false, reason: "prompt required" }, { status: 400 });
+  const transcriptPath = field("transcript_path");
+  const promptId = field("prompt_id");
+  const { firedAt } = hookFiredAt(url);
+  // Fire-and-forget: the hook must not block the provider's turn on relay round-trips.
+  void Promise.resolve(options.onUserPrompt({ prompt, transcriptPath, ...(promptId ? { promptId } : {}), ...(firedAt !== undefined ? { firedAt } : {}) })).catch(() => {});
+  return Response.json({ ok: true });
+}
+
+async function handleSessionBoundary(req: Request, options: ControlServerOptions): Promise<Response> {
+  if (!options.onSessionBoundary) return Response.json({ ok: false, reason: "session-boundary capture unavailable" });
+  const body = await req.json().catch(() => null);
+  const reason = isRecord(body) && typeof body.reason === "string" ? body.reason : undefined;
+  const transcriptPath = isRecord(body) && typeof body.transcriptPath === "string" ? body.transcriptPath : undefined;
+  // Fire-and-forget: a lifecycle hook must not block the provider compacting or exiting.
+  void Promise.resolve(options.onSessionBoundary({ reason, transcriptPath })).catch(() => {});
+  return Response.json({ ok: true });
+}
+
+async function handleLogLevel(req: Request): Promise<Response> {
+  const body = await req.json().catch(() => null);
+  const level = parseLogLevel(isRecord(body) && typeof body.level === "string" ? body.level : undefined);
+  if (!level) return Response.json({ error: `level must be one of: ${LOG_LEVELS.join(", ")}` }, { status: 400 });
+  const previous = logger.getLevel();
+  logger.setLevel(level);
+  logger.info("logger", `log level set to ${level} (was ${previous}) via control port`);
+  return Response.json({ ok: true, level, previous });
+}
+
+async function handleHookFatal(req: Request, options: ControlServerOptions): Promise<Response> {
+  const body = await req.json().catch(() => null);
+  const hook = isRecord(body) && typeof body.hook === "string" && body.hook.trim() ? body.hook.trim() : "unknown";
+  const error = isRecord(body) && typeof body.error === "string" ? body.error : "(no detail)";
+  // Never silent: a hook that couldn't handle its own failure lands here as FATAL.
+  logger.fatal(`hook:${hook}`, error);
+  try { options.onHookFatal?.({ hook, error }); } catch { /* reporting must never throw back at the hook */ }
+  return Response.json({ ok: true });
+}
+
+async function handleStatus(req: Request, options: ControlServerOptions): Promise<Response> {
+  const body = await req.json().catch(() => null) as Partial<ProviderStatusEvent> | null;
+  const status = body?.status;
+  if (status !== "idle" && status !== "busy" && status !== "offline" && status !== "error") {
+    return Response.json({ error: "status must be idle, busy, offline, or error" }, { status: 400 });
+  }
+  const reason = body?.reason;
+  if (reason !== undefined && reason !== "provider-turn" && reason !== "subagent" && reason !== "background-script") {
+    return Response.json({ error: "reason must be provider-turn, subagent, or background-script" }, { status: 400 });
+  }
+  const isWorkKind = (item: unknown): item is "provider-turn" | "subagent" | "background-script" =>
+    item === "provider-turn" || item === "subagent" || item === "background-script";
+  const clear = Array.isArray(body?.clear) ? body.clear.filter(isWorkKind) : undefined;
+  // #650: optional pids the runner reconciles against process liveness so a tracked
+  // command/shell killed out-of-band clears the busy marker instead of stranding it.
+  const pidsInput = Array.isArray((body as { pids?: unknown })?.pids) ? (body as { pids: unknown[] }).pids : undefined;
+  const pids = pidsInput
+    ?.map((p) => (typeof p === "number" ? p : Number(p)))
+    .filter((p) => Number.isInteger(p) && p > 0);
+  const timeline = statusTimelineEvent(body);
+  const update: ProviderStatusEvent = {
+    status,
+    ...(typeof body?.providerSessionId === "string" ? { providerSessionId: body.providerSessionId } : {}),
+    ...(reason ? { reason } : {}),
+    ...(clear?.length ? { clear: [...new Set(clear)] } : {}),
+    ...(timeline ? { timeline } : {}),
+    ...(typeof body?.id === "string" ? { id: body.id } : {}),
+    ...(pids?.length ? { pids } : {}),
+    ...(typeof body?.label === "string" ? { label: body.label } : {}),
+    ...(typeof body?.role === "string" ? { role: body.role } : {}),
+    ...(typeof body?.parentId === "string" ? { parentId: body.parentId } : {}),
+    ...(body?.metadata && typeof body.metadata === "object" && !Array.isArray(body.metadata) ? { metadata: body.metadata as Record<string, unknown> } : {}),
+  };
+  options.onStatus(update);
+  return Response.json({ ok: true, ...update });
+}
+
+// #286: the stop-failure hook reports a usage/rate-limit stall here. Build the
+// provider-neutral `blocked` state (reason rate_limit) so the dashboard shows it
+// distinctly (never as idle/available) and the relay's resume sweep can lift it
+// at reset. The agent stays `idle` underneath (the turn truly ended); the runner
+// carries the hold via its rateLimitHold field, not an active-work claim.
+async function handleRateLimit(req: Request, options: ControlServerOptions): Promise<Response> {
+  const body = await req.json().catch(() => null);
+  if (!isRecord(body)) return Response.json({ error: "invalid body" }, { status: 400 });
+  const errorType = typeof body.errorType === "string" && body.errorType ? body.errorType : "rate_limit";
+  // CC's reset_time is unix SECONDS ("if available"); normalize to ms for the relay.
+  const resetTimeSec = typeof body.resetTime === "number" && Number.isFinite(body.resetTime) ? body.resetTime : undefined;
+  const resetAt = resetTimeSec !== undefined ? Math.round(resetTimeSec * 1000) : undefined;
+  const errorMessage = typeof body.errorMessage === "string" && body.errorMessage ? body.errorMessage : undefined;
+  options.onStatus({
+    status: "idle",
+    clear: ["subagent"],
+    providerState: buildRateLimitProviderState({ errorType, resetAt, message: errorMessage, source: options.providerSourceId ?? "provider" }),
+  });
+  return Response.json({ ok: true, errorType, ...(resetAt ? { resetAt } : {}) });
+}
+
+function statusTimelineEvent(body: Partial<ProviderStatusEvent> | null): ProviderStatusEvent["timeline"] | undefined {
+  const timeline = body?.timeline;
+  if (!timeline || typeof timeline !== "object" || Array.isArray(timeline)) return undefined;
+  if (typeof timeline.status !== "string" || !timeline.status) return undefined;
+  return {
+    status: timeline.status,
+    ...(typeof timeline.id === "string" ? { id: timeline.id } : {}),
+    ...(typeof timeline.timestamp === "number" && Number.isFinite(timeline.timestamp) ? { timestamp: timeline.timestamp } : {}),
+  };
+}
+
+function parseJson(raw: string | Buffer): Record<string, unknown> | null {
+  try {
+    const parsed = JSON.parse(typeof raw === "string" ? raw : raw.toString());
+    return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed as Record<string, unknown> : null;
+  } catch {
+    return null;
+  }
+}