@humwick/runner 0.0.0-stage → 0.130.0

package/src/adapters/codex-token-usage-capture.ts

@@ -0,0 +1,163 @@
+// #2483 — per-turn Codex token-usage capture: the runner-side half of the token-usage spine's
+// Codex capture seam (sibling to claude-token-usage-capture.ts's #2482 seam, proven live via
+// #2482/#2491). Unlike Claude's transcript-tail (no flush hook exists for Codex, so the runner
+// has to read whatever bytes have been appended), Codex's app-server pushes usage LIVE on its
+// JSON-RPC event stream (`thread/tokenUsage/updated`, already parsed once by handleCodexEvent
+// for context-state tracking via codexTokenUsageContext) — no flush race, so this reads directly
+// off the same already-parsed notification params instead of tailing a file.
+//
+// Confirmed against a live rollout (`~/.codex/sessions/**/*.jsonl`, `token_count` events; the
+// app-server notification mirrors the same shape, camelCased): each event carries `total`
+// (cumulative-since-thread-start) and `last` (the delta consumed by the exchange that just
+// completed) token usage, each with input/cached-input/cache-write-input/output/reasoning-output
+// breakdowns. `last` is what we report — it matches the spine's per-turn semantics.
+//
+// #2483 review round 1, F1 — Codex's buckets are NOT disjoint (cachedInputTokens ⊂ inputTokens,
+// reasoningOutputTokens ⊂ outputTokens; a live fixture confirms input+output alone equals
+// totalTokens). The spine's convention (locked by src/db/token-usage.test.ts, set by Claude's
+// #2482) is DISJOINT buckets — its aggregate sums input+output+cacheRead+cacheWrite+reasoning as
+// the total. Posting Codex's raw inputTokens/outputTokens double-counts the cache/reasoning
+// sub-portions (~2x on the live fixture) — so cachedInputTokens/cacheWriteInputTokens are
+// subtracted out of inputTokens, and reasoningOutputTokens out of outputTokens, before posting.
+//
+// #2483 review round 1, F4 — the dedupe key needs `total.totalTokens` specifically (the
+// cumulative counter, which only ever grows within a thread — a compaction just mints a fresh
+// baseline, changing the key). Falling back to `last.totalTokens` when `total` doesn't carry a
+// usable totalTokens degrades to a key that can collide across two different exchanges (both
+// deltas the same size) and silently undercounts via the spine's ON CONFLICT collapse — so a
+// report with no usable cumulative total is dropped rather than posted under an unsafe key.
+//
+// #2483 review round 2, LOW-1 — a real-corpus check (0.3% of events) found `totalTokens > 0`
+// with an all-zero input/output breakdown: the only shape that breaks the F1 sum invariant on
+// real data (a nothing-attributable row would post as all-zero, silently undercounting). Reported
+// as explicit unknown usage (#3351) rather than guessed at.
+//
+// #2483 review round 2, MED-1 — the F2 rollout fallback must cache a MISS too, not just a hit:
+// an unresolvable model (the managed/restart no-model case) would otherwise re-run
+// findCodexRolloutPath (a recursive readdirSync of ~/.codex/sessions) + a synchronous ~1 MiB
+// read on EVERY thread/tokenUsage/updated event, forever, on the app-server event loop. The miss
+// is cached with a cooldown, keyed to threadId so a thread change re-arms it immediately.
+import { isRecord, stringValue } from "@humwick/sdk";
+import type { ProviderTokenUsageRecord, ProviderTokenUsageReporter } from "../adapter";
+import { codexModelFromRolloutPath, codexUsageSections, findCodexRolloutPath, numberValue } from "./codex-rollout-context";
+
+/** The spine's turn-less report shape (POST /api/agents/:id/token-usage, #2485). */
+export type CodexTokenUsageSpineRecord = ProviderTokenUsageRecord & { provider: "codex" };
+
+interface CodexTokenUsageBreakdown {
+  inputTokens: number;
+  cachedInputTokens: number;
+  cacheWriteInputTokens: number;
+  outputTokens: number;
+  reasoningOutputTokens: number;
+  totalTokens: number;
+}
+
+function breakdownFrom(usage: Record<string, unknown> | undefined): CodexTokenUsageBreakdown | undefined {
+  if (!usage) return undefined;
+  const inputTokens = numberValue(usage.inputTokens) ?? numberValue(usage.input_tokens);
+  const outputTokens = numberValue(usage.outputTokens) ?? numberValue(usage.output_tokens);
+  const totalTokens = numberValue(usage.totalTokens) ?? numberValue(usage.total_tokens);
+  if (inputTokens === undefined || outputTokens === undefined || totalTokens === undefined) return undefined;
+  return {
+    inputTokens,
+    cachedInputTokens: numberValue(usage.cachedInputTokens) ?? numberValue(usage.cached_input_tokens) ?? 0,
+    cacheWriteInputTokens: numberValue(usage.cacheWriteInputTokens) ?? numberValue(usage.cache_write_input_tokens) ?? 0,
+    outputTokens,
+    reasoningOutputTokens: numberValue(usage.reasoningOutputTokens) ?? numberValue(usage.reasoning_output_tokens) ?? 0,
+    totalTokens,
+  };
+}
+
+/**
+ * Build a spine report from a `thread/tokenUsage/updated` notification's params — the SAME
+ * params handleCodexEvent already reads for context-state tracking (codexTokenUsageContext).
+ * Returns undefined when the event carries no per-turn `last` usage, when that usage is a
+ * zero-token no-op, when `total.totalTokens` isn't available to key a safe dedupe (F4), or when
+ * the caller has no known model — skip rather than post an unattributable row (mirrors
+ * claude-transcript.ts dropping an entry with no model).
+ */
+export function codexTokenUsageReport(
+  params: Record<string, unknown> | undefined,
+  opts: { sessionId: string; model: string | undefined; now?: number },
+): CodexTokenUsageSpineRecord | undefined {
+  if (!opts.model) return undefined;
+  const sections = codexUsageSections(params);
+  if (!sections) return undefined;
+  const lastTotal = numberValue(sections.last?.totalTokens) ?? numberValue(sections.last?.total_tokens);
+  if (lastTotal === undefined || lastTotal <= 0) return undefined;
+  const totalTokensCumulative = numberValue(sections.total?.totalTokens) ?? numberValue(sections.total?.total_tokens);
+  if (totalTokensCumulative === undefined) return undefined;
+  const base = { sessionId: opts.sessionId, provider: "codex" as const, model: opts.model,
+    dedupeKey: `${opts.sessionId}:${totalTokensCumulative}:${lastTotal}`, createdAt: opts.now ?? Date.now() };
+  const lastBreakdown = breakdownFrom(sections.last);
+  // LOW-1 / #3351: positive usage without a readable input/output breakdown is real spend whose
+  // buckets are unknown; report it as unknown under the key a readable report would use.
+  if (!lastBreakdown || (lastBreakdown.inputTokens === 0 && lastBreakdown.outputTokens === 0)) {
+    return { ...base, inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0, usageStatus: "unknown" };
+  }
+  return {
+    ...base,
+    inputTokens: Math.max(0, lastBreakdown.inputTokens - lastBreakdown.cachedInputTokens - lastBreakdown.cacheWriteInputTokens),
+    outputTokens: Math.max(0, lastBreakdown.outputTokens - lastBreakdown.reasoningOutputTokens),
+    cacheReadTokens: lastBreakdown.cachedInputTokens,
+    cacheWriteTokens: lastBreakdown.cacheWriteInputTokens,
+    reasoningOutputTokens: lastBreakdown.reasoningOutputTokens,
+  };
+}
+
+/** How long a failed model-fallback scan is remembered before retrying (MED-1) — long enough
+ *  that a chatty thread can't spin the sync readdirSync+read on every event, short enough that a
+ *  rollout that shows up a little late (or a fresh thread reusing this process) is retried soon. */
+const TOKEN_USAGE_MODEL_PROBE_COOLDOWN_MS = 60_000;
+
+interface CodexTokenUsageModelProbe {
+  threadId: string;
+  model?: string;
+  missedAt?: number;
+}
+
+function readModelProbe(meta: Record<string, unknown>, threadId: string): CodexTokenUsageModelProbe | undefined {
+  const probe = meta.tokenUsageModelProbe;
+  if (!isRecord(probe) || probe.threadId !== threadId) return undefined; // #2483 review MED-1: keyed to
+  // threadId, so a thread change re-arms the probe immediately instead of inheriting a stale miss.
+  return { threadId, model: stringValue(probe.model), missedAt: numberValue(probe.missedAt) };
+}
+
+/**
+ * codex.ts's handleCodexEvent call site — kept to one line there so the reporter wiring doesn't
+ * grow that file past its size ratchet (#291). Resolves the model (the spawn config's, falling
+ * back to the rollout's most recent `turn_context.model` — #2483 review F2: a managed-
+ * policy/restart spawn resolves with no explicit model, and without this fallback every report
+ * for that agent would silently, permanently skip), builds the record, and reports it
+ * (best-effort; failures go to reportFailure).
+ */
+export function reportCodexTokenUsage(
+  params: Record<string, unknown> | undefined,
+  threadId: string | undefined,
+  process: { meta?: Record<string, unknown> } | undefined,
+  reporter: ProviderTokenUsageReporter | undefined,
+  now = Date.now(),
+): void {
+  if (!threadId || !reporter) return;
+  const meta = process?.meta ?? {};
+  const config = isRecord(meta.config) ? meta.config : undefined;
+  let model = stringValue(config?.model);
+  if (!model) {
+    const probe = readModelProbe(meta, threadId);
+    model = probe?.model;
+    const onCooldown = probe?.missedAt !== undefined && now - probe.missedAt < TOKEN_USAGE_MODEL_PROBE_COOLDOWN_MS;
+    if (!model && !onCooldown) {
+      const rolloutPath = stringValue(meta.threadPath) ?? findCodexRolloutPath(threadId);
+      model = rolloutPath ? codexModelFromRolloutPath(rolloutPath) : undefined;
+      if (process) {
+        process.meta = { ...meta, tokenUsageModelProbe: model ? { threadId, model } : { threadId, missedAt: now } };
+      }
+    }
+  }
+  // #3351 — an unresolved model is a label gap, not missing spend: report under "unknown".
+  if (!model) reporter.debug(`token-usage capture: no known model for Codex thread ${threadId}; reporting as unknown model`);
+  const record = codexTokenUsageReport(params, { sessionId: threadId, model: model ?? "unknown", now });
+  if (!record) return;
+  void reporter.report(record).catch((error) => reporter.reportFailure(error));
+}

package/src/adapters/codex-tool-output.ts

@@ -0,0 +1,53 @@
+import { relayEnv, writeChildEnv } from "@humwick/sdk/env";
+import { execFile } from "node:child_process";
+import { join } from "node:path";
+import { AGENT_REDACT_FILES, agentRedactConfigPath, agentRedactInstallDir, isRecord } from "@humwick/sdk";
+import { isAgentRedactInstall, resolveAgentRedact } from "../agent-redact";
+import { humwickHome } from "../config";
+
+const WITHHELD = "[agent-redact: output withheld because MCP result redaction failed]";
+
+export function codexMcpOutputRedactor(cwd: string): ((response: Response) => Promise<Response>) | undefined {
+  if (!isAgentRedactInstall(resolveAgentRedact())) return undefined;
+  return (response) => redactCodexMcpResponse(response, cwd);
+}
+
+// Codex's blocking PostToolUse replacement throws inside functions.exec (#3248).
+// Replace at the transport boundary instead; the existing hook remains a backstop.
+export async function redactCodexMcpResponse(response: Response, cwd: string, env: NodeJS.ProcessEnv = process.env): Promise<Response> {
+  let body: Record<string, unknown> = {};
+  let field = "result";
+  try {
+    const parsed: unknown = await response.json();
+    if (!isRecord(parsed)) throw new Error("invalid JSON-RPC response");
+    body = parsed;
+    field = "result" in body ? "result" : "error";
+    if (!(field in body)) return Response.json(body, { status: response.status });
+    const input = JSON.stringify({ cwd, tool_name: "mcp__agent_relay__result", tool_response: body[field] });
+    const binary = join(agentRedactInstallDir(relayEnv(env, "AGENT_RELAY_HOME") || humwickHome()), AGENT_REDACT_FILES.binary);
+    // The Claude renderer preserves JSON shape; both adapters use the same collector/masker.
+    const output = await new Promise<string>((resolve, reject) => {
+      const child = execFile(binary, ["hook", "claude"], {
+        cwd, env: writeChildEnv({}, { ...env, AGENT_REDACT_CONFIG: env.AGENT_REDACT_CONFIG || agentRedactConfigPath() }),
+        timeout: 10_000, killSignal: "SIGKILL", maxBuffer: Math.max(16_384, Buffer.byteLength(input) * 3),
+      }, (error, stdout, stderr) => error || stderr.trim() ? reject(new Error("masker failed")) : resolve(stdout));
+      child.stdin?.on("error", () => reject(new Error("masker input failed")));
+      child.stdin?.end(input);
+    });
+    // A successful hook writes nothing when no strings needed masking (#3255).
+    const masked: unknown = JSON.parse(output.trim() || "{}");
+    if (!isRecord(masked)) throw new Error("invalid masker output");
+    if (Object.keys(masked).length) {
+      const hook = masked.hookSpecificOutput;
+      if (!isRecord(hook) || hook.hookEventName !== "PostToolUse" || !("updatedToolOutput" in hook)) throw new Error("invalid replacement");
+      const replacement = hook.updatedToolOutput;
+      if (!isRecord(replacement) || (field === "result" && !Array.isArray(replacement.content))) throw new Error("invalid result shape");
+      body = { ...body, [field]: replacement };
+    }
+  } catch {
+    body = { jsonrpc: "2.0", id: body.id ?? null, ...(field === "error"
+      ? { error: { code: isRecord(body.error) && typeof body.error.code === "number" ? body.error.code : -32603, message: WITHHELD } }
+      : { result: { content: [{ type: "text", text: WITHHELD }], isError: true } }) };
+  }
+  return Response.json(body, { status: response.status });
+}

package/src/adapters/codex-turn-outcome.ts

@@ -0,0 +1,27 @@
+import { isRecord, stringValue, type MessageTurnEndReason } from "@humwick/sdk";
+import type { ProviderStatusEvent } from "../adapter";
+
+/** Accept both event method names and turn/thread statuses, including older dot notation. */
+export function codexEndedByFrom(value: string | undefined): MessageTurnEndReason {
+  return value === "systemError" || value?.includes("failed") ? "failed" : value?.includes("interrupted") ? "interrupted" : "completed";
+}
+
+/** Translate the provider's turn error once for event and thread/read completion paths. */
+export function codexTurnFailure(error: unknown, turnId?: string): ProviderStatusEvent {
+  const detail = isRecord(error) ? error : undefined;
+  const message = (typeof error === "string" ? error : stringValue(detail?.message)) || "Codex turn failed without error details.";
+  const additional = stringValue(detail?.additionalDetails);
+  const body = additional ? `${message}\n${additional}` : message;
+  const reason = detail?.code === "model_not_found" || detail?.codexErrorInfo === "modelNotFound"
+    || /\bmodel\b[^\n]*(?:not available|unavailable|does not exist|not found|not supported)|\bunknown model\b/i.test(message)
+    ? "model-unavailable" : "codex-turn-failed";
+  return {
+    status: "error", reason: "provider-turn", id: turnId, clear: ["provider-turn"],
+    providerState: { state: "failed", reason, message: body, source: "codex", terminal: true },
+    metadata: { terminalFailureReason: reason, terminalFailureMessage: body, ...(detail?.codexErrorInfo ? { codexErrorInfo: detail.codexErrorInfo } : {}) },
+    timeline: {
+      status: "provider.restart_decision", title: reason === "model-unavailable" ? "Codex model unavailable" : "Codex turn failed", body,
+      metadata: { eventType: "provider.restart_decision", decision: "stop-surface", reason },
+    },
+  };
+}

package/src/adapters/codex-version.ts

@@ -0,0 +1,64 @@
+// #1096 review, finding 4 — Codex hooks (CODEX_HOME/hooks.json with Claude-compatible
+// event names) are only consumed by Codex >= 0.124. An older Codex silently ignores the
+// file, so writing it and reporting `provisioning.hooks: applied` would be a lie. This
+// module probes the installed Codex version once and answers whether hook delivery is
+// actually honored, so the assembler can emit a truthful projection (a LOUD warning when
+// the guard cannot fire) instead of a silent absence.
+//
+// TODO(#1096): a post-deploy LIVE check is still owed — assert that a Codex >= 0.124 actually
+// executes CODEX_HOME/hooks.json for these event names (Stop/PreToolUse/…), not just that the
+// version string clears the gate. The version probe is necessary but not sufficient.
+
+import { codexVersionOverrideFromEnv } from "../config";
+import { spawnSyncBounded } from "../bounded-spawn-sync";
+
+// Codex releases from this version on ship stable hooks with Claude-compatible events.
+const CODEX_HOOKS_MIN_VERSION = [0, 124, 0] as const;
+
+let cachedProbe: boolean | undefined;
+
+export function codexVersionSupportsHooks(version: string): boolean {
+  const parts = version.split(".").map((n) => parseInt(n, 10));
+  const major = Number.isFinite(parts[0]) ? parts[0]! : 0;
+  const minor = Number.isFinite(parts[1]) ? parts[1]! : 0;
+  const patch = Number.isFinite(parts[2]) ? parts[2]! : 0;
+  const [rMajor, rMinor, rPatch] = CODEX_HOOKS_MIN_VERSION;
+  if (major !== rMajor) return major > rMajor;
+  if (minor !== rMinor) return minor > rMinor;
+  return patch >= rPatch;
+}
+
+export function parseCodexVersion(output: string): string | null {
+  const match = output.match(/(\d+)\.(\d+)\.(\d+)/);
+  return match ? match[0] : null;
+}
+
+function probeCodexVersion(command: string): string | null {
+  try {
+    const result = spawnSyncBounded([command, "--version"], { stdout: "pipe", stderr: "ignore" });
+    if (result.exitCode !== 0) return null;
+    return parseCodexVersion(result.stdout.toString());
+  } catch {
+    return null;
+  }
+}
+
+// Whether the installed Codex actually consumes managed hooks. An explicit
+// AGENT_RELAY_CODEX_VERSION override wins (deterministic tests + operator escape hatch);
+// otherwise the real `codex --version` is probed once and cached. An UNKNOWN version (probe
+// failed / Codex not installed) is treated as supported: we neither block launches nor cry
+// wolf on a machine that can't be inspected — we only claim "not applied" when we can PROVE
+// the version is too old.
+export function codexHooksSupported(command = "codex"): boolean {
+  const override = codexVersionOverrideFromEnv();
+  if (override !== undefined) return override.trim() === "" ? true : codexVersionSupportsHooks(override.trim());
+  if (cachedProbe !== undefined) return cachedProbe;
+  const version = probeCodexVersion(command);
+  cachedProbe = version ? codexVersionSupportsHooks(version) : true;
+  return cachedProbe;
+}
+
+// Test-only: drop the memoized probe so a suite can re-probe under a changed environment.
+export function resetCodexHooksProbeCache(): void {
+  cachedProbe = undefined;
+}