@agent-compose/sdk 0.8.4 → 0.8.6

package/src/runtimes/codex.ts

@@ -5,17 +5,28 @@
  * only stream-parse what it prints.
  *
  * Auth: set `OPENAI_API_KEY` (or `CODEX_API_KEY`) in the sandbox env via a
- * workflow secret. The runtime installs the `codex` CLI (`@openai/codex`) on
- * demand — no image baking needed; pair with `snapshots: { bootFrom: "reuse" }`
- * to install once and boot from the captured snapshot on every run after.
+ * workflow secret. The E2B session image bakes the pinned `codex` CLI
+ * (`@openai/codex@CODEX_CLI_VERSION`); on any other machine the runtime
+ * installs that same version on demand (pair with
+ * `snapshots: { bootFrom: "reuse" }` to install once and boot from the
+ * captured snapshot on every run after).
  *
  * Verified against codex-cli 0.124.0: `codex exec --json` + resume-by-thread,
- * with the command_execution / reasoning / agent_message item shapes below.
+ * with the command_execution / reasoning / agent_message item shapes below;
+ * the plan's `todo_list` item against codex-rs rust-v0.153.4 and
+ * rust-v0.159.2 (see codexPlanMessages); the command_execution /
+ * agent_message / turn.completed shapes re-verified live against 0.160.0
+ * (the pin), whose plan-tool sources are byte-identical to rust-v0.159.2.
  */
 
 import type { AgentMessage } from "../index.js";
-import { createCliAgentRuntime, shellQuote, type CliAgentSpec, type CliReasoningEffort } from "./_cli-agent.js";
+import type { AgentMessagePlan } from "../types/protocol.js";
+import {
+  MID_TURN_DELIVERED_ENV, MID_TURN_INBOX_ENV, createCliAgentRuntime, shellQuote,
+  type CliAgentSpec, type CliReasoningEffort,
+} from "./_cli-agent.js";
 import { formatError } from "../utils/errors.js";
+import { CODEX_CLI_VERSION } from "../sandbox/baked-clis.js";
 
 function now(): string { return new Date().toISOString(); }
 
@@ -24,6 +35,97 @@ function now(): string { return new Date().toISOString(); }
  *  overrides the bundled `@openai/codex` the adapter drives underneath. */
 const CODEX_ACP_ADAPTER = "@agentclientprotocol/codex-acp@0.1.0";
 
+/** The Bash PreToolUse hook rtk installs for Codex (`rtk init -g --codex`
+ *  writes exactly this entry — matcher `Bash`, command `rtk hook codex` —
+ *  into $CODEX_HOME/hooks.json). `rtk hook codex` exists since rtk 0.50.0;
+ *  the image bakes RTK_VERSION (sandbox/baked-clis.ts), which this payload
+ *  was run against. Codex's shell tool matches the `Bash` matcher (codex-rs
+ *  rust-v0.160.0, the pinned CODEX_CLI_VERSION; identical at
+ *  rust-v0.159.2), and the hook runs under `$SHELL -lc` with the PreToolUse
+ *  JSON on stdin. `rtk hook codex` answers `permissionDecision: "allow"` +
+ *  `updatedInput` rewriting `git status` to `rtk git status` when it has a
+ *  filter for the command; Codex applies the replacement before its own
+ *  approval and sandbox checks. For a command it cannot compress (an
+ *  unknown tool, a pipe into one, substitutions, heredocs, redirects, a
+ *  `RTK_DISABLED=1` prefix, an unknown permission mode) it prints nothing
+ *  and exits 0, and Codex runs the original unchanged.
+ *
+ *  The wrapper fails OPEN both ways, like RTK_BASH_HOOK_COMMAND
+ *  (claude-code.ts). `command -v` covers an image without rtk (the devbox,
+ *  a bare Vercel VM): silent, exit 0, so Codex runs the command instead of
+ *  failing every shell call's hook with 127. The trailing `exit 0` covers
+ *  an rtk that cannot answer: Codex treats a PreToolUse hook's exit 2 with
+ *  stderr as a BLOCK of the tool call (codex-rs hooks/src/events/
+ *  pre_tool_use.rs at rust-v0.160.0 — the model reads "Command blocked by
+ *  PreToolUse hook: <stderr>"), and clap exits 2 with its usage on stderr
+ *  for a subcommand it does not know. That was 2026-10-03: the image's rtk
+ *  was a cache-served 0.45.0 with no `hook codex`, and this hook — `exec`
+ *  handing rtk's exit code to Codex — blocked every shell command of every
+ *  codex session. rtk's hooks never exit non-zero on purpose (they fail
+ *  open with no stdout), so a non-zero exit is always a broken rtk, and the
+ *  compressor must never cost the worker its shell: the exit code is
+ *  dropped, and the smoke gate's codex-rtk-hook-rewrite check is what
+ *  proves the rewrite itself. */
+export const RTK_CODEX_HOOK_COMMAND =
+  "command -v rtk >/dev/null 2>&1 && rtk hook codex; exit 0";
+
+/** The awk program of the mid-turn hook below: the inbox lines not yet fed
+ *  (JSON string literals, one per line — `codexSpec.midTurnInput.messageLine`)
+ *  become ONE PostToolUse hook output. Each literal's quotes are stripped and
+ *  the bodies are joined with an escaped blank line; the bodies are already
+ *  JSON-escaped, so the result is one valid JSON string. Nothing is printed
+ *  when no line is due, and codex ignores an empty stdout. */
+const CODEX_MID_TURN_HOOK_AWK = String.raw`NR > fed && NR <= upto { s = substr($0, 2, length($0) - 2); out = (out == "" ? s : out "\\n\\n" s) } END { if (out != "") printf "{\"hookSpecificOutput\":{\"hookEventName\":\"PostToolUse\",\"additionalContext\":\"%s\"}}\n", out }`;
+
+/** The PostToolUse command hook that carries a mid-turn message into a
+ *  RUNNING codex turn (the 2026-10-02 relayed-steer incident: three of the
+ *  owner's instructions waited 48-51 minutes for a codex build to end).
+ *  Codex runs it, under `$SHELL -lc` with the event JSON on stdin and the
+ *  codex process's own environment, after every tool call it completes
+ *  (codex-rs core/src/tools/registry.rs → hook_runtime.rs at rust-v0.160.0,
+ *  the pin). The launch wrapper exported the turn's inbox and delivered-
+ *  counter paths into that environment (sdk _cli-agent.ts
+ *  toolHookInboxFragment); the hook forwards the inbox lines past the
+ *  counter as the hook's `additionalContext`, which codex records as
+ *  developer context in the live turn's history before the model's next
+ *  request (hook_runtime.rs record_additional_contexts), then advances the
+ *  counter — the same ack the server's inject lane trusts for the stdin
+ *  lane. Not a platform turn (no inbox in the environment): silent, exit 0.
+ *  Codex's default spill threshold for a hook's additional context is 2,500
+ *  tokens (hooks/src/output_spill.rs); a longer message reaches the model
+ *  as a preview plus a file pointer, which a steer never is. */
+export const CODEX_MID_TURN_HOOK_COMMAND =
+  // Drain the event JSON first, so codex's stdin write never meets a closed pipe.
+  "cat >/dev/null; "
+  + `[ -n "\${${MID_TURN_INBOX_ENV}:-}" ] && [ -f "$${MID_TURN_INBOX_ENV}" ] || exit 0; `
+  + `ac_fed=$(cat "$${MID_TURN_DELIVERED_ENV}" 2>/dev/null); ac_fed=\${ac_fed:-0}; `
+  + `ac_lines=$(wc -l < "$${MID_TURN_INBOX_ENV}" 2>/dev/null); ac_lines=\${ac_lines:-0}; `
+  + `[ "$ac_lines" -gt "$ac_fed" ] || exit 0; `
+  + `awk -v fed="$ac_fed" -v upto="$ac_lines" '${CODEX_MID_TURN_HOOK_AWK}' "$${MID_TURN_INBOX_ENV}" `
+  + `&& echo "$ac_lines" > "$${MID_TURN_DELIVERED_ENV}"; exit 0`;
+
+/** $CODEX_HOME/hooks.json for every platform Codex session (the server
+ *  writes it at boot, session-runtime-config.ts). Codex only RUNS a
+ *  user-level hook it has persisted trust for — the TUI's review prompt
+ *  has no headless counterpart, and `--dangerously-bypass-hook-trust` would
+ *  run a cloned repository's `.codex/hooks.json` unreviewed too — so the
+ *  server also writes the trust record for each of these exact hooks into
+ *  the config.toml it generates (sandbox/codex-hooks.ts). Verified live
+ *  against codex 0.159.2 and 0.160.0 for the rtk hook: with the record,
+ *  `codex exec` rewrote `git status` through rtk with no bypass flag;
+ *  without it, the hook was skipped.
+ *
+ *  The PostToolUse group carries NO matcher: codex runs a matcher-less hook
+ *  after every tool it completes (hooks/src/events/common.rs
+ *  matches_matcher: an absent matcher is a match), so a mid-turn message
+ *  lands at the next tool step whatever the tool was. */
+export const CODEX_PLATFORM_HOOKS = {
+  hooks: {
+    PreToolUse: [{ matcher: "Bash", hooks: [{ type: "command", command: RTK_CODEX_HOOK_COMMAND }] }],
+    PostToolUse: [{ hooks: [{ type: "command", command: CODEX_MID_TURN_HOOK_COMMAND }] }],
+  },
+} as const;
+
 /** Known-noise codex ADVISORY lines. codex emits these as `item.completed`
  *  error items on the `--json` stream (exec maps every `Warning` notification
  *  to an error item — verified against codex-cli 0.147.0), so without a filter
@@ -101,6 +203,41 @@ export function codexWriterLockPreflight(
     + `flock -n "$ac_lk" rm -f -- "$ac_lk" 2>/dev/null || true; fi; `;
 }
 
+/**
+ * codex's PLAN (its update_plan tool) on the `--json` stream: one
+ * `todo_list` item per turn, `item.started` on the plan's first update,
+ * `item.updated` (same id, the whole list) on every later one, and
+ * `item.completed` at the turn's end. That is codex-rs exec's
+ * event_processor_with_jsonl_output.rs, identical in rust-v0.153.4 and
+ * rust-v0.159.2, where it is the only `item.updated` codex emits; the item
+ * `{ id, type: "todo_list", items: [{ text, completed }] }` is the shape every
+ * todo_list item our machines have persisted carries. Each event maps to ONE
+ * whole-plan `plan` message, so the transcript's checklist, and the worker's
+ * status line, follow the plan while the turn runs (before this, the list
+ * surfaced once, at the turn's end, as a raw `todo_list` tool card).
+ *
+ * A step carries only `completed`: codex's `pending` and `in_progress` both
+ * serialize as false. Its plan tool allows at most one step in progress and
+ * a plan runs in order, so the first step not completed is the one under
+ * way; the rest are pending. The stream names no priority, so none is set.
+ */
+function codexPlanMessages(item: Record<string, unknown>, timestamp: string): AgentMessage[] {
+  const steps = Array.isArray(item.items) ? item.items : [];
+  let underWay = false;
+  const entries: AgentMessagePlan["entries"] = [];
+  for (const step of steps) {
+    if (typeof step !== "object" || step === null) continue;
+    const text = (step as { text?: unknown }).text;
+    if (typeof text !== "string" || text.trim().length === 0) continue;
+    let status: AgentMessagePlan["entries"][number]["status"];
+    if ((step as { completed?: unknown }).completed === true) status = "completed";
+    else if (!underWay) { status = "in_progress"; underWay = true; }
+    else status = "pending";
+    entries.push({ content: text.trim(), status });
+  }
+  return entries.length > 0 ? [{ type: "plan", entries, timestamp }] : [];
+}
+
 /** Exported for the fixture-based parity tests (ADR-0020): the legacy JSONL
  *  `mapEvent` is the golden the ACP normaliser is asserted equal to. Not part
  *  of the public runtime surface — `createCodexRuntime` stays the entry point. */
@@ -113,6 +250,22 @@ export const codexSpec: CliAgentSpec = {
   // every resume of the same thread, so supersede teardown must KILL it —
   // never detach it alive (server runner-kill.ts consumes this flag).
   exclusiveSessionWriter: true,
+  // Mid-turn input (the 2026-10-02 relayed-steer incident). codex reads
+  // stdin ONCE, as the prompt: `codex exec -` submits one UserTurn and never
+  // reads stdin again (codex-rs exec/src/lib.rs read_prompt_from_stdin and
+  // its single InitialOperation::UserTurn at rust-v0.160.0; the app-server's
+  // `turn/steer` is a different transport, not `codex exec`). So a running
+  // turn takes a message through the PostToolUse hook above: codex runs it
+  // after every tool call it completes, with the codex process's own
+  // environment; the hook prints the unfed inbox lines as `additionalContext`
+  // and codex records them as developer context in the live turn's history
+  // before the model's next request. One JSON string literal per inbox line,
+  // so the hook splices bodies without decoding. Codex builds the PostToolUse
+  // payload only for a tool call it counts as successful (tools/registry.rs),
+  // so a message lands at the next successful tool step. Source-verified at
+  // rust-v0.160.0 (the pinned CODEX_CLI_VERSION); the end-to-end run against
+  // a live codex is the acceptance check still owed.
+  midTurnInput: { transport: "tool-hook", messageLine: (text) => JSON.stringify(text) },
   // ACP-mode invocation (ADR-0020 increment 1). `codex` has no native `--acp`
   // flag; the adapter (a Rust binary shipped via npm) speaks ACP and drives a
   // compatible bundled `@openai/codex`. The runner attempts this first and
@@ -125,9 +278,11 @@ export const codexSpec: CliAgentSpec = {
     // (e.g. a sandbox-baked one) instead of the adapter's bundled copy.
     ...(process.env.CODEX_PATH ? { env: { CODEX_PATH: process.env.CODEX_PATH } } : {}),
   },
-  // Global npm install; symlink onto PATH only if the global bin dir isn't
-  // already there (so a non-login `sh -c` can find it).
-  install: 'sudo npm install -g @openai/codex && (command -v codex >/dev/null 2>&1 || sudo ln -sf "$(npm prefix -g)/bin/codex" /usr/local/bin/codex)',
+  // The E2B session image bakes this exact version (sandbox/baked-clis.ts),
+  // so this runs only on a machine whose image predates the bake: a global
+  // npm install of the SAME pin, symlinked onto PATH only if the global bin
+  // dir isn't already there (so a non-login `sh -c` can find it).
+  install: `sudo npm install -g @openai/codex@${CODEX_CLI_VERSION} && (command -v codex >/dev/null 2>&1 || sudo ln -sf "$(npm prefix -g)/bin/codex" /usr/local/bin/codex)`,
   // Codex reads the prompt from stdin when invoked as `codex exec ... -`.
   promptPayload: (prompt) => prompt,
   buildCommand: ({ promptPath, sessionId, model, cwd, effort }) => {
@@ -140,13 +295,13 @@ export const codexSpec: CliAgentSpec = {
       // for running in environments that are externally sandboxed".
       "--dangerously-bypass-approvals-and-sandbox",
       ...(model ? ["-m", shellQuote(model)] : []),
-      // codex's own reasoning knob — a config override, valid values
-      // none|minimal|low|medium|high|xhigh (codex docs; xhigh is the
-      // codex-max-tier deep-reasoning level). There is NO "max" in codex's
-      // vocabulary: the server rejects it for codex sessions
-      // (sessionEffortLockError), and this clamp to xhigh is the type-level
-      // backstop for a caller that bypasses that gate. The value comes from
-      // the closed CliReasoningEffort set, so it is shell-safe unquoted.
+      // codex's own reasoning knob — a config override. Every codex model
+      // takes low|medium|high|xhigh; only the GPT-6 and GPT-5.6 presets add
+      // max (codex 0.153+), so the platform offers codex up to xhigh: the
+      // server rejects "max" for codex sessions (sessionEffortLockError), and
+      // this clamp to xhigh is the type-level backstop for a caller that
+      // bypasses that gate. The value comes from the closed
+      // CliReasoningEffort set, so it is shell-safe unquoted.
       ...(effort ? ["-c", `model_reasoning_effort=${effort === "max" ? "xhigh" : effort}`] : []),
     ].join(" ");
     // Working directory via a shell `cd` — the idiom EVERY other runtime uses
@@ -172,11 +327,19 @@ export const codexSpec: CliAgentSpec = {
   mapEvent: (p): AgentMessage[] => {
     const ts = now();
     switch (p.type) {
+      case "item.updated": {
+        // Only the plan updates in place (codexPlanMessages); any other
+        // item surfaces on its started / completed events below.
+        const item = p.item as Record<string, unknown> | undefined;
+        return item?.type === "todo_list" ? codexPlanMessages(item, ts) : [];
+      }
       case "item.started":
       case "item.completed": {
         const item = p.item as Record<string, unknown> | undefined;
         if (!item) return [];
         const itype = String(item.type ?? "");
+        // The plan, whole, on its first update and at the turn's end.
+        if (itype === "todo_list") return codexPlanMessages(item, ts);
         // Text + reasoning land on completion (started carries no final text).
         if (itype === "agent_message") {
           return p.type === "item.completed" ? [{ type: "text", text: String(item.text ?? ""), timestamp: ts }] : [];
@@ -209,12 +372,17 @@ export const codexSpec: CliAgentSpec = {
           }
           return [{ type: "error", text, timestamp: ts }];
         }
-        // file_change / mcp_tool_call / web_search / todo: surface once, on completion.
+        // file_change / mcp_tool_call / web_search: surface once, on completion.
         if (p.type === "item.completed") {
           return [{ type: "tool_use", toolName: itype || "item", toolInput: item, toolUseId: String(item.id ?? ""), timestamp: ts }];
         }
         return [];
       }
+      // The turn's token classes as codex reports them (codex-rs
+      // exec_events.rs `Usage`): input, cached input, cache-write input,
+      // output and reasoning output. Every class is kept — cache writes land
+      // in the creation class, reasoning rides beside output (it is counted
+      // inside output_tokens, so the four totals stay additive).
       case "turn.completed": {
         const u = p.usage as Record<string, number> | undefined;
         if (!u) return [];
@@ -223,9 +391,10 @@ export const codexSpec: CliAgentSpec = {
           inputTokens: u.input_tokens ?? 0,
           outputTokens: u.output_tokens ?? 0,
           cacheReadTokens: u.cached_input_tokens ?? 0,
-          cacheCreationTokens: 0,
+          cacheCreationTokens: u.cache_write_input_tokens ?? 0,
           durationMs: 0,
           numTurns: 1,
+          ...(typeof u.reasoning_output_tokens === "number" ? { reasoningOutputTokens: u.reasoning_output_tokens } : {}),
           timestamp: ts,
         }];
       }
@@ -238,8 +407,8 @@ export const codexSpec: CliAgentSpec = {
   },
 };
 
-/** The effort levels codex actually has (`model_reasoning_effort`):
- *  low|medium|high|xhigh — no "max" (that level is Claude Code's alone). */
+/** The effort levels this runtime passes to codex (`model_reasoning_effort`):
+ *  low|medium|high|xhigh, the levels every codex model takes. */
 export type CodexReasoningEffort = Exclude<CliReasoningEffort, "max">;
 
 export interface CodexRuntimeConfig {

package/src/runtimes/openai-desktop.ts

@@ -3,6 +3,17 @@
  *
  * Works against DesktopSandboxProvider — no E2B SDK dependency here.
  * The action→screenshot loop runs until the model emits text, then yields it.
+ *
+ * COORDINATE SPACES (incident note). Every screenshot is downscaled to the
+ * model's DISPLAY geometry (1024×720) before it is sent, so the model emits
+ * coordinates in the SCALED screenshot's space — never the real desktop's.
+ * An earlier revision executed those coordinates against the real desktop
+ * using hardcoded real-geometry constants, so on any sandbox whose actual
+ * geometry differed, every click landed in the wrong place. The provider has
+ * no geometry call, so the real geometry is read from each screenshot buffer
+ * itself (sharp metadata), and the resulting per-capture `CaptureScale` maps
+ * the model's coordinates back onto the exact frame it saw. The old constants
+ * survive ONLY as a fallback for the metadata-missing case.
  */
 
 import OpenAI from "openai";
@@ -12,8 +23,11 @@ import { defineRuntime, formatError } from "../index.js";
 
 const DISPLAY_WIDTH  = 1024;
 const DISPLAY_HEIGHT = 720;
-const DESKTOP_WIDTH  = 1280;
-const DESKTOP_HEIGHT = 800;
+/** FALLBACK real-desktop geometry — consulted ONLY when sharp cannot read a
+ *  screenshot's dimensions. The truth is derived per capture from the raw
+ *  screenshot buffer itself (see `captureScaledScreenshot`). */
+const FALLBACK_DESKTOP_WIDTH  = 1280;
+const FALLBACK_DESKTOP_HEIGHT = 800;
 const MAX_TURNS = 40;
 
 export interface OpenAIDesktopRunnerOptions extends RuntimeOptions {
@@ -27,8 +41,9 @@ export interface OpenAIDesktopRunnerOptions extends RuntimeOptions {
 // so the exchange is described here instead of with SDK types.
 
 /** One model-emitted desktop action. Coordinates arrive in DISPLAY (model)
- *  space and are rescaled to the real desktop before dispatch. */
-interface ComputerAction {
+ *  space and are rescaled through the CURRENT capture's `CaptureScale`
+ *  before dispatch. Exported so tests can type their fixtures. */
+export interface ComputerAction {
   type: string;
   coordinate?: [number, number];
   startCoordinate?: [number, number];
@@ -99,12 +114,16 @@ export class OpenAIDesktopRunner implements ModelExecutionContract {
 
     yield { type: "init", sessionId: "e2b-desktop", timestamp: ts() };
 
-    const screenshot = await captureScaledScreenshot(sandbox);
+    const first = await captureScaledScreenshot(sandbox);
+    // The scale of the LATEST frame the model has seen — its next batch of
+    // coordinates is in that frame's space, so every fresh capture below
+    // replaces this before the frame is sent back.
+    let scale = first.scale;
 
     const messages: InputMessage[] = [{
       role: "user",
       content: [
-        { type: "input_image", image_url: `data:image/png;base64,${screenshot}` },
+        { type: "input_image", image_url: `data:image/png;base64,${first.b64}` },
         { type: "input_text", text: opts.prompt },
       ],
     }];
@@ -137,7 +156,7 @@ export class OpenAIDesktopRunner implements ModelExecutionContract {
 
       for (const call of computerCalls) {
         console.log(`${label}   → ${call.action.type}`);
-        await executeAction(sandbox, call.action).catch(
+        await executeAction(sandbox, call.action, scale).catch(
           (err: unknown) => console.warn(`${label}   Action failed (non-fatal): ${formatError(err)}`),
         );
       }
@@ -145,10 +164,11 @@ export class OpenAIDesktopRunner implements ModelExecutionContract {
       if (computerCalls.length === 0) break;
 
       const next = await captureScaledScreenshot(sandbox);
+      scale = next.scale;
       messages.push({ role: "user", content: computerCalls.map((call): InputPart => ({
         type: "computer_call_output",
         call_id: call.call_id,
-        output: { type: "input_image", image_url: `data:image/png;base64,${next}` },
+        output: { type: "input_image", image_url: `data:image/png;base64,${next.b64}` },
       })) });
     }
 
@@ -160,21 +180,58 @@ export class OpenAIDesktopRunner implements ModelExecutionContract {
 // Helpers — work against DesktopSandboxProvider interface
 // ---------------------------------------------------------------------------
 
-async function captureScaledScreenshot(sandbox: DesktopSandboxProvider): Promise<string> {
+/** Real-desktop pixels per model-DISPLAY pixel, derived from ONE capture.
+ *  Coordinates the model emits against that capture are multiplied by these
+ *  factors (and rounded) before touching the provider. */
+export interface CaptureScale {
+  scaleX: number;
+  scaleY: number;
+}
+
+/** One captured frame: the model-facing image plus the scale that maps the
+ *  model's coordinates back onto this frame's real desktop. */
+export interface ScaledCapture {
+  /** Base64 PNG resized to DISPLAY_WIDTH×DISPLAY_HEIGHT (fit "fill"). */
+  b64: string;
+  scale: CaptureScale;
+}
+
+/** Derive a capture's scale from its real pixel dimensions. The constants
+ *  are a FALLBACK only — used when sharp cannot read the frame's metadata;
+ *  a real dimension always wins. */
+export function scaleFromDimensions(width: number | undefined, height: number | undefined): CaptureScale {
+  return {
+    scaleX: (width  ?? FALLBACK_DESKTOP_WIDTH)  / DISPLAY_WIDTH,
+    scaleY: (height ?? FALLBACK_DESKTOP_HEIGHT) / DISPLAY_HEIGHT,
+  };
+}
+
+/** Capture one frame: read the REAL geometry from the raw screenshot's own
+ *  metadata, downscale to the model's DISPLAY geometry, and return both the
+ *  image and the scale that maps model coordinates back onto this frame. */
+export async function captureScaledScreenshot(sandbox: DesktopSandboxProvider): Promise<ScaledCapture> {
   const raw = await sandbox.screenshot();
-  const scaled = await sharp(raw)
+  const image = sharp(raw);
+  const { width, height } = await image.metadata();
+  const scaled = await image
     .resize(DISPLAY_WIDTH, DISPLAY_HEIGHT, { kernel: "lanczos3", fit: "fill" })
     .png()
     .toBuffer();
-  return scaled.toString("base64");
+  return { b64: scaled.toString("base64"), scale: scaleFromDimensions(width, height) };
 }
 
-function scaleX(x: number): number { return Math.round(x * DESKTOP_WIDTH / DISPLAY_WIDTH); }
-function scaleY(y: number): number { return Math.round(y * DESKTOP_HEIGHT / DISPLAY_HEIGHT); }
-
-async function executeAction(sandbox: DesktopSandboxProvider, action: ComputerAction): Promise<void> {
+/** Dispatch one model action against the provider, mapping every coordinate
+ *  through the CURRENT capture's scale. Non-coordinate payloads — scroll
+ *  deltas (ticks), typed text, key chords — pass through untouched. */
+export async function executeAction(
+  sandbox: DesktopSandboxProvider,
+  action: ComputerAction,
+  scale: CaptureScale,
+): Promise<void> {
+  const mapX = (x: number) => Math.round(x * scale.scaleX);
+  const mapY = (y: number) => Math.round(y * scale.scaleY);
   const [x, y] = action.coordinate
-    ? [scaleX(action.coordinate[0]), scaleY(action.coordinate[1])]
+    ? [mapX(action.coordinate[0]), mapY(action.coordinate[1])]
     : [0, 0];
 
   switch (action.type) {
@@ -185,12 +242,18 @@ async function executeAction(sandbox: DesktopSandboxProvider, action: ComputerAc
     case "move":         await sandbox.moveMouse(x, y); break;
     case "type":         await sandbox.write(action.text ?? ""); break;
     case "key":          await sandbox.press(action.key ?? ""); break;
-    case "scroll":       await sandbox.scroll(action.direction === "up" ? "up" : "down", action.ticks ?? 3); break;
+    case "scroll":
+      // The scroll POSITION is a coordinate — mapped (the provider has no
+      // positional scroll, so position lands via moveMouse). The scroll
+      // DELTA (ticks) is not a coordinate — untouched.
+      if (action.coordinate) await sandbox.moveMouse(x, y);
+      await sandbox.scroll(action.direction === "up" ? "up" : "down", action.ticks ?? 3);
+      break;
     case "drag":
       if (action.startCoordinate && action.endCoordinate) {
         await sandbox.drag(
-          [scaleX(action.startCoordinate[0]), scaleY(action.startCoordinate[1])],
-          [scaleX(action.endCoordinate[0]), scaleY(action.endCoordinate[1])],
+          [mapX(action.startCoordinate[0]), mapY(action.startCoordinate[1])],
+          [mapX(action.endCoordinate[0]), mapY(action.endCoordinate[1])],
         );
       }
       break;