@opengeni/runtime 0.2.1 → 0.2.3

package/dist/index.js

@@ -90,10 +90,10 @@ import {
   timeoutAgentError,
   timeoutControlResponse,
   verifyStreamToken
-} from "./chunk-2PO56VAL.js";
+} from "./chunk-KNW7AMQB.js";
 
 // src/index.ts
-import { AGENT_INSTRUCTIONS_CORE_PLACEHOLDER, collectSandboxEnvironment as collectSandboxEnvironment2, contextServerCompactThreshold, firstPartyMcpBaseUrl, resolveContextCompactionMode, resolveModelProvider, sandboxLifecycleHookIds } from "@opengeni/config";
+import { AGENT_INSTRUCTIONS_CORE_PLACEHOLDER, collectSandboxEnvironment as collectSandboxEnvironment2, contextInputBudgetTokens, contextServerCompactThreshold, firstPartyMcpBaseUrl, resolveContextCompactionMode, resolveModelProvider, sandboxLifecycleHookIds } from "@opengeni/config";
 import { CAPABILITY_DESCRIPTORS as CAPABILITY_DESCRIPTORS2, isClearedRunStateBlob, signDelegatedAccessToken } from "@opengeni/contracts";
 import {
   Agent,
@@ -509,6 +509,106 @@ function computerCallNormalizingFetch(base) {
   };
 }
 
+// src/image-history.ts
+var SCREENSHOT_OMITTED_PLACEHOLDER = "[screenshot omitted: an older desktop frame \u2014 the full image remains in the session event log]";
+var DATA_IMAGE_BASE64_PATTERN = /data:image\/[a-z0-9.+-]+;base64,[a-z0-9+/=_-]+/i;
+function elideStaleScreenshotImages(items, options = {}) {
+  const keepLast = Math.max(0, Math.floor(options.keepLast ?? 3));
+  const placeholder = options.placeholder ?? SCREENSHOT_OMITTED_PLACEHOLDER;
+  const occurrences = [];
+  for (let i = 0; i < items.length; i += 1) {
+    collectItemImageOccurrences(items[i], [i], placeholder, occurrences);
+  }
+  const elidedCount = Math.max(0, occurrences.length - keepLast);
+  if (elidedCount === 0) {
+    return { items: items.slice(), imageCount: occurrences.length, elidedCount: 0 };
+  }
+  const cloned = structuredClone(items);
+  for (const occurrence of occurrences.slice(0, elidedCount)) {
+    setPath(cloned, occurrence.path, occurrence.replacement);
+  }
+  return { items: cloned, imageCount: occurrences.length, elidedCount };
+}
+function collectItemImageOccurrences(item, path, placeholder, out) {
+  if (!isRecord(item)) {
+    return;
+  }
+  if (item.type === "message" && (item.role === "user" || item.role === "system")) {
+    return;
+  }
+  if (item.type === "computer_call_result" || item.type === "computer_call_output") {
+    collectComputerOutputImages(item, path, placeholder, out);
+    return;
+  }
+  if (item.type === "function_call_result" || item.type === "function_call_output") {
+    collectToolResultImages(item.output, [...path, "output"], placeholder, out);
+  }
+}
+function collectComputerOutputImages(item, path, placeholder, out) {
+  const output = item.output;
+  if (!isRecord(output) || output.type !== "computer_screenshot") {
+    return;
+  }
+  for (const key of ["data", "image_url", "imageUrl"]) {
+    if (isImageDataUrl(output[key])) {
+      out.push({ path: [...path, "output", key], replacement: placeholder });
+      return;
+    }
+  }
+}
+function collectToolResultImages(value, path, placeholder, out) {
+  if (typeof value === "string") {
+    if (isImageDataUrl(value)) {
+      out.push({ path, replacement: placeholder });
+    }
+    return;
+  }
+  if (Array.isArray(value)) {
+    for (let i = 0; i < value.length; i += 1) {
+      collectToolResultImages(value[i], [...path, i], placeholder, out);
+    }
+    return;
+  }
+  if (!isRecord(value)) {
+    return;
+  }
+  if (value.type === "input_image") {
+    for (const key of ["image", "imageUrl", "image_url"]) {
+      if (isImageDataUrl(value[key])) {
+        out.push({ path, replacement: { type: "input_text", text: placeholder } });
+        return;
+      }
+    }
+  }
+  for (const key of ["content", "text", "output"]) {
+    if (key in value) {
+      collectToolResultImages(value[key], [...path, key], placeholder, out);
+    }
+  }
+}
+function isImageDataUrl(value) {
+  return typeof value === "string" && DATA_IMAGE_BASE64_PATTERN.test(value);
+}
+function isRecord(value) {
+  return Boolean(value) && typeof value === "object" && !Array.isArray(value);
+}
+function setPath(root, path, value) {
+  if (path.length === 0) {
+    return;
+  }
+  let cursor = root;
+  for (let i = 0; i < path.length - 1; i += 1) {
+    const segment = path[i];
+    cursor = Array.isArray(cursor) ? cursor[segment] : cursor[segment];
+  }
+  const last = path[path.length - 1];
+  if (Array.isArray(cursor)) {
+    cursor[last] = value;
+  } else {
+    cursor[last] = value;
+  }
+}
+
 // src/codex-tool-search.ts
 import { toolSearchTool } from "@openai/agents";
 var CODEX_APPS_TOOL_PREFIX = "codex_apps__";
@@ -685,13 +785,22 @@ function installCodexToolSearch(agent, connectorNamespaces = NO_NAMESPACES) {
 
 // src/context-compaction.ts
 var COMPACTION_SUMMARY_MARKER = "opengeni_context_summary";
-var SUMMARY_PREFIX = [
-  "[CONTEXT CHECKPOINT] The earlier part of this conversation was automatically compacted to stay within the model context window.",
-  "Durable facts already live in the workspace notebook / document bases (via MCP) \u2014 the summary below is a light working-memory bridge, not a full transcript.",
-  "Trust it for current objective, decisions, blockers, deployed/infra state, and next steps; re-read the notebook for anything authoritative.",
+var SUMMARY_BUFFER_TOKENS = 2e4;
+var COMPACT_USER_MESSAGE_MAX_TOKENS = 2e4;
+var CLIENT_COMPACTION_TRIGGER_FRACTION = 0.9;
+var COMPACTION_PROMPT = [
+  "You are performing a CONTEXT CHECKPOINT COMPACTION. Create a handoff summary for another LLM that will resume the task.",
   "",
-  "SUMMARY:"
+  "Include:",
+  "- Current progress and key decisions made",
+  "- Important context, constraints, or user preferences",
+  "- What remains to be done (clear next steps)",
+  "- Any critical data, examples, or references needed to continue",
+  "",
+  "Be concise, structured, and focused on helping the next LLM seamlessly continue the work."
 ].join("\n");
+var SUMMARY_PREFIX = "Another language model started to solve this problem and produced a summary of its thinking process. You also have access to the state of the tools that were used by that language model. Use this to build on the work that has already been done and avoid duplicating work. Here is the summary produced by the other language model, use the information in this summary to assist with your own analysis:";
+var USER_MESSAGE_TRUNCATION_MARKER = "\n[... middle truncated for context compaction ...]\n";
 var RESULT_TYPE_BY_CALL_TYPE2 = {
   function_call: "function_call_result",
   computer_call: "computer_call_result",
@@ -735,6 +844,56 @@ function estimateTokens(items) {
   }
   return total;
 }
+function clientCompactionThresholdTokens(input) {
+  const available = Math.max(
+    0,
+    input.contextWindowTokens - input.contextReservedOutputTokens - SUMMARY_BUFFER_TOKENS
+  );
+  return Math.floor(available * CLIENT_COMPACTION_TRIGGER_FRACTION);
+}
+function decideClientCompaction(input) {
+  const thresholdTokens = clientCompactionThresholdTokens(input);
+  const recorded = typeof input.lastInputTokens === "number" && input.lastInputTokens > 0 ? input.lastInputTokens : 0;
+  const signalTokens = recorded > 0 ? recorded : estimateTokens(input.items);
+  if (input.items.length === 0) {
+    return { shouldCompact: false, reason: "no_history", signalTokens, thresholdTokens };
+  }
+  if (input.force) {
+    return { shouldCompact: true, reason: "force", signalTokens, thresholdTokens };
+  }
+  if (signalTokens > thresholdTokens) {
+    return { shouldCompact: true, reason: "above_threshold", signalTokens, thresholdTokens };
+  }
+  return { shouldCompact: false, reason: "below_threshold", signalTokens, thresholdTokens };
+}
+var CompactionNeededError = class extends Error {
+  signalTokens;
+  thresholdTokens;
+  signalSource;
+  constructor(input) {
+    super(
+      `Context compaction needed: signal ${input.signalTokens} tokens exceeded threshold ${input.thresholdTokens}`
+    );
+    this.name = "CompactionNeededError";
+    this.signalTokens = input.signalTokens;
+    this.thresholdTokens = input.thresholdTokens;
+    this.signalSource = input.signalSource;
+  }
+};
+function findCompactionNeededError(error, seen = /* @__PURE__ */ new WeakSet()) {
+  if (error instanceof CompactionNeededError) {
+    return error;
+  }
+  if (!error || typeof error !== "object") {
+    return null;
+  }
+  if (seen.has(error)) {
+    return null;
+  }
+  seen.add(error);
+  const record = error;
+  return findCompactionNeededError(record.cause, seen) ?? findCompactionNeededError(record.error, seen);
+}
 function findKeepBoundary(items, keepRecentTokens) {
   const boundaries = [];
   for (let i = 0; i < items.length; i += 1) {
@@ -771,150 +930,116 @@ function enforceInputBudget(items, maxTokens, trailingTokens = 0) {
     estimatedTokens: estimateTokens(kept) + Math.max(0, trailingTokens)
   };
 }
-function planCompaction(input) {
-  const softLimit = Math.floor(input.inputBudgetTokens * input.softFraction);
-  const hardLimit = Math.floor(input.inputBudgetTokens * input.hardFraction);
-  const recorded = typeof input.lastInputTokens === "number" && input.lastInputTokens > 0 ? input.lastInputTokens : 0;
-  const signalTokens = Math.max(recorded, estimateTokens(input.items));
-  const hardForced = signalTokens >= hardLimit;
-  const empty = {
-    shouldCompact: false,
-    reason: "below_threshold",
-    signalTokens,
-    hardForced,
-    boundaryIndex: input.items.length,
-    prefixItems: [],
-    priorSummaryItem: null,
-    tailItems: [...input.items]
-  };
-  if (!input.force && signalTokens < softLimit) {
-    return empty;
-  }
-  const effectiveKeepRecent = hardForced ? Math.min(
-    Math.floor(input.keepRecentTokens / 2),
-    Math.floor(input.inputBudgetTokens / 4)
-  ) : input.keepRecentTokens;
-  const boundaryIndex = findKeepBoundary(input.items, effectiveKeepRecent);
-  if (boundaryIndex <= 0) {
-    return { ...empty, reason: "no_boundary", boundaryIndex };
-  }
-  const prefix = input.items.slice(0, boundaryIndex);
-  const tailItems = input.items.slice(boundaryIndex);
-  let priorSummaryItem = null;
-  const prefixItems = [];
-  for (const item of prefix) {
-    if (isCompactionSummary(item)) {
-      priorSummaryItem = item;
+function buildCompactionPromptInput(items) {
+  return [
+    ...items,
+    {
+      type: "message",
+      role: "user",
+      content: COMPACTION_PROMPT
+    }
+  ];
+}
+function buildCompactionReplacementHistory(items, summaryBody) {
+  const history = [];
+  for (const item of items) {
+    if (!isUserMessage(item) || isCompactionSummary(item)) {
       continue;
     }
-    prefixItems.push(item);
-  }
-  if (prefixItems.length === 0) {
-    return { ...empty, reason: "nothing_to_summarize", boundaryIndex };
+    history.push(compactUserMessage(item));
   }
+  history.push(buildSummaryItem(summaryBody));
+  return history;
+}
+function buildSummaryItem(summaryBody) {
+  const trimmed = summaryBody.trim();
   return {
-    shouldCompact: true,
-    reason: "compact",
-    signalTokens,
-    hardForced,
-    boundaryIndex,
-    prefixItems,
-    priorSummaryItem,
-    tailItems
+    type: "message",
+    role: "user",
+    content: `${SUMMARY_PREFIX}
+${trimmed}`,
+    [COMPACTION_SUMMARY_MARKER]: true
   };
 }
-function compactionSummaryText(item) {
-  if (!item) {
-    return "";
+function compactUserMessage(item) {
+  const text = messageText(item);
+  const next = { ...item };
+  if (estimatedTextTokens(text) > COMPACT_USER_MESSAGE_MAX_TOKENS) {
+    next.content = truncateMiddleByEstimatedTokens(text, COMPACT_USER_MESSAGE_MAX_TOKENS);
+    return next;
+  }
+  next.content = contentWithoutImages(item);
+  return next;
+}
+function estimatedTextTokens(text) {
+  return Math.ceil(text.length / 4);
+}
+function truncateMiddleByEstimatedTokens(text, maxTokens) {
+  const maxChars = Math.max(0, maxTokens * 4);
+  if (text.length <= maxChars) {
+    return text;
   }
+  if (maxChars <= USER_MESSAGE_TRUNCATION_MARKER.length) {
+    return USER_MESSAGE_TRUNCATION_MARKER.slice(0, maxChars);
+  }
+  const keepChars = maxChars - USER_MESSAGE_TRUNCATION_MARKER.length;
+  const headChars = Math.ceil(keepChars / 2);
+  const tailChars = Math.floor(keepChars / 2);
+  return `${text.slice(0, headChars)}${USER_MESSAGE_TRUNCATION_MARKER}${text.slice(text.length - tailChars)}`;
+}
+function contentWithoutImages(item) {
+  const content = item.content;
+  if (!Array.isArray(content)) {
+    return content;
+  }
+  return content.filter((part) => {
+    if (!part || typeof part !== "object") {
+      return true;
+    }
+    const type = part.type;
+    return type !== "input_image" && type !== "image_url";
+  });
+}
+function messageText(item) {
   const content = item.content;
   if (typeof content === "string") {
-    return stripSummaryPrefix(content);
+    return content;
   }
   if (Array.isArray(content)) {
-    const text = content.map((part) => {
+    return content.map((part) => {
       if (part && typeof part === "object") {
-        const t = part.text;
-        return typeof t === "string" ? t : "";
+        const record = part;
+        if (typeof record.text === "string") {
+          return record.text;
+        }
+        if (typeof record.content === "string") {
+          return record.content;
+        }
       }
       return "";
     }).join("");
-    return stripSummaryPrefix(text);
   }
   return "";
 }
-function stripSummaryPrefix(text) {
-  const marker = "SUMMARY:";
-  const idx = text.indexOf(marker);
-  return idx >= 0 ? text.slice(idx + marker.length) : text;
-}
-function buildSummaryItem(summaryBody) {
-  return {
-    type: "message",
-    role: "user",
-    content: `${SUMMARY_PREFIX}${summaryBody}`,
-    [COMPACTION_SUMMARY_MARKER]: true
-  };
-}
-var SUMMARY_INSTRUCTIONS = [
-  "You are compacting the earlier part of a long-running agent conversation into a compact working-memory checkpoint so the agent can continue past the model's context limit.",
-  "Durable facts already live in the workspace notebook and document bases (via MCP). Do NOT re-derive or copy those; summarize POINTERS, not contents.",
-  "Capture, concisely and factually:",
-  "- The current objective and the key decisions made so far.",
-  "- Open blockers and anything in-progress.",
-  "- Deployed / infrastructure state that has changed (what exists now).",
-  "- Environment and credential facts BY REFERENCE ONLY \u2014 name the env var keys, secret names, or notebook/document ids; NEVER copy a secret value, token, key, or password.",
-  "- Concrete next steps.",
-  "Say explicitly that durable facts are in the notebook and that this summary lists pointers, not contents.",
-  "Output only the summary body \u2014 no preamble, no markdown headers, plain prose or terse bullets."
-].join("\n");
-function renderPrefixTranscript(items, priorSummaryText) {
-  const lines = [];
-  if (priorSummaryText.trim().length > 0) {
-    lines.push("PRIOR CHECKPOINT SUMMARY (fold this forward; it already replaced even older history):");
-    lines.push(priorSummaryText.trim());
-    lines.push("");
-    lines.push("CONVERSATION SINCE THAT CHECKPOINT:");
-  } else {
-    lines.push("CONVERSATION TO SUMMARIZE:");
-  }
-  for (const item of items) {
-    lines.push(renderItem(item));
-  }
-  return lines.join("\n");
+function renderCompactionPromptInputForChat(input) {
+  return input.map(renderItem).join("\n");
 }
 function renderItem(item) {
   const type = itemType2(item) ?? "unknown";
   if (type === "message") {
     const role = itemRole(item) ?? "assistant";
-    return `[${role}] ${truncate(messageText(item), 4e3)}`;
+    return `[${role}] ${truncateForTranscript(messageText(item), 4e3)}`;
   }
   if (type === "reasoning") {
     return "[reasoning] (omitted)";
   }
   if (RESULT_TYPES2.has(type)) {
-    return `[tool_result] ${truncate(resultText(item), 2e3)}`;
+    return `[tool_result] ${truncateForTranscript(resultText(item), 2e3)}`;
   }
   if (RESULT_TYPE_BY_CALL_TYPE2[type]) {
-    return `[tool_call ${type}] ${truncate(callText(item), 1e3)}`;
-  }
-  return `[${type}] ${truncate(safeStringify(item), 1e3)}`;
-}
-function messageText(item) {
-  const content = item.content;
-  if (typeof content === "string") {
-    return content;
-  }
-  if (Array.isArray(content)) {
-    return content.map((part) => {
-      if (part && typeof part === "object") {
-        const t = part.text;
-        return typeof t === "string" ? t : "";
-      }
-      return "";
-    }).join("");
+    return `[tool_call ${type}] ${truncateForTranscript(callText(item), 1e3)}`;
   }
-  return "";
+  return `[${type}] ${truncateForTranscript(safeStringify(item), 1e3)}`;
 }
 function resultText(item) {
   const output = item.output;
@@ -937,18 +1062,11 @@ function safeStringify(value) {
     return String(value);
   }
 }
-function truncate(text, max) {
+function truncateForTranscript(text, max) {
   if (text.length <= max) {
     return text;
   }
-  return `${text.slice(0, max)}\u2026 (${text.length - max} more chars)`;
-}
-function buildCompactionMessages(plan) {
-  const priorText = compactionSummaryText(plan.priorSummaryItem);
-  return {
-    system: SUMMARY_INSTRUCTIONS,
-    user: renderPrefixTranscript(plan.prefixItems, priorText)
-  };
+  return `${text.slice(0, max)}... (${text.length - max} more chars)`;
 }
 
 // src/sandbox-computer.ts
@@ -966,8 +1084,8 @@ var DEFAULT_DIMENSIONS = [1280, 800];
 var ACTION_YIELD_MS = 15e3;
 var SCROLL_NOTCH_PIXELS = 100;
 var SCROLL_MAX_CLICKS = 15;
-var SCREENSHOT_MAX_ATTEMPTS = 3;
-var SCREENSHOT_RETRY_DELAY_MS = 400;
+var SCREENSHOT_WARMUP_BUDGET_MS = 3e4;
+var SCREENSHOT_RETRY_DELAY_MS = 750;
 var KEYSYM = {
   ctrl: "ctrl",
   control: "ctrl",
@@ -1036,6 +1154,8 @@ var SandboxComputer = class {
   typeDelayMs;
   readOnly;
   tmp;
+  screenshotWarmupBudgetMs;
+  screenshotRetryDelayMs;
   constructor(session, opts = {}) {
     this.session = session;
     this.display = opts.display ?? DEFAULT_DISPLAY;
@@ -1046,6 +1166,8 @@ var SandboxComputer = class {
     this.typeDelayMs = opts.typeDelayMs ?? 12;
     this.readOnly = opts.readOnly ?? false;
     this.tmp = opts.screenshotTmpDir ?? "/tmp";
+    this.screenshotWarmupBudgetMs = opts.screenshotWarmupBudgetMs ?? SCREENSHOT_WARMUP_BUDGET_MS;
+    this.screenshotRetryDelayMs = opts.screenshotRetryDelayMs ?? SCREENSHOT_RETRY_DELAY_MS;
   }
   /** Rebind to a freshly resumed-by-id session after a box rollover / re-establish. */
   rebind(session) {
@@ -1090,10 +1212,13 @@ var SandboxComputer = class {
   }
   async screenshot() {
     let lastError;
-    for (let attempt = 0; attempt < SCREENSHOT_MAX_ATTEMPTS; attempt++) {
+    const deadline = Date.now() + this.screenshotWarmupBudgetMs;
+    let attempt = 0;
+    while (true) {
       if (attempt > 0) {
-        await new Promise((r) => setTimeout(r, SCREENSHOT_RETRY_DELAY_MS));
+        await new Promise((r) => setTimeout(r, this.screenshotRetryDelayMs));
       }
+      attempt++;
       const f = `${this.tmp}/og-shot-${Date.now()}-${Math.random().toString(36).slice(2)}.png`;
       try {
         await this.x(`scrot --pointer --overwrite ${f}`);
@@ -1107,6 +1232,9 @@ var SandboxComputer = class {
       } finally {
         await this.x(`rm -f ${f}`).catch(() => void 0);
       }
+      if (Date.now() + this.screenshotRetryDelayMs >= deadline) {
+        break;
+      }
     }
     if (lastError instanceof Error) {
       throw lastError;
@@ -1449,16 +1577,29 @@ var ComputerUseCapability = class extends Capability {
       // The SDK base exposes the bound runAs as a protected field.
       ...typeof this._runAs === "string" ? { runAs: this._runAs } : {}
     });
+    switch (this.args.toolMode) {
+      case "hosted":
+        return [this.hostedComputerTool(computer)];
+      case "function-image":
+        return computerFunctionTools(computer, this.args.readOnly ?? false, this.args.needsApproval, true);
+      case "function-text":
+        return computerFunctionTools(computer, this.args.readOnly ?? false, this.args.needsApproval, false);
+      case void 0:
+        break;
+    }
     if (supportsStructuredToolOutputTransport(this._modelInstance)) {
-      return [
-        computerTool({
-          computer,
-          ...this.args.needsApproval !== void 0 ? { needsApproval: this.args.needsApproval } : {}
-        })
-      ];
+      return [this.hostedComputerTool(computer)];
     }
     return computerFunctionTools(computer, this.args.readOnly ?? false, this.args.needsApproval, this.args.imageFunctionResults ?? false);
   }
+  /** The single HOSTED `computer_use_preview` tool bound to `computer` — identical
+   *  construction for the explicit "hosted" mode and the legacy structured-sniff path. */
+  hostedComputerTool(computer) {
+    return computerTool({
+      computer,
+      ...this.args.needsApproval !== void 0 ? { needsApproval: this.args.needsApproval } : {}
+    });
+  }
 };
 
 // src/index.ts
@@ -1622,20 +1763,17 @@ function configureOpenAI(settings) {
   }
   setDefaultModelProvider(router);
 }
-async function summarizeForCompaction(settings, messages, options = {}) {
+async function summarizeForCompaction(settings, input, options = {}) {
   const client = options.client ?? buildOpenAIClientFromSettings(settings);
   const api = options.api ?? "responses";
   const model = options.model ?? settings.openaiModel;
-  const maxTokens = options.maxOutputTokens ?? settings.contextSummaryMaxTokens;
+  const maxTokens = options.maxOutputTokens ?? SUMMARY_BUFFER_TOKENS;
   try {
     if (api === "chat") {
       const completion = await client.chat.completions.create({
         model,
         max_tokens: maxTokens,
-        messages: [
-          { role: "system", content: messages.system },
-          { role: "user", content: messages.user }
-        ]
+        messages: [{ role: "user", content: renderCompactionPromptInputForChat(input) }]
       });
       const text2 = completion.choices?.[0]?.message?.content;
       const trimmed2 = typeof text2 === "string" ? text2.trim() : "";
@@ -1648,10 +1786,7 @@ async function summarizeForCompaction(settings, messages, options = {}) {
       // built-in path (api "responses"), so gate it on the built-in provider.
       ...settings.openaiProvider === "azure" ? {} : { store: false },
       max_output_tokens: maxTokens,
-      input: [
-        { role: "system", content: messages.system },
-        { role: "user", content: messages.user }
-      ]
+      input
     });
     const text = extractResponseOutputText(response);
     const trimmed = text.trim();
@@ -1786,7 +1921,8 @@ function buildOpenGeniAgent(settings, resources, options = {}) {
     capabilities: buildAgentCapabilities(settings, options.packSkills ?? [], {
       compactionMode,
       contextWindowTokens,
-      ...options.structuredToolTransport !== void 0 ? { structuredToolTransport: options.structuredToolTransport } : {}
+      ...options.structuredToolTransport !== void 0 ? { structuredToolTransport: options.structuredToolTransport } : {},
+      ...options.computerToolMode !== void 0 ? { computerToolMode: options.computerToolMode } : {}
     })
   });
   agentFileDownloads.set(agent, normalizeSandboxFileDownloads(options.fileResourceDownloads ?? []).filter((download) => !download.content));
@@ -1828,17 +1964,13 @@ function buildAgentCapabilities(settings, packSkills, options = {}) {
   }
   caps.push(skills({ lazyFrom: lazySkillSourceWithPackSkills(packSkills) }));
   if (settings.computerUseEnabled && settings.sandboxDesktopEnabled && desktopCapableBackend(settings.sandboxBackend)) {
+    const explicitMode = options.computerToolMode;
     const computerCapability = computerUse({
       dimensions: [settings.streamResolutionWidth, settings.streamResolutionHeight],
       readOnly: settings.computerUseReadOnly,
-      // On the codex path the function tools deliver screenshots as a real image the
-      // model can see. The ChatGPT/Codex backend rejects HOSTED tool types but DOES
-      // accept `input_image` content items inside a `function_call_output` (proven by
-      // openai/codex codex-rs, whose view_image tool ships exactly that shape) — so a
-      // structured image tool result is seen, where a text data-URL would be unreadable.
-      ...options.structuredToolTransport === false ? { imageFunctionResults: true } : {}
+      ...explicitMode ? { toolMode: explicitMode } : options.structuredToolTransport === false ? { imageFunctionResults: true } : {}
     });
-    if (options.structuredToolTransport === false) {
+    if (!explicitMode && options.structuredToolTransport === false) {
       neutralizeStructuredToolTransport(computerCapability);
     }
     caps.push(computerCapability);
@@ -1899,7 +2031,7 @@ async function prepareAgentTools(settings, tools, options = {}) {
       }
       const error = connectedBestEffort.errors.get(failed);
       console.warn(
-        `[mcp] optional capability server "${failed.name}" failed to connect/list tools; skipping it for this turn`,
+        `[mcp] optional server "${failed.name}" failed to connect/list tools; skipping it for this turn`,
         error instanceof Error ? error.message : error
       );
     }
@@ -2223,6 +2355,52 @@ var normalizeComputerCallsFilter = ({ modelData }) => ({
     modelData.input
   )
 });
+function contextRobustnessFilterForSettings(settings, options = {}) {
+  const inputBudgetTokens = modelCallBudgetTokens(settings);
+  const clientCompactionMode = resolveContextCompactionMode(settings) === "client";
+  const compactionThresholdTokens = clientCompactionThresholdTokens(settings);
+  return ({ modelData }) => {
+    const images = elideStaleScreenshotImages(modelData.input);
+    if (images.elidedCount > 0) {
+      console.warn(
+        `per-call image history policy elided ${images.elidedCount} older screenshot image(s), keeping the last ${Math.min(3, images.imageCount)} full image(s)`
+      );
+    }
+    let input = images.items;
+    if (inputBudgetTokens !== void 0) {
+      const guarded = enforceInputBudget(
+        input,
+        inputBudgetTokens
+      );
+      if (guarded.trimmed) {
+        console.warn(
+          `per-call budget guard trimmed ${guarded.droppedCount} oldest history item(s) to fit input budget (${inputBudgetTokens} tokens); the over-budget model call was NOT sent`
+        );
+        input = guarded.items;
+      }
+    }
+    if (clientCompactionMode && options.throwOnCompactionNeeded) {
+      const reported = options.contextCompactionSignalTokens?.();
+      const hasReported = typeof reported === "number" && reported > 0;
+      const signalTokens = hasReported ? reported : estimateTokens(input);
+      if (signalTokens > compactionThresholdTokens) {
+        throw new CompactionNeededError({
+          signalTokens,
+          thresholdTokens: compactionThresholdTokens,
+          signalSource: hasReported ? "provider" : "estimate"
+        });
+      }
+    }
+    return { ...modelData, input };
+  };
+}
+function modelCallBudgetTokens(settings) {
+  if (resolveContextCompactionMode(settings) !== "client") {
+    return void 0;
+  }
+  const budget = contextInputBudgetTokens(settings);
+  return budget > 0 ? budget : void 0;
+}
 function composeCallModelInputFilters(filters) {
   return async (args) => {
     let modelData = args.modelData;
@@ -2232,11 +2410,12 @@ function composeCallModelInputFilters(filters) {
     return modelData;
   };
 }
-function callModelInputFilterForSettings(settings) {
+function callModelInputFilterForSettings(settings, options = {}) {
   const filters = [normalizeComputerCallsFilter];
   if (settings.openaiProviderItemIds === "strip") {
     filters.push(stripProviderItemIdsFilter);
   }
+  filters.push(contextRobustnessFilterForSettings(settings, options));
   return composeCallModelInputFilters(filters);
 }
 async function runAgentStream(agent, input, settings, overrides = {}) {
@@ -2273,7 +2452,13 @@ async function runAgentStream(agent, input, settings, overrides = {}) {
     }
     const decoratedClient = withSandboxLifecycleHooks(resourceClient2, ownedHooks, ownedHookContext);
     const ownedFilter = composeCallModelInputFilters(
-      [callModelInputFilterForSettings(settings), overrides.callModelInputFilter].filter(
+      [
+        callModelInputFilterForSettings(settings, {
+          throwOnCompactionNeeded: Boolean(overrides.contextCompactionSignalTokens),
+          ...overrides.contextCompactionSignalTokens ? { contextCompactionSignalTokens: overrides.contextCompactionSignalTokens } : {}
+        }),
+        overrides.callModelInputFilter
+      ].filter(
         (f) => Boolean(f)
       )
     );
@@ -2309,18 +2494,23 @@ async function runAgentStream(agent, input, settings, overrides = {}) {
   }) : void 0;
   const sandboxSessionState = prepared.sandboxSessionState ?? (prepared.serializedRunStateForSandbox && client ? await restoredSandboxSessionState(await RunState.fromString(agent, prepared.serializedRunStateForSandbox), client) : void 0);
   const callModelInputFilter = composeCallModelInputFilters(
-    [callModelInputFilterForSettings(settings), overrides.callModelInputFilter].filter(
+    [
+      callModelInputFilterForSettings(settings, {
+        throwOnCompactionNeeded: Boolean(overrides.contextCompactionSignalTokens),
+        ...overrides.contextCompactionSignalTokens ? { contextCompactionSignalTokens: overrides.contextCompactionSignalTokens } : {}
+      }),
+      overrides.callModelInputFilter
+    ].filter(
       (f) => Boolean(f)
     )
   );
   const runOptions = {
     stream: true,
     maxTurns: settings.agentMaxModelCallsPerTurn,
-    // Strip provider-assigned item ids from every model call (turn-start
-    // history replay AND mid-turn follow-ups) so requests never depend on the
-    // provider's server-side response store. A stored response can vanish
-    // between two calls of the same turn, failing the run with 400 "Item with
-    // id 'rs_…' not found"; with the ids gone the request is self-contained.
+    // Built-in per-call guard chain: normalize computer calls, optionally strip
+    // provider ids, elide stale screenshots in every mode, and trim to the input
+    // budget on the client-compaction path. This runs for turn-start replay AND
+    // every mid-turn follow-up.
     callModelInputFilter
   };
   void settings.disableOpenaiTracing;
@@ -3493,12 +3683,16 @@ function approvalIdentifier(item) {
 export {
   ActiveBackendUnresolvableError,
   CAPABILITY_DESCRIPTORS,
+  CLIENT_COMPACTION_TRIGGER_FRACTION,
+  COMPACTION_PROMPT,
   COMPACTION_SUMMARY_MARKER,
+  COMPACT_USER_MESSAGE_MAX_TOKENS,
   ChannelAConflictError,
   ChannelANotFoundError,
   ChannelAUnsupportedError,
   ChannelAValidationError,
   CodexSubscriptionUnavailableError,
+  CompactionNeededError,
   ComputerActionError,
   ComputerReadOnlyError,
   ComputerUnavailableError,
@@ -3520,12 +3714,13 @@ export {
   RecordingUnavailableError,
   RoutingSandboxSession,
   RoutingUnsupportedError,
+  SCREENSHOT_OMITTED_PLACEHOLDER,
   SELFHOSTED_DEFAULT_TIMEOUT_MS,
   SELFHOSTED_RECONNECT_WINDOW_MS,
   SELFHOSTED_RELAY_STREAM_PATH,
   STREAM_PORT,
   STREAM_TOKEN_DEFAULT_TTL_SECONDS,
-  SUMMARY_INSTRUCTIONS,
+  SUMMARY_BUFFER_TOKENS,
   SUMMARY_PREFIX,
   SandboxChannelAService,
   SandboxComputer,
@@ -3540,6 +3735,7 @@ export {
   TERMINAL_STREAM_PORT,
   TerminalServerError,
   TerminalServerUnsupportedError,
+  USER_MESSAGE_TRUNCATION_MARKER,
   agentErrorToControlError,
   agentsErrorRunState,
   applyMissingManifestEntries,
@@ -3550,7 +3746,8 @@ export {
   azureOpenAIDefaultQuery,
   backendSupportsOs,
   buildAgentCapabilities,
-  buildCompactionMessages,
+  buildCompactionPromptInput,
+  buildCompactionReplacementHistory,
   buildDisplayStackScript,
   buildManifest,
   buildModelInstance,
@@ -3562,21 +3759,24 @@ export {
   buildSummaryItem,
   buildTerminalServerScript,
   callModelInputFilterForSettings,
+  clientCompactionThresholdTokens,
   collectSandboxEnvironment,
-  compactionSummaryText,
   composeAgentInstructions,
   computerUse,
   configureOpenAI,
   contentTypeForCodec,
+  contextRobustnessFilterForSettings,
   coreInstructions,
   createProductionAgentRuntime,
   createSandboxClient,
   createSandboxClientForBackend,
+  decideClientCompaction,
   decodeModalSnapshotId,
   deletePriorPersistedSnapshot,
   deleteRecordingArtifacts,
   deserializeSandboxSessionStateEnvelope,
   desktopCapableBackend,
+  elideStaleScreenshotImages,
   enforceInputBudget,
   ensureDisplayStack,
   ensureReadableStreamFrom,
@@ -3587,6 +3787,7 @@ export {
   exposeStreamPort,
   extForCodec,
   extractResponseOutputText,
+  findCompactionNeededError,
   findKeepBoundary,
   isCompactionSummary,
   isExecSessionLostBanner,
@@ -3613,14 +3814,13 @@ export {
   parseNumstatZ,
   parsePorcelainV2,
   parseUnifiedPatch,
-  planCompaction,
   prefixedMcpToolName,
   prepareAgentTools,
   prepareRunInput,
   readRecordingBytes,
   readWorkspaceArchiveFromEnvelopeSessionState,
   recordingStorageKey,
-  renderPrefixTranscript,
+  renderCompactionPromptInputForChat,
   repositoryCloneCommand,
   repositoryUsesSandboxClone,
   resolveTurnModel,