@opengeni/runtime 0.3.0 → 0.4.0

package/dist/sandbox/index.js

@@ -95,7 +95,7 @@ import {
   timeoutAgentError,
   timeoutControlResponse,
   verifyStreamToken
-} from "../chunk-D5KU3QUC.js";
+} from "../chunk-HGQ252FL.js";
 export {
   ActiveBackendUnresolvableError,
   CAPABILITY_DESCRIPTORS,

package/package.json

@@ -1,7 +1,8 @@
 {
   "name": "@opengeni/runtime",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "type": "module",
+  "license": "Apache-2.0",
   "main": "./dist/index.js",
   "module": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -25,22 +26,19 @@
   },
   "scripts": {
     "build": "tsup",
-    "typecheck": "tsc --noEmit"
+    "typecheck": "tsc --noEmit",
+    "prepublishOnly": "bash ../../scripts/prepublish-guard"
   },
   "dependencies": {
     "@opengeni/agent-proto": "^0.2.1",
-    "@opengeni/config": "^0.2.4",
-    "@opengeni/contracts": "^0.6.0",
+    "@opengeni/config": "^0.3.0",
+    "@opengeni/contracts": "^0.9.0",
     "@openai/agents": "^0.11.6",
     "@openai/agents-extensions": "^0.11.6",
     "modal": "^0.7.4",
     "openai": "6.36.0",
     "@opengeni/codex": "^0.2.1"
   },
-  "devDependencies": {
-    "tsup": "^8.5.0",
-    "typescript": "^6.0.3"
-  },
   "repository": {
     "type": "git",
     "url": "git+https://github.com/Cloudgeni-ai/opengeni.git",

package/src/context-compaction.ts

@@ -8,6 +8,8 @@
  * from the active model-facing history; the database audit rows remain.
  */
 
+import { TOOL_CALL_RESULT_TYPE_BY_CALL_TYPE } from "./history-sanitizer";
+
 export type CompactionItem = Record<string, unknown>;
 
 /**
@@ -18,7 +20,10 @@ export const COMPACTION_SUMMARY_MARKER = "opengeni_context_summary";
 
 export const SUMMARY_BUFFER_TOKENS = 20_000;
 export const COMPACT_USER_MESSAGE_MAX_TOKENS = 20_000;
-export const CLIENT_COMPACTION_TRIGGER_FRACTION = 0.9;
+export const DEFAULT_COMPACTION_THRESHOLD_RATIO = 0.6;
+export const MIN_COMPACTION_THRESHOLD_RATIO = 0.3;
+export const MAX_COMPACTION_THRESHOLD_RATIO = 0.9;
+export const COMPACTION_FALLBACK_TARGET_RATIO = 0.5;
 
 // Verbatim from Codex CLI:
 // codex-rs/prompts/templates/compact/prompt.md
@@ -42,12 +47,7 @@ export const SUMMARY_PREFIX =
 export const USER_MESSAGE_TRUNCATION_MARKER =
   "\n[... middle truncated for context compaction ...]\n";
 
-const RESULT_TYPE_BY_CALL_TYPE: Record<string, string> = {
-  function_call: "function_call_result",
-  computer_call: "computer_call_result",
-  shell_call: "shell_call_output",
-  apply_patch_call: "apply_patch_call_output",
-};
+const RESULT_TYPE_BY_CALL_TYPE = TOOL_CALL_RESULT_TYPE_BY_CALL_TYPE;
 const RESULT_TYPES = new Set(Object.values(RESULT_TYPE_BY_CALL_TYPE));
 
 function itemType(item: unknown): string | undefined {
@@ -101,15 +101,19 @@ export function estimateTokens(items: readonly CompactionItem[]): number {
   return total;
 }
 
+export function clampCompactionThresholdRatio(value: number | undefined | null): number {
+  const numeric = typeof value === "number" && Number.isFinite(value)
+    ? value
+    : DEFAULT_COMPACTION_THRESHOLD_RATIO;
+  return Math.min(MAX_COMPACTION_THRESHOLD_RATIO, Math.max(MIN_COMPACTION_THRESHOLD_RATIO, numeric));
+}
+
 export function clientCompactionThresholdTokens(input: {
   contextWindowTokens: number;
   contextReservedOutputTokens: number;
+  contextCompactionThresholdRatio?: number | null;
 }): number {
-  const available = Math.max(
-    0,
-    input.contextWindowTokens - input.contextReservedOutputTokens - SUMMARY_BUFFER_TOKENS,
-  );
-  return Math.floor(available * CLIENT_COMPACTION_TRIGGER_FRACTION);
+  return Math.floor(Math.max(0, input.contextWindowTokens) * clampCompactionThresholdRatio(input.contextCompactionThresholdRatio));
 }
 
 export type ClientCompactionDecision = {
@@ -124,6 +128,7 @@ export function decideClientCompaction(input: {
   lastInputTokens?: number | null;
   contextWindowTokens: number;
   contextReservedOutputTokens: number;
+  contextCompactionThresholdRatio?: number | null;
   force?: boolean;
 }): ClientCompactionDecision {
   const thresholdTokens = clientCompactionThresholdTokens(input);
@@ -272,6 +277,70 @@ export function buildCompactionReplacementHistory(
   return history;
 }
 
+export type DeterministicFallbackCompactionInput = {
+  items: readonly CompactionItem[];
+  cause?: string;
+  targetTokens?: number;
+};
+
+export function buildDeterministicFallbackCompactionHistory(input: DeterministicFallbackCompactionInput): CompactionItem[] {
+  const sourceTokens = Math.max(1, estimateTokens(input.items));
+  const requestedTarget = typeof input.targetTokens === "number" && input.targetTokens > 0
+    ? Math.floor(input.targetTokens)
+    : Math.floor(sourceTokens * COMPACTION_FALLBACK_TARGET_RATIO);
+  const targetTokens = Math.max(1, Math.min(requestedTarget, Math.max(1, sourceTokens - 1)));
+  const summaryItem = buildSummaryItem([
+    "Non-LLM context compaction fallback.",
+    input.cause ? `Summarizer failure: ${input.cause}` : "Summarizer failure: unavailable.",
+    "Older assistant, tool, reasoning, and image history was dropped deterministically.",
+    "Retained context above is limited to initial instructions and the most recent user messages that fit the fallback budget; oversized retained messages may be middle-truncated.",
+  ].join("\n"));
+  const summaryTokens = estimateItemTokens(summaryItem);
+  let remaining = Math.max(0, targetTokens - summaryTokens);
+  const retainedInstructions: CompactionItem[] = [];
+
+  for (const item of input.items) {
+    if (!isInitialInstructionMessage(item)) {
+      if (isUserMessage(item)) {
+        break;
+      }
+      continue;
+    }
+    if (remaining <= 0) {
+      break;
+    }
+    const capped = compactMessageToEstimatedItemBudget(item, Math.min(2_000, remaining));
+    if (!capped) {
+      continue;
+    }
+    const tokens = estimateItemTokens(capped);
+    if (tokens <= remaining) {
+      retainedInstructions.push(capped);
+      remaining -= tokens;
+    }
+  }
+
+  const retainedUsers: CompactionItem[] = [];
+  for (let index = input.items.length - 1; index >= 0 && remaining > 0; index -= 1) {
+    const item = input.items[index]!;
+    if (!isUserMessage(item) || isCompactionSummary(item)) {
+      continue;
+    }
+    const capped = compactMessageToEstimatedItemBudget(item, Math.min(COMPACT_USER_MESSAGE_MAX_TOKENS, remaining));
+    if (!capped) {
+      continue;
+    }
+    const tokens = estimateItemTokens(capped);
+    if (tokens <= remaining) {
+      retainedUsers.push(capped);
+      remaining -= tokens;
+    }
+  }
+  retainedUsers.reverse();
+
+  return [...retainedInstructions, ...retainedUsers, summaryItem];
+}
+
 /**
  * Build the synthetic summary item (a plain user message) appended to the
  * rebuilt active history.
@@ -287,16 +356,41 @@ export function buildSummaryItem(summaryBody: string): CompactionItem {
 }
 
 function compactUserMessage(item: CompactionItem): CompactionItem {
+  return compactMessageToTokenBudget(item, COMPACT_USER_MESSAGE_MAX_TOKENS);
+}
+
+function compactMessageToTokenBudget(item: CompactionItem, maxTokens: number): CompactionItem {
   const text = messageText(item);
   const next = { ...item };
-  if (estimatedTextTokens(text) > COMPACT_USER_MESSAGE_MAX_TOKENS) {
-    next.content = truncateMiddleByEstimatedTokens(text, COMPACT_USER_MESSAGE_MAX_TOKENS);
+  if (estimatedTextTokens(text) > maxTokens) {
+    next.content = truncateMiddleByEstimatedTokens(text, maxTokens);
     return next;
   }
   next.content = contentWithoutImages(item);
   return next;
 }
 
+function compactMessageToEstimatedItemBudget(item: CompactionItem, maxItemTokens: number): CompactionItem | null {
+  if (maxItemTokens <= 0) {
+    return null;
+  }
+  const overheadTokens = estimateItemTokens({ ...item, content: "" });
+  let contentBudget = Math.max(0, maxItemTokens - overheadTokens);
+  for (let attempt = 0; attempt < 6; attempt += 1) {
+    const capped = compactMessageToTokenBudget(item, contentBudget);
+    const estimated = estimateItemTokens(capped);
+    if (estimated <= maxItemTokens) {
+      return capped;
+    }
+    const overage = estimated - maxItemTokens;
+    if (contentBudget <= 0) {
+      break;
+    }
+    contentBudget = Math.max(0, contentBudget - Math.max(1, overage));
+  }
+  return null;
+}
+
 function estimatedTextTokens(text: string): number {
   return Math.ceil(text.length / 4);
 }
@@ -353,8 +447,42 @@ function messageText(item: CompactionItem): string {
   return "";
 }
 
-export function renderCompactionPromptInputForChat(input: readonly CompactionItem[]): string {
-  return input.map(renderItem).join("\n");
+function isInitialInstructionMessage(item: unknown): boolean {
+  const type = itemType(item);
+  const role = itemRole(item);
+  return type === "message" && (role === "system" || role === "developer");
+}
+
+export type RenderCompactionTranscriptOptions = {
+  /**
+   * Hard cap for a rendered transcript. When set, oldest rendered items are
+   * dropped first while preserving the final checkpoint prompt; if the retained
+   * item itself is too large, it is middle-truncated instead of looping.
+   */
+  maxEstimatedTokens?: number;
+};
+
+export function renderCompactionPromptInputForChat(
+  input: readonly CompactionItem[],
+  options: RenderCompactionTranscriptOptions = {},
+): string {
+  const rendered = input.map(renderItem);
+  if (typeof options.maxEstimatedTokens !== "number" || options.maxEstimatedTokens <= 0) {
+    return rendered.join("\n");
+  }
+  const maxChars = Math.max(1, Math.floor(options.maxEstimatedTokens * 4));
+  const last = rendered.at(-1);
+  const prefix = rendered.slice(0, -1);
+  let kept = prefix.slice();
+  while (kept.length > 0 && transcriptLength([...kept, last].filter((line): line is string => line !== undefined)) > maxChars) {
+    kept.shift();
+  }
+  const lines = [...kept, last].filter((line): line is string => line !== undefined);
+  const joined = lines.join("\n");
+  if (joined.length <= maxChars) {
+    return joined;
+  }
+  return truncateMiddleByChars(joined, maxChars);
 }
 
 function renderItem(item: CompactionItem): string {
@@ -405,3 +533,23 @@ function truncateForTranscript(text: string, max: number): string {
   }
   return `${text.slice(0, max)}... (${text.length - max} more chars)`;
 }
+
+function transcriptLength(lines: readonly string[]): number {
+  if (lines.length === 0) {
+    return 0;
+  }
+  return lines.reduce((sum, line) => sum + line.length, 0) + Math.max(0, lines.length - 1);
+}
+
+function truncateMiddleByChars(text: string, maxChars: number): string {
+  if (text.length <= maxChars) {
+    return text;
+  }
+  if (maxChars <= USER_MESSAGE_TRUNCATION_MARKER.length) {
+    return USER_MESSAGE_TRUNCATION_MARKER.slice(0, maxChars);
+  }
+  const keepChars = maxChars - USER_MESSAGE_TRUNCATION_MARKER.length;
+  const headChars = Math.ceil(keepChars / 2);
+  const tailChars = Math.floor(keepChars / 2);
+  return `${text.slice(0, headChars)}${USER_MESSAGE_TRUNCATION_MARKER}${text.slice(text.length - tailChars)}`;
+}

package/src/history-sanitizer.ts

@@ -27,6 +27,8 @@
  * filtered, keeping the persisted audit trail intact.
  */
 
+import { SCREENSHOT_FAILURE_CARD_IMAGE_URL } from "./screenshot-error-card";
+
 /** A history item is any JSON object; we only inspect a few discriminator fields. */
 export type HistoryItem = Record<string, unknown>;
 
@@ -36,7 +38,7 @@ export type HistoryItem = Record<string, unknown>;
  * the one observed live, the rest are included so the same pairing logic holds
  * for every tool-call kind the SDK can emit.
  */
-const RESULT_TYPE_BY_CALL_TYPE: Record<string, string> = {
+export const TOOL_CALL_RESULT_TYPE_BY_CALL_TYPE: Record<string, string> = {
   function_call: "function_call_result",
   computer_call: "computer_call_result",
   shell_call: "shell_call_output",
@@ -50,6 +52,7 @@ const RESULT_TYPE_BY_CALL_TYPE: Record<string, string> = {
   tool_search_call: "tool_search_output",
 };
 
+const RESULT_TYPE_BY_CALL_TYPE = TOOL_CALL_RESULT_TYPE_BY_CALL_TYPE;
 const RESULT_TYPES = new Set(Object.values(RESULT_TYPE_BY_CALL_TYPE));
 
 function itemType(item: unknown): string | undefined {
@@ -594,42 +597,35 @@ export function rewriteComputerCallsToActionsOnly(body: unknown): boolean {
 }
 
 /**
- * The 1×1 transparent PNG placeholder used by the SDK for tool-approval-rejection
- * screenshots (`TOOL_APPROVAL_REJECTION_SCREENSHOT_DATA_URL` in agents-core
- * `toolExecution.mjs`). We reuse the exact same constant as a backstop for the
- * action-timeout 400: when an action times out the SDK's catch sets output='' and
- * builds `{type:"computer_call_output",output:{type:"computer_screenshot",image_url:""}}`.
- * Azure rejects `image_url:""` with "400 Invalid input[N].output.image_url". This
- * placeholder is a valid data URI the provider accepts, so the turn continues and
- * the model receives the next real screenshot on its following step.
- */
-const EMPTY_IMAGE_URL_PLACEHOLDER =
-  "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==";
-
-/**
- * Backstop for the action-timeout 400: walk the `input` array of a serialized
- * Responses request body and replace any `computer_call_output` item whose
- * `output.image_url` is an empty string, null, undefined, or otherwise not a
- * non-empty string with the 1×1 transparent PNG placeholder data URI.
- *
- * WHY THIS IS NEEDED. When a computer ACTION (click/type/scroll/drag) times out
- * at the 15-second yield window `SandboxComputer.x()` throws `ComputerActionError`.
- * The agents-core SDK `toolExecution.mjs` catch block sets `output = ''` and then
- * builds the wire item:
- *
- *   `{type:"computer_call_output", output:{type:"computer_screenshot", image_url:""}}`
- *
- * Azure rejects the whole request with:
- *
- *   `400 Invalid 'input[N].output.image_url'. Expected a valid URL, but got a
- *    value with an invalid format.`
- *
- * Our screenshot() fail-loud guard (which throws on empty frames) only runs when
- * the SDK calls screenshot() on a SUCCESS path — not on this action-error catch
- * path that sets output='' directly. This wire-level rewrite is the only seam that
- * catches both paths regardless of how the empty image_url was produced. It runs
- * in the same `computerCallNormalizingFetch` wrapper, so a single parse/rewrite
- * pass covers both the action/actions-only rewrite and this placeholder injection.
+ * Backstop for the empty `computer_call_output` image_url: walk the `input` array of
+ * a serialized Responses request body and replace any `computer_call_output` item
+ * whose `output.image_url` is empty/missing with a LEGIBLE "screen capture failed"
+ * error card ({@link SCREENSHOT_FAILURE_CARD_IMAGE_URL}).
+ *
+ * WHY A CARD, NOT A BLANK. An empty `image_url` reaches this seam ONLY when the
+ * computer op genuinely FAILED to produce a screen: agents-core's `toolExecution.mjs`
+ * catch sets `output = ''` when the action OR the follow-up `computer.screenshot()`
+ * throws, building `{type:"computer_call_output",output:{type:"computer_screenshot",
+ * image_url:""}}`. Azure then rejects the whole request with
+ * `400 Invalid 'input[N].output.image_url'`. The previous fix substituted a 1×1
+ * TRANSPARENT placeholder to dodge the 400 — but that reaches the model as a
+ * plausible BLANK DESKTOP it confidently reports ("the screen appears blank/empty"),
+ * turning a hard capture FAILURE into a silent, wrong observation. That is the worst
+ * failure mode for computer use, and it is exactly what the 0.1.3 TCC-denied incident
+ * produced. Substituting a legible error card instead makes the failure REACH THE
+ * MODEL as an error (the only channel the hosted `computer_use_preview` protocol has
+ * is the image), so the model stops and tells the user rather than hallucinating.
+ *
+ * WHY THIS IS SAFE (empty = failure, never an intentional blank). Post-af289e3 the
+ * intentional-blank cases carry a NON-empty data URI already: agents-core's
+ * tool-approval-rejection screenshot is its own non-empty 1×1 placeholder, and the
+ * SandboxComputer action-timeout now warn+returns to a REAL screenshot rather than an
+ * empty output. So an EMPTY image_url at this seam is unambiguously a capture/interact
+ * FAILURE — the error card is the correct substitution for every empty case, and this
+ * function never touches a non-empty (real screenshot OR intentional blank) output.
+ *
+ * The failure REASON (permission denied / null image / timeout / display down) is not
+ * on the card; it is logged worker-side by `NativeDesktopComputer.screenshot()`.
  *
  * Mutates `body` in place (the caller has already JSON.parsed a private copy).
  * Returns `true` iff at least one image_url was replaced.
@@ -657,9 +653,11 @@ export function rewriteEmptyComputerCallOutputImageUrls(body: unknown): boolean
     }
     const out = output as Record<string, unknown>;
     const imageUrl = out.image_url;
-    // Replace the image_url when it is not a non-empty string (covers: "", null, undefined, missing).
+    // Replace the image_url when it is not a non-empty string (covers: "", null,
+    // undefined, missing) — an empty output is always a genuine capture failure, so
+    // it becomes the legible error card, never a silent blank.
     if (typeof imageUrl !== "string" || imageUrl.length === 0) {
-      out.image_url = EMPTY_IMAGE_URL_PLACEHOLDER;
+      out.image_url = SCREENSHOT_FAILURE_CARD_IMAGE_URL;
       changed = true;
     }
   }