@opengeni/runtime 0.2.2 → 0.3.0

package/src/image-history.ts

@@ -0,0 +1,149 @@
+import type { AgentInputItem } from "@openai/agents";
+
+export const SCREENSHOT_OMITTED_PLACEHOLDER =
+  "[screenshot omitted: an older desktop frame — the full image remains in the session event log]";
+
+const DATA_IMAGE_BASE64_PATTERN = /data:image\/[a-z0-9.+-]+;base64,[a-z0-9+/=_-]+/i;
+
+type PathSegment = string | number;
+
+type ImageOccurrence = {
+  path: PathSegment[];
+  replacement: unknown;
+};
+
+export type ElideStaleScreenshotsResult<T> = {
+  items: T[];
+  imageCount: number;
+  elidedCount: number;
+};
+
+export type ElideStaleScreenshotsOptions = {
+  keepLast?: number;
+  placeholder?: string;
+};
+
+export function elideStaleScreenshotImages<T extends AgentInputItem>(
+  items: readonly T[],
+  options: ElideStaleScreenshotsOptions = {},
+): ElideStaleScreenshotsResult<T> {
+  const keepLast = Math.max(0, Math.floor(options.keepLast ?? 3));
+  const placeholder = options.placeholder ?? SCREENSHOT_OMITTED_PLACEHOLDER;
+  const occurrences: ImageOccurrence[] = [];
+  for (let i = 0; i < items.length; i += 1) {
+    collectItemImageOccurrences(items[i], [i], placeholder, occurrences);
+  }
+
+  const elidedCount = Math.max(0, occurrences.length - keepLast);
+  if (elidedCount === 0) {
+    return { items: items.slice(), imageCount: occurrences.length, elidedCount: 0 };
+  }
+
+  const cloned = structuredClone(items) as T[];
+  for (const occurrence of occurrences.slice(0, elidedCount)) {
+    setPath(cloned, occurrence.path, occurrence.replacement);
+  }
+  return { items: cloned, imageCount: occurrences.length, elidedCount };
+}
+
+function collectItemImageOccurrences(
+  item: unknown,
+  path: PathSegment[],
+  placeholder: string,
+  out: ImageOccurrence[],
+): void {
+  if (!isRecord(item)) {
+    return;
+  }
+  if (item.type === "message" && (item.role === "user" || item.role === "system")) {
+    return;
+  }
+  if (item.type === "computer_call_result" || item.type === "computer_call_output") {
+    collectComputerOutputImages(item, path, placeholder, out);
+    return;
+  }
+  if (item.type === "function_call_result" || item.type === "function_call_output") {
+    collectToolResultImages(item.output, [...path, "output"], placeholder, out);
+  }
+}
+
+function collectComputerOutputImages(
+  item: Record<string, unknown>,
+  path: PathSegment[],
+  placeholder: string,
+  out: ImageOccurrence[],
+): void {
+  const output = item.output;
+  if (!isRecord(output) || output.type !== "computer_screenshot") {
+    return;
+  }
+  for (const key of ["data", "image_url", "imageUrl"]) {
+    if (isImageDataUrl(output[key])) {
+      out.push({ path: [...path, "output", key], replacement: placeholder });
+      return;
+    }
+  }
+}
+
+function collectToolResultImages(
+  value: unknown,
+  path: PathSegment[],
+  placeholder: string,
+  out: ImageOccurrence[],
+): void {
+  if (typeof value === "string") {
+    if (isImageDataUrl(value)) {
+      out.push({ path, replacement: placeholder });
+    }
+    return;
+  }
+  if (Array.isArray(value)) {
+    for (let i = 0; i < value.length; i += 1) {
+      collectToolResultImages(value[i], [...path, i], placeholder, out);
+    }
+    return;
+  }
+  if (!isRecord(value)) {
+    return;
+  }
+  if (value.type === "input_image") {
+    for (const key of ["image", "imageUrl", "image_url"]) {
+      if (isImageDataUrl(value[key])) {
+        out.push({ path, replacement: { type: "input_text", text: placeholder } });
+        return;
+      }
+    }
+  }
+  for (const key of ["content", "text", "output"]) {
+    if (key in value) {
+      collectToolResultImages(value[key], [...path, key], placeholder, out);
+    }
+  }
+}
+
+function isImageDataUrl(value: unknown): value is string {
+  return typeof value === "string" && DATA_IMAGE_BASE64_PATTERN.test(value);
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return Boolean(value) && typeof value === "object" && !Array.isArray(value);
+}
+
+function setPath(root: unknown, path: PathSegment[], value: unknown): void {
+  if (path.length === 0) {
+    return;
+  }
+  let cursor = root;
+  for (let i = 0; i < path.length - 1; i += 1) {
+    const segment = path[i]!;
+    cursor = Array.isArray(cursor)
+      ? cursor[segment as number]
+      : (cursor as Record<string, unknown>)[segment as string];
+  }
+  const last = path[path.length - 1]!;
+  if (Array.isArray(cursor)) {
+    cursor[last as number] = value;
+  } else {
+    (cursor as Record<string, unknown>)[last as string] = value;
+  }
+}

package/src/index.ts

@@ -1,5 +1,5 @@
 import type { ConfiguredModel, ContextCompactionMode, ModelProviderApi, ResolvedModelProvider, Settings } from "@opengeni/config";
-import { AGENT_INSTRUCTIONS_CORE_PLACEHOLDER, collectSandboxEnvironment, contextServerCompactThreshold, firstPartyMcpBaseUrl, parseExposedPorts, resolveContextCompactionMode, resolveModelProvider, sandboxLifecycleHookIds } from "@opengeni/config";
+import { AGENT_INSTRUCTIONS_CORE_PLACEHOLDER, collectSandboxEnvironment, contextInputBudgetTokens, contextServerCompactThreshold, firstPartyMcpBaseUrl, parseExposedPorts, resolveContextCompactionMode, resolveModelProvider, sandboxLifecycleHookIds } from "@opengeni/config";
 import { CAPABILITY_DESCRIPTORS, isClearedRunStateBlob, signDelegatedAccessToken, type Permission, type ReasoningEffort, type ResourceRef, type SessionEventType, type ToolRef } from "@opengeni/contracts";
 import {
   Agent,
@@ -82,8 +82,17 @@ import { dirname, isAbsolute, join, posix as posixPath, relative } from "node:pa
 import { fileURLToPath } from "node:url";
 
 import { computerCallNormalizingFetch, normalizeComputerCallActions, sanitizeHistoryItemsForModel } from "./history-sanitizer";
+import { elideStaleScreenshotImages } from "./image-history";
 import { installCodexToolSearch } from "./codex-tool-search";
-import { enforceInputBudget, estimateItemTokens } from "./context-compaction";
+import {
+  CompactionNeededError,
+  SUMMARY_BUFFER_TOKENS,
+  clientCompactionThresholdTokens,
+  enforceInputBudget,
+  estimateItemTokens,
+  estimateTokens,
+  renderCompactionPromptInputForChat,
+} from "./context-compaction";
 import {
   createSandboxClient,
   deserializeSandboxSessionStateEnvelope,
@@ -134,22 +143,34 @@ export type { HistoryItem } from "./history-sanitizer";
 export { OpenAIChatCompletionsModel, OpenAIResponsesModel } from "@openai/agents";
 
 export {
-  planCompaction,
+  CompactionNeededError,
+  buildCompactionPromptInput,
+  buildCompactionReplacementHistory,
+  clientCompactionThresholdTokens,
+  decideClientCompaction,
   enforceInputBudget,
   buildSummaryItem,
-  buildCompactionMessages,
+  findCompactionNeededError,
   isCompactionSummary,
   isUserMessage,
   findKeepBoundary,
   estimateTokens,
   estimateItemTokens,
-  compactionSummaryText,
-  renderPrefixTranscript,
+  renderCompactionPromptInputForChat,
   COMPACTION_SUMMARY_MARKER,
+  COMPACTION_PROMPT,
+  COMPACT_USER_MESSAGE_MAX_TOKENS,
+  CLIENT_COMPACTION_TRIGGER_FRACTION,
+  SUMMARY_BUFFER_TOKENS,
   SUMMARY_PREFIX,
-  SUMMARY_INSTRUCTIONS,
+  USER_MESSAGE_TRUNCATION_MARKER,
 } from "./context-compaction";
-export type { CompactionItem, CompactionPlan, PlanCompactionInput } from "./context-compaction";
+export type { ClientCompactionDecision, CompactionItem } from "./context-compaction";
+export {
+  elideStaleScreenshotImages,
+  SCREENSHOT_OMITTED_PLACEHOLDER,
+} from "./image-history";
+export type { ElideStaleScreenshotsOptions, ElideStaleScreenshotsResult } from "./image-history";
 
 ensureReadableStreamFrom();
 
@@ -420,7 +441,7 @@ export class MultiProviderModelProvider implements ModelProvider {
 
   async getModel(modelName?: string): Promise<Model> {
     if (modelName) {
-      const resolved = resolveTurnModel(this.settings, modelName);
+      const resolved = resolveTurnModel(settingsForRunScopedModelResolution(this.settings, modelName), modelName);
       if (resolved) {
         // Fail-loud floor (defense in depth): a `codex/<slug>` id must only ever
         // resolve through the synthetic codex-subscription provider (which installs
@@ -458,6 +479,27 @@ export class MultiProviderModelProvider implements ModelProvider {
   }
 }
 
+function settingsForRunScopedModelResolution(settings: Settings, modelName: string): Settings {
+  if (modelName !== settings.openaiModel) {
+    return settings;
+  }
+  const builtinAllowed = splitOpenaiAllowedModels(settings.openaiAllowedModels);
+  const fallbackBuiltin = builtinAllowed.find((id) => id !== modelName);
+  if (!fallbackBuiltin) {
+    return settings;
+  }
+  // The worker sets runSettings.openaiModel to the turn's model. For namespaced
+  // registry ids configuredModels filters the built-in entry out, but a unique
+  // bare registry id would otherwise be claimed by the built-in only because of
+  // that per-turn override. Resolve the run-scoped router against the deployment
+  // allow-list head instead; real built-in models stay in the allow-list.
+  return builtinAllowed.includes(modelName) ? settings : { ...settings, openaiModel: fallbackBuiltin };
+}
+
+function splitOpenaiAllowedModels(value: string): string[] {
+  return value.split(",").map((item) => item.trim()).filter(Boolean);
+}
+
 /**
  * A `codex/<slug>` turn reached the model router but the workspace has no active
  * Codex subscription connected (the worker overlay never injected the synthetic
@@ -500,10 +542,10 @@ export function configureOpenAI(settings: Settings): void {
 
 /**
  * Run the compaction summarizer as one plain, tool-less, non-streaming model
- * call against the resolved provider. `system`/`user` come from
- * buildCompactionMessages. Returns the trimmed summary text, or null on any
+ * call against the resolved provider. `input` is the active history plus
+ * Codex's checkpoint prompt. Returns the trimmed summary text, or null on any
  * failure (the caller treats a failed summarize as "skip compaction this turn"
- * — never fatal). The call deliberately does NOT request reasoning encryption,
+ * - never fatal). The call deliberately does NOT request reasoning encryption,
  * tools, or server-side compaction; it is a self-contained summarize.
  *
  * Provider-aware: the summary always runs on the SAME provider that serves the
@@ -517,22 +559,19 @@ export function configureOpenAI(settings: Settings): void {
  */
 export async function summarizeForCompaction(
   settings: Settings,
-  messages: { system: string; user: string },
+  input: Array<Record<string, unknown>>,
   options: { client?: OpenAI; api?: ModelProviderApi; maxOutputTokens?: number; model?: string } = {},
 ): Promise<string | null> {
   const client = options.client ?? buildOpenAIClientFromSettings(settings);
   const api = options.api ?? "responses";
   const model = options.model ?? settings.openaiModel;
-  const maxTokens = options.maxOutputTokens ?? settings.contextSummaryMaxTokens;
+  const maxTokens = options.maxOutputTokens ?? SUMMARY_BUFFER_TOKENS;
   try {
     if (api === "chat") {
       const completion = await client.chat.completions.create({
         model,
         max_tokens: maxTokens,
-        messages: [
-          { role: "system", content: messages.system },
-          { role: "user", content: messages.user },
-        ],
+        messages: [{ role: "user", content: renderCompactionPromptInputForChat(input) }],
       } as any);
       const text = (completion as { choices?: Array<{ message?: { content?: unknown } }> }).choices?.[0]?.message?.content;
       const trimmed = typeof text === "string" ? text.trim() : "";
@@ -545,10 +584,7 @@ export async function summarizeForCompaction(
       // built-in path (api "responses"), so gate it on the built-in provider.
       ...(settings.openaiProvider === "azure" ? {} : { store: false }),
       max_output_tokens: maxTokens,
-      input: [
-        { role: "system", content: messages.system },
-        { role: "user", content: messages.user },
-      ],
+      input,
     } as any);
     const text = extractResponseOutputText(response);
     const trimmed = text.trim();
@@ -696,6 +732,14 @@ export type BuildAgentOptions = {
   // restyle the persona but never drop the goal-loop contract or environment
   // block.
   instructionsTemplate?: string;
+  // Per-SESSION persona/system instructions (the per-agent-type prompt lever an
+  // embedding host supplies at session create). Composed AFTER the workspace
+  // instructionsTemplate + the non-bypassable CORE, so it refines the workspace
+  // persona for this one session without dropping the goal-loop/environment
+  // contract. Rides the SAME instructions channel (system-level) — NEVER a user/
+  // timeline message. Omitted ⇒ the composed instructions are byte-identical to
+  // a workspace-only persona.
+  sessionInstructions?: string;
   // Skills delivered by enabled capability packs. They join the bundled
   // skills in the sandbox skill index (mounted under .agents/) so
   // skills/<name> references resolve like any other indexed skill.
@@ -778,6 +822,27 @@ export function composeAgentInstructions(template: string, workspaceEnvironment?
   return core ? `${template} ${core}` : template;
 }
 
+/**
+ * Appends the per-session persona instructions to the already-composed
+ * (workspace + CORE) instructions, joined by " " — exactly the join used
+ * throughout the persona composition. The session slice is intentionally LAST
+ * (session-specific refinement of the workspace persona). An absent/blank value
+ * is a no-op that returns the composed string byte-for-byte.
+ */
+export function appendSessionInstructions(composed: string, sessionInstructions?: string): string {
+  const trimmed = sessionInstructions?.trim();
+  return trimmed ? `${composed} ${trimmed}` : composed;
+}
+
+/**
+ * Appends the one-shot genesis title directive (genesis turn only), joined by
+ * " " and always LAST so a white-label persona template or a per-session
+ * instruction can't drop it. A no-op when the hint is absent.
+ */
+export function appendGenesisTitleDirective(instructions: string, genesisTitleHint?: boolean): string {
+  return genesisTitleHint ? `${instructions} ${GENESIS_TITLE_DIRECTIVE}` : instructions;
+}
+
 const agentFileDownloads = new WeakMap<object, SandboxFileDownload[]>();
 const agentRepositoryCloneHooks = new WeakMap<object, SandboxLifecycleHook[]>();
 // TOKEN-BROKER (B1): the per-turn git token seed, stashed alongside the agent's
@@ -822,9 +887,21 @@ export function buildOpenGeniAgent(settings: Settings, resources: ResourceRef[],
     // ownership + workspace-environment block) at the {{core}} marker, or
     // appends it when the template omits the marker. With the default template
     // and no environment this is byte-identical to the historical preamble.
-    instructions: options.genesisTitleHint
-      ? `${composeAgentInstructions(options.instructionsTemplate ?? settings.agentInstructionsTemplate, options.workspaceEnvironment)} ${GENESIS_TITLE_DIRECTIVE}`
-      : composeAgentInstructions(options.instructionsTemplate ?? settings.agentInstructionsTemplate, options.workspaceEnvironment),
+    // Persona composition order (all one system-level instructions string):
+    //   1. workspace instructionsTemplate (or deployment default) with the
+    //      non-bypassable CORE substituted at {{core}} — composeAgentInstructions,
+    //   2. + the per-session persona instructions (session-specific, LAST so it
+    //      refines the workspace persona),
+    //   3. + the one-shot genesis title directive (genesis turn only).
+    // With no session instructions and no genesis hint this is byte-identical to
+    // the historical composed instructions.
+    instructions: appendGenesisTitleDirective(
+      appendSessionInstructions(
+        composeAgentInstructions(options.instructionsTemplate ?? settings.agentInstructionsTemplate, options.workspaceEnvironment),
+        options.sessionInstructions,
+      ),
+      options.genesisTitleHint,
+    ),
     modelSettings: {
       reasoning: { effort: options.reasoningEffort ?? settings.openaiReasoningEffort, summary: "detailed" },
       // Server-side compaction (OpenAI platform) requires store=false: the
@@ -1573,6 +1650,7 @@ export type RunAgentStreamOptions = {
   sandboxClient?: unknown;
   sandboxEnvironment?: Record<string, string>;
   onRuntimeEvent?: (event: NormalizedRuntimeEvent) => Promise<void> | void;
+  contextCompactionSignalTokens?: () => number | null | undefined;
   // OWNERSHIP INVERSION (P1.2): an externally-owned, already-live sandbox
   // session resolved by the per-turn resume-by-id path. When present,
   // runAgentStream does NOT build (or resume, or discard) a client — it threads
@@ -1603,6 +1681,11 @@ export type RunAgentStreamOptions = {
   callModelInputFilter?: CallModelInputFilter;
 };
 
+export type ContextRobustnessFilterOptions = {
+  contextCompactionSignalTokens?: () => number | null | undefined;
+  throwOnCompactionNeeded?: boolean;
+};
+
 // One-shot directive appended to the agent's system prompt on the genesis turn
 // (see buildOpenGeniAgent's genesisTitleHint). Delivered through the
 // authoritative instructions channel so the model reliably obeys; references
@@ -1656,6 +1739,59 @@ export const normalizeComputerCallsFilter: CallModelInputFilter = ({ modelData }
   ) as unknown as AgentInputItem[],
 });
 
+export function contextRobustnessFilterForSettings(
+  settings: Settings,
+  options: ContextRobustnessFilterOptions = {},
+): CallModelInputFilter {
+  const inputBudgetTokens = modelCallBudgetTokens(settings);
+  const clientCompactionMode = resolveContextCompactionMode(settings) === "client";
+  const compactionThresholdTokens = clientCompactionThresholdTokens(settings);
+  return ({ modelData }) => {
+    const images = elideStaleScreenshotImages(modelData.input);
+    if (images.elidedCount > 0) {
+      console.warn(
+        `per-call image history policy elided ${images.elidedCount} older screenshot image(s), keeping the last ${Math.min(3, images.imageCount)} full image(s)`,
+      );
+    }
+    let input = images.items;
+    if (inputBudgetTokens !== undefined) {
+      const guarded = enforceInputBudget(
+        input as unknown as Array<Record<string, unknown>>,
+        inputBudgetTokens,
+      );
+      if (guarded.trimmed) {
+        console.warn(
+          `per-call budget guard trimmed ${guarded.droppedCount} oldest history item(s) to fit input budget (${inputBudgetTokens} tokens); the over-budget model call was NOT sent`,
+        );
+        input = guarded.items as unknown as AgentInputItem[];
+      }
+    }
+    if (clientCompactionMode && options.throwOnCompactionNeeded) {
+      const reported = options.contextCompactionSignalTokens?.();
+      const hasReported = typeof reported === "number" && reported > 0;
+      const signalTokens = hasReported
+        ? reported
+        : estimateTokens(input as unknown as Array<Record<string, unknown>>);
+      if (signalTokens > compactionThresholdTokens) {
+        throw new CompactionNeededError({
+          signalTokens,
+          thresholdTokens: compactionThresholdTokens,
+          signalSource: hasReported ? "provider" : "estimate",
+        });
+      }
+    }
+    return { ...modelData, input };
+  };
+}
+
+function modelCallBudgetTokens(settings: Settings): number | undefined {
+  if (resolveContextCompactionMode(settings) !== "client") {
+    return undefined;
+  }
+  const budget = contextInputBudgetTokens(settings);
+  return budget > 0 ? budget : undefined;
+}
+
 /**
  * Compose a list of callModelInputFilters into one, applied left-to-right so
  * each sees the prior filter's output.
@@ -1674,13 +1810,18 @@ function composeCallModelInputFilters(filters: CallModelInputFilter[]): CallMode
  * The model-input filter applied before every model call. The computer_call
  * action/actions normalizer is ALWAYS on (the Azure endpoint 400s without it);
  * the provider-item-id strip is layered on top when the configured policy
- * selects it.
+ * selects it; the context-robustness guard then elides stale screenshots on
+ * every mode and applies hard budget trimming only on the client-compaction path.
  */
-export function callModelInputFilterForSettings(settings: Settings): CallModelInputFilter | undefined {
+export function callModelInputFilterForSettings(
+  settings: Settings,
+  options: ContextRobustnessFilterOptions = {},
+): CallModelInputFilter | undefined {
   const filters: CallModelInputFilter[] = [normalizeComputerCallsFilter];
   if (settings.openaiProviderItemIds === "strip") {
     filters.push(stripProviderItemIdsFilter);
   }
+  filters.push(contextRobustnessFilterForSettings(settings, options));
   return composeCallModelInputFilters(filters);
 }
 
@@ -1759,7 +1900,15 @@ export async function runAgentStream(agent: Agent<any, any>, input: PreparedAgen
     // through the client during this run (it is inert for the provided session).
     const decoratedClient = withSandboxLifecycleHooks(resourceClient, ownedHooks, ownedHookContext);
     const ownedFilter = composeCallModelInputFilters(
-      [callModelInputFilterForSettings(settings), overrides.callModelInputFilter].filter(
+      [
+        callModelInputFilterForSettings(settings, {
+          throwOnCompactionNeeded: Boolean(overrides.contextCompactionSignalTokens),
+          ...(overrides.contextCompactionSignalTokens
+            ? { contextCompactionSignalTokens: overrides.contextCompactionSignalTokens }
+            : {}),
+        }),
+        overrides.callModelInputFilter,
+      ].filter(
         (f): f is CallModelInputFilter => Boolean(f),
       ),
     );
@@ -1806,23 +1955,31 @@ export async function runAgentStream(agent: Agent<any, any>, input: PreparedAgen
     ?? (prepared.serializedRunStateForSandbox && client
       ? await restoredSandboxSessionState(await RunState.fromString(agent, prepared.serializedRunStateForSandbox), client)
       : undefined);
-  // Strip provider item ids first, then apply any per-turn filter (genesis
-  // title directive). Composed left-to-right so the directive lands on the
-  // already-id-stripped input. A callModelInputFilter only shapes the per-call
-  // model input, never the persisted run-state history.
+  // Apply the built-in per-call filters (computer-call normalization, optional
+  // provider-id stripping, image/budget guard), then any per-turn filter
+  // (genesis title directive). A callModelInputFilter only shapes the per-call
+  // model input; the SDK persists filtered clones into its session view, while
+  // OpenGeni's durable conversation truth is still reconciled explicitly below.
   const callModelInputFilter = composeCallModelInputFilters(
-    [callModelInputFilterForSettings(settings), overrides.callModelInputFilter].filter(
+    [
+      callModelInputFilterForSettings(settings, {
+        throwOnCompactionNeeded: Boolean(overrides.contextCompactionSignalTokens),
+        ...(overrides.contextCompactionSignalTokens
+          ? { contextCompactionSignalTokens: overrides.contextCompactionSignalTokens }
+          : {}),
+      }),
+      overrides.callModelInputFilter,
+    ].filter(
       (f): f is CallModelInputFilter => Boolean(f),
     ),
   );
   const runOptions: Parameters<typeof run>[2] = {
     stream: true,
     maxTurns: settings.agentMaxModelCallsPerTurn,
-    // Strip provider-assigned item ids from every model call (turn-start
-    // history replay AND mid-turn follow-ups) so requests never depend on the
-    // provider's server-side response store. A stored response can vanish
-    // between two calls of the same turn, failing the run with 400 "Item with
-    // id 'rs_…' not found"; with the ids gone the request is self-contained.
+    // Built-in per-call guard chain: normalize computer calls, optionally strip
+    // provider ids, elide stale screenshots in every mode, and trim to the input
+    // budget on the client-compaction path. This runs for turn-start replay AND
+    // every mid-turn follow-up.
     callModelInputFilter,
   };
   void settings.disableOpenaiTracing;