@opengeni/runtime 0.2.0 → 0.2.2

package/dist/index.d.ts

@@ -70,12 +70,31 @@ declare class SandboxComputer implements Computer {
     drag(path: [number, number][]): Promise<void>;
     wait(): Promise<void>;
 }
+/**
+ * EXPLICIT tool-transport selection, decided by the caller that knows the
+ * provider's true wire identity (the worker's model resolution — see agent-turn.ts),
+ * NOT inferred from the bound model instance's constructor name. This is the
+ * HARDENING seam: `supportsStructuredToolOutputTransport` string-sniffs the
+ * constructor for "ChatCompletions", which a wrapped / proxied / minified model
+ * instance would defeat — silently handing a chat-completions provider the HOSTED
+ * `computer_use_preview` tool it 400s on every turn. When `toolMode` is set, tools()
+ * OBEYS it and never consults the sniff:
+ *   • "hosted"         → the single hosted `computer_use_preview` tool (Responses backends).
+ *   • "function-image" → the FUNCTION `computer_*` tools with screenshots delivered as a
+ *                        structured `{type:'image'}` output (the codex/ChatGPT backend,
+ *                        which rejects hosted tool types but SEES structured image results).
+ *   • "function-text"  → the FUNCTION tools with screenshots rendered as a text
+ *                        `data:…;base64` URL (chat-completions providers, which can't read
+ *                        structured image tool results).
+ */
+type ComputerToolMode = "hosted" | "function-image" | "function-text";
 type ComputerUseArgs = {
     dimensions?: [number, number];
     readOnly?: boolean;
     display?: string;
     needsApproval?: boolean | ((ctx: unknown, action: unknown) => boolean | Promise<boolean>);
     imageFunctionResults?: boolean;
+    toolMode?: ComputerToolMode;
 };
 declare function computerUse(args?: ComputerUseArgs): ComputerUseCapability;
 /**
@@ -101,6 +120,9 @@ declare class ComputerUseCapability extends Capability {
     readonly type = "computer-use";
     constructor(args?: ComputerUseArgs);
     tools(): Tool<unknown>[];
+    /** The single HOSTED `computer_use_preview` tool bound to `computer` — identical
+     *  construction for the explicit "hosted" mode and the legacy structured-sniff path. */
+    private hostedComputerTool;
 }
 
 /**
@@ -632,6 +654,7 @@ type BuildAgentOptions = {
     encryptedReasoning?: boolean;
     contextWindowTokens?: number;
     structuredToolTransport?: boolean;
+    computerToolMode?: ComputerToolMode;
     codexConnectorNamespaces?: ReadonlySet<string>;
     sandboxEnvironment?: Record<string, string>;
     activeSandboxBackend?: Settings["sandboxBackend"];
@@ -716,6 +739,7 @@ declare function buildAgentCapabilities(settings: Settings, packSkills: PackSkil
     compactionMode?: ContextCompactionMode;
     contextWindowTokens?: number;
     structuredToolTransport?: boolean;
+    computerToolMode?: ComputerToolMode;
 }): ReturnType<typeof Capabilities.default>;
 declare function sandboxRunAs(_settings: Settings): string | undefined;
 type PreparedAgentTools = {
@@ -909,4 +933,4 @@ declare function azureOpenAIDefaultQuery(settings: Pick<Settings, "azureOpenaiAp
  */
 declare function lazySkillSourceWithPackSkills(packSkills: PackSkill[]): LocalDirLazySkillSource;
 
-export { type AgentSegmentInput, type BuildAgentOptions, COMPACTION_SUMMARY_MARKER, CodexSubscriptionUnavailableError, type CompactionItem, type CompactionPlan, ComputerActionError, ComputerReadOnlyError, ComputerUnavailableError, type ComputerUseArgs, ComputerUseCapability, GENESIS_TITLE_DIRECTIVE, type HistoryItem, type ModelResponseUsage, MultiProviderModelProvider, type NormalizedRuntimeEvent, type OpenGeniRuntime, type PackSkill, type PackSkillFile, type PlanCompactionInput, type PrepareInputOptions, type PrepareToolsOptions, type PreparedAgentInput, type PreparedAgentTools, type ProductionRuntimeOverrides, type RunAgentStreamOptions, SUMMARY_INSTRUCTIONS, SUMMARY_PREFIX, SandboxComputer, type SandboxComputerOptions, type SandboxFileDownload, type SandboxLifecycleHook, type SandboxLifecycleHookContext, type SandboxLifecycleHookPhase, type WorkspaceEnvironmentContext, agentsErrorRunState, applyMissingManifestEntries, azureCliLoginCommand, azureOpenAIDefaultQuery, buildAgentCapabilities, buildCompactionMessages, buildManifest, buildModelInstance, buildOpenAIClientFromSettings, buildOpenGeniAgent, buildProviderClient, buildSummaryItem, callModelInputFilterForSettings, compactionSummaryText, composeAgentInstructions, computerUse, configureOpenAI, coreInstructions, createProductionAgentRuntime, enforceInputBudget, ensureReadableStreamFrom, estimateItemTokens, estimateTokens, extractResponseOutputText, findKeepBoundary, isCompactionSummary, isUserMessage, lazySkillSourceWithPackSkills, materializeSandboxFileDownloads, maxTurnsExceededRunState, modelResponseUsageFromSdkEvent, neutralizeToolSearchItemsInSerializedRunState, normalizeComputerCallsFilter, normalizeSdkEvent, normalizeToolOutputForEvent, planCompaction, prefixedMcpToolName, prepareAgentTools, prepareRunInput, renderPrefixTranscript, repositoryCloneCommand, repositoryUsesSandboxClone, resolveTurnModel, runAgentStream, runAzureCliLoginHook, runBeforeAgentStartHooks, runRepositoryCloneHook, sandboxCommandExitCode, sandboxCommandOutput, sandboxCommandStillRunning, sandboxFileDownloadsForAgent, sandboxLifecycleHooksForIds, sandboxRunAs, sanitizeHistoryItemsForModel, serializeApprovals, stripProviderItemIdsFilter, stripReasoningEncryptedContent, stripReasoningIdentityFromSerializedRunState, summarizeForCompaction, withManifestRefreshOnResume, withSandboxFileDownloads, withSandboxLifecycleHooks, workspaceEnvironmentInstructions };
+export { type AgentSegmentInput, type BuildAgentOptions, COMPACTION_SUMMARY_MARKER, CodexSubscriptionUnavailableError, type CompactionItem, type CompactionPlan, ComputerActionError, ComputerReadOnlyError, type ComputerToolMode, ComputerUnavailableError, type ComputerUseArgs, ComputerUseCapability, GENESIS_TITLE_DIRECTIVE, type HistoryItem, type ModelResponseUsage, MultiProviderModelProvider, type NormalizedRuntimeEvent, type OpenGeniRuntime, type PackSkill, type PackSkillFile, type PlanCompactionInput, type PrepareInputOptions, type PrepareToolsOptions, type PreparedAgentInput, type PreparedAgentTools, type ProductionRuntimeOverrides, type RunAgentStreamOptions, SUMMARY_INSTRUCTIONS, SUMMARY_PREFIX, SandboxComputer, type SandboxComputerOptions, type SandboxFileDownload, type SandboxLifecycleHook, type SandboxLifecycleHookContext, type SandboxLifecycleHookPhase, type WorkspaceEnvironmentContext, agentsErrorRunState, applyMissingManifestEntries, azureCliLoginCommand, azureOpenAIDefaultQuery, buildAgentCapabilities, buildCompactionMessages, buildManifest, buildModelInstance, buildOpenAIClientFromSettings, buildOpenGeniAgent, buildProviderClient, buildSummaryItem, callModelInputFilterForSettings, compactionSummaryText, composeAgentInstructions, computerUse, configureOpenAI, coreInstructions, createProductionAgentRuntime, enforceInputBudget, ensureReadableStreamFrom, estimateItemTokens, estimateTokens, extractResponseOutputText, findKeepBoundary, isCompactionSummary, isUserMessage, lazySkillSourceWithPackSkills, materializeSandboxFileDownloads, maxTurnsExceededRunState, modelResponseUsageFromSdkEvent, neutralizeToolSearchItemsInSerializedRunState, normalizeComputerCallsFilter, normalizeSdkEvent, normalizeToolOutputForEvent, planCompaction, prefixedMcpToolName, prepareAgentTools, prepareRunInput, renderPrefixTranscript, repositoryCloneCommand, repositoryUsesSandboxClone, resolveTurnModel, runAgentStream, runAzureCliLoginHook, runBeforeAgentStartHooks, runRepositoryCloneHook, sandboxCommandExitCode, sandboxCommandOutput, sandboxCommandStillRunning, sandboxFileDownloadsForAgent, sandboxLifecycleHooksForIds, sandboxRunAs, sanitizeHistoryItemsForModel, serializeApprovals, stripProviderItemIdsFilter, stripReasoningEncryptedContent, stripReasoningIdentityFromSerializedRunState, summarizeForCompaction, withManifestRefreshOnResume, withSandboxFileDownloads, withSandboxLifecycleHooks, workspaceEnvironmentInstructions };

package/dist/index.js

@@ -1449,16 +1449,29 @@ var ComputerUseCapability = class extends Capability {
       // The SDK base exposes the bound runAs as a protected field.
       ...typeof this._runAs === "string" ? { runAs: this._runAs } : {}
     });
+    switch (this.args.toolMode) {
+      case "hosted":
+        return [this.hostedComputerTool(computer)];
+      case "function-image":
+        return computerFunctionTools(computer, this.args.readOnly ?? false, this.args.needsApproval, true);
+      case "function-text":
+        return computerFunctionTools(computer, this.args.readOnly ?? false, this.args.needsApproval, false);
+      case void 0:
+        break;
+    }
     if (supportsStructuredToolOutputTransport(this._modelInstance)) {
-      return [
-        computerTool({
-          computer,
-          ...this.args.needsApproval !== void 0 ? { needsApproval: this.args.needsApproval } : {}
-        })
-      ];
+      return [this.hostedComputerTool(computer)];
     }
     return computerFunctionTools(computer, this.args.readOnly ?? false, this.args.needsApproval, this.args.imageFunctionResults ?? false);
   }
+  /** The single HOSTED `computer_use_preview` tool bound to `computer` — identical
+   *  construction for the explicit "hosted" mode and the legacy structured-sniff path. */
+  hostedComputerTool(computer) {
+    return computerTool({
+      computer,
+      ...this.args.needsApproval !== void 0 ? { needsApproval: this.args.needsApproval } : {}
+    });
+  }
 };
 
 // src/index.ts
@@ -1786,7 +1799,8 @@ function buildOpenGeniAgent(settings, resources, options = {}) {
     capabilities: buildAgentCapabilities(settings, options.packSkills ?? [], {
       compactionMode,
       contextWindowTokens,
-      ...options.structuredToolTransport !== void 0 ? { structuredToolTransport: options.structuredToolTransport } : {}
+      ...options.structuredToolTransport !== void 0 ? { structuredToolTransport: options.structuredToolTransport } : {},
+      ...options.computerToolMode !== void 0 ? { computerToolMode: options.computerToolMode } : {}
     })
   });
   agentFileDownloads.set(agent, normalizeSandboxFileDownloads(options.fileResourceDownloads ?? []).filter((download) => !download.content));
@@ -1828,17 +1842,13 @@ function buildAgentCapabilities(settings, packSkills, options = {}) {
   }
   caps.push(skills({ lazyFrom: lazySkillSourceWithPackSkills(packSkills) }));
   if (settings.computerUseEnabled && settings.sandboxDesktopEnabled && desktopCapableBackend(settings.sandboxBackend)) {
+    const explicitMode = options.computerToolMode;
     const computerCapability = computerUse({
       dimensions: [settings.streamResolutionWidth, settings.streamResolutionHeight],
       readOnly: settings.computerUseReadOnly,
-      // On the codex path the function tools deliver screenshots as a real image the
-      // model can see. The ChatGPT/Codex backend rejects HOSTED tool types but DOES
-      // accept `input_image` content items inside a `function_call_output` (proven by
-      // openai/codex codex-rs, whose view_image tool ships exactly that shape) — so a
-      // structured image tool result is seen, where a text data-URL would be unreadable.
-      ...options.structuredToolTransport === false ? { imageFunctionResults: true } : {}
+      ...explicitMode ? { toolMode: explicitMode } : options.structuredToolTransport === false ? { imageFunctionResults: true } : {}
     });
-    if (options.structuredToolTransport === false) {
+    if (!explicitMode && options.structuredToolTransport === false) {
       neutralizeStructuredToolTransport(computerCapability);
     }
     caps.push(computerCapability);
@@ -1899,7 +1909,7 @@ async function prepareAgentTools(settings, tools, options = {}) {
       }
       const error = connectedBestEffort.errors.get(failed);
       console.warn(
-        `[mcp] optional capability server "${failed.name}" failed to connect/list tools; skipping it for this turn`,
+        `[mcp] optional server "${failed.name}" failed to connect/list tools; skipping it for this turn`,
         error instanceof Error ? error.message : error
       );
     }