@opengeni/runtime 3.2.2 → 3.3.1-canary.38922642121001

package/src/index.ts

@@ -117,6 +117,7 @@ import { createSecretArgumentResolver } from "./secret-argument-resolution";
 import {
   approvalIdentifier,
   INTERACTION_REQUEST_HUMAN_MODEL_TOOL_NAME,
+  RequestHumanInteractionToolInput,
   CAPABILITY_DESCRIPTORS,
   DEFAULT_FIRST_PARTY_MCP_PERMISSIONS,
   DEFAULT_FIRST_PARTY_MCP_TOOLS,
@@ -303,6 +304,7 @@ import {
   applyDiff,
   RunContext,
   ToolGuardrailFunctionOutputFactory,
+  defineToolInputGuardrail,
   type AgentInputItem,
   type CallModelInputFilter,
   type MCPServer,
@@ -3881,6 +3883,23 @@ function activeModelToolInvocation(modelName: string): ModelToolInvocation | nul
   return invocation?.modelName === modelName ? invocation : null;
 }
 
+/** Why interaction_request_human arguments cannot be recorded as an
+ * intervention, in words the model can act on, or null when they can. */
+export function interactionRequestArgumentsProblem(serialized: string): string | null {
+  let value: unknown;
+  try {
+    value = JSON.parse(serialized);
+  } catch {
+    return "Invalid interaction_request_human arguments: they are not valid JSON. Nothing was requested; call it again with a JSON object matching its schema.";
+  }
+  const parsed = RequestHumanInteractionToolInput.safeParse(value);
+  if (parsed.success) return null;
+  const issues = parsed.error.issues
+    .slice(0, 5)
+    .map((issue) => `${issue.path.join(".") || "arguments"}: ${issue.message}`);
+  return `Invalid interaction_request_human arguments: ${issues.join("; ")}. Nothing was requested; call it again with corrected arguments.`;
+}
+
 /**
  * Turn the canonical interaction-request tool into a typed SDK interruption.
  * Its MCP/Codemode catalog entry and execution stay unchanged; this projection
@@ -3903,6 +3922,22 @@ function installInteractionInterventionPolicy(
       const originalInvoke = tool.invoke.bind(tool);
       return {
         ...tool,
+        // Runs before the approval pause, so invalid arguments come back to
+        // the model as a correctable tool result. Without it the call would
+        // pause first, the intervention could never be recorded, and the
+        // turn would fail.
+        inputGuardrails: [
+          ...(tool.inputGuardrails ?? []),
+          defineToolInputGuardrail({
+            name: "validate_interaction_request_human",
+            run: async ({ toolCall }) => {
+              const problem = interactionRequestArgumentsProblem(toolCall.arguments);
+              return problem
+                ? ToolGuardrailFunctionOutputFactory.rejectContent(problem)
+                : ToolGuardrailFunctionOutputFactory.allow();
+            },
+          }),
+        ],
         needsApproval: async (
           _runContext: Parameters<typeof originalNeedsApproval>[0],
           _parsedInput: Parameters<typeof originalNeedsApproval>[1],
@@ -8469,7 +8504,10 @@ export class PrefixedMcpServer implements MCPServer {
           return {
             ...tool,
             name,
-            ...(this.accountLabel
+            // Name the account only when the turn has several accounts of this
+            // connector to tell apart. With one account the line adds nothing
+            // the model can act on, and it would repeat on every tool.
+            ...(this.accountLabel?.account
               ? {
                   description:
                     `Account: ${this.accountLabel.model}. Use only this account; if the intended account for a write is unclear, ask before calling.\n${tool.description ?? ""}`.trimEnd(),

package/src/interaction-tool-json-schema.ts

@@ -0,0 +1,214 @@
+import type { AttemptToolJsonSchema } from "@opengeni/contracts";
+import { z } from "zod";
+
+type JsonSchemaNode = Record<string, unknown>;
+
+/** Keywords that only constrain validation. The tool's Zod schema still
+ * enforces them before execution, and rejects with the exact field. */
+const VALIDATION_ONLY_KEYWORDS = new Set(["$schema", "$id", "minLength", "maxLength"]);
+/** Larger numeric bounds are overflow guards, not guidance. */
+const MEANINGFUL_NUMERIC_BOUND = 100_000;
+const MEANINGFUL_MAX_ITEMS = 64;
+/** Longer patterns (UUIDs, control-character guards) are validation noise. */
+const MEANINGFUL_PATTERN_LENGTH = 32;
+
+/**
+ * The JSON Schema an interaction tool shows the model.
+ *
+ * Model providers expand every `$ref` when they render a tool, so a tagged
+ * union such as a browser action, whose locator union repeats in each of its
+ * twenty variants, costs several thousand tokens. Each tagged union of objects
+ * is shown as one object instead: the tag lists every variant, each field
+ * appears once, and one compact line names the fields each variant takes.
+ * Validation-only limits are left out. The tool's own Zod schema stays the
+ * authority and rejects any invalid combination with the exact field.
+ */
+export function interactionToolInputJsonSchema(schema: z.ZodType): AttemptToolJsonSchema {
+  const generated = z.toJSONSchema(schema, { target: "draft-2020-12" }) as JsonSchemaNode;
+  return modelFacingSchema(generated) as AttemptToolJsonSchema;
+}
+
+/** Per-variant field lists of a flattened union, for the enclosing summary. */
+const flattenedVariants = new WeakMap<JsonSchemaNode, Array<[string, string[]]>>();
+
+function modelFacingSchema(node: unknown): unknown {
+  if (Array.isArray(node)) return node.map(modelFacingSchema);
+  if (!isNode(node)) return node;
+  const out: JsonSchemaNode = {};
+  for (const [key, value] of Object.entries(node)) {
+    if (VALIDATION_ONLY_KEYWORDS.has(key)) continue;
+    if (key === "additionalProperties" && value === false) continue;
+    if (
+      (key === "minimum" || key === "maximum") &&
+      typeof value === "number" &&
+      Math.abs(value) >= MEANINGFUL_NUMERIC_BOUND
+    ) {
+      continue;
+    }
+    if (key === "minItems" && typeof value === "number" && value <= 1) continue;
+    if (key === "maxItems" && typeof value === "number" && value > MEANINGFUL_MAX_ITEMS) continue;
+    if (
+      key === "pattern" &&
+      typeof value === "string" &&
+      value.length > MEANINGFUL_PATTERN_LENGTH
+    ) {
+      continue;
+    }
+    out[key] = modelFacingSchema(value);
+  }
+  for (const unionKey of ["oneOf", "anyOf"] as const) {
+    const members = out[unionKey];
+    if (!Array.isArray(members)) continue;
+    const flattened = flattenTaggedUnion(members);
+    if (!flattened) continue;
+    delete out[unionKey];
+    const { description, ...rest } = flattened;
+    Object.assign(out, rest);
+    const own = typeof out.description === "string" ? out.description : "";
+    out.description = own ? `${own} ${String(description)}` : description;
+    flattenedVariants.set(out, flattenedVariants.get(flattened) ?? []);
+  }
+  return out;
+}
+
+/** One object for a union whose members are objects sharing a string tag.
+ * Null when the union is not such a tagged union. */
+function flattenTaggedUnion(members: unknown[]): JsonSchemaNode | null {
+  if (members.length < 2 || !members.every(isObjectSchema)) return null;
+  const objects = members as Array<JsonSchemaNode & { properties: Record<string, unknown> }>;
+  const tag = ["type", "kind", "operation"].find((candidate) =>
+    objects.every((member) => tagValues(member.properties[candidate]) !== null),
+  );
+  if (!tag) return null;
+
+  const variants: Array<[string, string[]]> = [];
+  const tagValuesInOrder: string[] = [];
+  const properties = new Map<string, unknown[]>();
+  for (const member of objects) {
+    const nested = flattenedVariants.get(member);
+    const required = new Set(Array.isArray(member.required) ? (member.required as string[]) : []);
+    if (nested) {
+      variants.push(...nested);
+    } else {
+      const fields = Object.keys(member.properties)
+        .filter((field) => field !== tag)
+        .map((field) => (required.has(field) ? field : `${field}?`));
+      for (const value of tagValues(member.properties[tag])!) variants.push([value, fields]);
+    }
+    for (const value of tagValues(member.properties[tag])!) {
+      if (!tagValuesInOrder.includes(value)) tagValuesInOrder.push(value);
+    }
+    for (const [field, schema] of Object.entries(member.properties)) {
+      if (field === tag) continue;
+      const seen = properties.get(field) ?? [];
+      if (!seen.some((existing) => sameSchema(existing, schema))) seen.push(schema);
+      properties.set(field, seen);
+    }
+  }
+
+  const merged: Record<string, unknown> = {
+    [tag]: { type: "string", enum: tagValuesInOrder },
+  };
+  for (const [field, schemas] of properties) {
+    merged[field] = schemas.length === 1 ? schemas[0] : mergeAlternatives(schemas);
+  }
+  const summary = variants
+    .map(([value, fields]) => (fields.length > 0 ? `${value}: ${fields.join(", ")}` : value))
+    .join("; ");
+  const flattened: JsonSchemaNode = {
+    type: "object",
+    properties: merged,
+    required: [tag],
+    description: `Fields by ${tag} (? = optional): ${summary}.`,
+  };
+  flattenedVariants.set(flattened, variants);
+  return flattened;
+}
+
+function mergeAlternatives(schemas: unknown[]): unknown {
+  if (schemas.every((schema) => tagValues(schema) !== null)) {
+    return { type: "string", enum: [...new Set(schemas.flatMap((schema) => tagValues(schema)!))] };
+  }
+  return { anyOf: schemas };
+}
+
+function tagValues(schema: unknown): string[] | null {
+  if (!isNode(schema)) return null;
+  if (typeof schema.const === "string") return [schema.const];
+  if (
+    Array.isArray(schema.enum) &&
+    schema.enum.length > 0 &&
+    schema.enum.every((value) => typeof value === "string")
+  ) {
+    return schema.enum as string[];
+  }
+  return null;
+}
+
+function isObjectSchema(schema: unknown): boolean {
+  return isNode(schema) && schema.type === "object" && isNode(schema.properties);
+}
+
+function sameSchema(left: unknown, right: unknown): boolean {
+  return JSON.stringify(left) === JSON.stringify(right);
+}
+
+function isNode(value: unknown): value is JsonSchemaNode {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+const MAX_REPORTED_ISSUES = 4;
+const MAX_ISSUE_MESSAGE_CHARS = 240;
+
+/**
+ * Name exactly what is wrong with invalid tool arguments. A union failure
+ * reports the variant the arguments chose (the one whose tag matched), so a
+ * click without a locator role says `action.locator.role`, not only "invalid".
+ */
+export function describeInvalidToolArguments(error: z.ZodError): string {
+  const issues = leafIssues(error.issues, []).slice(0, MAX_REPORTED_ISSUES);
+  const described = issues.map(({ path, message }) => {
+    const text = message.replace(/^Invalid input: /u, "").slice(0, MAX_ISSUE_MESSAGE_CHARS);
+    return path.length > 0 ? `${path.join(".")}: ${text}` : text;
+  });
+  return `Interaction tool arguments are invalid: ${described.join("; ")}.`;
+}
+
+type LeafIssue = { path: PropertyKey[]; message: string };
+
+function leafIssues(issues: readonly z.core.$ZodIssue[], base: PropertyKey[]): LeafIssue[] {
+  const leaves: LeafIssue[] = [];
+  for (const issue of issues) {
+    const path = [...base, ...issue.path];
+    if (issue.code === "invalid_union" && issue.errors.length > 0) {
+      leaves.push(...leafIssues(chosenUnionBranch(issue.errors), path));
+      continue;
+    }
+    if (issue.code === "unrecognized_keys") {
+      leaves.push({ path, message: `unexpected field ${issue.keys.join(", ")}` });
+      continue;
+    }
+    leaves.push({ path, message: issue.message });
+  }
+  return leaves;
+}
+
+/** The union branch the arguments meant: one whose tag matched, else the
+ * branch that reports the unknown tag, else the one with the fewest issues. */
+function chosenUnionBranch(branches: z.core.$ZodIssue[][]): z.core.$ZodIssue[] {
+  const tagMismatch = (issue: z.core.$ZodIssue): boolean =>
+    (issue.code === "invalid_union" && issue.errors.length === 0) ||
+    (issue.code === "invalid_value" &&
+      ["type", "kind", "operation"].includes(String(issue.path.at(-1))));
+  const matching = branches.filter((branch) => !branch.some(tagMismatch));
+  if (matching.length > 0) {
+    return matching.reduce((best, branch) => (branch.length < best.length ? branch : best));
+  }
+  const unknownTag = branches.find((branch) =>
+    branch.some((issue) => issue.code === "invalid_union" && issue.errors.length === 0),
+  );
+  return (
+    unknownTag?.filter((issue) => issue.code === "invalid_union") ??
+    branches.reduce((best, branch) => (branch.length < best.length ? branch : best))
+  );
+}

package/src/interaction-tools.ts

@@ -90,7 +90,10 @@ import {
   type InteractionTransport,
 } from "@opengeni/sdk";
 import { z } from "zod";
-import { browserActionInputJsonSchema } from "./browser-action-json-schema";
+import {
+  describeInvalidToolArguments,
+  interactionToolInputJsonSchema,
+} from "./interaction-tool-json-schema";
 import { guardedMcpFetch } from "./mcp-network";
 
 export const INTERACTION_ATTEMPT_TOOL_NAMES = FIRST_PARTY_IN_PROCESS_TOOL_NAMES;
@@ -786,10 +789,7 @@ export function createInteractionAttemptToolDefinitions(
       codemodePath: options.codemodePath,
       title: options.title,
       description: options.description,
-      inputSchema:
-        options.name === "browser_act"
-          ? browserActionInputJsonSchema(options.input)
-          : jsonSchema(options.input),
+      inputSchema: interactionToolInputJsonSchema(options.input),
       outputSchema: jsonSchema(options.output),
       annotations: {
         title: options.title,
@@ -2603,7 +2603,7 @@ async function safeInteractionExecution<TInput extends z.ZodType, TOutput extend
     return result;
   } catch (error) {
     if (error instanceof z.ZodError) {
-      return interactionErrorResult("invalid_arguments", "Interaction tool arguments are invalid.");
+      return interactionErrorResult("invalid_arguments", describeInvalidToolArguments(error));
     }
     // A failed read is still a useful tool result. Preserve the API's public
     // explanation; mutations and uncertain outcomes retain their failure path.

package/src/sandbox/browser-control-client.ts

@@ -87,6 +87,7 @@ import {
 } from "./browser-control-server";
 import { parseExecResponseBanner } from "./exec-banner";
 import { controllerStreamRequest, parseControllerStreamResponse } from "./browser-control-stream";
+import type { BrowserControlHttpCall, BrowserControlHttpOutcome } from "./browser-control-http";
 import {
   buildStreamUrl,
   exposedPortAllowsHostFetch,
@@ -177,6 +178,8 @@ export type BrowserControlPlacementSession = {
   ensureBrowserControl?: (
     request: BrowserControlEnsureRequest,
   ) => Promise<BrowserControlEnsureResponse>;
+  /** Agent-native controller transport, addressed by authority scope. */
+  browserControlHttp?: (request: BrowserControlHttpCall) => Promise<BrowserControlHttpOutcome>;
   openBrowserFrames?: (
     request: BrowserFramesOpenRequest,
   ) => Promise<{ channel: StreamChannel; endpoint: ExposedPortEndpoint }>;
@@ -1176,6 +1179,19 @@ export class BrowserControlClient {
       );
     }
 
+    const native = await this.requestNative(input, BROWSER_CONTROL_MAX_JSON_BYTES);
+    if (native) {
+      let text: string;
+      try {
+        text = new TextDecoder("utf-8", { fatal: true }).decode(native.body);
+      } catch (error) {
+        throw new BrowserControlProtocolError("browser controller response is not UTF-8", {
+          cause: error,
+        });
+      }
+      return parseEnvelope(text, native.status);
+    }
+
     if (
       this.session.execWithInput &&
       (input.body === undefined || Buffer.byteLength(JSON.stringify(input.body)) <= 32 * 1024)
@@ -1357,6 +1373,18 @@ export class BrowserControlClient {
       );
     }
 
+    const nativeFrame = await this.requestNative(
+      input,
+      input.surface === "browser" ? 24 * 1024 * 1024 : COMPUTER_SCREENSHOT_MAX_BYTES,
+    );
+    if (nativeFrame) {
+      if (nativeFrame.status < 200 || nativeFrame.status >= 300) {
+        parseEnvelope(new TextDecoder().decode(nativeFrame.body), nativeFrame.status);
+        throw new BrowserControlProtocolError("browser controller image response is invalid");
+      }
+      return controllerImageFrame(nativeFrame.body, nativeFrame.headers, input);
+    }
+
     if (this.session.execWithInput) {
       try {
         const response = await this.requestStream(
@@ -1512,8 +1540,69 @@ export class BrowserControlClient {
   }
 
   private async controllerPort(): Promise<number> {
+    if (!this.nativeAuthority || !this.session.ensureBrowserControl) return this.port;
+    return await this.ensureControllerPort(false);
+  }
+
+  /** The agent-native transport: one in-process loopback request on the
+   * Connected Machine, addressed by authority scope, with no process started.
+   * Null when the placement does not serve it; the caller then keeps its
+   * command transport. A controller that is not running receives nothing, so
+   * it is ensured and the request sent once more whatever its method. */
+  private async requestNative(
+    input: { method: string; path: string; token: string; body?: unknown; timeoutMs?: number },
+    maxResponseBytes: number,
+    retry = true,
+  ): Promise<{ status: number; headers: Map<string, string>; body: Uint8Array } | null> {
+    const authority = this.nativeAuthority;
+    if (!authority || !this.session.browserControlHttp || !this.session.ensureBrowserControl) {
+      return null;
+    }
+    const method = input.method;
+    if (method !== "GET" && method !== "POST" && method !== "PUT" && method !== "DELETE") {
+      throw new BrowserControlProtocolError("browser controller request method is invalid");
+    }
+    // Validates the path exactly as the command transport does.
+    localControllerUrl(this.port, input.path);
+    const body =
+      input.body === undefined ? undefined : new TextEncoder().encode(JSON.stringify(input.body));
+    if (body !== undefined && body.byteLength > BROWSER_CONTROL_MAX_JSON_BYTES) {
+      throw new RangeError("browser controller request body is too large");
+    }
+    let outcome: BrowserControlHttpOutcome;
+    try {
+      outcome = await this.session.browserControlHttp({
+        scopeId: authority.scopeId,
+        scopeGeneration: authority.scopeGeneration,
+        method,
+        path: input.path,
+        bearerToken: requireToken(input.token, "browser controller token"),
+        ...(body === undefined ? {} : { body }),
+        timeoutMs: boundedTimeout(input.timeoutMs ?? this.timeoutMs),
+        maxResponseBytes,
+      });
+    } catch (error) {
+      if (retry && method === "GET") {
+        await this.ensureControllerPort(true);
+        return await this.requestNative(input, maxResponseBytes, false);
+      }
+      throw new BrowserControlTransportError("browser controller request transport failed", {
+        cause: error,
+      });
+    }
+    if (outcome.kind === "unsupported") return null;
+    if (outcome.kind === "controller_absent") {
+      if (!retry) throw new BrowserControlTransportError("browser controller is not running");
+      await this.ensureControllerPort(true);
+      return await this.requestNative(input, maxResponseBytes, false);
+    }
+    return outcome;
+  }
+
+  private async ensureControllerPort(replace: boolean): Promise<number> {
     if (!this.nativeAuthority || !this.session.ensureBrowserControl) return this.port;
     const key = nativeControllerKey(this.nativeAuthority);
+    if (replace) nativeControllerPorts.delete(key);
     const cached = nativeControllerPorts.get(key);
     if (cached !== undefined) return cached;
     const ensured = await this.session.ensureBrowserControl({

package/src/sandbox/browser-control-http.ts

@@ -0,0 +1,32 @@
+/** One request to an agent-owned browser/computer controller, addressed by its
+ * exact authority scope rather than a local port. */
+export type BrowserControlHttpCall = {
+  scopeId: string;
+  scopeGeneration: string;
+  method: "GET" | "POST" | "PUT" | "DELETE";
+  /** Absolute controller path with optional query. */
+  path: string;
+  bearerToken: string;
+  /** JSON request body bytes, when the request has a body. */
+  body?: Uint8Array;
+  timeoutMs: number;
+  maxResponseBytes: number;
+};
+
+/** What a placement transport did with one controller request. */
+export type BrowserControlHttpOutcome =
+  /** The placement cannot serve it natively; nothing was sent. */
+  | { kind: "unsupported" }
+  /** No controller for that scope and generation accepted it; nothing was
+   * delivered, so the caller may ensure the controller and send again. */
+  | { kind: "controller_absent" }
+  | {
+      kind: "response";
+      status: number;
+      /** Lowercase header names. */
+      headers: Map<string, string>;
+      body: Uint8Array;
+    };
+
+/** Extra time beyond the request's own deadline for the reply to arrive. */
+export const BROWSER_CONTROL_HTTP_REPLY_GRACE_MS = 5_000;

package/src/sandbox/routing/backend-resolver.ts

@@ -67,6 +67,7 @@ export interface SelfhostedConnectionBinding {
   operationResourcePolicySupported: boolean;
   operationCpuQuotaSupported: boolean;
   transactionalFsWriteSupported?: boolean;
+  browserControlHttpSupported?: boolean;
 }
 
 export interface ActiveBackendResolverDeps {
@@ -406,6 +407,7 @@ export function makeActiveBackendResolver(
         operationResourcePolicySupported: connection.operationResourcePolicySupported,
         operationCpuQuotaSupported: connection.operationCpuQuotaSupported,
         transactionalFsWriteSupported: connection.transactionalFsWriteSupported === true,
+        browserControlHttpSupported: connection.browserControlHttpSupported === true,
         // The routing proxy caches this session by sandbox+epoch, but command
         // policy and runner authority are mutable. Re-read one coherent DB
         // snapshot only when a new exec/Git is admitted.