@opengeni/runtime 4.0.1 → 4.0.3-canary.36199476632001

package/src/interaction-tools.ts

@@ -9,10 +9,15 @@ import {
   BrowserActionReceipt,
   BrowserClipboard,
   BrowserDiagnosticBatch,
+  BrowserDomReadResponse,
+  BrowserDomReadLocator,
+  BrowserDomReadSelector,
+  BrowserDomSafeAttribute,
   BrowserIdentity,
   BrowserIdentityListResponse,
   BrowserIdentityMutationResponse,
   BrowserObservation,
+  InteractionSemanticNode,
   BrowserRevisionListResponse,
   BrowserSession,
   BrowserSessionMutationResponse,
@@ -61,6 +66,7 @@ import type {
 } from "@opengeni/codemode";
 import { OpenGeniApiError, OpenGeniClient, type InteractionTransport } from "@opengeni/sdk";
 import { z } from "zod";
+import { browserActionInputJsonSchema } from "./browser-action-json-schema";
 import { guardedMcpFetch } from "./mcp-network";
 
 export const INTERACTION_ATTEMPT_TOOL_NAMES = FIRST_PARTY_IN_PROCESS_TOOL_NAMES;
@@ -74,11 +80,200 @@ class InteractionExecutionResult<T> {
   ) {}
 }
 
+const BROWSER_AGENT_MAX_NODES = 60;
+const BROWSER_AGENT_MAX_NODE_BYTES = 12_000;
+const BROWSER_AGENT_MAX_FIELD_LENGTH = 180;
+const BROWSER_READ_MAX_NODES = 80;
+// A base64 image is journaled inside a 16 MiB Code Mode result. Leave room
+// for JSON framing, metadata, and other content instead of failing after capture.
+const BROWSER_TOOL_IMAGE_MAX_BYTES = 8 * 1024 * 1024;
+
+const BrowserAgentNode = z
+  .object({
+    ref: z.string().max(512),
+    role: z.string().max(256),
+    depth: z.number().int().nonnegative(),
+    name: z.string().max(BROWSER_AGENT_MAX_FIELD_LENGTH).optional(),
+    description: z.string().max(BROWSER_AGENT_MAX_FIELD_LENGTH).optional(),
+    value: z
+      .union([
+        z.string().max(BROWSER_AGENT_MAX_FIELD_LENGTH),
+        z.object({ redacted: z.literal(true), reason: z.string().max(32) }).strict(),
+      ])
+      .optional(),
+    states: z.array(z.string().max(BROWSER_AGENT_MAX_FIELD_LENGTH)).max(6),
+    actions: z.array(z.string().max(BROWSER_AGENT_MAX_FIELD_LENGTH)).max(6),
+  })
+  .strict();
+const BrowserAgentView = z
+  .object({
+    kind: z.literal("compact"),
+    sourceKind: z.enum(["snapshot", "diff", "none"]),
+    nodes: z.array(BrowserAgentNode).max(BROWSER_AGENT_MAX_NODES),
+    sourceNodeCount: z.number().int().nonnegative(),
+    omittedNodeCount: z.number().int().nonnegative(),
+    removedRefCount: z.number().int().nonnegative(),
+    clippedFieldCount: z.number().int().nonnegative(),
+    maxNodes: z.number().int().positive(),
+    maxNodeBytes: z.number().int().positive(),
+  })
+  .strict();
+const CompactBrowserObservation = BrowserObservation.safeExtend({
+  semantic: z.null(),
+  agentView: BrowserAgentView,
+});
+const BrowserAgentObservation = z.union([BrowserObservation, CompactBrowserObservation]);
+const BrowserAgentActionReceipt = BrowserActionReceipt.safeExtend({
+  observation: BrowserAgentObservation.nullable(),
+});
+const BrowserScreenshotInput = z
+  .object({
+    browserSessionId: z.string().uuid(),
+    targetId: z.string().min(1).max(512),
+    fullPage: z.boolean().optional(),
+    quality: z.number().int().min(1).max(100).optional(),
+  })
+  .strict();
+const BrowserScreenshotOutput = z
+  .object({
+    kind: z.literal("browser_screenshot"),
+    browserSessionId: z.string().uuid(),
+    targetId: z.string(),
+    frameId: z.string(),
+    targetGeneration: z.string(),
+    documentGeneration: z.string(),
+    mediaType: z.enum(["image/jpeg", "image/png"]),
+    width: z.number().int().positive(),
+    height: z.number().int().positive(),
+    deviceScaleFactor: z.number().positive(),
+    scrollX: z.number(),
+    scrollY: z.number(),
+    capturedAt: z.string(),
+    fullPage: z.boolean(),
+  })
+  .strict();
+const BrowserReadInput = z
+  .object({
+    browserSessionId: z.string().uuid(),
+    targetId: z.string().min(1).max(512),
+    mode: z.enum(["matches", "count", "subtree", "dom"]).optional(),
+    dom: z
+      .discriminatedUnion("kind", [
+        z
+          .object({
+            kind: z.literal("element"),
+            locator: BrowserDomReadLocator,
+            attributes: z.array(BrowserDomSafeAttribute).max(6).optional(),
+            maxChars: z.number().int().min(1).max(4_096).optional(),
+          })
+          .strict(),
+        z
+          .object({
+            kind: z.literal("count"),
+            selector: BrowserDomReadSelector,
+          })
+          .strict(),
+      ])
+      .optional(),
+    expectedTargetGeneration: z.string().min(1).max(256).optional(),
+    expectedDocumentGeneration: z.string().min(1).max(256).nullable().optional(),
+    expectedFrameId: z.string().min(1).max(256).nullable().optional(),
+    ref: z.string().min(1).max(512).optional(),
+    scopeRef: z.string().min(1).max(512).optional(),
+    role: z.string().min(1).max(256).optional(),
+    nameContains: z.string().min(1).max(512).optional(),
+    textContains: z.string().min(1).max(512).optional(),
+    state: z.string().min(1).max(128).optional(),
+    action: z.string().min(1).max(128).optional(),
+    limit: z.number().int().min(1).max(BROWSER_READ_MAX_NODES).optional(),
+  })
+  .strict()
+  .superRefine((value, context) => {
+    if (value.mode === "subtree" && !value.ref) {
+      context.addIssue({ code: "custom", path: ["ref"], message: "subtree requires a ref" });
+    }
+    if ((value.mode === "dom") !== (value.dom !== undefined)) {
+      context.addIssue({
+        code: "custom",
+        path: ["dom"],
+        message: "dom query is required only with mode=dom",
+      });
+    }
+    if (
+      value.mode === "dom" &&
+      [
+        value.ref,
+        value.scopeRef,
+        value.role,
+        value.nameContains,
+        value.textContains,
+        value.state,
+        value.action,
+        value.limit,
+      ].some((field) => field !== undefined)
+    ) {
+      context.addIssue({
+        code: "custom",
+        path: ["mode"],
+        message: "accessibility filters cannot be combined with a DOM query",
+      });
+    }
+    if (
+      value.mode !== "dom" &&
+      [
+        value.expectedTargetGeneration,
+        value.expectedDocumentGeneration,
+        value.expectedFrameId,
+      ].some((field) => field !== undefined)
+    ) {
+      context.addIssue({
+        code: "custom",
+        path: ["mode"],
+        message: "DOM generation fences require mode=dom",
+      });
+    }
+  });
+const BrowserReadOutput = z
+  .object({
+    browserSessionId: z.string().uuid(),
+    targetId: z.string(),
+    observationId: z.string(),
+    targetGeneration: z.string(),
+    documentGeneration: z.string().nullable(),
+    frameId: z.string().nullable(),
+    source: z.literal("accessibility"),
+    mode: z.enum(["matches", "count", "subtree"]),
+    scopeFound: z.boolean(),
+    nodes: z.array(BrowserAgentNode).max(BROWSER_READ_MAX_NODES),
+    totalMatches: z.number().int().nonnegative(),
+    omittedMatches: z.number().int().nonnegative(),
+    clippedFieldCount: z.number().int().nonnegative(),
+    maxNodeBytes: z.number().int().positive(),
+  })
+  .strict();
+const BrowserDomAgentReadOutput = BrowserDomReadResponse.safeExtend({
+  source: z.literal("dom"),
+});
+const BrowserReadToolOutput = z.discriminatedUnion("source", [
+  BrowserReadOutput,
+  BrowserDomAgentReadOutput,
+]);
+
+type BrowserSemanticNodeValue = z.infer<typeof InteractionSemanticNode>;
+type FlatBrowserNode = {
+  node: BrowserSemanticNodeValue;
+  depth: number;
+  index: number;
+  withinScope: boolean;
+};
+
 const TOOL_PERMISSION = {
   interaction_discover: "sessions:read",
   browser_open: "sessions:control",
   browser_tabs: "sessions:control",
   browser_observe: "sessions:read",
+  browser_read: "sessions:read",
+  browser_screenshot: "sessions:read",
   browser_act: "sessions:control",
   browser_clipboard: "sessions:read",
   browser_debug: "sessions:read",
@@ -183,6 +378,8 @@ const BrowserObserveInput = z
   .object({
     browserSessionId: z.string().uuid(),
     targetId: z.string().min(1).max(512),
+    view: z.enum(["compact", "full"]).optional(),
+    includeScreenshot: z.boolean().optional(),
   })
   .strict();
 const BrowserActInput = z
@@ -192,6 +389,7 @@ const BrowserActInput = z
     expectedTargetGeneration: z.string().min(1).max(256).optional(),
     expectedDocumentGeneration: z.string().min(1).max(256).nullable().optional(),
     expectedFrameId: z.string().min(1).max(256).nullable().optional(),
+    view: z.enum(["compact", "full", "none"]).optional(),
     action: z.union([BrowserAction, BrowserActionBatch]),
   })
   .strict();
@@ -375,13 +573,18 @@ const ComputerObserveInput = z
     targetId: z.string().min(1).max(512),
   })
   .strict();
+const ComputerObserveOutput = ComputerObservation.extend({
+  capturedFrame: z
+    .object({ width: z.number().int().positive(), height: z.number().int().positive() })
+    .strict(),
+}).strict();
 const ComputerClipboardInput = z.object({ computerSessionId: z.string().uuid() }).strict();
 const ComputerActInput = z
   .object({
     computerSessionId: z.string().uuid(),
     targetId: z.string().min(1).max(512),
     expectedTargetGeneration: z.string().min(1).max(256).optional(),
-    expectedObservationId: z.string().uuid().nullable().optional(),
+    expectedObservationId: ComputerObservation.shape.observationId.nullable().optional(),
     expectedFrameId: z.string().min(1).max(256).nullable().optional(),
     action: ComputerAction,
   })
@@ -437,7 +640,10 @@ export function createInteractionAttemptToolDefinitions(
       codemodePath: options.codemodePath,
       title: options.title,
       description: options.description,
-      inputSchema: jsonSchema(options.input),
+      inputSchema:
+        options.name === "browser_act"
+          ? browserActionInputJsonSchema(options.input)
+          : jsonSchema(options.input),
       outputSchema: jsonSchema(options.output),
       annotations: {
         title: options.title,
@@ -510,7 +716,7 @@ export function createInteractionAttemptToolDefinitions(
     codemodePath: ["interaction", "browser", "open"],
     title: "Open or reuse browser",
     description:
-      "Open a managed BrowserSession on the current agent placement, reuse a relevant compatible live session by default, attach to an explicit workspace BrowserSession, or open an attached Chrome profile by passing placement={kind:'attached_device',deviceId}. This does not infer or attach the user's existing Chrome: for requests about 'my browser', 'my tabs', or current Chrome, call interaction_discover first and select an actual attachedBrowsers device; if none exists, explain that the Chrome extension must be connected instead of silently creating a blank managed browser. Managed Chromium defaults to headed so OAuth and later human interaction use a supported browser; request headless=true only for agent-only work that will not require sign-in or human control. Returns exact session and tab state.",
+      "Open a managed BrowserSession on the current agent placement, reuse a relevant compatible live session by default, attach to an explicit workspace BrowserSession, or open an attached Chrome profile by passing placement={kind:'attached_device',deviceId}. This does not infer or attach the user's existing Chrome: for requests about 'my browser', 'my tabs', or current Chrome, call interaction_discover first and select an actual attachedBrowsers device; if none exists, explain that the Chrome extension must be connected instead of silently creating a blank managed browser. BrowserSessions persist across tool calls; shell-launched browser daemons do not survive remote-command cleanup. Never switch to the user's attached Chrome as a fallback for a failed managed browser unless the user requested that profile. A new attached session opens a dedicated tab. Managed Chromium defaults to headed so OAuth and later human interaction use a supported browser; request headless=true only for agent-only work that will not require sign-in or human control. Returns exact session and tab state.",
     input: BrowserOpenInput,
     output: BrowserOpenOutput,
     readOnly: false,
@@ -524,7 +730,7 @@ export function createInteractionAttemptToolDefinitions(
     codemodePath: ["interaction", "browser", "tabs"],
     title: "Manage browser tabs",
     description:
-      "List, open, select, or close tabs in one exact BrowserSession. Returns the authoritative complete tab list after the operation.",
+      "List, open, logically select, or close tabs in one exact BrowserSession. Selection changes the BrowserSession's default target, not the visible desktop tab. New attached-Chrome tabs open in the background. Use browser_act activate only when foregrounding the owned tab is explicitly intended. Returns the authoritative complete tab list after the operation.",
     input: BrowserTabsInput,
     output: BrowserTargetListResponse,
     readOnly: false,
@@ -556,17 +762,161 @@ export function createInteractionAttemptToolDefinitions(
     codemodePath: ["interaction", "browser", "observe"],
     title: "Observe browser tab",
     description:
-      "Read one tab's current URL/title, causal generations, compact semantic accessibility tree, dialog, and diagnostic counts without taking control.",
+      "Read one tab's URL, title, viewport, causal generations, and a bounded accessibility view. Compact view is the default: agentView lists selected refs and reports omitted nodes and clipped fields; it is not a complete page tree. Set view=full for the complete accessibility snapshot. Set includeScreenshot=true to add a current still image as tool image content; browser_screenshot is faster when only pixels are needed.",
     input: BrowserObserveInput,
-    output: BrowserObservation,
+    output: BrowserAgentObservation,
     readOnly: true,
     idempotent: true,
-    execute: async (value) =>
-      await input.transport.observeBrowserTarget(
+    execute: async (value) => {
+      let observation = await input.transport.observeBrowserTarget(
         input.workspaceId,
         value.browserSessionId,
         value.targetId,
-      ),
+      );
+      if (!value.includeScreenshot)
+        return projectBrowserObservation(observation, value.view ?? "compact");
+      const frame = await input.transport.captureBrowserTarget(
+        input.workspaceId,
+        value.browserSessionId,
+        value.targetId,
+      );
+      if (
+        observation.target.targetGeneration !== frame.targetGeneration ||
+        observation.target.documentGeneration !== frame.documentGeneration
+      ) {
+        observation = await input.transport.observeBrowserTarget(
+          input.workspaceId,
+          value.browserSessionId,
+          value.targetId,
+        );
+      }
+      if (
+        observation.target.targetGeneration !== frame.targetGeneration ||
+        observation.target.documentGeneration !== frame.documentGeneration
+      ) {
+        throw new Error("browser target changed while its visual observation was captured");
+      }
+      return new InteractionExecutionResult(
+        projectBrowserObservation(observation, value.view ?? "compact"),
+        [browserToolImageContent(frame.data, frame.mediaType)],
+      );
+    },
+  });
+
+  add({
+    name: "browser_read",
+    codemodePath: ["interaction", "browser", "read"],
+    title: "Read focused browser content",
+    description:
+      "Search, count, or read a subtree from one tab's accessibility snapshot; filter by ref, role, name, accessibility text, state, or action, and scopeRef. AX mode refreshes the full tree internally and cannot return DOM attributes or editable input values. Use mode=dom with a ref, CSS, test-id, or placeholder locator for bounded element text, editable value, or allowlisted attributes, or a CSS selector for count. Content-matching role/label/text locators and CSS attribute/pseudo selectors are rejected for DOM reads to prevent secret-value probing. DOM reads use exact causal fences and only fixed, read-only browser-side projection; callers cannot supply JavaScript. Ref DOM locators refresh accessibility; CSS/test-id/placeholder locators avoid that scan.",
+    input: BrowserReadInput,
+    output: BrowserReadToolOutput,
+    readOnly: true,
+    idempotent: true,
+    execute: async (value) => {
+      if (value.mode === "dom") {
+        const state =
+          value.expectedTargetGeneration !== undefined &&
+          value.expectedDocumentGeneration !== undefined &&
+          value.expectedFrameId !== undefined
+            ? null
+            : await input.transport.getBrowserTargetState(
+                input.workspaceId,
+                value.browserSessionId,
+                value.targetId,
+              );
+        const fences = {
+          expectedTargetGeneration: value.expectedTargetGeneration ?? state!.targetGeneration,
+          expectedDocumentGeneration:
+            value.expectedDocumentGeneration === undefined
+              ? state!.documentGeneration
+              : value.expectedDocumentGeneration,
+          expectedFrameId:
+            value.expectedFrameId === undefined ? state!.frameId : value.expectedFrameId,
+        };
+        const documentGeneration = fences.expectedDocumentGeneration;
+        const frameId = fences.expectedFrameId;
+        if (documentGeneration === null || frameId === null) {
+          throw new OpenGeniApiError(409, "browser target has no inspectable document", {
+            code: "document_unavailable",
+            retryable: true,
+          });
+        }
+        const inspectedFences = {
+          expectedTargetGeneration: fences.expectedTargetGeneration,
+          expectedDocumentGeneration: documentGeneration,
+          expectedFrameId: frameId,
+        };
+        const dom = value.dom!;
+        const request =
+          dom.kind === "element"
+            ? {
+                ...inspectedFences,
+                kind: "element" as const,
+                locator: dom.locator,
+                ...(dom.attributes ? { attributes: dom.attributes } : {}),
+                maxChars: dom.maxChars ?? 4_096,
+              }
+            : { ...inspectedFences, kind: "count" as const, selector: dom.selector };
+        return {
+          ...(await input.transport.readBrowserDom(
+            input.workspaceId,
+            value.browserSessionId,
+            value.targetId,
+            request,
+          )),
+          source: "dom" as const,
+        };
+      }
+      return readBrowserObservation(
+        await input.transport.observeBrowserTarget(
+          input.workspaceId,
+          value.browserSessionId,
+          value.targetId,
+        ),
+        value,
+      );
+    },
+  });
+
+  add({
+    name: "browser_screenshot",
+    codemodePath: ["interaction", "browser", "screenshot"],
+    title: "Capture browser screenshot",
+    description:
+      "Capture a current browser tab screenshot without reading its accessibility tree. The image is returned as tool image content, with small structured frame metadata. Set fullPage=true to capture the whole scrollable page when supported; large pages may take longer or exceed capture bounds. Set quality lower to reduce JPEG bytes if a large capture exceeds the Code Mode image limit.",
+    input: BrowserScreenshotInput,
+    output: BrowserScreenshotOutput,
+    readOnly: true,
+    idempotent: true,
+    execute: async (value) => {
+      const frame = await input.transport.captureBrowserTarget(
+        input.workspaceId,
+        value.browserSessionId,
+        value.targetId,
+        {},
+        { fullPage: value.fullPage ?? false, ...(value.quality ? { quality: value.quality } : {}) },
+      );
+      return new InteractionExecutionResult(
+        {
+          kind: "browser_screenshot" as const,
+          browserSessionId: frame.browserSessionId,
+          targetId: frame.targetId,
+          frameId: frame.frameId,
+          targetGeneration: frame.targetGeneration,
+          documentGeneration: frame.documentGeneration,
+          mediaType: frame.mediaType,
+          width: frame.width,
+          height: frame.height,
+          deviceScaleFactor: frame.deviceScaleFactor,
+          scrollX: frame.scrollX,
+          scrollY: frame.scrollY,
+          capturedAt: frame.capturedAt,
+          fullPage: value.fullPage ?? false,
+        },
+        [browserToolImageContent(frame.data, frame.mediaType)],
+      );
+    },
   });
 
   add({
@@ -574,29 +924,42 @@ export function createInteractionAttemptToolDefinitions(
     codemodePath: ["interaction", "browser", "act"],
     title: "Act in browser tab",
     description:
-      "Perform one semantic-first browser action or bounded batch, including setting a managed browser's web permission for this tab's exact current top-level origin. Omit generation fences to use a fresh observation automatically; provide them to require exact previously observed state. Returns the durable receipt and changed observation.",
+      "Perform one semantic-first browser action or bounded batch. Use viewport to set page width, height, desktop/mobile layout and touch emulation, then check the measured viewport in the returned observation; emulation does not prove physical mobile-browser behavior. Use history back/forward for tab navigation; keypress shortcuts are page input and may not navigate browser history. The explicit activate action foregrounds the target in the user's desktop browser; use only when that is intended. Permission actions set a managed browser's web permission for this tab's exact current top-level origin. Omit generation fences to fetch fresh generation metadata without scanning accessibility; supplying all three fences skips that state read while preserving controller validation. The default receipt includes a compact accessibility view with explicit omissions; set view=full for the complete observation, or view=none when no post-action tree is needed.",
     input: BrowserActInput,
-    output: BrowserActionReceipt,
+    output: BrowserAgentActionReceipt,
     readOnly: false,
     idempotent: true,
     execute: async (value, context) => {
-      const current = await input.transport.observeBrowserTarget(
+      const current =
+        value.expectedTargetGeneration !== undefined &&
+        value.expectedDocumentGeneration !== undefined &&
+        value.expectedFrameId !== undefined
+          ? null
+          : await input.transport.getBrowserTargetState(
+              input.workspaceId,
+              value.browserSessionId,
+              value.targetId,
+            );
+      const receipt = await input.transport.actInBrowser(
         input.workspaceId,
         value.browserSessionId,
-        value.targetId,
+        {
+          operationId: context.operationId,
+          targetId: value.targetId,
+          expectedTargetGeneration: value.expectedTargetGeneration ?? current!.targetGeneration,
+          expectedDocumentGeneration:
+            value.expectedDocumentGeneration === undefined
+              ? current!.documentGeneration
+              : value.expectedDocumentGeneration,
+          expectedFrameId:
+            value.expectedFrameId === undefined ? current!.frameId : value.expectedFrameId,
+          ...(value.view === "none" ? { observationMode: "none" as const } : {}),
+          action: value.action,
+        },
       );
-      return await input.transport.actInBrowser(input.workspaceId, value.browserSessionId, {
-        operationId: context.operationId,
-        targetId: value.targetId,
-        expectedTargetGeneration: value.expectedTargetGeneration ?? current.target.targetGeneration,
-        expectedDocumentGeneration:
-          value.expectedDocumentGeneration === undefined
-            ? current.target.documentGeneration
-            : value.expectedDocumentGeneration,
-        expectedFrameId:
-          value.expectedFrameId === undefined ? current.frameId : value.expectedFrameId,
-        action: value.action,
-      });
+      return receipt.observation && (value.view ?? "compact") === "compact"
+        ? { ...receipt, observation: projectBrowserObservation(receipt.observation, "compact") }
+        : receipt;
     },
   });
 
@@ -931,9 +1294,9 @@ export function createInteractionAttemptToolDefinitions(
     codemodePath: ["interaction", "computer", "observe"],
     title: "Observe app or window",
     description:
-      "Read one ComputerSession app/window/screen target, causal generation, semantic accessibility tree, focus, frame identity, and a bounded current screenshot without taking control.",
+      "Read one ComputerSession app/window/screen target, causal generation, semantic accessibility tree, focus, frame identity, and a bounded current screenshot without taking control. capturedFrame gives the screenshot's exact pixel dimensions; use those pixels for pointer coordinates, not the target's native bounds or an image-preview size.",
     input: ComputerObserveInput,
-    output: ComputerObservation,
+    output: ComputerObserveOutput,
     readOnly: true,
     idempotent: true,
     execute: async (value) => {
@@ -957,13 +1320,20 @@ export function createInteractionAttemptToolDefinitions(
       if (observation.target.targetGeneration !== frame.targetGeneration) {
         throw new Error("computer target changed while its visual observation was captured");
       }
-      return new InteractionExecutionResult({ ...observation, frameId: frame.frameId }, [
+      return new InteractionExecutionResult(
         {
-          type: "image",
-          data: Buffer.from(frame.data).toString("base64"),
-          mimeType: frame.mediaType,
+          ...observation,
+          frameId: frame.frameId,
+          capturedFrame: { width: frame.width, height: frame.height },
         },
-      ]);
+        [
+          {
+            type: "image",
+            data: Buffer.from(frame.data).toString("base64"),
+            mimeType: frame.mediaType,
+          },
+        ],
+      );
     },
   });
 
@@ -986,31 +1356,38 @@ export function createInteractionAttemptToolDefinitions(
     codemodePath: ["interaction", "computer", "act"],
     title: "Act in app or window",
     description:
-      "Perform one action in an exact ComputerSession target. Prefer semantic actions from computer_observe: on macOS they can invoke controls and set values without foregrounding the app. Pointer, keyboard, target focus, screen actions, and clipboard paste use the physical graphical seat and may change the user's foreground app; use them only when foreground control is explicitly intended. Omit fences to use a fresh observation automatically. Returns the durable causal receipt.",
+      "Perform one action in an exact ComputerSession target. Prefer semantic actions from computer_observe: on macOS they can invoke controls and set values without foregrounding the app. Pointer, keyboard, target focus, screen actions, and clipboard paste use the physical graphical seat and may change the user's foreground app; use them only when foreground control is explicitly intended. For pointer actions, set action.frameId from the observed frame and use coordinates in the returned screenshot's pixels; do not scale them to target.bounds. A pixel frame fence applies only to pointer actions; semantic actions use the observation fence even if a frame ID is supplied. Supplying the target generation and relevant action fence preserves the exact observation without refreshing it. Omit fences to use a fresh observation automatically. Returns the durable causal receipt.",
     input: ComputerActInput,
     output: ComputerActionReceipt,
     readOnly: false,
     idempotent: true,
     execute: async (value, context) => {
-      const current = await input.transport.observeComputerTarget(
-        input.workspaceId,
-        value.computerSessionId,
-        value.targetId,
-      );
+      const current =
+        value.expectedTargetGeneration !== undefined &&
+        (value.action.type === "pointer"
+          ? (value.expectedFrameId ?? value.action.frameId) != null
+          : value.expectedObservationId != null)
+          ? null
+          : await input.transport.observeComputerTarget(
+              input.workspaceId,
+              value.computerSessionId,
+              value.targetId,
+            );
       return await input.transport.actInComputer(input.workspaceId, value.computerSessionId, {
         operationId: context.operationId,
         targetId: value.targetId,
-        expectedTargetGeneration: value.expectedTargetGeneration ?? current.target.targetGeneration,
+        expectedTargetGeneration:
+          value.expectedTargetGeneration ?? current!.target.targetGeneration,
         expectedObservationId:
           value.expectedObservationId === undefined
-            ? current.observationId
+            ? (current?.observationId ?? null)
             : value.expectedObservationId,
         expectedFrameId:
-          value.expectedFrameId === undefined
-            ? value.action.type === "pointer"
+          value.action.type === "pointer"
+            ? value.expectedFrameId === undefined
               ? value.action.frameId
-              : null
-            : value.expectedFrameId,
+              : value.expectedFrameId
+            : null,
         action: value.action,
       });
     },
@@ -1131,7 +1508,8 @@ async function openBrowser(
   if (value.browserSessionId) {
     session = await transport.getBrowserSession(workspaceId, value.browserSessionId);
   } else {
-    const listed = await transport.listBrowserSessions(workspaceId);
+    const listed =
+      value.mode === "new" ? { sessions: [] } : await transport.listBrowserSessions(workspaceId);
     // The agent-facing browser is human-capable by default. A headless session
     // is a deliberately narrower execution mode and must never be silently
     // reused for an omitted/default headed request: OAuth providers such as
@@ -1141,7 +1519,18 @@ async function openBrowser(
       value.mode === "new"
         ? null
         : newestRelevant(
-            listed.sessions.filter((candidate) => candidate.headless === requestedHeadless),
+            listed.sessions.filter(
+              (candidate) =>
+                candidate.headless === requestedHeadless &&
+                compatibleInteractionPlacement(candidate.placement, value.placement) &&
+                (value.identityId === undefined || candidate.identityId === value.identityId) &&
+                (value.baseRevisionId === undefined ||
+                  candidate.baseRevisionId === value.baseRevisionId) &&
+                (value.networkRouteId === undefined ||
+                  candidate.networkRouteId === value.networkRouteId) &&
+                (value.linkedComputerSessionId === undefined ||
+                  candidate.linkedComputerSessionId === value.linkedComputerSessionId),
+            ),
             sourceSessionId,
           );
     if (reusable) {
@@ -1205,8 +1594,14 @@ async function openComputer(
   if (value.computerSessionId) {
     session = await transport.getComputerSession(workspaceId, value.computerSessionId);
   } else {
-    const listed = await transport.listComputerSessions(workspaceId);
-    const reusable = value.mode === "new" ? null : newestRelevant(listed.sessions, sourceSessionId);
+    const listed =
+      value.mode === "new" ? { sessions: [] } : await transport.listComputerSessions(workspaceId);
+    const reusable = newestRelevant(
+      listed.sessions.filter((candidate) =>
+        compatibleInteractionPlacement(candidate.placement, value.placement),
+      ),
+      sourceSessionId,
+    );
     session = reusable
       ? reusable
       : (
@@ -1227,6 +1622,16 @@ async function openComputer(
   };
 }
 
+function compatibleInteractionPlacement(
+  candidate: z.infer<typeof InteractionPlacement>,
+  requested: z.infer<typeof InteractionPlacement> | undefined,
+): boolean {
+  if (!requested) return candidate.kind !== "attached_device";
+  return Object.entries(requested).every(
+    ([key, value]) => (candidate as Record<string, unknown>)[key] === value,
+  );
+}
+
 function newestRelevant<
   T extends {
     lifecycle: string;
@@ -1270,6 +1675,195 @@ function assertInterventionResumeMatches(
   }
 }
 
+function flattenBrowserNodes(
+  roots: readonly BrowserSemanticNodeValue[],
+  scopeRef?: string,
+): FlatBrowserNode[] {
+  const flattened: FlatBrowserNode[] = [];
+  const stack = [...roots].reverse().map((node) => ({ node, depth: 0, withinScope: !scopeRef }));
+  while (stack.length > 0) {
+    const next = stack.pop()!;
+    const withinScope = next.withinScope || next.node.ref === scopeRef;
+    flattened.push({
+      node: next.node,
+      depth: next.depth,
+      index: flattened.length,
+      withinScope,
+    });
+    for (const child of [...(next.node.children ?? [])].reverse()) {
+      stack.push({ node: child, depth: next.depth + 1, withinScope });
+    }
+  }
+  return flattened;
+}
+
+function boundedAgentField(value: string, onClip: () => void): string {
+  if (value.length <= BROWSER_AGENT_MAX_FIELD_LENGTH) return value;
+  onClip();
+  return `${value.slice(0, BROWSER_AGENT_MAX_FIELD_LENGTH - 1)}…`;
+}
+
+function projectAgentNode(flat: FlatBrowserNode): {
+  node: z.infer<typeof BrowserAgentNode>;
+  clippedFieldCount: number;
+} {
+  let clippedFieldCount = 0;
+  const clip = () => {
+    clippedFieldCount += 1;
+  };
+  const source = flat.node;
+  const nodes = {
+    ref: source.ref,
+    role: source.role,
+    depth: flat.depth,
+    ...(source.name !== undefined ? { name: boundedAgentField(source.name, clip) } : {}),
+    ...(source.description !== undefined
+      ? { description: boundedAgentField(source.description, clip) }
+      : {}),
+    ...(source.value !== undefined
+      ? {
+          value:
+            typeof source.value === "string" ? boundedAgentField(source.value, clip) : source.value,
+        }
+      : {}),
+    states: source.states.slice(0, 6).map((state) => boundedAgentField(state, clip)),
+    actions: source.actions.slice(0, 6).map((action) => boundedAgentField(action, clip)),
+  };
+  if (source.states.length > 6) clippedFieldCount += source.states.length - 6;
+  if (source.actions.length > 6) clippedFieldCount += source.actions.length - 6;
+  return { node: nodes, clippedFieldCount };
+}
+
+function selectAgentNodes(
+  flattened: readonly FlatBrowserNode[],
+  maxNodes: number,
+  maxBytes: number,
+  prioritize = true,
+): { nodes: z.infer<typeof BrowserAgentNode>[]; clippedFieldCount: number } {
+  const priority = (flat: FlatBrowserNode): number => {
+    const { node } = flat;
+    if (node.actions.length > 0) return 0;
+    if (["heading", "dialog", "alert", "status", "navigation", "main"].includes(node.role))
+      return 1;
+    if (["text", "paragraph", "listitem"].includes(node.role) && node.name) return 2;
+    if (node.name || node.value || node.description) return 3;
+    return 4;
+  };
+  const candidates = prioritize
+    ? [...flattened].sort((a, b) => priority(a) - priority(b) || a.index - b.index)
+    : flattened;
+  const selected: Array<{ index: number; node: z.infer<typeof BrowserAgentNode> }> = [];
+  let usedBytes = 0;
+  let clippedFieldCount = 0;
+  for (const flat of candidates) {
+    if (selected.length >= maxNodes) break;
+    const projected = projectAgentNode(flat);
+    const bytes = Buffer.byteLength(JSON.stringify(projected.node), "utf8");
+    if (usedBytes + bytes > maxBytes) continue;
+    selected.push({ index: flat.index, node: projected.node });
+    usedBytes += bytes;
+    clippedFieldCount += projected.clippedFieldCount;
+  }
+  return {
+    nodes: selected.sort((a, b) => a.index - b.index).map((entry) => entry.node),
+    clippedFieldCount,
+  };
+}
+
+function projectBrowserObservation(
+  observation: z.infer<typeof BrowserObservation>,
+  view: "compact" | "full",
+): z.infer<typeof BrowserAgentObservation> {
+  if (view === "full") return observation;
+  const semantic = observation.semantic;
+  const roots =
+    semantic?.kind === "snapshot"
+      ? semantic.roots
+      : semantic?.kind === "diff"
+        ? semantic.changed
+        : [];
+  const flattened = flattenBrowserNodes(roots);
+  const projected = selectAgentNodes(
+    flattened,
+    BROWSER_AGENT_MAX_NODES,
+    BROWSER_AGENT_MAX_NODE_BYTES,
+  );
+  return {
+    ...observation,
+    semantic: null,
+    agentView: {
+      kind: "compact",
+      sourceKind: semantic?.kind ?? "none",
+      nodes: projected.nodes,
+      sourceNodeCount: flattened.length,
+      omittedNodeCount: flattened.length - projected.nodes.length,
+      removedRefCount: semantic?.kind === "diff" ? semantic.removedRefs.length : 0,
+      clippedFieldCount: projected.clippedFieldCount,
+      maxNodes: BROWSER_AGENT_MAX_NODES,
+      maxNodeBytes: BROWSER_AGENT_MAX_NODE_BYTES,
+    },
+  };
+}
+
+function readBrowserObservation(
+  observation: z.infer<typeof BrowserObservation>,
+  input: z.output<typeof BrowserReadInput>,
+): z.infer<typeof BrowserReadOutput> {
+  if (observation.semantic?.kind !== "snapshot") {
+    throw new Error("browser focused read requires a complete accessibility snapshot");
+  }
+  const mode = input.mode && input.mode !== "dom" ? input.mode : "matches";
+  const scopeRef = mode === "subtree" ? input.ref : input.scopeRef;
+  const flattened = flattenBrowserNodes(observation.semantic.roots, scopeRef);
+  const scopeFound = !scopeRef || flattened.some((entry) => entry.node.ref === scopeRef);
+  const includes = (source: string | undefined, needle: string | undefined): boolean =>
+    needle === undefined || (source ?? "").toLowerCase().includes(needle.toLowerCase());
+  const matches = flattened.filter(({ node, withinScope }) => {
+    if (!withinScope) return false;
+    if (mode !== "subtree" && input.ref && node.ref !== input.ref) return false;
+    if (input.role && node.role.toLowerCase() !== input.role.toLowerCase()) return false;
+    if (!includes(node.name, input.nameContains)) return false;
+    if (
+      input.textContains &&
+      ![node.name, node.description, typeof node.value === "string" ? node.value : undefined].some(
+        (candidate) => includes(candidate, input.textContains),
+      )
+    )
+      return false;
+    if (
+      input.state &&
+      !node.states.some((state) => state.toLowerCase() === input.state!.toLowerCase())
+    )
+      return false;
+    if (
+      input.action &&
+      !node.actions.some((action) => action.toLowerCase() === input.action!.toLowerCase())
+    )
+      return false;
+    return true;
+  });
+  const projected =
+    mode === "count"
+      ? { nodes: [] as z.infer<typeof BrowserAgentNode>[], clippedFieldCount: 0 }
+      : selectAgentNodes(matches, input.limit ?? 40, BROWSER_AGENT_MAX_NODE_BYTES, false);
+  return {
+    browserSessionId: observation.browserSessionId,
+    targetId: observation.target.id,
+    observationId: observation.observationId,
+    targetGeneration: observation.target.targetGeneration,
+    documentGeneration: observation.target.documentGeneration,
+    frameId: observation.frameId,
+    source: "accessibility",
+    mode,
+    scopeFound,
+    nodes: projected.nodes,
+    totalMatches: matches.length,
+    omittedMatches: matches.length - projected.nodes.length,
+    clippedFieldCount: projected.clippedFieldCount,
+    maxNodeBytes: BROWSER_AGENT_MAX_NODE_BYTES,
+  };
+}
+
 async function safeInteractionExecution<TInput extends z.ZodType, TOutput extends z.ZodType>(
   inputSchema: TInput,
   outputSchema: TOutput,
@@ -1311,6 +1905,20 @@ async function safeInteractionExecution<TInput extends z.ZodType, TOutput extend
   }
 }
 
+function browserToolImageContent(
+  data: Uint8Array,
+  mimeType: string,
+): AttemptToolResultValue["content"][number] {
+  if (data.byteLength > BROWSER_TOOL_IMAGE_MAX_BYTES) {
+    throw new OpenGeniApiError(
+      413,
+      "Browser screenshot exceeds the Code Mode image limit; capture the viewport or lower JPEG quality.",
+      { code: "browser_screenshot_too_large", retryable: false },
+    );
+  }
+  return { type: "image", data: Buffer.from(data).toString("base64"), mimeType };
+}
+
 function interactionErrorResult(
   code: string,
   message: string,