@opengeni/codemode 0.6.1 → 0.6.2-canary.36079201993001

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -64,13 +64,27 @@ import { openGeni } from "@opengeni/codemode";
64
64
 
65
65
  const browser = await openGeni.browsers.open({ initialUrl: "http://127.0.0.1:3000" });
66
66
  const tab = await browser.tabs.selected();
67
+ const page = await tab.observe(); // Compact refs and explicit omission counts.
68
+ const buttons = await tab.read({ role: "button", nameContains: "Save", limit: 5 });
69
+ const email = await tab.read({ mode: "dom", dom: { kind: "element", locator: { kind: "css", selector: "#email" }, attributes: ["placeholder"] } });
67
70
  await tab.getByRole("button", { name: "Save" }).click();
71
+ const still = await tab.screenshot({ quality: 40 });
72
+ console.log(still.path); // Open this local PNG/JPEG/WebP with view_image.
68
73
 
69
74
  const computer = await openGeni.computers.open();
70
75
  const app = await computer.apps.focused();
71
76
  await app.getByRole("button", { name: "1" }).invoke();
72
77
  ```
73
78
 
79
+ `tab.observeFull()` retrieves the complete accessibility snapshot when compact
80
+ refs or a focused `tab.read()` query are insufficient. The default focused read
81
+ searches accessibility; `mode: "dom"` returns bounded text, editable values,
82
+ safe attributes, or counts. Sensitive fields are redacted. DOM CSS reads accept
83
+ simple tag, class, id, descendant, and child selectors; attribute and pseudo
84
+ selectors are rejected to prevent secret-value probing. Browser image blocks
85
+ are bounded below the Code Mode journal limit; for an oversized screenshot,
86
+ capture the viewport or lower JPEG quality.
87
+
74
88
  Both surfaces return the same durable tool receipts. Human approval, catalog
75
89
  generation, operation idempotency, and outcome-unknown behavior remain enforced
76
90
  by the shared attempt executor. Catalog, approval, authorization, input
@@ -84,6 +98,24 @@ directly cannot bypass approval. The dispatcher keeps its claim alive during
84
98
  gateway preparation and reuses a deterministic durable tool-created event if a
85
99
  pre-execution claim is reclaimed.
86
100
 
101
+ `tab.screenshot({ fullPage?: boolean, saveTo?: string })` calls the same atomic
102
+ `interaction.browser.screenshot` tool and writes its image block to a private
103
+ local file. It returns the absolute path, MIME type, byte count, and small frame
104
+ metadata, without printing base64 pixels. Use the agent's `view_image` tool on
105
+ that path to inspect the still. `saveTo` writes to an exact caller-chosen path
106
+ and refuses to overwrite an existing file. `CodemodeClient.callPath` remains
107
+ available when code needs the complete MCP result, including image blocks.
108
+ Default image paths are private OS scratch files; later image calls remove this
109
+ client's scratch directories older than 24 hours. Use `saveTo` when a file must
110
+ remain available longer; that path is caller-owned and never pruned by Code Mode.
111
+ The generated `tools.*` namespace normally returns structured output directly.
112
+ When a typed tool returns image blocks, it instead returns
113
+ `{ structuredContent, images, otherContent }`: `images` contains local paths,
114
+ and `otherContent` preserves text and other non-image blocks. Generated
115
+ declarations express this union. Open `images[n].path` with `view_image`.
116
+ The facade and typed wrapper keep base64 out of program output; raw `callPath`
117
+ retains it and should not be printed wholesale.
118
+
87
119
  `CodemodeCallOptions.signal` cancels only the caller's HTTP/polling observation.
88
120
  It does not request server cancellation and cannot prove that an operation
89
121
  stopped. The attempt/turn lifecycle remains the only cancellation authority; a
@@ -0,0 +1,22 @@
1
+ import type { AttemptToolContent, AttemptToolResult } from "@opengeni/contracts";
2
+ type ImageMimeType = "image/png" | "image/jpeg" | "image/webp";
3
+ export type CodemodeLocalImage = Readonly<{
4
+ /** Local path readable with the agent's `view_image` tool. */
5
+ path: string;
6
+ mimeType: ImageMimeType;
7
+ sizeBytes: number;
8
+ }>;
9
+ /** Typed Code Mode calls with image blocks retain every structured and non-image block. */
10
+ export type CodemodeMultimodalValue<T> = Readonly<{
11
+ structuredContent: T;
12
+ images: readonly CodemodeLocalImage[];
13
+ otherContent: readonly Exclude<AttemptToolContent, {
14
+ type: "image";
15
+ }>[];
16
+ }>;
17
+ /** Preserve image blocks as private local files instead of base64 in program output. */
18
+ export declare function materializeCodemodeImages(result: AttemptToolResult, options?: {
19
+ saveTo?: string;
20
+ requireOne?: boolean;
21
+ }): Promise<CodemodeLocalImage[]>;
22
+ export {};
package/dist/index.d.ts CHANGED
@@ -115,7 +115,7 @@ export declare class CodemodeClient {
115
115
  private callResolved;
116
116
  /** Resolve and call one exact generated namespace path without parsing a wire name. */
117
117
  callPath(path: readonly string[], argumentsValue?: Record<string, unknown>, options?: CodemodeCallOptions): Promise<AttemptToolResultValue>;
118
- /** Return structured content when the catalog declares it; otherwise retain the full MCP result. */
118
+ /** Return structured content, or a local-image wrapper for typed multimodal results. */
119
119
  callPathValue(path: readonly string[], argumentsValue?: Record<string, unknown>, options?: CodemodeCallOptions): Promise<unknown>;
120
120
  private submit;
121
121
  private read;
@@ -151,6 +151,7 @@ export declare function decodeCodemodeDispatchAck(input: Uint8Array): CodemodeDi
151
151
  export declare function parseVerifiedAttemptToolCatalog(input: unknown): AttemptToolCatalogValue;
152
152
  export * from "./environment.js";
153
153
  export * from "./interaction.js";
154
+ export * from "./images.js";
154
155
  export * from "./artifacts.js";
155
156
  export * from "./structured.js";
156
157
  export * from "./declarations.js";
package/dist/index.js CHANGED
@@ -26,6 +26,101 @@ import {
26
26
  } from "@opengeni/tool-gateway";
27
27
  import { siteSessionPath } from "@opengeni/contracts/site-session-http";
28
28
 
29
+ // src/images.ts
30
+ import { lstat, mkdir, mkdtemp, readdir, rm, writeFile } from "fs/promises";
31
+ import { tmpdir } from "os";
32
+ import { dirname, join, resolve } from "path";
33
+ var MAX_IMAGE_BASE64_CHARS = 16 * 1024 * 1024;
34
+ var TEMP_IMAGE_PREFIX = "opengeni-codemode-image-";
35
+ var TEMP_IMAGE_MAX_AGE_MS = 24 * 60 * 60 * 1e3;
36
+ var TEMP_IMAGE_PRUNE_INTERVAL_MS = 60 * 60 * 1e3;
37
+ var lastPruneAt = 0;
38
+ async function materializeCodemodeImages(result, options = {}) {
39
+ const blocks = result.content.filter((content) => content.type === "image");
40
+ if (options.requireOne && blocks.length !== 1) {
41
+ throw new Error("Codemode browser screenshot must return exactly one image block");
42
+ }
43
+ if (blocks.length === 0) return [];
44
+ if (options.saveTo !== void 0 && blocks.length !== 1) {
45
+ throw new Error("An explicit image path requires exactly one image block");
46
+ }
47
+ let totalBase64Chars = 0;
48
+ const decoded = blocks.map((block) => {
49
+ const mimeType = supportedMimeType(block.mimeType);
50
+ totalBase64Chars += block.data.length;
51
+ if (block.data.length === 0 || block.data.length > MAX_IMAGE_BASE64_CHARS || totalBase64Chars > MAX_IMAGE_BASE64_CHARS) {
52
+ throw new Error("Codemode image exceeds the Code Mode result limit");
53
+ }
54
+ const bytes = Buffer.from(block.data, "base64");
55
+ if (bytes.length === 0 || bytes.toString("base64") !== block.data || !matchesImageType(bytes, mimeType)) {
56
+ throw new Error("Codemode image bytes are invalid");
57
+ }
58
+ return { bytes, mimeType };
59
+ });
60
+ let directory;
61
+ if (options.saveTo === void 0) {
62
+ await pruneExpiredPrivateImages();
63
+ directory = await mkdtemp(join(tmpdir(), TEMP_IMAGE_PREFIX));
64
+ } else {
65
+ if (options.saveTo.length === 0) throw new TypeError("Image saveTo must be a nonempty path");
66
+ await mkdir(dirname(resolve(options.saveTo)), { recursive: true });
67
+ }
68
+ const images = [];
69
+ try {
70
+ for (const [index, image] of decoded.entries()) {
71
+ const extension = image.mimeType === "image/jpeg" ? "jpg" : image.mimeType.split("/")[1];
72
+ const path = options.saveTo === void 0 ? join(
73
+ directory,
74
+ decoded.length === 1 ? `frame.${extension}` : `image-${index + 1}.${extension}`
75
+ ) : resolve(options.saveTo);
76
+ await writeFile(path, image.bytes, { flag: "wx", mode: 384 });
77
+ images.push({ path, mimeType: image.mimeType, sizeBytes: image.bytes.length });
78
+ }
79
+ } catch (error) {
80
+ if (directory) await rm(directory, { recursive: true, force: true });
81
+ throw error;
82
+ }
83
+ return images;
84
+ }
85
+ async function pruneExpiredPrivateImages() {
86
+ const now = Date.now();
87
+ if (now - lastPruneAt < TEMP_IMAGE_PRUNE_INTERVAL_MS) return;
88
+ lastPruneAt = now;
89
+ if (typeof process.getuid !== "function") return;
90
+ try {
91
+ let removed = 0;
92
+ for (const entry of await readdir(tmpdir(), { withFileTypes: true })) {
93
+ if (!entry.isDirectory() || !/^opengeni-codemode-image-[A-Za-z0-9]{6}$/u.test(entry.name)) {
94
+ continue;
95
+ }
96
+ const path = join(tmpdir(), entry.name);
97
+ const info = await lstat(path);
98
+ if (!info.isDirectory() || info.uid !== process.getuid() || (info.mode & 63) !== 0 || now - info.mtimeMs < TEMP_IMAGE_MAX_AGE_MS) {
99
+ continue;
100
+ }
101
+ await rm(path, { recursive: true, force: true });
102
+ if (++removed >= 64) break;
103
+ }
104
+ } catch {
105
+ }
106
+ }
107
+ function supportedMimeType(mimeType) {
108
+ if (mimeType === "image/png" || mimeType === "image/jpeg" || mimeType === "image/webp") {
109
+ return mimeType;
110
+ }
111
+ throw new Error(`Codemode image has unsupported type: ${mimeType}`);
112
+ }
113
+ function matchesImageType(bytes, mimeType) {
114
+ switch (mimeType) {
115
+ case "image/png":
116
+ return bytes.length >= 8 && bytes.subarray(0, 8).equals(Buffer.from("89504e470d0a1a0a", "hex"));
117
+ case "image/jpeg":
118
+ return bytes.length >= 3 && bytes[0] === 255 && bytes[1] === 216 && bytes[2] === 255;
119
+ case "image/webp":
120
+ return bytes.length >= 12 && bytes.toString("ascii", 0, 4) === "RIFF" && bytes.toString("ascii", 8, 12) === "WEBP";
121
+ }
122
+ }
123
+
29
124
  // src/environment.ts
30
125
  import { readFile } from "fs/promises";
31
126
  var CODEMODE_ENVIRONMENT = {
@@ -305,6 +400,8 @@ var PATH2 = {
305
400
  browserOpen: ["interaction", "browser", "open"],
306
401
  browserTabs: ["interaction", "browser", "tabs"],
307
402
  browserObserve: ["interaction", "browser", "observe"],
403
+ browserRead: ["interaction", "browser", "read"],
404
+ browserScreenshot: ["interaction", "browser", "screenshot"],
308
405
  browserAct: ["interaction", "browser", "act"],
309
406
  browserClipboard: ["interaction", "browser", "clipboard"],
310
407
  browserDebug: ["interaction", "browser", "debug"],
@@ -419,6 +516,16 @@ var CodemodeBrowser = class {
419
516
  async observe(targetId, callOptions = {}) {
420
517
  return await this.tabs.use(await this.tabs.resolveId(targetId, callOptions)).observe(callOptions);
421
518
  }
519
+ async observeFull(targetId, callOptions = {}) {
520
+ return await this.tabs.use(await this.tabs.resolveId(targetId, callOptions)).observeFull(callOptions);
521
+ }
522
+ async read(options = {}, callOptions = {}) {
523
+ const { targetId, ...query } = options;
524
+ return await this.tabs.use(await this.tabs.resolveId(targetId, callOptions)).read(query, callOptions);
525
+ }
526
+ async screenshot(options = {}, callOptions = {}) {
527
+ return await this.tabs.use(await this.tabs.resolveId(options.targetId, callOptions)).screenshot(options, callOptions);
528
+ }
422
529
  async act(action, options = {}, callOptions = {}) {
423
530
  const targetId = await this.tabs.resolveId(options.targetId, callOptions);
424
531
  return await this.tabs.use(targetId).act(action, options, callOptions);
@@ -609,6 +716,35 @@ var CodemodeBrowserTab = class {
609
716
  callOptions
610
717
  );
611
718
  }
719
+ async observeFull(callOptions = {}) {
720
+ return await callStructured(
721
+ this.client,
722
+ PATH2.browserObserve,
723
+ { browserSessionId: this.browserSessionId, targetId: this.id, view: "full" },
724
+ callOptions
725
+ );
726
+ }
727
+ async read(options = {}, callOptions = {}) {
728
+ return await callStructured(
729
+ this.client,
730
+ PATH2.browserRead,
731
+ { browserSessionId: this.browserSessionId, targetId: this.id, ...options },
732
+ callOptions
733
+ );
734
+ }
735
+ async screenshot(options = {}, callOptions = {}) {
736
+ const result = await (await this.client()).callPath(
737
+ PATH2.browserScreenshot,
738
+ {
739
+ browserSessionId: this.browserSessionId,
740
+ targetId: this.id,
741
+ ...options.fullPage === void 0 ? {} : { fullPage: options.fullPage },
742
+ ...options.quality === void 0 ? {} : { quality: options.quality }
743
+ },
744
+ callOptions
745
+ );
746
+ return await materializeBrowserScreenshot(result, options.saveTo);
747
+ }
612
748
  async act(action, fences = {}, callOptions = {}) {
613
749
  return await callStructured(
614
750
  this.client,
@@ -1068,6 +1204,24 @@ var CodemodeComputerLocator = class {
1068
1204
  );
1069
1205
  }
1070
1206
  };
1207
+ async function materializeBrowserScreenshot(result, saveTo) {
1208
+ if (result.isError) throw new CodemodeToolExecutionError(result);
1209
+ if (!result.structuredContent) {
1210
+ throw new Error("Codemode tool interaction.browser.screenshot returned no structured content");
1211
+ }
1212
+ const image = result.content.find((content) => content.type === "image");
1213
+ if (typeof result.structuredContent.mediaType === "string" && image && result.structuredContent.mediaType !== image.mimeType) {
1214
+ throw new Error("Codemode browser screenshot image type differs from frame metadata");
1215
+ }
1216
+ const images = await materializeCodemodeImages(result, {
1217
+ ...saveTo === void 0 ? {} : { saveTo },
1218
+ requireOne: true
1219
+ });
1220
+ return {
1221
+ ...images[0],
1222
+ metadata: result.structuredContent
1223
+ };
1224
+ }
1071
1225
  function createOpenGeniCodemode(client = () => environmentCodemodeClient()) {
1072
1226
  return new OpenGeniCodemode(client);
1073
1227
  }
@@ -1084,6 +1238,7 @@ function generateCodemodeDeclarations(catalog, options = {}) {
1084
1238
  interfaceName: "CodemodeGeneratedTools",
1085
1239
  callOptionsType: "CodemodeCallOptions",
1086
1240
  fallbackResultType: "CodemodeToolResult",
1241
+ typedResultWrapperType: "CodemodeMultimodalValue",
1087
1242
  generatedBy: "@opengeni/codemode via @opengeni/tool-gateway",
1088
1243
  catalogDigestLabel: "Attempt catalog digest"
1089
1244
  }
@@ -1399,7 +1554,7 @@ var CodemodeClient = class {
1399
1554
  options
1400
1555
  )).result;
1401
1556
  }
1402
- /** Return structured content when the catalog declares it; otherwise retain the full MCP result. */
1557
+ /** Return structured content, or a local-image wrapper for typed multimodal results. */
1403
1558
  async callPathValue(path, argumentsValue = {}, options = {}) {
1404
1559
  if (path.length < 2 || path.some((segment) => segment.length === 0)) {
1405
1560
  throw new AttemptToolNotFoundError();
@@ -1416,7 +1571,15 @@ var CodemodeClient = class {
1416
1571
  `Codemode tool ${path.join(".")} declared outputSchema but returned no structured content`
1417
1572
  );
1418
1573
  }
1419
- return result.structuredContent;
1574
+ if (!result.content.some((content) => content.type === "image")) {
1575
+ return result.structuredContent;
1576
+ }
1577
+ const images = await materializeCodemodeImages(result);
1578
+ return {
1579
+ structuredContent: result.structuredContent,
1580
+ images,
1581
+ otherContent: result.content.filter((content) => content.type !== "image")
1582
+ };
1420
1583
  }
1421
1584
  async submit(operationId, catalogDigest, identity, argumentsValue, signal) {
1422
1585
  const response = await this.request("/calls", {
@@ -1656,14 +1819,14 @@ function parseCodemodeApiError(input) {
1656
1819
  }
1657
1820
  async function abortableDelay(delayMs, signal) {
1658
1821
  if (!signal) {
1659
- await new Promise((resolve) => setTimeout(resolve, delayMs));
1822
+ await new Promise((resolve2) => setTimeout(resolve2, delayMs));
1660
1823
  return;
1661
1824
  }
1662
1825
  throwIfAborted(signal);
1663
- await new Promise((resolve, reject) => {
1826
+ await new Promise((resolve2, reject) => {
1664
1827
  const timer = setTimeout(() => {
1665
1828
  signal.removeEventListener("abort", onAbort);
1666
- resolve();
1829
+ resolve2();
1667
1830
  }, delayMs);
1668
1831
  const onAbort = () => {
1669
1832
  clearTimeout(timer);
@@ -1728,6 +1891,7 @@ export {
1728
1891
  environmentCodemodeClient,
1729
1892
  generateCodemodeDeclarations,
1730
1893
  jsonSchemaToTypeScript,
1894
+ materializeCodemodeImages,
1731
1895
  openGeni,
1732
1896
  parseVerifiedAttemptToolCatalog,
1733
1897
  tools