@opengeni/codemode 0.6.1 → 0.6.2-canary.36079201993001

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@opengeni/codemode",
3
- "version": "0.6.1",
3
+ "version": "0.6.2-canary.36079201993001",
4
4
  "description": "Attempt-frozen programmatic tool catalog and execution authority for OpenGeni.",
5
5
  "license": "Apache-2.0",
6
6
  "repository": {
@@ -34,9 +34,9 @@
34
34
  "prepublishOnly": "bash ../../scripts/prepublish-guard"
35
35
  },
36
36
  "dependencies": {
37
- "@opengeni/contracts": "^5.1.0",
38
- "@opengeni/sdk": "^7.1.0",
39
- "@opengeni/tool-gateway": "^0.1.12"
37
+ "@opengeni/contracts": "^5.1.1-canary.36079201993001",
38
+ "@opengeni/sdk": "^7.1.1-canary.36079201993001",
39
+ "@opengeni/tool-gateway": "^0.1.13-canary.36079201993001"
40
40
  },
41
41
  "engines": {
42
42
  "node": ">=18"
@@ -23,6 +23,7 @@ export function generateCodemodeDeclarations(
23
23
  interfaceName: "CodemodeGeneratedTools",
24
24
  callOptionsType: "CodemodeCallOptions",
25
25
  fallbackResultType: "CodemodeToolResult",
26
+ typedResultWrapperType: "CodemodeMultimodalValue",
26
27
  generatedBy: "@opengeni/codemode via @opengeni/tool-gateway",
27
28
  catalogDigestLabel: "Attempt catalog digest",
28
29
  },
package/src/images.ts ADDED
@@ -0,0 +1,145 @@
1
+ import { lstat, mkdir, mkdtemp, readdir, rm, writeFile } from "node:fs/promises";
2
+ import { tmpdir } from "node:os";
3
+ import { dirname, join, resolve } from "node:path";
4
+
5
+ import type { AttemptToolContent, AttemptToolResult } from "@opengeni/contracts";
6
+
7
+ const MAX_IMAGE_BASE64_CHARS = 16 * 1024 * 1024;
8
+ const TEMP_IMAGE_PREFIX = "opengeni-codemode-image-";
9
+ const TEMP_IMAGE_MAX_AGE_MS = 24 * 60 * 60 * 1000;
10
+ const TEMP_IMAGE_PRUNE_INTERVAL_MS = 60 * 60 * 1000;
11
+ let lastPruneAt = 0;
12
+ type ImageMimeType = "image/png" | "image/jpeg" | "image/webp";
13
+
14
+ export type CodemodeLocalImage = Readonly<{
15
+ /** Local path readable with the agent's `view_image` tool. */
16
+ path: string;
17
+ mimeType: ImageMimeType;
18
+ sizeBytes: number;
19
+ }>;
20
+
21
+ /** Typed Code Mode calls with image blocks retain every structured and non-image block. */
22
+ export type CodemodeMultimodalValue<T> = Readonly<{
23
+ structuredContent: T;
24
+ images: readonly CodemodeLocalImage[];
25
+ otherContent: readonly Exclude<AttemptToolContent, { type: "image" }>[];
26
+ }>;
27
+
28
+ /** Preserve image blocks as private local files instead of base64 in program output. */
29
+ export async function materializeCodemodeImages(
30
+ result: AttemptToolResult,
31
+ options: { saveTo?: string; requireOne?: boolean } = {},
32
+ ): Promise<CodemodeLocalImage[]> {
33
+ const blocks = result.content.filter((content) => content.type === "image");
34
+ if (options.requireOne && blocks.length !== 1) {
35
+ throw new Error("Codemode browser screenshot must return exactly one image block");
36
+ }
37
+ if (blocks.length === 0) return [];
38
+ if (options.saveTo !== undefined && blocks.length !== 1) {
39
+ throw new Error("An explicit image path requires exactly one image block");
40
+ }
41
+
42
+ let totalBase64Chars = 0;
43
+ const decoded = blocks.map((block) => {
44
+ const mimeType = supportedMimeType(block.mimeType);
45
+ totalBase64Chars += block.data.length;
46
+ if (
47
+ block.data.length === 0 ||
48
+ block.data.length > MAX_IMAGE_BASE64_CHARS ||
49
+ totalBase64Chars > MAX_IMAGE_BASE64_CHARS
50
+ ) {
51
+ throw new Error("Codemode image exceeds the Code Mode result limit");
52
+ }
53
+ const bytes = Buffer.from(block.data, "base64");
54
+ if (
55
+ bytes.length === 0 ||
56
+ bytes.toString("base64") !== block.data ||
57
+ !matchesImageType(bytes, mimeType)
58
+ ) {
59
+ throw new Error("Codemode image bytes are invalid");
60
+ }
61
+ return { bytes, mimeType };
62
+ });
63
+
64
+ let directory: string | undefined;
65
+ if (options.saveTo === undefined) {
66
+ await pruneExpiredPrivateImages();
67
+ directory = await mkdtemp(join(tmpdir(), TEMP_IMAGE_PREFIX));
68
+ } else {
69
+ if (options.saveTo.length === 0) throw new TypeError("Image saveTo must be a nonempty path");
70
+ await mkdir(dirname(resolve(options.saveTo)), { recursive: true });
71
+ }
72
+ const images: CodemodeLocalImage[] = [];
73
+ try {
74
+ for (const [index, image] of decoded.entries()) {
75
+ const extension = image.mimeType === "image/jpeg" ? "jpg" : image.mimeType.split("/")[1]!;
76
+ const path =
77
+ options.saveTo === undefined
78
+ ? join(
79
+ directory!,
80
+ decoded.length === 1 ? `frame.${extension}` : `image-${index + 1}.${extension}`,
81
+ )
82
+ : resolve(options.saveTo);
83
+ await writeFile(path, image.bytes, { flag: "wx", mode: 0o600 });
84
+ images.push({ path, mimeType: image.mimeType, sizeBytes: image.bytes.length });
85
+ }
86
+ } catch (error) {
87
+ if (directory) await rm(directory, { recursive: true, force: true });
88
+ throw error;
89
+ }
90
+ return images;
91
+ }
92
+
93
+ /** Opportunistic cleanup across short-lived scripts; caller-owned saveTo paths are never touched. */
94
+ async function pruneExpiredPrivateImages(): Promise<void> {
95
+ const now = Date.now();
96
+ if (now - lastPruneAt < TEMP_IMAGE_PRUNE_INTERVAL_MS) return;
97
+ lastPruneAt = now;
98
+ if (typeof process.getuid !== "function") return;
99
+ try {
100
+ let removed = 0;
101
+ for (const entry of await readdir(tmpdir(), { withFileTypes: true })) {
102
+ if (!entry.isDirectory() || !/^opengeni-codemode-image-[A-Za-z0-9]{6}$/u.test(entry.name)) {
103
+ continue;
104
+ }
105
+ const path = join(tmpdir(), entry.name);
106
+ const info = await lstat(path);
107
+ if (
108
+ !info.isDirectory() ||
109
+ info.uid !== process.getuid() ||
110
+ (info.mode & 0o077) !== 0 ||
111
+ now - info.mtimeMs < TEMP_IMAGE_MAX_AGE_MS
112
+ ) {
113
+ continue;
114
+ }
115
+ await rm(path, { recursive: true, force: true });
116
+ if (++removed >= 64) break;
117
+ }
118
+ } catch {
119
+ // Temp cleanup must not block a live screenshot.
120
+ }
121
+ }
122
+
123
+ function supportedMimeType(mimeType: string): ImageMimeType {
124
+ if (mimeType === "image/png" || mimeType === "image/jpeg" || mimeType === "image/webp") {
125
+ return mimeType;
126
+ }
127
+ throw new Error(`Codemode image has unsupported type: ${mimeType}`);
128
+ }
129
+
130
+ function matchesImageType(bytes: Buffer, mimeType: ImageMimeType): boolean {
131
+ switch (mimeType) {
132
+ case "image/png":
133
+ return (
134
+ bytes.length >= 8 && bytes.subarray(0, 8).equals(Buffer.from("89504e470d0a1a0a", "hex"))
135
+ );
136
+ case "image/jpeg":
137
+ return bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff;
138
+ case "image/webp":
139
+ return (
140
+ bytes.length >= 12 &&
141
+ bytes.toString("ascii", 0, 4) === "RIFF" &&
142
+ bytes.toString("ascii", 8, 12) === "WEBP"
143
+ );
144
+ }
145
+ }
package/src/index.ts CHANGED
@@ -37,6 +37,7 @@ import {
37
37
  } from "@opengeni/tool-gateway";
38
38
 
39
39
  import { siteSessionPath } from "@opengeni/contracts/site-session-http";
40
+ import { materializeCodemodeImages } from "./images";
40
41
 
41
42
  export type { AttemptToolCatalog, AttemptToolCatalogEntry } from "@opengeni/contracts";
42
43
 
@@ -364,7 +365,7 @@ export class CodemodeClient {
364
365
  ).result;
365
366
  }
366
367
 
367
- /** Return structured content when the catalog declares it; otherwise retain the full MCP result. */
368
+ /** Return structured content, or a local-image wrapper for typed multimodal results. */
368
369
  async callPathValue(
369
370
  path: readonly string[],
370
371
  argumentsValue: Record<string, unknown> = {},
@@ -385,7 +386,15 @@ export class CodemodeClient {
385
386
  `Codemode tool ${path.join(".")} declared outputSchema but returned no structured content`,
386
387
  );
387
388
  }
388
- return result.structuredContent;
389
+ if (!result.content.some((content) => content.type === "image")) {
390
+ return result.structuredContent;
391
+ }
392
+ const images = await materializeCodemodeImages(result);
393
+ return {
394
+ structuredContent: result.structuredContent,
395
+ images,
396
+ otherContent: result.content.filter((content) => content.type !== "image"),
397
+ };
389
398
  }
390
399
 
391
400
  private async submit(
@@ -762,6 +771,7 @@ async function abortableDelay(delayMs: number, signal?: AbortSignal): Promise<vo
762
771
 
763
772
  export * from "./environment";
764
773
  export * from "./interaction";
774
+ export * from "./images";
765
775
  export * from "./artifacts";
766
776
  export * from "./structured";
767
777
  export * from "./declarations";
@@ -1,4 +1,5 @@
1
1
  import type {
2
+ AttemptToolResult,
2
3
  AttachedBrowserDevice,
3
4
  AuthRun,
4
5
  AuthRunListResponse,
@@ -8,6 +9,9 @@ import type {
8
9
  BrowserClipboard,
9
10
  BrowserDiagnosticBatch,
10
11
  BrowserDiagnosticKind,
12
+ BrowserDomReadLocator,
13
+ BrowserDomReadResponse,
14
+ BrowserDomSafeAttribute,
11
15
  BrowserIdentity,
12
16
  BrowserIdentityListResponse,
13
17
  BrowserIdentityMutationResponse,
@@ -47,7 +51,8 @@ import type {
47
51
  import type { CodemodeCallOptions, CodemodeClient } from "./index";
48
52
  import { environmentCodemodeClient, type CodemodeClientProvider } from "./environment";
49
53
  import { CodemodeArtifactCollection } from "./artifacts";
50
- import { callStructured, codemodeClientProvider } from "./structured";
54
+ import { materializeCodemodeImages, type CodemodeLocalImage } from "./images";
55
+ import { callStructured, codemodeClientProvider, CodemodeToolExecutionError } from "./structured";
51
56
 
52
57
  export { CodemodeToolExecutionError } from "./structured";
53
58
 
@@ -56,6 +61,8 @@ const PATH = {
56
61
  browserOpen: ["interaction", "browser", "open"],
57
62
  browserTabs: ["interaction", "browser", "tabs"],
58
63
  browserObserve: ["interaction", "browser", "observe"],
64
+ browserRead: ["interaction", "browser", "read"],
65
+ browserScreenshot: ["interaction", "browser", "screenshot"],
59
66
  browserAct: ["interaction", "browser", "act"],
60
67
  browserClipboard: ["interaction", "browser", "clipboard"],
61
68
  browserDebug: ["interaction", "browser", "debug"],
@@ -108,6 +115,124 @@ export type BrowserActionFences = {
108
115
  expectedFrameId?: string | null | undefined;
109
116
  };
110
117
 
118
+ export type CodemodeBrowserScreenshotOptions = {
119
+ /** Capture the entire scrollable page when supported by the browser. */
120
+ fullPage?: boolean | undefined;
121
+ /** JPEG quality, 1–100. Reduce it if a large image exceeds the journal bound. */
122
+ quality?: number | undefined;
123
+ /** Save the still at this exact path. Existing files are never overwritten. */
124
+ saveTo?: string | undefined;
125
+ };
126
+
127
+ export type CodemodeBrowserScreenshotMetadata = Readonly<{
128
+ kind: "browser_screenshot";
129
+ browserSessionId: string;
130
+ targetId: string;
131
+ frameId: string;
132
+ targetGeneration: string;
133
+ documentGeneration: string;
134
+ mediaType: "image/png" | "image/jpeg";
135
+ width: number;
136
+ height: number;
137
+ deviceScaleFactor: number;
138
+ scrollX: number;
139
+ scrollY: number;
140
+ capturedAt: string;
141
+ fullPage: boolean;
142
+ }>;
143
+
144
+ export type CodemodeBrowserScreenshot = CodemodeLocalImage &
145
+ Readonly<{
146
+ /** Small structured frame metadata; pixel bytes stay in the local image file. */
147
+ metadata: CodemodeBrowserScreenshotMetadata;
148
+ }>;
149
+
150
+ export type CodemodeBrowserAgentNode = Readonly<{
151
+ ref: string;
152
+ role: string;
153
+ depth: number;
154
+ name?: string;
155
+ description?: string;
156
+ value?: string | { redacted: true; reason: string };
157
+ states: readonly string[];
158
+ actions: readonly string[];
159
+ }>;
160
+
161
+ export type CodemodeBrowserAgentView = Readonly<{
162
+ kind: "compact";
163
+ sourceKind: "snapshot" | "diff" | "none";
164
+ nodes: readonly CodemodeBrowserAgentNode[];
165
+ sourceNodeCount: number;
166
+ omittedNodeCount: number;
167
+ removedRefCount: number;
168
+ clippedFieldCount: number;
169
+ maxNodes: number;
170
+ maxNodeBytes: number;
171
+ }>;
172
+
173
+ export type CodemodeBrowserAgentObservation = BrowserObservation & {
174
+ /** Present on the compact default; absent when the full tree was requested. */
175
+ agentView?: CodemodeBrowserAgentView;
176
+ };
177
+
178
+ export type CodemodeBrowserActionReceipt = Omit<BrowserActionReceipt, "observation"> & {
179
+ observation: CodemodeBrowserAgentObservation | null;
180
+ };
181
+
182
+ export type CodemodeBrowserActionOptions = BrowserActionFences & {
183
+ view?: "compact" | "full" | "none" | undefined;
184
+ };
185
+
186
+ export type CodemodeBrowserAxReadOptions = Readonly<{
187
+ mode?: "matches" | "count" | "subtree";
188
+ ref?: string;
189
+ scopeRef?: string;
190
+ role?: string;
191
+ nameContains?: string;
192
+ textContains?: string;
193
+ state?: string;
194
+ action?: string;
195
+ limit?: number;
196
+ }>;
197
+
198
+ export type CodemodeBrowserDomReadOptions = BrowserActionFences &
199
+ Readonly<{
200
+ mode: "dom";
201
+ dom:
202
+ | {
203
+ kind: "element";
204
+ locator: BrowserDomReadLocator;
205
+ attributes?: readonly BrowserDomSafeAttribute[];
206
+ maxChars?: number;
207
+ }
208
+ | { kind: "count"; selector: string };
209
+ }>;
210
+
211
+ export type CodemodeBrowserReadOptions =
212
+ | CodemodeBrowserAxReadOptions
213
+ | CodemodeBrowserDomReadOptions;
214
+
215
+ export type CodemodeBrowserAxReadResult = Readonly<{
216
+ browserSessionId: string;
217
+ targetId: string;
218
+ observationId: string;
219
+ targetGeneration: string;
220
+ documentGeneration: string | null;
221
+ frameId: string | null;
222
+ source: "accessibility";
223
+ mode: "matches" | "count" | "subtree";
224
+ scopeFound: boolean;
225
+ nodes: readonly CodemodeBrowserAgentNode[];
226
+ totalMatches: number;
227
+ omittedMatches: number;
228
+ clippedFieldCount: number;
229
+ maxNodeBytes: number;
230
+ }>;
231
+
232
+ export type CodemodeBrowserReadResult =
233
+ | CodemodeBrowserAxReadResult
234
+ | (BrowserDomReadResponse & { source: "dom" });
235
+
111
236
  export type ComputerActionFences = {
112
237
  expectedTargetGeneration?: string | undefined;
113
238
  expectedObservationId?: string | null | undefined;
@@ -273,17 +398,45 @@ export class CodemodeBrowser {
273
398
  async observe(
274
399
  targetId?: string,
275
400
  callOptions: CodemodeCallOptions = {},
276
- ): Promise<BrowserObservation> {
401
+ ): Promise<CodemodeBrowserAgentObservation> {
277
402
  return await this.tabs
278
403
  .use(await this.tabs.resolveId(targetId, callOptions))
279
404
  .observe(callOptions);
280
405
  }
281
406
 
407
+ async observeFull(
408
+ targetId?: string,
409
+ callOptions: CodemodeCallOptions = {},
410
+ ): Promise<BrowserObservation> {
411
+ return await this.tabs
412
+ .use(await this.tabs.resolveId(targetId, callOptions))
413
+ .observeFull(callOptions);
414
+ }
415
+
416
+ async read(
417
+ options: CodemodeBrowserReadOptions & { targetId?: string | undefined } = {},
418
+ callOptions: CodemodeCallOptions = {},
419
+ ): Promise<CodemodeBrowserReadResult> {
420
+ const { targetId, ...query } = options;
421
+ return await this.tabs
422
+ .use(await this.tabs.resolveId(targetId, callOptions))
423
+ .read(query, callOptions);
424
+ }
425
+
426
+ async screenshot(
427
+ options: CodemodeBrowserScreenshotOptions & { targetId?: string | undefined } = {},
428
+ callOptions: CodemodeCallOptions = {},
429
+ ): Promise<CodemodeBrowserScreenshot> {
430
+ return await this.tabs
431
+ .use(await this.tabs.resolveId(options.targetId, callOptions))
432
+ .screenshot(options, callOptions);
433
+ }
434
+
282
435
  async act(
283
436
  action: BrowserAction,
284
- options: BrowserActionFences & { targetId?: string | undefined } = {},
437
+ options: CodemodeBrowserActionOptions & { targetId?: string | undefined } = {},
285
438
  callOptions: CodemodeCallOptions = {},
286
- ): Promise<BrowserActionReceipt> {
439
+ ): Promise<CodemodeBrowserActionReceipt> {
287
440
  const targetId = await this.tabs.resolveId(options.targetId, callOptions);
288
441
  return await this.tabs.use(targetId).act(action, options, callOptions);
289
442
  }
@@ -541,7 +694,7 @@ export class CodemodeBrowserTab {
541
694
  readonly id: string,
542
695
  ) {}
543
696
 
544
- async observe(callOptions: CodemodeCallOptions = {}): Promise<BrowserObservation> {
697
+ async observe(callOptions: CodemodeCallOptions = {}): Promise<CodemodeBrowserAgentObservation> {
545
698
  return await callStructured(
546
699
  this.client,
547
700
  PATH.browserObserve,
@@ -550,11 +703,51 @@ export class CodemodeBrowserTab {
550
703
  );
551
704
  }
552
705
 
706
+ async observeFull(callOptions: CodemodeCallOptions = {}): Promise<BrowserObservation> {
707
+ return await callStructured(
708
+ this.client,
709
+ PATH.browserObserve,
710
+ { browserSessionId: this.browserSessionId, targetId: this.id, view: "full" },
711
+ callOptions,
712
+ );
713
+ }
714
+
715
+ async read(
716
+ options: CodemodeBrowserReadOptions = {},
717
+ callOptions: CodemodeCallOptions = {},
718
+ ): Promise<CodemodeBrowserReadResult> {
719
+ return await callStructured(
720
+ this.client,
721
+ PATH.browserRead,
722
+ { browserSessionId: this.browserSessionId, targetId: this.id, ...options },
723
+ callOptions,
724
+ );
725
+ }
726
+
727
+ async screenshot(
728
+ options: CodemodeBrowserScreenshotOptions = {},
729
+ callOptions: CodemodeCallOptions = {},
730
+ ): Promise<CodemodeBrowserScreenshot> {
731
+ const result = await (
732
+ await this.client()
733
+ ).callPath(
734
+ PATH.browserScreenshot,
735
+ {
736
+ browserSessionId: this.browserSessionId,
737
+ targetId: this.id,
738
+ ...(options.fullPage === undefined ? {} : { fullPage: options.fullPage }),
739
+ ...(options.quality === undefined ? {} : { quality: options.quality }),
740
+ },
741
+ callOptions,
742
+ );
743
+ return await materializeBrowserScreenshot(result, options.saveTo);
744
+ }
745
+
553
746
  async act(
554
747
  action: BrowserAction,
555
- fences: BrowserActionFences = {},
748
+ fences: CodemodeBrowserActionOptions = {},
556
749
  callOptions: CodemodeCallOptions = {},
557
- ): Promise<BrowserActionReceipt> {
750
+ ): Promise<CodemodeBrowserActionReceipt> {
558
751
  return await callStructured(
559
752
  this.client,
560
753
  PATH.browserAct,
@@ -1207,6 +1400,33 @@ export class CodemodeComputerLocator {
1207
1400
  }
1208
1401
  }
1209
1402
 
1403
+ /** Keep screenshot pixels out of Code Mode text output; `view_image` reads the local file. */
1404
+ async function materializeBrowserScreenshot(
1405
+ result: AttemptToolResult,
1406
+ saveTo: string | undefined,
1407
+ ): Promise<CodemodeBrowserScreenshot> {
1408
+ if (result.isError) throw new CodemodeToolExecutionError(result);
1409
+ if (!result.structuredContent) {
1410
+ throw new Error("Codemode tool interaction.browser.screenshot returned no structured content");
1411
+ }
1412
+ const image = result.content.find((content) => content.type === "image");
1413
+ if (
1414
+ typeof result.structuredContent.mediaType === "string" &&
1415
+ image &&
1416
+ result.structuredContent.mediaType !== image.mimeType
1417
+ ) {
1418
+ throw new Error("Codemode browser screenshot image type differs from frame metadata");
1419
+ }
1420
+ const images = await materializeCodemodeImages(result, {
1421
+ ...(saveTo === undefined ? {} : { saveTo }),
1422
+ requireOne: true,
1423
+ });
1424
+ return {
1425
+ ...images[0]!,
1426
+ metadata: result.structuredContent as CodemodeBrowserScreenshotMetadata,
1427
+ };
1428
+ }
1429
+
1210
1430
  export function createOpenGeniCodemode(
1211
1431
  client: CodemodeClient | CodemodeClientProvider = () => environmentCodemodeClient(),
1212
1432
  ) {