@opengeni/codemode 0.6.1 → 0.6.2-canary.36079201993001
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -0
- package/dist/images.d.ts +22 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +169 -5
- package/dist/index.js.map +1 -1
- package/dist/interaction.d.ts +137 -831
- package/package.json +4 -4
- package/src/declarations.ts +1 -0
- package/src/images.ts +145 -0
- package/src/index.ts +12 -2
- package/src/interaction.ts +227 -7
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@opengeni/codemode",
|
|
3
|
-
"version": "0.6.
|
|
3
|
+
"version": "0.6.2-canary.36079201993001",
|
|
4
4
|
"description": "Attempt-frozen programmatic tool catalog and execution authority for OpenGeni.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"repository": {
|
|
@@ -34,9 +34,9 @@
|
|
|
34
34
|
"prepublishOnly": "bash ../../scripts/prepublish-guard"
|
|
35
35
|
},
|
|
36
36
|
"dependencies": {
|
|
37
|
-
"@opengeni/contracts": "^5.1.
|
|
38
|
-
"@opengeni/sdk": "^7.1.
|
|
39
|
-
"@opengeni/tool-gateway": "^0.1.
|
|
37
|
+
"@opengeni/contracts": "^5.1.1-canary.36079201993001",
|
|
38
|
+
"@opengeni/sdk": "^7.1.1-canary.36079201993001",
|
|
39
|
+
"@opengeni/tool-gateway": "^0.1.13-canary.36079201993001"
|
|
40
40
|
},
|
|
41
41
|
"engines": {
|
|
42
42
|
"node": ">=18"
|
package/src/declarations.ts
CHANGED
|
@@ -23,6 +23,7 @@ export function generateCodemodeDeclarations(
|
|
|
23
23
|
interfaceName: "CodemodeGeneratedTools",
|
|
24
24
|
callOptionsType: "CodemodeCallOptions",
|
|
25
25
|
fallbackResultType: "CodemodeToolResult",
|
|
26
|
+
typedResultWrapperType: "CodemodeMultimodalValue",
|
|
26
27
|
generatedBy: "@opengeni/codemode via @opengeni/tool-gateway",
|
|
27
28
|
catalogDigestLabel: "Attempt catalog digest",
|
|
28
29
|
},
|
package/src/images.ts
ADDED
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
import { lstat, mkdir, mkdtemp, readdir, rm, writeFile } from "node:fs/promises";
|
|
2
|
+
import { tmpdir } from "node:os";
|
|
3
|
+
import { dirname, join, resolve } from "node:path";
|
|
4
|
+
|
|
5
|
+
import type { AttemptToolContent, AttemptToolResult } from "@opengeni/contracts";
|
|
6
|
+
|
|
7
|
+
const MAX_IMAGE_BASE64_CHARS = 16 * 1024 * 1024;
|
|
8
|
+
const TEMP_IMAGE_PREFIX = "opengeni-codemode-image-";
|
|
9
|
+
const TEMP_IMAGE_MAX_AGE_MS = 24 * 60 * 60 * 1000;
|
|
10
|
+
const TEMP_IMAGE_PRUNE_INTERVAL_MS = 60 * 60 * 1000;
|
|
11
|
+
let lastPruneAt = 0;
|
|
12
|
+
type ImageMimeType = "image/png" | "image/jpeg" | "image/webp";
|
|
13
|
+
|
|
14
|
+
export type CodemodeLocalImage = Readonly<{
|
|
15
|
+
/** Local path readable with the agent's `view_image` tool. */
|
|
16
|
+
path: string;
|
|
17
|
+
mimeType: ImageMimeType;
|
|
18
|
+
sizeBytes: number;
|
|
19
|
+
}>;
|
|
20
|
+
|
|
21
|
+
/** Typed Code Mode calls with image blocks retain every structured and non-image block. */
|
|
22
|
+
export type CodemodeMultimodalValue<T> = Readonly<{
|
|
23
|
+
structuredContent: T;
|
|
24
|
+
images: readonly CodemodeLocalImage[];
|
|
25
|
+
otherContent: readonly Exclude<AttemptToolContent, { type: "image" }>[];
|
|
26
|
+
}>;
|
|
27
|
+
|
|
28
|
+
/** Preserve image blocks as private local files instead of base64 in program output. */
|
|
29
|
+
export async function materializeCodemodeImages(
|
|
30
|
+
result: AttemptToolResult,
|
|
31
|
+
options: { saveTo?: string; requireOne?: boolean } = {},
|
|
32
|
+
): Promise<CodemodeLocalImage[]> {
|
|
33
|
+
const blocks = result.content.filter((content) => content.type === "image");
|
|
34
|
+
if (options.requireOne && blocks.length !== 1) {
|
|
35
|
+
throw new Error("Codemode browser screenshot must return exactly one image block");
|
|
36
|
+
}
|
|
37
|
+
if (blocks.length === 0) return [];
|
|
38
|
+
if (options.saveTo !== undefined && blocks.length !== 1) {
|
|
39
|
+
throw new Error("An explicit image path requires exactly one image block");
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
let totalBase64Chars = 0;
|
|
43
|
+
const decoded = blocks.map((block) => {
|
|
44
|
+
const mimeType = supportedMimeType(block.mimeType);
|
|
45
|
+
totalBase64Chars += block.data.length;
|
|
46
|
+
if (
|
|
47
|
+
block.data.length === 0 ||
|
|
48
|
+
block.data.length > MAX_IMAGE_BASE64_CHARS ||
|
|
49
|
+
totalBase64Chars > MAX_IMAGE_BASE64_CHARS
|
|
50
|
+
) {
|
|
51
|
+
throw new Error("Codemode image exceeds the Code Mode result limit");
|
|
52
|
+
}
|
|
53
|
+
const bytes = Buffer.from(block.data, "base64");
|
|
54
|
+
if (
|
|
55
|
+
bytes.length === 0 ||
|
|
56
|
+
bytes.toString("base64") !== block.data ||
|
|
57
|
+
!matchesImageType(bytes, mimeType)
|
|
58
|
+
) {
|
|
59
|
+
throw new Error("Codemode image bytes are invalid");
|
|
60
|
+
}
|
|
61
|
+
return { bytes, mimeType };
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
let directory: string | undefined;
|
|
65
|
+
if (options.saveTo === undefined) {
|
|
66
|
+
await pruneExpiredPrivateImages();
|
|
67
|
+
directory = await mkdtemp(join(tmpdir(), TEMP_IMAGE_PREFIX));
|
|
68
|
+
} else {
|
|
69
|
+
if (options.saveTo.length === 0) throw new TypeError("Image saveTo must be a nonempty path");
|
|
70
|
+
await mkdir(dirname(resolve(options.saveTo)), { recursive: true });
|
|
71
|
+
}
|
|
72
|
+
const images: CodemodeLocalImage[] = [];
|
|
73
|
+
try {
|
|
74
|
+
for (const [index, image] of decoded.entries()) {
|
|
75
|
+
const extension = image.mimeType === "image/jpeg" ? "jpg" : image.mimeType.split("/")[1]!;
|
|
76
|
+
const path =
|
|
77
|
+
options.saveTo === undefined
|
|
78
|
+
? join(
|
|
79
|
+
directory!,
|
|
80
|
+
decoded.length === 1 ? `frame.${extension}` : `image-${index + 1}.${extension}`,
|
|
81
|
+
)
|
|
82
|
+
: resolve(options.saveTo);
|
|
83
|
+
await writeFile(path, image.bytes, { flag: "wx", mode: 0o600 });
|
|
84
|
+
images.push({ path, mimeType: image.mimeType, sizeBytes: image.bytes.length });
|
|
85
|
+
}
|
|
86
|
+
} catch (error) {
|
|
87
|
+
if (directory) await rm(directory, { recursive: true, force: true });
|
|
88
|
+
throw error;
|
|
89
|
+
}
|
|
90
|
+
return images;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** Opportunistic cleanup across short-lived scripts; caller-owned saveTo paths are never touched. */
|
|
94
|
+
async function pruneExpiredPrivateImages(): Promise<void> {
|
|
95
|
+
const now = Date.now();
|
|
96
|
+
if (now - lastPruneAt < TEMP_IMAGE_PRUNE_INTERVAL_MS) return;
|
|
97
|
+
lastPruneAt = now;
|
|
98
|
+
if (typeof process.getuid !== "function") return;
|
|
99
|
+
try {
|
|
100
|
+
let removed = 0;
|
|
101
|
+
for (const entry of await readdir(tmpdir(), { withFileTypes: true })) {
|
|
102
|
+
if (!entry.isDirectory() || !/^opengeni-codemode-image-[A-Za-z0-9]{6}$/u.test(entry.name)) {
|
|
103
|
+
continue;
|
|
104
|
+
}
|
|
105
|
+
const path = join(tmpdir(), entry.name);
|
|
106
|
+
const info = await lstat(path);
|
|
107
|
+
if (
|
|
108
|
+
!info.isDirectory() ||
|
|
109
|
+
info.uid !== process.getuid() ||
|
|
110
|
+
(info.mode & 0o077) !== 0 ||
|
|
111
|
+
now - info.mtimeMs < TEMP_IMAGE_MAX_AGE_MS
|
|
112
|
+
) {
|
|
113
|
+
continue;
|
|
114
|
+
}
|
|
115
|
+
await rm(path, { recursive: true, force: true });
|
|
116
|
+
if (++removed >= 64) break;
|
|
117
|
+
}
|
|
118
|
+
} catch {
|
|
119
|
+
// Temp cleanup must not block a live screenshot.
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function supportedMimeType(mimeType: string): ImageMimeType {
|
|
124
|
+
if (mimeType === "image/png" || mimeType === "image/jpeg" || mimeType === "image/webp") {
|
|
125
|
+
return mimeType;
|
|
126
|
+
}
|
|
127
|
+
throw new Error(`Codemode image has unsupported type: ${mimeType}`);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
function matchesImageType(bytes: Buffer, mimeType: ImageMimeType): boolean {
|
|
131
|
+
switch (mimeType) {
|
|
132
|
+
case "image/png":
|
|
133
|
+
return (
|
|
134
|
+
bytes.length >= 8 && bytes.subarray(0, 8).equals(Buffer.from("89504e470d0a1a0a", "hex"))
|
|
135
|
+
);
|
|
136
|
+
case "image/jpeg":
|
|
137
|
+
return bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff;
|
|
138
|
+
case "image/webp":
|
|
139
|
+
return (
|
|
140
|
+
bytes.length >= 12 &&
|
|
141
|
+
bytes.toString("ascii", 0, 4) === "RIFF" &&
|
|
142
|
+
bytes.toString("ascii", 8, 12) === "WEBP"
|
|
143
|
+
);
|
|
144
|
+
}
|
|
145
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -37,6 +37,7 @@ import {
|
|
|
37
37
|
} from "@opengeni/tool-gateway";
|
|
38
38
|
|
|
39
39
|
import { siteSessionPath } from "@opengeni/contracts/site-session-http";
|
|
40
|
+
import { materializeCodemodeImages } from "./images";
|
|
40
41
|
|
|
41
42
|
export type { AttemptToolCatalog, AttemptToolCatalogEntry } from "@opengeni/contracts";
|
|
42
43
|
|
|
@@ -364,7 +365,7 @@ export class CodemodeClient {
|
|
|
364
365
|
).result;
|
|
365
366
|
}
|
|
366
367
|
|
|
367
|
-
/** Return structured content
|
|
368
|
+
/** Return structured content, or a local-image wrapper for typed multimodal results. */
|
|
368
369
|
async callPathValue(
|
|
369
370
|
path: readonly string[],
|
|
370
371
|
argumentsValue: Record<string, unknown> = {},
|
|
@@ -385,7 +386,15 @@ export class CodemodeClient {
|
|
|
385
386
|
`Codemode tool ${path.join(".")} declared outputSchema but returned no structured content`,
|
|
386
387
|
);
|
|
387
388
|
}
|
|
388
|
-
|
|
389
|
+
if (!result.content.some((content) => content.type === "image")) {
|
|
390
|
+
return result.structuredContent;
|
|
391
|
+
}
|
|
392
|
+
const images = await materializeCodemodeImages(result);
|
|
393
|
+
return {
|
|
394
|
+
structuredContent: result.structuredContent,
|
|
395
|
+
images,
|
|
396
|
+
otherContent: result.content.filter((content) => content.type !== "image"),
|
|
397
|
+
};
|
|
389
398
|
}
|
|
390
399
|
|
|
391
400
|
private async submit(
|
|
@@ -762,6 +771,7 @@ async function abortableDelay(delayMs: number, signal?: AbortSignal): Promise<vo
|
|
|
762
771
|
|
|
763
772
|
export * from "./environment";
|
|
764
773
|
export * from "./interaction";
|
|
774
|
+
export * from "./images";
|
|
765
775
|
export * from "./artifacts";
|
|
766
776
|
export * from "./structured";
|
|
767
777
|
export * from "./declarations";
|
package/src/interaction.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type {
|
|
2
|
+
AttemptToolResult,
|
|
2
3
|
AttachedBrowserDevice,
|
|
3
4
|
AuthRun,
|
|
4
5
|
AuthRunListResponse,
|
|
@@ -8,6 +9,9 @@ import type {
|
|
|
8
9
|
BrowserClipboard,
|
|
9
10
|
BrowserDiagnosticBatch,
|
|
10
11
|
BrowserDiagnosticKind,
|
|
12
|
+
BrowserDomReadLocator,
|
|
13
|
+
BrowserDomReadResponse,
|
|
14
|
+
BrowserDomSafeAttribute,
|
|
11
15
|
BrowserIdentity,
|
|
12
16
|
BrowserIdentityListResponse,
|
|
13
17
|
BrowserIdentityMutationResponse,
|
|
@@ -47,7 +51,8 @@ import type {
|
|
|
47
51
|
import type { CodemodeCallOptions, CodemodeClient } from "./index";
|
|
48
52
|
import { environmentCodemodeClient, type CodemodeClientProvider } from "./environment";
|
|
49
53
|
import { CodemodeArtifactCollection } from "./artifacts";
|
|
50
|
-
import {
|
|
54
|
+
import { materializeCodemodeImages, type CodemodeLocalImage } from "./images";
|
|
55
|
+
import { callStructured, codemodeClientProvider, CodemodeToolExecutionError } from "./structured";
|
|
51
56
|
|
|
52
57
|
export { CodemodeToolExecutionError } from "./structured";
|
|
53
58
|
|
|
@@ -56,6 +61,8 @@ const PATH = {
|
|
|
56
61
|
browserOpen: ["interaction", "browser", "open"],
|
|
57
62
|
browserTabs: ["interaction", "browser", "tabs"],
|
|
58
63
|
browserObserve: ["interaction", "browser", "observe"],
|
|
64
|
+
browserRead: ["interaction", "browser", "read"],
|
|
65
|
+
browserScreenshot: ["interaction", "browser", "screenshot"],
|
|
59
66
|
browserAct: ["interaction", "browser", "act"],
|
|
60
67
|
browserClipboard: ["interaction", "browser", "clipboard"],
|
|
61
68
|
browserDebug: ["interaction", "browser", "debug"],
|
|
@@ -108,6 +115,124 @@ export type BrowserActionFences = {
|
|
|
108
115
|
expectedFrameId?: string | null | undefined;
|
|
109
116
|
};
|
|
110
117
|
|
|
118
|
+
export type CodemodeBrowserScreenshotOptions = {
|
|
119
|
+
/** Capture the entire scrollable page when supported by the browser. */
|
|
120
|
+
fullPage?: boolean | undefined;
|
|
121
|
+
/** JPEG quality, 1–100. Reduce it if a large image exceeds the journal bound. */
|
|
122
|
+
quality?: number | undefined;
|
|
123
|
+
/** Save the still at this exact path. Existing files are never overwritten. */
|
|
124
|
+
saveTo?: string | undefined;
|
|
125
|
+
};
|
|
126
|
+
|
|
127
|
+
export type CodemodeBrowserScreenshotMetadata = Readonly<{
|
|
128
|
+
kind: "browser_screenshot";
|
|
129
|
+
browserSessionId: string;
|
|
130
|
+
targetId: string;
|
|
131
|
+
frameId: string;
|
|
132
|
+
targetGeneration: string;
|
|
133
|
+
documentGeneration: string;
|
|
134
|
+
mediaType: "image/png" | "image/jpeg";
|
|
135
|
+
width: number;
|
|
136
|
+
height: number;
|
|
137
|
+
deviceScaleFactor: number;
|
|
138
|
+
scrollX: number;
|
|
139
|
+
scrollY: number;
|
|
140
|
+
capturedAt: string;
|
|
141
|
+
fullPage: boolean;
|
|
142
|
+
}>;
|
|
143
|
+
|
|
144
|
+
export type CodemodeBrowserScreenshot = CodemodeLocalImage &
|
|
145
|
+
Readonly<{
|
|
146
|
+
/** Small structured frame metadata; pixel bytes stay in the local image file. */
|
|
147
|
+
metadata: CodemodeBrowserScreenshotMetadata;
|
|
148
|
+
}>;
|
|
149
|
+
|
|
150
|
+
export type CodemodeBrowserAgentNode = Readonly<{
|
|
151
|
+
ref: string;
|
|
152
|
+
role: string;
|
|
153
|
+
depth: number;
|
|
154
|
+
name?: string;
|
|
155
|
+
description?: string;
|
|
156
|
+
value?: string | { redacted: true; reason: string };
|
|
157
|
+
states: readonly string[];
|
|
158
|
+
actions: readonly string[];
|
|
159
|
+
}>;
|
|
160
|
+
|
|
161
|
+
export type CodemodeBrowserAgentView = Readonly<{
|
|
162
|
+
kind: "compact";
|
|
163
|
+
sourceKind: "snapshot" | "diff" | "none";
|
|
164
|
+
nodes: readonly CodemodeBrowserAgentNode[];
|
|
165
|
+
sourceNodeCount: number;
|
|
166
|
+
omittedNodeCount: number;
|
|
167
|
+
removedRefCount: number;
|
|
168
|
+
clippedFieldCount: number;
|
|
169
|
+
maxNodes: number;
|
|
170
|
+
maxNodeBytes: number;
|
|
171
|
+
}>;
|
|
172
|
+
|
|
173
|
+
export type CodemodeBrowserAgentObservation = BrowserObservation & {
|
|
174
|
+
/** Present on the compact default; absent when the full tree was requested. */
|
|
175
|
+
agentView?: CodemodeBrowserAgentView;
|
|
176
|
+
};
|
|
177
|
+
|
|
178
|
+
export type CodemodeBrowserActionReceipt = Omit<BrowserActionReceipt, "observation"> & {
|
|
179
|
+
observation: CodemodeBrowserAgentObservation | null;
|
|
180
|
+
};
|
|
181
|
+
|
|
182
|
+
export type CodemodeBrowserActionOptions = BrowserActionFences & {
|
|
183
|
+
view?: "compact" | "full" | "none" | undefined;
|
|
184
|
+
};
|
|
185
|
+
|
|
186
|
+
export type CodemodeBrowserAxReadOptions = Readonly<{
|
|
187
|
+
mode?: "matches" | "count" | "subtree";
|
|
188
|
+
ref?: string;
|
|
189
|
+
scopeRef?: string;
|
|
190
|
+
role?: string;
|
|
191
|
+
nameContains?: string;
|
|
192
|
+
textContains?: string;
|
|
193
|
+
state?: string;
|
|
194
|
+
action?: string;
|
|
195
|
+
limit?: number;
|
|
196
|
+
}>;
|
|
197
|
+
|
|
198
|
+
export type CodemodeBrowserDomReadOptions = BrowserActionFences &
|
|
199
|
+
Readonly<{
|
|
200
|
+
mode: "dom";
|
|
201
|
+
dom:
|
|
202
|
+
| {
|
|
203
|
+
kind: "element";
|
|
204
|
+
locator: BrowserDomReadLocator;
|
|
205
|
+
attributes?: readonly BrowserDomSafeAttribute[];
|
|
206
|
+
maxChars?: number;
|
|
207
|
+
}
|
|
208
|
+
| { kind: "count"; selector: string };
|
|
209
|
+
}>;
|
|
210
|
+
|
|
211
|
+
export type CodemodeBrowserReadOptions =
|
|
212
|
+
| CodemodeBrowserAxReadOptions
|
|
213
|
+
| CodemodeBrowserDomReadOptions;
|
|
214
|
+
|
|
215
|
+
export type CodemodeBrowserAxReadResult = Readonly<{
|
|
216
|
+
browserSessionId: string;
|
|
217
|
+
targetId: string;
|
|
218
|
+
observationId: string;
|
|
219
|
+
targetGeneration: string;
|
|
220
|
+
documentGeneration: string | null;
|
|
221
|
+
frameId: string | null;
|
|
222
|
+
source: "accessibility";
|
|
223
|
+
mode: "matches" | "count" | "subtree";
|
|
224
|
+
scopeFound: boolean;
|
|
225
|
+
nodes: readonly CodemodeBrowserAgentNode[];
|
|
226
|
+
totalMatches: number;
|
|
227
|
+
omittedMatches: number;
|
|
228
|
+
clippedFieldCount: number;
|
|
229
|
+
maxNodeBytes: number;
|
|
230
|
+
}>;
|
|
231
|
+
|
|
232
|
+
export type CodemodeBrowserReadResult =
|
|
233
|
+
| CodemodeBrowserAxReadResult
|
|
234
|
+
| (BrowserDomReadResponse & { source: "dom" });
|
|
235
|
+
|
|
111
236
|
export type ComputerActionFences = {
|
|
112
237
|
expectedTargetGeneration?: string | undefined;
|
|
113
238
|
expectedObservationId?: string | null | undefined;
|
|
@@ -273,17 +398,45 @@ export class CodemodeBrowser {
|
|
|
273
398
|
async observe(
|
|
274
399
|
targetId?: string,
|
|
275
400
|
callOptions: CodemodeCallOptions = {},
|
|
276
|
-
): Promise<
|
|
401
|
+
): Promise<CodemodeBrowserAgentObservation> {
|
|
277
402
|
return await this.tabs
|
|
278
403
|
.use(await this.tabs.resolveId(targetId, callOptions))
|
|
279
404
|
.observe(callOptions);
|
|
280
405
|
}
|
|
281
406
|
|
|
407
|
+
async observeFull(
|
|
408
|
+
targetId?: string,
|
|
409
|
+
callOptions: CodemodeCallOptions = {},
|
|
410
|
+
): Promise<BrowserObservation> {
|
|
411
|
+
return await this.tabs
|
|
412
|
+
.use(await this.tabs.resolveId(targetId, callOptions))
|
|
413
|
+
.observeFull(callOptions);
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
async read(
|
|
417
|
+
options: CodemodeBrowserReadOptions & { targetId?: string | undefined } = {},
|
|
418
|
+
callOptions: CodemodeCallOptions = {},
|
|
419
|
+
): Promise<CodemodeBrowserReadResult> {
|
|
420
|
+
const { targetId, ...query } = options;
|
|
421
|
+
return await this.tabs
|
|
422
|
+
.use(await this.tabs.resolveId(targetId, callOptions))
|
|
423
|
+
.read(query, callOptions);
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
async screenshot(
|
|
427
|
+
options: CodemodeBrowserScreenshotOptions & { targetId?: string | undefined } = {},
|
|
428
|
+
callOptions: CodemodeCallOptions = {},
|
|
429
|
+
): Promise<CodemodeBrowserScreenshot> {
|
|
430
|
+
return await this.tabs
|
|
431
|
+
.use(await this.tabs.resolveId(options.targetId, callOptions))
|
|
432
|
+
.screenshot(options, callOptions);
|
|
433
|
+
}
|
|
434
|
+
|
|
282
435
|
async act(
|
|
283
436
|
action: BrowserAction,
|
|
284
|
-
options:
|
|
437
|
+
options: CodemodeBrowserActionOptions & { targetId?: string | undefined } = {},
|
|
285
438
|
callOptions: CodemodeCallOptions = {},
|
|
286
|
-
): Promise<
|
|
439
|
+
): Promise<CodemodeBrowserActionReceipt> {
|
|
287
440
|
const targetId = await this.tabs.resolveId(options.targetId, callOptions);
|
|
288
441
|
return await this.tabs.use(targetId).act(action, options, callOptions);
|
|
289
442
|
}
|
|
@@ -541,7 +694,7 @@ export class CodemodeBrowserTab {
|
|
|
541
694
|
readonly id: string,
|
|
542
695
|
) {}
|
|
543
696
|
|
|
544
|
-
async observe(callOptions: CodemodeCallOptions = {}): Promise<
|
|
697
|
+
async observe(callOptions: CodemodeCallOptions = {}): Promise<CodemodeBrowserAgentObservation> {
|
|
545
698
|
return await callStructured(
|
|
546
699
|
this.client,
|
|
547
700
|
PATH.browserObserve,
|
|
@@ -550,11 +703,51 @@ export class CodemodeBrowserTab {
|
|
|
550
703
|
);
|
|
551
704
|
}
|
|
552
705
|
|
|
706
|
+
async observeFull(callOptions: CodemodeCallOptions = {}): Promise<BrowserObservation> {
|
|
707
|
+
return await callStructured(
|
|
708
|
+
this.client,
|
|
709
|
+
PATH.browserObserve,
|
|
710
|
+
{ browserSessionId: this.browserSessionId, targetId: this.id, view: "full" },
|
|
711
|
+
callOptions,
|
|
712
|
+
);
|
|
713
|
+
}
|
|
714
|
+
|
|
715
|
+
async read(
|
|
716
|
+
options: CodemodeBrowserReadOptions = {},
|
|
717
|
+
callOptions: CodemodeCallOptions = {},
|
|
718
|
+
): Promise<CodemodeBrowserReadResult> {
|
|
719
|
+
return await callStructured(
|
|
720
|
+
this.client,
|
|
721
|
+
PATH.browserRead,
|
|
722
|
+
{ browserSessionId: this.browserSessionId, targetId: this.id, ...options },
|
|
723
|
+
callOptions,
|
|
724
|
+
);
|
|
725
|
+
}
|
|
726
|
+
|
|
727
|
+
async screenshot(
|
|
728
|
+
options: CodemodeBrowserScreenshotOptions = {},
|
|
729
|
+
callOptions: CodemodeCallOptions = {},
|
|
730
|
+
): Promise<CodemodeBrowserScreenshot> {
|
|
731
|
+
const result = await (
|
|
732
|
+
await this.client()
|
|
733
|
+
).callPath(
|
|
734
|
+
PATH.browserScreenshot,
|
|
735
|
+
{
|
|
736
|
+
browserSessionId: this.browserSessionId,
|
|
737
|
+
targetId: this.id,
|
|
738
|
+
...(options.fullPage === undefined ? {} : { fullPage: options.fullPage }),
|
|
739
|
+
...(options.quality === undefined ? {} : { quality: options.quality }),
|
|
740
|
+
},
|
|
741
|
+
callOptions,
|
|
742
|
+
);
|
|
743
|
+
return await materializeBrowserScreenshot(result, options.saveTo);
|
|
744
|
+
}
|
|
745
|
+
|
|
553
746
|
async act(
|
|
554
747
|
action: BrowserAction,
|
|
555
|
-
fences:
|
|
748
|
+
fences: CodemodeBrowserActionOptions = {},
|
|
556
749
|
callOptions: CodemodeCallOptions = {},
|
|
557
|
-
): Promise<
|
|
750
|
+
): Promise<CodemodeBrowserActionReceipt> {
|
|
558
751
|
return await callStructured(
|
|
559
752
|
this.client,
|
|
560
753
|
PATH.browserAct,
|
|
@@ -1207,6 +1400,33 @@ export class CodemodeComputerLocator {
|
|
|
1207
1400
|
}
|
|
1208
1401
|
}
|
|
1209
1402
|
|
|
1403
|
+
/** Keep screenshot pixels out of Code Mode text output; `view_image` reads the local file. */
|
|
1404
|
+
async function materializeBrowserScreenshot(
|
|
1405
|
+
result: AttemptToolResult,
|
|
1406
|
+
saveTo: string | undefined,
|
|
1407
|
+
): Promise<CodemodeBrowserScreenshot> {
|
|
1408
|
+
if (result.isError) throw new CodemodeToolExecutionError(result);
|
|
1409
|
+
if (!result.structuredContent) {
|
|
1410
|
+
throw new Error("Codemode tool interaction.browser.screenshot returned no structured content");
|
|
1411
|
+
}
|
|
1412
|
+
const image = result.content.find((content) => content.type === "image");
|
|
1413
|
+
if (
|
|
1414
|
+
typeof result.structuredContent.mediaType === "string" &&
|
|
1415
|
+
image &&
|
|
1416
|
+
result.structuredContent.mediaType !== image.mimeType
|
|
1417
|
+
) {
|
|
1418
|
+
throw new Error("Codemode browser screenshot image type differs from frame metadata");
|
|
1419
|
+
}
|
|
1420
|
+
const images = await materializeCodemodeImages(result, {
|
|
1421
|
+
...(saveTo === undefined ? {} : { saveTo }),
|
|
1422
|
+
requireOne: true,
|
|
1423
|
+
});
|
|
1424
|
+
return {
|
|
1425
|
+
...images[0]!,
|
|
1426
|
+
metadata: result.structuredContent as CodemodeBrowserScreenshotMetadata,
|
|
1427
|
+
};
|
|
1428
|
+
}
|
|
1429
|
+
|
|
1210
1430
|
export function createOpenGeniCodemode(
|
|
1211
1431
|
client: CodemodeClient | CodemodeClientProvider = () => environmentCodemodeClient(),
|
|
1212
1432
|
) {
|