@opengeni/codemode 0.6.1 → 0.6.2-canary.36199476632001
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -0
- package/dist/images.d.ts +22 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +169 -5
- package/dist/index.js.map +1 -1
- package/dist/interaction.d.ts +137 -831
- package/package.json +4 -4
- package/src/declarations.ts +1 -0
- package/src/images.ts +145 -0
- package/src/index.ts +12 -2
- package/src/interaction.ts +227 -7
package/README.md
CHANGED
|
@@ -64,13 +64,27 @@ import { openGeni } from "@opengeni/codemode";
|
|
|
64
64
|
|
|
65
65
|
const browser = await openGeni.browsers.open({ initialUrl: "http://127.0.0.1:3000" });
|
|
66
66
|
const tab = await browser.tabs.selected();
|
|
67
|
+
const page = await tab.observe(); // Compact refs and explicit omission counts.
|
|
68
|
+
const buttons = await tab.read({ role: "button", nameContains: "Save", limit: 5 });
|
|
69
|
+
const email = await tab.read({ mode: "dom", dom: { kind: "element", locator: { kind: "css", selector: "#email" }, attributes: ["placeholder"] } });
|
|
67
70
|
await tab.getByRole("button", { name: "Save" }).click();
|
|
71
|
+
const still = await tab.screenshot({ quality: 40 });
|
|
72
|
+
console.log(still.path); // Open this local PNG/JPEG/WebP with view_image.
|
|
68
73
|
|
|
69
74
|
const computer = await openGeni.computers.open();
|
|
70
75
|
const app = await computer.apps.focused();
|
|
71
76
|
await app.getByRole("button", { name: "1" }).invoke();
|
|
72
77
|
```
|
|
73
78
|
|
|
79
|
+
`tab.observeFull()` retrieves the complete accessibility snapshot when compact
|
|
80
|
+
refs or a focused `tab.read()` query are insufficient. The default focused read
|
|
81
|
+
searches accessibility; `mode: "dom"` returns bounded text, editable values,
|
|
82
|
+
safe attributes, or counts. Sensitive fields are redacted. DOM CSS reads accept
|
|
83
|
+
simple tag, class, id, descendant, and child selectors; attribute and pseudo
|
|
84
|
+
selectors are rejected to prevent secret-value probing. Browser image blocks
|
|
85
|
+
are bounded below the Code Mode journal limit; for an oversized screenshot,
|
|
86
|
+
capture the viewport or lower JPEG quality.
|
|
87
|
+
|
|
74
88
|
Both surfaces return the same durable tool receipts. Human approval, catalog
|
|
75
89
|
generation, operation idempotency, and outcome-unknown behavior remain enforced
|
|
76
90
|
by the shared attempt executor. Catalog, approval, authorization, input
|
|
@@ -84,6 +98,24 @@ directly cannot bypass approval. The dispatcher keeps its claim alive during
|
|
|
84
98
|
gateway preparation and reuses a deterministic durable tool-created event if a
|
|
85
99
|
pre-execution claim is reclaimed.
|
|
86
100
|
|
|
101
|
+
`tab.screenshot({ fullPage?: boolean, saveTo?: string })` calls the same atomic
|
|
102
|
+
`interaction.browser.screenshot` tool and writes its image block to a private
|
|
103
|
+
local file. It returns the absolute path, MIME type, byte count, and small frame
|
|
104
|
+
metadata, without printing base64 pixels. Use the agent's `view_image` tool on
|
|
105
|
+
that path to inspect the still. `saveTo` writes to an exact caller-chosen path
|
|
106
|
+
and refuses to overwrite an existing file. `CodemodeClient.callPath` remains
|
|
107
|
+
available when code needs the complete MCP result, including image blocks.
|
|
108
|
+
Default image paths are private OS scratch files; later image calls remove this
|
|
109
|
+
client's scratch directories older than 24 hours. Use `saveTo` when a file must
|
|
110
|
+
remain available longer; that path is caller-owned and never pruned by Code Mode.
|
|
111
|
+
The generated `tools.*` namespace normally returns structured output directly.
|
|
112
|
+
When a typed tool returns image blocks, it instead returns
|
|
113
|
+
`{ structuredContent, images, otherContent }`: `images` contains local paths,
|
|
114
|
+
and `otherContent` preserves text and other non-image blocks. Generated
|
|
115
|
+
declarations express this union. Open `images[n].path` with `view_image`.
|
|
116
|
+
The facade and typed wrapper keep base64 out of program output; raw `callPath`
|
|
117
|
+
retains it and should not be printed wholesale.
|
|
118
|
+
|
|
87
119
|
`CodemodeCallOptions.signal` cancels only the caller's HTTP/polling observation.
|
|
88
120
|
It does not request server cancellation and cannot prove that an operation
|
|
89
121
|
stopped. The attempt/turn lifecycle remains the only cancellation authority; a
|
package/dist/images.d.ts
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { AttemptToolContent, AttemptToolResult } from "@opengeni/contracts";
|
|
2
|
+
type ImageMimeType = "image/png" | "image/jpeg" | "image/webp";
|
|
3
|
+
export type CodemodeLocalImage = Readonly<{
|
|
4
|
+
/** Local path readable with the agent's `view_image` tool. */
|
|
5
|
+
path: string;
|
|
6
|
+
mimeType: ImageMimeType;
|
|
7
|
+
sizeBytes: number;
|
|
8
|
+
}>;
|
|
9
|
+
/** Typed Code Mode calls with image blocks retain every structured and non-image block. */
|
|
10
|
+
export type CodemodeMultimodalValue<T> = Readonly<{
|
|
11
|
+
structuredContent: T;
|
|
12
|
+
images: readonly CodemodeLocalImage[];
|
|
13
|
+
otherContent: readonly Exclude<AttemptToolContent, {
|
|
14
|
+
type: "image";
|
|
15
|
+
}>[];
|
|
16
|
+
}>;
|
|
17
|
+
/** Preserve image blocks as private local files instead of base64 in program output. */
|
|
18
|
+
export declare function materializeCodemodeImages(result: AttemptToolResult, options?: {
|
|
19
|
+
saveTo?: string;
|
|
20
|
+
requireOne?: boolean;
|
|
21
|
+
}): Promise<CodemodeLocalImage[]>;
|
|
22
|
+
export {};
|
package/dist/index.d.ts
CHANGED
|
@@ -115,7 +115,7 @@ export declare class CodemodeClient {
|
|
|
115
115
|
private callResolved;
|
|
116
116
|
/** Resolve and call one exact generated namespace path without parsing a wire name. */
|
|
117
117
|
callPath(path: readonly string[], argumentsValue?: Record<string, unknown>, options?: CodemodeCallOptions): Promise<AttemptToolResultValue>;
|
|
118
|
-
/** Return structured content
|
|
118
|
+
/** Return structured content, or a local-image wrapper for typed multimodal results. */
|
|
119
119
|
callPathValue(path: readonly string[], argumentsValue?: Record<string, unknown>, options?: CodemodeCallOptions): Promise<unknown>;
|
|
120
120
|
private submit;
|
|
121
121
|
private read;
|
|
@@ -151,6 +151,7 @@ export declare function decodeCodemodeDispatchAck(input: Uint8Array): CodemodeDi
|
|
|
151
151
|
export declare function parseVerifiedAttemptToolCatalog(input: unknown): AttemptToolCatalogValue;
|
|
152
152
|
export * from "./environment.js";
|
|
153
153
|
export * from "./interaction.js";
|
|
154
|
+
export * from "./images.js";
|
|
154
155
|
export * from "./artifacts.js";
|
|
155
156
|
export * from "./structured.js";
|
|
156
157
|
export * from "./declarations.js";
|
package/dist/index.js
CHANGED
|
@@ -26,6 +26,101 @@ import {
|
|
|
26
26
|
} from "@opengeni/tool-gateway";
|
|
27
27
|
import { siteSessionPath } from "@opengeni/contracts/site-session-http";
|
|
28
28
|
|
|
29
|
+
// src/images.ts
|
|
30
|
+
import { lstat, mkdir, mkdtemp, readdir, rm, writeFile } from "fs/promises";
|
|
31
|
+
import { tmpdir } from "os";
|
|
32
|
+
import { dirname, join, resolve } from "path";
|
|
33
|
+
var MAX_IMAGE_BASE64_CHARS = 16 * 1024 * 1024;
|
|
34
|
+
var TEMP_IMAGE_PREFIX = "opengeni-codemode-image-";
|
|
35
|
+
var TEMP_IMAGE_MAX_AGE_MS = 24 * 60 * 60 * 1e3;
|
|
36
|
+
var TEMP_IMAGE_PRUNE_INTERVAL_MS = 60 * 60 * 1e3;
|
|
37
|
+
var lastPruneAt = 0;
|
|
38
|
+
async function materializeCodemodeImages(result, options = {}) {
|
|
39
|
+
const blocks = result.content.filter((content) => content.type === "image");
|
|
40
|
+
if (options.requireOne && blocks.length !== 1) {
|
|
41
|
+
throw new Error("Codemode browser screenshot must return exactly one image block");
|
|
42
|
+
}
|
|
43
|
+
if (blocks.length === 0) return [];
|
|
44
|
+
if (options.saveTo !== void 0 && blocks.length !== 1) {
|
|
45
|
+
throw new Error("An explicit image path requires exactly one image block");
|
|
46
|
+
}
|
|
47
|
+
let totalBase64Chars = 0;
|
|
48
|
+
const decoded = blocks.map((block) => {
|
|
49
|
+
const mimeType = supportedMimeType(block.mimeType);
|
|
50
|
+
totalBase64Chars += block.data.length;
|
|
51
|
+
if (block.data.length === 0 || block.data.length > MAX_IMAGE_BASE64_CHARS || totalBase64Chars > MAX_IMAGE_BASE64_CHARS) {
|
|
52
|
+
throw new Error("Codemode image exceeds the Code Mode result limit");
|
|
53
|
+
}
|
|
54
|
+
const bytes = Buffer.from(block.data, "base64");
|
|
55
|
+
if (bytes.length === 0 || bytes.toString("base64") !== block.data || !matchesImageType(bytes, mimeType)) {
|
|
56
|
+
throw new Error("Codemode image bytes are invalid");
|
|
57
|
+
}
|
|
58
|
+
return { bytes, mimeType };
|
|
59
|
+
});
|
|
60
|
+
let directory;
|
|
61
|
+
if (options.saveTo === void 0) {
|
|
62
|
+
await pruneExpiredPrivateImages();
|
|
63
|
+
directory = await mkdtemp(join(tmpdir(), TEMP_IMAGE_PREFIX));
|
|
64
|
+
} else {
|
|
65
|
+
if (options.saveTo.length === 0) throw new TypeError("Image saveTo must be a nonempty path");
|
|
66
|
+
await mkdir(dirname(resolve(options.saveTo)), { recursive: true });
|
|
67
|
+
}
|
|
68
|
+
const images = [];
|
|
69
|
+
try {
|
|
70
|
+
for (const [index, image] of decoded.entries()) {
|
|
71
|
+
const extension = image.mimeType === "image/jpeg" ? "jpg" : image.mimeType.split("/")[1];
|
|
72
|
+
const path = options.saveTo === void 0 ? join(
|
|
73
|
+
directory,
|
|
74
|
+
decoded.length === 1 ? `frame.${extension}` : `image-${index + 1}.${extension}`
|
|
75
|
+
) : resolve(options.saveTo);
|
|
76
|
+
await writeFile(path, image.bytes, { flag: "wx", mode: 384 });
|
|
77
|
+
images.push({ path, mimeType: image.mimeType, sizeBytes: image.bytes.length });
|
|
78
|
+
}
|
|
79
|
+
} catch (error) {
|
|
80
|
+
if (directory) await rm(directory, { recursive: true, force: true });
|
|
81
|
+
throw error;
|
|
82
|
+
}
|
|
83
|
+
return images;
|
|
84
|
+
}
|
|
85
|
+
async function pruneExpiredPrivateImages() {
|
|
86
|
+
const now = Date.now();
|
|
87
|
+
if (now - lastPruneAt < TEMP_IMAGE_PRUNE_INTERVAL_MS) return;
|
|
88
|
+
lastPruneAt = now;
|
|
89
|
+
if (typeof process.getuid !== "function") return;
|
|
90
|
+
try {
|
|
91
|
+
let removed = 0;
|
|
92
|
+
for (const entry of await readdir(tmpdir(), { withFileTypes: true })) {
|
|
93
|
+
if (!entry.isDirectory() || !/^opengeni-codemode-image-[A-Za-z0-9]{6}$/u.test(entry.name)) {
|
|
94
|
+
continue;
|
|
95
|
+
}
|
|
96
|
+
const path = join(tmpdir(), entry.name);
|
|
97
|
+
const info = await lstat(path);
|
|
98
|
+
if (!info.isDirectory() || info.uid !== process.getuid() || (info.mode & 63) !== 0 || now - info.mtimeMs < TEMP_IMAGE_MAX_AGE_MS) {
|
|
99
|
+
continue;
|
|
100
|
+
}
|
|
101
|
+
await rm(path, { recursive: true, force: true });
|
|
102
|
+
if (++removed >= 64) break;
|
|
103
|
+
}
|
|
104
|
+
} catch {
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
function supportedMimeType(mimeType) {
|
|
108
|
+
if (mimeType === "image/png" || mimeType === "image/jpeg" || mimeType === "image/webp") {
|
|
109
|
+
return mimeType;
|
|
110
|
+
}
|
|
111
|
+
throw new Error(`Codemode image has unsupported type: ${mimeType}`);
|
|
112
|
+
}
|
|
113
|
+
function matchesImageType(bytes, mimeType) {
|
|
114
|
+
switch (mimeType) {
|
|
115
|
+
case "image/png":
|
|
116
|
+
return bytes.length >= 8 && bytes.subarray(0, 8).equals(Buffer.from("89504e470d0a1a0a", "hex"));
|
|
117
|
+
case "image/jpeg":
|
|
118
|
+
return bytes.length >= 3 && bytes[0] === 255 && bytes[1] === 216 && bytes[2] === 255;
|
|
119
|
+
case "image/webp":
|
|
120
|
+
return bytes.length >= 12 && bytes.toString("ascii", 0, 4) === "RIFF" && bytes.toString("ascii", 8, 12) === "WEBP";
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
29
124
|
// src/environment.ts
|
|
30
125
|
import { readFile } from "fs/promises";
|
|
31
126
|
var CODEMODE_ENVIRONMENT = {
|
|
@@ -305,6 +400,8 @@ var PATH2 = {
|
|
|
305
400
|
browserOpen: ["interaction", "browser", "open"],
|
|
306
401
|
browserTabs: ["interaction", "browser", "tabs"],
|
|
307
402
|
browserObserve: ["interaction", "browser", "observe"],
|
|
403
|
+
browserRead: ["interaction", "browser", "read"],
|
|
404
|
+
browserScreenshot: ["interaction", "browser", "screenshot"],
|
|
308
405
|
browserAct: ["interaction", "browser", "act"],
|
|
309
406
|
browserClipboard: ["interaction", "browser", "clipboard"],
|
|
310
407
|
browserDebug: ["interaction", "browser", "debug"],
|
|
@@ -419,6 +516,16 @@ var CodemodeBrowser = class {
|
|
|
419
516
|
async observe(targetId, callOptions = {}) {
|
|
420
517
|
return await this.tabs.use(await this.tabs.resolveId(targetId, callOptions)).observe(callOptions);
|
|
421
518
|
}
|
|
519
|
+
async observeFull(targetId, callOptions = {}) {
|
|
520
|
+
return await this.tabs.use(await this.tabs.resolveId(targetId, callOptions)).observeFull(callOptions);
|
|
521
|
+
}
|
|
522
|
+
async read(options = {}, callOptions = {}) {
|
|
523
|
+
const { targetId, ...query } = options;
|
|
524
|
+
return await this.tabs.use(await this.tabs.resolveId(targetId, callOptions)).read(query, callOptions);
|
|
525
|
+
}
|
|
526
|
+
async screenshot(options = {}, callOptions = {}) {
|
|
527
|
+
return await this.tabs.use(await this.tabs.resolveId(options.targetId, callOptions)).screenshot(options, callOptions);
|
|
528
|
+
}
|
|
422
529
|
async act(action, options = {}, callOptions = {}) {
|
|
423
530
|
const targetId = await this.tabs.resolveId(options.targetId, callOptions);
|
|
424
531
|
return await this.tabs.use(targetId).act(action, options, callOptions);
|
|
@@ -609,6 +716,35 @@ var CodemodeBrowserTab = class {
|
|
|
609
716
|
callOptions
|
|
610
717
|
);
|
|
611
718
|
}
|
|
719
|
+
async observeFull(callOptions = {}) {
|
|
720
|
+
return await callStructured(
|
|
721
|
+
this.client,
|
|
722
|
+
PATH2.browserObserve,
|
|
723
|
+
{ browserSessionId: this.browserSessionId, targetId: this.id, view: "full" },
|
|
724
|
+
callOptions
|
|
725
|
+
);
|
|
726
|
+
}
|
|
727
|
+
async read(options = {}, callOptions = {}) {
|
|
728
|
+
return await callStructured(
|
|
729
|
+
this.client,
|
|
730
|
+
PATH2.browserRead,
|
|
731
|
+
{ browserSessionId: this.browserSessionId, targetId: this.id, ...options },
|
|
732
|
+
callOptions
|
|
733
|
+
);
|
|
734
|
+
}
|
|
735
|
+
async screenshot(options = {}, callOptions = {}) {
|
|
736
|
+
const result = await (await this.client()).callPath(
|
|
737
|
+
PATH2.browserScreenshot,
|
|
738
|
+
{
|
|
739
|
+
browserSessionId: this.browserSessionId,
|
|
740
|
+
targetId: this.id,
|
|
741
|
+
...options.fullPage === void 0 ? {} : { fullPage: options.fullPage },
|
|
742
|
+
...options.quality === void 0 ? {} : { quality: options.quality }
|
|
743
|
+
},
|
|
744
|
+
callOptions
|
|
745
|
+
);
|
|
746
|
+
return await materializeBrowserScreenshot(result, options.saveTo);
|
|
747
|
+
}
|
|
612
748
|
async act(action, fences = {}, callOptions = {}) {
|
|
613
749
|
return await callStructured(
|
|
614
750
|
this.client,
|
|
@@ -1068,6 +1204,24 @@ var CodemodeComputerLocator = class {
|
|
|
1068
1204
|
);
|
|
1069
1205
|
}
|
|
1070
1206
|
};
|
|
1207
|
+
async function materializeBrowserScreenshot(result, saveTo) {
|
|
1208
|
+
if (result.isError) throw new CodemodeToolExecutionError(result);
|
|
1209
|
+
if (!result.structuredContent) {
|
|
1210
|
+
throw new Error("Codemode tool interaction.browser.screenshot returned no structured content");
|
|
1211
|
+
}
|
|
1212
|
+
const image = result.content.find((content) => content.type === "image");
|
|
1213
|
+
if (typeof result.structuredContent.mediaType === "string" && image && result.structuredContent.mediaType !== image.mimeType) {
|
|
1214
|
+
throw new Error("Codemode browser screenshot image type differs from frame metadata");
|
|
1215
|
+
}
|
|
1216
|
+
const images = await materializeCodemodeImages(result, {
|
|
1217
|
+
...saveTo === void 0 ? {} : { saveTo },
|
|
1218
|
+
requireOne: true
|
|
1219
|
+
});
|
|
1220
|
+
return {
|
|
1221
|
+
...images[0],
|
|
1222
|
+
metadata: result.structuredContent
|
|
1223
|
+
};
|
|
1224
|
+
}
|
|
1071
1225
|
function createOpenGeniCodemode(client = () => environmentCodemodeClient()) {
|
|
1072
1226
|
return new OpenGeniCodemode(client);
|
|
1073
1227
|
}
|
|
@@ -1084,6 +1238,7 @@ function generateCodemodeDeclarations(catalog, options = {}) {
|
|
|
1084
1238
|
interfaceName: "CodemodeGeneratedTools",
|
|
1085
1239
|
callOptionsType: "CodemodeCallOptions",
|
|
1086
1240
|
fallbackResultType: "CodemodeToolResult",
|
|
1241
|
+
typedResultWrapperType: "CodemodeMultimodalValue",
|
|
1087
1242
|
generatedBy: "@opengeni/codemode via @opengeni/tool-gateway",
|
|
1088
1243
|
catalogDigestLabel: "Attempt catalog digest"
|
|
1089
1244
|
}
|
|
@@ -1399,7 +1554,7 @@ var CodemodeClient = class {
|
|
|
1399
1554
|
options
|
|
1400
1555
|
)).result;
|
|
1401
1556
|
}
|
|
1402
|
-
/** Return structured content
|
|
1557
|
+
/** Return structured content, or a local-image wrapper for typed multimodal results. */
|
|
1403
1558
|
async callPathValue(path, argumentsValue = {}, options = {}) {
|
|
1404
1559
|
if (path.length < 2 || path.some((segment) => segment.length === 0)) {
|
|
1405
1560
|
throw new AttemptToolNotFoundError();
|
|
@@ -1416,7 +1571,15 @@ var CodemodeClient = class {
|
|
|
1416
1571
|
`Codemode tool ${path.join(".")} declared outputSchema but returned no structured content`
|
|
1417
1572
|
);
|
|
1418
1573
|
}
|
|
1419
|
-
|
|
1574
|
+
if (!result.content.some((content) => content.type === "image")) {
|
|
1575
|
+
return result.structuredContent;
|
|
1576
|
+
}
|
|
1577
|
+
const images = await materializeCodemodeImages(result);
|
|
1578
|
+
return {
|
|
1579
|
+
structuredContent: result.structuredContent,
|
|
1580
|
+
images,
|
|
1581
|
+
otherContent: result.content.filter((content) => content.type !== "image")
|
|
1582
|
+
};
|
|
1420
1583
|
}
|
|
1421
1584
|
async submit(operationId, catalogDigest, identity, argumentsValue, signal) {
|
|
1422
1585
|
const response = await this.request("/calls", {
|
|
@@ -1656,14 +1819,14 @@ function parseCodemodeApiError(input) {
|
|
|
1656
1819
|
}
|
|
1657
1820
|
async function abortableDelay(delayMs, signal) {
|
|
1658
1821
|
if (!signal) {
|
|
1659
|
-
await new Promise((
|
|
1822
|
+
await new Promise((resolve2) => setTimeout(resolve2, delayMs));
|
|
1660
1823
|
return;
|
|
1661
1824
|
}
|
|
1662
1825
|
throwIfAborted(signal);
|
|
1663
|
-
await new Promise((
|
|
1826
|
+
await new Promise((resolve2, reject) => {
|
|
1664
1827
|
const timer = setTimeout(() => {
|
|
1665
1828
|
signal.removeEventListener("abort", onAbort);
|
|
1666
|
-
|
|
1829
|
+
resolve2();
|
|
1667
1830
|
}, delayMs);
|
|
1668
1831
|
const onAbort = () => {
|
|
1669
1832
|
clearTimeout(timer);
|
|
@@ -1728,6 +1891,7 @@ export {
|
|
|
1728
1891
|
environmentCodemodeClient,
|
|
1729
1892
|
generateCodemodeDeclarations,
|
|
1730
1893
|
jsonSchemaToTypeScript,
|
|
1894
|
+
materializeCodemodeImages,
|
|
1731
1895
|
openGeni,
|
|
1732
1896
|
parseVerifiedAttemptToolCatalog,
|
|
1733
1897
|
tools
|