@opengeni/runtime 3.3.0 → 3.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-LJT5NPWT.js +9 -0
- package/dist/assets/codemode-client.json +1 -1
- package/dist/browser-current-tab.d.ts +46 -0
- package/dist/browser-model-text.d.ts +126 -0
- package/dist/{chunk-XRX57ZQO.js → chunk-3DFH3ZHU.js} +784 -121
- package/dist/chunk-3DFH3ZHU.js.map +1 -0
- package/dist/{chunk-HJSIPTFA.js → chunk-4XDJ4TJA.js} +2 -2
- package/dist/{chunk-XQ23VCZ4.js → chunk-6FQ6KEII.js} +3 -3
- package/dist/{chunk-7LJ7KDPI.js → chunk-ERD4ICVB.js} +4 -3
- package/dist/{chunk-EK7IW2FR.js → chunk-ETTCGPBT.js} +224 -6
- package/dist/chunk-ETTCGPBT.js.map +1 -0
- package/dist/{chunk-7OUG7WBV.js → chunk-GWRBALGQ.js} +181 -13
- package/dist/chunk-GWRBALGQ.js.map +1 -0
- package/dist/{chunk-YG26XE7B.js → chunk-JYDG2A7H.js} +2 -2
- package/dist/{chunk-RSHQSLEJ.js → chunk-KIL4FGX5.js} +3 -3
- package/dist/{chunk-NWS4HPB7.js → chunk-MIAL5I4O.js} +2 -2
- package/dist/{chunk-Z5U6WE22.js → chunk-PRZPGRIK.js} +2 -2
- package/dist/{chunk-KCPW4K7L.js → chunk-PZQUUBEB.js} +2 -2
- package/dist/{chunk-5WJOPANS.js → chunk-Y5THZH5N.js} +7 -7
- package/dist/codex-tool-search.d.ts +9 -0
- package/dist/gmail-rest-mcp.js +8 -8
- package/dist/index.d.ts +3 -0
- package/dist/index.js +14 -12
- package/dist/interaction-tool-json-schema.d.ts +27 -0
- package/dist/{lazy-manifest-AJ6XPJ4F.js → lazy-manifest-BLZ3UZLX.js} +4 -4
- package/dist/lazy-tool-transport.d.ts +10 -0
- package/dist/{localSources-SSIEH5F5.js → localSources-I6SIQ6CX.js} +6 -6
- package/dist/model-call.js +3 -3
- package/dist/sandbox/browser-control-client.d.ts +77 -0
- package/dist/sandbox/index.d.ts +1 -1
- package/dist/sandbox/index.js +7 -7
- package/dist/sandbox/selfhosted/session.d.ts +3 -0
- package/dist/slack-rest-mcp.js +8 -8
- package/dist/workspace-tool-gateway.js +12 -12
- package/package.json +14 -14
- package/src/browser-current-tab.ts +139 -0
- package/src/browser-model-text.ts +378 -0
- package/src/codex-tool-search.ts +53 -0
- package/src/index.ts +39 -1
- package/src/interaction-tool-json-schema.ts +281 -0
- package/src/interaction-tools.ts +425 -235
- package/src/lazy-tool-transport.ts +229 -10
- package/src/model-input.ts +12 -1
- package/src/sandbox/browser-control-client.ts +336 -0
- package/src/sandbox/index.ts +4 -0
- package/src/sandbox/selfhosted/session.ts +38 -0
- package/dist/agent-CTPLA3HP.js +0 -9
- package/dist/browser-action-json-schema.d.ts +0 -4
- package/dist/chunk-7OUG7WBV.js.map +0 -1
- package/dist/chunk-EK7IW2FR.js.map +0 -1
- package/dist/chunk-XRX57ZQO.js.map +0 -1
- package/src/browser-action-json-schema.ts +0 -10
- /package/dist/{agent-CTPLA3HP.js.map → agent-LJT5NPWT.js.map} +0 -0
- /package/dist/{chunk-HJSIPTFA.js.map → chunk-4XDJ4TJA.js.map} +0 -0
- /package/dist/{chunk-XQ23VCZ4.js.map → chunk-6FQ6KEII.js.map} +0 -0
- /package/dist/{chunk-7LJ7KDPI.js.map → chunk-ERD4ICVB.js.map} +0 -0
- /package/dist/{chunk-YG26XE7B.js.map → chunk-JYDG2A7H.js.map} +0 -0
- /package/dist/{chunk-RSHQSLEJ.js.map → chunk-KIL4FGX5.js.map} +0 -0
- /package/dist/{chunk-NWS4HPB7.js.map → chunk-MIAL5I4O.js.map} +0 -0
- /package/dist/{chunk-Z5U6WE22.js.map → chunk-PRZPGRIK.js.map} +0 -0
- /package/dist/{chunk-KCPW4K7L.js.map → chunk-PZQUUBEB.js.map} +0 -0
- /package/dist/{chunk-5WJOPANS.js.map → chunk-Y5THZH5N.js.map} +0 -0
- /package/dist/{lazy-manifest-AJ6XPJ4F.js.map → lazy-manifest-BLZ3UZLX.js.map} +0 -0
- /package/dist/{localSources-SSIEH5F5.js.map → localSources-I6SIQ6CX.js.map} +0 -0
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
// The browser tab an agent is working in.
|
|
2
|
+
//
|
|
3
|
+
// Browser tools may omit browserSessionId and targetId to keep working in the
|
|
4
|
+
// tab the agent used last, the way a person keeps using the tab in front of
|
|
5
|
+
// them. A step then costs a few output tokens instead of two identifiers, and
|
|
6
|
+
// output tokens dominate a browser step's model latency. The remembered tab
|
|
7
|
+
// lives for one turn attempt; a call that omits identifiers before any browser
|
|
8
|
+
// call in that attempt resolves this session's most recently used live
|
|
9
|
+
// browser and its selected tab.
|
|
10
|
+
|
|
11
|
+
export class BrowserTargetResolutionError extends Error {
|
|
12
|
+
constructor(
|
|
13
|
+
readonly code: "browser_session_required" | "browser_target_required",
|
|
14
|
+
message: string,
|
|
15
|
+
) {
|
|
16
|
+
super(message);
|
|
17
|
+
this.name = "BrowserTargetResolutionError";
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export type CurrentBrowserTabDeps = {
|
|
22
|
+
/** This agent session's live BrowserSessions. */
|
|
23
|
+
listLiveSessions: () => Promise<ReadonlyArray<{ id: string; lastUsedAt: string }>>;
|
|
24
|
+
/** The tabs of one BrowserSession. */
|
|
25
|
+
listTargets: (
|
|
26
|
+
browserSessionId: string,
|
|
27
|
+
) => Promise<ReadonlyArray<{ id: string; kind: string; selected: boolean }>>;
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
export type ResolvedBrowserSession = {
|
|
31
|
+
browserSessionId: string;
|
|
32
|
+
/** True when the identifier came from memory or lookup, not the caller. */
|
|
33
|
+
implicit: boolean;
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
export type ResolvedBrowserTarget = ResolvedBrowserSession & {
|
|
37
|
+
targetId: string;
|
|
38
|
+
implicitTarget: boolean;
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
export class CurrentBrowserTab {
|
|
42
|
+
private current: { browserSessionId: string; targetId: string | null } | null = null;
|
|
43
|
+
|
|
44
|
+
constructor(private readonly deps: CurrentBrowserTabDeps) {}
|
|
45
|
+
|
|
46
|
+
/** Record the session and, when known, the tab a successful call used. */
|
|
47
|
+
use(browserSessionId: string, targetId?: string | null): void {
|
|
48
|
+
if (targetId === undefined) {
|
|
49
|
+
if (this.current?.browserSessionId !== browserSessionId) {
|
|
50
|
+
this.current = { browserSessionId, targetId: null };
|
|
51
|
+
}
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
54
|
+
this.current = { browserSessionId, targetId };
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
forgetTarget(browserSessionId: string, targetId: string): void {
|
|
58
|
+
if (this.current?.browserSessionId === browserSessionId && this.current.targetId === targetId) {
|
|
59
|
+
this.current = { browserSessionId, targetId: null };
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
forgetSession(browserSessionId: string): void {
|
|
64
|
+
if (this.current?.browserSessionId === browserSessionId) this.current = null;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** A call that relied on remembered identifiers failed: resolve afresh next time. */
|
|
68
|
+
failed(resolved: ResolvedBrowserSession & { implicitTarget?: boolean }): void {
|
|
69
|
+
if (resolved.implicit) this.forgetSession(resolved.browserSessionId);
|
|
70
|
+
else if (
|
|
71
|
+
resolved.implicitTarget &&
|
|
72
|
+
this.current?.browserSessionId === resolved.browserSessionId
|
|
73
|
+
) {
|
|
74
|
+
this.current = { browserSessionId: resolved.browserSessionId, targetId: null };
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
async session(browserSessionId: string | undefined): Promise<ResolvedBrowserSession> {
|
|
79
|
+
if (browserSessionId !== undefined) return { browserSessionId, implicit: false };
|
|
80
|
+
if (this.current) return { browserSessionId: this.current.browserSessionId, implicit: true };
|
|
81
|
+
const sessions = [...(await this.deps.listLiveSessions())].sort((left, right) =>
|
|
82
|
+
right.lastUsedAt.localeCompare(left.lastUsedAt),
|
|
83
|
+
);
|
|
84
|
+
const latest = sessions[0];
|
|
85
|
+
if (!latest) {
|
|
86
|
+
throw new BrowserTargetResolutionError(
|
|
87
|
+
"browser_session_required",
|
|
88
|
+
"This session has no open browser. Call browser_open first, or pass browserSessionId.",
|
|
89
|
+
);
|
|
90
|
+
}
|
|
91
|
+
return { browserSessionId: latest.id, implicit: true };
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
async target(value: {
|
|
95
|
+
browserSessionId?: string | undefined;
|
|
96
|
+
targetId?: string | undefined;
|
|
97
|
+
}): Promise<ResolvedBrowserTarget> {
|
|
98
|
+
const session = await this.session(value.browserSessionId);
|
|
99
|
+
if (value.targetId !== undefined) {
|
|
100
|
+
return { ...session, targetId: value.targetId, implicitTarget: false };
|
|
101
|
+
}
|
|
102
|
+
if (this.current?.browserSessionId === session.browserSessionId && this.current.targetId) {
|
|
103
|
+
return { ...session, targetId: this.current.targetId, implicitTarget: true };
|
|
104
|
+
}
|
|
105
|
+
const pages = (await this.deps.listTargets(session.browserSessionId)).filter(
|
|
106
|
+
(target) => target.kind === "page" || target.kind === "popup",
|
|
107
|
+
);
|
|
108
|
+
const chosen =
|
|
109
|
+
pages.find((target) => target.selected) ?? (pages.length === 1 ? pages[0] : null);
|
|
110
|
+
if (!chosen) {
|
|
111
|
+
throw new BrowserTargetResolutionError(
|
|
112
|
+
"browser_target_required",
|
|
113
|
+
pages.length === 0
|
|
114
|
+
? "This browser has no open tab. Open one with browser_tabs operation=open."
|
|
115
|
+
: `This browser has ${pages.length} tabs and none is selected. Pass targetId from browser_tabs operation=list.`,
|
|
116
|
+
);
|
|
117
|
+
}
|
|
118
|
+
return { ...session, targetId: chosen.id, implicitTarget: true };
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** An element ref passed where a locator is expected, as a bare string. */
|
|
123
|
+
export function expandRefShorthand(action: unknown): unknown {
|
|
124
|
+
if (!action || typeof action !== "object" || Array.isArray(action)) return action;
|
|
125
|
+
const record = action as Record<string, unknown>;
|
|
126
|
+
let expanded: Record<string, unknown> | null = null;
|
|
127
|
+
for (const field of ["locator", "from", "to"] as const) {
|
|
128
|
+
const value = record[field];
|
|
129
|
+
if (typeof value === "string") {
|
|
130
|
+
expanded ??= { ...record };
|
|
131
|
+
expanded[field] = { kind: "ref", ref: value };
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
if (Array.isArray(record.actions)) {
|
|
135
|
+
expanded ??= { ...record };
|
|
136
|
+
expanded.actions = record.actions.map(expandRefShorthand);
|
|
137
|
+
}
|
|
138
|
+
return expanded ?? action;
|
|
139
|
+
}
|
|
@@ -0,0 +1,378 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The model's text view of browser tool results.
|
|
3
|
+
*
|
|
4
|
+
* A browser tool result keeps its complete typed value in `structuredContent`
|
|
5
|
+
* for Code Mode, the API and the UI. What the model reads is this text: an
|
|
6
|
+
* outline with one line per page element, in the role/name vocabulary that
|
|
7
|
+
* locators use. It repeats no envelope keys, quoting or bookkeeping ids the
|
|
8
|
+
* model never sends back, which makes a page view about half the tokens of
|
|
9
|
+
* the same value as JSON.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
export type BrowserTextNode = {
|
|
13
|
+
ref: string;
|
|
14
|
+
role: string;
|
|
15
|
+
depth: number;
|
|
16
|
+
name?: string | undefined;
|
|
17
|
+
description?: string | undefined;
|
|
18
|
+
value?: string | { redacted: true; reason: string } | undefined;
|
|
19
|
+
states?: readonly string[] | undefined;
|
|
20
|
+
actions?: readonly string[] | undefined;
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
export type BrowserTextTarget = {
|
|
24
|
+
id: string;
|
|
25
|
+
kind: string;
|
|
26
|
+
title: string;
|
|
27
|
+
url: string;
|
|
28
|
+
selected?: boolean | undefined;
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
export type BrowserTextObservation = {
|
|
32
|
+
target: BrowserTextTarget;
|
|
33
|
+
viewport?:
|
|
34
|
+
| {
|
|
35
|
+
width: number;
|
|
36
|
+
height: number;
|
|
37
|
+
deviceScaleFactor?: number | undefined;
|
|
38
|
+
maxTouchPoints?: number | undefined;
|
|
39
|
+
}
|
|
40
|
+
| null
|
|
41
|
+
| undefined;
|
|
42
|
+
focusedRef?: string | null | undefined;
|
|
43
|
+
changedRegions?: readonly unknown[] | undefined;
|
|
44
|
+
diagnostics?:
|
|
45
|
+
| {
|
|
46
|
+
consoleErrorCount: number;
|
|
47
|
+
failedRequestCount: number;
|
|
48
|
+
downloadCount: number;
|
|
49
|
+
pageErrorCount: number;
|
|
50
|
+
}
|
|
51
|
+
| null
|
|
52
|
+
| undefined;
|
|
53
|
+
dialog?: { type: string; message: string; defaultPrompt?: string | undefined } | null | undefined;
|
|
54
|
+
screenshot?: unknown;
|
|
55
|
+
agentView: {
|
|
56
|
+
sourceKind: string;
|
|
57
|
+
nodes: readonly BrowserTextNode[];
|
|
58
|
+
omittedNodeCount?: number | undefined;
|
|
59
|
+
removedRefCount?: number | undefined;
|
|
60
|
+
clippedFieldCount?: number | undefined;
|
|
61
|
+
};
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
/** Text, structure and table-layout roles. Locate these by text or role and
|
|
65
|
+
* name; listing their refs would cost every page view several tokens per line
|
|
66
|
+
* for nodes that are almost never acted on. A complete view, a focused read,
|
|
67
|
+
* and any node that reports actions still show every ref. */
|
|
68
|
+
const REFLESS_ROLES: ReadonlySet<string> = new Set([
|
|
69
|
+
"article",
|
|
70
|
+
"banner",
|
|
71
|
+
"blockquote",
|
|
72
|
+
"caption",
|
|
73
|
+
"cell",
|
|
74
|
+
"code",
|
|
75
|
+
"columnheader",
|
|
76
|
+
"complementary",
|
|
77
|
+
"contentinfo",
|
|
78
|
+
"definition",
|
|
79
|
+
"deletion",
|
|
80
|
+
"document",
|
|
81
|
+
"emphasis",
|
|
82
|
+
"figure",
|
|
83
|
+
"form",
|
|
84
|
+
"generic",
|
|
85
|
+
"group",
|
|
86
|
+
"heading",
|
|
87
|
+
"insertion",
|
|
88
|
+
"LabelText",
|
|
89
|
+
"LineBreak",
|
|
90
|
+
"list",
|
|
91
|
+
"listitem",
|
|
92
|
+
"log",
|
|
93
|
+
"main",
|
|
94
|
+
"mark",
|
|
95
|
+
"navigation",
|
|
96
|
+
"none",
|
|
97
|
+
"note",
|
|
98
|
+
"paragraph",
|
|
99
|
+
"presentation",
|
|
100
|
+
"region",
|
|
101
|
+
"row",
|
|
102
|
+
"rowgroup",
|
|
103
|
+
"rowheader",
|
|
104
|
+
"Section",
|
|
105
|
+
"separator",
|
|
106
|
+
"status",
|
|
107
|
+
"strong",
|
|
108
|
+
"subscript",
|
|
109
|
+
"superscript",
|
|
110
|
+
"table",
|
|
111
|
+
"term",
|
|
112
|
+
"text",
|
|
113
|
+
"time",
|
|
114
|
+
]);
|
|
115
|
+
|
|
116
|
+
/** States every page repeats that tell the model nothing it can act on. */
|
|
117
|
+
function shownState(role: string, state: string): boolean {
|
|
118
|
+
if (state === "focusable" || state === "settable" || state === "editable") return false;
|
|
119
|
+
if (state.startsWith("editable=") || state.startsWith("url=")) return false;
|
|
120
|
+
if (state.endsWith("=")) return false;
|
|
121
|
+
if (role === "document" && state === "focused") return false;
|
|
122
|
+
return true;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function quoted(value: string): string {
|
|
126
|
+
return JSON.stringify(value);
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
export function browserNodeLine(node: BrowserTextNode, allRefs: boolean, baseDepth = 0): string {
|
|
130
|
+
const parts = [`${" ".repeat(Math.max(0, node.depth - baseDepth))}- ${node.role}`];
|
|
131
|
+
if (node.name) parts.push(quoted(node.name));
|
|
132
|
+
if (allRefs || (node.actions?.length ?? 0) > 0 || !REFLESS_ROLES.has(node.role)) {
|
|
133
|
+
parts.push(`[${node.ref}]`);
|
|
134
|
+
}
|
|
135
|
+
if (node.value !== undefined && node.value !== "") {
|
|
136
|
+
parts.push(typeof node.value === "string" ? `value=${quoted(node.value)}` : "value=<redacted>");
|
|
137
|
+
}
|
|
138
|
+
if (node.description) parts.push(`description=${quoted(node.description)}`);
|
|
139
|
+
for (const state of node.states ?? []) if (shownState(node.role, state)) parts.push(state);
|
|
140
|
+
return parts.join(" ");
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
export function browserTargetLine(target: BrowserTextTarget): string {
|
|
144
|
+
const kind = target.kind === "page" ? "" : ` (${target.kind})`;
|
|
145
|
+
const selected = target.selected ? " selected" : "";
|
|
146
|
+
return `Tab ${target.id}${kind}${selected}: ${quoted(target.title)} ${target.url}`;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/** Indent relative to the shallowest listed node, so a view of a deeply
|
|
150
|
+
* nested part of the page does not spend tokens on leading spaces. */
|
|
151
|
+
function shallowestDepth(nodes: readonly BrowserTextNode[]): number {
|
|
152
|
+
let base = Number.POSITIVE_INFINITY;
|
|
153
|
+
for (const node of nodes) base = Math.min(base, node.depth);
|
|
154
|
+
return Number.isFinite(base) ? base : 0;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function plural(count: number, noun: string): string {
|
|
158
|
+
if (count === 1) return `${count} ${noun}`;
|
|
159
|
+
return `${count} ${noun}${/(?:s|x|ch|sh)$/.test(noun) ? "es" : "s"}`;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
export function browserObservationText(
|
|
163
|
+
observation: BrowserTextObservation,
|
|
164
|
+
options: { allRefs?: boolean } = {},
|
|
165
|
+
): string {
|
|
166
|
+
const allRefs = options.allRefs === true;
|
|
167
|
+
const lines = [browserTargetLine({ ...observation.target, selected: false })];
|
|
168
|
+
const viewport = observation.viewport;
|
|
169
|
+
if (viewport) {
|
|
170
|
+
const extras = [
|
|
171
|
+
viewport.deviceScaleFactor && viewport.deviceScaleFactor !== 1
|
|
172
|
+
? `scale ${viewport.deviceScaleFactor}`
|
|
173
|
+
: null,
|
|
174
|
+
viewport.maxTouchPoints ? `${viewport.maxTouchPoints} touch points` : null,
|
|
175
|
+
].filter((entry): entry is string => entry !== null);
|
|
176
|
+
lines.push(
|
|
177
|
+
`Viewport ${viewport.width}x${viewport.height}${extras.length ? ` (${extras.join(", ")})` : ""}`,
|
|
178
|
+
);
|
|
179
|
+
}
|
|
180
|
+
const dialog = observation.dialog;
|
|
181
|
+
if (dialog) {
|
|
182
|
+
const prompt =
|
|
183
|
+
dialog.type === "prompt" && dialog.defaultPrompt
|
|
184
|
+
? ` default ${quoted(dialog.defaultPrompt)}`
|
|
185
|
+
: "";
|
|
186
|
+
lines.push(`Open ${dialog.type} dialog: ${quoted(dialog.message)}${prompt}`);
|
|
187
|
+
}
|
|
188
|
+
const diagnostics = observation.diagnostics;
|
|
189
|
+
if (diagnostics) {
|
|
190
|
+
const reported = [
|
|
191
|
+
diagnostics.consoleErrorCount ? plural(diagnostics.consoleErrorCount, "console error") : null,
|
|
192
|
+
diagnostics.pageErrorCount ? plural(diagnostics.pageErrorCount, "page error") : null,
|
|
193
|
+
diagnostics.failedRequestCount
|
|
194
|
+
? plural(diagnostics.failedRequestCount, "failed request")
|
|
195
|
+
: null,
|
|
196
|
+
diagnostics.downloadCount ? plural(diagnostics.downloadCount, "download") : null,
|
|
197
|
+
].filter((entry): entry is string => entry !== null);
|
|
198
|
+
if (reported.length) lines.push(`Page reported ${reported.join(", ")}`);
|
|
199
|
+
}
|
|
200
|
+
if (observation.changedRegions && observation.changedRegions.length > 0) {
|
|
201
|
+
lines.push(`Changed regions: ${JSON.stringify(observation.changedRegions)}`);
|
|
202
|
+
}
|
|
203
|
+
if (observation.screenshot) lines.push(`Screenshot: ${JSON.stringify(observation.screenshot)}`);
|
|
204
|
+
const view = observation.agentView;
|
|
205
|
+
const nodes = view.nodes;
|
|
206
|
+
if (
|
|
207
|
+
observation.focusedRef &&
|
|
208
|
+
!nodes.some(
|
|
209
|
+
(node) => node.ref === observation.focusedRef && (node.states ?? []).includes("focused"),
|
|
210
|
+
)
|
|
211
|
+
) {
|
|
212
|
+
lines.push(`Focused element: ${observation.focusedRef}`);
|
|
213
|
+
}
|
|
214
|
+
if (view.sourceKind === "none") {
|
|
215
|
+
lines.push("No accessibility view is available for this tab.");
|
|
216
|
+
} else {
|
|
217
|
+
if (view.sourceKind === "diff") lines.push("Changed elements:");
|
|
218
|
+
const base = shallowestDepth(nodes);
|
|
219
|
+
for (const node of nodes) lines.push(browserNodeLine(node, allRefs, base));
|
|
220
|
+
}
|
|
221
|
+
const notes = [
|
|
222
|
+
view.omittedNodeCount ? `${plural(view.omittedNodeCount, "element")} not shown` : null,
|
|
223
|
+
view.removedRefCount ? `${plural(view.removedRefCount, "earlier element")} removed` : null,
|
|
224
|
+
view.clippedFieldCount ? `${plural(view.clippedFieldCount, "long field")} clipped` : null,
|
|
225
|
+
].filter((entry): entry is string => entry !== null);
|
|
226
|
+
if (notes.length) lines.push(`(${notes.join("; ")})`);
|
|
227
|
+
return lines.join("\n");
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
export type BrowserTextResolvedTarget = {
|
|
231
|
+
actionIndex: number;
|
|
232
|
+
ref: string | null;
|
|
233
|
+
role: string;
|
|
234
|
+
name: string | null;
|
|
235
|
+
container: { role: string; name: string | null } | null;
|
|
236
|
+
};
|
|
237
|
+
|
|
238
|
+
function resolvedTargetText(target: BrowserTextResolvedTarget): string {
|
|
239
|
+
const parts = [`#${target.actionIndex} ${target.role}`];
|
|
240
|
+
if (target.name) parts.push(quoted(target.name));
|
|
241
|
+
if (target.ref) parts.push(`[${target.ref}]`);
|
|
242
|
+
if (target.container) {
|
|
243
|
+
parts.push(
|
|
244
|
+
`in ${target.container.role}${target.container.name ? ` ${quoted(target.container.name)}` : ""}`,
|
|
245
|
+
);
|
|
246
|
+
}
|
|
247
|
+
return parts.join(" ");
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
export function browserActionText(receipt: {
|
|
251
|
+
targetId: string;
|
|
252
|
+
state: string;
|
|
253
|
+
error?:
|
|
254
|
+
| ({ code: string; message: string; retryable: boolean } & Record<string, unknown>)
|
|
255
|
+
| null
|
|
256
|
+
| undefined;
|
|
257
|
+
targets?: readonly BrowserTextResolvedTarget[] | undefined;
|
|
258
|
+
observation?: BrowserTextObservation | null | undefined;
|
|
259
|
+
allRefs?: boolean | undefined;
|
|
260
|
+
}): string {
|
|
261
|
+
const lines = [`Action ${receipt.state} on tab ${receipt.targetId}.`];
|
|
262
|
+
if (receipt.error) {
|
|
263
|
+
const { code, message, retryable, ...details } = receipt.error;
|
|
264
|
+
lines.push(
|
|
265
|
+
`Error ${code}${retryable ? " (retryable)" : ""}: ${message}${
|
|
266
|
+
Object.keys(details).length ? ` ${JSON.stringify(details)}` : ""
|
|
267
|
+
}`,
|
|
268
|
+
);
|
|
269
|
+
}
|
|
270
|
+
if (receipt.targets && receipt.targets.length > 0) {
|
|
271
|
+
lines.push(`Acted on: ${receipt.targets.map(resolvedTargetText).join("; ")}`);
|
|
272
|
+
}
|
|
273
|
+
if (receipt.observation) {
|
|
274
|
+
lines.push(browserObservationText(receipt.observation, { allRefs: receipt.allRefs === true }));
|
|
275
|
+
}
|
|
276
|
+
return lines.join("\n");
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
const PLACEMENT_LABELS: Record<string, string> = {
|
|
280
|
+
sandbox_group: "the managed sandbox",
|
|
281
|
+
connected_machine: "a Connected Machine",
|
|
282
|
+
external_provider: "a remote browser provider",
|
|
283
|
+
attached_device: "the user's own Chrome",
|
|
284
|
+
};
|
|
285
|
+
|
|
286
|
+
export function browserSessionLine(session: {
|
|
287
|
+
id: string;
|
|
288
|
+
lifecycle: string;
|
|
289
|
+
placement: { kind: string };
|
|
290
|
+
engine?: string | null | undefined;
|
|
291
|
+
headless?: boolean | null | undefined;
|
|
292
|
+
identityId?: string | null | undefined;
|
|
293
|
+
failureCode?: string | null | undefined;
|
|
294
|
+
}): string {
|
|
295
|
+
const engine = session.engine ?? "browser";
|
|
296
|
+
const mode =
|
|
297
|
+
session.placement.kind === "attached_device" ? "" : session.headless ? " headless" : " headed";
|
|
298
|
+
const placement = PLACEMENT_LABELS[session.placement.kind] ?? session.placement.kind;
|
|
299
|
+
const extras = [
|
|
300
|
+
session.identityId ? `identity ${session.identityId}` : null,
|
|
301
|
+
session.failureCode ? `failure ${session.failureCode}` : null,
|
|
302
|
+
].filter((entry): entry is string => entry !== null);
|
|
303
|
+
return `BrowserSession ${session.id} (${session.lifecycle}): ${engine}${mode} on ${placement}${
|
|
304
|
+
extras.length ? `, ${extras.join(", ")}` : ""
|
|
305
|
+
}`;
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
type AttachedProfileText = { id: string; name: string; profileLabel?: string | undefined };
|
|
309
|
+
|
|
310
|
+
function attachedProfileText(profile: AttachedProfileText): string {
|
|
311
|
+
return `${quoted(profile.name)}${profile.profileLabel ? ` (${profile.profileLabel})` : ""} deviceId ${profile.id}`;
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
export function browserOpenText(output: {
|
|
315
|
+
session: Parameters<typeof browserSessionLine>[0];
|
|
316
|
+
targets: readonly BrowserTextTarget[];
|
|
317
|
+
otherTabCount?: number | undefined;
|
|
318
|
+
observation?: BrowserTextObservation | undefined;
|
|
319
|
+
attachedProfile?: AttachedProfileText | undefined;
|
|
320
|
+
otherAttachedProfiles?: readonly AttachedProfileText[] | undefined;
|
|
321
|
+
}): string {
|
|
322
|
+
const lines = [browserSessionLine(output.session)];
|
|
323
|
+
if (output.attachedProfile) {
|
|
324
|
+
lines.push(`Attached Chrome profile: ${attachedProfileText(output.attachedProfile)}`);
|
|
325
|
+
}
|
|
326
|
+
if (output.otherAttachedProfiles && output.otherAttachedProfiles.length > 0) {
|
|
327
|
+
lines.push(
|
|
328
|
+
`Other attached profiles: ${output.otherAttachedProfiles.map(attachedProfileText).join("; ")}`,
|
|
329
|
+
);
|
|
330
|
+
}
|
|
331
|
+
const observedId = output.observation?.target.id;
|
|
332
|
+
for (const target of output.targets) {
|
|
333
|
+
if (target.id !== observedId) lines.push(browserTargetLine(target));
|
|
334
|
+
}
|
|
335
|
+
if (output.otherTabCount) {
|
|
336
|
+
lines.push(`${plural(output.otherTabCount, "other tab")} (browser_tabs list shows them)`);
|
|
337
|
+
}
|
|
338
|
+
if (output.observation) lines.push(browserObservationText(output.observation));
|
|
339
|
+
else if (output.targets.length === 0) lines.push("No tabs are open.");
|
|
340
|
+
return lines.join("\n");
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
export function browserTabsText(output: {
|
|
344
|
+
browserSessionId: string;
|
|
345
|
+
targets: readonly BrowserTextTarget[];
|
|
346
|
+
mutationObservation?: BrowserTextObservation | undefined;
|
|
347
|
+
}): string {
|
|
348
|
+
const lines = [
|
|
349
|
+
`BrowserSession ${output.browserSessionId} has ${plural(output.targets.length, "tab")}:`,
|
|
350
|
+
];
|
|
351
|
+
for (const target of output.targets) lines.push(browserTargetLine(target));
|
|
352
|
+
if (output.mutationObservation) lines.push(browserObservationText(output.mutationObservation));
|
|
353
|
+
return lines.join("\n");
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
export function browserReadText(output: {
|
|
357
|
+
targetId: string;
|
|
358
|
+
mode: string;
|
|
359
|
+
scopeFound: boolean;
|
|
360
|
+
nodes: readonly BrowserTextNode[];
|
|
361
|
+
totalMatches: number;
|
|
362
|
+
omittedMatches: number;
|
|
363
|
+
clippedFieldCount: number;
|
|
364
|
+
}): string {
|
|
365
|
+
const lines = [
|
|
366
|
+
`Tab ${output.targetId}: ${plural(output.totalMatches, "match")} (${output.mode})${
|
|
367
|
+
output.scopeFound ? "" : "; the scope element was not found"
|
|
368
|
+
}`,
|
|
369
|
+
];
|
|
370
|
+
const base = shallowestDepth(output.nodes);
|
|
371
|
+
for (const node of output.nodes) lines.push(browserNodeLine(node, true, base));
|
|
372
|
+
const notes = [
|
|
373
|
+
output.omittedMatches ? `${plural(output.omittedMatches, "match")} not shown` : null,
|
|
374
|
+
output.clippedFieldCount ? `${plural(output.clippedFieldCount, "long field")} clipped` : null,
|
|
375
|
+
].filter((entry): entry is string => entry !== null);
|
|
376
|
+
if (notes.length) lines.push(`(${notes.join("; ")})`);
|
|
377
|
+
return lines.join("\n");
|
|
378
|
+
}
|
package/src/codex-tool-search.ts
CHANGED
|
@@ -135,6 +135,59 @@ function familyCompanions<T extends Tool & { name: string }>(disclosed: T[], poo
|
|
|
135
135
|
return companions;
|
|
136
136
|
}
|
|
137
137
|
|
|
138
|
+
/**
|
|
139
|
+
* The tool that starts each interaction family. It stays in the first request,
|
|
140
|
+
* so a browser or computer task begins with its first real call instead of a
|
|
141
|
+
* search round trip, and opening discloses the rest of the family right after
|
|
142
|
+
* the open result (see the lazy-tool transport). Only a tool the attempt
|
|
143
|
+
* actually prepared can be visible: a first-party interaction tool, or one the
|
|
144
|
+
* `interaction` server listed, never a same-named tool from another server.
|
|
145
|
+
*/
|
|
146
|
+
const INTERACTION_ENTRY_TOOLS: ReadonlySet<string> = new Set(["browser_open", "computer_open"]);
|
|
147
|
+
|
|
148
|
+
/** True when a prepared function tool is an interaction family's entry tool. */
|
|
149
|
+
export function isInteractionEntryFunctionTool(
|
|
150
|
+
tool: unknown,
|
|
151
|
+
mcpServerIds: ReadonlySet<string> = NO_MCP_SERVER_IDS,
|
|
152
|
+
modelServerIds?: ReadonlyMap<string, string>,
|
|
153
|
+
): boolean {
|
|
154
|
+
if (!tool || typeof tool !== "object" || (tool as { type?: unknown }).type !== "function") {
|
|
155
|
+
return false;
|
|
156
|
+
}
|
|
157
|
+
const name = (tool as { name?: unknown }).name;
|
|
158
|
+
if (typeof name !== "string" || !isInteractionEntryToolName(name)) return false;
|
|
159
|
+
const serverId = mcpServerIdForTool(tool, mcpServerIds, modelServerIds);
|
|
160
|
+
return serverId === null || serverId === INTERACTION_MCP_SERVER_ID;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/** True when a model tool name is an interaction family's entry tool. */
|
|
164
|
+
export function isInteractionEntryToolName(name: string): boolean {
|
|
165
|
+
const short = interactionToolName(name);
|
|
166
|
+
return short !== null && INTERACTION_ENTRY_TOOLS.has(short);
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* The companions an entry tool discloses once it is called: the rest of its
|
|
171
|
+
* family that is still searchable, within the same byte budgets as a search.
|
|
172
|
+
*/
|
|
173
|
+
export function interactionEntryCompanions(entryName: string, searchable: Tool[]): Tool[] {
|
|
174
|
+
if (!isInteractionEntryToolName(entryName)) return [];
|
|
175
|
+
const pool = searchable.filter(
|
|
176
|
+
(tool): tool is Tool & { name: string } =>
|
|
177
|
+
tool.type === "function" && typeof (tool as { name?: unknown }).name === "string",
|
|
178
|
+
);
|
|
179
|
+
const disclosed: Tool[] = [];
|
|
180
|
+
let disclosedBytes = 0;
|
|
181
|
+
for (const tool of familyCompanions([{ name: entryName } as Tool & { name: string }], pool)) {
|
|
182
|
+
const toolBytes = mcpSerializedSizeBytes(tool);
|
|
183
|
+
if (toolBytes > MCP_MAX_TOOL_DEFINITION_BYTES) continue;
|
|
184
|
+
if (disclosedBytes + toolBytes > MCP_MAX_TOOL_SEARCH_DISCLOSURE_BYTES) continue;
|
|
185
|
+
disclosedBytes += toolBytes;
|
|
186
|
+
disclosed.push(tool);
|
|
187
|
+
}
|
|
188
|
+
return disclosed;
|
|
189
|
+
}
|
|
190
|
+
|
|
138
191
|
/**
|
|
139
192
|
* True when a prepared MCP function tool is one of the harness control tools
|
|
140
193
|
* served by the authorized first-party `opengeni` server. These stay in the
|
package/src/index.ts
CHANGED
|
@@ -117,6 +117,7 @@ import { createSecretArgumentResolver } from "./secret-argument-resolution";
|
|
|
117
117
|
import {
|
|
118
118
|
approvalIdentifier,
|
|
119
119
|
INTERACTION_REQUEST_HUMAN_MODEL_TOOL_NAME,
|
|
120
|
+
RequestHumanInteractionToolInput,
|
|
120
121
|
CAPABILITY_DESCRIPTORS,
|
|
121
122
|
DEFAULT_FIRST_PARTY_MCP_PERMISSIONS,
|
|
122
123
|
DEFAULT_FIRST_PARTY_MCP_TOOLS,
|
|
@@ -303,6 +304,7 @@ import {
|
|
|
303
304
|
applyDiff,
|
|
304
305
|
RunContext,
|
|
305
306
|
ToolGuardrailFunctionOutputFactory,
|
|
307
|
+
defineToolInputGuardrail,
|
|
306
308
|
type AgentInputItem,
|
|
307
309
|
type CallModelInputFilter,
|
|
308
310
|
type MCPServer,
|
|
@@ -3881,6 +3883,23 @@ function activeModelToolInvocation(modelName: string): ModelToolInvocation | nul
|
|
|
3881
3883
|
return invocation?.modelName === modelName ? invocation : null;
|
|
3882
3884
|
}
|
|
3883
3885
|
|
|
3886
|
+
/** Why interaction_request_human arguments cannot be recorded as an
|
|
3887
|
+
* intervention, in words the model can act on, or null when they can. */
|
|
3888
|
+
export function interactionRequestArgumentsProblem(serialized: string): string | null {
|
|
3889
|
+
let value: unknown;
|
|
3890
|
+
try {
|
|
3891
|
+
value = JSON.parse(serialized);
|
|
3892
|
+
} catch {
|
|
3893
|
+
return "Invalid interaction_request_human arguments: they are not valid JSON. Nothing was requested; call it again with a JSON object matching its schema.";
|
|
3894
|
+
}
|
|
3895
|
+
const parsed = RequestHumanInteractionToolInput.safeParse(value);
|
|
3896
|
+
if (parsed.success) return null;
|
|
3897
|
+
const issues = parsed.error.issues
|
|
3898
|
+
.slice(0, 5)
|
|
3899
|
+
.map((issue) => `${issue.path.join(".") || "arguments"}: ${issue.message}`);
|
|
3900
|
+
return `Invalid interaction_request_human arguments: ${issues.join("; ")}. Nothing was requested; call it again with corrected arguments.`;
|
|
3901
|
+
}
|
|
3902
|
+
|
|
3884
3903
|
/**
|
|
3885
3904
|
* Turn the canonical interaction-request tool into a typed SDK interruption.
|
|
3886
3905
|
* Its MCP/Codemode catalog entry and execution stay unchanged; this projection
|
|
@@ -3903,6 +3922,22 @@ function installInteractionInterventionPolicy(
|
|
|
3903
3922
|
const originalInvoke = tool.invoke.bind(tool);
|
|
3904
3923
|
return {
|
|
3905
3924
|
...tool,
|
|
3925
|
+
// Runs before the approval pause, so invalid arguments come back to
|
|
3926
|
+
// the model as a correctable tool result. Without it the call would
|
|
3927
|
+
// pause first, the intervention could never be recorded, and the
|
|
3928
|
+
// turn would fail.
|
|
3929
|
+
inputGuardrails: [
|
|
3930
|
+
...(tool.inputGuardrails ?? []),
|
|
3931
|
+
defineToolInputGuardrail({
|
|
3932
|
+
name: "validate_interaction_request_human",
|
|
3933
|
+
run: async ({ toolCall }) => {
|
|
3934
|
+
const problem = interactionRequestArgumentsProblem(toolCall.arguments);
|
|
3935
|
+
return problem
|
|
3936
|
+
? ToolGuardrailFunctionOutputFactory.rejectContent(problem)
|
|
3937
|
+
: ToolGuardrailFunctionOutputFactory.allow();
|
|
3938
|
+
},
|
|
3939
|
+
}),
|
|
3940
|
+
],
|
|
3906
3941
|
needsApproval: async (
|
|
3907
3942
|
_runContext: Parameters<typeof originalNeedsApproval>[0],
|
|
3908
3943
|
_parsedInput: Parameters<typeof originalNeedsApproval>[1],
|
|
@@ -8469,7 +8504,10 @@ export class PrefixedMcpServer implements MCPServer {
|
|
|
8469
8504
|
return {
|
|
8470
8505
|
...tool,
|
|
8471
8506
|
name,
|
|
8472
|
-
|
|
8507
|
+
// Name the account only when the turn has several accounts of this
|
|
8508
|
+
// connector to tell apart. With one account the line adds nothing
|
|
8509
|
+
// the model can act on, and it would repeat on every tool.
|
|
8510
|
+
...(this.accountLabel?.account
|
|
8473
8511
|
? {
|
|
8474
8512
|
description:
|
|
8475
8513
|
`Account: ${this.accountLabel.model}. Use only this account; if the intended account for a write is unclear, ask before calling.\n${tool.description ?? ""}`.trimEnd(),
|