@opengeni/runtime 3.2.2 → 3.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-LJT5NPWT.js +9 -0
- package/dist/assets/codemode-client.json +1 -1
- package/dist/browser-current-tab.d.ts +46 -0
- package/dist/browser-model-text.d.ts +126 -0
- package/dist/{chunk-2AUDDDO2.js → chunk-3DFH3ZHU.js} +784 -121
- package/dist/chunk-3DFH3ZHU.js.map +1 -0
- package/dist/{chunk-HJSIPTFA.js → chunk-4XDJ4TJA.js} +2 -2
- package/dist/{chunk-XQ23VCZ4.js → chunk-6FQ6KEII.js} +3 -3
- package/dist/{chunk-7LJ7KDPI.js → chunk-ERD4ICVB.js} +4 -3
- package/dist/{chunk-462CH77G.js → chunk-ETTCGPBT.js} +400 -9
- package/dist/chunk-ETTCGPBT.js.map +1 -0
- package/dist/{chunk-7OUG7WBV.js → chunk-GWRBALGQ.js} +181 -13
- package/dist/chunk-GWRBALGQ.js.map +1 -0
- package/dist/{chunk-YG26XE7B.js → chunk-JYDG2A7H.js} +2 -2
- package/dist/{chunk-RSHQSLEJ.js → chunk-KIL4FGX5.js} +3 -3
- package/dist/{chunk-NWS4HPB7.js → chunk-MIAL5I4O.js} +2 -2
- package/dist/{chunk-Z5U6WE22.js → chunk-PRZPGRIK.js} +2 -2
- package/dist/{chunk-KCPW4K7L.js → chunk-PZQUUBEB.js} +2 -2
- package/dist/{chunk-5WJOPANS.js → chunk-Y5THZH5N.js} +7 -7
- package/dist/codex-tool-search.d.ts +9 -0
- package/dist/gmail-rest-mcp.js +8 -8
- package/dist/index.d.ts +3 -0
- package/dist/index.js +14 -12
- package/dist/interaction-tool-json-schema.d.ts +27 -0
- package/dist/{lazy-manifest-AJ6XPJ4F.js → lazy-manifest-BLZ3UZLX.js} +4 -4
- package/dist/lazy-tool-transport.d.ts +10 -0
- package/dist/{localSources-SSIEH5F5.js → localSources-I6SIQ6CX.js} +6 -6
- package/dist/model-call.js +3 -3
- package/dist/sandbox/browser-control-client.d.ts +87 -0
- package/dist/sandbox/browser-control-http.d.ts +33 -0
- package/dist/sandbox/index.d.ts +1 -1
- package/dist/sandbox/index.js +7 -7
- package/dist/sandbox/routing/backend-resolver.d.ts +1 -0
- package/dist/sandbox/selfhosted/session.d.ts +22 -0
- package/dist/slack-rest-mcp.js +8 -8
- package/dist/workspace-tool-gateway.js +12 -12
- package/package.json +14 -14
- package/src/browser-current-tab.ts +139 -0
- package/src/browser-model-text.ts +378 -0
- package/src/bundled_site_skills/opengeni-sites/SKILL.md +46 -12
- package/src/codex-tool-search.ts +53 -0
- package/src/index.ts +39 -1
- package/src/interaction-tool-json-schema.ts +281 -0
- package/src/interaction-tools.ts +425 -235
- package/src/lazy-tool-transport.ts +229 -10
- package/src/model-input.ts +12 -1
- package/src/sandbox/browser-control-client.ts +425 -0
- package/src/sandbox/browser-control-http.ts +32 -0
- package/src/sandbox/index.ts +4 -0
- package/src/sandbox/routing/backend-resolver.ts +2 -0
- package/src/sandbox/selfhosted/session.ts +154 -0
- package/dist/agent-CTPLA3HP.js +0 -9
- package/dist/browser-action-json-schema.d.ts +0 -4
- package/dist/chunk-2AUDDDO2.js.map +0 -1
- package/dist/chunk-462CH77G.js.map +0 -1
- package/dist/chunk-7OUG7WBV.js.map +0 -1
- package/src/browser-action-json-schema.ts +0 -10
- /package/dist/{agent-CTPLA3HP.js.map → agent-LJT5NPWT.js.map} +0 -0
- /package/dist/{chunk-HJSIPTFA.js.map → chunk-4XDJ4TJA.js.map} +0 -0
- /package/dist/{chunk-XQ23VCZ4.js.map → chunk-6FQ6KEII.js.map} +0 -0
- /package/dist/{chunk-7LJ7KDPI.js.map → chunk-ERD4ICVB.js.map} +0 -0
- /package/dist/{chunk-YG26XE7B.js.map → chunk-JYDG2A7H.js.map} +0 -0
- /package/dist/{chunk-RSHQSLEJ.js.map → chunk-KIL4FGX5.js.map} +0 -0
- /package/dist/{chunk-NWS4HPB7.js.map → chunk-MIAL5I4O.js.map} +0 -0
- /package/dist/{chunk-Z5U6WE22.js.map → chunk-PRZPGRIK.js.map} +0 -0
- /package/dist/{chunk-KCPW4K7L.js.map → chunk-PZQUUBEB.js.map} +0 -0
- /package/dist/{chunk-5WJOPANS.js.map → chunk-Y5THZH5N.js.map} +0 -0
- /package/dist/{lazy-manifest-AJ6XPJ4F.js.map → lazy-manifest-BLZ3UZLX.js.map} +0 -0
- /package/dist/{localSources-SSIEH5F5.js.map → localSources-I6SIQ6CX.js.map} +0 -0
|
@@ -0,0 +1,378 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The model's text view of browser tool results.
|
|
3
|
+
*
|
|
4
|
+
* A browser tool result keeps its complete typed value in `structuredContent`
|
|
5
|
+
* for Code Mode, the API and the UI. What the model reads is this text: an
|
|
6
|
+
* outline with one line per page element, in the role/name vocabulary that
|
|
7
|
+
* locators use. It repeats no envelope keys, quoting or bookkeeping ids the
|
|
8
|
+
* model never sends back, which makes a page view about half the tokens of
|
|
9
|
+
* the same value as JSON.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
export type BrowserTextNode = {
|
|
13
|
+
ref: string;
|
|
14
|
+
role: string;
|
|
15
|
+
depth: number;
|
|
16
|
+
name?: string | undefined;
|
|
17
|
+
description?: string | undefined;
|
|
18
|
+
value?: string | { redacted: true; reason: string } | undefined;
|
|
19
|
+
states?: readonly string[] | undefined;
|
|
20
|
+
actions?: readonly string[] | undefined;
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
export type BrowserTextTarget = {
|
|
24
|
+
id: string;
|
|
25
|
+
kind: string;
|
|
26
|
+
title: string;
|
|
27
|
+
url: string;
|
|
28
|
+
selected?: boolean | undefined;
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
export type BrowserTextObservation = {
|
|
32
|
+
target: BrowserTextTarget;
|
|
33
|
+
viewport?:
|
|
34
|
+
| {
|
|
35
|
+
width: number;
|
|
36
|
+
height: number;
|
|
37
|
+
deviceScaleFactor?: number | undefined;
|
|
38
|
+
maxTouchPoints?: number | undefined;
|
|
39
|
+
}
|
|
40
|
+
| null
|
|
41
|
+
| undefined;
|
|
42
|
+
focusedRef?: string | null | undefined;
|
|
43
|
+
changedRegions?: readonly unknown[] | undefined;
|
|
44
|
+
diagnostics?:
|
|
45
|
+
| {
|
|
46
|
+
consoleErrorCount: number;
|
|
47
|
+
failedRequestCount: number;
|
|
48
|
+
downloadCount: number;
|
|
49
|
+
pageErrorCount: number;
|
|
50
|
+
}
|
|
51
|
+
| null
|
|
52
|
+
| undefined;
|
|
53
|
+
dialog?: { type: string; message: string; defaultPrompt?: string | undefined } | null | undefined;
|
|
54
|
+
screenshot?: unknown;
|
|
55
|
+
agentView: {
|
|
56
|
+
sourceKind: string;
|
|
57
|
+
nodes: readonly BrowserTextNode[];
|
|
58
|
+
omittedNodeCount?: number | undefined;
|
|
59
|
+
removedRefCount?: number | undefined;
|
|
60
|
+
clippedFieldCount?: number | undefined;
|
|
61
|
+
};
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
/** Text, structure and table-layout roles. Locate these by text or role and
|
|
65
|
+
* name; listing their refs would cost every page view several tokens per line
|
|
66
|
+
* for nodes that are almost never acted on. A complete view, a focused read,
|
|
67
|
+
* and any node that reports actions still show every ref. */
|
|
68
|
+
const REFLESS_ROLES: ReadonlySet<string> = new Set([
|
|
69
|
+
"article",
|
|
70
|
+
"banner",
|
|
71
|
+
"blockquote",
|
|
72
|
+
"caption",
|
|
73
|
+
"cell",
|
|
74
|
+
"code",
|
|
75
|
+
"columnheader",
|
|
76
|
+
"complementary",
|
|
77
|
+
"contentinfo",
|
|
78
|
+
"definition",
|
|
79
|
+
"deletion",
|
|
80
|
+
"document",
|
|
81
|
+
"emphasis",
|
|
82
|
+
"figure",
|
|
83
|
+
"form",
|
|
84
|
+
"generic",
|
|
85
|
+
"group",
|
|
86
|
+
"heading",
|
|
87
|
+
"insertion",
|
|
88
|
+
"LabelText",
|
|
89
|
+
"LineBreak",
|
|
90
|
+
"list",
|
|
91
|
+
"listitem",
|
|
92
|
+
"log",
|
|
93
|
+
"main",
|
|
94
|
+
"mark",
|
|
95
|
+
"navigation",
|
|
96
|
+
"none",
|
|
97
|
+
"note",
|
|
98
|
+
"paragraph",
|
|
99
|
+
"presentation",
|
|
100
|
+
"region",
|
|
101
|
+
"row",
|
|
102
|
+
"rowgroup",
|
|
103
|
+
"rowheader",
|
|
104
|
+
"Section",
|
|
105
|
+
"separator",
|
|
106
|
+
"status",
|
|
107
|
+
"strong",
|
|
108
|
+
"subscript",
|
|
109
|
+
"superscript",
|
|
110
|
+
"table",
|
|
111
|
+
"term",
|
|
112
|
+
"text",
|
|
113
|
+
"time",
|
|
114
|
+
]);
|
|
115
|
+
|
|
116
|
+
/** States every page repeats that tell the model nothing it can act on. */
|
|
117
|
+
function shownState(role: string, state: string): boolean {
|
|
118
|
+
if (state === "focusable" || state === "settable" || state === "editable") return false;
|
|
119
|
+
if (state.startsWith("editable=") || state.startsWith("url=")) return false;
|
|
120
|
+
if (state.endsWith("=")) return false;
|
|
121
|
+
if (role === "document" && state === "focused") return false;
|
|
122
|
+
return true;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function quoted(value: string): string {
|
|
126
|
+
return JSON.stringify(value);
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
export function browserNodeLine(node: BrowserTextNode, allRefs: boolean, baseDepth = 0): string {
|
|
130
|
+
const parts = [`${" ".repeat(Math.max(0, node.depth - baseDepth))}- ${node.role}`];
|
|
131
|
+
if (node.name) parts.push(quoted(node.name));
|
|
132
|
+
if (allRefs || (node.actions?.length ?? 0) > 0 || !REFLESS_ROLES.has(node.role)) {
|
|
133
|
+
parts.push(`[${node.ref}]`);
|
|
134
|
+
}
|
|
135
|
+
if (node.value !== undefined && node.value !== "") {
|
|
136
|
+
parts.push(typeof node.value === "string" ? `value=${quoted(node.value)}` : "value=<redacted>");
|
|
137
|
+
}
|
|
138
|
+
if (node.description) parts.push(`description=${quoted(node.description)}`);
|
|
139
|
+
for (const state of node.states ?? []) if (shownState(node.role, state)) parts.push(state);
|
|
140
|
+
return parts.join(" ");
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
export function browserTargetLine(target: BrowserTextTarget): string {
|
|
144
|
+
const kind = target.kind === "page" ? "" : ` (${target.kind})`;
|
|
145
|
+
const selected = target.selected ? " selected" : "";
|
|
146
|
+
return `Tab ${target.id}${kind}${selected}: ${quoted(target.title)} ${target.url}`;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/** Indent relative to the shallowest listed node, so a view of a deeply
|
|
150
|
+
* nested part of the page does not spend tokens on leading spaces. */
|
|
151
|
+
function shallowestDepth(nodes: readonly BrowserTextNode[]): number {
|
|
152
|
+
let base = Number.POSITIVE_INFINITY;
|
|
153
|
+
for (const node of nodes) base = Math.min(base, node.depth);
|
|
154
|
+
return Number.isFinite(base) ? base : 0;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function plural(count: number, noun: string): string {
|
|
158
|
+
if (count === 1) return `${count} ${noun}`;
|
|
159
|
+
return `${count} ${noun}${/(?:s|x|ch|sh)$/.test(noun) ? "es" : "s"}`;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
export function browserObservationText(
|
|
163
|
+
observation: BrowserTextObservation,
|
|
164
|
+
options: { allRefs?: boolean } = {},
|
|
165
|
+
): string {
|
|
166
|
+
const allRefs = options.allRefs === true;
|
|
167
|
+
const lines = [browserTargetLine({ ...observation.target, selected: false })];
|
|
168
|
+
const viewport = observation.viewport;
|
|
169
|
+
if (viewport) {
|
|
170
|
+
const extras = [
|
|
171
|
+
viewport.deviceScaleFactor && viewport.deviceScaleFactor !== 1
|
|
172
|
+
? `scale ${viewport.deviceScaleFactor}`
|
|
173
|
+
: null,
|
|
174
|
+
viewport.maxTouchPoints ? `${viewport.maxTouchPoints} touch points` : null,
|
|
175
|
+
].filter((entry): entry is string => entry !== null);
|
|
176
|
+
lines.push(
|
|
177
|
+
`Viewport ${viewport.width}x${viewport.height}${extras.length ? ` (${extras.join(", ")})` : ""}`,
|
|
178
|
+
);
|
|
179
|
+
}
|
|
180
|
+
const dialog = observation.dialog;
|
|
181
|
+
if (dialog) {
|
|
182
|
+
const prompt =
|
|
183
|
+
dialog.type === "prompt" && dialog.defaultPrompt
|
|
184
|
+
? ` default ${quoted(dialog.defaultPrompt)}`
|
|
185
|
+
: "";
|
|
186
|
+
lines.push(`Open ${dialog.type} dialog: ${quoted(dialog.message)}${prompt}`);
|
|
187
|
+
}
|
|
188
|
+
const diagnostics = observation.diagnostics;
|
|
189
|
+
if (diagnostics) {
|
|
190
|
+
const reported = [
|
|
191
|
+
diagnostics.consoleErrorCount ? plural(diagnostics.consoleErrorCount, "console error") : null,
|
|
192
|
+
diagnostics.pageErrorCount ? plural(diagnostics.pageErrorCount, "page error") : null,
|
|
193
|
+
diagnostics.failedRequestCount
|
|
194
|
+
? plural(diagnostics.failedRequestCount, "failed request")
|
|
195
|
+
: null,
|
|
196
|
+
diagnostics.downloadCount ? plural(diagnostics.downloadCount, "download") : null,
|
|
197
|
+
].filter((entry): entry is string => entry !== null);
|
|
198
|
+
if (reported.length) lines.push(`Page reported ${reported.join(", ")}`);
|
|
199
|
+
}
|
|
200
|
+
if (observation.changedRegions && observation.changedRegions.length > 0) {
|
|
201
|
+
lines.push(`Changed regions: ${JSON.stringify(observation.changedRegions)}`);
|
|
202
|
+
}
|
|
203
|
+
if (observation.screenshot) lines.push(`Screenshot: ${JSON.stringify(observation.screenshot)}`);
|
|
204
|
+
const view = observation.agentView;
|
|
205
|
+
const nodes = view.nodes;
|
|
206
|
+
if (
|
|
207
|
+
observation.focusedRef &&
|
|
208
|
+
!nodes.some(
|
|
209
|
+
(node) => node.ref === observation.focusedRef && (node.states ?? []).includes("focused"),
|
|
210
|
+
)
|
|
211
|
+
) {
|
|
212
|
+
lines.push(`Focused element: ${observation.focusedRef}`);
|
|
213
|
+
}
|
|
214
|
+
if (view.sourceKind === "none") {
|
|
215
|
+
lines.push("No accessibility view is available for this tab.");
|
|
216
|
+
} else {
|
|
217
|
+
if (view.sourceKind === "diff") lines.push("Changed elements:");
|
|
218
|
+
const base = shallowestDepth(nodes);
|
|
219
|
+
for (const node of nodes) lines.push(browserNodeLine(node, allRefs, base));
|
|
220
|
+
}
|
|
221
|
+
const notes = [
|
|
222
|
+
view.omittedNodeCount ? `${plural(view.omittedNodeCount, "element")} not shown` : null,
|
|
223
|
+
view.removedRefCount ? `${plural(view.removedRefCount, "earlier element")} removed` : null,
|
|
224
|
+
view.clippedFieldCount ? `${plural(view.clippedFieldCount, "long field")} clipped` : null,
|
|
225
|
+
].filter((entry): entry is string => entry !== null);
|
|
226
|
+
if (notes.length) lines.push(`(${notes.join("; ")})`);
|
|
227
|
+
return lines.join("\n");
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
export type BrowserTextResolvedTarget = {
|
|
231
|
+
actionIndex: number;
|
|
232
|
+
ref: string | null;
|
|
233
|
+
role: string;
|
|
234
|
+
name: string | null;
|
|
235
|
+
container: { role: string; name: string | null } | null;
|
|
236
|
+
};
|
|
237
|
+
|
|
238
|
+
function resolvedTargetText(target: BrowserTextResolvedTarget): string {
|
|
239
|
+
const parts = [`#${target.actionIndex} ${target.role}`];
|
|
240
|
+
if (target.name) parts.push(quoted(target.name));
|
|
241
|
+
if (target.ref) parts.push(`[${target.ref}]`);
|
|
242
|
+
if (target.container) {
|
|
243
|
+
parts.push(
|
|
244
|
+
`in ${target.container.role}${target.container.name ? ` ${quoted(target.container.name)}` : ""}`,
|
|
245
|
+
);
|
|
246
|
+
}
|
|
247
|
+
return parts.join(" ");
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
export function browserActionText(receipt: {
|
|
251
|
+
targetId: string;
|
|
252
|
+
state: string;
|
|
253
|
+
error?:
|
|
254
|
+
| ({ code: string; message: string; retryable: boolean } & Record<string, unknown>)
|
|
255
|
+
| null
|
|
256
|
+
| undefined;
|
|
257
|
+
targets?: readonly BrowserTextResolvedTarget[] | undefined;
|
|
258
|
+
observation?: BrowserTextObservation | null | undefined;
|
|
259
|
+
allRefs?: boolean | undefined;
|
|
260
|
+
}): string {
|
|
261
|
+
const lines = [`Action ${receipt.state} on tab ${receipt.targetId}.`];
|
|
262
|
+
if (receipt.error) {
|
|
263
|
+
const { code, message, retryable, ...details } = receipt.error;
|
|
264
|
+
lines.push(
|
|
265
|
+
`Error ${code}${retryable ? " (retryable)" : ""}: ${message}${
|
|
266
|
+
Object.keys(details).length ? ` ${JSON.stringify(details)}` : ""
|
|
267
|
+
}`,
|
|
268
|
+
);
|
|
269
|
+
}
|
|
270
|
+
if (receipt.targets && receipt.targets.length > 0) {
|
|
271
|
+
lines.push(`Acted on: ${receipt.targets.map(resolvedTargetText).join("; ")}`);
|
|
272
|
+
}
|
|
273
|
+
if (receipt.observation) {
|
|
274
|
+
lines.push(browserObservationText(receipt.observation, { allRefs: receipt.allRefs === true }));
|
|
275
|
+
}
|
|
276
|
+
return lines.join("\n");
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
const PLACEMENT_LABELS: Record<string, string> = {
|
|
280
|
+
sandbox_group: "the managed sandbox",
|
|
281
|
+
connected_machine: "a Connected Machine",
|
|
282
|
+
external_provider: "a remote browser provider",
|
|
283
|
+
attached_device: "the user's own Chrome",
|
|
284
|
+
};
|
|
285
|
+
|
|
286
|
+
export function browserSessionLine(session: {
|
|
287
|
+
id: string;
|
|
288
|
+
lifecycle: string;
|
|
289
|
+
placement: { kind: string };
|
|
290
|
+
engine?: string | null | undefined;
|
|
291
|
+
headless?: boolean | null | undefined;
|
|
292
|
+
identityId?: string | null | undefined;
|
|
293
|
+
failureCode?: string | null | undefined;
|
|
294
|
+
}): string {
|
|
295
|
+
const engine = session.engine ?? "browser";
|
|
296
|
+
const mode =
|
|
297
|
+
session.placement.kind === "attached_device" ? "" : session.headless ? " headless" : " headed";
|
|
298
|
+
const placement = PLACEMENT_LABELS[session.placement.kind] ?? session.placement.kind;
|
|
299
|
+
const extras = [
|
|
300
|
+
session.identityId ? `identity ${session.identityId}` : null,
|
|
301
|
+
session.failureCode ? `failure ${session.failureCode}` : null,
|
|
302
|
+
].filter((entry): entry is string => entry !== null);
|
|
303
|
+
return `BrowserSession ${session.id} (${session.lifecycle}): ${engine}${mode} on ${placement}${
|
|
304
|
+
extras.length ? `, ${extras.join(", ")}` : ""
|
|
305
|
+
}`;
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
type AttachedProfileText = { id: string; name: string; profileLabel?: string | undefined };
|
|
309
|
+
|
|
310
|
+
function attachedProfileText(profile: AttachedProfileText): string {
|
|
311
|
+
return `${quoted(profile.name)}${profile.profileLabel ? ` (${profile.profileLabel})` : ""} deviceId ${profile.id}`;
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
export function browserOpenText(output: {
|
|
315
|
+
session: Parameters<typeof browserSessionLine>[0];
|
|
316
|
+
targets: readonly BrowserTextTarget[];
|
|
317
|
+
otherTabCount?: number | undefined;
|
|
318
|
+
observation?: BrowserTextObservation | undefined;
|
|
319
|
+
attachedProfile?: AttachedProfileText | undefined;
|
|
320
|
+
otherAttachedProfiles?: readonly AttachedProfileText[] | undefined;
|
|
321
|
+
}): string {
|
|
322
|
+
const lines = [browserSessionLine(output.session)];
|
|
323
|
+
if (output.attachedProfile) {
|
|
324
|
+
lines.push(`Attached Chrome profile: ${attachedProfileText(output.attachedProfile)}`);
|
|
325
|
+
}
|
|
326
|
+
if (output.otherAttachedProfiles && output.otherAttachedProfiles.length > 0) {
|
|
327
|
+
lines.push(
|
|
328
|
+
`Other attached profiles: ${output.otherAttachedProfiles.map(attachedProfileText).join("; ")}`,
|
|
329
|
+
);
|
|
330
|
+
}
|
|
331
|
+
const observedId = output.observation?.target.id;
|
|
332
|
+
for (const target of output.targets) {
|
|
333
|
+
if (target.id !== observedId) lines.push(browserTargetLine(target));
|
|
334
|
+
}
|
|
335
|
+
if (output.otherTabCount) {
|
|
336
|
+
lines.push(`${plural(output.otherTabCount, "other tab")} (browser_tabs list shows them)`);
|
|
337
|
+
}
|
|
338
|
+
if (output.observation) lines.push(browserObservationText(output.observation));
|
|
339
|
+
else if (output.targets.length === 0) lines.push("No tabs are open.");
|
|
340
|
+
return lines.join("\n");
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
export function browserTabsText(output: {
|
|
344
|
+
browserSessionId: string;
|
|
345
|
+
targets: readonly BrowserTextTarget[];
|
|
346
|
+
mutationObservation?: BrowserTextObservation | undefined;
|
|
347
|
+
}): string {
|
|
348
|
+
const lines = [
|
|
349
|
+
`BrowserSession ${output.browserSessionId} has ${plural(output.targets.length, "tab")}:`,
|
|
350
|
+
];
|
|
351
|
+
for (const target of output.targets) lines.push(browserTargetLine(target));
|
|
352
|
+
if (output.mutationObservation) lines.push(browserObservationText(output.mutationObservation));
|
|
353
|
+
return lines.join("\n");
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
export function browserReadText(output: {
|
|
357
|
+
targetId: string;
|
|
358
|
+
mode: string;
|
|
359
|
+
scopeFound: boolean;
|
|
360
|
+
nodes: readonly BrowserTextNode[];
|
|
361
|
+
totalMatches: number;
|
|
362
|
+
omittedMatches: number;
|
|
363
|
+
clippedFieldCount: number;
|
|
364
|
+
}): string {
|
|
365
|
+
const lines = [
|
|
366
|
+
`Tab ${output.targetId}: ${plural(output.totalMatches, "match")} (${output.mode})${
|
|
367
|
+
output.scopeFound ? "" : "; the scope element was not found"
|
|
368
|
+
}`,
|
|
369
|
+
];
|
|
370
|
+
const base = shallowestDepth(output.nodes);
|
|
371
|
+
for (const node of output.nodes) lines.push(browserNodeLine(node, true, base));
|
|
372
|
+
const notes = [
|
|
373
|
+
output.omittedMatches ? `${plural(output.omittedMatches, "match")} not shown` : null,
|
|
374
|
+
output.clippedFieldCount ? `${plural(output.clippedFieldCount, "long field")} clipped` : null,
|
|
375
|
+
].filter((entry): entry is string => entry !== null);
|
|
376
|
+
if (notes.length) lines.push(`(${notes.join("; ")})`);
|
|
377
|
+
return lines.join("\n");
|
|
378
|
+
}
|
|
@@ -18,8 +18,9 @@ Site on the web is built in (see "Publish to the web").
|
|
|
18
18
|
|
|
19
19
|
A Site never needs its own server, API key, or hosting provider to reach
|
|
20
20
|
Opengeni. Model access, agent conversations, and workspace tools all come from
|
|
21
|
-
the host bridge: `createOpenGeniSiteClient` from `@opengeni/sdk/site
|
|
22
|
-
`@opengeni/react/session-ui`
|
|
21
|
+
the host bridge: `createOpenGeniSiteClient` from `@opengeni/sdk/site`. Use its
|
|
22
|
+
`ai.run` helper for inline results, or `@opengeni/react/session-ui` when the app
|
|
23
|
+
needs a conversation surface. The parent page owns the
|
|
23
24
|
credentials and workspace identity; Site code receives neither, so "keep
|
|
24
25
|
credentials server-side" is already satisfied by that bridge.
|
|
25
26
|
|
|
@@ -239,6 +240,33 @@ const client = createOpenGeniSiteClient();
|
|
|
239
240
|
const issues = await client.tools.linear.issues_list({ state: "Todo" });
|
|
240
241
|
```
|
|
241
242
|
|
|
243
|
+
### Inline AI: the app chooses the interaction
|
|
244
|
+
|
|
245
|
+
Use `site.ai.run({ initialMessage: appDefinedPrompt, idempotencyKey: actionId })`
|
|
246
|
+
to return AI output inside the app without mounting chat or displaying history.
|
|
247
|
+
Prompts, context, controls and results are app-designed; Draft, Summarize or
|
|
248
|
+
Classify are merely possible examples, never required buttons or templates.
|
|
249
|
+
Use `onText` for incremental text and `onSession` to retain a native handle before
|
|
250
|
+
waiting. A result has `text`, `sessionId`, `turnId`, `after` and `status`:
|
|
251
|
+
`completed`, `pending` (approval/input needed), or `cancelled`; failures reject.
|
|
252
|
+
Validate any app-defined structured output before using it.
|
|
253
|
+
|
|
254
|
+
This is ordinary native execution with the same model access, subscriptions,
|
|
255
|
+
permissions, tool selection, approvals, metering and outsider caps. Aborting the
|
|
256
|
+
observation does not cancel the task. `site.ai.result(id, { turnId })` replays
|
|
257
|
+
without creating more work. Supplying `after` returns subsequent text only;
|
|
258
|
+
retain earlier text or omit it for a full replay. Keep retry keys stable per
|
|
259
|
+
action and handles in viewer-private state/storage, not shared Site Data.
|
|
260
|
+
|
|
261
|
+
Site AI is private per viewer by default and does not clutter the workspace
|
|
262
|
+
sidebar or ordinary search. Signed-in members use their real human identity;
|
|
263
|
+
guests use existing cookie/visitor continuity, not a shared anonymous chat.
|
|
264
|
+
If organization policy disallows private execution, surface its explicit
|
|
265
|
+
refusal; never fall back to shared sessions. Privacy does not automatically
|
|
266
|
+
extend to separately published artifacts or shared Site Data.
|
|
267
|
+
|
|
268
|
+
### Optional conversation UI
|
|
269
|
+
|
|
242
270
|
For embedded conversations, `site.client` is the ordinary Opengeni SDK client;
|
|
243
271
|
`site.workspaceId` is a host-resolved routing alias. Use the normal React
|
|
244
272
|
complete conversation surface—do not implement session REST or SSE yourself:
|
|
@@ -259,7 +287,7 @@ const site = createOpenGeniSiteClient();
|
|
|
259
287
|
```
|
|
260
288
|
|
|
261
289
|
`sessionId` is the existing session or the id returned by `site.client.createSession`.
|
|
262
|
-
For a normal chat app,
|
|
290
|
+
For a normal chat app, a conversation selector and “New conversation” are optional.
|
|
263
291
|
Create on first Send rather than mounting the page; keep the selected id in React
|
|
264
292
|
state and reopen existing conversations through the SDK.
|
|
265
293
|
Keep first-send creation single-flight; reuse its `idempotencyKey` when retrying,
|
|
@@ -272,14 +300,17 @@ write origin metadata yourself. The local preview has no published identity, so
|
|
|
272
300
|
this filter returns no Site history there; test newly created/current conversations
|
|
273
301
|
locally and the selector after publication.
|
|
274
302
|
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
303
|
+
Do not automatically create a project just to group Site chats. Site execution
|
|
304
|
+
does not appear in ordinary workspace discovery, even for its owner. The host
|
|
305
|
+
Site page provides a viewer-scoped Conversations panel independently of app
|
|
306
|
+
navigation. Opening existing sessions never changes their origin or audience.
|
|
307
|
+
Page code cannot widen the Site boundary by supplying project, visibility,
|
|
308
|
+
tools or credential fields; use ordinary authoring tools for project management.
|
|
309
|
+
|
|
310
|
+
Per-Site AI costs are in Insights: actual credit charges are distinct from
|
|
311
|
+
API-equivalent subscription/own-key estimates and unbilled platform counters.
|
|
312
|
+
Usage is for limits. Do not build private transcript or viewer analytics into
|
|
313
|
+
shared dashboards just because aggregate costs are visible there.
|
|
283
314
|
|
|
284
315
|
`SessionConversation` connects timeline/history, durable composer drafts,
|
|
285
316
|
queue display/edit/delete/steer, pause/resume, and human-input forms. Prefer it
|
|
@@ -667,8 +698,11 @@ workspace first.
|
|
|
667
698
|
`rate_limited`, `site_budget_exhausted` and retryable `site_budget_busy` (429), and
|
|
668
699
|
`site_platform_brake_engaged` (503) as normal UI states.
|
|
669
700
|
- The same Site HTML and Site Data work there; no rebuild is needed.
|
|
670
|
-
-
|
|
701
|
+
- Published-host agents get this Site's `site_data_query` and `site_data_write`,
|
|
671
702
|
plus `site_sql_run` when SQL is enabled; members also get read-only `site_sql_query`.
|
|
703
|
+
Member tools are intersected with live permissions: a member without artifact
|
|
704
|
+
publishing can still use AI without write tools, and without data-read permission
|
|
705
|
+
can use plain AI without data tools. Never request extra permission just to chat.
|
|
672
706
|
The server pins the Site and binds calls to the conversation's viewer, including
|
|
673
707
|
collection ownership and query audiences. No management tools or sandbox are
|
|
674
708
|
supplied; publication settings have no sandbox opt-in. Members get no Site
|
package/src/codex-tool-search.ts
CHANGED
|
@@ -135,6 +135,59 @@ function familyCompanions<T extends Tool & { name: string }>(disclosed: T[], poo
|
|
|
135
135
|
return companions;
|
|
136
136
|
}
|
|
137
137
|
|
|
138
|
+
/**
|
|
139
|
+
* The tool that starts each interaction family. It stays in the first request,
|
|
140
|
+
* so a browser or computer task begins with its first real call instead of a
|
|
141
|
+
* search round trip, and opening discloses the rest of the family right after
|
|
142
|
+
* the open result (see the lazy-tool transport). Only a tool the attempt
|
|
143
|
+
* actually prepared can be visible: a first-party interaction tool, or one the
|
|
144
|
+
* `interaction` server listed, never a same-named tool from another server.
|
|
145
|
+
*/
|
|
146
|
+
const INTERACTION_ENTRY_TOOLS: ReadonlySet<string> = new Set(["browser_open", "computer_open"]);
|
|
147
|
+
|
|
148
|
+
/** True when a prepared function tool is an interaction family's entry tool. */
|
|
149
|
+
export function isInteractionEntryFunctionTool(
|
|
150
|
+
tool: unknown,
|
|
151
|
+
mcpServerIds: ReadonlySet<string> = NO_MCP_SERVER_IDS,
|
|
152
|
+
modelServerIds?: ReadonlyMap<string, string>,
|
|
153
|
+
): boolean {
|
|
154
|
+
if (!tool || typeof tool !== "object" || (tool as { type?: unknown }).type !== "function") {
|
|
155
|
+
return false;
|
|
156
|
+
}
|
|
157
|
+
const name = (tool as { name?: unknown }).name;
|
|
158
|
+
if (typeof name !== "string" || !isInteractionEntryToolName(name)) return false;
|
|
159
|
+
const serverId = mcpServerIdForTool(tool, mcpServerIds, modelServerIds);
|
|
160
|
+
return serverId === null || serverId === INTERACTION_MCP_SERVER_ID;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/** True when a model tool name is an interaction family's entry tool. */
|
|
164
|
+
export function isInteractionEntryToolName(name: string): boolean {
|
|
165
|
+
const short = interactionToolName(name);
|
|
166
|
+
return short !== null && INTERACTION_ENTRY_TOOLS.has(short);
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* The companions an entry tool discloses once it is called: the rest of its
|
|
171
|
+
* family that is still searchable, within the same byte budgets as a search.
|
|
172
|
+
*/
|
|
173
|
+
export function interactionEntryCompanions(entryName: string, searchable: Tool[]): Tool[] {
|
|
174
|
+
if (!isInteractionEntryToolName(entryName)) return [];
|
|
175
|
+
const pool = searchable.filter(
|
|
176
|
+
(tool): tool is Tool & { name: string } =>
|
|
177
|
+
tool.type === "function" && typeof (tool as { name?: unknown }).name === "string",
|
|
178
|
+
);
|
|
179
|
+
const disclosed: Tool[] = [];
|
|
180
|
+
let disclosedBytes = 0;
|
|
181
|
+
for (const tool of familyCompanions([{ name: entryName } as Tool & { name: string }], pool)) {
|
|
182
|
+
const toolBytes = mcpSerializedSizeBytes(tool);
|
|
183
|
+
if (toolBytes > MCP_MAX_TOOL_DEFINITION_BYTES) continue;
|
|
184
|
+
if (disclosedBytes + toolBytes > MCP_MAX_TOOL_SEARCH_DISCLOSURE_BYTES) continue;
|
|
185
|
+
disclosedBytes += toolBytes;
|
|
186
|
+
disclosed.push(tool);
|
|
187
|
+
}
|
|
188
|
+
return disclosed;
|
|
189
|
+
}
|
|
190
|
+
|
|
138
191
|
/**
|
|
139
192
|
* True when a prepared MCP function tool is one of the harness control tools
|
|
140
193
|
* served by the authorized first-party `opengeni` server. These stay in the
|
package/src/index.ts
CHANGED
|
@@ -117,6 +117,7 @@ import { createSecretArgumentResolver } from "./secret-argument-resolution";
|
|
|
117
117
|
import {
|
|
118
118
|
approvalIdentifier,
|
|
119
119
|
INTERACTION_REQUEST_HUMAN_MODEL_TOOL_NAME,
|
|
120
|
+
RequestHumanInteractionToolInput,
|
|
120
121
|
CAPABILITY_DESCRIPTORS,
|
|
121
122
|
DEFAULT_FIRST_PARTY_MCP_PERMISSIONS,
|
|
122
123
|
DEFAULT_FIRST_PARTY_MCP_TOOLS,
|
|
@@ -303,6 +304,7 @@ import {
|
|
|
303
304
|
applyDiff,
|
|
304
305
|
RunContext,
|
|
305
306
|
ToolGuardrailFunctionOutputFactory,
|
|
307
|
+
defineToolInputGuardrail,
|
|
306
308
|
type AgentInputItem,
|
|
307
309
|
type CallModelInputFilter,
|
|
308
310
|
type MCPServer,
|
|
@@ -3881,6 +3883,23 @@ function activeModelToolInvocation(modelName: string): ModelToolInvocation | nul
|
|
|
3881
3883
|
return invocation?.modelName === modelName ? invocation : null;
|
|
3882
3884
|
}
|
|
3883
3885
|
|
|
3886
|
+
/** Why interaction_request_human arguments cannot be recorded as an
|
|
3887
|
+
* intervention, in words the model can act on, or null when they can. */
|
|
3888
|
+
export function interactionRequestArgumentsProblem(serialized: string): string | null {
|
|
3889
|
+
let value: unknown;
|
|
3890
|
+
try {
|
|
3891
|
+
value = JSON.parse(serialized);
|
|
3892
|
+
} catch {
|
|
3893
|
+
return "Invalid interaction_request_human arguments: they are not valid JSON. Nothing was requested; call it again with a JSON object matching its schema.";
|
|
3894
|
+
}
|
|
3895
|
+
const parsed = RequestHumanInteractionToolInput.safeParse(value);
|
|
3896
|
+
if (parsed.success) return null;
|
|
3897
|
+
const issues = parsed.error.issues
|
|
3898
|
+
.slice(0, 5)
|
|
3899
|
+
.map((issue) => `${issue.path.join(".") || "arguments"}: ${issue.message}`);
|
|
3900
|
+
return `Invalid interaction_request_human arguments: ${issues.join("; ")}. Nothing was requested; call it again with corrected arguments.`;
|
|
3901
|
+
}
|
|
3902
|
+
|
|
3884
3903
|
/**
|
|
3885
3904
|
* Turn the canonical interaction-request tool into a typed SDK interruption.
|
|
3886
3905
|
* Its MCP/Codemode catalog entry and execution stay unchanged; this projection
|
|
@@ -3903,6 +3922,22 @@ function installInteractionInterventionPolicy(
|
|
|
3903
3922
|
const originalInvoke = tool.invoke.bind(tool);
|
|
3904
3923
|
return {
|
|
3905
3924
|
...tool,
|
|
3925
|
+
// Runs before the approval pause, so invalid arguments come back to
|
|
3926
|
+
// the model as a correctable tool result. Without it the call would
|
|
3927
|
+
// pause first, the intervention could never be recorded, and the
|
|
3928
|
+
// turn would fail.
|
|
3929
|
+
inputGuardrails: [
|
|
3930
|
+
...(tool.inputGuardrails ?? []),
|
|
3931
|
+
defineToolInputGuardrail({
|
|
3932
|
+
name: "validate_interaction_request_human",
|
|
3933
|
+
run: async ({ toolCall }) => {
|
|
3934
|
+
const problem = interactionRequestArgumentsProblem(toolCall.arguments);
|
|
3935
|
+
return problem
|
|
3936
|
+
? ToolGuardrailFunctionOutputFactory.rejectContent(problem)
|
|
3937
|
+
: ToolGuardrailFunctionOutputFactory.allow();
|
|
3938
|
+
},
|
|
3939
|
+
}),
|
|
3940
|
+
],
|
|
3906
3941
|
needsApproval: async (
|
|
3907
3942
|
_runContext: Parameters<typeof originalNeedsApproval>[0],
|
|
3908
3943
|
_parsedInput: Parameters<typeof originalNeedsApproval>[1],
|
|
@@ -8469,7 +8504,10 @@ export class PrefixedMcpServer implements MCPServer {
|
|
|
8469
8504
|
return {
|
|
8470
8505
|
...tool,
|
|
8471
8506
|
name,
|
|
8472
|
-
|
|
8507
|
+
// Name the account only when the turn has several accounts of this
|
|
8508
|
+
// connector to tell apart. With one account the line adds nothing
|
|
8509
|
+
// the model can act on, and it would repeat on every tool.
|
|
8510
|
+
...(this.accountLabel?.account
|
|
8473
8511
|
? {
|
|
8474
8512
|
description:
|
|
8475
8513
|
`Account: ${this.accountLabel.model}. Use only this account; if the intended account for a write is unclear, ask before calling.\n${tool.description ?? ""}`.trimEnd(),
|