@opengeni/runtime 4.0.0 → 4.0.1-canary.35916963870001
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/codemode-client.json +1 -1
- package/dist/{chunk-WIHICNWS.js → chunk-JFMOX5OE.js} +63 -11
- package/dist/chunk-JFMOX5OE.js.map +1 -0
- package/dist/{chunk-5HJEVOMJ.js → chunk-ROCWOHGM.js} +138 -30
- package/dist/chunk-ROCWOHGM.js.map +1 -0
- package/dist/index.d.ts +8 -1
- package/dist/index.js +6 -2
- package/dist/operational-instructions.d.ts +1 -1
- package/dist/sandbox/browser-control-client.d.ts +20 -0
- package/dist/sandbox/index.js +1 -1
- package/dist/sandbox/routing/routing-session.d.ts +2 -1
- package/dist/sandbox/turn-tool-cancellation.d.ts +1 -1
- package/dist/workspace-tool-gateway.js +2 -2
- package/package.json +11 -11
- package/src/bundled_default_skills/opengeni-help/SKILL.md +14 -0
- package/src/bundled_management_skills/opengeni-skills/SKILL.md +1 -1
- package/src/bundled_site_skills/opengeni-sites/SKILL.md +23 -2
- package/src/index.ts +97 -16
- package/src/interaction-tools.ts +72 -13
- package/src/operational-instructions.ts +5 -1
- package/src/sandbox/browser-control-client.ts +109 -23
- package/src/sandbox/routing/routing-session.ts +6 -1
- package/src/sandbox/turn-tool-cancellation.ts +12 -2
- package/dist/chunk-5HJEVOMJ.js.map +0 -1
- package/dist/chunk-WIHICNWS.js.map +0 -1
package/src/interaction-tools.ts
CHANGED
|
@@ -183,6 +183,7 @@ const BrowserObserveInput = z
|
|
|
183
183
|
.object({
|
|
184
184
|
browserSessionId: z.string().uuid(),
|
|
185
185
|
targetId: z.string().min(1).max(512),
|
|
186
|
+
includeScreenshot: z.boolean().optional(),
|
|
186
187
|
})
|
|
187
188
|
.strict();
|
|
188
189
|
const BrowserActInput = z
|
|
@@ -381,7 +382,7 @@ const ComputerActInput = z
|
|
|
381
382
|
computerSessionId: z.string().uuid(),
|
|
382
383
|
targetId: z.string().min(1).max(512),
|
|
383
384
|
expectedTargetGeneration: z.string().min(1).max(256).optional(),
|
|
384
|
-
expectedObservationId:
|
|
385
|
+
expectedObservationId: ComputerObservation.shape.observationId.nullable().optional(),
|
|
385
386
|
expectedFrameId: z.string().min(1).max(256).nullable().optional(),
|
|
386
387
|
action: ComputerAction,
|
|
387
388
|
})
|
|
@@ -510,7 +511,7 @@ export function createInteractionAttemptToolDefinitions(
|
|
|
510
511
|
codemodePath: ["interaction", "browser", "open"],
|
|
511
512
|
title: "Open or reuse browser",
|
|
512
513
|
description:
|
|
513
|
-
"Open a managed BrowserSession on the current agent placement, reuse a relevant compatible live session by default, attach to an explicit workspace BrowserSession, or open an attached Chrome profile by passing placement={kind:'attached_device',deviceId}. This does not infer or attach the user's existing Chrome: for requests about 'my browser', 'my tabs', or current Chrome, call interaction_discover first and select an actual attachedBrowsers device; if none exists, explain that the Chrome extension must be connected instead of silently creating a blank managed browser. Managed Chromium defaults to headed so OAuth and later human interaction use a supported browser; request headless=true only for agent-only work that will not require sign-in or human control. Returns exact session and tab state.",
|
|
514
|
+
"Open a managed BrowserSession on the current agent placement, reuse a relevant compatible live session by default, attach to an explicit workspace BrowserSession, or open an attached Chrome profile by passing placement={kind:'attached_device',deviceId}. This does not infer or attach the user's existing Chrome: for requests about 'my browser', 'my tabs', or current Chrome, call interaction_discover first and select an actual attachedBrowsers device; if none exists, explain that the Chrome extension must be connected instead of silently creating a blank managed browser. BrowserSessions persist across tool calls; shell-launched browser daemons do not survive remote-command cleanup. Never switch to the user's attached Chrome as a fallback for a failed managed browser unless the user requested that profile. A new attached session opens a dedicated tab. Managed Chromium defaults to headed so OAuth and later human interaction use a supported browser; request headless=true only for agent-only work that will not require sign-in or human control. Returns exact session and tab state.",
|
|
514
515
|
input: BrowserOpenInput,
|
|
515
516
|
output: BrowserOpenOutput,
|
|
516
517
|
readOnly: false,
|
|
@@ -524,7 +525,7 @@ export function createInteractionAttemptToolDefinitions(
|
|
|
524
525
|
codemodePath: ["interaction", "browser", "tabs"],
|
|
525
526
|
title: "Manage browser tabs",
|
|
526
527
|
description:
|
|
527
|
-
"List, open, select, or close tabs in one exact BrowserSession. Returns the authoritative complete tab list after the operation.",
|
|
528
|
+
"List, open, logically select, or close tabs in one exact BrowserSession. Selection changes the BrowserSession's default target, not the visible desktop tab. New attached-Chrome tabs open in the background. Use browser_act activate only when foregrounding the owned tab is explicitly intended. Returns the authoritative complete tab list after the operation.",
|
|
528
529
|
input: BrowserTabsInput,
|
|
529
530
|
output: BrowserTargetListResponse,
|
|
530
531
|
readOnly: false,
|
|
@@ -556,17 +557,47 @@ export function createInteractionAttemptToolDefinitions(
|
|
|
556
557
|
codemodePath: ["interaction", "browser", "observe"],
|
|
557
558
|
title: "Observe browser tab",
|
|
558
559
|
description:
|
|
559
|
-
"Read one tab's current URL/title, causal generations, compact semantic accessibility tree, dialog, and diagnostic counts without taking control.",
|
|
560
|
+
"Read one tab's current URL/title, measured page viewport, causal generations, compact semantic accessibility tree, dialog, and diagnostic counts without taking control. Set includeScreenshot=true to receive a current still image as tool image content; the structured screenshot field is only a retained-artifact reference and can remain null.",
|
|
560
561
|
input: BrowserObserveInput,
|
|
561
562
|
output: BrowserObservation,
|
|
562
563
|
readOnly: true,
|
|
563
564
|
idempotent: true,
|
|
564
|
-
execute: async (value) =>
|
|
565
|
-
await input.transport.observeBrowserTarget(
|
|
565
|
+
execute: async (value) => {
|
|
566
|
+
let observation = await input.transport.observeBrowserTarget(
|
|
566
567
|
input.workspaceId,
|
|
567
568
|
value.browserSessionId,
|
|
568
569
|
value.targetId,
|
|
569
|
-
)
|
|
570
|
+
);
|
|
571
|
+
if (!value.includeScreenshot) return observation;
|
|
572
|
+
const frame = await input.transport.captureBrowserTarget(
|
|
573
|
+
input.workspaceId,
|
|
574
|
+
value.browserSessionId,
|
|
575
|
+
value.targetId,
|
|
576
|
+
);
|
|
577
|
+
if (
|
|
578
|
+
observation.target.targetGeneration !== frame.targetGeneration ||
|
|
579
|
+
observation.target.documentGeneration !== frame.documentGeneration
|
|
580
|
+
) {
|
|
581
|
+
observation = await input.transport.observeBrowserTarget(
|
|
582
|
+
input.workspaceId,
|
|
583
|
+
value.browserSessionId,
|
|
584
|
+
value.targetId,
|
|
585
|
+
);
|
|
586
|
+
}
|
|
587
|
+
if (
|
|
588
|
+
observation.target.targetGeneration !== frame.targetGeneration ||
|
|
589
|
+
observation.target.documentGeneration !== frame.documentGeneration
|
|
590
|
+
) {
|
|
591
|
+
throw new Error("browser target changed while its visual observation was captured");
|
|
592
|
+
}
|
|
593
|
+
return new InteractionExecutionResult(observation, [
|
|
594
|
+
{
|
|
595
|
+
type: "image",
|
|
596
|
+
data: Buffer.from(frame.data).toString("base64"),
|
|
597
|
+
mimeType: frame.mediaType,
|
|
598
|
+
},
|
|
599
|
+
]);
|
|
600
|
+
},
|
|
570
601
|
});
|
|
571
602
|
|
|
572
603
|
add({
|
|
@@ -574,7 +605,7 @@ export function createInteractionAttemptToolDefinitions(
|
|
|
574
605
|
codemodePath: ["interaction", "browser", "act"],
|
|
575
606
|
title: "Act in browser tab",
|
|
576
607
|
description:
|
|
577
|
-
"Perform one semantic-first browser action or bounded batch,
|
|
608
|
+
"Perform one semantic-first browser action or bounded batch. Use viewport to set page width, height, desktop/mobile layout and touch emulation, then check the measured viewport in the returned observation; emulation does not prove physical mobile-browser behavior. Use history back/forward for tab navigation; keypress shortcuts are page input and may not navigate browser history. The explicit activate action foregrounds the target in the user's desktop browser; use only when that is intended. Permission actions set a managed browser's web permission for this tab's exact current top-level origin. Omit generation fences to use a fresh observation automatically; provide them to require exact previously observed state. Returns the durable receipt and changed observation.",
|
|
578
609
|
input: BrowserActInput,
|
|
579
610
|
output: BrowserActionReceipt,
|
|
580
611
|
readOnly: false,
|
|
@@ -986,7 +1017,7 @@ export function createInteractionAttemptToolDefinitions(
|
|
|
986
1017
|
codemodePath: ["interaction", "computer", "act"],
|
|
987
1018
|
title: "Act in app or window",
|
|
988
1019
|
description:
|
|
989
|
-
"Perform one action in an exact ComputerSession target. Prefer semantic actions from computer_observe: on macOS they can invoke controls and set values without foregrounding the app. Pointer, keyboard, target focus, screen actions, and clipboard paste use the physical graphical seat and may change the user's foreground app; use them only when foreground control is explicitly intended. Omit fences to use a fresh observation automatically. Returns the durable causal receipt.",
|
|
1020
|
+
"Perform one action in an exact ComputerSession target. Prefer semantic actions from computer_observe: on macOS they can invoke controls and set values without foregrounding the app. Pointer, keyboard, target focus, screen actions, and clipboard paste use the physical graphical seat and may change the user's foreground app; use them only when foreground control is explicitly intended. For pointer actions, set action.frameId from the observed frame and use coordinates in the returned screenshot's pixels; do not scale them to target.bounds. Omit fences to use a fresh observation automatically. Returns the durable causal receipt.",
|
|
990
1021
|
input: ComputerActInput,
|
|
991
1022
|
output: ComputerActionReceipt,
|
|
992
1023
|
readOnly: false,
|
|
@@ -1131,7 +1162,8 @@ async function openBrowser(
|
|
|
1131
1162
|
if (value.browserSessionId) {
|
|
1132
1163
|
session = await transport.getBrowserSession(workspaceId, value.browserSessionId);
|
|
1133
1164
|
} else {
|
|
1134
|
-
const listed =
|
|
1165
|
+
const listed =
|
|
1166
|
+
value.mode === "new" ? { sessions: [] } : await transport.listBrowserSessions(workspaceId);
|
|
1135
1167
|
// The agent-facing browser is human-capable by default. A headless session
|
|
1136
1168
|
// is a deliberately narrower execution mode and must never be silently
|
|
1137
1169
|
// reused for an omitted/default headed request: OAuth providers such as
|
|
@@ -1141,7 +1173,18 @@ async function openBrowser(
|
|
|
1141
1173
|
value.mode === "new"
|
|
1142
1174
|
? null
|
|
1143
1175
|
: newestRelevant(
|
|
1144
|
-
listed.sessions.filter(
|
|
1176
|
+
listed.sessions.filter(
|
|
1177
|
+
(candidate) =>
|
|
1178
|
+
candidate.headless === requestedHeadless &&
|
|
1179
|
+
compatibleInteractionPlacement(candidate.placement, value.placement) &&
|
|
1180
|
+
(value.identityId === undefined || candidate.identityId === value.identityId) &&
|
|
1181
|
+
(value.baseRevisionId === undefined ||
|
|
1182
|
+
candidate.baseRevisionId === value.baseRevisionId) &&
|
|
1183
|
+
(value.networkRouteId === undefined ||
|
|
1184
|
+
candidate.networkRouteId === value.networkRouteId) &&
|
|
1185
|
+
(value.linkedComputerSessionId === undefined ||
|
|
1186
|
+
candidate.linkedComputerSessionId === value.linkedComputerSessionId),
|
|
1187
|
+
),
|
|
1145
1188
|
sourceSessionId,
|
|
1146
1189
|
);
|
|
1147
1190
|
if (reusable) {
|
|
@@ -1205,8 +1248,14 @@ async function openComputer(
|
|
|
1205
1248
|
if (value.computerSessionId) {
|
|
1206
1249
|
session = await transport.getComputerSession(workspaceId, value.computerSessionId);
|
|
1207
1250
|
} else {
|
|
1208
|
-
const listed =
|
|
1209
|
-
|
|
1251
|
+
const listed =
|
|
1252
|
+
value.mode === "new" ? { sessions: [] } : await transport.listComputerSessions(workspaceId);
|
|
1253
|
+
const reusable = newestRelevant(
|
|
1254
|
+
listed.sessions.filter((candidate) =>
|
|
1255
|
+
compatibleInteractionPlacement(candidate.placement, value.placement),
|
|
1256
|
+
),
|
|
1257
|
+
sourceSessionId,
|
|
1258
|
+
);
|
|
1210
1259
|
session = reusable
|
|
1211
1260
|
? reusable
|
|
1212
1261
|
: (
|
|
@@ -1227,6 +1276,16 @@ async function openComputer(
|
|
|
1227
1276
|
};
|
|
1228
1277
|
}
|
|
1229
1278
|
|
|
1279
|
+
function compatibleInteractionPlacement(
|
|
1280
|
+
candidate: z.infer<typeof InteractionPlacement>,
|
|
1281
|
+
requested: z.infer<typeof InteractionPlacement> | undefined,
|
|
1282
|
+
): boolean {
|
|
1283
|
+
if (!requested) return candidate.kind !== "attached_device";
|
|
1284
|
+
return Object.entries(requested).every(
|
|
1285
|
+
([key, value]) => (candidate as Record<string, unknown>)[key] === value,
|
|
1286
|
+
);
|
|
1287
|
+
}
|
|
1288
|
+
|
|
1230
1289
|
function newestRelevant<
|
|
1231
1290
|
T extends {
|
|
1232
1291
|
lifecycle: string;
|
|
@@ -83,6 +83,8 @@ Display images with . Use the exac
|
|
|
83
83
|
|
|
84
84
|
Publish files you deliberately deliver so they are retained and discoverable in Artifacts; do not publish every temporary file. Reuse retained references for unchanged outputs. Source-code navigation may still use workspace file links. Inline HTML stays in chat unless explicitly saved as a Site.
|
|
85
85
|
|
|
86
|
+
For published files, [Open file](artifact:<artifactId>) opens the retained file in Artifacts.  displays images, video, audio, or PDFs inline in the OpenGeni console, with an Artifact link for other formats. Replace <artifactId> with the exact artifact.artifactId from the publication receipt and use a descriptive label. Sites and native documents keep their tool-returned canonical links. Never substitute a storage URL or a sandbox path for a published artifact reference.
|
|
87
|
+
|
|
86
88
|
# Rules for getting work done
|
|
87
89
|
|
|
88
90
|
- When you search for text or files, you reach first for \`rg\` or \`rg --files\`; they are much faster than alternatives like \`grep\`. If \`rg\` is unavailable, you use the next best tool without fuss.
|
|
@@ -120,6 +122,8 @@ When presented with clarifying questions or objections from the user, lead with
|
|
|
120
122
|
|
|
121
123
|
If completion requires new authority, external coordination, or a meaningful expansion beyond the user’s implied intent and task scope (e.g. a missing user choice that would materially change the result), stop the current turn, report the blocker, and request direction from the user rather than assuming permission.
|
|
122
124
|
|
|
125
|
+
Decide the design before building it. First check whether OpenGeni already provides the capability natively (for example, a Site reaches the model and workspace tools through the host bridge and needs no server of its own). If a native path fits, use it. Follow the project's established architecture and choices the user has already authorized or delegated; the user need not name each host, provider, or credential. Ask before making a new external commitment or materially departing from the established architecture beyond the authorized scope: name the candidate designs, and do not start parallel work that assumes the unresolved choice. The absence of a native path alone does not require a question; continue with routine implementation choices within scope.
|
|
126
|
+
|
|
123
127
|
# Destructive Actions
|
|
124
128
|
|
|
125
129
|
Be cautious with commands or API calls that can delete, overwrite, or otherwise make data difficult to recover.
|
|
@@ -152,7 +156,7 @@ Skills are reusable instructions supplied dynamically for the current session. W
|
|
|
152
156
|
|
|
153
157
|
# Integration setup
|
|
154
158
|
|
|
155
|
-
Use available integration tools directly. If the task needs an integration you cannot access, discover it with \`capability_catalog_search\` before declaring a setup blocker. For a suitable match with \`setup.nextAction\`, call \`capability_authorization_request\` with the returned capability ID and a brief task-specific rationale to show its Connect card in chat. Requesting the card does not need integration-management permission; the authenticated human must authorize setup. After setup, rediscover the tools and continue the task, verifying the access it needs. If access remains blocked, explain the specific blocker from the returned facts. If either setup tool is unavailable, report the missing setup path.
|
|
159
|
+
Use available integration tools directly. If the task needs an integration you cannot access, discover it with \`capability_catalog_search\` before declaring a setup blocker. For a suitable match with \`setup.nextAction\`, call \`capability_authorization_request\` with the returned capability ID and a brief task-specific rationale to show its Connect card in chat. Requesting the card does not need integration-management permission; the authenticated human must authorize setup. A card is for an integration required by the authorized design, including established or delegated choices. Resolve out-of-scope architecture choices before requesting setup. After setup, rediscover the tools and continue the task, verifying the access it needs. If access remains blocked, explain the specific blocker from the returned facts. If either setup tool is unavailable, report the missing setup path.
|
|
156
160
|
|
|
157
161
|
# Session coordination
|
|
158
162
|
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { createHash, randomUUID } from "node:crypto";
|
|
2
2
|
import { posix as posixPath } from "node:path";
|
|
3
|
+
import { parseBrowserFrameMetadata, type BrowserFrameMetadata } from "@opengeni/sdk";
|
|
3
4
|
import {
|
|
4
5
|
BROWSER_CONTROL_MAX_JSON_BYTES,
|
|
5
6
|
BROWSER_CONTROL_PORT,
|
|
@@ -165,6 +166,29 @@ export type ComputerControlFrame = {
|
|
|
165
166
|
metadata: ComputerControlFrameMetadata;
|
|
166
167
|
};
|
|
167
168
|
|
|
169
|
+
export type BrowserControlFrame = {
|
|
170
|
+
data: Uint8Array;
|
|
171
|
+
mediaType: "image/jpeg" | "image/png";
|
|
172
|
+
metadataHeader: string;
|
|
173
|
+
metadata: BrowserFrameMetadata;
|
|
174
|
+
};
|
|
175
|
+
|
|
176
|
+
type ExpectedBrowserFrameEvidence = {
|
|
177
|
+
browserSessionId: string;
|
|
178
|
+
controllerGeneration: string;
|
|
179
|
+
targetId: string;
|
|
180
|
+
};
|
|
181
|
+
|
|
182
|
+
type FrameRequest = {
|
|
183
|
+
method: "GET";
|
|
184
|
+
path: string;
|
|
185
|
+
token: string;
|
|
186
|
+
timeoutMs?: number;
|
|
187
|
+
} & (
|
|
188
|
+
| { surface: "browser"; expectedFrameEvidence: ExpectedBrowserFrameEvidence }
|
|
189
|
+
| { surface: "computer"; expectedFrameEvidence: ExpectedComputerFrameEvidence }
|
|
190
|
+
);
|
|
191
|
+
|
|
168
192
|
export type ComputerControlFrameMetadata = {
|
|
169
193
|
frameId: string;
|
|
170
194
|
computerSessionId: string;
|
|
@@ -947,7 +971,17 @@ export class BrowserControlClient {
|
|
|
947
971
|
expectedFrameEvidence: ExpectedComputerFrameEvidence;
|
|
948
972
|
timeoutMs?: number;
|
|
949
973
|
}): Promise<ComputerControlFrame> {
|
|
950
|
-
return await this.requestBytes(input);
|
|
974
|
+
return (await this.requestBytes({ ...input, surface: "computer" })) as ComputerControlFrame;
|
|
975
|
+
}
|
|
976
|
+
|
|
977
|
+
async requestBrowserBytesForSession(input: {
|
|
978
|
+
method: "GET";
|
|
979
|
+
path: string;
|
|
980
|
+
token: string;
|
|
981
|
+
expectedFrameEvidence: ExpectedBrowserFrameEvidence;
|
|
982
|
+
timeoutMs?: number;
|
|
983
|
+
}): Promise<BrowserControlFrame> {
|
|
984
|
+
return (await this.requestBytes({ ...input, surface: "browser" })) as BrowserControlFrame;
|
|
951
985
|
}
|
|
952
986
|
|
|
953
987
|
private async requestJson(
|
|
@@ -1124,15 +1158,9 @@ export class BrowserControlClient {
|
|
|
1124
1158
|
}
|
|
1125
1159
|
|
|
1126
1160
|
private async requestBytes(
|
|
1127
|
-
input:
|
|
1128
|
-
method: "GET";
|
|
1129
|
-
path: string;
|
|
1130
|
-
token: string;
|
|
1131
|
-
expectedFrameEvidence: ExpectedComputerFrameEvidence;
|
|
1132
|
-
timeoutMs?: number;
|
|
1133
|
-
},
|
|
1161
|
+
input: FrameRequest,
|
|
1134
1162
|
retryNativeEndpoint = true,
|
|
1135
|
-
): Promise<ComputerControlFrame> {
|
|
1163
|
+
): Promise<ComputerControlFrame | BrowserControlFrame> {
|
|
1136
1164
|
const requireHostFetch = this.session.requireHostFetchController === true;
|
|
1137
1165
|
if (this.session.resolveExposedPort && !this.session.ensureBrowserControl) {
|
|
1138
1166
|
try {
|
|
@@ -1247,10 +1275,10 @@ export class BrowserControlClient {
|
|
|
1247
1275
|
const data = await readBytes(
|
|
1248
1276
|
this.session,
|
|
1249
1277
|
responsePath,
|
|
1250
|
-
COMPUTER_SCREENSHOT_MAX_BYTES,
|
|
1278
|
+
input.surface === "browser" ? 24 * 1024 * 1024 : COMPUTER_SCREENSHOT_MAX_BYTES,
|
|
1251
1279
|
timeoutMs,
|
|
1252
1280
|
);
|
|
1253
|
-
return
|
|
1281
|
+
return controllerImageFrame(data, headers, input);
|
|
1254
1282
|
} catch (error) {
|
|
1255
1283
|
if (
|
|
1256
1284
|
error instanceof BrowserControlRequestError ||
|
|
@@ -1432,6 +1460,19 @@ export class BrowserControlSessionClient {
|
|
|
1432
1460
|
);
|
|
1433
1461
|
}
|
|
1434
1462
|
|
|
1463
|
+
async capture(targetId: string): Promise<BrowserControlFrame> {
|
|
1464
|
+
return await this.parent.requestBrowserBytesForSession({
|
|
1465
|
+
method: "GET",
|
|
1466
|
+
path: this.targetPath(targetId, "screenshot?format=jpeg&quality=55"),
|
|
1467
|
+
token: this.viewToken,
|
|
1468
|
+
expectedFrameEvidence: {
|
|
1469
|
+
browserSessionId: this.reference.browserSessionId,
|
|
1470
|
+
controllerGeneration: this.reference.controllerGeneration,
|
|
1471
|
+
targetId: requireOpaqueId(targetId, "browser target id"),
|
|
1472
|
+
},
|
|
1473
|
+
});
|
|
1474
|
+
}
|
|
1475
|
+
|
|
1435
1476
|
async readClipboard(): Promise<BrowserClipboardValue> {
|
|
1436
1477
|
return BrowserClipboard.parse(
|
|
1437
1478
|
await this.parent.requestForSession({
|
|
@@ -1840,15 +1881,9 @@ async function requestExposedController(
|
|
|
1840
1881
|
|
|
1841
1882
|
async function requestExposedControllerBytes(
|
|
1842
1883
|
endpoint: ExposedPortEndpoint,
|
|
1843
|
-
input:
|
|
1844
|
-
method: "GET";
|
|
1845
|
-
path: string;
|
|
1846
|
-
token: string;
|
|
1847
|
-
expectedFrameEvidence: ExpectedComputerFrameEvidence;
|
|
1848
|
-
timeoutMs?: number;
|
|
1849
|
-
},
|
|
1884
|
+
input: FrameRequest,
|
|
1850
1885
|
defaultTimeoutMs: number,
|
|
1851
|
-
): Promise<ComputerControlFrame> {
|
|
1886
|
+
): Promise<ComputerControlFrame | BrowserControlFrame> {
|
|
1852
1887
|
const token = requireToken(input.token, "browser controller token");
|
|
1853
1888
|
const timeoutMs = boundedTimeout(input.timeoutMs ?? defaultTimeoutMs);
|
|
1854
1889
|
const streamUrl = exposedControllerUrl(endpoint, input.path);
|
|
@@ -1872,12 +1907,63 @@ async function requestExposedControllerBytes(
|
|
|
1872
1907
|
throw new BrowserControlProtocolError("browser controller image response is invalid");
|
|
1873
1908
|
}
|
|
1874
1909
|
const declaredLength = Number(response.headers.get("content-length"));
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
|
|
1910
|
+
const maxBytes = input.surface === "browser" ? 24 * 1024 * 1024 : COMPUTER_SCREENSHOT_MAX_BYTES;
|
|
1911
|
+
if (Number.isFinite(declaredLength) && declaredLength > maxBytes) {
|
|
1912
|
+
await response.body?.cancel("frame exceeds its byte bound").catch(() => undefined);
|
|
1913
|
+
throw new RangeError("frame exceeds its byte bound");
|
|
1878
1914
|
}
|
|
1879
1915
|
const data = new Uint8Array(await response.arrayBuffer());
|
|
1880
|
-
return
|
|
1916
|
+
return controllerImageFrame(data, response.headers, input);
|
|
1917
|
+
}
|
|
1918
|
+
|
|
1919
|
+
function controllerImageFrame(
|
|
1920
|
+
data: Uint8Array,
|
|
1921
|
+
headers: { get(name: string): string | null | undefined },
|
|
1922
|
+
input: FrameRequest,
|
|
1923
|
+
): ComputerControlFrame | BrowserControlFrame {
|
|
1924
|
+
return input.surface === "browser"
|
|
1925
|
+
? browserControlFrame(data, headers, input.expectedFrameEvidence)
|
|
1926
|
+
: computerControlFrame(data, headers, input.expectedFrameEvidence);
|
|
1927
|
+
}
|
|
1928
|
+
|
|
1929
|
+
function browserControlFrame(
|
|
1930
|
+
data: Uint8Array,
|
|
1931
|
+
headers: { get(name: string): string | null | undefined },
|
|
1932
|
+
expected: ExpectedBrowserFrameEvidence,
|
|
1933
|
+
): BrowserControlFrame {
|
|
1934
|
+
if (data.byteLength < 1 || data.byteLength > 24 * 1024 * 1024) {
|
|
1935
|
+
throw new RangeError("browser frame exceeds its byte bound");
|
|
1936
|
+
}
|
|
1937
|
+
const get = (name: string): string | null => headers.get(name.toLowerCase()) ?? null;
|
|
1938
|
+
const mediaType = get("content-type")?.split(";", 1)[0]?.trim().toLowerCase();
|
|
1939
|
+
if (mediaType !== "image/jpeg" && mediaType !== "image/png") {
|
|
1940
|
+
throw new BrowserControlProtocolError("browser frame media type is invalid");
|
|
1941
|
+
}
|
|
1942
|
+
const metadataHeader = get("x-opengeni-browser-frame");
|
|
1943
|
+
if (
|
|
1944
|
+
!metadataHeader ||
|
|
1945
|
+
metadataHeader.length > 64 * 1024 ||
|
|
1946
|
+
!/^[A-Za-z0-9_-]+$/u.test(metadataHeader)
|
|
1947
|
+
) {
|
|
1948
|
+
throw new BrowserControlProtocolError("browser frame metadata is invalid");
|
|
1949
|
+
}
|
|
1950
|
+
let metadata: BrowserFrameMetadata;
|
|
1951
|
+
try {
|
|
1952
|
+
metadata = parseBrowserFrameMetadata(
|
|
1953
|
+
JSON.parse(Buffer.from(metadataHeader, "base64url").toString("utf8")),
|
|
1954
|
+
);
|
|
1955
|
+
} catch {
|
|
1956
|
+
throw new BrowserControlProtocolError("browser frame metadata is invalid");
|
|
1957
|
+
}
|
|
1958
|
+
if (
|
|
1959
|
+
metadata.browserSessionId !== expected.browserSessionId ||
|
|
1960
|
+
metadata.controllerGeneration !== expected.controllerGeneration ||
|
|
1961
|
+
metadata.targetId !== expected.targetId ||
|
|
1962
|
+
metadata.mediaType !== mediaType
|
|
1963
|
+
) {
|
|
1964
|
+
throw new BrowserControlProtocolError("browser frame evidence does not match its request");
|
|
1965
|
+
}
|
|
1966
|
+
return { data, mediaType, metadataHeader, metadata };
|
|
1881
1967
|
}
|
|
1882
1968
|
|
|
1883
1969
|
function computerControlFrame(
|
|
@@ -295,6 +295,7 @@ export interface RoutingSandboxSessionDeps {
|
|
|
295
295
|
adoptProcessAsBackgroundCommand?: (input: {
|
|
296
296
|
backend: ResolvedActiveBackend;
|
|
297
297
|
process: RoutingRetainedProcess;
|
|
298
|
+
command?: string | undefined;
|
|
298
299
|
}) => Promise<void>;
|
|
299
300
|
/** Called only when an operation against the default/home backend throws a
|
|
300
301
|
* non-fence error. Wiring may classify definitive provider disappearance and
|
|
@@ -2125,7 +2126,10 @@ export class RoutingSandboxSession implements RoutableBackendSession {
|
|
|
2125
2126
|
/** Make a retained process session-owned before exposing its live locator.
|
|
2126
2127
|
* Provider yield alone is not adoption: short commands can still finish and
|
|
2127
2128
|
* return inline during the model-facing eager-wait window. */
|
|
2128
|
-
async adoptRetainedProcessAsBackgroundCommand(
|
|
2129
|
+
async adoptRetainedProcessAsBackgroundCommand(
|
|
2130
|
+
providerSessionId: number,
|
|
2131
|
+
command?: string,
|
|
2132
|
+
): Promise<void> {
|
|
2129
2133
|
const record = this.retainedProcess(providerSessionId);
|
|
2130
2134
|
if (!this.canAdoptRetainedProcessAsBackgroundCommand(providerSessionId)) {
|
|
2131
2135
|
throw new RoutingUnsupportedError("backgroundCommandAdoption", record.backend.kind);
|
|
@@ -2142,6 +2146,7 @@ export class RoutingSandboxSession implements RoutableBackendSession {
|
|
|
2142
2146
|
record.backgroundAdoption ??= this.deps.adoptProcessAsBackgroundCommand({
|
|
2143
2147
|
backend: record.backend,
|
|
2144
2148
|
process: record.process,
|
|
2149
|
+
command,
|
|
2145
2150
|
});
|
|
2146
2151
|
try {
|
|
2147
2152
|
await record.backgroundAdoption;
|
|
@@ -67,6 +67,8 @@ type ShellProcessIdentity = {
|
|
|
67
67
|
|
|
68
68
|
type ActiveShellSession = {
|
|
69
69
|
sessionId: number;
|
|
70
|
+
/** Original model command, before adding process-control shell wrappers. */
|
|
71
|
+
command?: string;
|
|
70
72
|
markerPath: string | null;
|
|
71
73
|
token: string | null;
|
|
72
74
|
interactive: boolean;
|
|
@@ -97,7 +99,10 @@ type CommandCancellationSession = {
|
|
|
97
99
|
/** Whether the provider locator remains controllable from another worker
|
|
98
100
|
* process after this turn returns. */
|
|
99
101
|
canAdoptRetainedProcessAsBackgroundCommand?(providerSessionId: number): boolean;
|
|
100
|
-
adoptRetainedProcessAsBackgroundCommand?(
|
|
102
|
+
adoptRetainedProcessAsBackgroundCommand?(
|
|
103
|
+
providerSessionId: number,
|
|
104
|
+
command?: string,
|
|
105
|
+
): Promise<void>;
|
|
101
106
|
retainedProcessIdentity?(providerSessionId: number): { id: string } | null;
|
|
102
107
|
cancelSupervisedCommand?(providerSessionId: number, reason: "explicit_stop"): Promise<boolean>;
|
|
103
108
|
writeStdinForProcessRead?(args: {
|
|
@@ -1203,6 +1208,7 @@ class TurnToolCancellationControllerImpl implements TurnToolCancellationControll
|
|
|
1203
1208
|
// invoke a helper through the mutable current route.
|
|
1204
1209
|
this.shellSessions.set(retainedProcess.providerSessionId, {
|
|
1205
1210
|
sessionId: retainedProcess.providerSessionId,
|
|
1211
|
+
command: parsed.cmd,
|
|
1206
1212
|
markerPath,
|
|
1207
1213
|
token,
|
|
1208
1214
|
interactive,
|
|
@@ -1232,6 +1238,7 @@ class TurnToolCancellationControllerImpl implements TurnToolCancellationControll
|
|
|
1232
1238
|
if (sessionId !== null) {
|
|
1233
1239
|
const state: ActiveShellSession = {
|
|
1234
1240
|
sessionId,
|
|
1241
|
+
command: parsed.cmd,
|
|
1235
1242
|
markerPath: useRemoteOpCancellation ? null : markerPath,
|
|
1236
1243
|
token: useRemoteOpCancellation ? null : token,
|
|
1237
1244
|
interactive,
|
|
@@ -1481,7 +1488,10 @@ class TurnToolCancellationControllerImpl implements TurnToolCancellationControll
|
|
|
1481
1488
|
return `${runningCommandBanner(state.sessionId, output)}\nThis process is turn-scoped; it will stop when this turn ends or is interrupted.\n`;
|
|
1482
1489
|
}
|
|
1483
1490
|
if (state.processSession?.adoptRetainedProcessAsBackgroundCommand) {
|
|
1484
|
-
await state.processSession.adoptRetainedProcessAsBackgroundCommand(
|
|
1491
|
+
await state.processSession.adoptRetainedProcessAsBackgroundCommand(
|
|
1492
|
+
state.sessionId,
|
|
1493
|
+
state.command,
|
|
1494
|
+
);
|
|
1485
1495
|
this.shellSessions.delete(state.sessionId);
|
|
1486
1496
|
}
|
|
1487
1497
|
return runningCommandBanner(
|