@geminixiang/mikan 1.4.0 → 1.5.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/dist/adapters/commands/desktop.d.ts +5 -0
- package/dist/adapters/commands/desktop.d.ts.map +1 -0
- package/dist/adapters/commands/desktop.js +58 -0
- package/dist/adapters/commands/desktop.js.map +1 -0
- package/dist/adapters/commands/manifest.d.ts.map +1 -1
- package/dist/adapters/commands/manifest.js +8 -0
- package/dist/adapters/commands/manifest.js.map +1 -1
- package/dist/adapters/commands/registry.d.ts.map +1 -1
- package/dist/adapters/commands/registry.js +2 -0
- package/dist/adapters/commands/registry.js.map +1 -1
- package/dist/adapters/commands/types.d.ts +3 -1
- package/dist/adapters/commands/types.d.ts.map +1 -1
- package/dist/adapters/commands/types.js.map +1 -1
- package/dist/adapters/slack/app-config.d.ts +27 -0
- package/dist/adapters/slack/app-config.d.ts.map +1 -0
- package/dist/adapters/slack/app-config.js +241 -0
- package/dist/adapters/slack/app-config.js.map +1 -0
- package/dist/adapters/slack/types.d.ts +46 -1
- package/dist/adapters/slack/types.d.ts.map +1 -1
- package/dist/adapters/slack/types.js.map +1 -1
- package/dist/adapters/web/desktop/index.d.ts +16 -0
- package/dist/adapters/web/desktop/index.d.ts.map +1 -0
- package/dist/adapters/web/desktop/index.js +163 -0
- package/dist/adapters/web/desktop/index.js.map +1 -0
- package/dist/adapters/web/desktop/page.d.ts +2 -0
- package/dist/adapters/web/desktop/page.d.ts.map +1 -0
- package/dist/adapters/web/desktop/page.js +83 -0
- package/dist/adapters/web/desktop/page.js.map +1 -0
- package/dist/adapters/web/desktop/types.d.ts +8 -0
- package/dist/adapters/web/desktop/types.d.ts.map +1 -0
- package/dist/adapters/web/desktop/types.js +2 -0
- package/dist/adapters/web/desktop/types.js.map +1 -0
- package/dist/adapters/web/server.d.ts +2 -0
- package/dist/adapters/web/server.d.ts.map +1 -1
- package/dist/adapters/web/server.js +19 -0
- package/dist/adapters/web/server.js.map +1 -1
- package/dist/cli/boot.d.ts.map +1 -1
- package/dist/cli/boot.js +4 -1
- package/dist/cli/boot.js.map +1 -1
- package/dist/cli/onboard.d.ts +3 -0
- package/dist/cli/onboard.d.ts.map +1 -1
- package/dist/cli/onboard.js +79 -3
- package/dist/cli/onboard.js.map +1 -1
- package/dist/cli/slack.d.ts +9 -0
- package/dist/cli/slack.d.ts.map +1 -0
- package/dist/cli/slack.js +96 -0
- package/dist/cli/slack.js.map +1 -0
- package/dist/cli/types.d.ts +2 -1
- package/dist/cli/types.d.ts.map +1 -1
- package/dist/cli/types.js.map +1 -1
- package/dist/harness/desktop.d.ts +16 -0
- package/dist/harness/desktop.d.ts.map +1 -0
- package/dist/harness/desktop.js +6 -0
- package/dist/harness/desktop.js.map +1 -0
- package/dist/harness/execution-resolver.d.ts +1 -0
- package/dist/harness/execution-resolver.d.ts.map +1 -1
- package/dist/harness/execution-resolver.js +5 -0
- package/dist/harness/execution-resolver.js.map +1 -1
- package/dist/harness/presenter.d.ts.map +1 -1
- package/dist/harness/presenter.js +6 -1
- package/dist/harness/presenter.js.map +1 -1
- package/dist/harness/runner.d.ts.map +1 -1
- package/dist/harness/runner.js +22 -1
- package/dist/harness/runner.js.map +1 -1
- package/dist/harness/session.d.ts +1 -1
- package/dist/harness/session.d.ts.map +1 -1
- package/dist/harness/session.js +30 -9
- package/dist/harness/session.js.map +1 -1
- package/dist/harness/tools/codemode.d.ts.map +1 -1
- package/dist/harness/tools/codemode.js +2 -2
- package/dist/harness/tools/codemode.js.map +1 -1
- package/dist/harness/tools/index.d.ts +2 -2
- package/dist/harness/tools/index.d.ts.map +1 -1
- package/dist/harness/tools/index.js +10 -2
- package/dist/harness/tools/index.js.map +1 -1
- package/dist/harness/tools/jev-browser.d.ts +12 -1
- package/dist/harness/tools/jev-browser.d.ts.map +1 -1
- package/dist/harness/tools/jev-browser.js +189 -30
- package/dist/harness/tools/jev-browser.js.map +1 -1
- package/dist/harness/tools/pi-tools.d.ts.map +1 -1
- package/dist/harness/tools/pi-tools.js +2 -1
- package/dist/harness/tools/pi-tools.js.map +1 -1
- package/dist/harness/tools/types.d.ts +4 -0
- package/dist/harness/tools/types.d.ts.map +1 -1
- package/dist/harness/tools/types.js.map +1 -1
- package/dist/harness/types.d.ts +2 -1
- package/dist/harness/types.d.ts.map +1 -1
- package/dist/harness/types.js.map +1 -1
- package/dist/main.js +29 -0
- package/dist/main.js.map +1 -1
- package/dist/runtime/conversation-runtime.js +7 -0
- package/dist/runtime/conversation-runtime.js.map +1 -1
- package/dist/sandbox/desktop.d.ts +30 -0
- package/dist/sandbox/desktop.d.ts.map +1 -0
- package/dist/sandbox/desktop.js +112 -0
- package/dist/sandbox/desktop.js.map +1 -0
- package/dist/sandbox/provisioner.d.ts +4 -1
- package/dist/sandbox/provisioner.d.ts.map +1 -1
- package/dist/sandbox/provisioner.js +21 -0
- package/dist/sandbox/provisioner.js.map +1 -1
- package/dist/sandbox/types.d.ts +19 -0
- package/dist/sandbox/types.d.ts.map +1 -1
- package/dist/sandbox/types.js.map +1 -1
- package/dist/types.d.ts +17 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/package.json +9 -8
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"jev-browser.d.ts","sourceRoot":"","sources":["../../../src/harness/tools/jev-browser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,+BAA+B,CAAC;AAC/D,OAAO,EAAE,IAAI,EAAe,MAAM,SAAS,CAAC;AAG5C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC;AACnE,OAAO,EAAkC,KAAK,YAAY,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"jev-browser.d.ts","sourceRoot":"","sources":["../../../src/harness/tools/jev-browser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,+BAA+B,CAAC;AAC/D,OAAO,EAAE,IAAI,EAAe,MAAM,SAAS,CAAC;AAG5C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC;AACnE,OAAO,EAAkC,KAAK,YAAY,EAAE,MAAM,WAAW,CAAC;AAM9E,eAAO,MAAM,gBAAgB,gBAAgB,CAAC;AAe9C,QAAA,MAAM,gBAAgB;;;;;;;;;EA6CpB,CAAC;AAIH,UAAU,OAAO;IACf,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd;AAcD,UAAU,gBAAgB;IACxB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAMD,iBAAS,kBAAkB,CAAC,YAAY,EAAE,OAAO,EAAE,KAAK,EAAE,gBAAgB,GAAG,SAAS,GAAG,MAAM,CAY9F;AA2HD,iBAAS,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAEnD;AAuDD,iBAAS,cAAc,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG;IACtD,KAAK,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IAC3B,IAAI,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IAC1B,MAAM,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;CAC7B,CAOA;AAmBD,UAAU,YAAY;IACpB,SAAS,EAAE,YAAY,CAAC;IACxB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACjC;AAED,iBAAS,iBAAiB,CACxB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC7B,WAAW,EAAE,OAAO,EACpB,aAAa,EAAE,OAAO,EACtB,QAAQ,SAAK,GACZ,YAAY,CAiDd;AAED,MAAM,MAAM,kBAAkB,GAAG,CAC/B,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,WAAW,KACjB,OAAO,CAAC,MAAM,CAAC,CAAC;AAkCrB,iBAAS,aAAa,CACpB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE;IAAE,MAAM,CAAC,EAAE,OAAO,CAAA;CAAE,GAAG,SAAS,CAAC,EACzD,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,MAAM,GAAG,SAAS,CAUpB;AAcD,UAAU,cAAc;IACtB,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACvC;AAED,MAAM,MAAM,cAAc,GAAG,SAAS,CAAC,OAAO,gBAAgB,CAAC,GAAG;IAChE,UAAU,CAAC,OAAO,EAAE,cAAc,GAAG,SAAS,GAAG,IAAI,CAAC;CACvD,CAAC;AAEF,UAAU,iBAAiB;IACzB,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AA0cD,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,MAAM,YAAY,GAAG,SAAS,EACtC,YAAY,CAAC,EAAE,kBAAkB,EACjC,OAAO,GAAE,iBAAsB,GAC9B,cAAc,CAqBhB;AAED,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,cAAc,EAAE,QAAQ,EAAE,kBAAkB,EAAE,CAAC"}
|
|
@@ -4,6 +4,7 @@ import { BACKGROUND_CONTEXT, withAbortSignal } from "@earendil-works/chord/conte
|
|
|
4
4
|
import { evaluateWithJev } from "../jev.js";
|
|
5
5
|
import { LABEL_PARAMETER } from "./host-fn-tool.js";
|
|
6
6
|
import { isRecord } from "../../unknown-values.js";
|
|
7
|
+
import { DESKTOP_COMMAND, DESKTOP_RUNTIME_DIR, parseDesktopStatus } from "../../sandbox/desktop.js";
|
|
7
8
|
export const JEV_BROWSER_TOOL = "jev_browser";
|
|
8
9
|
const AGENT_BROWSER_BIN = "agent-browser";
|
|
9
10
|
const AGENT_BROWSER_UNAVAILABLE = "agent-browser CLI is unavailable in the current sandbox. Ask the operator to provision " +
|
|
@@ -38,6 +39,9 @@ const jevBrowserSchema = Type.Object({
|
|
|
38
39
|
minimum: 1,
|
|
39
40
|
maximum: HARD_MAX_STEPS,
|
|
40
41
|
})),
|
|
42
|
+
handOff: Type.Optional(Type.String({
|
|
43
|
+
description: "Give the browser to the person on the office desktop for a step only they can do, such as signing in or solving a challenge: what they should do, in their language. Use alone. It opens the desktop, moves the current page onto it, posts the step with a desktop link, and ends your turn. Continue after they say they are done.",
|
|
44
|
+
})),
|
|
41
45
|
});
|
|
42
46
|
function lifecycleOf(result) {
|
|
43
47
|
return result.data?.lifecycle;
|
|
@@ -57,21 +61,64 @@ function describeContinuity(expectedOpen, first) {
|
|
|
57
61
|
function tabNumber(tab) {
|
|
58
62
|
return Number(tab.tabId.replace(/^t/, ""));
|
|
59
63
|
}
|
|
60
|
-
async function closeOldestTabs(
|
|
61
|
-
const listed = await runAgentBrowser(
|
|
64
|
+
async function closeOldestTabs(browser, agentTargets, keep, signal) {
|
|
65
|
+
const listed = await runAgentBrowser(browser, ["tab", "list"], signal);
|
|
62
66
|
const tabs = listed.success && Array.isArray(listed.data?.tabs) ? listed.data.tabs : [];
|
|
63
|
-
const
|
|
67
|
+
const opened = agentTargets
|
|
68
|
+
? tabs.filter((tab) => tab.targetId !== undefined && agentTargets.has(tab.targetId))
|
|
69
|
+
: tabs;
|
|
70
|
+
const oldest = opened
|
|
64
71
|
.filter((tab) => !tab.active)
|
|
65
72
|
.toSorted((a, b) => tabNumber(a) - tabNumber(b))
|
|
66
|
-
.slice(0, Math.max(0,
|
|
73
|
+
.slice(0, Math.max(0, opened.length - keep));
|
|
67
74
|
const closed = [];
|
|
68
75
|
for (const tab of oldest) {
|
|
69
|
-
const result = await runAgentBrowser(
|
|
70
|
-
if (result.success)
|
|
71
|
-
|
|
76
|
+
const result = await runAgentBrowser(browser, ["tab", "close", tab.tabId], signal);
|
|
77
|
+
if (!result.success)
|
|
78
|
+
continue;
|
|
79
|
+
if (tab.targetId)
|
|
80
|
+
agentTargets?.delete(tab.targetId);
|
|
81
|
+
closed.push(tab.url ?? tab.tabId);
|
|
72
82
|
}
|
|
73
83
|
return closed;
|
|
74
84
|
}
|
|
85
|
+
function targetIdOf(result) {
|
|
86
|
+
const data = result.data;
|
|
87
|
+
return isRecord(data) && typeof data.targetId === "string" ? data.targetId : undefined;
|
|
88
|
+
}
|
|
89
|
+
async function readDesktopStatus(env, signal) {
|
|
90
|
+
let stdout = "";
|
|
91
|
+
const result = await env.exec([DESKTOP_COMMAND, "status"], {
|
|
92
|
+
timeout: COMMAND_TIMEOUT_SECONDS,
|
|
93
|
+
onOutput: (text, _context, info) => {
|
|
94
|
+
if (info.stream === "stdout")
|
|
95
|
+
stdout += text;
|
|
96
|
+
},
|
|
97
|
+
}, signal ? withAbortSignal(signal, BACKGROUND_CONTEXT) : BACKGROUND_CONTEXT);
|
|
98
|
+
if (!result.ok || result.value.exitCode !== 0)
|
|
99
|
+
return { display: undefined, control: "agent" };
|
|
100
|
+
return parseDesktopStatus(stdout);
|
|
101
|
+
}
|
|
102
|
+
function desktopLaunchEnv(display) {
|
|
103
|
+
return {
|
|
104
|
+
AGENT_BROWSER_HEADED: "1",
|
|
105
|
+
AGENT_BROWSER_ARGS: "--ozone-platform=wayland",
|
|
106
|
+
WAYLAND_DISPLAY: `${DESKTOP_RUNTIME_DIR}/${display}`,
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
const PERSON_HAS_CONTROL = "A person is using this browser on the office desktop. Do not use the browser until they hand it back: they press Hand back to agent on the desktop page or close it. Tell them what you need and end your turn.";
|
|
110
|
+
const NO_DESKTOP_STATUS = { display: undefined, control: "agent" };
|
|
111
|
+
function personHasControlResult() {
|
|
112
|
+
return {
|
|
113
|
+
content: [
|
|
114
|
+
{
|
|
115
|
+
type: "text",
|
|
116
|
+
text: JSON.stringify({ status: "person-has-control", message: PERSON_HAS_CONTROL }),
|
|
117
|
+
},
|
|
118
|
+
],
|
|
119
|
+
details: undefined,
|
|
120
|
+
};
|
|
121
|
+
}
|
|
75
122
|
const CLICK_TARGET_KEY = "click_target";
|
|
76
123
|
const TYPE_TARGET_KEY = "type_target";
|
|
77
124
|
const SELECT_TARGET_KEY = "select_target";
|
|
@@ -98,10 +145,11 @@ const TEXT_VALUE_INSTRUCTIONS = 'Return a JSON object with exactly one key, "tex
|
|
|
98
145
|
function truncate(text, max) {
|
|
99
146
|
return text.length > max ? `${text.slice(0, max)}…` : text;
|
|
100
147
|
}
|
|
101
|
-
async function runAgentBrowser(
|
|
148
|
+
async function runAgentBrowser(browser, args, signal) {
|
|
102
149
|
const output = { stdout: "", stderr: "" };
|
|
103
|
-
const result = await env.exec([AGENT_BROWSER_BIN, "--session", sessionId, ...args, "--json"], {
|
|
150
|
+
const result = await browser.env.exec([AGENT_BROWSER_BIN, "--session", browser.sessionId, ...args, "--json"], {
|
|
104
151
|
timeout: COMMAND_TIMEOUT_SECONDS,
|
|
152
|
+
env: browser.launchEnv,
|
|
105
153
|
onOutput: (text, _context, info) => {
|
|
106
154
|
output[info.stream] += text;
|
|
107
155
|
},
|
|
@@ -119,7 +167,12 @@ async function runAgentBrowser(env, sessionId, args, signal) {
|
|
|
119
167
|
}
|
|
120
168
|
if (stdout.trim()) {
|
|
121
169
|
try {
|
|
122
|
-
|
|
170
|
+
const parsed = JSON.parse(stdout);
|
|
171
|
+
const lifecycle = lifecycleOf(parsed);
|
|
172
|
+
if (lifecycle?.launched === true || lifecycle?.relaunchedBrowser === true) {
|
|
173
|
+
browser.onLaunch?.(browser.launchEnv !== undefined);
|
|
174
|
+
}
|
|
175
|
+
return parsed;
|
|
123
176
|
}
|
|
124
177
|
catch (error) {
|
|
125
178
|
if (code === 0)
|
|
@@ -232,24 +285,106 @@ function resolveTarget(operation, answers, singles) {
|
|
|
232
285
|
const choice = answers[key]?.choice;
|
|
233
286
|
return typeof choice === "string" ? choice : undefined;
|
|
234
287
|
}
|
|
235
|
-
async function openPage(
|
|
288
|
+
async function openPage(browser, url, inNewTab, signal) {
|
|
236
289
|
if (!inNewTab)
|
|
237
|
-
return runAgentBrowser(
|
|
238
|
-
const tab = await runAgentBrowser(
|
|
290
|
+
return runAgentBrowser(browser, ["open", url], signal);
|
|
291
|
+
const tab = await runAgentBrowser(browser, ["tab", "new"], signal);
|
|
239
292
|
if (!tab.success)
|
|
240
293
|
return tab;
|
|
241
|
-
return runAgentBrowser(
|
|
294
|
+
return runAgentBrowser(browser, ["open", url], signal);
|
|
295
|
+
}
|
|
296
|
+
const NO_PAGE_TO_HAND_OFF = "The browser has no page to hand over. Open the page with url first; it now opens on the desktop. Then call handOff again.";
|
|
297
|
+
const HAND_OFF_UNAVAILABLE = "Handing the browser to a person needs the office desktop and a direct message to send its link privately; this conversation has neither, so ask the person in chat instead.";
|
|
298
|
+
async function setBrowserControl(env, owner, signal) {
|
|
299
|
+
const result = await env.exec([DESKTOP_COMMAND, "control", owner], { timeout: COMMAND_TIMEOUT_SECONDS }, signal ? withAbortSignal(signal, BACKGROUND_CONTEXT) : BACKGROUND_CONTEXT);
|
|
300
|
+
return result.ok && result.value.exitCode === 0;
|
|
301
|
+
}
|
|
302
|
+
function handedOffResult() {
|
|
303
|
+
return {
|
|
304
|
+
content: [
|
|
305
|
+
{
|
|
306
|
+
type: "text",
|
|
307
|
+
text: JSON.stringify({
|
|
308
|
+
status: "handed-off",
|
|
309
|
+
message: "The person controls the browser now and was asked to do this step.",
|
|
310
|
+
}),
|
|
311
|
+
},
|
|
312
|
+
],
|
|
313
|
+
details: { handedOff: true },
|
|
314
|
+
terminate: true,
|
|
315
|
+
};
|
|
316
|
+
}
|
|
317
|
+
function newSessionId() {
|
|
318
|
+
return `mikan-jb-${randomUUID()}`;
|
|
242
319
|
}
|
|
243
|
-
function createUnlockedJevBrowserTool(getEnv, generateText) {
|
|
244
|
-
|
|
320
|
+
function createUnlockedJevBrowserTool(getEnv, generateText, options) {
|
|
321
|
+
let sessionId = newSessionId();
|
|
322
|
+
const agentTargets = new Set();
|
|
323
|
+
let handOff;
|
|
245
324
|
let browserOpen = false;
|
|
325
|
+
let openedOnDesktop = false;
|
|
326
|
+
const desktopStatus = (env, signal) => options.desktop ? readDesktopStatus(env, signal) : Promise.resolve(NO_DESKTOP_STATUS);
|
|
327
|
+
const target = (env, launchEnv) => ({
|
|
328
|
+
env,
|
|
329
|
+
sessionId,
|
|
330
|
+
launchEnv,
|
|
331
|
+
onLaunch: (onDesktop) => {
|
|
332
|
+
openedOnDesktop = onDesktop;
|
|
333
|
+
},
|
|
334
|
+
});
|
|
335
|
+
async function moveBrowserToDesktop(env, display, signal) {
|
|
336
|
+
const headless = target(env);
|
|
337
|
+
let launchedNow = false;
|
|
338
|
+
headless.onLaunch = () => {
|
|
339
|
+
launchedNow = true;
|
|
340
|
+
};
|
|
341
|
+
const current = await runAgentBrowser(headless, ["get", "url"], signal);
|
|
342
|
+
const url = current.success && !launchedNow ? current.data?.url : undefined;
|
|
343
|
+
await runAgentBrowser(headless, ["close"], signal);
|
|
344
|
+
agentTargets.clear();
|
|
345
|
+
browserOpen = false;
|
|
346
|
+
openedOnDesktop = false;
|
|
347
|
+
sessionId = newSessionId();
|
|
348
|
+
if (!url)
|
|
349
|
+
throw new Error(NO_PAGE_TO_HAND_OFF);
|
|
350
|
+
const reopened = await runAgentBrowser(target(env, desktopLaunchEnv(display)), ["open", url], signal);
|
|
351
|
+
if (!reopened.success) {
|
|
352
|
+
throw new Error(`Could not reopen ${url} on the desktop: ${reopened.error}. ${NO_PAGE_TO_HAND_OFF}`);
|
|
353
|
+
}
|
|
354
|
+
const opened = targetIdOf(reopened);
|
|
355
|
+
if (opened)
|
|
356
|
+
agentTargets.add(opened);
|
|
357
|
+
browserOpen = true;
|
|
358
|
+
}
|
|
359
|
+
async function handBrowserToPerson(env, step, signal) {
|
|
360
|
+
if (!handOff || !(await handOff.openDesktop()))
|
|
361
|
+
throw new Error(HAND_OFF_UNAVAILABLE);
|
|
362
|
+
const desktop = await desktopStatus(env, signal);
|
|
363
|
+
if (!desktop.display)
|
|
364
|
+
throw new Error(HAND_OFF_UNAVAILABLE);
|
|
365
|
+
if (!openedOnDesktop)
|
|
366
|
+
await moveBrowserToDesktop(env, desktop.display, signal);
|
|
367
|
+
if (!(await setBrowserControl(env, "user", signal)))
|
|
368
|
+
throw new Error(HAND_OFF_UNAVAILABLE);
|
|
369
|
+
try {
|
|
370
|
+
await handOff.announce(step);
|
|
371
|
+
}
|
|
372
|
+
catch (error) {
|
|
373
|
+
await setBrowserControl(env, "agent").catch(() => false);
|
|
374
|
+
throw error;
|
|
375
|
+
}
|
|
376
|
+
return handedOffResult();
|
|
377
|
+
}
|
|
246
378
|
let closedByCall = false;
|
|
247
379
|
return {
|
|
380
|
+
setHandOff: (next) => {
|
|
381
|
+
handOff = next;
|
|
382
|
+
},
|
|
248
383
|
name: JEV_BROWSER_TOOL,
|
|
249
384
|
label: "jev browser",
|
|
250
385
|
description: [
|
|
251
386
|
"Control a real Chrome browser in this conversation's sandbox (not the host): give a goal that Jev drives step by step (click, type, select, scroll, wait), run raw agent-browser commands, or both; commands run first.",
|
|
252
|
-
`Each thread has one browser that stays open across calls; url opens a new tab, and past ${BROWSER_TAB_LIMIT} tabs the oldest closes.`,
|
|
387
|
+
`Each thread has one browser that stays open across calls; url opens a new tab, and past ${BROWSER_TAB_LIMIT} tabs it opened the oldest closes; tabs a person opened stay.`,
|
|
253
388
|
"Read the page from lastPageSnapshot (accessibility text; the tool does not summarize) and command output from commandResults. A browserContinuity of unknown is missing metadata, not a failure.",
|
|
254
389
|
'CLI success is not goal completion: verify the visible state. To submit a typed input, run ["press","Enter"]: press takes a key, not a ref. After navigation, snapshot again and use the new @refs. If content sits in an iframe, set frame instead of scrolling. find may act, not only read. When syntax is unclear, run ["<command>","--help"]; after one informed retry, report the blocker instead of trying more selectors.',
|
|
255
390
|
"If agent-browser is missing from the sandbox, report it; never install it on the host.",
|
|
@@ -271,10 +406,22 @@ function createUnlockedJevBrowserTool(getEnv, generateText) {
|
|
|
271
406
|
throw new Error('press takes a key, not a target ref: use ["press","Enter"] after focusing the input. Check ["press","--help"] for native syntax.');
|
|
272
407
|
}
|
|
273
408
|
}
|
|
409
|
+
if (args.handOff !== undefined) {
|
|
410
|
+
if (args.url || args.goal || args.commands?.length || args.close || args.frame) {
|
|
411
|
+
throw new Error("Use handOff alone, without url, goal, commands, frame, or close.");
|
|
412
|
+
}
|
|
413
|
+
return handBrowserToPerson(env, args.handOff, signal);
|
|
414
|
+
}
|
|
415
|
+
const desktop = await desktopStatus(env, signal);
|
|
416
|
+
if (desktop.control === "user")
|
|
417
|
+
return personHasControlResult();
|
|
418
|
+
const browser = target(env, desktop.display ? desktopLaunchEnv(desktop.display) : undefined);
|
|
419
|
+
const ownTabs = desktop.display ? agentTargets : undefined;
|
|
274
420
|
if (args.close === true && !args.url && !args.goal && !args.commands?.length) {
|
|
275
|
-
const result = await runAgentBrowser(
|
|
421
|
+
const result = await runAgentBrowser(browser, ["close"], signal);
|
|
276
422
|
if (!result.success)
|
|
277
423
|
throw new Error(`Failed to close the browser: ${result.error}`);
|
|
424
|
+
agentTargets.clear();
|
|
278
425
|
browserOpen = false;
|
|
279
426
|
closedByCall = true;
|
|
280
427
|
return {
|
|
@@ -307,26 +454,32 @@ function createUnlockedJevBrowserTool(getEnv, generateText) {
|
|
|
307
454
|
try {
|
|
308
455
|
if (args.url) {
|
|
309
456
|
if (browserOpen) {
|
|
310
|
-
closedTabs.push(...(await closeOldestTabs(
|
|
457
|
+
closedTabs.push(...(await closeOldestTabs(browser, ownTabs, BROWSER_TAB_LIMIT - 1, signal)));
|
|
311
458
|
}
|
|
312
|
-
const openResult = await openPage(
|
|
459
|
+
const openResult = await openPage(browser, args.url, browserOpen, signal);
|
|
313
460
|
captureLifecycle(openResult);
|
|
314
461
|
if (!openResult.success) {
|
|
315
462
|
throw new Error(`Failed to open ${args.url}: ${openResult.error}`);
|
|
316
463
|
}
|
|
464
|
+
const opened = targetIdOf(openResult);
|
|
465
|
+
if (opened)
|
|
466
|
+
agentTargets.add(opened);
|
|
317
467
|
browserOpen = true;
|
|
318
468
|
closedByCall = false;
|
|
319
469
|
}
|
|
320
470
|
if (args.frame !== undefined) {
|
|
321
|
-
const switched = await runAgentBrowser(
|
|
471
|
+
const switched = await runAgentBrowser(browser, ["frame", args.frame], signal);
|
|
322
472
|
if (!switched.success)
|
|
323
473
|
throw new Error(`Failed to switch browser frame: ${switched.error}`);
|
|
324
474
|
}
|
|
325
475
|
for (const command of args.commands ?? []) {
|
|
326
476
|
if (signal?.aborted)
|
|
327
477
|
throw new Error("Operation aborted");
|
|
328
|
-
const result = await runAgentBrowser(
|
|
478
|
+
const result = await runAgentBrowser(browser, command, signal);
|
|
329
479
|
captureLifecycle(result);
|
|
480
|
+
const newTab = command[0] === "tab" && command[1] === "new" ? targetIdOf(result) : undefined;
|
|
481
|
+
if (newTab)
|
|
482
|
+
agentTargets.add(newTab);
|
|
330
483
|
commandResults.push({
|
|
331
484
|
command,
|
|
332
485
|
success: result.success,
|
|
@@ -362,7 +515,12 @@ function createUnlockedJevBrowserTool(getEnv, generateText) {
|
|
|
362
515
|
for (let step = 1; step <= maxSteps; step++) {
|
|
363
516
|
if (signal?.aborted)
|
|
364
517
|
throw new Error("Operation aborted");
|
|
365
|
-
|
|
518
|
+
if (step > 1 && (await desktopStatus(env, signal)).control === "user") {
|
|
519
|
+
status = "blocked";
|
|
520
|
+
message = PERSON_HAS_CONTROL;
|
|
521
|
+
break;
|
|
522
|
+
}
|
|
523
|
+
const snap = await runAgentBrowser(browser, ["snapshot"], signal);
|
|
366
524
|
captureLifecycle(snap);
|
|
367
525
|
if (!snap.success || !snap.data) {
|
|
368
526
|
status = "blocked";
|
|
@@ -430,7 +588,7 @@ function createUnlockedJevBrowserTool(getEnv, generateText) {
|
|
|
430
588
|
const command = operation === "WAIT"
|
|
431
589
|
? ["wait", "1000"]
|
|
432
590
|
: ["scroll", operation === "SCROLL_DOWN" ? "down" : "up", "500"];
|
|
433
|
-
const action = await runAgentBrowser(
|
|
591
|
+
const action = await runAgentBrowser(browser, command, signal);
|
|
434
592
|
if (!action.success) {
|
|
435
593
|
status = "blocked";
|
|
436
594
|
message = `${operation} failed: ${action.error}`;
|
|
@@ -447,7 +605,7 @@ function createUnlockedJevBrowserTool(getEnv, generateText) {
|
|
|
447
605
|
}
|
|
448
606
|
const targetInfo = refs[targetRef];
|
|
449
607
|
if (operation === "CLICK") {
|
|
450
|
-
const clickResult = await runAgentBrowser(
|
|
608
|
+
const clickResult = await runAgentBrowser(browser, ["click", `@${targetRef}`], signal);
|
|
451
609
|
if (!clickResult.success) {
|
|
452
610
|
status = "blocked";
|
|
453
611
|
message = `Click on ${targetRef} failed: ${clickResult.error}`;
|
|
@@ -469,7 +627,7 @@ function createUnlockedJevBrowserTool(getEnv, generateText) {
|
|
|
469
627
|
forSelect: operation === "SELECT",
|
|
470
628
|
}, signal);
|
|
471
629
|
const command = operation === "SELECT" ? "select" : "fill";
|
|
472
|
-
const actResult = await runAgentBrowser(
|
|
630
|
+
const actResult = await runAgentBrowser(browser, [command, `@${targetRef}`, text], signal);
|
|
473
631
|
if (!actResult.success) {
|
|
474
632
|
status = "blocked";
|
|
475
633
|
message = `${command} on ${targetRef} failed: ${actResult.error}`;
|
|
@@ -478,7 +636,7 @@ function createUnlockedJevBrowserTool(getEnv, generateText) {
|
|
|
478
636
|
history.push({ step, operation, target: targetRef, label: targetInfo?.name, text });
|
|
479
637
|
}
|
|
480
638
|
if (status === "step-limit") {
|
|
481
|
-
const finalSnapshot = await runAgentBrowser(
|
|
639
|
+
const finalSnapshot = await runAgentBrowser(browser, ["snapshot"], signal);
|
|
482
640
|
if (!finalSnapshot.success || !finalSnapshot.data) {
|
|
483
641
|
throw new Error(`Final snapshot failed: ${finalSnapshot.error}`);
|
|
484
642
|
}
|
|
@@ -488,10 +646,11 @@ function createUnlockedJevBrowserTool(getEnv, generateText) {
|
|
|
488
646
|
}
|
|
489
647
|
finally {
|
|
490
648
|
if (args.close === true) {
|
|
491
|
-
await runAgentBrowser(
|
|
649
|
+
await runAgentBrowser(browser, ["close"])
|
|
492
650
|
.then((result) => {
|
|
493
651
|
if (!result.success)
|
|
494
652
|
return;
|
|
653
|
+
agentTargets.clear();
|
|
495
654
|
browserOpen = false;
|
|
496
655
|
closedByCall = true;
|
|
497
656
|
})
|
|
@@ -520,8 +679,8 @@ function createUnlockedJevBrowserTool(getEnv, generateText) {
|
|
|
520
679
|
},
|
|
521
680
|
};
|
|
522
681
|
}
|
|
523
|
-
export function createJevBrowserTool(getEnv, generateText) {
|
|
524
|
-
const tool = createUnlockedJevBrowserTool(getEnv, generateText);
|
|
682
|
+
export function createJevBrowserTool(getEnv, generateText, options = {}) {
|
|
683
|
+
const tool = createUnlockedJevBrowserTool(getEnv, generateText, options);
|
|
525
684
|
const execute = tool.execute.bind(tool);
|
|
526
685
|
let executionTail = Promise.resolve();
|
|
527
686
|
return {
|