@geminixiang/mikan 1.4.0 → 1.5.0-beta.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. package/CHANGELOG.md +29 -0
  2. package/dist/adapters/commands/desktop.d.ts +5 -0
  3. package/dist/adapters/commands/desktop.d.ts.map +1 -0
  4. package/dist/adapters/commands/desktop.js +58 -0
  5. package/dist/adapters/commands/desktop.js.map +1 -0
  6. package/dist/adapters/commands/manifest.d.ts.map +1 -1
  7. package/dist/adapters/commands/manifest.js +8 -0
  8. package/dist/adapters/commands/manifest.js.map +1 -1
  9. package/dist/adapters/commands/registry.d.ts.map +1 -1
  10. package/dist/adapters/commands/registry.js +2 -0
  11. package/dist/adapters/commands/registry.js.map +1 -1
  12. package/dist/adapters/commands/types.d.ts +3 -1
  13. package/dist/adapters/commands/types.d.ts.map +1 -1
  14. package/dist/adapters/commands/types.js.map +1 -1
  15. package/dist/adapters/slack/app-config.d.ts +27 -0
  16. package/dist/adapters/slack/app-config.d.ts.map +1 -0
  17. package/dist/adapters/slack/app-config.js +241 -0
  18. package/dist/adapters/slack/app-config.js.map +1 -0
  19. package/dist/adapters/slack/types.d.ts +46 -1
  20. package/dist/adapters/slack/types.d.ts.map +1 -1
  21. package/dist/adapters/slack/types.js.map +1 -1
  22. package/dist/adapters/web/desktop/index.d.ts +16 -0
  23. package/dist/adapters/web/desktop/index.d.ts.map +1 -0
  24. package/dist/adapters/web/desktop/index.js +163 -0
  25. package/dist/adapters/web/desktop/index.js.map +1 -0
  26. package/dist/adapters/web/desktop/page.d.ts +2 -0
  27. package/dist/adapters/web/desktop/page.d.ts.map +1 -0
  28. package/dist/adapters/web/desktop/page.js +83 -0
  29. package/dist/adapters/web/desktop/page.js.map +1 -0
  30. package/dist/adapters/web/desktop/types.d.ts +8 -0
  31. package/dist/adapters/web/desktop/types.d.ts.map +1 -0
  32. package/dist/adapters/web/desktop/types.js +2 -0
  33. package/dist/adapters/web/desktop/types.js.map +1 -0
  34. package/dist/adapters/web/server.d.ts +2 -0
  35. package/dist/adapters/web/server.d.ts.map +1 -1
  36. package/dist/adapters/web/server.js +19 -0
  37. package/dist/adapters/web/server.js.map +1 -1
  38. package/dist/cli/boot.d.ts.map +1 -1
  39. package/dist/cli/boot.js +4 -1
  40. package/dist/cli/boot.js.map +1 -1
  41. package/dist/cli/onboard.d.ts +3 -0
  42. package/dist/cli/onboard.d.ts.map +1 -1
  43. package/dist/cli/onboard.js +79 -3
  44. package/dist/cli/onboard.js.map +1 -1
  45. package/dist/cli/slack.d.ts +9 -0
  46. package/dist/cli/slack.d.ts.map +1 -0
  47. package/dist/cli/slack.js +96 -0
  48. package/dist/cli/slack.js.map +1 -0
  49. package/dist/cli/types.d.ts +2 -1
  50. package/dist/cli/types.d.ts.map +1 -1
  51. package/dist/cli/types.js.map +1 -1
  52. package/dist/harness/desktop.d.ts +16 -0
  53. package/dist/harness/desktop.d.ts.map +1 -0
  54. package/dist/harness/desktop.js +6 -0
  55. package/dist/harness/desktop.js.map +1 -0
  56. package/dist/harness/execution-resolver.d.ts +1 -0
  57. package/dist/harness/execution-resolver.d.ts.map +1 -1
  58. package/dist/harness/execution-resolver.js +5 -0
  59. package/dist/harness/execution-resolver.js.map +1 -1
  60. package/dist/harness/presenter.d.ts.map +1 -1
  61. package/dist/harness/presenter.js +6 -1
  62. package/dist/harness/presenter.js.map +1 -1
  63. package/dist/harness/runner.d.ts.map +1 -1
  64. package/dist/harness/runner.js +22 -1
  65. package/dist/harness/runner.js.map +1 -1
  66. package/dist/harness/session.d.ts +1 -1
  67. package/dist/harness/session.d.ts.map +1 -1
  68. package/dist/harness/session.js +30 -9
  69. package/dist/harness/session.js.map +1 -1
  70. package/dist/harness/tools/codemode.d.ts.map +1 -1
  71. package/dist/harness/tools/codemode.js +2 -2
  72. package/dist/harness/tools/codemode.js.map +1 -1
  73. package/dist/harness/tools/index.d.ts +2 -2
  74. package/dist/harness/tools/index.d.ts.map +1 -1
  75. package/dist/harness/tools/index.js +10 -2
  76. package/dist/harness/tools/index.js.map +1 -1
  77. package/dist/harness/tools/jev-browser.d.ts +12 -1
  78. package/dist/harness/tools/jev-browser.d.ts.map +1 -1
  79. package/dist/harness/tools/jev-browser.js +189 -30
  80. package/dist/harness/tools/jev-browser.js.map +1 -1
  81. package/dist/harness/tools/pi-tools.d.ts.map +1 -1
  82. package/dist/harness/tools/pi-tools.js +2 -1
  83. package/dist/harness/tools/pi-tools.js.map +1 -1
  84. package/dist/harness/tools/types.d.ts +4 -0
  85. package/dist/harness/tools/types.d.ts.map +1 -1
  86. package/dist/harness/tools/types.js.map +1 -1
  87. package/dist/harness/types.d.ts +2 -1
  88. package/dist/harness/types.d.ts.map +1 -1
  89. package/dist/harness/types.js.map +1 -1
  90. package/dist/main.js +29 -0
  91. package/dist/main.js.map +1 -1
  92. package/dist/runtime/conversation-runtime.js +7 -0
  93. package/dist/runtime/conversation-runtime.js.map +1 -1
  94. package/dist/sandbox/desktop.d.ts +30 -0
  95. package/dist/sandbox/desktop.d.ts.map +1 -0
  96. package/dist/sandbox/desktop.js +112 -0
  97. package/dist/sandbox/desktop.js.map +1 -0
  98. package/dist/sandbox/provisioner.d.ts +4 -1
  99. package/dist/sandbox/provisioner.d.ts.map +1 -1
  100. package/dist/sandbox/provisioner.js +21 -0
  101. package/dist/sandbox/provisioner.js.map +1 -1
  102. package/dist/sandbox/types.d.ts +19 -0
  103. package/dist/sandbox/types.d.ts.map +1 -1
  104. package/dist/sandbox/types.js.map +1 -1
  105. package/dist/types.d.ts +17 -1
  106. package/dist/types.d.ts.map +1 -1
  107. package/dist/types.js.map +1 -1
  108. package/package.json +9 -8
@@ -1 +1 @@
1
- {"version":3,"file":"jev-browser.d.ts","sourceRoot":"","sources":["../../../src/harness/tools/jev-browser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,+BAA+B,CAAC;AAC/D,OAAO,EAAE,IAAI,EAAe,MAAM,SAAS,CAAC;AAG5C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC;AACnE,OAAO,EAAkC,KAAK,YAAY,EAAE,MAAM,WAAW,CAAC;AAI9E,eAAO,MAAM,gBAAgB,gBAAgB,CAAC;AAe9C,QAAA,MAAM,gBAAgB;;;;;;;;EAuCpB,CAAC;AAIH,UAAU,OAAO;IACf,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd;AAcD,UAAU,gBAAgB;IACxB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAMD,iBAAS,kBAAkB,CAAC,YAAY,EAAE,OAAO,EAAE,KAAK,EAAE,gBAAgB,GAAG,SAAS,GAAG,MAAM,CAY9F;AA4ED,iBAAS,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAEnD;AA2CD,iBAAS,cAAc,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG;IACtD,KAAK,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IAC3B,IAAI,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IAC1B,MAAM,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;CAC7B,CAOA;AAmBD,UAAU,YAAY;IACpB,SAAS,EAAE,YAAY,CAAC;IACxB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACjC;AAED,iBAAS,iBAAiB,CACxB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC7B,WAAW,EAAE,OAAO,EACpB,aAAa,EAAE,OAAO,EACtB,QAAQ,SAAK,GACZ,YAAY,CAiDd;AAED,MAAM,MAAM,kBAAkB,GAAG,CAC/B,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,WAAW,KACjB,OAAO,CAAC,MAAM,CAAC,CAAC;AAkCrB,iBAAS,aAAa,CACpB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE;IAAE,MAAM,CAAC,EAAE,OAAO,CAAA;CAAE,GAAG,SAAS,CAAC,EACzD,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,MAAM,GAAG,SAAS,CAUpB;AAuWD,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,MAAM,YAAY,GAAG,SAAS,EACtC,YAAY,CAAC,EAAE,kBAAkB,GAChC,SAAS,CAAC,OAAO,gBAAgB,CAAC,CAqBpC;AAED,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,cAAc,EAAE,QAAQ,EAAE,kBAAkB,EAAE,CAAC"}
1
+ {"version":3,"file":"jev-browser.d.ts","sourceRoot":"","sources":["../../../src/harness/tools/jev-browser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,+BAA+B,CAAC;AAC/D,OAAO,EAAE,IAAI,EAAe,MAAM,SAAS,CAAC;AAG5C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC;AACnE,OAAO,EAAkC,KAAK,YAAY,EAAE,MAAM,WAAW,CAAC;AAM9E,eAAO,MAAM,gBAAgB,gBAAgB,CAAC;AAe9C,QAAA,MAAM,gBAAgB;;;;;;;;;EA6CpB,CAAC;AAIH,UAAU,OAAO;IACf,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd;AAcD,UAAU,gBAAgB;IACxB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAMD,iBAAS,kBAAkB,CAAC,YAAY,EAAE,OAAO,EAAE,KAAK,EAAE,gBAAgB,GAAG,SAAS,GAAG,MAAM,CAY9F;AA2HD,iBAAS,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAEnD;AAuDD,iBAAS,cAAc,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG;IACtD,KAAK,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IAC3B,IAAI,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IAC1B,MAAM,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;CAC7B,CAOA;AAmBD,UAAU,YAAY;IACpB,SAAS,EAAE,YAAY,CAAC;IACxB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACjC;AAED,iBAAS,iBAAiB,CACxB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC7B,WAAW,EAAE,OAAO,EACpB,aAAa,EAAE,OAAO,EACtB,QAAQ,SAAK,GACZ,YAAY,CAiDd;AAED,MAAM,MAAM,kBAAkB,GAAG,CAC/B,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,WAAW,KACjB,OAAO,CAAC,MAAM,CAAC,CAAC;AAkCrB,iBAAS,aAAa,CACpB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE;IAAE,MAAM,CAAC,EAAE,OAAO,CAAA;CAAE,GAAG,SAAS,CAAC,EACzD,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,MAAM,GAAG,SAAS,CAUpB;AAcD,UAAU,cAAc;IACtB,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACvC;AAED,MAAM,MAAM,cAAc,GAAG,SAAS,CAAC,OAAO,gBAAgB,CAAC,GAAG;IAChE,UAAU,CAAC,OAAO,EAAE,cAAc,GAAG,SAAS,GAAG,IAAI,CAAC;CACvD,CAAC;AAEF,UAAU,iBAAiB;IACzB,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AA0cD,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,MAAM,YAAY,GAAG,SAAS,EACtC,YAAY,CAAC,EAAE,kBAAkB,EACjC,OAAO,GAAE,iBAAsB,GAC9B,cAAc,CAqBhB;AAED,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,cAAc,EAAE,QAAQ,EAAE,kBAAkB,EAAE,CAAC"}
@@ -4,6 +4,7 @@ import { BACKGROUND_CONTEXT, withAbortSignal } from "@earendil-works/chord/conte
4
4
  import { evaluateWithJev } from "../jev.js";
5
5
  import { LABEL_PARAMETER } from "./host-fn-tool.js";
6
6
  import { isRecord } from "../../unknown-values.js";
7
+ import { DESKTOP_COMMAND, DESKTOP_RUNTIME_DIR, parseDesktopStatus } from "../../sandbox/desktop.js";
7
8
  export const JEV_BROWSER_TOOL = "jev_browser";
8
9
  const AGENT_BROWSER_BIN = "agent-browser";
9
10
  const AGENT_BROWSER_UNAVAILABLE = "agent-browser CLI is unavailable in the current sandbox. Ask the operator to provision " +
@@ -38,6 +39,9 @@ const jevBrowserSchema = Type.Object({
38
39
  minimum: 1,
39
40
  maximum: HARD_MAX_STEPS,
40
41
  })),
42
+ handOff: Type.Optional(Type.String({
43
+ description: "Give the browser to the person on the office desktop for a step only they can do, such as signing in or solving a challenge: what they should do, in their language. Use alone. It opens the desktop, moves the current page onto it, posts the step with a desktop link, and ends your turn. Continue after they say they are done.",
44
+ })),
41
45
  });
42
46
  function lifecycleOf(result) {
43
47
  return result.data?.lifecycle;
@@ -57,21 +61,64 @@ function describeContinuity(expectedOpen, first) {
57
61
  function tabNumber(tab) {
58
62
  return Number(tab.tabId.replace(/^t/, ""));
59
63
  }
60
- async function closeOldestTabs(env, sessionId, keep, signal) {
61
- const listed = await runAgentBrowser(env, sessionId, ["tab", "list"], signal);
64
+ async function closeOldestTabs(browser, agentTargets, keep, signal) {
65
+ const listed = await runAgentBrowser(browser, ["tab", "list"], signal);
62
66
  const tabs = listed.success && Array.isArray(listed.data?.tabs) ? listed.data.tabs : [];
63
- const oldest = tabs
67
+ const opened = agentTargets
68
+ ? tabs.filter((tab) => tab.targetId !== undefined && agentTargets.has(tab.targetId))
69
+ : tabs;
70
+ const oldest = opened
64
71
  .filter((tab) => !tab.active)
65
72
  .toSorted((a, b) => tabNumber(a) - tabNumber(b))
66
- .slice(0, Math.max(0, tabs.length - keep));
73
+ .slice(0, Math.max(0, opened.length - keep));
67
74
  const closed = [];
68
75
  for (const tab of oldest) {
69
- const result = await runAgentBrowser(env, sessionId, ["tab", "close", tab.tabId], signal);
70
- if (result.success)
71
- closed.push(tab.url ?? tab.tabId);
76
+ const result = await runAgentBrowser(browser, ["tab", "close", tab.tabId], signal);
77
+ if (!result.success)
78
+ continue;
79
+ if (tab.targetId)
80
+ agentTargets?.delete(tab.targetId);
81
+ closed.push(tab.url ?? tab.tabId);
72
82
  }
73
83
  return closed;
74
84
  }
85
+ function targetIdOf(result) {
86
+ const data = result.data;
87
+ return isRecord(data) && typeof data.targetId === "string" ? data.targetId : undefined;
88
+ }
89
+ async function readDesktopStatus(env, signal) {
90
+ let stdout = "";
91
+ const result = await env.exec([DESKTOP_COMMAND, "status"], {
92
+ timeout: COMMAND_TIMEOUT_SECONDS,
93
+ onOutput: (text, _context, info) => {
94
+ if (info.stream === "stdout")
95
+ stdout += text;
96
+ },
97
+ }, signal ? withAbortSignal(signal, BACKGROUND_CONTEXT) : BACKGROUND_CONTEXT);
98
+ if (!result.ok || result.value.exitCode !== 0)
99
+ return { display: undefined, control: "agent" };
100
+ return parseDesktopStatus(stdout);
101
+ }
102
+ function desktopLaunchEnv(display) {
103
+ return {
104
+ AGENT_BROWSER_HEADED: "1",
105
+ AGENT_BROWSER_ARGS: "--ozone-platform=wayland",
106
+ WAYLAND_DISPLAY: `${DESKTOP_RUNTIME_DIR}/${display}`,
107
+ };
108
+ }
109
+ const PERSON_HAS_CONTROL = "A person is using this browser on the office desktop. Do not use the browser until they hand it back: they press Hand back to agent on the desktop page or close it. Tell them what you need and end your turn.";
110
+ const NO_DESKTOP_STATUS = { display: undefined, control: "agent" };
111
+ function personHasControlResult() {
112
+ return {
113
+ content: [
114
+ {
115
+ type: "text",
116
+ text: JSON.stringify({ status: "person-has-control", message: PERSON_HAS_CONTROL }),
117
+ },
118
+ ],
119
+ details: undefined,
120
+ };
121
+ }
75
122
  const CLICK_TARGET_KEY = "click_target";
76
123
  const TYPE_TARGET_KEY = "type_target";
77
124
  const SELECT_TARGET_KEY = "select_target";
@@ -98,10 +145,11 @@ const TEXT_VALUE_INSTRUCTIONS = 'Return a JSON object with exactly one key, "tex
98
145
  function truncate(text, max) {
99
146
  return text.length > max ? `${text.slice(0, max)}…` : text;
100
147
  }
101
- async function runAgentBrowser(env, sessionId, args, signal) {
148
+ async function runAgentBrowser(browser, args, signal) {
102
149
  const output = { stdout: "", stderr: "" };
103
- const result = await env.exec([AGENT_BROWSER_BIN, "--session", sessionId, ...args, "--json"], {
150
+ const result = await browser.env.exec([AGENT_BROWSER_BIN, "--session", browser.sessionId, ...args, "--json"], {
104
151
  timeout: COMMAND_TIMEOUT_SECONDS,
152
+ env: browser.launchEnv,
105
153
  onOutput: (text, _context, info) => {
106
154
  output[info.stream] += text;
107
155
  },
@@ -119,7 +167,12 @@ async function runAgentBrowser(env, sessionId, args, signal) {
119
167
  }
120
168
  if (stdout.trim()) {
121
169
  try {
122
- return JSON.parse(stdout);
170
+ const parsed = JSON.parse(stdout);
171
+ const lifecycle = lifecycleOf(parsed);
172
+ if (lifecycle?.launched === true || lifecycle?.relaunchedBrowser === true) {
173
+ browser.onLaunch?.(browser.launchEnv !== undefined);
174
+ }
175
+ return parsed;
123
176
  }
124
177
  catch (error) {
125
178
  if (code === 0)
@@ -232,24 +285,106 @@ function resolveTarget(operation, answers, singles) {
232
285
  const choice = answers[key]?.choice;
233
286
  return typeof choice === "string" ? choice : undefined;
234
287
  }
235
- async function openPage(env, sessionId, url, inNewTab, signal) {
288
+ async function openPage(browser, url, inNewTab, signal) {
236
289
  if (!inNewTab)
237
- return runAgentBrowser(env, sessionId, ["open", url], signal);
238
- const tab = await runAgentBrowser(env, sessionId, ["tab", "new"], signal);
290
+ return runAgentBrowser(browser, ["open", url], signal);
291
+ const tab = await runAgentBrowser(browser, ["tab", "new"], signal);
239
292
  if (!tab.success)
240
293
  return tab;
241
- return runAgentBrowser(env, sessionId, ["open", url], signal);
294
+ return runAgentBrowser(browser, ["open", url], signal);
295
+ }
296
+ const NO_PAGE_TO_HAND_OFF = "The browser has no page to hand over. Open the page with url first; it now opens on the desktop. Then call handOff again.";
297
+ const HAND_OFF_UNAVAILABLE = "Handing the browser to a person needs the office desktop and a direct message to send its link privately; this conversation has neither, so ask the person in chat instead.";
298
+ async function setBrowserControl(env, owner, signal) {
299
+ const result = await env.exec([DESKTOP_COMMAND, "control", owner], { timeout: COMMAND_TIMEOUT_SECONDS }, signal ? withAbortSignal(signal, BACKGROUND_CONTEXT) : BACKGROUND_CONTEXT);
300
+ return result.ok && result.value.exitCode === 0;
301
+ }
302
+ function handedOffResult() {
303
+ return {
304
+ content: [
305
+ {
306
+ type: "text",
307
+ text: JSON.stringify({
308
+ status: "handed-off",
309
+ message: "The person controls the browser now and was asked to do this step.",
310
+ }),
311
+ },
312
+ ],
313
+ details: { handedOff: true },
314
+ terminate: true,
315
+ };
316
+ }
317
+ function newSessionId() {
318
+ return `mikan-jb-${randomUUID()}`;
242
319
  }
243
- function createUnlockedJevBrowserTool(getEnv, generateText) {
244
- const sessionId = `mikan-jb-${randomUUID()}`;
320
+ function createUnlockedJevBrowserTool(getEnv, generateText, options) {
321
+ let sessionId = newSessionId();
322
+ const agentTargets = new Set();
323
+ let handOff;
245
324
  let browserOpen = false;
325
+ let openedOnDesktop = false;
326
+ const desktopStatus = (env, signal) => options.desktop ? readDesktopStatus(env, signal) : Promise.resolve(NO_DESKTOP_STATUS);
327
+ const target = (env, launchEnv) => ({
328
+ env,
329
+ sessionId,
330
+ launchEnv,
331
+ onLaunch: (onDesktop) => {
332
+ openedOnDesktop = onDesktop;
333
+ },
334
+ });
335
+ async function moveBrowserToDesktop(env, display, signal) {
336
+ const headless = target(env);
337
+ let launchedNow = false;
338
+ headless.onLaunch = () => {
339
+ launchedNow = true;
340
+ };
341
+ const current = await runAgentBrowser(headless, ["get", "url"], signal);
342
+ const url = current.success && !launchedNow ? current.data?.url : undefined;
343
+ await runAgentBrowser(headless, ["close"], signal);
344
+ agentTargets.clear();
345
+ browserOpen = false;
346
+ openedOnDesktop = false;
347
+ sessionId = newSessionId();
348
+ if (!url)
349
+ throw new Error(NO_PAGE_TO_HAND_OFF);
350
+ const reopened = await runAgentBrowser(target(env, desktopLaunchEnv(display)), ["open", url], signal);
351
+ if (!reopened.success) {
352
+ throw new Error(`Could not reopen ${url} on the desktop: ${reopened.error}. ${NO_PAGE_TO_HAND_OFF}`);
353
+ }
354
+ const opened = targetIdOf(reopened);
355
+ if (opened)
356
+ agentTargets.add(opened);
357
+ browserOpen = true;
358
+ }
359
+ async function handBrowserToPerson(env, step, signal) {
360
+ if (!handOff || !(await handOff.openDesktop()))
361
+ throw new Error(HAND_OFF_UNAVAILABLE);
362
+ const desktop = await desktopStatus(env, signal);
363
+ if (!desktop.display)
364
+ throw new Error(HAND_OFF_UNAVAILABLE);
365
+ if (!openedOnDesktop)
366
+ await moveBrowserToDesktop(env, desktop.display, signal);
367
+ if (!(await setBrowserControl(env, "user", signal)))
368
+ throw new Error(HAND_OFF_UNAVAILABLE);
369
+ try {
370
+ await handOff.announce(step);
371
+ }
372
+ catch (error) {
373
+ await setBrowserControl(env, "agent").catch(() => false);
374
+ throw error;
375
+ }
376
+ return handedOffResult();
377
+ }
246
378
  let closedByCall = false;
247
379
  return {
380
+ setHandOff: (next) => {
381
+ handOff = next;
382
+ },
248
383
  name: JEV_BROWSER_TOOL,
249
384
  label: "jev browser",
250
385
  description: [
251
386
  "Control a real Chrome browser in this conversation's sandbox (not the host): give a goal that Jev drives step by step (click, type, select, scroll, wait), run raw agent-browser commands, or both; commands run first.",
252
- `Each thread has one browser that stays open across calls; url opens a new tab, and past ${BROWSER_TAB_LIMIT} tabs the oldest closes.`,
387
+ `Each thread has one browser that stays open across calls; url opens a new tab, and past ${BROWSER_TAB_LIMIT} tabs it opened the oldest closes; tabs a person opened stay.`,
253
388
  "Read the page from lastPageSnapshot (accessibility text; the tool does not summarize) and command output from commandResults. A browserContinuity of unknown is missing metadata, not a failure.",
254
389
  'CLI success is not goal completion: verify the visible state. To submit a typed input, run ["press","Enter"]: press takes a key, not a ref. After navigation, snapshot again and use the new @refs. If content sits in an iframe, set frame instead of scrolling. find may act, not only read. When syntax is unclear, run ["<command>","--help"]; after one informed retry, report the blocker instead of trying more selectors.',
255
390
  "If agent-browser is missing from the sandbox, report it; never install it on the host.",
@@ -271,10 +406,22 @@ function createUnlockedJevBrowserTool(getEnv, generateText) {
271
406
  throw new Error('press takes a key, not a target ref: use ["press","Enter"] after focusing the input. Check ["press","--help"] for native syntax.');
272
407
  }
273
408
  }
409
+ if (args.handOff !== undefined) {
410
+ if (args.url || args.goal || args.commands?.length || args.close || args.frame) {
411
+ throw new Error("Use handOff alone, without url, goal, commands, frame, or close.");
412
+ }
413
+ return handBrowserToPerson(env, args.handOff, signal);
414
+ }
415
+ const desktop = await desktopStatus(env, signal);
416
+ if (desktop.control === "user")
417
+ return personHasControlResult();
418
+ const browser = target(env, desktop.display ? desktopLaunchEnv(desktop.display) : undefined);
419
+ const ownTabs = desktop.display ? agentTargets : undefined;
274
420
  if (args.close === true && !args.url && !args.goal && !args.commands?.length) {
275
- const result = await runAgentBrowser(env, sessionId, ["close"], signal);
421
+ const result = await runAgentBrowser(browser, ["close"], signal);
276
422
  if (!result.success)
277
423
  throw new Error(`Failed to close the browser: ${result.error}`);
424
+ agentTargets.clear();
278
425
  browserOpen = false;
279
426
  closedByCall = true;
280
427
  return {
@@ -307,26 +454,32 @@ function createUnlockedJevBrowserTool(getEnv, generateText) {
307
454
  try {
308
455
  if (args.url) {
309
456
  if (browserOpen) {
310
- closedTabs.push(...(await closeOldestTabs(env, sessionId, BROWSER_TAB_LIMIT - 1, signal)));
457
+ closedTabs.push(...(await closeOldestTabs(browser, ownTabs, BROWSER_TAB_LIMIT - 1, signal)));
311
458
  }
312
- const openResult = await openPage(env, sessionId, args.url, browserOpen, signal);
459
+ const openResult = await openPage(browser, args.url, browserOpen, signal);
313
460
  captureLifecycle(openResult);
314
461
  if (!openResult.success) {
315
462
  throw new Error(`Failed to open ${args.url}: ${openResult.error}`);
316
463
  }
464
+ const opened = targetIdOf(openResult);
465
+ if (opened)
466
+ agentTargets.add(opened);
317
467
  browserOpen = true;
318
468
  closedByCall = false;
319
469
  }
320
470
  if (args.frame !== undefined) {
321
- const switched = await runAgentBrowser(env, sessionId, ["frame", args.frame], signal);
471
+ const switched = await runAgentBrowser(browser, ["frame", args.frame], signal);
322
472
  if (!switched.success)
323
473
  throw new Error(`Failed to switch browser frame: ${switched.error}`);
324
474
  }
325
475
  for (const command of args.commands ?? []) {
326
476
  if (signal?.aborted)
327
477
  throw new Error("Operation aborted");
328
- const result = await runAgentBrowser(env, sessionId, command, signal);
478
+ const result = await runAgentBrowser(browser, command, signal);
329
479
  captureLifecycle(result);
480
+ const newTab = command[0] === "tab" && command[1] === "new" ? targetIdOf(result) : undefined;
481
+ if (newTab)
482
+ agentTargets.add(newTab);
330
483
  commandResults.push({
331
484
  command,
332
485
  success: result.success,
@@ -362,7 +515,12 @@ function createUnlockedJevBrowserTool(getEnv, generateText) {
362
515
  for (let step = 1; step <= maxSteps; step++) {
363
516
  if (signal?.aborted)
364
517
  throw new Error("Operation aborted");
365
- const snap = await runAgentBrowser(env, sessionId, ["snapshot"], signal);
518
+ if (step > 1 && (await desktopStatus(env, signal)).control === "user") {
519
+ status = "blocked";
520
+ message = PERSON_HAS_CONTROL;
521
+ break;
522
+ }
523
+ const snap = await runAgentBrowser(browser, ["snapshot"], signal);
366
524
  captureLifecycle(snap);
367
525
  if (!snap.success || !snap.data) {
368
526
  status = "blocked";
@@ -430,7 +588,7 @@ function createUnlockedJevBrowserTool(getEnv, generateText) {
430
588
  const command = operation === "WAIT"
431
589
  ? ["wait", "1000"]
432
590
  : ["scroll", operation === "SCROLL_DOWN" ? "down" : "up", "500"];
433
- const action = await runAgentBrowser(env, sessionId, command, signal);
591
+ const action = await runAgentBrowser(browser, command, signal);
434
592
  if (!action.success) {
435
593
  status = "blocked";
436
594
  message = `${operation} failed: ${action.error}`;
@@ -447,7 +605,7 @@ function createUnlockedJevBrowserTool(getEnv, generateText) {
447
605
  }
448
606
  const targetInfo = refs[targetRef];
449
607
  if (operation === "CLICK") {
450
- const clickResult = await runAgentBrowser(env, sessionId, ["click", `@${targetRef}`], signal);
608
+ const clickResult = await runAgentBrowser(browser, ["click", `@${targetRef}`], signal);
451
609
  if (!clickResult.success) {
452
610
  status = "blocked";
453
611
  message = `Click on ${targetRef} failed: ${clickResult.error}`;
@@ -469,7 +627,7 @@ function createUnlockedJevBrowserTool(getEnv, generateText) {
469
627
  forSelect: operation === "SELECT",
470
628
  }, signal);
471
629
  const command = operation === "SELECT" ? "select" : "fill";
472
- const actResult = await runAgentBrowser(env, sessionId, [command, `@${targetRef}`, text], signal);
630
+ const actResult = await runAgentBrowser(browser, [command, `@${targetRef}`, text], signal);
473
631
  if (!actResult.success) {
474
632
  status = "blocked";
475
633
  message = `${command} on ${targetRef} failed: ${actResult.error}`;
@@ -478,7 +636,7 @@ function createUnlockedJevBrowserTool(getEnv, generateText) {
478
636
  history.push({ step, operation, target: targetRef, label: targetInfo?.name, text });
479
637
  }
480
638
  if (status === "step-limit") {
481
- const finalSnapshot = await runAgentBrowser(env, sessionId, ["snapshot"], signal);
639
+ const finalSnapshot = await runAgentBrowser(browser, ["snapshot"], signal);
482
640
  if (!finalSnapshot.success || !finalSnapshot.data) {
483
641
  throw new Error(`Final snapshot failed: ${finalSnapshot.error}`);
484
642
  }
@@ -488,10 +646,11 @@ function createUnlockedJevBrowserTool(getEnv, generateText) {
488
646
  }
489
647
  finally {
490
648
  if (args.close === true) {
491
- await runAgentBrowser(env, sessionId, ["close"])
649
+ await runAgentBrowser(browser, ["close"])
492
650
  .then((result) => {
493
651
  if (!result.success)
494
652
  return;
653
+ agentTargets.clear();
495
654
  browserOpen = false;
496
655
  closedByCall = true;
497
656
  })
@@ -520,8 +679,8 @@ function createUnlockedJevBrowserTool(getEnv, generateText) {
520
679
  },
521
680
  };
522
681
  }
523
- export function createJevBrowserTool(getEnv, generateText) {
524
- const tool = createUnlockedJevBrowserTool(getEnv, generateText);
682
+ export function createJevBrowserTool(getEnv, generateText, options = {}) {
683
+ const tool = createUnlockedJevBrowserTool(getEnv, generateText, options);
525
684
  const execute = tool.execute.bind(tool);
526
685
  let executionTail = Promise.resolve();
527
686
  return {