@mirasim/computer-use 0.0.0-stage → 0.1.0-rc.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (341) hide show
  1. package/LICENSE +202 -0
  2. package/NOTICE +16 -0
  3. package/README.md +154 -2
  4. package/THIRD_PARTY_NOTICES +488 -0
  5. package/bin/mirasim-cu.mjs +30 -0
  6. package/dist/entries/cli.d.ts +2 -0
  7. package/dist/entries/cli.js +32 -0
  8. package/dist/entries/commands/approvals.d.ts +1 -0
  9. package/dist/entries/commands/approvals.js +17 -0
  10. package/dist/entries/commands/bridge-host.d.ts +1 -0
  11. package/dist/entries/commands/bridge-host.js +7 -0
  12. package/dist/entries/commands/bridge.d.ts +1 -0
  13. package/dist/entries/commands/bridge.js +16 -0
  14. package/dist/entries/commands/browser-state.d.ts +1 -0
  15. package/dist/entries/commands/browser-state.js +8 -0
  16. package/dist/entries/commands/capture-server.d.ts +1 -0
  17. package/dist/entries/commands/capture-server.js +5 -0
  18. package/dist/entries/commands/help.d.ts +2 -0
  19. package/dist/entries/commands/help.js +1 -0
  20. package/dist/entries/commands/native-host-options.d.ts +32 -0
  21. package/dist/entries/commands/native-host-options.js +7 -0
  22. package/dist/entries/commands/setup-macos.d.ts +1 -0
  23. package/dist/entries/commands/setup-macos.js +4 -0
  24. package/dist/entries/commands/setup.d.ts +1 -0
  25. package/dist/entries/commands/setup.js +15 -0
  26. package/dist/entries/mcp.d.ts +1 -0
  27. package/dist/entries/mcp.js +6 -0
  28. package/dist/entries/version.d.ts +1 -0
  29. package/dist/entries/version.js +1 -0
  30. package/dist/index.d.ts +11 -0
  31. package/dist/index.js +1 -0
  32. package/dist/platform/browser/acquire.d.ts +15 -0
  33. package/dist/platform/browser/acquire.js +1 -0
  34. package/dist/platform/browser/bridge/acquire-bridge.d.ts +7 -0
  35. package/dist/platform/browser/bridge/acquire-bridge.js +3 -0
  36. package/dist/platform/browser/bridge/client.d.ts +69 -0
  37. package/dist/platform/browser/bridge/client.js +1 -0
  38. package/dist/platform/browser/bridge/extension-channel.d.ts +12 -0
  39. package/dist/platform/browser/bridge/extension-channel.js +1 -0
  40. package/dist/platform/browser/bridge/extension-id.d.ts +8 -0
  41. package/dist/platform/browser/bridge/extension-id.js +1 -0
  42. package/dist/platform/browser/bridge/extension-updates.d.ts +37 -0
  43. package/dist/platform/browser/bridge/extension-updates.js +1 -0
  44. package/dist/platform/browser/bridge/model-commands.d.ts +20 -0
  45. package/dist/platform/browser/bridge/model-commands.js +1 -0
  46. package/dist/platform/browser/bridge/model-core.d.ts +59 -0
  47. package/dist/platform/browser/bridge/model-core.js +1 -0
  48. package/dist/platform/browser/bridge/model.d.ts +4 -0
  49. package/dist/platform/browser/bridge/model.js +1 -0
  50. package/dist/platform/browser/bridge/native-frames.d.ts +9 -0
  51. package/dist/platform/browser/bridge/native-frames.js +1 -0
  52. package/dist/platform/browser/bridge/native-host.d.ts +24 -0
  53. package/dist/platform/browser/bridge/native-host.js +5 -0
  54. package/dist/platform/browser/bridge/runtime-contexts.d.ts +12 -0
  55. package/dist/platform/browser/bridge/runtime-contexts.js +1 -0
  56. package/dist/platform/browser/bridge/server-core.d.ts +66 -0
  57. package/dist/platform/browser/bridge/server-core.js +1 -0
  58. package/dist/platform/browser/bridge/server.d.ts +13 -0
  59. package/dist/platform/browser/bridge/server.js +1 -0
  60. package/dist/platform/browser/bridge-tabs.d.ts +22 -0
  61. package/dist/platform/browser/bridge-tabs.js +1 -0
  62. package/dist/platform/browser/cdp-connection.d.ts +3 -0
  63. package/dist/platform/browser/cdp-connection.js +1 -0
  64. package/dist/platform/browser/chrome-binary.d.ts +6 -0
  65. package/dist/platform/browser/chrome-binary.js +1 -0
  66. package/dist/platform/browser/config.d.ts +29 -0
  67. package/dist/platform/browser/config.js +1 -0
  68. package/dist/platform/browser/devtools-port.d.ts +3 -0
  69. package/dist/platform/browser/devtools-port.js +2 -0
  70. package/dist/platform/browser/downloads.d.ts +22 -0
  71. package/dist/platform/browser/downloads.js +1 -0
  72. package/dist/platform/browser/driver-connect.d.ts +13 -0
  73. package/dist/platform/browser/driver-connect.js +1 -0
  74. package/dist/platform/browser/driver-core.d.ts +50 -0
  75. package/dist/platform/browser/driver-core.js +1 -0
  76. package/dist/platform/browser/driver-tabs.d.ts +16 -0
  77. package/dist/platform/browser/driver-tabs.js +1 -0
  78. package/dist/platform/browser/driver-verbs.d.ts +13 -0
  79. package/dist/platform/browser/driver-verbs.js +2 -0
  80. package/dist/platform/browser/driver.d.ts +38 -0
  81. package/dist/platform/browser/driver.js +3 -0
  82. package/dist/platform/browser/focus-guard.d.ts +2 -0
  83. package/dist/platform/browser/focus-guard.js +1 -0
  84. package/dist/platform/browser/journal.d.ts +21 -0
  85. package/dist/platform/browser/journal.js +2 -0
  86. package/dist/platform/browser/keep-in-tab.d.ts +1 -0
  87. package/dist/platform/browser/keep-in-tab.js +1 -0
  88. package/dist/platform/browser/messages.d.ts +25 -0
  89. package/dist/platform/browser/messages.js +8 -0
  90. package/dist/platform/browser/navigate.d.ts +4 -0
  91. package/dist/platform/browser/navigate.js +2 -0
  92. package/dist/platform/browser/occupancy.d.ts +2 -0
  93. package/dist/platform/browser/occupancy.js +2 -0
  94. package/dist/platform/browser/overlay-guard.d.ts +3 -0
  95. package/dist/platform/browser/overlay-guard.js +1 -0
  96. package/dist/platform/browser/save-picker.d.ts +2 -0
  97. package/dist/platform/browser/save-picker.js +1 -0
  98. package/dist/platform/browser/session-files.d.ts +11 -0
  99. package/dist/platform/browser/session-files.js +1 -0
  100. package/dist/platform/browser/state.d.ts +25 -0
  101. package/dist/platform/browser/state.js +2 -0
  102. package/dist/platform/browser/tabs.d.ts +15 -0
  103. package/dist/platform/browser/tabs.js +2 -0
  104. package/dist/platform/browser/window-placement.d.ts +8 -0
  105. package/dist/platform/browser/window-placement.js +1 -0
  106. package/dist/platform/chromium/actions-value.d.ts +11 -0
  107. package/dist/platform/chromium/actions-value.js +5 -0
  108. package/dist/platform/chromium/actions.d.ts +20 -0
  109. package/dist/platform/chromium/actions.js +2 -0
  110. package/dist/platform/chromium/checkable-target.d.ts +1 -0
  111. package/dist/platform/chromium/checkable-target.js +1 -0
  112. package/dist/platform/chromium/collect-viewport.d.ts +1 -0
  113. package/dist/platform/chromium/collect-viewport.js +1 -0
  114. package/dist/platform/chromium/collect.d.ts +11 -0
  115. package/dist/platform/chromium/collect.js +1 -0
  116. package/dist/platform/chromium/coverer-label.d.ts +8 -0
  117. package/dist/platform/chromium/coverer-label.js +1 -0
  118. package/dist/platform/chromium/frames.d.ts +2 -0
  119. package/dist/platform/chromium/frames.js +1 -0
  120. package/dist/platform/chromium/headings.d.ts +1 -0
  121. package/dist/platform/chromium/headings.js +1 -0
  122. package/dist/platform/chromium/locate.d.ts +2 -0
  123. package/dist/platform/chromium/locate.js +1 -0
  124. package/dist/platform/chromium/page-acts.d.ts +9 -0
  125. package/dist/platform/chromium/page-acts.js +1 -0
  126. package/dist/platform/chromium/page-base.d.ts +33 -0
  127. package/dist/platform/chromium/page-base.js +1 -0
  128. package/dist/platform/chromium/page-driver.d.ts +18 -0
  129. package/dist/platform/chromium/page-driver.js +2 -0
  130. package/dist/platform/chromium/page-refs.d.ts +4 -0
  131. package/dist/platform/chromium/page-refs.js +1 -0
  132. package/dist/platform/chromium/page-text.d.ts +15 -0
  133. package/dist/platform/chromium/page-text.js +1 -0
  134. package/dist/platform/chromium/page-waits.d.ts +7 -0
  135. package/dist/platform/chromium/page-waits.js +1 -0
  136. package/dist/platform/chromium/preview-stream.d.ts +37 -0
  137. package/dist/platform/chromium/preview-stream.js +1 -0
  138. package/dist/platform/chromium/preview.d.ts +3 -0
  139. package/dist/platform/chromium/preview.js +1 -0
  140. package/dist/platform/chromium/read-page-script.d.ts +1 -0
  141. package/dist/platform/chromium/read-page-script.js +1 -0
  142. package/dist/platform/chromium/read-page.d.ts +11 -0
  143. package/dist/platform/chromium/read-page.js +6 -0
  144. package/dist/platform/chromium/run-js.d.ts +4 -0
  145. package/dist/platform/chromium/run-js.js +2 -0
  146. package/dist/platform/chromium/scroll-regions.d.ts +1 -0
  147. package/dist/platform/chromium/scroll-regions.js +1 -0
  148. package/dist/platform/chromium/scroll-scripts.d.ts +3 -0
  149. package/dist/platform/chromium/scroll-scripts.js +1 -0
  150. package/dist/platform/chromium/scroll.d.ts +18 -0
  151. package/dist/platform/chromium/scroll.js +5 -0
  152. package/dist/platform/chromium/select-text.d.ts +4 -0
  153. package/dist/platform/chromium/select-text.js +2 -0
  154. package/dist/platform/chromium/shot.d.ts +10 -0
  155. package/dist/platform/chromium/shot.js +2 -0
  156. package/dist/platform/chromium/unreached.d.ts +8 -0
  157. package/dist/platform/chromium/unreached.js +1 -0
  158. package/dist/platform/cli-command.d.ts +1 -0
  159. package/dist/platform/cli-command.js +1 -0
  160. package/dist/platform/driver-base.d.ts +83 -0
  161. package/dist/platform/driver-base.js +1 -0
  162. package/dist/platform/driver-diagnostics.d.ts +30 -0
  163. package/dist/platform/driver-diagnostics.js +4 -0
  164. package/dist/platform/driver-lines.d.ts +15 -0
  165. package/dist/platform/driver-lines.js +2 -0
  166. package/dist/platform/driver-narrate.d.ts +26 -0
  167. package/dist/platform/driver-narrate.js +3 -0
  168. package/dist/platform/driver-refs.d.ts +16 -0
  169. package/dist/platform/driver-refs.js +1 -0
  170. package/dist/platform/driver-render.d.ts +26 -0
  171. package/dist/platform/driver-render.js +4 -0
  172. package/dist/platform/driver-select.d.ts +31 -0
  173. package/dist/platform/driver-select.js +2 -0
  174. package/dist/platform/driver-settle.d.ts +22 -0
  175. package/dist/platform/driver-settle.js +4 -0
  176. package/dist/platform/driver-verdict.d.ts +11 -0
  177. package/dist/platform/driver-verdict.js +5 -0
  178. package/dist/platform/driver-wait.d.ts +9 -0
  179. package/dist/platform/driver-wait.js +1 -0
  180. package/dist/platform/driver-waits.d.ts +10 -0
  181. package/dist/platform/driver-waits.js +1 -0
  182. package/dist/platform/driver.d.ts +39 -0
  183. package/dist/platform/driver.js +15 -0
  184. package/dist/platform/electron/driver.d.ts +30 -0
  185. package/dist/platform/electron/driver.js +7 -0
  186. package/dist/platform/electron/preflight.d.ts +9 -0
  187. package/dist/platform/electron/preflight.js +1 -0
  188. package/dist/platform/electron/verbs.d.ts +3 -0
  189. package/dist/platform/electron/verbs.js +1 -0
  190. package/dist/platform/home.d.ts +2 -0
  191. package/dist/platform/home.js +1 -0
  192. package/dist/platform/macos/act-element.d.ts +27 -0
  193. package/dist/platform/macos/act-element.js +1 -0
  194. package/dist/platform/macos/app-names.d.ts +2 -0
  195. package/dist/platform/macos/app-names.js +2 -0
  196. package/dist/platform/macos/app-notes.d.ts +2 -0
  197. package/dist/platform/macos/app-notes.js +86 -0
  198. package/dist/platform/macos/approvals.d.ts +10 -0
  199. package/dist/platform/macos/approvals.js +2 -0
  200. package/dist/platform/macos/ax-client.d.ts +53 -0
  201. package/dist/platform/macos/ax-client.js +3 -0
  202. package/dist/platform/macos/ax-errors.d.ts +16 -0
  203. package/dist/platform/macos/ax-errors.js +1 -0
  204. package/dist/platform/macos/ax-shapes.d.ts +53 -0
  205. package/dist/platform/macos/ax-shapes.js +1 -0
  206. package/dist/platform/macos/capture-client.d.ts +11 -0
  207. package/dist/platform/macos/capture-client.js +2 -0
  208. package/dist/platform/macos/click-verb.d.ts +4 -0
  209. package/dist/platform/macos/click-verb.js +1 -0
  210. package/dist/platform/macos/driver-apps.d.ts +59 -0
  211. package/dist/platform/macos/driver-apps.js +2 -0
  212. package/dist/platform/macos/driver-read.d.ts +47 -0
  213. package/dist/platform/macos/driver-read.js +1 -0
  214. package/dist/platform/macos/driver-targets.d.ts +9 -0
  215. package/dist/platform/macos/driver-targets.js +1 -0
  216. package/dist/platform/macos/driver.d.ts +65 -0
  217. package/dist/platform/macos/driver.js +2 -0
  218. package/dist/platform/macos/element-lines.d.ts +12 -0
  219. package/dist/platform/macos/element-lines.js +1 -0
  220. package/dist/platform/macos/element-shapes.d.ts +54 -0
  221. package/dist/platform/macos/element-shapes.js +0 -0
  222. package/dist/platform/macos/element-text.d.ts +3 -0
  223. package/dist/platform/macos/element-text.js +2 -0
  224. package/dist/platform/macos/elements.d.ts +26 -0
  225. package/dist/platform/macos/elements.js +1 -0
  226. package/dist/platform/macos/extra-verbs.d.ts +2 -0
  227. package/dist/platform/macos/extra-verbs.js +1 -0
  228. package/dist/platform/macos/interaction-notes.d.ts +2 -0
  229. package/dist/platform/macos/interaction-notes.js +1 -0
  230. package/dist/platform/macos/interaction-policy.d.ts +5 -0
  231. package/dist/platform/macos/interaction-policy.js +1 -0
  232. package/dist/platform/macos/menu-commands.d.ts +22 -0
  233. package/dist/platform/macos/menu-commands.js +5 -0
  234. package/dist/platform/macos/mouse.d.ts +23 -0
  235. package/dist/platform/macos/mouse.js +1 -0
  236. package/dist/platform/macos/narrate.d.ts +20 -0
  237. package/dist/platform/macos/narrate.js +1 -0
  238. package/dist/platform/macos/permissions.d.ts +5 -0
  239. package/dist/platform/macos/permissions.js +1 -0
  240. package/dist/platform/macos/press-rungs.d.ts +49 -0
  241. package/dist/platform/macos/press-rungs.js +5 -0
  242. package/dist/platform/macos/preview-stream.d.ts +15 -0
  243. package/dist/platform/macos/preview-stream.js +1 -0
  244. package/dist/platform/macos/prompt-fragment.d.ts +1 -0
  245. package/dist/platform/macos/prompt-fragment.js +1 -0
  246. package/dist/platform/macos/scroll-pages.d.ts +1 -0
  247. package/dist/platform/macos/scroll-pages.js +1 -0
  248. package/dist/platform/macos/scroll-verb.d.ts +5 -0
  249. package/dist/platform/macos/scroll-verb.js +1 -0
  250. package/dist/platform/macos/select-text-verb.d.ts +8 -0
  251. package/dist/platform/macos/select-text-verb.js +1 -0
  252. package/dist/platform/macos/type-verb.d.ts +5 -0
  253. package/dist/platform/macos/type-verb.js +1 -0
  254. package/dist/platform/ownership.d.ts +20 -0
  255. package/dist/platform/ownership.js +1 -0
  256. package/dist/platform/platform-package.d.ts +1 -0
  257. package/dist/platform/platform-package.js +1 -0
  258. package/dist/platform/preview.d.ts +39 -0
  259. package/dist/platform/preview.js +1 -0
  260. package/dist/platform/profile.d.ts +20 -0
  261. package/dist/platform/profile.js +1 -0
  262. package/dist/platform/registry.d.ts +12 -0
  263. package/dist/platform/registry.js +1 -0
  264. package/dist/platform/tools/api-contract.d.ts +33 -0
  265. package/dist/platform/tools/api-contract.js +3 -0
  266. package/dist/platform/tools/api-expect.d.ts +3 -0
  267. package/dist/platform/tools/api-expect.js +1 -0
  268. package/dist/platform/tools/api-run.d.ts +6 -0
  269. package/dist/platform/tools/api-run.js +1 -0
  270. package/dist/platform/tools/api-target.d.ts +22 -0
  271. package/dist/platform/tools/api-target.js +1 -0
  272. package/dist/platform/tools/api-wait.d.ts +3 -0
  273. package/dist/platform/tools/api-wait.js +1 -0
  274. package/dist/platform/tools/api.d.ts +7 -0
  275. package/dist/platform/tools/api.js +1 -0
  276. package/dist/platform/tools/find-target.d.ts +54 -0
  277. package/dist/platform/tools/find-target.js +1 -0
  278. package/dist/platform/tools/index.d.ts +8 -0
  279. package/dist/platform/tools/index.js +1 -0
  280. package/dist/platform/tools/normalize.d.ts +2 -0
  281. package/dist/platform/tools/normalize.js +1 -0
  282. package/dist/platform/tools/repl-budget.d.ts +8 -0
  283. package/dist/platform/tools/repl-budget.js +4 -0
  284. package/dist/platform/tools/repl-code.d.ts +1 -0
  285. package/dist/platform/tools/repl-code.js +4 -0
  286. package/dist/platform/tools/repl-docs.d.ts +5 -0
  287. package/dist/platform/tools/repl-docs.js +4 -0
  288. package/dist/platform/tools/repl-state.d.ts +21 -0
  289. package/dist/platform/tools/repl-state.js +0 -0
  290. package/dist/platform/tools/repl.d.ts +8 -0
  291. package/dist/platform/tools/repl.js +19 -0
  292. package/dist/platform/tools/shot-file.d.ts +7 -0
  293. package/dist/platform/tools/shot-file.js +1 -0
  294. package/dist/platform/tools/validate.d.ts +2 -0
  295. package/dist/platform/tools/validate.js +1 -0
  296. package/dist/platform/types.d.ts +220 -0
  297. package/dist/platform/types.js +1 -0
  298. package/dist/platform/windows/app-gate.d.ts +24 -0
  299. package/dist/platform/windows/app-gate.js +2 -0
  300. package/dist/platform/windows/background-verb.d.ts +13 -0
  301. package/dist/platform/windows/background-verb.js +1 -0
  302. package/dist/platform/windows/dates.d.ts +2 -0
  303. package/dist/platform/windows/dates.js +1 -0
  304. package/dist/platform/windows/driver-apps.d.ts +9 -0
  305. package/dist/platform/windows/driver-apps.js +1 -0
  306. package/dist/platform/windows/driver-read.d.ts +12 -0
  307. package/dist/platform/windows/driver-read.js +3 -0
  308. package/dist/platform/windows/driver-targets.d.ts +58 -0
  309. package/dist/platform/windows/driver-targets.js +1 -0
  310. package/dist/platform/windows/driver-yield.d.ts +10 -0
  311. package/dist/platform/windows/driver-yield.js +1 -0
  312. package/dist/platform/windows/driver.d.ts +39 -0
  313. package/dist/platform/windows/driver.js +2 -0
  314. package/dist/platform/windows/elements.d.ts +66 -0
  315. package/dist/platform/windows/elements.js +2 -0
  316. package/dist/platform/windows/launch.d.ts +23 -0
  317. package/dist/platform/windows/launch.js +1 -0
  318. package/dist/platform/windows/menu-verb.d.ts +7 -0
  319. package/dist/platform/windows/menu-verb.js +6 -0
  320. package/dist/platform/windows/prompt-fragment.d.ts +2 -0
  321. package/dist/platform/windows/prompt-fragment.js +1 -0
  322. package/dist/platform/windows/text-verbs.d.ts +10 -0
  323. package/dist/platform/windows/text-verbs.js +5 -0
  324. package/dist/platform/windows/uia-client.d.ts +32 -0
  325. package/dist/platform/windows/uia-client.js +3 -0
  326. package/dist/platform/windows/verbs.d.ts +2 -0
  327. package/dist/platform/windows/verbs.js +1 -0
  328. package/extension/background.js +280 -0
  329. package/extension/connect.html +27 -0
  330. package/extension/connect.js +28 -0
  331. package/extension/group.js +283 -0
  332. package/extension/icons/icon-128.png +0 -0
  333. package/extension/icons/icon-16.png +0 -0
  334. package/extension/icons/icon-32.png +0 -0
  335. package/extension/icons/icon-48.png +0 -0
  336. package/extension/manifest.json +36 -0
  337. package/extension/relay.js +361 -0
  338. package/extension/runtime-info.js +9 -0
  339. package/extension/transport.js +81 -0
  340. package/package.json +76 -4
  341. package/scripts/extension.mjs +111 -0
@@ -0,0 +1,66 @@
1
+ import http from "node:http";
2
+ import type { Duplex } from "node:stream";
3
+ import { type ExtensionChannel } from "./extension-channel.ts";
4
+ import { BridgeModel } from "./model.ts";
5
+ import { ExtensionUpdates } from "./extension-updates.ts";
6
+ export interface BridgeServerOptions {
7
+ port?: number;
8
+ openUrl?: (url: string) => void | Promise<void>;
9
+ extensionId?: string;
10
+ socketPath?: string;
11
+ log?: (...args: unknown[]) => void;
12
+ }
13
+ export declare class ExtensionConnection {
14
+ private readonly channel;
15
+ protected readonly log: (...args: unknown[]) => void;
16
+ protected readonly warn: (...args: unknown[]) => void;
17
+ private readonly callbacks;
18
+ private lastId;
19
+ onEvent: ((method: string, params: unknown[]) => void) | null;
20
+ onClose: (() => void) | null;
21
+ constructor(channel: ExtensionChannel, log: (...args: unknown[]) => void, warn: (...args: unknown[]) => void);
22
+ get transport(): ExtensionChannel["kind"];
23
+ get pendingCount(): number;
24
+ get isOpen(): boolean;
25
+ send(method: string, params: unknown[]): Promise<unknown>;
26
+ close(reason: string): void;
27
+ private onMessage;
28
+ private dispose;
29
+ }
30
+ export declare abstract class BridgeServerCore {
31
+ protected abstract handleUpgrade(req: http.IncomingMessage, socket: Duplex, head: Buffer): void;
32
+ protected readonly uuid: `${string}-${string}-${string}-${string}-${string}`;
33
+ private readonly httpServer;
34
+ protected readonly wss: import("ws").Server<typeof import("ws").WebSocket, typeof http.IncomingMessage>;
35
+ protected readonly model: BridgeModel;
36
+ readonly extensionId: string;
37
+ readonly extensionIds: string[];
38
+ protected readonly updates: ExtensionUpdates;
39
+ protected activeOperations: number;
40
+ protected readonly log: (...args: unknown[]) => void;
41
+ protected readonly warn: (...args: unknown[]) => void;
42
+ private readonly openUrl;
43
+ protected port: number;
44
+ protected extension: ExtensionConnection | null;
45
+ protected extensionInitialized: boolean;
46
+ protected extensionSession: string | null;
47
+ protected readonly standby: Set<ExtensionConnection>;
48
+ protected readyWaiters: Array<() => void>;
49
+ private socketServer;
50
+ private readonly socketPath;
51
+ private connectInFlight;
52
+ protected keepalive: NodeJS.Timeout | null;
53
+ constructor(options?: BridgeServerOptions);
54
+ start(): Promise<void>;
55
+ private startNativeLeg;
56
+ protected abstract attachExtension(channel: ExtensionChannel): void;
57
+ cdpUrl(): string;
58
+ connectPageUrl(): string;
59
+ extensionConnected(): boolean;
60
+ ensureExtension(): Promise<void>;
61
+ private waitExtensionReady;
62
+ stop(): void;
63
+ protected runOperation<T>(operation: () => Promise<T>): Promise<T>;
64
+ private handleHttp;
65
+ private stripUuid;
66
+ }
@@ -0,0 +1 @@
1
+ import{randomUUID as m}from"node:crypto";import w from"node:http";import h from"node:fs";import x from"node:net";import v from"node:path";import{WebSocketServer as y}from"ws";import{socketChannel as b}from"./extension-channel.js";import{BridgeModel as S}from"./model.js";import{MIRASIM_EXTENSION_ID as E,extensionManifest as I}from"./extension-id.js";import{ExtensionUpdates as k}from"./extension-updates.js";const T=15e3,f=3e4,u=3e3;class L{channel;log;warn;callbacks=new Map;lastId=0;onEvent=null;onClose=null;constructor(e,t,s){this.channel=e,this.log=t,this.warn=s,e.onMessage(r=>this.onMessage(r)),e.onClose(()=>this.dispose())}get transport(){return this.channel.kind}get pendingCount(){return this.callbacks.size}get isOpen(){return this.channel.isOpen}send(e,t){if(!this.channel.isOpen)return Promise.reject(new Error("Extension connection closed"));const s=++this.lastId,r=Date.now();return new Promise((i,n)=>{const o=e==="bridge.ping"?null:setTimeout(()=>{this.callbacks.has(s)&&this.warn(`slow: ${p(e,t)} pending ${u}ms`)},u);o?.unref?.();const c=setTimeout(()=>{if(!this.callbacks.delete(s))return;o&&clearTimeout(o);const a=`Extension did not answer ${p(e,t)} within ${f}ms`;n(new Error(a)),this.close("Extension command timed out")},f);c.unref?.();const d=()=>{clearTimeout(c),o&&clearTimeout(o)};this.callbacks.set(s,{resolve:a=>{d();const g=Date.now()-r;g>=u&&e!=="bridge.ping"&&this.warn(`${p(e,t)} answered after ${g}ms`),i(a)},reject:a=>{d(),n(a)}});try{this.channel.send(JSON.stringify({id:s,method:e,params:t}))}catch(a){this.callbacks.delete(s),d(),n(a),this.close("Extension transport failed")}})}close(e){try{this.channel.close(e)}finally{this.dispose()}}onMessage(e){let t;try{t=JSON.parse(e)}catch(s){this.log("malformed extension message:",s);return}if(t.id!==void 0&&this.callbacks.has(t.id)){const s=this.callbacks.get(t.id);this.callbacks.delete(t.id),t.error!==void 0?s.reject(new Error(t.error)):s.resolve(t.result);return}t.method&&this.onEvent?.(t.method,t.params??[])}dispose(){for(const e of this.callbacks.values())e.reject(new Error("Extension connection closed"));this.callbacks.clear(),this.onClose?.(),this.onClose=null}}function p(l,e){if(l!=="chrome.debugger.sendCommand")return l;const t=e[0],s=typeof e[1]=="string"?e[1]:l,r=[t?.tabId===void 0?null:`tab=${t.tabId}`,t?.sessionId?`session=${t.sessionId}`:null].filter(Boolean);return r.length?`${s} ${r.join(" ")}`:s}class R{uuid=m();httpServer;wss=new y({noServer:!0});model;extensionId;extensionIds;updates;activeOperations=0;log;warn;openUrl;port=0;extension=null;extensionInitialized=!1;extensionSession=null;standby=new Set;readyWaiters=[];socketServer=null;socketPath;connectInFlight=null;keepalive=null;constructor(e={}){this.log=e.log??(()=>{}),this.warn=e.log??((...t)=>console.error("[bridge]",...t)),this.extensionId=e.extensionId??E,this.extensionIds=[this.extensionId],this.openUrl=e.openUrl??null,this.model=new S((t,s)=>{if(this.updates.reloading)throw new Error("The Mirasim extension is reloading; reconnect before continuing.");if(!this.extension)throw new Error("Bridge extension not connected");return this.extension.send(t,s)},this.log),this.updates=new k(I(),()=>!this.extension||this.model.clientCount()>0||this.activeOperations>0||this.extension.pendingCount>0,async()=>{if(!this.extension)throw new Error("The Mirasim extension disconnected before reloading.");return await this.extension.send("bridge.reload",[])}),this.port=e.port??0,this.socketPath=e.socketPath??null,this.httpServer=w.createServer((t,s)=>this.handleHttp(t,s)),this.httpServer.on("upgrade",(t,s,r)=>{this.handleUpgrade(t,s,r)})}async start(){await new Promise((e,t)=>{this.httpServer.once("error",t),this.httpServer.listen(this.port,"127.0.0.1",()=>e())}),this.port=this.httpServer.address().port,await this.startNativeLeg()}async startNativeLeg(){const e=this.socketPath;if(!e)return;const t=e.startsWith("\\\\.\\pipe\\");if(!t)try{h.mkdirSync(v.dirname(e),{recursive:!0})}catch(i){this.warn("native leg unavailable:",i);return}const s=x.createServer(i=>{i.on("error",n=>this.log("native socket error:",n)),this.attachExtension(b(i,n=>this.warn("native leg:",n.message)))});s.on("error",i=>this.warn("native leg listener:",i));const r=()=>new Promise((i,n)=>{s.once("error",n),s.listen(e,()=>i())});try{await r()}catch(i){if(t||i.code!=="EADDRINUSE"){this.warn("native leg unavailable:",i);return}try{h.unlinkSync(e),await r()}catch(n){this.warn("native leg unavailable:",n);return}}if(!t)try{h.chmodSync(e,384)}catch{}this.socketServer=s,this.log("native leg listening on",e)}cdpUrl(){return`http://127.0.0.1:${this.port}/${this.uuid}`}connectPageUrl(){const e=`ws://127.0.0.1:${this.port}/extension/${this.uuid}`;return`chrome-extension://${this.extensionId}/connect.html?relay=${encodeURIComponent(e)}`}extensionConnected(){return this.extension!==null}async ensureExtension(){this.extension&&this.extensionInitialized&&!this.updates.reloading||(this.connectInFlight||(this.connectInFlight=(async()=>{const e=this.waitExtensionReady();this.openUrl&&await this.openUrl(this.connectPageUrl());let t;const s=new Promise((r,i)=>{t=setTimeout(()=>i(new Error("The mirasim bridge extension did not connect \u2014 is it installed and enabled in your browser?")),T),t.unref?.()});try{await Promise.race([e,s])}finally{clearTimeout(t)}})(),this.connectInFlight.catch(()=>{}).finally(()=>{this.connectInFlight=null})),await this.connectInFlight)}waitExtensionReady(){return this.extension&&this.extensionInitialized&&!this.updates.reloading?Promise.resolve():new Promise(e=>this.readyWaiters.push(e))}stop(){this.updates.reset(),this.keepalive&&clearInterval(this.keepalive),this.extension?.close("Bridge stopped");for(const e of this.wss.clients)e.close(1e3,"Bridge stopped");if(this.httpServer.close(),this.socketServer?.close(),this.socketServer=null,this.socketPath&&!this.socketPath.startsWith("\\\\.\\pipe\\"))try{h.unlinkSync(this.socketPath)}catch{}}async runOperation(e){if(this.updates.reloading)throw new Error("The Mirasim extension is reloading; reconnect before continuing.");this.activeOperations+=1;try{return await e()}finally{this.activeOperations-=1}}handleHttp(e,t){const s=new URL(e.url??"/",`http://127.0.0.1:${this.port}`),r=this.stripUuid(s.pathname);if(r===null){t.writeHead(404).end();return}const i=(n,o)=>{const c=JSON.stringify(o,null,2);t.writeHead(n,{"Content-Type":"application/json; charset=UTF-8","Content-Length":Buffer.byteLength(c)}),t.end(c)};if(r==="/status"&&e.method==="GET"){const n=this.extensionConnected()&&this.extensionInitialized;i(200,{extensionConnected:n,extension:this.updates.status(n),standbyBrowsers:this.standby.size});return}if(r==="/extension/reload"&&e.method==="POST"){if(e.headers.origin){i(403,{error:"Reload must be requested by the local host."});return}try{this.updates.request(),i(202,{scheduled:!0})}catch(n){i(409,{error:n instanceof Error?n.message:String(n)})}return}if(r==="/recycle"&&e.method==="POST"){this.runOperation(()=>this.model.recycleAgentTabs()).then(n=>i(200,n),n=>i(502,{error:n instanceof Error?n.message:String(n)}));return}if(r==="/json/version"&&e.method==="GET"){i(200,{Browser:"mirasim-bridge","Protocol-Version":"1.3",webSocketDebuggerUrl:`ws://127.0.0.1:${this.port}/cdp/${this.uuid}`});return}if((r==="/json"||r==="/json/list")&&e.method==="GET"){this.runOperation(()=>this.model.listTargets()).then(n=>i(200,n),n=>i(500,{error:n instanceof Error?n.message:String(n)}));return}if(r==="/json/new"&&(e.method==="PUT"||e.method==="GET")){const n=decodeURIComponent(s.search.slice(1));(async()=>{try{await this.ensureExtension();const{targetId:o}=await this.runOperation(()=>this.model.createTarget(n||void 0));i(200,{id:o,type:"page",url:n||"about:blank"})}catch(o){i(500,{error:o instanceof Error?o.message:String(o)})}})();return}t.writeHead(404).end()}stripUuid(e){const t=`/${this.uuid}`;if(!e.startsWith(t))return null;const s=e.slice(t.length).replace(/\/$/,"");return s.startsWith("/")?s:null}}export{R as BridgeServerCore,L as ExtensionConnection};
@@ -0,0 +1,13 @@
1
+ import http from "node:http";
2
+ import type { Duplex } from "node:stream";
3
+ import { BridgeServerCore } from "./server-core.ts";
4
+ import { type ExtensionChannel } from "./extension-channel.ts";
5
+ export type { BridgeServerOptions } from "./server-core.ts";
6
+ export declare class BridgeServer extends BridgeServerCore {
7
+ protected handleUpgrade(req: http.IncomingMessage, socket: Duplex, head: Buffer): Promise<void>;
8
+ protected attachExtension(channel: ExtensionChannel): void;
9
+ private admitExtension;
10
+ private activateExtension;
11
+ private onExtensionEvent;
12
+ private attachCdpClient;
13
+ }
@@ -0,0 +1 @@
1
+ import"node:http";import{WebSocket as l}from"ws";import"./model.js";import{BridgeServerCore as c,ExtensionConnection as u}from"./server-core.js";import{webSocketChannel as m}from"./extension-channel.js";import{parseExtensionRuntime as f}from"./extension-updates.js";const x=2e4,p=5e3,g=5e3;class T extends c{async handleUpgrade(t,e,n){const i=new URL(t.url??"/",`http://127.0.0.1:${this.port}`),s=t.headers.origin;if(i.pathname===`/extension/${this.uuid}`){if(s&&!this.extensionIds.some(o=>s===`chrome-extension://${o}`)){e.destroy();return}this.wss.handleUpgrade(t,e,n,o=>this.attachExtension(m(o)));return}if(i.pathname===`/cdp/${this.uuid}`){if(s){e.destroy();return}try{await this.ensureExtension()}catch(o){this.log("rejecting CDP client:",o),e.destroy();return}this.wss.handleUpgrade(t,e,n,o=>this.attachCdpClient(o));return}e.destroy()}attachExtension(t){const e=new u(t,this.log,this.warn);if(!this.extension){this.activateExtension(e,[]);return}const n=[];let i=!1;e.onEvent=(s,o)=>{if(n.push([s,o]),!i&&s==="extension.connected"){i=!0;const a=typeof o[0]=="string"?o[0]:null;this.admitExtension(e,n,a)}},e.onClose=()=>{this.standby.delete(e)}}async admitExtension(t,e,n){const i=this.extension,s=!n||!this.extensionSession||n===this.extensionSession;if(i&&!s&&await v(i,p)){if(this.extension!==i)return this.admitExtension(t,e,n);if(!t.isOpen)return;t.onEvent=null,this.standby.add(t),this.warn("another browser's extension connected while one is active; it waits in standby until the active browser goes");return}t.isOpen&&this.activateExtension(t,e)}activateExtension(t,e){const n=this.extension;this.extension=null,n?.close("Replaced by a new extension connection"),n&&this.model.onExtensionDisconnected(),this.keepalive&&clearInterval(this.keepalive),this.updates.reset(),this.extension=t,this.extensionInitialized=!1,this.extensionSession=null,t.onEvent=(i,s)=>{this.extension===t&&this.onExtensionEvent(i,s)},t.onClose=()=>{this.extension===t&&(this.extension=null,this.extensionSession=null,this.updates.reset(),this.extensionInitialized=!1,this.keepalive&&clearInterval(this.keepalive),this.keepalive=null,this.model.onExtensionDisconnected(),setTimeout(()=>{if(!this.extension){for(const i of this.standby)i.close("The active browser left; reconnect to take over");this.standby.clear()}},g).unref?.())};for(const[i,s]of e)this.onExtensionEvent(i,s);this.keepalive=setInterval(()=>{t.send("bridge.ping",[]).catch(()=>{})},x),this.keepalive.unref?.()}onExtensionEvent(t,e){switch(t){case"chrome.tabs.onCreated":{const[n,i]=e;this.model.onTabCreated(n??{},i??{});break}case"chrome.tabs.onRemoved":this.model.onTabRemoved(e[0]);break;case"chrome.tabs.onReplaced":this.model.onTabReplaced(...e);break;case"chrome.debugger.onEvent":{const[n,i,s]=e;this.model.onDebuggerEvent(n??{},i,s);break}case"chrome.debugger.onDetach":this.model.onDebuggerDetach(e[0]??{});break;case"extension.connected":{this.extensionSession=typeof e[0]=="string"?e[0]:null,this.model.onExtensionConnected(e[0]);break}case"extension.initialized":{const n=f(e[0]);if(n&&!this.extensionIds.includes(n.id)){this.extension?.close("Unexpected extension identity");return}this.updates.runtime=n,this.extensionInitialized=!0;const i=this.readyWaiters;this.readyWaiters=[];for(const s of i)s();break}case"extension.updateAvailable":{const n=e[0];if(typeof n!="string"||!/^\d+(\.\d+){0,3}$/.test(n))break;this.updates.updateVersion=n;try{this.updates.request()}catch(i){this.log("extension update waits for manual reload:",i)}break}default:this.log("unknown extension event:",t)}}attachCdpClient(t){if(this.updates.reloading){t.close(1013,"The Mirasim extension is reloading");return}const e=this.model.addClient(n=>{t.readyState===l.OPEN&&t.send(JSON.stringify(n))});t.on("message",n=>{let i;try{i=JSON.parse(n.toString())}catch(s){this.log("malformed CDP message:",s);return}(async()=>{const{id:s,sessionId:o,method:a,params:h}=i;a==="Page.getFrameTree"&&s!==void 0&&o&&e.trackFrameTree(s,o);try{const r=await this.runOperation(()=>this.model.handleCommand(e,a??"",h,o));e.send({id:s,sessionId:o,result:r})}catch(r){e.send({id:s,sessionId:o,error:{message:r instanceof Error?r.message:String(r)}})}})()}),t.on("close",()=>this.model.removeClient(e)),t.on("error",()=>this.model.removeClient(e))}}async function v(d,t){if(!d.isOpen)return!1;let e;const n=new Promise(i=>{e=setTimeout(()=>i(!1),t)});try{return await Promise.race([d.send("bridge.ping",[]).then(()=>!0,()=>!1),n])}finally{clearTimeout(e)}}export{T as BridgeServer};
@@ -0,0 +1,22 @@
1
+ import type { Browser, Page } from "playwright-core";
2
+ interface BridgeTabInfo {
3
+ tabId: number;
4
+ title: string;
5
+ url: string;
6
+ unloaded: boolean;
7
+ browserSession: string;
8
+ }
9
+ export declare function bridgeRequest<T>(browser: Browser, method: string, params?: Record<string, unknown>): Promise<T>;
10
+ export declare function bridgeTabs(browser: Browser): Promise<{
11
+ unloaded: BridgeTabInfo[];
12
+ browserSession: string | null;
13
+ }>;
14
+ export declare function bridgeBrowserSession(browser: Browser): Promise<string | null>;
15
+ export declare function restoreBridgeTab(browser: Browser, tabId: number, targetId: string, browserSession: string): Promise<{
16
+ page: Page;
17
+ viewport?: {
18
+ width: number;
19
+ height: number;
20
+ };
21
+ }>;
22
+ export {};
@@ -0,0 +1 @@
1
+ import{pageTargetInfo as w}from"./tabs.js";async function s(n,e,t={}){const o=await n.newBrowserCDPSession();try{return await o.send(e,t)}finally{await o.detach().catch(()=>{})}}async function c(n){if(n.version()!=="mirasim-bridge")return{unloaded:[],browserSession:null};try{const e=await s(n,"Mirasim.tabs");return{unloaded:e.tabs.filter(o=>o.unloaded).map(o=>({...o,browserSession:e.browserSession??""})),browserSession:e.browserSession||null}}catch(e){if(/wasn't found|not found|not allowed|unknown method/i.test(String(e)))return{unloaded:[],browserSession:null};throw e}}async function u(n){return(await c(n).catch(()=>null))?.browserSession??null}async function f(n,e,t,o){const a=await s(n,"Mirasim.restoreTab",{tabId:e,targetId:t,browserSession:o}),i=Date.now()+1e4;for(;Date.now()<i;){for(const r of n.contexts().flatMap(l=>l.pages()))if(!r.isClosed()&&(await w(r).catch(()=>null))?.targetId===t)return{page:r,viewport:a.viewport};await new Promise(r=>setTimeout(r,50))}throw new Error(`Restored tab ${e} did not become readable; no action was sent to a replacement page.`)}export{u as bridgeBrowserSession,s as bridgeRequest,c as bridgeTabs,f as restoreBridgeTab};
@@ -0,0 +1,3 @@
1
+ import { type Browser } from "playwright-core";
2
+ export declare function browserArtifactsDir(): string;
3
+ export declare function connectBrowserCdp(url: string, timeout?: number): Promise<Browser>;
@@ -0,0 +1 @@
1
+ import{mkdir as n}from"node:fs/promises";import{join as e}from"node:path";import{chromium as i}from"playwright-core";import{sharedHome as c}from"../home.js";function m(){return e(c(),"downloads",".staging")}async function p(t,r){const o=m();return await n(o,{recursive:!0,mode:448}),i.connectOverCDP(t,{artifactsDir:o,...r===void 0?{}:{timeout:r}})}export{m as browserArtifactsDir,p as connectBrowserCdp};
@@ -0,0 +1,6 @@
1
+ import { type BrowserConfig } from "./config.ts";
2
+ export declare function isAttachedProfile(cfg?: BrowserConfig): boolean;
3
+ export declare function needsNoSandbox(platform?: NodeJS.Platform, uid?: number | undefined): boolean;
4
+ export declare function needsShmFallback(platform?: NodeJS.Platform, bytes?: number | null): boolean;
5
+ export declare function findChrome(cfg?: BrowserConfig): string | null;
6
+ export declare function chromeMissingError(): Error;
@@ -0,0 +1 @@
1
+ import{existsSync as i,statfsSync as p}from"node:fs";import{homedir as a}from"node:os";import{delimiter as u,join as s}from"node:path";import{browserConfigFromEnv as m}from"./config.js";import{msgChromeMissing as h}from"./messages.js";function P(o=m()){return o.attached}function v(o=process.platform,r=typeof process.getuid=="function"?process.getuid():void 0){return o==="linux"&&r===0}function G(o=process.platform,r=o==="linux"?f():null){return o==="linux"&&r!==null&&r<512*1024*1024}function f(){try{const o=p("/dev/shm");return o.blocks*o.bsize}catch{return null}}function S(o=m()){const r=o.chromePath;if(r)return i(r)?r:null;const n=process.platform==="darwin"?["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",s(a(),"Applications/Google Chrome.app/Contents/MacOS/Google Chrome"),"/Applications/Chromium.app/Contents/MacOS/Chromium"]:process.platform==="win32"?g():["/opt/google/chrome/chrome","/usr/bin/google-chrome","/usr/bin/google-chrome-stable","/usr/bin/chromium","/usr/bin/chromium-browser"];for(const e of n)if(i(e))return e;const t=process.platform==="win32"?["chrome.exe","chromium.exe"]:["google-chrome","google-chrome-stable","chromium","chromium-browser"];for(const e of(process.env.PATH??"").split(u))for(const l of t){const c=s(e,l);if(e&&i(c))return c}return null}function g(){const o=[process.env.ProgramFiles,process.env["ProgramFiles(x86)"],process.env.LOCALAPPDATA],r=["Google/Chrome/Application/chrome.exe","Chromium/Application/chrome.exe"];return[...o.flatMap(n=>n?r.map(t=>s(n,t)):[]),"C:/Program Files/Google/Chrome/Application/chrome.exe","C:/Program Files (x86)/Google/Chrome/Application/chrome.exe"]}function d(){return process.platform==="darwin"?"brew install --cask google-chrome":process.platform==="win32"?"download it from https://www.google.com/chrome/":"e.g. apt install chromium, or the vendor .deb from google.com/chrome"}function F(){return new Error(h(d()))}export{F as chromeMissingError,S as findChrome,P as isAttachedProfile,v as needsNoSandbox,G as needsShmFallback};
@@ -0,0 +1,29 @@
1
+ export interface TabClaim {
2
+ key: string;
3
+ isolated: boolean;
4
+ }
5
+ export interface BrowserConfig {
6
+ readonly profileDir: string;
7
+ readonly attached: boolean;
8
+ readonly chromePath?: string;
9
+ readonly tab: TabClaim | null;
10
+ readonly tabReuse: boolean;
11
+ readonly runSlug?: string;
12
+ readonly window: "offscreen" | "normal";
13
+ readonly cdpUrl?: string;
14
+ readonly cdpStrict: boolean;
15
+ readonly extension: boolean;
16
+ readonly home: string;
17
+ }
18
+ export interface BrowserConfigOptions {
19
+ profileDir?: string;
20
+ attached?: boolean;
21
+ chromePath?: string;
22
+ tab?: string;
23
+ runSlug?: string;
24
+ window?: "offscreen" | "normal";
25
+ cdpUrl?: string;
26
+ cdpStrict?: boolean;
27
+ extension?: boolean;
28
+ }
29
+ export declare function browserConfigFromEnv(opts?: BrowserConfigOptions, env?: NodeJS.ProcessEnv): BrowserConfig;
@@ -0,0 +1 @@
1
+ import{join as o}from"node:path";import{computerUseHome as n}from"../home.js";function c(e){const r=/^(name|ctx):(.+)$/.exec(e??"");if(!r)return null;const t=r[2].trim().toLowerCase();return/^[a-z0-9][a-z0-9-]{0,31}$/.test(t)?{key:t,isolated:r[1]==="ctx"}:null}function a(e={},r=process.env){const t=e.tab??r.COMPUTER_USE_TAB,i=e.cdpUrl??(r.COMPUTER_USE_CDP_URL?.trim()||void 0);return Object.freeze({profileDir:e.profileDir??U(r),attached:e.attached??r.COMPUTER_USE_ATTACHED==="1",chromePath:e.chromePath??(r.COMPUTER_USE_CHROME?.trim()||void 0),tab:c(t),tabReuse:t==="reuse",runSlug:e.runSlug??(r.COMPUTER_USE_RUN_SLUG?.trim()||void 0),window:e.window??(r.COMPUTER_USE_WINDOW==="offscreen"?"offscreen":"normal"),cdpUrl:i,cdpStrict:e.cdpStrict??(e.cdpUrl!==void 0||r.COMPUTER_USE_CDP_STRICT==="1"),extension:e.extension??r.COMPUTER_USE_BROWSER?.trim()==="extension",home:n(r)})}function U(e=process.env){return e.COMPUTER_USE_PROFILE_DIR||o(n(e),"chrome")}export{a as browserConfigFromEnv};
@@ -0,0 +1,3 @@
1
+ export declare function readDevToolsPort(profileDir: string): number | null;
2
+ export declare function pingDevTools(port: number, timeoutMs?: number): Promise<boolean>;
3
+ export declare function waitForDevToolsPort(profileDir: string, timeoutMs: number, gaveUp: () => boolean): Promise<number | null>;
@@ -0,0 +1,2 @@
1
+ import{readFileSync as s}from"node:fs";import{join as l}from"node:path";const a=r=>new Promise(o=>setTimeout(o,r));function c(r){try{const o=s(l(r,"DevToolsActivePort"),"utf8").split(`
2
+ `)[0]?.trim(),t=o?Number.parseInt(o,10):Number.NaN;return Number.isInteger(t)&&t>0&&t<65536?t:null}catch{return null}}async function f(r,o=700){const t=new AbortController,n=setTimeout(()=>t.abort(),o);try{return(await fetch(`http://127.0.0.1:${r}/json/version`,{signal:t.signal})).ok}catch{return!1}finally{clearTimeout(n)}}async function p(r,o,t){const n=Date.now()+o;let e=20;for(;;){const i=c(r);if(i!==null)return i;if(t()||Date.now()>=n)return null;await a(e),e=Math.min(150,Math.round(e*1.6))}}export{f as pingDevTools,c as readDevToolsPort,p as waitForDevToolsPort};
@@ -0,0 +1,22 @@
1
+ import type { Browser, Page } from "playwright-core";
2
+ import type { ActResult, VerbSpec } from "../types.ts";
3
+ export declare const DOWNLOAD_VERB: VerbSpec;
4
+ export declare class BrowserDownloads {
5
+ private watches;
6
+ private saving;
7
+ private root;
8
+ private source;
9
+ private bridge;
10
+ private connect;
11
+ private snapshot;
12
+ arm(browser: Browser, page: Page): Promise<ActResult>;
13
+ private markSource;
14
+ private stuckHint;
15
+ private inPlace;
16
+ private persist;
17
+ private refresh;
18
+ private applyNativeReceipt;
19
+ query(id: string, timeoutMs?: number, stop?: boolean, browser?: Browser): Promise<ActResult>;
20
+ private result;
21
+ close(): Promise<void>;
22
+ }
@@ -0,0 +1 @@
1
+ import{randomUUID as D}from"node:crypto";import{copyFile as E,mkdir as F,stat as I,unlink as O}from"node:fs/promises";import{hostname as $}from"node:os";import{basename as p,dirname as A,join as b,resolve as T}from"node:path";import{Type as u}from"typebox";import{browserArtifactsDir as v}from"./cdp-connection.js";import{SAVE_PICKER_REGISTRY as B}from"./save-picker.js";import{readableFrames as C}from"../chromium/frames.js";import{computerUseHome as _}from"../home.js";const G={kind:"download",readOnly:!0,schema:u.Object({kind:u.Literal("download"),id:u.Optional(u.String({description:"Watch or download ID returned by an earlier download call; omit to arm a new watch BEFORE clicking or pressing the control that downloads."})),timeoutMs:u.Optional(u.Integer({minimum:0,maximum:15e3,description:"Query wait budget, 0\u201315000 ms (default 10000). Timeout keeps the ID and returns the current state; query again later."})),stop:u.Optional(u.Boolean({description:"Stop collecting new downloads for this watch; existing downloads keep progressing and remain queryable."}))},{description:`Return a structured browser download receipt. First call without id to watch THIS tab, then trigger the download through an interaction verb, then query with id. Results include state, downloads with IDs, final filename/path, host and failure reason; ambiguous candidates are never silently selected. Files completed on this machine remain usable after the session ends. A query that runs out of budget while a native save panel is likely waiting for a person says so in hint. On tabs this session opened, a page's own save dialog (showSaveFilePicker) never opens: the file arrives here as an ordinary download marked source "save-picker", in the browser's download folder rather than where the page would have asked.`})},R=(w,e,t)=>w.send(e,t),q=w=>new Promise(e=>setTimeout(e,w));class K{watches=new Map;saving=new Set;root=null;source=null;bridge=!1;async connect(e){if(this.root&&this.source===e)return;await this.root?.detach().catch(()=>{}),this.root=null;const t=await e.newBrowserCDPSession();try{const o=await t.send("Browser.getVersion");this.bridge=o.product==="Chrome/mirasim-bridge",this.root=t,this.source=e,e.once("disconnected",()=>{for(const r of this.watches.values())r.page.context().browser()===e&&r.stop()})}catch(o){throw await t.detach().catch(()=>{}),o}}async snapshot(e,t){if(!this.root)throw new Error("The original browser connection is unavailable");return await R(this.root,"Mirasim.downloads",{since:e,until:t})}async arm(e,t){if(this.watches.size>=100)return{ok:!1,note:"This session already has 100 download watches; start a new session before arming more."};await this.connect(e);const o=new Date(Date.now()-1e3).toISOString(),r=this.bridge?await this.snapshot(o):null;for(const s of this.watches.values())s.page===t&&s.stop();const a={id:`watch-${D()}`,page:t,since:o,before:new Set(r?.items.map(s=>s.id)),host:r?.host??$(),receipts:[],native:this.bridge,session:null,lastEvent:Date.now(),marks:new Set,stop:()=>{}};if(this.bridge){const s=await t.context().newCDPSession(t);a.session=s;const i=d=>{a.lastEvent=Date.now();const h={id:d.guid,state:"in_progress",url:d.url,filename:d.suggestedFilename,path:null,host:a.host};a.receipts.push(h),this.markSource(a,h)},l=d=>{const h=a.receipts.find(m=>m.id===d.guid);h&&d.state==="canceled"&&(h.state="failed",h.error="Browser canceled the download")};s.on("Page.downloadWillBegin",i),s.on("Page.downloadProgress",l),a.stop=()=>{a.until??=new Date().toISOString(),s.off("Page.downloadWillBegin",i),s.off("Page.downloadProgress",l),t.off("close",a.stop)};try{await s.send("Page.enable")}catch(d){throw a.stop(),await s.detach().catch(()=>{}),d}t.once("close",a.stop)}else{const s=await t.context().newCDPSession(t);a.session=s;const i=this.root,l=new Map,d=new Set,h=n=>{d.add(n.frame.id);for(const c of n.childFrames??[])h(c)},m=n=>{d.add(n.frameId)},y=n=>{d.add(n.frame.id)},P=n=>{d.delete(n.frameId)},k=n=>{if(!d.has(n.frameId))return;if(a.lastEvent=Date.now(),!/^[A-Za-z0-9_-]{1,128}$/.test(n.guid)){a.receipts.push({id:`download-${D()}`,state:"failed",url:n.url,filename:n.suggestedFilename,path:null,host:a.host,error:"The browser supplied an invalid download identifier"});return}const c={id:`download-${n.guid}`,state:"in_progress",url:n.url,filename:n.suggestedFilename,path:null,host:a.host};a.receipts.push(c),this.markSource(a,c);let f;const g=new Promise(x=>{f=x});l.set(n.guid,{receipt:c,finish:f}),this.saving.add(g),g.finally(()=>this.saving.delete(g))},S=n=>{const c=l.get(n.guid);if(!(!c||n.state==="inProgress"))if(l.delete(n.guid),n.state==="canceled")c.receipt.state="failed",c.receipt.error="Browser canceled the download",c.finish();else if(n.state==="completed"){const f=b(v(),n.guid);(!!n.filePath&&T(A(n.filePath))!==T(v())?this.inPlace(n.filePath,c.receipt):this.persist(n.filePath||f,c.receipt)).then(async()=>{c.receipt.state==="complete"&&(!n.filePath||n.filePath===f)&&await O(f).catch(()=>{})}).finally(c.finish)}else l.set(n.guid,c)};s.on("Page.frameAttached",m),s.on("Page.frameNavigated",y),s.on("Page.frameDetached",P),i.on("Browser.downloadWillBegin",k),i.on("Browser.downloadProgress",S),a.stop=()=>{a.until??=new Date().toISOString(),i.off("Browser.downloadWillBegin",k),s.off("Page.frameAttached",m),s.off("Page.frameNavigated",y),s.off("Page.frameDetached",P),t.off("close",a.stop)},a.dispose=()=>{i.off("Browser.downloadProgress",S);for(const{receipt:n,finish:c}of l.values())n.state="unavailable",n.error="The download connection closed before completion",c();l.clear()};try{await s.send("Page.enable"),h((await s.send("Page.getFrameTree")).frameTree);const{targetInfo:n}=await s.send("Target.getTargetInfo"),{browserContextIds:c}=await i.send("Target.getBrowserContexts"),f=n.browserContextId;await i.send("Browser.setDownloadBehavior",{behavior:"allowAndName",downloadPath:v(),eventsEnabled:!0,...f&&c.includes(f)?{browserContextId:f}:{}})}catch(n){throw a.stop(),a.dispose(),await s.detach().catch(()=>{}),n}t.once("close",a.stop)}return this.watches.set(a.id,a),this.result(a,"armed",!1)}markSource(e,t){if(!t.url.startsWith("blob:"))return;const o=s=>s.evaluate(([i,l])=>!!globalThis[Symbol.for(i)]?.has(l),[B,t.url]).catch(()=>!1);let r;const a=Promise.race([Promise.all(C(e.page).map(o)).then(s=>{s.includes(!0)&&(t.source="save-picker")}),new Promise(s=>{r=setTimeout(s,1e3)})]).finally(()=>{clearTimeout(r),e.marks.delete(a)});e.marks.add(a)}async stuckHint(e,t,o){if(t.some(i=>i.state==="in_progress"&&i.awaitingLocation))return`Chrome is asking where to save this file (its "Ask where to save each file" setting): a native save panel is open in the user's Chrome, out of the browser driver's reach. Bind Google Chrome with the macOS driver (for a background tab the panel is a window of its own titled Save) and press Save or Cancel in that panel, or ask the user to turn the setting off; then query this ID again.`;if(o!=="armed"||e.page.isClosed())return;const r=i=>i.evaluate(l=>typeof globalThis.showSaveFilePicker=="function"&&!globalThis[Symbol.for(l)],B).catch(()=>!1);let a;return await Promise.race([Promise.all(C(e.page).map(r)).then(i=>i.includes(!0)),new Promise(i=>{a=setTimeout(()=>i(!1),1e3)})]).finally(()=>clearTimeout(a))?"No download began. This tab keeps the browser's real save dialog (it is not one this session opened), so if the control saves through the page's own save dialog (showSaveFilePicker), a native save panel may be waiting for a person: bind the browser's app with the macOS driver and press Save or Cancel there. If the tab was not its window's active tab, the browser refused the panel and the page got AbortError instead: make the tab active and trigger the save again.":void 0}async inPlace(e,t){try{await I(e),t.filename=p(e.replace(/\\/g,"/"))||t.filename,t.path=e,t.state="complete"}catch(o){t.state="failed",t.error=o instanceof Error?o.message:String(o)}}async persist(e,t){try{const o=b(_(),"downloads",t.id),r=p(t.filename.replace(/\\/g,"/")).replace(/[\x00-\x1f:<>"|?*]/g,"_").slice(0,180);t.filename=r&&r!=="."&&r!==".."?r:"download",await F(o,{recursive:!0,mode:448});const a=b(o,t.filename);await E(e,a),await I(a),t.path=a,t.state="complete"}catch(o){t.state="failed",t.error=o instanceof Error?o.message:String(o)}}async refresh(e){if(!e.native||!e.receipts.length)return;if(!this.bridge)throw new Error("This watch belongs to the original extension browser");const{items:t}=await this.snapshot(e.since,e.until);for(const o of e.receipts){if(o.state==="ambiguous")continue;const r=t.filter(i=>!e.before.has(i.id)&&(i.finalUrl===o.url||i.url===o.url)),a=e.receipts.filter(i=>i.url===o.url);if(r.length>1||a.length>1){o.state="ambiguous",o.path=null,delete o.nativeId,o.error="Multiple downloads match this page's event; no file was selected",o.candidates=r.map(i=>({nativeId:i.id,path:i.filename,state:i.state}));continue}const s=r[0];if(o.nativeId!==void 0&&s?.id!==o.nativeId){o.state="unavailable",o.path=null,o.error="The identified download is no longer in Chrome's ledger";continue}s&&(this.applyNativeReceipt(o,s),s.state==="in_progress"&&!s.filename?o.awaitingLocation=!0:delete o.awaitingLocation)}}applyNativeReceipt(e,t){e.nativeId=t.id,e.filename=p(t.filename.replace(/\\/g,"/"))||e.filename,e.state=t.state==="interrupted"?"failed":t.state==="complete"?"complete":"in_progress",e.path=e.state==="complete"?t.filename:null,delete e.error,t.error&&(e.error=t.error),e.state==="complete"&&!t.exists&&(e.state="failed",e.path=null,e.error="The browser reports that the downloaded file no longer exists")}async query(e,t=1e4,o=!1,r){const a=this.watches.get(e)??[...this.watches.values()].find(i=>i.receipts.some(l=>l.id===e));if(!a)return{ok:!1,note:`Unknown download ID: ${e}. Use an ID returned by this session.`};o&&a.stop();const s=Date.now()+t;for(;;){try{a.native&&r?.isConnected()&&await this.connect(r),await this.refresh(a),await Promise.all(a.marks)}catch(d){return this.result(a,"unavailable",!1,e,`Download status unavailable: ${d instanceof Error?d.message:String(d)}`)}const i=e===a.id?a.receipts:a.receipts.filter(d=>d.id===e),l=i.length>1?"ambiguous":i[0]?.state??(a.page.isClosed()||a.until?"failed":"armed");if((l==="complete"||l==="failed"||l==="ambiguous"||l==="unavailable")&&Date.now()-a.lastEvent>=250)return this.result(a,l,!1,e);if(Date.now()>=s)return this.result(a,l,!0,e,void 0,await this.stuckHint(a,i,l));await q(Math.min(100,s-Date.now()))}}result(e,t,o,r=e.id,a,s){const i=r===e.id?e.receipts:e.receipts.filter(h=>h.id===r),l=a?i.map(h=>({...h,state:"unavailable",path:null,error:a})):i,d=i.filter(h=>h.source==="save-picker").length;return{ok:!0,value:{id:r,state:t,host:e.host,timedOut:o,downloads:structuredClone(l),...a?{error:a}:t==="failed"&&!i.length?{error:"The watch stopped or its tab closed before a download began"}:{},...s?{hint:s}:{}},note:`download ${r}: ${t}${o?"; wait budget ended \u2014 query this ID again":""}`+(d?`; ${d===1?"this file":`${d} of these files`} came from the page's own save dialog, saved as an ordinary download instead \u2014 it is at the path shown, not where the page would have asked`:"")+(s?`; ${s}`:"")}}async close(){for(const t of this.watches.values())t.stop();let e;await Promise.race([Promise.all([...this.saving]),new Promise(t=>{e=setTimeout(t,5e3)})]),e&&clearTimeout(e);for(const t of this.watches.values())t.dispose?.(),await t.session?.detach().catch(()=>{});await this.root?.detach().catch(()=>{}),this.root=null,this.source=null}}export{K as BrowserDownloads,G as DOWNLOAD_VERB};
@@ -0,0 +1,13 @@
1
+ import { BrowserDriverTabs } from "./driver-tabs.ts";
2
+ export declare abstract class BrowserDriverConnect extends BrowserDriverTabs {
3
+ private endpoints;
4
+ protected ensure(): Promise<void>;
5
+ private extensionGoneMessage;
6
+ private attachRefusal;
7
+ protected noteBridgeSession(session: string | null): boolean;
8
+ private ensureInner;
9
+ private publishClaim;
10
+ private wireContext;
11
+ private installKeepInTab;
12
+ private reclaimTab;
13
+ }
@@ -0,0 +1 @@
1
+ import{connectBrowserCdp as o}from"./cdp-connection.js";import{acquireManagedBrowser as u,openBlankTab as b,publishTabIdentity as g}from"./acquire.js";import{acquireBridge as m,bridgeJustStarted as x,extensionHeldElsewhere as E,waitForExtension as I}from"./bridge/acquire-bridge.js";import{claimIsolatedTab as h,claimNamedTab as T,claimSessionTab as y,newBackgroundPage as v,pageTargetInfo as k}from"./tabs.js";import{bridgeBrowserSession as c}from"./bridge-tabs.js";import{BROWSER_SWITCHED_NOTE as S,isBridgeExtensionGone as D,msgBridgeExtensionElsewhere as C,msgBridgeExtensionGone as l,msgBridgeJustStarted as B,msgEndpointUnavailable as _,REOPENED_NOTE as d,SURFACE_REIDENTIFIED_NOTE as N,TAB_DETACHED_NOTE as O,TAB_REOPENED_AFTER_SELF_CLOSE_NOTE as P,TAB_REOPENED_NOTE as R,tabRefusedNote as U}from"./messages.js";import{SurfaceUnreadableError as f}from"../types.js";import{KEEP_IN_TAB_JS as w}from"./keep-in-tab.js";import{ActionJournal as A}from"./journal.js";import{captureEvidenceShot as $}from"../chromium/shot.js";import{ensureRefSelector as F}from"../chromium/page-refs.js";import{REIDENTIFIED_WHY as W}from"./driver-core.js";import{BrowserDriverTabs as G}from"./driver-tabs.js";class et extends G{endpoints(){return this.explicit?[this.cdpUrl]:[...new Set([this.cdpUrl,this.fallbackCdpUrl])]}async ensure(){try{await this.ensureInner()}catch(e){throw D(e)?new f(await this.extensionGoneMessage()):e}}async extensionGoneMessage(){if(!this.cfg.extension||this.cfg.cdpUrl)return l();const e=await E(this.cfg.home);return e?C(e):x(this.cfg.home,9e4)?B():l()}async attachRefusal(e){if(!e||e==="about:blank"||!this.browser)return null;for(let i=0;i<2;i++){i&&await new Promise(t=>setTimeout(t,500));try{const t=await this.browser.newBrowserCDPSession();try{const a=(await t.send("Mirasim.attachRefusals")).refusals?.find(r=>r.url===e);if(a)return a.reason}finally{await t.detach().catch(()=>{})}}catch{return null}}return null}noteBridgeSession(e){if(!e)return!1;const i=this.bridgeSession;return this.bridgeSession=e,!i||i===e?!1:(this.resetObservationState(),this.active=void 0,this.appendNote(S),!0)}async ensureInner(){if(this.browser?.isConnected()){if(!this.active)return;const t=await this.ownPages();if(!t.length&&this.active.isClosed()){if(this.closedItsLastTab||this.noteBridgeSession(await c(this.browser)))return;this.resetObservationState();const s=this.active?.url()??"";this.active=this.attached?await this.openOwnWindow():this.tab?await this.reclaimTab():await v(this.context),this.claim(this.active);const a=!this.active.url()||this.active.url()==="about:blank";this.pendingNote=a?this.wokeAfterSelfClose?P:this.managedProfileDir&&!this.tab?d:this.explicit&&!this.managedProfileDir&&!this.attached?(r=>r?U(r):O)(await this.attachRefusal(s)):R:N,this.wokeAfterSelfClose=!1,await this.publishClaim()}else if(this.active.isClosed()){const s=t.filter(n=>n.url()!=="about:blank"),a=s[s.length-1]??t[t.length-1],r=await this.recognise(a).catch(()=>null);this.makeActive(a,r==="same"?W:"the tab it was on was closed")}return}if(this.browser&&(this.browser=null,this.managedProfileDir&&(this.pendingNote=d),this.resetObservationState(),this.active=void 0),this.cfg.extension&&!this.cfg.cdpUrl){const t=await m(this.cfg.home);if(this.cdpUrl=t.cdpUrl,!await I(this.cdpUrl,t.fresh?0:15e3,this.cfg.home))throw new f(await this.extensionGoneMessage())}const e=this.endpoints();let i;for(const t of e){try{this.browser=await o(t)}catch(s){if(i=s,await b(t))try{this.browser=await o(t)}catch(a){i=a}}if(this.browser){await F(this.browser),this.cdpUrl=t;break}}if(!this.browser&&!this.explicit){const t=await u(this.cfg.profileDir,this.cfg);this.browser=t.browser,this.cdpUrl=t.cdpUrl,this.managedProfileDir=t.profileDir,this.journal=new A(t.profileDir,()=>$(()=>this.page()),()=>{try{return this.page().url()}catch{return""}},this.cfg),this.attached=this.cfg.attached}if(!this.browser){const t=i instanceof Error?i.message:String(i??"unknown error");throw new Error(_(this.cdpUrl,t))}if(this.context=this.browser.contexts()[0]??await this.browser.newContext(),this.noteBridgeSession(await c(this.browser)),this.attached)this.active=await this.openOwnWindow();else if(this.tab?.isolated){const t=await h(this.browser,this.tab.key);this.active=t.page,this.ctxId=t.ctxId,this.pageCtxIds.set(t.page,t.ctxId),this.context=this.active.context()}else this.active=await y(this.context,this.tab?.key??null,this.cfg)??void 0;this.active&&this.claim(this.active),this.wireContext(),await this.installKeepInTab(this.active),await this.publishClaim()}async publishClaim(){if(!(!this.tab||!this.managedProfileDir||!this.active))try{const{targetId:e,ctxId:i}=await k(this.active);g(this.managedProfileDir,{key:this.tab.key,isolated:this.tab.isolated,ctxId:i,targetId:e})}catch{}}wireContext(){this.context.on("page",e=>{this.windowAppeared=!0,this.adoptWhenReal(e)}),this.context.on("dialog",e=>{(async()=>{const i=e.page();if(this.attached&&(!i||!await this.ownsPage(i).catch(()=>!1)))return;const t=e.type(),s=this.armedDialog;this.armedDialog=null;const a=s?s.accept:t==="alert"||t==="beforeunload";let r=!0;try{await(a?e.accept(s?.text):e.dismiss())}catch{r=!1}if(t==="beforeunload")return;const n=`a JS dialog appeared: \xAB${e.message().slice(0,120)}\xBB`,p=r?s?`${n} \u2014 ${a?"accepted":"dismissed"} as you asked`+(s.text!==void 0?` with text \xAB${s.text.slice(0,60)}\xBB`:""):`${n} \u2014 it was auto-${a?"accepted":"dismissed"}; if the flow needs a confirm, it did not happen`:`${n} \u2014 it was already answered before your ${a?"accept":"dismiss"} could land (another client shares this browser), so the page took an answer you did not choose. Check what the page did before acting on it`;this.appendNote(p)})()})}async installKeepInTab(e){this.keepInTabInstalled||(this.attached||(this.keepInTabInstalled=!0,await this.context.addInitScript(w).catch(()=>{this.keepInTabInstalled=!1})),await e?.evaluate(w).catch(()=>{}))}async reclaimTab(){const e=this.tab;if(!e.isolated)return T(this.context,e.key);const{page:i,ctxId:t}=await h(this.browser,e.key);return this.ctxId=t,this.pageCtxIds.set(i,t),i.context()!==this.context&&(this.context=i.context(),this.keepInTabInstalled=!1,this.wireContext(),await this.installKeepInTab(i)),i}}export{et as BrowserDriverConnect};
@@ -0,0 +1,50 @@
1
+ import { type Browser, type BrowserContext, type Page } from "playwright-core";
2
+ import { ChromiumPageDriver } from "../chromium/page-driver.ts";
3
+ import { type BrowserConfig, type BrowserConfigOptions, type TabClaim } from "./config.ts";
4
+ import { ActionJournal } from "./journal.ts";
5
+ import { BrowserDownloads } from "./downloads.ts";
6
+ export declare const ADOPTED_WHY = "a tab this session opened navigated to real content";
7
+ export declare const APPEARED_WHY = "a tab this session did not open appeared in the browser and reached real content";
8
+ export declare const REIDENTIFIED_WHY = "this session's own tab came back under a new id, because the connection to the browser dropped and returned";
9
+ export declare abstract class BrowserDriverCore extends ChromiumPageDriver {
10
+ protected readonly downloads: BrowserDownloads;
11
+ protected browser: Browser | null;
12
+ protected context: BrowserContext;
13
+ protected active: Page | undefined;
14
+ protected tabIds: WeakMap<Page, string>;
15
+ protected nextTabId: number;
16
+ protected managedProfileDir: string | null;
17
+ protected pendingNote: string | null;
18
+ protected armedDialog: {
19
+ accept: boolean;
20
+ text?: string;
21
+ } | null;
22
+ protected armedChooser: ((fc: {
23
+ setFiles(p: string[]): Promise<void>;
24
+ }) => void) | null;
25
+ protected keepInTabInstalled: boolean;
26
+ protected attached: boolean;
27
+ protected windowAppeared: boolean;
28
+ protected readonly tab: TabClaim | null;
29
+ protected ctxId: string | null;
30
+ protected readonly pageCtxIds: WeakMap<Page, string>;
31
+ protected readonly mine: WeakSet<Page>;
32
+ protected readonly savePickerPages: WeakSet<Page>;
33
+ protected readonly ownedTargetIds: Set<string>;
34
+ protected activeTargetId: string | null;
35
+ protected journal: ActionJournal | null;
36
+ protected closedItsLastTab: boolean;
37
+ protected bridgeSession: string | null;
38
+ protected wokeAfterSelfClose: boolean;
39
+ protected ownWindowId: number | null;
40
+ protected windowIds: WeakMap<Page, number>;
41
+ protected readonly explicit: boolean;
42
+ protected readonly fallbackCdpUrl: string;
43
+ protected readonly cfg: BrowserConfig;
44
+ constructor(cdpUrl?: string, fallbackCdpUrl?: string, cfgOpts?: BrowserConfigOptions, env?: NodeJS.ProcessEnv);
45
+ protected cdpUrl: string;
46
+ protected takePendingNote(): string | null;
47
+ protected appendNote(note: string): void;
48
+ protected makeActive(p: Page, why: string): void;
49
+ protected abstract rememberIdentity(p: Page): void;
50
+ }
@@ -0,0 +1 @@
1
+ import"playwright-core";import{ChromiumPageDriver as i}from"../chromium/page-driver.js";import{browserConfigFromEnv as s}from"./config.js";import"./acquire.js";import"./messages.js";import"./journal.js";import{BrowserDownloads as r}from"./downloads.js";const a="http://127.0.0.1:9222",v="a tab this session opened navigated to real content",I="a tab this session did not open appeared in the browser and reached real content",N="this session's own tab came back under a new id, because the connection to the browser dropped and returned";class k extends i{downloads=new r;browser=null;context;active;tabIds=new WeakMap;nextTabId=1;managedProfileDir=null;pendingNote=null;armedDialog=null;armedChooser=null;keepInTabInstalled=!1;attached=!1;windowAppeared=!1;tab;ctxId=null;pageCtxIds=new WeakMap;mine=new WeakSet;savePickerPages=new WeakSet;ownedTargetIds=new Set;activeTargetId=null;journal=null;closedItsLastTab=!1;bridgeSession=null;wokeAfterSelfClose=!1;ownWindowId=null;windowIds=new WeakMap;explicit;fallbackCdpUrl;cfg;constructor(e,t=a,n={},o=process.env){super(),this.cfg=s({...n,...e!==void 0?{cdpUrl:e}:{}},o),this.explicit=this.cfg.cdpStrict||this.cfg.extension&&!this.cfg.cdpUrl,this.tab=this.cfg.attached?null:this.cfg.tab,this.fallbackCdpUrl=t,this.cdpUrl=this.cfg.cdpUrl??t}cdpUrl;takePendingNote(){const e=this.pendingNote;return this.pendingNote=null,e}appendNote(e){this.pendingNote?this.pendingNote.includes(e)||(this.pendingNote=`${this.pendingNote}; ${e}`):this.pendingNote=e}makeActive(e,t){if(this.active===e)return;this.active=e,this.rememberIdentity(e),this.resetObservationState();const n=`the active tab changed (${t}) \u2014 nothing was lost, but refs from earlier observations belong to the previous tab, so re-observe before acting on them`;this.appendNote(n),this.journal?.publishAction("focus",{auto:!0},null,{ok:!0})}}export{v as ADOPTED_WHY,I as APPEARED_WHY,k as BrowserDriverCore,N as REIDENTIFIED_WHY};
@@ -0,0 +1,16 @@
1
+ import { type Page } from "playwright-core";
2
+ import { BrowserDriverCore } from "./driver-core.ts";
3
+ export declare abstract class BrowserDriverTabs extends BrowserDriverCore {
4
+ protected claim(p: Page): void;
5
+ private armSavePicker;
6
+ protected rememberIdentity(p: Page): void;
7
+ protected recognise(p: Page): Promise<"same" | "ours" | null>;
8
+ private ownsIsolatedPage;
9
+ protected adoptWhenReal(p: Page): void;
10
+ protected page(): Page;
11
+ protected openOwnWindow(): Promise<Page>;
12
+ protected ownsPage(p: Page): Promise<boolean>;
13
+ protected ownPages(): Promise<Page[]>;
14
+ protected visiblePages(): Promise<Page[]>;
15
+ protected tabId(p: Page): string;
16
+ }
@@ -0,0 +1 @@
1
+ import"playwright-core";import{userWantsVisible as f}from"./acquire.js";import{pageContextId as l,pageTargetInfo as h}from"./tabs.js";import{msgOwnWindowFailed as m,NO_TAB_OPEN_NOTE as u,NO_TAB_YET_NOTE as g}from"./messages.js";import{SAVE_PICKER_JS as d}from"./save-picker.js";import{SurfaceUnreadableError as c}from"../types.js";import{ADOPTED_WHY as n,APPEARED_WHY as w,BrowserDriverCore as I,REIDENTIFIED_WHY as v}from"./driver-core.js";const b=r=>new Promise(t=>setTimeout(t,r));class D extends I{claim(t){this.mine.add(t),this.rememberIdentity(t),this.armSavePicker(t)}armSavePicker(t){this.savePickerPages.has(t)||(this.savePickerPages.add(t),t.addInitScript(d).catch(()=>{}),t.evaluate(d).catch(()=>{}))}rememberIdentity(t){h(t).then(({targetId:e})=>{e&&(this.ownedTargetIds.add(e),this.active===t&&(this.activeTargetId=e))}).catch(()=>{})}async recognise(t){const{targetId:e}=await h(t).catch(()=>({targetId:""}));return!e||!this.ownedTargetIds.has(e)?null:(this.mine.add(t),this.armSavePicker(t),e===this.activeTargetId?"same":"ours")}async ownsIsolatedPage(t){let e=this.pageCtxIds.get(t);return e===void 0&&(e=await l(t).catch(()=>""),this.pageCtxIds.set(t,e)),e!==""&&e===this.ctxId}adoptWhenReal(t){const e=()=>{if(this.tab?.isolated){this.ownsIsolatedPage(t).then(s=>{s&&(this.claim(t),this.makeActive(t,n))}).catch(()=>{});return}if(this.tab){t.opener().then(s=>{s&&this.mine.has(s)&&(this.claim(t),this.makeActive(t,n))}).catch(()=>{});return}if(!this.attached){if(this.mine.has(t)){this.makeActive(t,n);return}this.recognise(t).then(s=>this.makeActive(t,s==="same"?v:s==="ours"?n:w)).catch(()=>this.makeActive(t,w));return}this.ownsPage(t).then(s=>{s&&this.makeActive(t,n)}).catch(()=>{})};if(t.url()!=="about:blank"){e();return}const i=s=>{s!==t.mainFrame()||t.isClosed()||t.url()==="about:blank"||(clearTimeout(a),t.off("framenavigated",i),e())},a=setTimeout(()=>t.off("framenavigated",i),1500);a.unref?.(),t.on("framenavigated",i)}page(){if(this.closedItsLastTab)throw new c(u);if(!this.active)throw new c(g);if(this.active.isClosed()){const t=this.context.pages().filter(i=>!i.isClosed()&&(this.tab?this.tab.isolated?this.pageCtxIds.get(i)===this.ctxId:this.mine.has(i):!this.attached||this.ownWindowId===null||this.windowIds.get(i)===this.ownWindowId)),e=t.filter(i=>i.url()!=="about:blank");this.makeActive(e[e.length-1]??t[t.length-1]??this.active,"the tab it was on was closed")}return this.active}async openOwnWindow(){const t=new Set(this.context.pages()),e=await this.browser.newBrowserCDPSession();try{await e.send("Target.createTarget",{url:"about:blank",newWindow:!0,background:!0})}finally{await e.detach().catch(()=>{})}let i;for(let s=0;s<40&&(i=this.context.pages().find(o=>!t.has(o)&&!o.isClosed()),!i);s++)await b(25);if(!i)throw new Error(m());const a=await this.context.newCDPSession(i);try{const{windowId:s}=await a.send("Browser.getWindowForTarget");this.ownWindowId=s,this.windowIds.set(i,s),this.cfg.window==="offscreen"&&!(this.managedProfileDir&&f(this.managedProfileDir))&&await a.send("Browser.setWindowBounds",{windowId:s,bounds:{left:-4e3,top:200}})}catch{}finally{await a.detach().catch(()=>{})}return i}async ownsPage(t){if(!this.attached||this.ownWindowId===null)return!0;let e=this.windowIds.get(t);if(e===void 0)try{const i=await this.context.newCDPSession(t);try{e=(await i.send("Browser.getWindowForTarget")).windowId}finally{await i.detach().catch(()=>{})}this.windowIds.set(t,e)}catch{return!1}return e===this.ownWindowId}async ownPages(){const t=this.context.pages().filter(i=>!i.isClosed());if(this.tab?.isolated){const i=[];for(const a of t)await this.ownsIsolatedPage(a)&&i.push(a);return i}if(this.tab)return t.filter(i=>this.mine.has(i));if(!this.managedProfileDir)return t.filter(i=>this.mine.has(i));if(!this.attached||this.ownWindowId===null)return t;const e=[];for(const i of t)await this.ownsPage(i)&&e.push(i);return e}async visiblePages(){return this.managedProfileDir||this.attached||this.tab?this.ownPages():this.context.pages().filter(t=>!t.isClosed())}tabId(t){let e=this.tabIds.get(t);return e||(e=`t${this.nextTabId++}`,this.tabIds.set(t,e)),e}}export{D as BrowserDriverTabs};
@@ -0,0 +1,13 @@
1
+ import { type Page } from "playwright-core";
2
+ import { type ActResult } from "../types.ts";
3
+ import { BrowserDriverConnect } from "./driver-connect.ts";
4
+ export declare abstract class BrowserDriverVerbs extends BrowserDriverConnect {
5
+ private pid;
6
+ private cdpSessions;
7
+ protected readonly overridden: Set<Page>;
8
+ perform(kind: string, a: Record<string, any>): Promise<ActResult>;
9
+ private browserPid;
10
+ protected cdpFor(page: Page): Promise<import("playwright-core").CDPSession>;
11
+ private ensureComposited;
12
+ private dispatch;
13
+ }
@@ -0,0 +1,2 @@
1
+ import{readFileSync as g}from"node:fs";import"playwright-core";import"../types.js";import{NO_TAB_OPEN_NOTE as m,NO_TAB_YET_NOTE as b}from"./messages.js";import{endpointFileFor as v,keepInBackground as y}from"./acquire.js";import{isFrontmost as k,returnFocusIfTaken as $}from"./focus-guard.js";import{BrowserDriverConnect as _}from"./driver-connect.js";import{landedNote as f,NAV_TIMEOUT_MS as c,navVerdict as d}from"./navigate.js";import{newBackgroundPage as w}from"./tabs.js";import{detectInterference as x,overlaySensitive as P}from"./overlay-guard.js";import{performUploadToRef as T,refuseMissingFiles as I}from"../chromium/actions.js";import{performRunJs as S}from"../chromium/run-js.js";import{withPageCapture as A}from"../chromium/shot.js";class z extends _{pid;cdpSessions=new WeakMap;overridden=new Set;async perform(i,t){if(i==="download"&&typeof t.id=="string")return await this.downloads.query(t.id,t.timeoutMs,t.stop,this.browser??void 0);if(i==="open_url"||i==="new_tab")this.wokeAfterSelfClose=this.closedItsLastTab,this.closedItsLastTab=!1;else if(this.closedItsLastTab)return{ok:!1,note:m};await this.ensure();let r=null;if(!this.active){if(i!=="open_url"&&i!=="new_tab")return{ok:!1,note:b};if(this.active=await w(this.context),this.claim(this.active),i==="new_tab"){if(r=this.tabId(this.active),!t.url)return{ok:!0,note:`new tab ${r} is now the target`};i="open_url"}}this.hookNavigation(),await this.ensureComposited();const s=await k(this.browserPid());if(this.lastPoint=null,this.cfg.extension&&P(i)){const o=await x(this.page());if(o){const h=`another extension or overlay is over the page right now \u2014 ${o}. Close it (dismiss the popup, click away from the autofill suggestion, switch off the interfering extension for this site), then re-observe and act. Acting now would land on the overlay, not the target.`;return this.journal?.publishAction(i,t,this.lastPoint,{ok:!1,note:h}),{ok:!1,note:h}}}let e;try{const o=this.page();e=await A(o,()=>this.page()!==o||o.isClosed()?Promise.resolve({ok:!1,note:"the target changed while waiting for capture \u2014 re-observe the intended tab before acting"}):this.dispatch(i,t))}catch(o){const h=o instanceof Error?o.message.split(`
2
+ `)[0]??"dispatch failed":String(o);throw this.journal?.publishAction(i,t,this.lastPoint,{ok:!1,note:h}),o}r&&(e={...e,note:`new tab ${r} is now the target; ${e.note??""}`}),this.journal?.publishAction(i,t,this.lastPoint,e),this.managedProfileDir&&!this.closedItsLastTab&&i!=="activate"&&await y(this.context,this.page(),this.managedProfileDir,this.cfg),this.windowAppeared&&(this.windowAppeared=!1,!s&&i!=="activate"&&await $(this.browserPid()));const n=this.takePendingNote();return n&&(e.note=e.note?`${n}. ${e.note}`:n),e}browserPid(){if(this.pid===void 0){this.pid=null;try{const i=JSON.parse(g(v(this.managedProfileDir),"utf8"));typeof i.pid=="number"&&(this.pid=i.pid)}catch{}}return this.pid}async cdpFor(i){let t=this.cdpSessions.get(i);return t||(t=await this.context.newCDPSession(i),this.cdpSessions.set(i,t)),t}async ensureComposited(){if(this.managedProfileDir)try{const i=await this.cdpFor(this.page()),{windowId:t,bounds:r}=await i.send("Browser.getWindowForTarget");if(r.windowState!=="minimized")return;await i.send("Browser.setWindowBounds",{windowId:t,bounds:{windowState:"normal"}}),this.cfg.window==="offscreen"&&await i.send("Browser.setWindowBounds",{windowId:t,bounds:{left:-4e3,top:200}})}catch{}}async dispatch(i,t){const r=this.page();switch(i){case"download":return t.stop||t.timeoutMs!==void 0?{ok:!1,note:"download stop/timeoutMs require an existing id; omit them to arm a watch"}:await this.downloads.arm(this.browser,r);case"upload":{const s=Array.isArray(t.paths)?t.paths:[],e=I(s);if(e)return e;if(t.ref){const o=await T(s,t.ref,()=>this.locate(t.ref,t.name,!1),()=>this.missing(t.ref,t.name));return!o.ok&&o.note?.includes("did not take the files")?{...o,note:`${o.note}; for a button that opens a chooser, drop ref and click the button next`}:o}this.armedChooser&&r.off("filechooser",this.armedChooser);const n=o=>{o.setFiles(s)};return this.armedChooser=n,r.once("filechooser",n),{ok:!0,note:`the next file chooser will receive ${s.length} file${s.length===1?"":"s"} \u2014 nothing has changed yet; click the control that opens it`}}case"dialog":return this.armedDialog={accept:t.answer==="accept",...typeof t.text=="string"?{text:t.text}:{}},{ok:!0,note:`the next JS dialog will be ${t.answer}ed \u2014 nothing on the page has changed yet; the answer is used when the action that raises the dialog runs`};case"open_url":try{return await r.goto(t.url,{waitUntil:"domcontentloaded",timeout:c}),{ok:!0,note:f("opened",t.url,r.url())}}catch(s){return d(s)}case"activate":return await r.bringToFront(),{ok:!0,note:"activated this browser tab and its window \u2014 foreground focus was requested explicitly"};case"run_js":return await S(()=>r,t.code);case"back":{const s=r.url();try{return await r.goBack({waitUntil:"commit",timeout:c})===null&&r.url()===s?{ok:!0,note:"no earlier page in this tab's history"}:{ok:!0,note:`went back to ${r.url()}`}}catch(e){return d(e)}}case"reload":{const s=r.url();try{return await r.reload({waitUntil:"domcontentloaded",timeout:c}),{ok:!0,note:f("reloaded",s,r.url())}}catch(e){return d(e)}}case"new_tab":{const s=await w(this.context);this.claim(s),this.active=s,this.resetObservationState();const e=this.tabId(s);if(typeof t.url=="string"&&t.url){try{await s.goto(t.url,{waitUntil:"domcontentloaded",timeout:c})}catch(n){const o=d(n);return{...o,note:`new tab ${e} is now the target; ${o.note??"its navigation did not finish"}`}}return{ok:!0,note:`new tab ${e} is now the target; ${f("opened",t.url,s.url())}`}}return{ok:!0,note:`new tab ${e} is now the target`}}case"viewport":{const s=await this.cdpFor(r);if(t.reset)return await s.send("Emulation.clearDeviceMetricsOverride"),this.overridden.delete(r),{ok:!0,note:"viewport override cleared \u2014 this tab is back to the real window size"};const e=Math.round(Number(t.width)||1200),n=Math.round(Number(t.height)||1700);await s.send("Emulation.setDeviceMetricsOverride",{width:e,height:n,deviceScaleFactor:0,mobile:!1}),this.overridden.add(r);const o=await r.evaluate("({ width: innerWidth, height: innerHeight })");return o.width!==e||o.height!==n?{ok:!1,note:`viewport ${e}x${n} did not take effect \u2014 the page reports ${o.width}x${o.height}`}:{ok:!0,note:`viewport of this tab set to ${e}x${n} \u2014 navigate AFTER this, not before, or a list already mounted keeps the rows it has; the size is the user's own again on viewport({reset:true}) or when this session ends`}}case"close_tab":{const s=(await this.ownPages()).filter(a=>!a.isClosed());if(t.all){if(typeof t.id=="string"&&t.id)return{ok:!1,note:"close_tab takes id OR all, not both \u2014 all closes every tab this session owns"};for(const a of s)await a.close().catch(()=>{});return this.closedItsLastTab=!0,{ok:!0,note:`closed ${s.length} tab${s.length===1?"":"s"} \u2014 every one this session opened or claimed; the user's own tabs were not touched. open_url or new_tab starts a fresh one`}}const e=typeof t.id=="string"&&t.id?t.id.replace(`${this.platform}/`,""):this.tabId(r);let n=s.find(a=>this.tabId(a)===e),o=!1;if(!n){const a=(await this.visiblePages()).find(p=>this.tabId(p)===e);if(!a)return{ok:!1,note:`no tab with id ${e} \u2014 call computer.surfaces() for current ids`};if(!t.confirmed)return{ok:!1,note:`${e} is the user's own tab, not this session's. Ask them in the conversation whether to close it \u2014 say which page \u2014 and call close_tab again with confirmed: true if they agree. Reading it and acting in it needs no permission; only closing does`};n=a,o=!0}const h=n===this.active;await n.close().catch(()=>{});const l=o?" \u2014 the user's own, closed on the agreement you reported":"";if(!h)return{ok:!0,note:`closed tab ${e}${l}; the target is unchanged`};const u=s.find(a=>a!==n&&!a.isClosed());return u?(this.active=u,this.resetObservationState(),{ok:!0,note:`closed tab ${e}${l}; the target moved to ${this.tabId(u)} \u2014 re-observe before acting on any earlier ref`}):(this.closedItsLastTab=!0,{ok:!0,note:`closed tab ${e}${l} \u2014 this session has no tab left; open_url or new_tab starts a fresh one`})}}return await this.dispatchPage(i,t)??{ok:!1,note:`unknown action kind ${i}`}}}export{z as BrowserDriverVerbs};
@@ -0,0 +1,38 @@
1
+ import type { Page } from "playwright-core";
2
+ import { type ActResult, type HeaderChanges, type OwnershipResource, type Snapshot, type TargetInfo, type ToolResult, type VerbSpec, type Verdict } from "../types.ts";
3
+ import { BrowserDriverVerbs } from "./driver-verbs.ts";
4
+ export declare class BrowserDriver extends BrowserDriverVerbs {
5
+ private unloadedSelection;
6
+ activeTarget(): Promise<string | null>;
7
+ ownershipResources(): Promise<OwnershipResource[]>;
8
+ prepareTarget(): Promise<void>;
9
+ protected page(): Page;
10
+ protected adoptWhenReal(page: Page): void;
11
+ protected previewTarget(): Promise<import("../types.ts").PreviewTarget | null>;
12
+ readonly platform = "browser";
13
+ turnEnded(info: {
14
+ sessionId: string;
15
+ turnId: string;
16
+ hookEventName: string;
17
+ keepBrowserTabs?: boolean;
18
+ }): Promise<void>;
19
+ extraVerbs(): VerbSpec[];
20
+ observeElements(opts?: {
21
+ prefix?: string;
22
+ forceImage?: boolean;
23
+ afterAction?: string;
24
+ }): Promise<ToolResult>;
25
+ observeImage(opts?: Parameters<BrowserDriverVerbs["observeImage"]>[0]): Promise<ToolResult>;
26
+ private noTabNote;
27
+ private noTabObservation;
28
+ protected staleRefCause(): string;
29
+ promptFragment(): string;
30
+ listTargets(): Promise<TargetInfo[]>;
31
+ protected noteBridgeSession(session: string | null): boolean;
32
+ actionResources(kind: string, args: Record<string, unknown>): Promise<OwnershipResource[]>;
33
+ focusTarget(id: string): Promise<ActResult>;
34
+ close(): Promise<void>;
35
+ protected journalVerdict(verdict: Verdict): void;
36
+ protected renderHeadLines(s: Snapshot, targets: TargetInfo[], targetsUnchanged?: boolean, changes?: HeaderChanges): string[];
37
+ protected blankSurfaceHint(): Promise<string>;
38
+ }
@@ -0,0 +1,3 @@
1
+ import{Type as i}from"typebox";import{SurfaceUnreadableError as d,text as h}from"../types.js";import{NO_TAB_OPEN_NOTE as c,NO_TAB_YET_NOTE as u}from"./messages.js";import{PASTE_VERB as b,SELECT_OPTION_VERB as p}from"../chromium/actions-value.js";import{SELECT_TEXT_VERB as w}from"../chromium/select-text.js";import{DIALOG_VERB as f,DRAG_VERB as g,UPLOAD_VERB as m}from"../chromium/actions.js";import{RUN_JS_CODE_DESC as v}from"../chromium/run-js.js";import{BrowserDriverVerbs as k}from"./driver-verbs.js";import{previewPage as y}from"../chromium/preview.js";import{withPageCapture as S}from"../chromium/shot.js";import{DOWNLOAD_VERB as T}from"./downloads.js";import{bridgeRequest as O,bridgeTabs as E,restoreBridgeTab as _}from"./bridge-tabs.js";const P=5e3,N=1500;async function R(l){if(!l.url())return"";let e;try{return await Promise.race([l.title().catch(()=>""),new Promise(t=>{e=setTimeout(()=>t(""),N),e.unref?.()})])}finally{e&&clearTimeout(e)}}class G extends k{unloadedSelection=null;async activeTarget(){return this.unloadedSelection?.id??super.activeTarget()}async ownershipResources(){return this.unloadedSelection?[{key:`chromium:${this.unloadedSelection.targetId}`,mode:"exclusive"}]:super.ownershipResources()}async prepareTarget(){const e=this.unloadedSelection;if(!e)return;const{page:t,viewport:s}=await _(this.browser,e.tabId,e.targetId,e.browserSession);this.tabIds.set(t,e.id),this.active=t,this.unloadedSelection=null,this.resetObservationState();const r=await t.evaluate("({ width: innerWidth, height: innerHeight })");if(r.width<=0||r.height<=0){const a=await this.perform("viewport",s??{});if(!a.ok)throw new d(`The restored tab has no usable viewport: ${a.note??"resize failed"}`);this.appendNote("Chrome restored this background tab with a zero-size viewport; applied a temporary viewport, cleared when this driver disconnects.")}this.appendNote("Chrome had unloaded this tab; restored the same tab in the background. Read its current state before continuing.")}page(){if(this.unloadedSelection)throw new d(`Tab ${this.unloadedSelection.id} has not been restored; no other page was observed or operated.`);return super.page()}adoptWhenReal(e){this.unloadedSelection||super.adoptWhenReal(e)}previewTarget(){return this.unloadedSelection||!this.browser?.isConnected()||!this.active||this.active.isClosed()?Promise.resolve(null):y(this.active,this.cdpUrl,"browser")}platform="browser";async turnEnded(e){if(this.attached||e.keepBrowserTabs===!0)return;const t=await this.ownPages().catch(()=>[]);for(const s of t)await s.close().catch(()=>{});t.length&&(this.closedItsLastTab=!0,this.resetObservationState())}extraVerbs(){return[{kind:"open_url",schema:i.Object({kind:i.Literal("open_url"),url:i.String({description:"Full URL to open. Only URLs that appeared on a page or were given in the task \u2014 never guess domains."})})},{kind:"activate",schema:i.Object({kind:i.Literal("activate")},{description:"Bring this browser tab and its window to the foreground. This changes the user's foreground window: explain the effect and obtain permission unless it is already authorized. focus() only selects the target. Embedded writes normally wait for the person to yield the window; activate() is an explicit foreground handoff, revoked by subsequent human input. Do not use it to bypass an input wait without the user handing over control."})},{kind:"back",schema:i.Object({kind:i.Literal("back")})},{kind:"reload",schema:i.Object({kind:i.Literal("reload")})},{kind:"new_tab",schema:i.Object({kind:i.Literal("new_tab"),url:i.Optional(i.String({description:"URL to open in it; omit for a blank tab"}))},{description:"Open a NEW tab of this session's own and make it the target \u2014 the current tab keeps its state, and both are listed as targets afterwards."})},{kind:"close_tab",schema:i.Object({kind:i.Literal("close_tab"),id:i.Optional(i.String({description:"Tab id from computer.surfaces(); omit to close the one you are on"})),all:i.Optional(i.Boolean({description:"Close every tab this session opened or claimed \u2014 the tidy-up at the end of a job. Never the user's own, whatever else is set. Refused together with id."})),confirmed:i.Optional(i.Boolean({description:"Only for a tab marked (user's): you ASKED the user in the conversation and they said yes. Never set it because closing would be convenient \u2014 the first refusal tells you to ask, and asking is the point."}))},{description:"Close a tab. This session's own always; one of the user's (marked (user's) in the listing) only after you asked them in the conversation and they agreed \u2014 then pass confirmed: true."})},{kind:"viewport",schema:i.Object({kind:i.Literal("viewport"),width:i.Optional(i.Number({description:"Viewport width in CSS pixels (default 1200)"})),height:i.Optional(i.Number({description:"Viewport height in CSS pixels \u2014 this is the one that decides how much of a long list exists (default 1700)"})),reset:i.Optional(i.Boolean({description:"Put the tab back to the real window size"}))},{description:"Give THIS tab a viewport of your own size, so a virtualized list (a feed, a search page, a long transcript) mounts more of itself per read. Set it BEFORE opening the page: rows mount against the viewport in effect at load."})},{kind:"run_js",readOnly:!0,schema:i.Object({kind:i.Literal("run_js"),code:i.String({description:v})},{description:"Ask the PAGE something no observation carries \u2014 read its data in bulk, its stored state, or an API it exposes. This is a stranger's site holding the user's logged-in session: read and compute here, act through the interaction verbs."})},g,f,m,T,p,w,b]}async observeElements(e={}){const t=await this.noTabNote();return t?this.noTabObservation(t,e.prefix):super.observeElements(e)}async observeImage(e={}){const t=await this.noTabNote();return t?this.noTabObservation(t,e.prefix):super.observeImage(e)}async noTabNote(){return this.closedItsLastTab?c:this.unloadedSelection?null:(await this.ensure(),this.active?null:u)}noTabObservation(e,t){const s=this.takePendingNote();return h(`${t?`${t}
2
+ `:""}${s?`NOTE: ${s}.
3
+ `:""}${e}.`,{verdict:"unknown"})}staleRefCause(){return"it predates a navigation, a reload or a tab switch"}promptFragment(){return"Grounding: observations are text lists of interactive elements with refs, for the CURRENT viewport only. Act by ref from the latest observation. When what you want is NOT in the list: if you can name it, w.find('<name>') searches EVERY control on the page and hands back a handle \u2014 act on it directly, the click scrolls it into view, so do not scroll first. Scroll only to discover what you cannot name. Observations may show only changes since the previous one; refs stay valid across observations (until navigation). Use w.text() to read full page text."}async listTargets(){await this.ensure();const{unloaded:e,browserSession:t}=!this.tab&&!this.attached&&!this.managedProfileDir?await E(this.browser):{unloaded:[],browserSession:null};this.noteBridgeSession(t);const s=new Set(await this.ownPages()),r=await this.visiblePages(),a=await Promise.all(r.map(async o=>({id:this.tabId(o),kind:"tab",title:await R(o)||o.url(),active:!this.unloadedSelection&&o===this.active,own:s.has(o)})));for(const o of e){const n=`unloaded:${o.tabId}:${o.browserSession}`;a.push({id:n,kind:"tab",title:`${o.title||o.url} [unloaded \u2014 use this handle to restore in background]`,active:this.unloadedSelection?.id===n,own:!1,unloaded:!0})}return a}noteBridgeSession(e){const t=super.noteBridgeSession(e);return t&&(this.unloadedSelection=null),t}async actionResources(e,t){if(e!=="close_tab")return[];const s=typeof t.id=="string"?t.id.replace(/^browser\//,""):null,r=t.all?await this.ownPages():s?(await this.visiblePages()).filter(a=>this.tabId(a)===s):[this.page()];return Promise.all(r.filter(a=>!a.isClosed()).map(a=>this.pageOwnership(a)))}async focusTarget(e){await this.ensure();const t=/^unloaded:(\d+):(.*)$/.exec(e);if(t&&!this.tab&&!this.attached&&!this.managedProfileDir&&this.browser.version()==="mirasim-bridge"){const r=Number(t[1]),a=t[2],{targetId:o}=await O(this.browser,"Mirasim.tabTarget",{tabId:r,browserSession:a});return this.unloadedSelection={id:e,tabId:r,targetId:o,browserSession:a},this.closedItsLastTab=!1,{ok:!0,note:"Selected unloaded tab; restoration follows target ownership."}}let s={ok:!1,note:`no target with id ${e}; call computer.surfaces() for current ids`};for(const r of await this.visiblePages())if(!(r.isClosed()||this.tabId(r)!==e)){this.unloadedSelection=null,this.active=r,this.closedItsLastTab=!1,s={ok:!0};break}return this.journal?.publishAction("focus",{target:e},null,s),s}async close(){this.unloadedSelection=null,await this.downloads.close(),await this.journal?.drain(P);for(const e of this.overridden)e.isClosed()||await S(e,()=>this.cdpFor(e).then(t=>t.send("Emulation.clearDeviceMetricsOverride"))).catch(()=>{});if(this.overridden.clear(),!this.managedProfileDir&&!this.tab&&this.browser)for(const e of await this.ownPages().catch(()=>[]))e.url()==="about:blank"&&await e.close().catch(()=>{});await this.browser?.close().catch(()=>{}),this.browser=null,this.resetObservationState(),this.resetPageState(),await this.disconnected()}journalVerdict(e){this.journal?.publishVerdict(e)}renderHeadLines(e,t,s=!1,r){const a=super.renderHeadLines(e,t,s,r);if(e.url==="about:blank"&&e.elements.length===0&&!e.text){const o=t.some(n=>n.title!=="about:blank");a.push(o?"NOTE: this tab is blank \u2014 your page may be in another tab; computer.surfaces() lists it \u2014 act through its handle.":"NOTE: this tab is blank and nothing has been opened in it \u2014 navigate (openUrl({url})) instead of observing again.")}return a}async blankSurfaceHint(){try{return this.page().url()!=="about:blank"?"":(await this.ownPages()).some(t=>t!==this.page()&&t.url()!=="about:blank")?"the active tab is a blank page \u2014 your page may be in another tab; computer.surfaces() lists it \u2014 act through its handle":""}catch{return""}}}export{G as BrowserDriver};
@@ -0,0 +1,2 @@
1
+ export declare function isFrontmost(browserPid: number | null): Promise<boolean>;
2
+ export declare function returnFocusIfTaken(browserPid: number | null): Promise<void>;
@@ -0,0 +1 @@
1
+ import{execFile as s}from"node:child_process";function r(n){return new Promise(i=>{s("/usr/bin/lsappinfo",n,{timeout:700},(t,e)=>{i(t?null:e)})})}async function o(n){const i=await r(["info","-only","pid",n]),t=Number(i?.match(/"pid"\s*=\s*(\d+)/)?.[1]);return Number.isFinite(t)?t:null}async function f(n){if(process.platform!=="darwin"||!n)return!1;const i=(await r(["front"]))?.trim().replace(/:$/,"");return!!i&&await o(i)===n}async function l(n){if(process.platform!=="darwin"||!n)return;const t=(await r(["visibleProcessList"]))?.match(/ASN:[^\s:"]*(?:-"[^"]*")?/g)?.map(e=>e.replace(/-"[^"]*"$/,""))??[];if(!(t.length<2)&&await o(t[0])===n)for(const e of t.slice(1)){if(await o(e)===n)continue;const a=(await r(["info","-only","bundleID",e]))?.match(/"CFBundleIdentifier"="([^"]+)"/)?.[1];if(!a)return;await new Promise(u=>{s("/usr/bin/open",["-b",a],{timeout:1500},()=>u())});return}}export{f as isFrontmost,l as returnFocusIfTaken};
@@ -0,0 +1,21 @@
1
+ import type { ActResult } from "../types.ts";
2
+ import { type BrowserConfig } from "./config.ts";
3
+ export declare class ActionJournal {
4
+ private readonly profileDir;
5
+ private readonly takeShot;
6
+ private readonly currentUrl;
7
+ private readonly cfg;
8
+ private chain;
9
+ private shotSeq;
10
+ constructor(profileDir: string, takeShot: () => Promise<Buffer | null>, currentUrl: () => string, cfg?: BrowserConfig);
11
+ publishAction(kind: string, a: Record<string, any>, point: {
12
+ x: number;
13
+ y: number;
14
+ } | null, res: ActResult): void;
15
+ publishVerdict(verdict: string): void;
16
+ drain(ms: number): Promise<void>;
17
+ private writeJournalLine;
18
+ private appendActionLine;
19
+ private nextShotSeq;
20
+ private pruneShots;
21
+ }
@@ -0,0 +1,2 @@
1
+ import{appendFile as u,mkdir as s,readdir as c,rm as l,stat as p,writeFile as a}from"node:fs/promises";import{dirname as m,join as h}from"node:path";import{actionsFileFor as g,shotsDirFor as y}from"./acquire.js";import{browserConfigFromEnv as w}from"./config.js";const S=256*1024,f=500;function x(){const n=process.env.COMPUTER_USE_SHOTS_KEEP?.trim();if(!n)return f;const i=Number.parseInt(n,10);return Number.isInteger(i)&&i>=0?i:f}class _{profileDir;takeShot;currentUrl;cfg;chain=Promise.resolve();shotSeq=null;constructor(i,t,r,e=w()){this.profileDir=i,this.takeShot=t,this.currentUrl=r,this.cfg=e}publishAction(i,t,r,e){const o={ts:Date.now(),kind:i,url:this.currentUrl()};typeof t.x=="number"&&typeof t.y=="number"?(o.x=t.x,o.y=t.y):r&&(o.x=r.x,o.y=r.y),typeof t.name=="string"&&t.name&&(o.name=t.name),i==="set_text"&&typeof t.text=="string"&&(o.text=t.text),i==="keys"&&typeof t.chord=="string"&&(o.chord=t.chord),i==="scroll"&&(typeof t.direction=="string"&&(o.direction=t.direction),typeof t.to=="string"&&(o.to=t.to),typeof t.into=="string"&&(o.into=t.into)),i==="focus"&&typeof t.target=="string"&&(o.target=t.target),i==="focus"&&t.auto===!0&&(o.auto=!0),e.journal&&Object.assign(o,e.journal),o.ok=e.ok,!e.ok&&e.note&&(o.note=e.note),this.chain=this.chain.then(()=>this.writeJournalLine(o)).catch(()=>{})}publishVerdict(i){const t={ts:Date.now(),kind:"verdict",verdict:i,url:this.currentUrl()};this.chain=this.chain.then(()=>this.appendActionLine(t)).catch(()=>{})}async drain(i){let t;try{await Promise.race([this.chain.catch(()=>{}),new Promise(r=>{t=setTimeout(r,i)})])}finally{t&&clearTimeout(t)}}async writeJournalLine(i){const t=x();if(t>0){const r=await this.takeShot();if(r){const e=y(this.profileDir,this.cfg);await s(e,{recursive:!0});const o=await this.nextShotSeq(e);await a(h(e,`${o}.jpg`),r).then(()=>!0,()=>!1)&&(i.shot=`shots/${o}.jpg`),await this.pruneShots(e,t)}}await this.appendActionLine(i)}async appendActionLine(i){const t=g(this.profileDir,this.cfg);await s(m(t),{recursive:!0}),await p(t).then(e=>e.size).catch(()=>0)>S&&await a(t,""),await u(t,`${JSON.stringify(i)}
2
+ `)}async nextShotSeq(i){if(this.shotSeq===null){let t=0;for(const r of await c(i).catch(()=>[])){const e=/^(\d+)\.jpg$/.exec(r);e&&(t=Math.max(t,Number(e[1])))}this.shotSeq=t}return++this.shotSeq}async pruneShots(i,t){const r=[];for(const e of await c(i).catch(()=>[])){const o=/^(\d+)\.jpg$/.exec(e);o&&r.push(Number(o[1]))}r.sort((e,o)=>e-o);for(const e of r.slice(0,Math.max(0,r.length-t)))await l(h(i,`${e}.jpg`),{force:!0}).catch(()=>{})}}export{_ as ActionJournal};
@@ -0,0 +1 @@
1
+ export declare const KEEP_IN_TAB_JS = "(() => {\n if (window.__computerUseKeepInTab) return;\n window.__computerUseKeepInTab = true;\n // _self/_parent/_top all stay in this window; _blank and any NAME open another one.\n const opensTab = (t) => !!t && !/^_(self|parent|top)$/i.test(t);\n const keep = (e) => {\n // composedPath() pierces shadow roots, where e.target is the host and closest() is blind.\n const path = e.composedPath ? e.composedPath() : [e.target];\n for (const n of path) {\n if (!n || (n.tagName !== 'A' && n.tagName !== 'FORM')) continue;\n if (opensTab(n.getAttribute('target'))) n.removeAttribute('target');\n break;\n }\n const base = document.querySelector('base[target]');\n if (base && opensTab(base.getAttribute('target'))) base.removeAttribute('target');\n };\n document.addEventListener('click', keep, true);\n document.addEventListener('submit', keep, true);\n const nativeOpen = window.open.bind(window);\n const blank = (u) => !u || /^(about:blank|#)$/i.test(String(u).trim());\n // A features string asking for a separate WINDOW (the popup token or any geometry) \u2014 the\n // OAuth/captcha handshake idiom. A tab-shaped open has no features at all.\n const popup = (f) => !!f && /(^|[\\s,])(popup|width|height|left|top|innerWidth|innerHeight|outerWidth|outerHeight|screenX|screenY)(\\s*=|$|[\\s,])/i.test(String(f));\n window.open = function (url, name, features) {\n if (blank(url)) return nativeOpen(url, name, features); // no destination: a window to write into\n if (popup(features)) return nativeOpen(url, name, features); // a handshake popup needs a real opener\n\n window.location.href = String(url);\n return {\n closed: false, focus() {}, blur() {}, close() {}, postMessage() {},\n get location() { return window.location; },\n set location(v) { window.location.href = String(v); },\n };\n };\n})()";
@@ -0,0 +1 @@
1
+ const e=("((()=>{if(window.__computerUseKeepInTab)return;window.__computerUseKeepInTab=!0;const i=t=>!!t&&!/^_(self|parent|top)$/i.test(t),r=t=>{const o=t.composedPath?t.composedPath():[t.target];for(const n of o)if(!(!n||n.tagName!==\"A\"&&n.tagName!==\"FORM\")){i(n.getAttribute(\"target\"))&&n.removeAttribute(\"target\");break}const e=document.querySelector(\"base[target]\");e&&i(e.getAttribute(\"target\"))&&e.removeAttribute(\"target\")};document.addEventListener(\"click\",r,!0),document.addEventListener(\"submit\",r,!0);const s=window.open.bind(window),c=t=>!t||/^(about:blank|#)$/i.test(String(t).trim()),a=t=>!!t&&/(^|[\\s,])(popup|width|height|left|top|innerWidth|innerHeight|outerWidth|outerHeight|screenX|screenY)(\\s*=|$|[\\s,])/i.test(String(t));window.open=function(t,o,e){return c(t)||a(e)?s(t,o,e):(window.location.href=String(t),{closed:!1,focus(){},blur(){},close(){},postMessage(){},get location(){return window.location},set location(n){window.location.href=String(n)}})}})())");export{e as KEEP_IN_TAB_JS};
@@ -0,0 +1,25 @@
1
+ export declare function msgChromeMissing(installHint: string): string;
2
+ export declare function msgAttachedDeadEnd(profileDir: string, why: string): string;
3
+ export declare const msgAttachedWhyNoPage: () => string;
4
+ export declare const msgAttachedWhyHandshake: () => string;
5
+ export declare const msgAttachedWhyNoPort: () => string;
6
+ export declare const msgOwnWindowFailed: () => string;
7
+ export declare function msgChromeBinaryUnrunnable(exe: string, code: string): string;
8
+ export declare function msgChromeStartFailed(detail: string): string;
9
+ export declare const msgChromeExitedImmediately: () => string;
10
+ export declare const msgChromeNoPort: () => string;
11
+ export declare const msgChromeRefusedAttach: () => string;
12
+ export declare function msgBridgeExtensionGone(): string;
13
+ export declare function msgBridgeExtensionElsewhere(owner: string): string;
14
+ export declare function msgBridgeJustStarted(): string;
15
+ export declare function isBridgeExtensionGone(e: unknown): boolean;
16
+ export declare function msgEndpointUnavailable(cdpUrl: string, detail: string): string;
17
+ export declare const REOPENED_NOTE: string;
18
+ export declare const TAB_REOPENED_NOTE: string;
19
+ export declare const TAB_DETACHED_NOTE: string;
20
+ export declare function tabRefusedNote(reason: string): string;
21
+ export declare const TAB_REOPENED_AFTER_SELF_CLOSE_NOTE: string;
22
+ export declare const NO_TAB_OPEN_NOTE: string;
23
+ export declare const NO_TAB_YET_NOTE: string;
24
+ export declare const BROWSER_SWITCHED_NOTE: string;
25
+ export declare const SURFACE_REIDENTIFIED_NOTE: string;
@@ -0,0 +1,8 @@
1
+ function n(e){return`mirasim-cu: Google Chrome was not found, so no browser could be started.
2
+ Fix one of:
3
+ - install Google Chrome (${e})
4
+ - point COMPUTER_USE_CHROME at a Chrome/Chromium binary
5
+ - point COMPUTER_USE_CDP_URL at an already-running browser's CDP endpoint`}function s(e,t){return`mirasim-cu: cannot take over this Chrome \u2014 ${t}.
6
+ This profile belongs to the user (${e}), so it will not be force-closed.
7
+ Quit Chrome (\u2318Q) and retry.`}const r=()=>"its debugging port answers, but no usable page can be opened",i=()=>"its debugging port answers, but the handshake timed out (the browser may be frozen)",a=()=>"it is already running, but without a debugging port",h=()=>"mirasim-cu: could not open the agent's own window in the user's Chrome.";function d(e,t){return`mirasim-cu: cannot run the Chrome binary at ${e} (${t}). Point COMPUTER_USE_CHROME at a working Chrome/Chromium.`}function c(e){return`mirasim-cu: Chrome could not be started (${e}).`}const g=()=>"mirasim-cu: Chrome exited immediately instead of publishing a debugging port \u2014 another browser may already own this profile directory.",m=()=>"mirasim-cu: Chrome never published a debugging port within 15s.",b=()=>"mirasim-cu: Chrome started but its debugger refused to attach.",o="https://chromewebstore.google.com/detail/mirasim/khflooofmndpmafhofjdcpjfgnlpikhg";function l(){return`The mirasim bridge extension is not connected, so this browser cannot be reached. Nothing you can do from here fixes it and retrying will not help: ask the user to install or enable the Mirasim extension from the Chrome Web Store (${o}) and remove any older copy loaded unpacked at chrome://extensions, which no longer connects; then continue.`}function u(e){return`The Mirasim extension is connected to another program's browser bridge (${e}), not to this one \u2014 it serves one bridge at a time and that one comes first. Retrying will not help: ask the user to turn browser control off in that program (${e}), then continue.`}function f(){return`The browser bridge started moments ago and the Mirasim extension has not connected yet. An installed extension checks again within a minute; wait about a minute and retry once. If it still does not connect, ask the user to install or enable the Mirasim extension from the Chrome Web Store (${o}).`}function p(e){return/Bridge extension not connected|bridge extension did not connect/i.test(e instanceof Error?e.message:String(e))}function w(e,t){return`mirasim-cu: explicit CDP endpoint ${e} is unavailable.
8
+ (${t})`}const x="the browser had been closed (most likely by the user), so a fresh one was opened in the background and the page is blank \u2014 re-observe, and navigate again before acting on refs",E="this surface's tab had been closed (most likely by the user), so a fresh blank tab was claimed \u2014 re-observe, and navigate again before acting on refs",C="this surface's tab stopped answering \u2014 it was closed, or the browser took its debugger away, in which case it is still open in the user's window but out of reach from here. A fresh blank tab was claimed \u2014 re-observe, and navigate again before acting on refs. If the same page drops again right after it loads, it cannot be driven through this browser: a password manager's autofill (measured: iCloud Passwords, on any page with a password field) puts its own extension frame into the page, and Chrome then refuses this extension's debugger there. Do not reopen it; tell the user \u2014 turning that extension off for the site, or the managed browser (its own profile), drives it";function y(e){const t=`this surface's tab is out of reach: it is still open in the user's window, but Chrome refused this extension's debugger on it ("${e}"). A fresh blank tab was claimed. `;return/chrome-extension:\/\/ URL of different extension/.test(e)?t+"Another extension has put its own frame into that page \u2014 a password manager's autofill does this on any page with a password field (measured: iCloud Passwords) \u2014 and while it is there no retry from here gets in. Do not reopen the page; tell the user that turning that extension off for this site, or the managed browser (its own profile), lets it be driven":t+"Reopening it will most likely be refused the same way \u2014 tell the user rather than retrying"}const _="this session had closed its last tab, so opening one started a fresh blank tab \u2014 nothing went wrong and nobody took the browser back; navigate before acting on refs",T="this session has no tab open \u2014 it closed its last one and nothing was reopened, so the browser is left as the user had it. open_url or new_tab starts a fresh tab when you need one",k="this session holds no tab yet \u2014 none is opened until asked. Bind one of the listed tabs to act in it, or new_tab opens one of the session's own",O="the browser behind the extension changed \u2014 a different browser, or this one after a restart, is connected now, and the tabs listed are ITS tabs. Nothing was done in it and this session holds no tab there; handles and refs from before belonged to the previous browser and are gone. If this is not the browser the task is about, stop and tell the user; otherwise bind one of the listed tabs, or new_tab opens one of the session's own",N="the connection to this browser dropped and came back, so this surface was re-identified \u2014 the page below is the one it is on now, and refs from before are dead; re-observe before acting";export{O as BROWSER_SWITCHED_NOTE,T as NO_TAB_OPEN_NOTE,k as NO_TAB_YET_NOTE,x as REOPENED_NOTE,N as SURFACE_REIDENTIFIED_NOTE,C as TAB_DETACHED_NOTE,_ as TAB_REOPENED_AFTER_SELF_CLOSE_NOTE,E as TAB_REOPENED_NOTE,p as isBridgeExtensionGone,s as msgAttachedDeadEnd,i as msgAttachedWhyHandshake,r as msgAttachedWhyNoPage,a as msgAttachedWhyNoPort,u as msgBridgeExtensionElsewhere,l as msgBridgeExtensionGone,f as msgBridgeJustStarted,d as msgChromeBinaryUnrunnable,g as msgChromeExitedImmediately,n as msgChromeMissing,m as msgChromeNoPort,b as msgChromeRefusedAttach,c as msgChromeStartFailed,w as msgEndpointUnavailable,h as msgOwnWindowFailed,y as tabRefusedNote};
@@ -0,0 +1,4 @@
1
+ import type { ActResult } from "../types.ts";
2
+ export declare const NAV_TIMEOUT_MS = 15000;
3
+ export declare function navVerdict(e: unknown): ActResult;
4
+ export declare function landedNote(verb: "opened" | "reloaded", requested: string, final: string): string;