@mirasim/computer-use 0.1.0-rc.6 → 0.1.0-rc.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/platform/browser/bridge/model-commands.js +1 -1
- package/dist/platform/browser/bridge/model-core.d.ts +7 -0
- package/dist/platform/browser/bridge/model-core.js +1 -1
- package/dist/platform/browser/bridge/server.js +1 -1
- package/dist/platform/browser/bridge-tabs.d.ts +3 -0
- package/dist/platform/browser/bridge-tabs.js +1 -1
- package/dist/platform/browser/driver-verbs.js +2 -2
- package/dist/platform/browser/driver.d.ts +2 -0
- package/dist/platform/browser/driver.js +3 -2
- package/dist/platform/browser/messages.d.ts +5 -0
- package/dist/platform/browser/messages.js +2 -2
- package/dist/platform/chromium/actions.js +2 -2
- package/dist/platform/chromium/run-js.d.ts +5 -1
- package/dist/platform/chromium/run-js.js +2 -2
- package/dist/platform/electron/driver.js +1 -1
- package/dist/platform/electron/verbs.js +1 -1
- package/dist/platform/macos/type-verb.js +1 -1
- package/dist/platform/tools/api-contract.d.ts +1 -0
- package/dist/platform/tools/api.js +1 -1
- package/dist/platform/tools/repl-docs.js +1 -1
- package/dist/platform/tools/repl.js +12 -12
- package/extension/_locales/en/messages.json +179 -0
- package/extension/_locales/zh_CN/messages.json +179 -0
- package/extension/access.js +245 -0
- package/extension/ask.html +26 -0
- package/extension/ask.js +20 -0
- package/extension/background.js +79 -3
- package/extension/group.js +335 -79
- package/extension/icons/icon-128.png +0 -0
- package/extension/icons/icon-16.png +0 -0
- package/extension/icons/icon-32.png +0 -0
- package/extension/icons/icon-48.png +0 -0
- package/extension/manifest.json +8 -2
- package/extension/panel.css +86 -0
- package/extension/panel.html +86 -0
- package/extension/panel.js +192 -0
- package/extension/relay.js +264 -5
- package/extension/toolbar.js +172 -0
- package/package.json +5 -5
- package/scripts/extension.mjs +4 -2
package/extension/relay.js
CHANGED
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
*/
|
|
17
17
|
|
|
18
18
|
import { RUNTIME_INFO } from "./runtime-info.js";
|
|
19
|
+
import { AccessError } from "./access.js";
|
|
19
20
|
|
|
20
21
|
export function debugLog(...args) {
|
|
21
22
|
// eslint-disable-next-line no-console
|
|
@@ -35,6 +36,18 @@ const ALLOWED_COMMANDS = new Set([
|
|
|
35
36
|
"chrome.downloads.search",
|
|
36
37
|
]);
|
|
37
38
|
|
|
39
|
+
/** What a paused agent may still do: stay connected, let go of what it holds, and list the tabs it
|
|
40
|
+
* may use (still filtered by the scope), so it can tell a pause from tabs that went away. */
|
|
41
|
+
const ALLOWED_WHILE_PAUSED = new Set(["bridge.ping", "bridge.reload", "bridge.releaseTab", "bridge.info", "chrome.debugger.detach",
|
|
42
|
+
"chrome.tabs.query", "chrome.debugger.getTargets"]);
|
|
43
|
+
|
|
44
|
+
/** Cookie reads whose answer is filtered by the site rules (`cookies` in the result). */
|
|
45
|
+
const COOKIE_READS = new Set(["Network.getAllCookies", "Network.getCookies", "Storage.getCookies"]);
|
|
46
|
+
/** Cookie writes checked cookie by cookie (`cookies` in the params). */
|
|
47
|
+
const COOKIE_WRITES = new Set(["Network.setCookies", "Storage.setCookies"]);
|
|
48
|
+
/** Commands that wipe every site's cookies at once, the blocked ones included. */
|
|
49
|
+
const COOKIE_WIPES = new Set(["Network.clearBrowserCookies", "Storage.clearCookies"]);
|
|
50
|
+
|
|
38
51
|
/** chrome.* events forwarded to the bridge (positional args, as the listener got them). */
|
|
39
52
|
const FORWARDED_EVENTS = [
|
|
40
53
|
"chrome.debugger.onEvent",
|
|
@@ -74,6 +87,27 @@ export class RelayConnection {
|
|
|
74
87
|
this._closed = false;
|
|
75
88
|
this._pendingReattach = new Set();
|
|
76
89
|
this._recentReattach = new Set();
|
|
90
|
+
/** Tabs the bridge has been told about (`attachTab`, a forwarded popup) and not told to forget. */
|
|
91
|
+
this._announced = new Set();
|
|
92
|
+
/**
|
|
93
|
+
* Tabs taken away from the agent (a blocked site, out of scope), with why: from the moment
|
|
94
|
+
* this is set no command reaches them and no event leaves them, before the asynchronous
|
|
95
|
+
* detach has even started.
|
|
96
|
+
* @type {Map<number, {reason: string, url?: string}>}
|
|
97
|
+
*/
|
|
98
|
+
this._revoked = new Map();
|
|
99
|
+
/** A tab's child sessions (out-of-process iframes, workers): sessionId → targetId. */
|
|
100
|
+
this._childTargets = new Map();
|
|
101
|
+
/** Child sessions on a blocked site: nothing goes in or out. */
|
|
102
|
+
this._blockedSessions = new Set();
|
|
103
|
+
/** What the bridge said about itself (`bridge.info`); null from a bridge that predates it. */
|
|
104
|
+
this.bridgeInfo = null;
|
|
105
|
+
/**
|
|
106
|
+
* Who decides what the agent may reach (group.js), asked before every tab is listed,
|
|
107
|
+
* attached or driven. Absent, nothing is refused — a test seam, never the product.
|
|
108
|
+
* @type {{policy: import("./access.js").AccessPolicy, visible(tab: chrome.tabs.Tab): boolean, authorizeAttach(tabId: number): Promise<void>, noteCreated?(tab: chrome.tabs.Tab): void} | undefined}
|
|
109
|
+
*/
|
|
110
|
+
this.gate = undefined;
|
|
77
111
|
|
|
78
112
|
/** @type {(() => void) | undefined} */
|
|
79
113
|
this.onclose = undefined;
|
|
@@ -135,6 +169,16 @@ export class RelayConnection {
|
|
|
135
169
|
this._send({ method: "extension.updateAvailable", params: [version] });
|
|
136
170
|
}
|
|
137
171
|
|
|
172
|
+
/** Tell the bridge how far the user lets the agent reach, so it can say so when it lists
|
|
173
|
+
* tabs. A bridge that predates this logs an unknown event and carries on. */
|
|
174
|
+
reportAccess(access) {
|
|
175
|
+
if (!this._closed) this._send({ method: "extension.access", params: [access] });
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
isAnnounced(tabId) {
|
|
179
|
+
return this._announced.has(tabId);
|
|
180
|
+
}
|
|
181
|
+
|
|
138
182
|
close(reason) {
|
|
139
183
|
this._transport.close(reason);
|
|
140
184
|
// ws.onclose fires asynchronously; run the bookkeeping now so no CDP event
|
|
@@ -149,9 +193,31 @@ export class RelayConnection {
|
|
|
149
193
|
* them itself (`createTarget`). */
|
|
150
194
|
attachTab(tab) {
|
|
151
195
|
if (this._closed || (this._attachedTabs.has(tab.id) && !tab.discarded && tab.status !== "unloaded")) return;
|
|
196
|
+
if (this.gate && !this.gate.visible(tab)) return;
|
|
197
|
+
this._revoked.delete(tab.id);
|
|
198
|
+
this._announced.add(tab.id);
|
|
152
199
|
this._send({ method: "chrome.tabs.onCreated", params: [tab, { agent: false }] });
|
|
153
200
|
}
|
|
154
201
|
|
|
202
|
+
/**
|
|
203
|
+
* Take a tab away from the agent: detach it, and unless `keepListed` (the pause), make the
|
|
204
|
+
* bridge forget it as if it had closed (it is no longer the agent's to see). Commands to it are
|
|
205
|
+
* refused from this call on, with `reason` (access.js `refusal`), until it is attached or
|
|
206
|
+
* announced again.
|
|
207
|
+
*/
|
|
208
|
+
revoke(tabId, reason, { url, keepListed = false } = {}) {
|
|
209
|
+
if (this._closed) return;
|
|
210
|
+
this._revoked.set(tabId, { reason, url });
|
|
211
|
+
this._pendingReattach.delete(tabId);
|
|
212
|
+
if (this._attachedTabs.has(tabId)) {
|
|
213
|
+
chrome.debugger.detach({ tabId }).catch(() => {});
|
|
214
|
+
this._notifyDetached(tabId);
|
|
215
|
+
this._send({ method: "chrome.debugger.onDetach", params: [{ tabId }, "target_closed"] });
|
|
216
|
+
}
|
|
217
|
+
for (const [sessionId, child] of this._childTargets) if (child.tabId === tabId) this._forgetChild(sessionId);
|
|
218
|
+
if (!keepListed && this._announced.delete(tabId)) this._send({ method: "chrome.tabs.onRemoved", params: [tabId, { revoked: true }] });
|
|
219
|
+
}
|
|
220
|
+
|
|
155
221
|
/** Detach a tab the user dragged out of the group. chrome.debugger.detach does not fire
|
|
156
222
|
* onDetach for the caller, so the event the bridge needs is synthesized. */
|
|
157
223
|
detachTab(tabId) {
|
|
@@ -203,6 +269,9 @@ export class RelayConnection {
|
|
|
203
269
|
clearInterval(this._silenceTimer);
|
|
204
270
|
this._pendingReattach.clear();
|
|
205
271
|
this._recentReattach.clear();
|
|
272
|
+
this._announced.clear();
|
|
273
|
+
this._childTargets.clear();
|
|
274
|
+
this._blockedSessions.clear();
|
|
206
275
|
for (const remove of this._eventListeners) remove();
|
|
207
276
|
this._eventListeners = [];
|
|
208
277
|
for (const tabId of [...this._attachedTabs]) {
|
|
@@ -218,6 +287,19 @@ export class RelayConnection {
|
|
|
218
287
|
const tabId = this._tabIdForEvent(fullMethod, args);
|
|
219
288
|
if (tabId === undefined) return;
|
|
220
289
|
if (!this._attachedTabs.has(tabId) && !this._forwardsWhenDetached(fullMethod)) return;
|
|
290
|
+
if (fullMethod === "chrome.debugger.onEvent" && !this._passesEvent(args)) return;
|
|
291
|
+
if (fullMethod === "chrome.tabs.onCreated") {
|
|
292
|
+
// A popup of a tab the agent drives: in scope as its opener is, and announced like one.
|
|
293
|
+
if (this._revoked.has(tabId)) return;
|
|
294
|
+
this.gate?.noteCreated?.(args[0]);
|
|
295
|
+
if (this.gate && !this.gate.visible(args[0])) return;
|
|
296
|
+
if (args[0]?.id !== undefined) this._announced.add(args[0].id);
|
|
297
|
+
}
|
|
298
|
+
if (fullMethod === "chrome.tabs.onRemoved" || fullMethod === "chrome.tabs.onReplaced") {
|
|
299
|
+
const gone = fullMethod === "chrome.tabs.onRemoved" ? args[0] : args[1];
|
|
300
|
+
this._announced.delete(gone);
|
|
301
|
+
this._revoked.delete(gone);
|
|
302
|
+
}
|
|
221
303
|
this._send({ method: fullMethod, params: args });
|
|
222
304
|
if (fullMethod === "chrome.debugger.onDetach") {
|
|
223
305
|
const reason = args[1];
|
|
@@ -231,6 +313,66 @@ export class RelayConnection {
|
|
|
231
313
|
}
|
|
232
314
|
}
|
|
233
315
|
|
|
316
|
+
/**
|
|
317
|
+
* Does a debugger event leave for the bridge? Not from a revoked tab or a blocked child
|
|
318
|
+
* session, and a navigation onto a blocked site is where the tab (or the iframe) is taken
|
|
319
|
+
* away: the event that says what the page now is never reaches the agent.
|
|
320
|
+
*/
|
|
321
|
+
_passesEvent([source, method, params]) {
|
|
322
|
+
const tabId = source?.tabId;
|
|
323
|
+
if (this._revoked.has(tabId)) return false;
|
|
324
|
+
if (source?.sessionId && this._blockedSessions.has(source.sessionId)) return false;
|
|
325
|
+
const policy = this.gate?.policy;
|
|
326
|
+
if (!policy) return true;
|
|
327
|
+
if (method === "Target.attachedToTarget") {
|
|
328
|
+
const verdict = policy.siteVerdict(params?.targetInfo?.url);
|
|
329
|
+
if (params?.sessionId) this._childTargets.set(params.sessionId, { tabId, targetId: params?.targetInfo?.targetId, forwarded: !verdict });
|
|
330
|
+
if (verdict) {
|
|
331
|
+
this._blockChild(tabId, params.sessionId, verdict, params.targetInfo.url);
|
|
332
|
+
return false;
|
|
333
|
+
}
|
|
334
|
+
return true;
|
|
335
|
+
}
|
|
336
|
+
if (method === "Target.detachedFromTarget") {
|
|
337
|
+
const child = params?.sessionId ? this._childTargets.get(params.sessionId) : undefined;
|
|
338
|
+
if (params?.sessionId) this._forgetChild(params.sessionId);
|
|
339
|
+
// The bridge hears of a child's end only if it heard of the child.
|
|
340
|
+
return child?.forwarded ?? true;
|
|
341
|
+
}
|
|
342
|
+
if (method === "Page.frameNavigated") {
|
|
343
|
+
const frame = params?.frame;
|
|
344
|
+
const verdict = policy.siteVerdict(frame?.url);
|
|
345
|
+
if (!verdict) return true;
|
|
346
|
+
if (!source?.sessionId && !frame?.parentId) {
|
|
347
|
+
policy.record({ reason: verdict, tabId, url: frame.url });
|
|
348
|
+
this.revoke(tabId, verdict, { url: frame.url });
|
|
349
|
+
this.ontabrevoked?.(tabId);
|
|
350
|
+
return false;
|
|
351
|
+
}
|
|
352
|
+
const child = source?.sessionId ? this._childTargets.get(source.sessionId) : undefined;
|
|
353
|
+
if (child && (frame.id === child.targetId || !frame.parentId)) {
|
|
354
|
+
this._blockChild(tabId, source.sessionId, verdict, frame.url);
|
|
355
|
+
return false;
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
return true;
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
/** An iframe or worker on a blocked site: let it run (it may be paused waiting for a
|
|
362
|
+
* debugger), then let go of it, so the page works and the agent never sees inside it. */
|
|
363
|
+
_blockChild(tabId, sessionId, reason, url) {
|
|
364
|
+
if (!sessionId || this._blockedSessions.has(sessionId)) return;
|
|
365
|
+
this._blockedSessions.add(sessionId);
|
|
366
|
+
this.gate?.policy.record({ reason, tabId, url, detail: "frame" });
|
|
367
|
+
void chrome.debugger.sendCommand({ tabId, sessionId }, "Runtime.runIfWaitingForDebugger").catch(() => {})
|
|
368
|
+
.then(() => chrome.debugger.sendCommand({ tabId }, "Target.detachFromTarget", { sessionId }).catch(() => {}));
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
_forgetChild(sessionId) {
|
|
372
|
+
this._childTargets.delete(sessionId);
|
|
373
|
+
this._blockedSessions.delete(sessionId);
|
|
374
|
+
}
|
|
375
|
+
|
|
234
376
|
_maybeScheduleReattach(tabId) {
|
|
235
377
|
if (this._closed) return false;
|
|
236
378
|
if (this._recentReattach.has(tabId)) return false;
|
|
@@ -318,26 +460,135 @@ export class RelayConnection {
|
|
|
318
460
|
this.releaseTab((message.params ?? [])[0]);
|
|
319
461
|
return {};
|
|
320
462
|
}
|
|
463
|
+
if (message.method === "bridge.info") {
|
|
464
|
+
const info = (message.params ?? [])[0];
|
|
465
|
+
this.bridgeInfo = info && typeof info === "object" ? { name: String(info.name ?? ""), version: String(info.version ?? "") } : null;
|
|
466
|
+
this.onbridgeinfo?.();
|
|
467
|
+
return {};
|
|
468
|
+
}
|
|
321
469
|
if (!ALLOWED_COMMANDS.has(message.method)) throw new Error(`Unknown method: ${message.method}`);
|
|
322
470
|
const args = message.params ?? [];
|
|
323
|
-
|
|
471
|
+
const policy = this.gate?.policy;
|
|
472
|
+
if (policy?.paused && !ALLOWED_WHILE_PAUSED.has(message.method)) {
|
|
473
|
+
policy.record({ reason: "paused" });
|
|
474
|
+
throw new AccessError("paused");
|
|
475
|
+
}
|
|
476
|
+
if (this.gate) return await this._gatedCommand(message.method, args);
|
|
477
|
+
return await this._runCommand(message.method, args);
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
/** Every chrome.* command, held against the user's access settings first (access.js). */
|
|
481
|
+
async _gatedCommand(method, args) {
|
|
482
|
+
const { policy } = this.gate;
|
|
483
|
+
switch (method) {
|
|
484
|
+
case "chrome.tabs.query": {
|
|
485
|
+
const tabs = (await this._runCommand(method, args)) ?? [];
|
|
486
|
+
return Array.isArray(tabs) ? tabs.filter((tab) => this.gate.visible(tab)) : tabs;
|
|
487
|
+
}
|
|
488
|
+
case "chrome.debugger.getTargets": {
|
|
489
|
+
const visible = new Set((await chrome.tabs.query({})).filter((tab) => this.gate.visible(tab)).map((tab) => tab.id));
|
|
490
|
+
const targets = (await this._runCommand(method, args)) ?? [];
|
|
491
|
+
return targets.filter((t) => t.tabId !== undefined && visible.has(t.tabId));
|
|
492
|
+
}
|
|
493
|
+
case "chrome.debugger.attach": {
|
|
494
|
+
const tabId = args[0]?.tabId;
|
|
495
|
+
await this.gate.authorizeAttach(tabId);
|
|
496
|
+
this._revoked.delete(tabId);
|
|
497
|
+
return await this._runCommand(method, args);
|
|
498
|
+
}
|
|
499
|
+
case "chrome.debugger.sendCommand":
|
|
500
|
+
return await this._gatedCdp(args);
|
|
501
|
+
case "chrome.tabs.create": {
|
|
502
|
+
const url = args[0]?.url;
|
|
503
|
+
const verdict = policy.siteVerdict(url);
|
|
504
|
+
if (verdict) {
|
|
505
|
+
policy.record({ reason: verdict, url });
|
|
506
|
+
throw new AccessError(verdict, { url });
|
|
507
|
+
}
|
|
508
|
+
return await this._runCommand(method, args);
|
|
509
|
+
}
|
|
510
|
+
case "chrome.tabs.remove":
|
|
511
|
+
case "chrome.tabs.reload": {
|
|
512
|
+
const ids = Array.isArray(args[0]) ? args[0] : [args[0]];
|
|
513
|
+
for (const tabId of ids) {
|
|
514
|
+
const tab = await chrome.tabs.get(tabId).catch(() => null);
|
|
515
|
+
if (tab && !this.gate.visible(tab)) {
|
|
516
|
+
policy.record({ reason: "out-of-scope", tabId });
|
|
517
|
+
throw new AccessError("out-of-scope", { tabId });
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
return await this._runCommand(method, args);
|
|
521
|
+
}
|
|
522
|
+
case "chrome.downloads.search": {
|
|
523
|
+
const items = (await this._runCommand(method, args)) ?? [];
|
|
524
|
+
return items.filter((item) => !policy.siteVerdict(item.finalUrl || item.url) && !policy.siteVerdict(item.referrer));
|
|
525
|
+
}
|
|
526
|
+
default:
|
|
527
|
+
return await this._runCommand(method, args);
|
|
528
|
+
}
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
/** A CDP command for a tab (or one of its child sessions), held against the site rules. */
|
|
532
|
+
async _gatedCdp(args) {
|
|
533
|
+
const { policy } = this.gate;
|
|
534
|
+
const [debuggee, cdpMethod, params] = args;
|
|
535
|
+
const tabId = debuggee?.tabId;
|
|
536
|
+
const revoked = this._revoked.get(tabId);
|
|
537
|
+
if (revoked) throw new AccessError(revoked.reason, { tabId, url: revoked.url });
|
|
538
|
+
if (debuggee?.sessionId && this._blockedSessions.has(debuggee.sessionId)) throw new AccessError("site-blocked", { url: "this frame's site" });
|
|
539
|
+
const refuse = (reason, details = {}) => {
|
|
540
|
+
policy.record({ reason, tabId, ...details });
|
|
541
|
+
throw new AccessError(reason, { tabId, ...details });
|
|
542
|
+
};
|
|
543
|
+
if (cdpMethod === "Page.navigate") {
|
|
544
|
+
const verdict = policy.siteVerdict(params?.url);
|
|
545
|
+
if (verdict) refuse(verdict, { url: params.url });
|
|
546
|
+
}
|
|
547
|
+
if (COOKIE_WIPES.has(cdpMethod) && policy.restrictive()) refuse("clear-cookies");
|
|
548
|
+
if (COOKIE_WRITES.has(cdpMethod)) {
|
|
549
|
+
for (const cookie of params?.cookies ?? []) {
|
|
550
|
+
const domain = cookie?.domain || hostFromUrl(cookie?.url);
|
|
551
|
+
if (!policy.cookieAllowed(domain)) refuse("cookie-site", { url: domain });
|
|
552
|
+
}
|
|
553
|
+
}
|
|
554
|
+
if (cdpMethod === "Network.setCookie" || cdpMethod === "Network.deleteCookies") {
|
|
555
|
+
const domain = params?.domain || hostFromUrl(params?.url);
|
|
556
|
+
if (!policy.cookieAllowed(domain)) refuse("cookie-site", { url: domain });
|
|
557
|
+
}
|
|
558
|
+
if (cdpMethod === "Storage.clearDataForOrigin" || cdpMethod === "Storage.clearDataForStorageKey") {
|
|
559
|
+
const origin = params?.origin ?? params?.storageKey;
|
|
560
|
+
const verdict = policy.siteVerdict(origin);
|
|
561
|
+
if (verdict) refuse(verdict, { url: origin });
|
|
562
|
+
}
|
|
563
|
+
const result = await this._runCommand("chrome.debugger.sendCommand", args);
|
|
564
|
+
if (COOKIE_READS.has(cdpMethod) && Array.isArray(result?.cookies))
|
|
565
|
+
return { ...result, cookies: result.cookies.filter((c) => policy.cookieAllowed(c.domain)) };
|
|
566
|
+
return result;
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
/** Run one allow-listed chrome.* command as asked, with the attach bookkeeping. */
|
|
570
|
+
async _runCommand(method, args) {
|
|
571
|
+
if (method === "chrome.debugger.attach") {
|
|
324
572
|
const target = args[0];
|
|
325
573
|
// A zombie attach from a previous service-worker life throws "Another debugger is
|
|
326
574
|
// already attached" against ourselves. Clearing our own leftover is safe; a REAL
|
|
327
575
|
// other debugger (devtools) re-throws on the second try and that error is honest.
|
|
328
576
|
try {
|
|
329
|
-
await invokeChromeMethod(
|
|
577
|
+
await invokeChromeMethod(method, args);
|
|
330
578
|
} catch (e) {
|
|
331
579
|
if (!String(e?.message).includes("already attached")) throw e;
|
|
332
580
|
await chrome.debugger.detach({ tabId: target?.tabId }).catch(() => {});
|
|
333
|
-
await invokeChromeMethod(
|
|
581
|
+
await invokeChromeMethod(method, args);
|
|
334
582
|
}
|
|
335
583
|
if (target?.tabId !== undefined) this._notifyAttached(target.tabId);
|
|
336
584
|
return {};
|
|
337
585
|
}
|
|
338
|
-
const result = await invokeChromeMethod(
|
|
586
|
+
const result = await invokeChromeMethod(method, args);
|
|
339
587
|
// Said before the reply leaves: the bridge attaches the tab only once it has the reply.
|
|
340
|
-
if (
|
|
588
|
+
if (method === "chrome.tabs.create" && result?.id !== undefined) {
|
|
589
|
+
this.ontabcreatedforagent?.(result.id);
|
|
590
|
+
this._announced.add(result.id);
|
|
591
|
+
}
|
|
341
592
|
return result ?? {};
|
|
342
593
|
}
|
|
343
594
|
|
|
@@ -346,6 +597,14 @@ export class RelayConnection {
|
|
|
346
597
|
}
|
|
347
598
|
}
|
|
348
599
|
|
|
600
|
+
function hostFromUrl(url) {
|
|
601
|
+
try {
|
|
602
|
+
return url ? new URL(url).hostname : "";
|
|
603
|
+
} catch {
|
|
604
|
+
return "";
|
|
605
|
+
}
|
|
606
|
+
}
|
|
607
|
+
|
|
349
608
|
/** Resolve chrome.<api>.<member> for reflective invocation and listener setup. */
|
|
350
609
|
function resolveChromeMember(fullMethod) {
|
|
351
610
|
const parts = fullMethod.split(".");
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The toolbar icon, per tab: what a click does, and what the badge and tooltip say.
|
|
3
|
+
*
|
|
4
|
+
* - On a tab the agent could use but has not been handed, a click hands it over (the tab joins
|
|
5
|
+
* the mirasim group of its window, group.js `addToMirasimGroup`). Chrome only fires
|
|
6
|
+
* `action.onClicked` for a tab with no popup, so those tabs get an empty popup.
|
|
7
|
+
* - Everywhere else — a tab already handed over, a browser page, a blocked site — a click opens
|
|
8
|
+
* the panel (panel.html), where the user can take a tab back. One click never takes a tab
|
|
9
|
+
* away from the agent mid-task.
|
|
10
|
+
*
|
|
11
|
+
* The icon says whether the agent may use the tab at all, as Playwriter's does: in colour on a
|
|
12
|
+
* tab handed over (in the mirasim group, or one the agent opened), greyed out everywhere else —
|
|
13
|
+
* so a tab in the group reads differently from one outside it even while the agent is idle.
|
|
14
|
+
* Badges add what the colour cannot: ✓ the agent holds this tab right now; × a site rule blocks
|
|
15
|
+
* it; II the agent is paused; "new" until the panel has been opened once (0.1.3 changed what
|
|
16
|
+
* the agent may reach by default). Chrome resets tab-specific action state on navigation, so
|
|
17
|
+
* every tab update re-applies it.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { PANEL_SEEN_KEY } from "./access.js";
|
|
21
|
+
import { GROUP_TITLE, addToMirasimGroup, isNonDebuggableUrl } from "./group.js";
|
|
22
|
+
import { debugLog } from "./relay.js";
|
|
23
|
+
|
|
24
|
+
const PANEL = "panel.html";
|
|
25
|
+
const BADGES = {
|
|
26
|
+
paused: { text: "II", color: "#b45309" },
|
|
27
|
+
using: { text: "✓", color: "#2a58c0" },
|
|
28
|
+
blocked: { text: "×", color: "#6b7280" },
|
|
29
|
+
fresh: { text: "new", color: "#2a58c0" },
|
|
30
|
+
none: { text: "", color: "#2a58c0" },
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
const t = (key) => chrome.i18n.getMessage(key) || key;
|
|
34
|
+
|
|
35
|
+
const ICON_SIZES = [16, 32];
|
|
36
|
+
/** The toolbar icon greyed out, drawn once from the shipped PNGs: no second set of files to keep
|
|
37
|
+
* in step with the brand mark (mirasim scripts/make-app-icons.mjs writes only the colour one). */
|
|
38
|
+
let greyIcon = null;
|
|
39
|
+
async function greyedIcon() {
|
|
40
|
+
greyIcon ??= (async () => {
|
|
41
|
+
const out = {};
|
|
42
|
+
for (const size of ICON_SIZES) {
|
|
43
|
+
const blob = await (await fetch(chrome.runtime.getURL(`icons/icon-${size}.png`))).blob();
|
|
44
|
+
const bitmap = await createImageBitmap(blob);
|
|
45
|
+
const canvas = new OffscreenCanvas(size, size);
|
|
46
|
+
const ctx = canvas.getContext("2d");
|
|
47
|
+
ctx.drawImage(bitmap, 0, 0, size, size);
|
|
48
|
+
const img = ctx.getImageData(0, 0, size, size);
|
|
49
|
+
const d = img.data;
|
|
50
|
+
for (let i = 0; i < d.length; i += 4) {
|
|
51
|
+
const grey = Math.round(0.3 * d[i] + 0.59 * d[i + 1] + 0.11 * d[i + 2]);
|
|
52
|
+
d[i] = d[i + 1] = d[i + 2] = grey;
|
|
53
|
+
d[i + 3] = Math.round(d[i + 3] * 0.55);
|
|
54
|
+
}
|
|
55
|
+
out[size] = img;
|
|
56
|
+
}
|
|
57
|
+
return out;
|
|
58
|
+
})().catch((e) => {
|
|
59
|
+
greyIcon = null;
|
|
60
|
+
throw e;
|
|
61
|
+
});
|
|
62
|
+
return await greyIcon;
|
|
63
|
+
}
|
|
64
|
+
const COLOUR_ICON = Object.fromEntries(ICON_SIZES.map((size) => [size, `icons/icon-${size}.png`]));
|
|
65
|
+
|
|
66
|
+
export class Toolbar {
|
|
67
|
+
/**
|
|
68
|
+
* @param {import("./access.js").AccessPolicy} policy
|
|
69
|
+
* @param {() => import("./group.js").MirasimTabGroup | null} activeGroup
|
|
70
|
+
*/
|
|
71
|
+
constructor(policy, activeGroup) {
|
|
72
|
+
this._policy = policy;
|
|
73
|
+
this._activeGroup = activeGroup;
|
|
74
|
+
this._seen = true;
|
|
75
|
+
this._ready = chrome.storage.local
|
|
76
|
+
.get(PANEL_SEEN_KEY)
|
|
77
|
+
.then((v) => {
|
|
78
|
+
this._seen = v?.[PANEL_SEEN_KEY] === true;
|
|
79
|
+
})
|
|
80
|
+
.catch(() => {});
|
|
81
|
+
|
|
82
|
+
chrome.action.onClicked.addListener((tab) => void this._onClicked(tab));
|
|
83
|
+
chrome.tabs.onUpdated.addListener((tabId) => void this.refresh(tabId));
|
|
84
|
+
chrome.tabs.onCreated.addListener((tab) => tab.id !== undefined && void this.refresh(tab.id));
|
|
85
|
+
chrome.tabGroups.onUpdated.addListener(() => void this.refreshAll());
|
|
86
|
+
chrome.tabGroups.onRemoved.addListener(() => void this.refreshAll());
|
|
87
|
+
chrome.storage.onChanged.addListener((changes, area) => {
|
|
88
|
+
if (area === "local" && changes[PANEL_SEEN_KEY]) {
|
|
89
|
+
this._seen = changes[PANEL_SEEN_KEY].newValue === true;
|
|
90
|
+
void this.refreshAll();
|
|
91
|
+
}
|
|
92
|
+
});
|
|
93
|
+
policy.onChange(() => void this.refreshAll());
|
|
94
|
+
void policy.ready.then(() => this.refreshAll());
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** A click on a tab with no popup: hand it to the agent. */
|
|
98
|
+
async _onClicked(tab) {
|
|
99
|
+
if (tab?.id === undefined) return;
|
|
100
|
+
try {
|
|
101
|
+
await addToMirasimGroup(tab.id);
|
|
102
|
+
} catch (e) {
|
|
103
|
+
debugLog("hand-over failed:", e);
|
|
104
|
+
await chrome.action.setTitle({ tabId: tab.id, title: `${t("handOverFailed")} ${e?.message ?? e}` }).catch(() => {});
|
|
105
|
+
return;
|
|
106
|
+
}
|
|
107
|
+
await this.refresh(tab.id);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
async refreshAll() {
|
|
111
|
+
await this._ready;
|
|
112
|
+
const tabs = await chrome.tabs.query({}).catch(() => []);
|
|
113
|
+
await Promise.all(tabs.map((tab) => this._apply(tab)));
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
async refresh(tabId) {
|
|
117
|
+
await this._ready;
|
|
118
|
+
const tab = await chrome.tabs.get(tabId).catch(() => null);
|
|
119
|
+
if (tab) await this._apply(tab);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** Where a tab stands, for the toolbar and the panel alike. */
|
|
123
|
+
async stateOf(tab) {
|
|
124
|
+
const group = this._activeGroup();
|
|
125
|
+
const handedOver = tab.groupId !== undefined && tab.groupId >= 0
|
|
126
|
+
&& (await chrome.tabGroups.get(tab.groupId).catch(() => null))?.title === GROUP_TITLE;
|
|
127
|
+
return {
|
|
128
|
+
connected: !!group,
|
|
129
|
+
agents: !!group?.isAgentTab(tab.id),
|
|
130
|
+
restricted: isNonDebuggableUrl(tab.url) || isNonDebuggableUrl(tab.pendingUrl),
|
|
131
|
+
siteVerdict: this._policy.siteVerdict(tab.url) ?? this._policy.siteVerdict(tab.pendingUrl),
|
|
132
|
+
handedOver,
|
|
133
|
+
using: !!group?.isAttached(tab.id),
|
|
134
|
+
granted: this._policy.grants.has(tab.id),
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
async _apply(tab) {
|
|
139
|
+
if (tab.id === undefined) return;
|
|
140
|
+
const s = await this.stateOf(tab);
|
|
141
|
+
const joinable = !s.handedOver && !s.agents && !s.restricted && !s.siteVerdict;
|
|
142
|
+
const reachable = (s.handedOver || s.agents || s.using) && !s.restricted && !s.siteVerdict;
|
|
143
|
+
let badge = BADGES.none;
|
|
144
|
+
let tip;
|
|
145
|
+
if (s.using) (badge = BADGES.using), (tip = t("tipUsing"));
|
|
146
|
+
else if (s.siteVerdict) (badge = BADGES.blocked), (tip = t("tipBlocked"));
|
|
147
|
+
else if (s.restricted) tip = t("tipRestricted");
|
|
148
|
+
else if (s.handedOver || s.agents) tip = t("tipHandedOver");
|
|
149
|
+
else tip = t("tipJoin");
|
|
150
|
+
if (this._policy.paused) badge = BADGES.paused;
|
|
151
|
+
else if (badge === BADGES.none && !this._seen) badge = BADGES.fresh;
|
|
152
|
+
const prefix = this._policy.paused ? t("tipPausedPrefix") : !s.connected ? t("tipOfflinePrefix") : "";
|
|
153
|
+
const tabId = tab.id;
|
|
154
|
+
let icon;
|
|
155
|
+
try {
|
|
156
|
+
icon = reachable ? { path: COLOUR_ICON } : { imageData: await greyedIcon() };
|
|
157
|
+
} catch (e) {
|
|
158
|
+
debugLog("grey icon failed:", e);
|
|
159
|
+
}
|
|
160
|
+
try {
|
|
161
|
+
await Promise.all([
|
|
162
|
+
icon ? chrome.action.setIcon({ tabId, ...icon }) : Promise.resolve(),
|
|
163
|
+
chrome.action.setPopup({ tabId, popup: joinable ? "" : PANEL }),
|
|
164
|
+
chrome.action.setBadgeText({ tabId, text: badge.text }),
|
|
165
|
+
chrome.action.setBadgeBackgroundColor({ tabId, color: badge.color }),
|
|
166
|
+
chrome.action.setTitle({ tabId, title: `${prefix}${tip}` }),
|
|
167
|
+
]);
|
|
168
|
+
} catch {
|
|
169
|
+
/* tab already closed */
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mirasim/computer-use",
|
|
3
|
-
"version": "0.1.0-rc.
|
|
3
|
+
"version": "0.1.0-rc.7",
|
|
4
4
|
"description": "Computer use for any agent: an MCP server that drives browsers, macOS apps, Windows apps and Electron apps through their accessibility trees.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"keywords": [
|
|
@@ -70,9 +70,9 @@
|
|
|
70
70
|
"access": "public"
|
|
71
71
|
},
|
|
72
72
|
"optionalDependencies": {
|
|
73
|
-
"@mirasim/computer-use-darwin-arm64": "0.1.0-rc.
|
|
74
|
-
"@mirasim/computer-use-darwin-x64": "0.1.0-rc.
|
|
75
|
-
"@mirasim/computer-use-win32-x64": "0.1.0-rc.
|
|
76
|
-
"@mirasim/computer-use-win32-arm64": "0.1.0-rc.
|
|
73
|
+
"@mirasim/computer-use-darwin-arm64": "0.1.0-rc.7",
|
|
74
|
+
"@mirasim/computer-use-darwin-x64": "0.1.0-rc.7",
|
|
75
|
+
"@mirasim/computer-use-win32-x64": "0.1.0-rc.7",
|
|
76
|
+
"@mirasim/computer-use-win32-arm64": "0.1.0-rc.7"
|
|
77
77
|
}
|
|
78
78
|
}
|
package/scripts/extension.mjs
CHANGED
|
@@ -14,8 +14,10 @@ import { parseArgs } from 'node:util';
|
|
|
14
14
|
import { crc32 } from 'node:zlib';
|
|
15
15
|
|
|
16
16
|
export const extensionSource = fileURLToPath(new URL('../extension/', import.meta.url));
|
|
17
|
-
const files = ['background.js', 'group.js', 'relay.js', 'runtime-info.js', 'transport.js',
|
|
18
|
-
'connect.html', 'connect.js',
|
|
17
|
+
const files = ['background.js', 'group.js', 'relay.js', 'runtime-info.js', 'transport.js', 'access.js', 'toolbar.js',
|
|
18
|
+
'connect.html', 'connect.js', 'panel.html', 'panel.css', 'panel.js', 'ask.html', 'ask.js',
|
|
19
|
+
'_locales/en/messages.json', '_locales/zh_CN/messages.json',
|
|
20
|
+
...[16, 32, 48, 128].map((size) => `icons/icon-${size}.png`)];
|
|
19
21
|
|
|
20
22
|
export function extensionId(key) {
|
|
21
23
|
return [...createHash('sha256').update(Buffer.from(key, 'base64')).digest().subarray(0, 16)]
|