@mirasim/computer-use 0.1.0-rc.6 → 0.1.0-rc.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/platform/browser/bridge/model-commands.js +1 -1
- package/dist/platform/browser/bridge/model-core.d.ts +7 -0
- package/dist/platform/browser/bridge/model-core.js +1 -1
- package/dist/platform/browser/bridge/server.js +1 -1
- package/dist/platform/browser/bridge-tabs.d.ts +3 -0
- package/dist/platform/browser/bridge-tabs.js +1 -1
- package/dist/platform/browser/driver-verbs.js +2 -2
- package/dist/platform/browser/driver.d.ts +2 -0
- package/dist/platform/browser/driver.js +3 -2
- package/dist/platform/browser/messages.d.ts +5 -0
- package/dist/platform/browser/messages.js +2 -2
- package/dist/platform/chromium/actions.js +2 -2
- package/dist/platform/chromium/run-js.d.ts +5 -1
- package/dist/platform/chromium/run-js.js +2 -2
- package/dist/platform/electron/driver.js +1 -1
- package/dist/platform/electron/verbs.js +1 -1
- package/dist/platform/macos/type-verb.js +1 -1
- package/dist/platform/tools/api-contract.d.ts +1 -0
- package/dist/platform/tools/api.js +1 -1
- package/dist/platform/tools/repl-docs.js +1 -1
- package/dist/platform/tools/repl.js +12 -12
- package/extension/_locales/en/messages.json +179 -0
- package/extension/_locales/zh_CN/messages.json +179 -0
- package/extension/access.js +245 -0
- package/extension/ask.html +26 -0
- package/extension/ask.js +20 -0
- package/extension/background.js +79 -3
- package/extension/group.js +335 -79
- package/extension/icons/icon-128.png +0 -0
- package/extension/icons/icon-16.png +0 -0
- package/extension/icons/icon-32.png +0 -0
- package/extension/icons/icon-48.png +0 -0
- package/extension/manifest.json +8 -2
- package/extension/panel.css +86 -0
- package/extension/panel.html +86 -0
- package/extension/panel.js +192 -0
- package/extension/relay.js +264 -5
- package/extension/toolbar.js +172 -0
- package/package.json +5 -5
- package/scripts/extension.mjs +4 -2
package/README.md
CHANGED
|
@@ -67,7 +67,9 @@ approval_mode = "approve"
|
|
|
67
67
|
Chrome is required), keeping its profile under `~/.mirasim-computer-use` so logins carry over
|
|
68
68
|
between sessions. To drive the browser you are signed into instead, install the
|
|
69
69
|
[Mirasim extension](https://chromewebstore.google.com/detail/mirasim/khflooofmndpmafhofjdcpjfgnlpikhg)
|
|
70
|
-
and add `-e COMPUTER_USE_BROWSER=extension`.
|
|
70
|
+
and add `-e COMPUTER_USE_BROWSER=extension`. By default the agent sees only the tabs it opens and
|
|
71
|
+
the ones you hand it: click the extension's toolbar icon on a tab to hand that tab over, and use
|
|
72
|
+
its panel to take tabs back, pause the agent, widen what it may reach or block sites.
|
|
71
73
|
|
|
72
74
|
In extension mode the first browser call starts a bridge in the background and registers it as
|
|
73
75
|
the extension's native-messaging host: a record under `~/.mirasim-computer-use/native-host/`
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{FAKE_WINDOW as g,tabIsUnloaded as w}from"./client.js";import{BridgeModelCore as f}from"./model-core.js";import{hostname as T}from"node:os";class p extends f{restoringTabs=new Map;async handleCommand(e,
|
|
1
|
+
import{FAKE_WINDOW as g,tabIsUnloaded as w}from"./client.js";import{BridgeModelCore as f}from"./model-core.js";import{hostname as T}from"node:os";class p extends f{restoringTabs=new Map;async handleCommand(e,a,t,r){if(r&&e.browserSessions.has(r))return await this.handleCommand(e,a,t,void 0);const n=t??{};switch(a){case"Mirasim.tabs":return{tabs:await this.listTargets(e),browserSession:this.browserSession,access:this.access};case"Mirasim.access":return{access:this.access};case"Mirasim.tabTarget":return{targetId:await this.tabTarget(Number(n.tabId),String(n.browserSession??""))};case"Mirasim.restoreTab":{const{attachment:s,result:i}=await this.restoreTab(Number(n.tabId),String(n.targetId??""),String(n.browserSession??""));return e.heldTabs.add(s.tabId),this.announce(e,s),i}case"Mirasim.holdOnly":{const s=typeof n.targetId=="string"&&n.targetId?this.tabIdForTarget(n.targetId):void 0;return this.holdOnly(e,s),s===void 0||!e.heldTabs.has(s)?{held:!1}:{held:!0,tabId:s,browserSession:this.browserSession}}case"Mirasim.closeTab":{this.assertBrowserSession(String(n.browserSession??""));const s=Number(n.tabId);if(!this.knownTabs.has(s))throw new Error(`Tab ${s} was closed; list tabs again.`);return await this.sendToExtension("chrome.tabs.remove",[s]),{}}case"Browser.getVersion":return{protocolVersion:"1.3",product:"Chrome/mirasim-bridge",userAgent:"mirasim-bridge/1.0"};case"Browser.setDownloadBehavior":return{};case"Mirasim.downloads":{if(typeof n.since!="string"||!Number.isFinite(Date.parse(n.since)))throw new Error("Downloads require an ISO start time");if(n.until!==void 0&&(typeof n.until!="string"||!Number.isFinite(Date.parse(n.until))))throw new Error("Downloads require an ISO end time");const s=await this.sendToExtension("chrome.downloads.search",[{startedAfter:n.since,...n.until===void 0?{}:{startedBefore:n.until}}]);return{host:T(),items:s}}case"Browser.getWindowForTarget":return structuredClone(g);case"Browser.setWindowBounds":return{};case"Mirasim.attachRefusals":{const s=Date.now()-6e4;return{refusals:[...this.attachRefusals.values()].filter(i=>i.at>=s).map(({url:i,reason:d})=>({url:i,reason:d}))}}case"Browser.close":throw new Error("Browser.close is not available on the mirasim bridge \u2014 the browser is the user's own");case"Target.setAutoAttach":if(!r)return await this.enableAutoAttach(e),{};break;case"Target.createTarget":return await this.createTarget(typeof n.url=="string"?n.url:void 0);case"Target.closeTarget":return await this.closeTarget(typeof n.targetId=="string"?n.targetId:void 0);case"Target.attachToTarget":if(!r)return await this.attachToTarget(e,typeof n.targetId=="string"?n.targetId:void 0);break;case"Target.attachToBrowserTarget":if(!r)return{sessionId:e.browserAlias()};break;case"Runtime.enable":{if(!r)throw new Error("Runtime.enable requires a target session");const s=this.resolveSession(e,r),i=this.runtimeForSession(s.tabId,s.cdpSessionId),d=e.enableRuntime(r);try{const c=await i.enable(()=>this.sendToExtension("chrome.debugger.sendCommand",[{tabId:s.tabId,...s.cdpSessionId?{sessionId:s.cdpSessionId}:{}},a,...t===void 0?[]:[t]]));if(d)for(const u of i.values())e.runtimeEvent(r,"Runtime.executionContextCreated",u,!0);return c}catch(c){throw d&&e.disableRuntime(r),c}}case"Runtime.disable":if(!r)throw new Error("Runtime.disable requires a target session");return this.resolveSession(e,r),e.disableRuntime(r),{};case"Target.detachFromTarget":{const s=typeof n.sessionId=="string"?n.sessionId:void 0,i=s?e.virtualSessions.get(s):void 0;if(s&&i!==void 0)return e.virtualSessions.delete(s),e.forgetSession(s),this.releaseUnusedTab(i),{};if(s&&e.browserSessions.delete(s))return{};const d=s?this.childSessionOwner.get(s):void 0;if(s&&d===void 0)return{};if(s&&d!==void 0)return await this.sendToExtension("chrome.debugger.sendCommand",[{tabId:d},a,{sessionId:s}]);break}}if(!r)return await this.sendBrowserCommand(a,t);const o=this.resolveSession(e,r);return await this.sendToExtension("chrome.debugger.sendCommand",[{tabId:o.tabId,...o.cdpSessionId?{sessionId:o.cdpSessionId}:{}},a,...t===void 0?[]:[t]])}async enableAutoAttach(e){e.autoAttach=!0,await this.reconcileKnownTabs();const a=[...this.knownTabs].filter(([t,r])=>!w(r)&&this.wants(e,t)).map(([t])=>t);await Promise.all(a.map(t=>this.attachAndAnnounce(e,t).catch(r=>this.log("attach failed:",r))))}async reconcileKnownTabs(){let e;try{e=await this.sendToExtension("chrome.tabs.query",[{}])??[]}catch(t){this.log("tab reconcile failed:",t);return}const a=new Map(e.filter(t=>t.id!==void 0).map(t=>[t.id,t]));for(const t of[...this.knownTabs.keys()]){const r=a.get(t);if(!r)this.knownTabs.delete(t),this.agentTabs.delete(t),this.dropTab(t);else{const n=w(this.knownTabs.get(t));if(this.knownTabs.set(t,r),w(r)&&this.attachments.has(t))this.dropTab(t),this.sendToExtension("bridge.releaseTab",[t]).catch(o=>this.log("release unloaded tab failed:",o));else if(n&&!w(r))for(const o of this.clients)this.wants(o,t)&&this.attachAndAnnounce(o,t).catch(s=>this.log("restored tab attach failed:",s))}}}assertBrowserSession(e){if(!e||!this.browserSession)throw new Error("Reload the Mirasim extension before restoring tabs: this connection has no browser-session identity.");if(e!==this.browserSession)throw new Error("The browser restarted; list tabs and bind the intended target again.")}async tabTarget(e,a){if(this.assertBrowserSession(a),await this.reconcileKnownTabs(),!this.knownTabs.has(e))throw new Error(`Tab ${e} was closed; list tabs again.`);let t;try{t=await this.sendToExtension("chrome.debugger.getTargets",[])}catch(n){throw String(n).includes("Unknown method")?new Error("Reload the Mirasim extension to enable restoring unloaded tabs."):n}const r=t.find(n=>n.tabId===e);if(this.assertBrowserSession(a),!r?.id)throw new Error(`Chrome did not identify tab ${e}; refusing to restore an unknown target.`);return r.id}restoreTab(e,a,t){this.assertBrowserSession(t);const r=`${t}:${e}`,n=this.restoringTabs.get(r);if(n)return n.then(s=>{if(s.result.targetId!==a)throw new Error("Tab identity changed; list and bind it again before acting.");return s});const o=(async()=>{if(!a)throw new Error("Restoring a tab requires its claimed target identity.");if(await this.tabTarget(e,t)!==a)throw new Error(`Tab ${e} changed identity before restoration; no reload was sent.`);const s=this.knownTabs.get(e);if(!s)throw new Error(`Tab ${e} was closed; list tabs again.`);const i=w(s);if(i){try{await this.sendToExtension("chrome.tabs.reload",[e])}catch(b){const l=b instanceof Error?b.message:String(b);throw new Error(l.includes("Unknown method")?"Reload the Mirasim extension to enable restoring unloaded tabs.":`Could not restore tab ${e}: ${l}`)}const h=Date.now()+1e4;for(;w(this.knownTabs.get(e)??{discarded:!0});){if(Date.now()>=h)throw new Error(`Tab ${e} remained unloaded after restoration; no action was sent to its page.`);if(await new Promise(b=>setTimeout(b,50)),await this.reconcileKnownTabs(),!this.knownTabs.has(e))throw new Error(`Tab ${e} closed while restoring; no replacement tab was opened.`)}}const d=await this.ensureAttached(e);if(d.targetInfo.targetId!==a)throw new Error(`Tab ${e} changed identity while restoring; list and bind it again before acting.`);const c=[...this.knownTabs.values()].filter(h=>h.windowId===s.windowId&&h.splitViewId===s.splitViewId&&(h.width??0)>0&&(h.height??0)>0),u=c.find(h=>h.id===e)??c.find(h=>h.active)??c[0];return{attachment:d,result:{targetId:d.targetInfo.targetId,reloaded:i,...u?{viewport:{width:u.width,height:u.height}}:{}}}})();return this.restoringTabs.set(r,o),o.finally(()=>this.restoringTabs.delete(r)).catch(()=>{}),o}async createTarget(e){const a=await this.sendToExtension("chrome.tabs.create",[{url:e||"about:blank",active:!1}]);if(a?.id===void 0)throw new Error("Failed to create tab");this.knownTabs.set(a.id,a),this.agentTabs.add(a.id);const t=await this.ensureAttached(a.id);for(const r of this.clients)r.autoAttach&&this.announce(r,t);return{targetId:t.targetInfo.targetId}}async recycleAgentTabs(){await this.reconcileKnownTabs();const e=[...this.agentTabs].filter(t=>this.knownTabs.has(t));let a=0;for(const t of e){try{await this.sendToExtension("chrome.tabs.remove",[t]),a+=1}catch(r){this.log("recycle failed for tab",t,r)}this.knownTabs.delete(t),this.agentTabs.delete(t),this.dropTab(t)}return{closed:a}}async closeTarget(e){const a=e?this.tabIdForTarget(e):void 0;return a===void 0?{success:!1}:(await this.sendToExtension("chrome.tabs.remove",[a]),{success:!0})}async attachToTarget(e,a){const t=a?this.tabIdForTarget(a):void 0;if(t===void 0)throw new Error(`No target with id: ${a}`);return await this.ensureAttached(t),{sessionId:e.virtualAlias(t)}}async sendBrowserCommand(e,a){if(e==="Target.getTargetInfo"&&!a?.targetId)return{targetInfo:{targetId:"mirasim-browser",type:"browser",title:"",url:"",attached:!0}};const t=this.attachments.values().next().value,r=t?.tabId??[...this.knownTabs].find(([,n])=>!w(n))?.[0];if(r===void 0)throw new Error(`No tab to forward browser-level command: ${e}`);t||await this.ensureAttached(r);try{return await this.sendToExtension("chrome.debugger.sendCommand",[{tabId:r},e,...a===void 0?[]:[a]])}finally{t||this.releaseUnusedTab(r)}}}export{p as BridgeModelCommands};
|
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
export type SendToExtension = (method: string, params: unknown[]) => Promise<unknown>;
|
|
2
2
|
import { BridgeClient, type BridgeTab, type TabAttachment, type CdpMessage } from "./client.ts";
|
|
3
3
|
import { RuntimeDomain } from "./runtime-contexts.ts";
|
|
4
|
+
export interface ExtensionAccess {
|
|
5
|
+
scope: "group" | "all" | "ask";
|
|
6
|
+
siteRules: boolean;
|
|
7
|
+
paused: boolean;
|
|
8
|
+
}
|
|
4
9
|
export { BridgeClient, type BridgeTab, type TabAttachment } from "./client.ts";
|
|
5
10
|
export declare abstract class BridgeModelCore {
|
|
6
11
|
protected readonly sendToExtension: SendToExtension;
|
|
@@ -8,6 +13,7 @@ export declare abstract class BridgeModelCore {
|
|
|
8
13
|
protected abstract reconcileKnownTabs(): Promise<void>;
|
|
9
14
|
protected knownTabs: Map<number, BridgeTab>;
|
|
10
15
|
protected agentTabs: Set<number>;
|
|
16
|
+
access: ExtensionAccess | null;
|
|
11
17
|
protected browserSession: string | null;
|
|
12
18
|
protected attachments: Map<number, TabAttachment>;
|
|
13
19
|
private attaching;
|
|
@@ -24,6 +30,7 @@ export declare abstract class BridgeModelCore {
|
|
|
24
30
|
addClient(send: (message: CdpMessage) => void): BridgeClient;
|
|
25
31
|
removeClient(client: BridgeClient): void;
|
|
26
32
|
clientCount(): number;
|
|
33
|
+
setAccess(raw: unknown): void;
|
|
27
34
|
onTabCreated(tab: BridgeTab, opener?: {
|
|
28
35
|
agent?: boolean;
|
|
29
36
|
}): void;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{BridgeClient as u,STAND_IN_PREFIX as h,tabIsUnloaded as c}from"./client.js";import{RuntimeDomain as g,isRuntimeContextEvent as T}from"./runtime-contexts.js";import{BridgeClient as
|
|
1
|
+
import{BridgeClient as u,STAND_IN_PREFIX as h,tabIsUnloaded as c}from"./client.js";import{RuntimeDomain as g,isRuntimeContextEvent as T}from"./runtime-contexts.js";import{BridgeClient as I}from"./client.js";const w=1e3;function b(l,e){const t=s=>{try{return new URL(s).href.replace(/\?$/,"")}catch{return s}};return t(l)===t(e)}class y{sendToExtension;log;knownTabs=new Map;agentTabs=new Set;access=null;browserSession=null;attachments=new Map;attaching=new Map;childSessionOwner=new Map;attachRefusals=new Map;windowOpens=[];clients=new Set;nextClientId=0;constructor(e,t=()=>{}){this.sendToExtension=e,this.log=t}addClient(e){const t=new u(++this.nextClientId,e);return this.clients.add(t),t}removeClient(e){this.clients.delete(e),e.dispose();const t=new Set([...e.sessionByTabId.keys(),...e.virtualSessions.values()]);for(const s of t)this.releaseUnusedTab(s)}clientCount(){return this.clients.size}setAccess(e){const t=e;!t||typeof t!="object"||!["group","all","ask"].includes(String(t.scope))||(this.access={scope:t.scope,siteRules:t.siteRules===!0,paused:t.paused===!0})}onTabCreated(e,t={}){if(e.id===void 0)return;const s=!this.knownTabs.has(e.id);if(this.knownTabs.set(e.id,e),t.agent&&this.agentTabs.add(e.id),c(e)){this.attachments.has(e.id)&&(this.dropTab(e.id),this.sendToExtension("bridge.releaseTab",[e.id]).catch(n=>this.log("release unloaded tab failed:",n)));return}s&&this.claimPopup(e);for(const n of this.clients)this.wants(n,e.id)&&this.attachAndAnnounce(n,e.id).catch(i=>this.log("auto-attach failed:",i))}claimPopup(e){if(this.agentTabs.has(e.id))return;const t=Date.now();this.windowOpens=this.windowOpens.filter(a=>t-a.at<=w);const s=e.url??"",n=this.windowOpens.findIndex(a=>!s||s==="about:blank"||a.url==="about:blank"||b(a.url,s));if(n<0)return;const[i]=this.windowOpens.splice(n,1);for(const a of this.clients)!a.autoAttach||!a.sessionByTabId.has(i.from)||(this.agentTabs.has(i.from)||a.keptTabs.has(i.from)?a.keptTabs.add(e.id):a.heldTabs.add(e.id))}onWindowOpen(e,t){const s=t?.url;[...this.clients].some(n=>n.autoAttach&&n.sessionByTabId.has(e))&&this.windowOpens.push({from:e,url:typeof s=="string"?s:"",at:Date.now()})}onTabRemoved(e){this.knownTabs.delete(e),this.agentTabs.delete(e);for(const t of this.clients)t.heldTabs.delete(e),t.keptTabs.delete(e);this.dropTab(e)}onTabReplaced(e,t){const s=this.agentTabs.has(t),n=[...this.clients].filter(a=>a.heldTabs.has(t)),i=[...this.clients].filter(a=>a.keptTabs.has(t));this.onTabRemoved(t),s&&this.agentTabs.add(e);for(const a of n)a.heldTabs.add(e);for(const a of i)a.keptTabs.add(e)}onDebuggerEvent(e,t,s){const n=e.tabId;if(n===void 0||!this.attachments.has(n)||e.sessionId&&this.childSessionOwner.get(e.sessionId)!==n)return;t==="Page.windowOpen"&&this.onWindowOpen(n,s);const i=s?.sessionId;if(t==="Target.attachedToTarget"&&i)this.childSessionOwner.set(i,n);else if(t==="Target.detachedFromTarget"&&i){this.childSessionOwner.delete(i),this.attachments.get(n)?.runtimes.delete(i);for(const o of this.clients)o.forgetSession(i)}if(T(t)&&this.runtimeForSession(n,e.sessionId).update(t,s),e.sessionId){for(const o of this.clients)t.startsWith("Runtime.")?o.runtimeEvent(e.sessionId,t,s):o.send({sessionId:e.sessionId,method:t,params:s});return}for(const o of this.clients){const r=o.sessionByTabId.get(n);r&&(t.startsWith("Runtime.")?o.runtimeEvent(r,t,s):o.send({sessionId:r,method:t,params:s}));for(const[d,f]of o.virtualSessions)f===n&&(t.startsWith("Runtime.")?o.runtimeEvent(d,t,s):o.send({sessionId:d,method:t,params:s}))}}onDebuggerDetach(e){e.tabId!==void 0&&this.dropTab(e.tabId)}onExtensionDisconnected(){for(const e of[...this.attachments.keys()])this.dropTab(e);this.knownTabs.clear(),this.childSessionOwner.clear()}onExtensionConnected(e){this.access=null;const t=!!e&&e===this.browserSession;t||this.agentTabs.clear(),this.log(`extension connected: ${t?"same":"new"} browser session, kept ${this.agentTabs.size} agent tab mark(s)`),this.browserSession=e??null}async listTargets(e){await this.reconcileKnownTabs();let t=[];try{t=await this.sendToExtension("chrome.tabs.query",[{}])??[]}catch(i){this.log("live tab query failed:",i)}const s=new Map(t.filter(i=>i.id!==void 0).map(i=>[i.id,i])),n=[];for(const i of this.knownTabs.keys()){const a=s.get(i),o=this.attachments.get(i)?.targetInfo;n.push({tabId:i,id:typeof o?.targetId=="string"?o.targetId:`${h}${i}`,type:"page",url:a?.url??(typeof o?.url=="string"?o.url:""),title:a?.title??(typeof o?.title=="string"?o.title:""),unloaded:c(a??this.knownTabs.get(i)),...e?{held:e.sessionByTabId.has(i)}:{}})}return n}holdOnly(e,t){for(const s of[...e.heldTabs])s!==t&&!this.agentTabs.has(s)&&this.releaseClientTab(e,s)}releaseClientTab(e,t){e.heldTabs.delete(t);const s=this.attachments.get(t),n=e.dropTab(t);n&&s&&e.send({method:"Target.detachedFromTarget",params:{sessionId:n,targetId:s.targetInfo.targetId}}),this.releaseUnusedTab(t)}wants(e,t){return e.autoAttach&&(this.agentTabs.has(t)||e.keptTabs.has(t)||e.heldTabs.has(t))}releaseUnusedTab(e){!this.attachments.has(e)||[...this.clients].some(t=>t.sessionByTabId.has(e)||[...t.virtualSessions.values()].includes(e))||(this.dropTab(e),this.sendToExtension("bridge.releaseTab",[e]).catch(t=>this.log("release failed:",t)))}runtimeForSession(e,t){const s=this.attachments.get(e);if(!s)throw new Error(`No debugger attachment for tab: ${e}`);const n=t??"";let i=s.runtimes.get(n);return i||(i=new g,s.runtimes.set(n,i)),i}resolveSession(e,t){const s=e.tabBySessionId.get(t);if(s!==void 0)return{tabId:s};const n=e.virtualSessions.get(t);if(n!==void 0)return{tabId:n};const i=this.childSessionOwner.get(t);if(i!==void 0)return{tabId:i,cdpSessionId:t};throw new Error(`No tab for sessionId: ${t}`)}tabIdForTarget(e){for(const s of this.attachments.values())if(s.targetInfo.targetId===e)return s.tabId;if(!e.startsWith(h))return;const t=Number(e.slice(h.length));return Number.isInteger(t)&&this.knownTabs.has(t)?t:void 0}async attachAndAnnounce(e,t){let s;try{s=await this.ensureAttached(t)}catch(n){this.attachRefusals.set(t,{url:this.knownTabs.get(t)?.url??"",reason:n instanceof Error?n.message:String(n),at:Date.now()});for(const i of this.attachRefusals.keys()){if(this.attachRefusals.size<=20)break;this.attachRefusals.delete(i)}throw n}this.attachRefusals.delete(t),this.announce(e,s)}announce(e,t){if(e.sessionByTabId.has(t.tabId))return;const s=e.tabAlias(t.tabId);e.send({method:"Target.attachedToTarget",params:{sessionId:s,targetInfo:{...t.targetInfo,attached:!0},waitingForDebugger:!1}})}ensureAttached(e){const t=this.knownTabs.get(e);if(!t)return Promise.reject(new Error(`Tab ${e} was closed; list tabs again.`));if(c(t))return Promise.reject(new Error(`Tab ${e} is unloaded by Chrome. Restore this tab before attaching its renderer.`));const s=this.attachments.get(e);if(s)return Promise.resolve(s);const n=this.attaching.get(e);if(n)return n;const i=(async()=>{await this.sendToExtension("chrome.debugger.attach",[{tabId:e},"1.3"]);const a=await this.sendToExtension("chrome.debugger.sendCommand",[{tabId:e},"Target.getTargetInfo"]);if(!a?.targetInfo?.targetId)throw new Error(`Tab ${e} reported no targetInfo`);const o={tabId:e,targetInfo:a.targetInfo,runtimes:new Map};return this.attachments.set(e,o),o})();return this.attaching.set(e,i),i.catch(()=>{}).finally(()=>this.attaching.delete(e)),i}dropTab(e){const t=this.attachments.get(e);this.attachments.delete(e);for(const[s,n]of this.childSessionOwner)if(n===e){this.childSessionOwner.delete(s);for(const i of this.clients)i.forgetSession(s)}for(const s of this.clients){const n=s.dropTab(e);n&&t&&s.send({method:"Target.detachedFromTarget",params:{sessionId:n,targetId:t.targetInfo.targetId}})}}}export{I as BridgeClient,y as BridgeModelCore};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import"node:http";import{WebSocket as l}from"ws";import"./model.js";import{BridgeServerCore as c,ExtensionConnection as u}from"./server-core.js";import{webSocketChannel as m}from"./extension-channel.js";import{parseExtensionRuntime as f}from"./extension-updates.js";const
|
|
1
|
+
import"node:http";import{WebSocket as l}from"ws";import"./model.js";import{BridgeServerCore as c,ExtensionConnection as u}from"./server-core.js";import{webSocketChannel as m}from"./extension-channel.js";import{parseExtensionRuntime as f}from"./extension-updates.js";import{ownVersion as x}from"../../../entries/version.js";const p=2e4,g=5e3,v=5e3;function b(){const r=x();return{name:process.versions.electron?"Mirasim":"Computer Use",version:r==="0.0.0"?"":r}}class D extends c{async handleUpgrade(t,e,n){const i=new URL(t.url??"/",`http://127.0.0.1:${this.port}`),s=t.headers.origin;if(i.pathname===`/extension/${this.uuid}`){if(s&&!this.extensionIds.some(o=>s===`chrome-extension://${o}`)){e.destroy();return}this.wss.handleUpgrade(t,e,n,o=>this.attachExtension(m(o)));return}if(i.pathname===`/cdp/${this.uuid}`){if(s){e.destroy();return}try{await this.ensureExtension()}catch(o){this.log("rejecting CDP client:",o),e.destroy();return}this.wss.handleUpgrade(t,e,n,o=>this.attachCdpClient(o));return}e.destroy()}attachExtension(t){const e=new u(t,this.log,this.warn);if(!this.extension){this.activateExtension(e,[]);return}const n=[];let i=!1;e.onEvent=(s,o)=>{if(n.push([s,o]),!i&&s==="extension.connected"){i=!0;const d=typeof o[0]=="string"?o[0]:null;this.admitExtension(e,n,d)}},e.onClose=()=>{this.standby.delete(e)}}async admitExtension(t,e,n){const i=this.extension,s=!n||!this.extensionSession||n===this.extensionSession;if(i&&!s&&await E(i,g)){if(this.extension!==i)return this.admitExtension(t,e,n);if(!t.isOpen)return;t.onEvent=null,this.standby.add(t),this.warn("another browser's extension connected while one is active; it waits in standby until the active browser goes");return}t.isOpen&&this.activateExtension(t,e)}activateExtension(t,e){const n=this.extension;this.extension=null,n?.close("Replaced by a new extension connection"),n&&this.model.onExtensionDisconnected(),this.keepalive&&clearInterval(this.keepalive),this.updates.reset(),this.extension=t,this.extensionInitialized=!1,this.extensionSession=null,t.onEvent=(i,s)=>{this.extension===t&&this.onExtensionEvent(i,s)},t.onClose=()=>{this.extension===t&&(this.extension=null,this.extensionSession=null,this.updates.reset(),this.extensionInitialized=!1,this.keepalive&&clearInterval(this.keepalive),this.keepalive=null,this.model.onExtensionDisconnected(),setTimeout(()=>{if(!this.extension){for(const i of this.standby)i.close("The active browser left; reconnect to take over");this.standby.clear()}},v).unref?.())};for(const[i,s]of e)this.onExtensionEvent(i,s);this.keepalive=setInterval(()=>{t.send("bridge.ping",[]).catch(()=>{})},p),this.keepalive.unref?.()}onExtensionEvent(t,e){switch(t){case"chrome.tabs.onCreated":{const[n,i]=e;this.model.onTabCreated(n??{},i??{});break}case"chrome.tabs.onRemoved":this.model.onTabRemoved(e[0]);break;case"chrome.tabs.onReplaced":this.model.onTabReplaced(...e);break;case"chrome.debugger.onEvent":{const[n,i,s]=e;this.model.onDebuggerEvent(n??{},i,s);break}case"chrome.debugger.onDetach":this.model.onDebuggerDetach(e[0]??{});break;case"extension.connected":{this.extensionSession=typeof e[0]=="string"?e[0]:null,this.model.onExtensionConnected(e[0]);break}case"extension.initialized":{const n=f(e[0]);if(n&&!this.extensionIds.includes(n.id)){this.extension?.close("Unexpected extension identity");return}this.updates.runtime=n,this.extensionInitialized=!0,this.extension?.send("bridge.info",[b()]).catch(()=>{});const i=this.readyWaiters;this.readyWaiters=[];for(const s of i)s();break}case"extension.access":this.model.setAccess(e[0]);break;case"extension.updateAvailable":{const n=e[0];if(typeof n!="string"||!/^\d+(\.\d+){0,3}$/.test(n))break;this.updates.updateVersion=n;try{this.updates.request()}catch(i){this.log("extension update waits for manual reload:",i)}break}default:this.log("unknown extension event:",t)}}attachCdpClient(t){if(this.updates.reloading){t.close(1013,"The Mirasim extension is reloading");return}const e=this.model.addClient(n=>{t.readyState===l.OPEN&&t.send(JSON.stringify(n))});t.on("message",n=>{let i;try{i=JSON.parse(n.toString())}catch(s){this.log("malformed CDP message:",s);return}(async()=>{const{id:s,sessionId:o,method:d,params:h}=i;d==="Page.getFrameTree"&&s!==void 0&&o&&e.trackFrameTree(s,o);try{const a=await this.runOperation(()=>this.model.handleCommand(e,d??"",h,o));e.send({id:s,sessionId:o,result:a})}catch(a){e.send({id:s,sessionId:o,error:{message:a instanceof Error?a.message:String(a)}})}})()}),t.on("close",()=>this.model.removeClient(e)),t.on("error",()=>this.model.removeClient(e))}}async function E(r,t){if(!r.isOpen)return!1;let e;const n=new Promise(i=>{e=setTimeout(()=>i(!1),t)});try{return await Promise.race([r.send("bridge.ping",[]).then(()=>!0,()=>!1),n])}finally{clearTimeout(e)}}export{D as BridgeServer};
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { Browser, Page } from "playwright-core";
|
|
2
|
+
import type { ExtensionAccess } from "./bridge/model-core.ts";
|
|
2
3
|
interface BridgeTabInfo {
|
|
3
4
|
tabId: number;
|
|
4
5
|
title: string;
|
|
@@ -11,7 +12,9 @@ export declare function bridgeRequest<T>(browser: Browser, method: string, param
|
|
|
11
12
|
export declare function bridgeTabs(browser: Browser): Promise<{
|
|
12
13
|
dormant: BridgeTabInfo[];
|
|
13
14
|
browserSession: string | null;
|
|
15
|
+
access: ExtensionAccess | null;
|
|
14
16
|
}>;
|
|
17
|
+
export declare function bridgeAccess(browser: Browser): Promise<ExtensionAccess | null>;
|
|
15
18
|
export declare function bridgeBrowserSession(browser: Browser): Promise<string | null>;
|
|
16
19
|
export declare function restoreBridgeTab(browser: Browser, tabId: number, targetId: string, browserSession: string): Promise<{
|
|
17
20
|
page: Page;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{pageTargetInfo as
|
|
1
|
+
import{pageTargetInfo as l}from"./tabs.js";async function a(n,e,t={}){const r=await n.newBrowserCDPSession();try{return await r.send(e,t)}finally{await r.detach().catch(()=>{})}}async function u(n){if(n.version()!=="mirasim-bridge")return{dormant:[],browserSession:null,access:null};try{const e=await a(n,"Mirasim.tabs");return{dormant:e.tabs.filter(r=>r.unloaded||r.held===!1).map(r=>({...r,browserSession:e.browserSession??""})),browserSession:e.browserSession||null,access:e.access??null}}catch(e){if(/wasn't found|not found|not allowed|unknown method/i.test(String(e)))return{dormant:[],browserSession:null,access:null};throw e}}async function w(n){if(n.version()!=="mirasim-bridge")return null;try{return(await a(n,"Mirasim.access")).access??null}catch{return null}}async function f(n){return(await u(n).catch(()=>null))?.browserSession??null}async function m(n,e,t,r){const o=await a(n,"Mirasim.restoreTab",{tabId:e,targetId:t,browserSession:r}),i=Date.now()+1e4;for(;Date.now()<i;){for(const s of n.contexts().flatMap(c=>c.pages()))if(!s.isClosed()&&(await l(s).catch(()=>null))?.targetId===t)return{page:s,reloaded:o.reloaded??!0,viewport:o.viewport};await new Promise(s=>setTimeout(s,50))}throw new Error(`Restored tab ${e} did not become readable; no action was sent to a replacement page.`)}async function b(n,e){return await a(n,"Mirasim.holdOnly",e?{targetId:e}:{})}async function p(n,e,t){await a(n,"Mirasim.closeTab",{tabId:e,browserSession:t})}export{w as bridgeAccess,f as bridgeBrowserSession,a as bridgeRequest,u as bridgeTabs,p as closeBridgeTab,b as holdOnlyBridgeTab,m as restoreBridgeTab};
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import{readFileSync as g}from"node:fs";import"playwright-core";import"../types.js";import{NO_TAB_OPEN_NOTE as
|
|
2
|
-
`)[0]??"dispatch failed":String(a);throw this.journal?.publishAction(o,t,this.lastPoint,{ok:!1,note:c}),a}e&&(s={...s,note:`new tab ${e} is now the target; ${s.note??""}`}),this.journal?.publishAction(o,t,this.lastPoint,s),this.managedProfileDir&&!this.closedItsLastTab&&o!=="activate"&&await
|
|
1
|
+
import{readFileSync as g}from"node:fs";import"playwright-core";import"../types.js";import{NO_TAB_OPEN_NOTE as v,NO_TAB_YET_NOTE as k,userTabCloseRefusal as y}from"./messages.js";import{endpointFileFor as $,keepInBackground as _}from"./acquire.js";import{isFrontmost as T,returnFocusIfTaken as x}from"./focus-guard.js";import{BrowserDriverConnect as P}from"./driver-connect.js";import{landedNote as w,NAV_TIMEOUT_MS as l,navVerdict as d}from"./navigate.js";import{newBackgroundPage as m}from"./tabs.js";import{detectInterference as S,overlaySensitive as I}from"./overlay-guard.js";import{performUploadToRef as A,refuseMissingFiles as C}from"../chromium/actions.js";import{performRunJs as F}from"../chromium/run-js.js";import{withPageCapture as N}from"../chromium/shot.js";const p=u=>`new tab ${u} is now the target \u2014 newTab answers with this note, not a handle, and the handle it was called on stays on its own tab: const t = await computer.surface('${u}') binds the new one`;class V extends P{async closeDormantTab(o,t){return null}pid;cdpSessions=new WeakMap;overridden=new Set;async perform(o,t){if(o==="download"&&typeof t.id=="string")return await this.downloads.query(t.id,t.timeoutMs,t.stop,this.browser??void 0);if(o==="open_url"||o==="new_tab")this.wokeAfterSelfClose=this.closedItsLastTab,this.closedItsLastTab=!1;else if(this.closedItsLastTab)return{ok:!1,note:v};const r=this.wokeAfterSelfClose;await this.ensure();let e=null;if(o==="new_tab"&&r&&!this.wokeAfterSelfClose&&this.active&&(!this.active.url()||this.active.url()==="about:blank")){if(e=this.tabId(this.active),!t.url)return{ok:!0,note:`new tab ${e} is now the target`};o="open_url"}if(!this.active){if(o!=="open_url"&&o!=="new_tab")return{ok:!1,note:k};if(this.active=await m(this.context),this.claim(this.active),o==="new_tab"){if(e=this.tabId(this.active),!t.url)return{ok:!0,note:`new tab ${e} is now the target`};o="open_url"}}this.hookNavigation(),await this.ensureComposited();const i=await T(this.browserPid());if(this.lastPoint=null,this.cfg.extension&&I(o)){const a=await S(this.page());if(a){const c=`another extension or overlay is over the page right now \u2014 ${a}. Close it (dismiss the popup, click away from the autofill suggestion, switch off the interfering extension for this site), then re-observe and act. Acting now would land on the overlay, not the target.`;return this.journal?.publishAction(o,t,this.lastPoint,{ok:!1,note:c}),{ok:!1,note:c}}}let s;try{const a=this.page();s=await N(a,()=>this.page()!==a||a.isClosed()?Promise.resolve({ok:!1,note:"the target changed while waiting for capture \u2014 re-observe the intended tab before acting"}):this.dispatch(o,t))}catch(a){const c=a instanceof Error?a.message.split(`
|
|
2
|
+
`)[0]??"dispatch failed":String(a);throw this.journal?.publishAction(o,t,this.lastPoint,{ok:!1,note:c}),a}e&&(s={...s,note:`new tab ${e} is now the target; ${s.note??""}`}),this.journal?.publishAction(o,t,this.lastPoint,s),this.managedProfileDir&&!this.closedItsLastTab&&o!=="activate"&&await _(this.context,this.page(),this.managedProfileDir,this.cfg),this.windowAppeared&&(this.windowAppeared=!1,!i&&o!=="activate"&&await x(this.browserPid()));const n=this.takePendingNote();return n&&(s.note=s.note?`${n}. ${s.note}`:n),s}browserPid(){if(this.pid===void 0){this.pid=null;try{const o=JSON.parse(g($(this.managedProfileDir),"utf8"));typeof o.pid=="number"&&(this.pid=o.pid)}catch{}}return this.pid}async cdpFor(o){let t=this.cdpSessions.get(o);return t||(t=await this.context.newCDPSession(o),this.cdpSessions.set(o,t)),t}async ensureComposited(){if(this.managedProfileDir)try{const o=await this.cdpFor(this.page()),{windowId:t,bounds:r}=await o.send("Browser.getWindowForTarget");if(r.windowState!=="minimized")return;await o.send("Browser.setWindowBounds",{windowId:t,bounds:{windowState:"normal"}}),this.cfg.window==="offscreen"&&await o.send("Browser.setWindowBounds",{windowId:t,bounds:{left:-4e3,top:200}})}catch{}}async dispatch(o,t){const r=this.page();switch(o){case"download":return t.stop||t.timeoutMs!==void 0?{ok:!1,note:"download stop/timeoutMs require an existing id; omit them to arm a watch"}:await this.downloads.arm(this.browser,r);case"upload":{const e=Array.isArray(t.paths)?t.paths:[],i=C(e);if(i)return i;if(t.ref){const n=await A(e,t.ref,()=>this.locate(t.ref,t.name,!1),()=>this.missing(t.ref,t.name));return!n.ok&&n.note?.includes("did not take the files")?{...n,note:`${n.note}; for a button that opens a chooser, drop ref and click the button next`}:n}this.armedChooser&&r.off("filechooser",this.armedChooser);const s=n=>{n.setFiles(e)};return this.armedChooser=s,r.once("filechooser",s),{ok:!0,note:`the next file chooser will receive ${e.length} file${e.length===1?"":"s"} \u2014 nothing has changed yet; click the control that opens it`}}case"dialog":return this.armedDialog={accept:t.answer==="accept",...typeof t.text=="string"?{text:t.text}:{}},{ok:!0,note:`the next JS dialog will be ${t.answer}ed \u2014 nothing on the page has changed yet; the answer is used when the action that raises the dialog runs`};case"open_url":try{return await r.goto(t.url,{waitUntil:"domcontentloaded",timeout:l}),{ok:!0,note:w("opened",t.url,r.url())}}catch(e){return d(e)}case"activate":return await r.bringToFront(),{ok:!0,note:"activated this browser tab and its window \u2014 foreground focus was requested explicitly"};case"run_js":return await F(()=>r,t.code,t.ms,t.budgetMs);case"back":{const e=r.url();try{return await r.goBack({waitUntil:"commit",timeout:l})===null&&r.url()===e?{ok:!0,note:"no earlier page in this tab's history"}:{ok:!0,note:`went back to ${r.url()}`}}catch(i){return d(i)}}case"reload":{const e=r.url();try{return await r.reload({waitUntil:"domcontentloaded",timeout:l}),{ok:!0,note:w("reloaded",e,r.url())}}catch(i){return d(i)}}case"new_tab":{const e=await m(this.context);this.claim(e),this.active=e,this.resetObservationState();const i=this.tabId(e);if(typeof t.url=="string"&&t.url){try{await e.goto(t.url,{waitUntil:"domcontentloaded",timeout:l})}catch(s){const n=d(s);return{...n,note:`${p(i)}; ${n.note??"its navigation did not finish"}`}}return{ok:!0,note:`${p(i)}; ${w("opened",t.url,e.url())}`}}return{ok:!0,note:p(i)}}case"viewport":{const e=await this.cdpFor(r);if(t.reset)return await e.send("Emulation.clearDeviceMetricsOverride"),this.overridden.delete(r),{ok:!0,note:"viewport override cleared \u2014 this tab is back to the real window size"};const i=Math.round(Number(t.width)||1200),s=Math.round(Number(t.height)||1700);await e.send("Emulation.setDeviceMetricsOverride",{width:i,height:s,deviceScaleFactor:0,mobile:!1}),this.overridden.add(r);const n=await r.evaluate("({ width: innerWidth, height: innerHeight })");return n.width!==i||n.height!==s?{ok:!1,note:`viewport ${i}x${s} did not take effect \u2014 the page reports ${n.width}x${n.height}`}:{ok:!0,note:`viewport of this tab set to ${i}x${s} \u2014 navigate AFTER this, not before, or a list already mounted keeps the rows it has; the size is the user's own again on viewport({reset:true}) or when this session ends`}}case"close_tab":{const e=(await this.ownPages()).filter(h=>!h.isClosed());if(t.all){if(typeof t.id=="string"&&t.id)return{ok:!1,note:"close_tab takes id OR all, not both \u2014 all closes every tab this session owns"};for(const h of e)await h.close().catch(()=>{});return this.closedItsLastTab=!0,{ok:!0,note:`closed ${e.length} tab${e.length===1?"":"s"} \u2014 every one this session opened or claimed; the user's own tabs were not touched. open_url or new_tab starts a fresh one`}}const i=typeof t.id=="string"&&t.id?t.id.replace(`${this.platform}/`,""):this.tabId(r);let s=e.find(h=>this.tabId(h)===i),n=!1;if(!s){const h=(await this.visiblePages()).find(b=>this.tabId(b)===i);if(!h)return await this.closeDormantTab(i,t.confirmed===!0)??{ok:!1,note:`no tab with id ${i} \u2014 call computer.surfaces() for current ids`};if(!t.confirmed)return{ok:!1,note:y(i)};s=h,n=!0}const a=s===this.active;await s.close().catch(()=>{});const c=n?" \u2014 the user's own, closed on the agreement you reported":"";if(!a)return{ok:!0,note:`closed tab ${i}${c}; the target is unchanged`};const f=e.find(h=>h!==s&&!h.isClosed());return f?(this.active=f,this.resetObservationState(),{ok:!0,note:`closed tab ${i}${c}; the target moved to ${this.tabId(f)} \u2014 re-observe before acting on any earlier ref`}):(this.closedItsLastTab=!0,{ok:!0,note:`closed tab ${i}${c} \u2014 this session has no tab left; open_url or new_tab starts a fresh one`})}}return await this.dispatchPage(o,t)??{ok:!1,note:`unknown action kind ${o}`}}}export{V as BrowserDriverVerbs};
|
|
@@ -6,6 +6,7 @@ export declare class BrowserDriver extends BrowserDriverVerbs {
|
|
|
6
6
|
private borrowed;
|
|
7
7
|
private reported;
|
|
8
8
|
private holdUnsupported;
|
|
9
|
+
private access;
|
|
9
10
|
activeTarget(): Promise<string | null>;
|
|
10
11
|
ownershipResources(): Promise<OwnershipResource[]>;
|
|
11
12
|
prepareTarget(): Promise<void>;
|
|
@@ -40,6 +41,7 @@ export declare class BrowserDriver extends BrowserDriverVerbs {
|
|
|
40
41
|
actionResources(kind: string, args: Record<string, unknown>): Promise<OwnershipResource[]>;
|
|
41
42
|
focusTarget(id: string): Promise<ActResult>;
|
|
42
43
|
perform(kind: string, a: Record<string, any>): Promise<ActResult>;
|
|
44
|
+
private userPaused;
|
|
43
45
|
private selectTarget;
|
|
44
46
|
protected closeDormantTab(id: string, confirmed: boolean): Promise<ActResult | null>;
|
|
45
47
|
close(): Promise<void>;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
-
import{Type as i}from"typebox";import{SurfaceUnreadableError as
|
|
1
|
+
import{Type as i}from"typebox";import{SurfaceUnreadableError as u,text as w}from"../types.js";import{NO_TAB_OPEN_NOTE as p,NO_TAB_YET_NOTE as m,PAUSED_BY_USER as h,accessMissNote as f,userTabCloseRefusal as g}from"./messages.js";import{PASTE_VERB as v,SELECT_OPTION_VERB as y}from"../chromium/actions-value.js";import{SELECT_TEXT_VERB as S}from"../chromium/select-text.js";import{DIALOG_VERB as k,DRAG_VERB as T,UPLOAD_VERB as O}from"../chromium/actions.js";import{RUN_JS_CODE_DESC as E,RUN_JS_MAX_MS as _,RUN_JS_MIN_MS as I,RUN_JS_MS_DESC as N}from"../chromium/run-js.js";import{BrowserDriverVerbs as P}from"./driver-verbs.js";import{previewPage as R}from"../chromium/preview.js";import{withPageCapture as B}from"../chromium/shot.js";import{DOWNLOAD_VERB as L}from"./downloads.js";import{bridgeAccess as $,bridgeRequest as U,bridgeTabs as D,closeBridgeTab as C,holdOnlyBridgeTab as x,restoreBridgeTab as A}from"./bridge-tabs.js";import{pageTargetInfo as j}from"./tabs.js";const V=5e3,z=1500,M=/^(?:[a-z][a-z\d+.-]*:\/\/|(?:localhost|\d{1,3}(?:\.\d{1,3}){3}|\[[\da-f:]+\])(?::\d+)?(?:[/?#]|$)|[\w.-]+:\d+(?:[/?#]|$))/i;async function H(d){if(!d.url())return"";let e;try{return await Promise.race([d.title().catch(()=>""),new Promise(t=>{e=setTimeout(()=>t(""),z),e.unref?.()})])}finally{e&&clearTimeout(e)}}const b=/^tab:(\d+):(.*)$/;class ie extends P{dormantSelection=null;borrowed=new Map;reported=null;holdUnsupported=!1;access=null;async activeTarget(){return this.dormantSelection?.id??super.activeTarget()}async ownershipResources(){return this.dormantSelection?[{key:`chromium:${this.dormantSelection.targetId}`,mode:"exclusive"}]:super.ownershipResources()}async prepareTarget(){const e=this.dormantSelection;if(!e)return;const{page:t,reloaded:s,viewport:r}=await A(this.browser,e.tabId,e.targetId,e.browserSession);if(this.tabIds.set(t,e.id),this.borrowed.set(t,e),this.active=t,this.reported=t,this.dormantSelection=null,this.resetObservationState(),!s)return;const a=await t.evaluate("({ width: innerWidth, height: innerHeight })");if(a.width<=0||a.height<=0){const n=await this.perform("viewport",r??{});if(!n.ok)throw new u(`The restored tab has no usable viewport: ${n.note??"resize failed"}`);this.appendNote("Chrome restored this background tab with a zero-size viewport; applied a temporary viewport, cleared when this driver disconnects.")}this.appendNote("Chrome had unloaded this tab; restored the same tab in the background. Read its current state before continuing.")}onBridge(){return!this.tab&&!this.attached&&!this.managedProfileDir&&this.browser?.version()==="mirasim-bridge"}async syncHeld(){if(this.holdUnsupported||!this.onBridge())return;const e=this.active&&!this.active.isClosed()?this.active:void 0;if(e!==this.reported){this.reported=e;for(const t of[...this.borrowed.keys()])t.isClosed()&&this.borrowed.delete(t);try{const t=e?(await j(e)).targetId:void 0,s=await x(this.browser,t);if(!e||!t||!s.held||s.tabId===void 0||!s.browserSession)return;const r=`tab:${s.tabId}:${s.browserSession}`;this.tabIds.has(e)||this.tabIds.set(e,r),this.borrowed.set(e,{id:this.tabIds.get(e),tabId:s.tabId,targetId:t,browserSession:s.browserSession})}catch(t){/wasn't found|not found|unknown method/i.test(String(t))&&(this.holdUnsupported=!0),this.reported=null}}}async releaseIdle(){await this.syncHeld();const e=this.active,t=e?this.borrowed.get(e):void 0;t&&(this.active=void 0,this.borrowed.clear(),await this.syncHeld(),this.dormantSelection=t,this.resetObservationState())}page(){if(this.dormantSelection)throw new u(`Tab ${this.dormantSelection.id} has not been restored; no other page was observed or operated.`);return super.page()}adoptWhenReal(e){this.dormantSelection||super.adoptWhenReal(e)}previewTarget(){return this.dormantSelection||!this.browser?.isConnected()||!this.active||this.active.isClosed()?Promise.resolve(null):R(this.active,this.cdpUrl,"browser")}platform="browser";async turnEnded(e){if(this.attached||e.keepBrowserTabs===!0)return;const t=await this.ownPages().catch(()=>[]);for(const s of t)await s.close().catch(()=>{});t.length&&(this.closedItsLastTab=!0,this.resetObservationState())}extraVerbs(){return[{kind:"open_url",schema:i.Object({kind:i.Literal("open_url"),url:i.String({description:"Full URL to open. Only URLs that appeared on a page or were given in the task \u2014 never guess domains."})})},{kind:"activate",schema:i.Object({kind:i.Literal("activate")},{description:"Bring this browser tab and its window to the foreground. This changes the user's foreground window: explain the effect and obtain permission unless it is already authorized. focus() only selects the target. Embedded writes normally wait for the person to yield the window; activate() is an explicit foreground handoff, revoked by subsequent human input. Do not use it to bypass an input wait without the user handing over control."})},{kind:"back",schema:i.Object({kind:i.Literal("back")})},{kind:"reload",schema:i.Object({kind:i.Literal("reload")})},{kind:"new_tab",schema:i.Object({kind:i.Literal("new_tab"),url:i.Optional(i.String({description:"URL to open in it; omit for a blank tab"}))},{description:"Open a NEW tab of this session's own and make it the target \u2014 the current tab keeps its state, and both are listed as targets afterwards. It answers with a note naming the new tab's id, not a handle; the handle it was called on stays on its own tab, so bind the new one with computer.surface('<that id>')."})},{kind:"close_tab",schema:i.Object({kind:i.Literal("close_tab"),id:i.Optional(i.String({description:"Tab id from computer.surfaces(); omit to close the one you are on"})),all:i.Optional(i.Boolean({description:"Close every tab this session opened or claimed \u2014 for a tidy-up the host's tab policy or the user asks for. Never the user's own, whatever else is set. Refused together with id."})),confirmed:i.Optional(i.Boolean({description:"Only for a tab marked (user's): you ASKED the user in the conversation and they said yes. Never set it because closing would be convenient \u2014 the first refusal tells you to ask, and asking is the point."}))},{description:"Close a tab: the tabs you open outlive your turn and nothing closes them for you. Leave them open unless the host states a tab policy that asks you to close them, or the user asks \u2014 the person may want to see the page, and a later turn may need it again. Never close a page that is waiting for the person (a sign-in, a security-key or 2FA prompt, a form for them to confirm). This session's own always; one of the user's (marked (user's) in the listing) only after you asked them in the conversation and they agreed \u2014 then pass confirmed: true."})},{kind:"viewport",schema:i.Object({kind:i.Literal("viewport"),width:i.Optional(i.Number({description:"Viewport width in CSS pixels (default 1200)"})),height:i.Optional(i.Number({description:"Viewport height in CSS pixels \u2014 this is the one that decides how much of a long list exists (default 1700)"})),reset:i.Optional(i.Boolean({description:"Put the tab back to the real window size"}))},{description:"Give THIS tab a viewport of your own size, so a virtualized list (a feed, a search page, a long transcript) mounts more of itself per read. Set it BEFORE opening the page: rows mount against the viewport in effect at load."})},{kind:"run_js",readOnly:!0,schema:i.Object({kind:i.Literal("run_js"),code:i.String({description:E}),ms:i.Optional(i.Integer({minimum:I,maximum:_,description:N}))},{description:"Ask the PAGE something no observation carries \u2014 read its data in bulk, its stored state, or an API it exposes. This is a stranger's site holding the user's logged-in session: read and compute here, act through the interaction verbs."})},T,k,O,L,y,S,v]}async observeElements(e={}){const t=await this.noTabNote();if(t)return this.noTabObservation(t,e.prefix);try{return await super.observeElements(e)}catch(s){throw await this.userPaused()?new Error(h):s}}async observeImage(e={}){const t=await this.noTabNote();if(t)return this.noTabObservation(t,e.prefix);try{return await super.observeImage(e)}catch(s){throw await this.userPaused()?new Error(h):s}}async noTabNote(){return this.closedItsLastTab?p:this.dormantSelection?null:(await this.ensure(),this.active?null:m)}async heldNothing(){return this.closedItsLastTab?"closed":this.dormantSelection?null:(await this.ensure(),this.active?null:"not-yet")}noTabObservation(e,t){const s=this.takePendingNote();return w(`${t?`${t}
|
|
2
2
|
`:""}${s?`NOTE: ${s}.
|
|
3
|
-
`:""}${e}.`,{verdict:"unknown"})}staleRefCause(){return"it predates a navigation, a reload or a tab switch"}explainMiss(e){const t=e.trim();if(!
|
|
3
|
+
`:""}${e}.`,{verdict:"unknown"})}staleRefCause(){return"it predates a navigation, a reload or a tab switch"}explainMiss(e){const t=this.access?f(this.access):null,s=e.trim();if(!M.test(s))return t;const r=JSON.stringify(/^[a-z][a-z\d+.-]*:\/\//i.test(s)?s:`http://${s}`),a=`mirasim-cu: computer.surface() binds a tab that is already open, by its id or title \u2014 never by URL \u2014 and opens nothing. Bind a tab and go there: const w = await computer.surface('<id>'); await w.openUrl({url: ${r}}) \u2014 or await w.newTab({url: ${r}}) to leave that tab as it is.`;return t?`${a}
|
|
4
|
+
${t}`:a}promptFragment(){return"Grounding: observations are text lists of interactive elements with refs, for the CURRENT viewport only. Act by ref from the latest observation. When what you want is NOT in the list: if you can name it, w.find('<name>') searches EVERY control on the page and hands back a handle \u2014 act on it directly, the click scrolls it into view, so do not scroll first. Scroll only to discover what you cannot name. Observations may show only changes since the previous one; refs stay valid across observations (until navigation). Use w.text() to read full page text."}async listTargets(){await this.ensure();const{dormant:e,browserSession:t,access:s}=!this.tab&&!this.attached&&!this.managedProfileDir?await D(this.browser):{dormant:[],browserSession:null,access:null};this.access=s,this.noteBridgeSession(t);const r=new Set(await this.ownPages()),a=await this.visiblePages(),n=await Promise.all(a.map(async o=>({id:this.tabId(o),kind:"tab",title:await H(o)||o.url(),active:!this.dormantSelection&&o===this.active,own:r.has(o),mine:this.mine.has(o)})));for(const o of e){const c=`tab:${o.tabId}:${o.browserSession}`,l=o.title||o.url;n.push({id:c,kind:"tab",title:o.unloaded?`${l} [unloaded \u2014 use this handle to restore in background]`:l,active:this.dormantSelection?.id===c,own:!1,...o.unloaded?{unloaded:!0}:{}})}return n}noteBridgeSession(e){const t=super.noteBridgeSession(e);return t&&(this.dormantSelection=null,this.borrowed.clear(),this.reported=null),t}async actionResources(e,t){if(e!=="close_tab")return[];const s=typeof t.id=="string"?t.id.replace(/^browser\//,""):null,r=t.all?await this.ownPages():s?(await this.visiblePages()).filter(a=>this.tabId(a)===s):[this.page()];return Promise.all(r.filter(a=>!a.isClosed()).map(a=>this.pageOwnership(a)))}async focusTarget(e){await this.ensure();const t=await this.selectTarget(e);return await this.syncHeld(),t}async perform(e,t){try{const s=await super.perform(e,t);return!s.ok&&await this.userPaused()?{...s,note:h}:s}catch(s){throw await this.userPaused()?new Error(h):s}finally{await this.syncHeld()}}async userPaused(){return this.onBridge()?(await $(this.browser))?.paused===!0:!1}async selectTarget(e){let t={ok:!1,note:`no target with id ${e}; call computer.surfaces() for current ids`};for(const r of await this.visiblePages())if(!(r.isClosed()||this.tabId(r)!==e)){this.dormantSelection=null,this.active=r,this.closedItsLastTab=!1,t={ok:!0};break}const s=t.ok?null:b.exec(e);if(s&&!this.tab&&!this.attached&&!this.managedProfileDir&&this.browser.version()==="mirasim-bridge"){const r=Number(s[1]),a=s[2],{targetId:n}=await U(this.browser,"Mirasim.tabTarget",{tabId:r,browserSession:a});return this.dormantSelection={id:e,tabId:r,targetId:n,browserSession:a},this.active=void 0,this.closedItsLastTab=!1,{ok:!0}}return this.journal?.publishAction("focus",{target:e},null,t),t}async closeDormantTab(e,t){const s=b.exec(e);if(!s||!this.browser||this.browser.version()!=="mirasim-bridge")return null;if(!t)return{ok:!1,note:g(e)};this.dormantSelection?.id===e&&(this.dormantSelection=null);try{await C(this.browser,Number(s[1]),s[2])}catch(r){return{ok:!1,note:`could not close tab ${e}: ${r instanceof Error?r.message:String(r)}`}}return{ok:!0,note:`closed tab ${e} \u2014 the user's own, closed on the agreement you reported; the target is unchanged`}}async close(){this.dormantSelection=null,this.borrowed.clear(),this.reported=null,await this.downloads.close(),await this.journal?.drain(V);for(const e of this.overridden)e.isClosed()||await B(e,()=>this.cdpFor(e).then(t=>t.send("Emulation.clearDeviceMetricsOverride"))).catch(()=>{});if(this.overridden.clear(),!this.managedProfileDir&&!this.tab&&this.browser)for(const e of await this.ownPages().catch(()=>[]))e.url()==="about:blank"&&await e.close().catch(()=>{});await this.browser?.close().catch(()=>{}),this.browser=null,this.resetObservationState(),this.resetPageState(),await this.disconnected()}journalVerdict(e){this.journal?.publishVerdict(e)}renderHeadLines(e,t,s=!1,r){const a=super.renderHeadLines(e,t,s,r);if(e.url==="about:blank"&&e.elements.length===0&&!e.text){const n=t.some(o=>o.title!=="about:blank");a.push(n?"NOTE: this tab is blank \u2014 your page may be in another tab; computer.surfaces() lists it \u2014 act through its handle.":"NOTE: this tab is blank and nothing has been opened in it \u2014 navigate (openUrl({url})) instead of observing again.")}return a}async blankSurfaceHint(){try{return this.page().url()!=="about:blank"?"":(await this.ownPages()).some(t=>t!==this.page()&&t.url()!=="about:blank")?"the active tab is a blank page \u2014 your page may be in another tab; computer.surfaces() lists it \u2014 act through its handle":""}catch{return""}}}export{ie as BrowserDriver};
|
|
@@ -24,3 +24,8 @@ export declare const NO_TAB_YET_NOTE: string;
|
|
|
24
24
|
export declare const BROWSER_SWITCHED_NOTE: string;
|
|
25
25
|
export declare const SURFACE_REIDENTIFIED_NOTE: string;
|
|
26
26
|
export declare function userTabCloseRefusal(id: string): string;
|
|
27
|
+
export declare function accessMissNote(access: {
|
|
28
|
+
scope: string;
|
|
29
|
+
siteRules: boolean;
|
|
30
|
+
}): string | null;
|
|
31
|
+
export declare const PAUSED_BY_USER: string;
|
|
@@ -4,5 +4,5 @@ Fix one of:
|
|
|
4
4
|
- point COMPUTER_USE_CHROME at a Chrome/Chromium binary
|
|
5
5
|
- point COMPUTER_USE_CDP_URL at an already-running browser's CDP endpoint`}function s(e,t){return`mirasim-cu: cannot take over this Chrome \u2014 ${t}.
|
|
6
6
|
This profile belongs to the user (${e}), so it will not be force-closed.
|
|
7
|
-
Quit Chrome (\u2318Q) and retry.`}const r=()=>"its debugging port answers, but no usable page can be opened",i=()=>"its debugging port answers, but the handshake timed out (the browser may be frozen)",a=()=>"it is already running, but without a debugging port",h=()=>"mirasim-cu: could not open the agent's own window in the user's Chrome.";function d(e,t){return`mirasim-cu: cannot run the Chrome binary at ${e} (${t}). Point COMPUTER_USE_CHROME at a working Chrome/Chromium.`}function c(e){return`mirasim-cu: Chrome could not be started (${e}).`}const
|
|
8
|
-
(${t})`}const x="the browser had been closed (most likely by the user), so a fresh one was opened in the background and the page is blank \u2014 re-observe, and navigate again before acting on refs",E="this surface's tab had been closed (most likely by the user), so a fresh blank tab was claimed \u2014 re-observe, and navigate again before acting on refs",y="this surface's tab stopped answering \u2014 it was closed, or the browser took its debugger away, in which case it is still open in the user's window but out of reach from here. A fresh blank tab was claimed \u2014 re-observe, and navigate again before acting on refs. If the same page drops again right after it loads, it cannot be driven through this browser: a password manager's autofill (measured: iCloud Passwords, on any page with a password field) puts its own extension frame into the page, and Chrome then refuses this extension's debugger there. Do not reopen it; tell the user \u2014 turning that extension off for the site, or the managed browser (its own profile), drives it";function C(e){const t=`this surface's tab is out of reach: it is still open in the user's window, but Chrome refused this extension's debugger on it ("${e}"). A fresh blank tab was claimed. `;return/chrome-extension:\/\/ URL of different extension/.test(e)?t+"Another extension has put its own frame into that page \u2014 a password manager's autofill does this on any page with a password field (measured: iCloud Passwords) \u2014 and while it is there no retry from here gets in. Do not reopen the page; tell the user that turning that extension off for this site, or the managed browser (its own profile), lets it be driven":t+"Reopening it will most likely be refused the same way \u2014 tell the user rather than retrying"}const _="this session had closed its last tab, so opening one started a fresh blank tab \u2014 nothing went wrong and nobody took the browser back; navigate before acting on refs",T="this session has no tab open \u2014 it closed its last one and nothing was reopened, so the browser is left as the user had it. open_url or new_tab starts a fresh tab when you need one",k="this session holds no tab yet \u2014 none is opened until asked. Bind one of the listed tabs to act in it, or new_tab opens one of the session's own",
|
|
7
|
+
Quit Chrome (\u2318Q) and retry.`}const r=()=>"its debugging port answers, but no usable page can be opened",i=()=>"its debugging port answers, but the handshake timed out (the browser may be frozen)",a=()=>"it is already running, but without a debugging port",h=()=>"mirasim-cu: could not open the agent's own window in the user's Chrome.";function d(e,t){return`mirasim-cu: cannot run the Chrome binary at ${e} (${t}). Point COMPUTER_USE_CHROME at a working Chrome/Chromium.`}function c(e){return`mirasim-cu: Chrome could not be started (${e}).`}const u=()=>"mirasim-cu: Chrome exited immediately instead of publishing a debugging port \u2014 another browser may already own this profile directory.",l=()=>"mirasim-cu: Chrome never published a debugging port within 15s.",m=()=>"mirasim-cu: Chrome started but its debugger refused to attach.",o="https://chromewebstore.google.com/detail/mirasim/khflooofmndpmafhofjdcpjfgnlpikhg";function g(){return`The mirasim bridge extension is not connected, so this browser cannot be reached. Nothing you can do from here fixes it and retrying will not help: ask the user to install or enable the Mirasim extension from the Chrome Web Store (${o}) and remove any older copy loaded unpacked at chrome://extensions, which no longer connects; then continue.`}function b(e){return`The Mirasim extension is connected to another program's browser bridge (${e}), not to this one \u2014 it serves one bridge at a time and that one comes first. Retrying will not help: ask the user to turn browser control off in that program (${e}), then continue.`}function f(){return`The browser bridge started moments ago and the Mirasim extension has not connected yet. An installed extension checks again within a minute; wait about a minute and retry once. If it still does not connect, ask the user to install or enable the Mirasim extension from the Chrome Web Store (${o}).`}function p(e){return/Bridge extension not connected|bridge extension did not connect/i.test(e instanceof Error?e.message:String(e))}function w(e,t){return`mirasim-cu: explicit CDP endpoint ${e} is unavailable.
|
|
8
|
+
(${t})`}const x="the browser had been closed (most likely by the user), so a fresh one was opened in the background and the page is blank \u2014 re-observe, and navigate again before acting on refs",E="this surface's tab had been closed (most likely by the user), so a fresh blank tab was claimed \u2014 re-observe, and navigate again before acting on refs",y="this surface's tab stopped answering \u2014 it was closed, or the browser took its debugger away, in which case it is still open in the user's window but out of reach from here. A fresh blank tab was claimed \u2014 re-observe, and navigate again before acting on refs. If the same page drops again right after it loads, it cannot be driven through this browser: a password manager's autofill (measured: iCloud Passwords, on any page with a password field) puts its own extension frame into the page, and Chrome then refuses this extension's debugger there. Do not reopen it; tell the user \u2014 turning that extension off for the site, or the managed browser (its own profile), drives it";function C(e){const t=`this surface's tab is out of reach: it is still open in the user's window, but Chrome refused this extension's debugger on it ("${e}"). A fresh blank tab was claimed. `;return/chrome-extension:\/\/ URL of different extension/.test(e)?t+"Another extension has put its own frame into that page \u2014 a password manager's autofill does this on any page with a password field (measured: iCloud Passwords) \u2014 and while it is there no retry from here gets in. Do not reopen the page; tell the user that turning that extension off for this site, or the managed browser (its own profile), lets it be driven":t+"Reopening it will most likely be refused the same way \u2014 tell the user rather than retrying"}const _="this session had closed its last tab, so opening one started a fresh blank tab \u2014 nothing went wrong and nobody took the browser back; navigate before acting on refs",T="this session has no tab open \u2014 it closed its last one and nothing was reopened, so the browser is left as the user had it. open_url or new_tab starts a fresh tab when you need one",k="this session holds no tab yet \u2014 none is opened until asked. Bind one of the listed tabs to act in it, or new_tab opens one of the session's own",R="the browser behind the extension changed \u2014 a different browser, or this one after a restart, is connected now, and the tabs listed are ITS tabs. Nothing was done in it and this session holds no tab there; handles and refs from before belonged to the previous browser and are gone. If this is not the browser the task is about, stop and tell the user; otherwise bind one of the listed tabs, or new_tab opens one of the session's own",N="the connection to this browser dropped and came back, so this surface was re-identified \u2014 the page below is the one it is on now, and refs from before are dead; re-observe before acting";function O(e){return`${e} is the user's own tab, not this session's. Ask them in the conversation whether to close it \u2014 say which page \u2014 and call close_tab again with confirmed: true if they agree. Reading it and acting in it needs no permission; only closing does`}function v(e){const t=[];return e.scope==="group"&&t.push("the user lets the agent see only the tabs it opened and the ones handed to it (the Mirasim tab group), so a page open in another tab is not listed \u2014 ask them to click the Mirasim toolbar icon on that tab to hand it over, then list surfaces again"),e.siteRules&&t.push("tabs on sites the user blocked in Mirasim are not listed either, and cannot be opened"),t.length?`mirasim-cu: ${t.join("; ")}.`:null}const A="The user paused the agent from the Mirasim extension: every browser command is refused until they resume it. Tell them, and wait for them to resume before trying again";export{R as BROWSER_SWITCHED_NOTE,T as NO_TAB_OPEN_NOTE,k as NO_TAB_YET_NOTE,A as PAUSED_BY_USER,x as REOPENED_NOTE,N as SURFACE_REIDENTIFIED_NOTE,y as TAB_DETACHED_NOTE,_ as TAB_REOPENED_AFTER_SELF_CLOSE_NOTE,E as TAB_REOPENED_NOTE,v as accessMissNote,p as isBridgeExtensionGone,s as msgAttachedDeadEnd,i as msgAttachedWhyHandshake,r as msgAttachedWhyNoPage,a as msgAttachedWhyNoPort,b as msgBridgeExtensionElsewhere,g as msgBridgeExtensionGone,f as msgBridgeJustStarted,d as msgChromeBinaryUnrunnable,u as msgChromeExitedImmediately,n as msgChromeMissing,l as msgChromeNoPort,m as msgChromeRefusedAttach,c as msgChromeStartFailed,w as msgEndpointUnavailable,h as msgOwnWindowFailed,C as tabRefusedNote,O as userTabCloseRefusal};
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import{existsSync as
|
|
2
|
-
`)[0]??"":String(
|
|
1
|
+
import{existsSync as v}from"node:fs";import{Type as e}from"typebox";import{coverOf as S,isEditorProxy as E}from"./coverer-label.js";async function D(o,t,i){let r;try{r=await o.evaluate(s=>{const u=s.matches&&s.matches(":disabled")?"natively disabled":s.getAttribute&&s.getAttribute("aria-disabled")==="true"?"aria-disabled":null,p=s.closest&&s.closest("form")||s.ownerDocument,g=!!(p&&p.querySelector&&p.querySelector("input:invalid, select:invalid, textarea:invalid"));return{why:u,unmet:g}},void 0,{timeout:2e3})}catch{return null}const{why:a,unmet:f}=r;if(!a)return null;const l=`${t??"the target"}${i?` "${i}"`:""}`;return{ok:!1,journal:{refused:a,gate:f?"fields":"state"},note:`${l} is ${a} \u2014 ${a==="aria-disabled"?"the page only LABELS it disabled, so a click by x/y from a screenshot may still reach it":"the browser delivers no input to it"}. The page is gating this control, so acting on a different element will not unlock it: ${f?"fill what it is waiting on (the [empty] / [invalid] fields in the list), then re-observe":"no unfilled field is waiting here, so the gate is a STATE, not a form: some step has not happened yet (nothing selected, an empty history, a mode not entered). Do that step, then re-observe"} \u2014 the [disabled] mark disappears once it becomes usable`}}const y=2e3,x=8e3;async function I(o,t,i,r,a){try{return await o.click({...i,timeout:y}),{ok:!0}}catch(l){const c=k(l,y);if(c)return c;if(!R(l))throw l}if(await E(t)){try{await o.click({...i,force:!0,timeout:x-y})}catch(l){const c=k(l,x-y);if(c)return c;throw l}return{ok:!0}}const f=await $(t,r,a);if(f)return{ok:!1,journal:{covered:!0},note:f};try{return await o.click({...i,timeout:x-y}),{ok:!0}}catch(l){const c=k(l,x-y);if(c)return c;const h=await $(t,r,a);if(!h)throw l;return{ok:!1,journal:{covered:!0},note:h}}}function k(o,t){const i=o;if(i?.name!=="TimeoutError")return null;const r=String(i.message??"");if(/click action done/.test(r))return{ok:!0,note:"clicked \u2014 it started a navigation that was still loading when the click returned; wait({loaded}) or wait({text}) for the next page before reading it"};if(/performing \w+ action/.test(r))throw new Error(`the click was sent but not confirmed within ${t/1e3}s \u2014 it may have landed; observe before clicking again`);return null}function R(o){const t=o;return t?.name==="TimeoutError"&&!/performing \w+ action/.test(String(t.message??""))}async function $(o,t,i){const r=await S(o);if(!r)return null;const a=`${t??"the target"}${i?` "${i}"`:""}`;return r.pseudo?`${a} is covered by the ${r.pseudo} layer of \xAB${r.who}\xBB, which takes the click instead \u2014 that is the page's own styling, not an element: it is in no element list and there is nothing to close, so a user's mouse click does not reach this control either. The keyboard still does (Tab to it, then Enter or Space)`:`${a} is covered by \xAB${r.who}\xBB, which took the click instead \u2014 the element is the right one, so acting on a different one will not help: close or dismiss what is on top (find('close') reaches its close control when the page marks one; an unmarked one shows only in a screenshot), or `+(t?`scroll({into:'${t}'}) to bring it clear`:"scroll until this control sits clear of it")+", then act on this ref again"}const A={kind:"dialog",schema:e.Object({kind:e.Literal("dialog"),answer:e.Union([e.Literal("accept"),e.Literal("dismiss")],{description:'"accept" or "dismiss" for the next JS dialog. Default without this verb: alert accepted, confirm/prompt dismissed.'}),text:e.Optional(e.String({description:"Text to type into a prompt() before accepting"}))})};function L(o){const t=o.filter(i=>!v(i));return t.length?{ok:!1,note:`no such file: ${t.join(", ")} \u2014 give an absolute path to a file that exists on this machine`}:null}async function N(o){if(await o.evaluate(r=>r.tagName==="INPUT"&&r.type==="file").catch(()=>!1))return o;const i=o.locator('xpath=ancestor-or-self::*[.//input[@type="file"]][1]//input[@type="file"]');return await i.count().catch(()=>0)===1?i:null}async function _(o,t,i,r){const a=await i();if(!a)return await r();try{await a.setInputFiles(o,{timeout:8e3})}catch(f){const l=f instanceof Error?f.message.split(`
|
|
2
|
+
`)[0]??"":String(f);return{ok:!1,note:`${t} did not take the files (${l}) \u2014 pass ref only for an element the list shows as \`file\``}}return{ok:!0,note:`set ${o.length} file${o.length===1?"":"s"} on ${t}`}}const j={kind:"upload",schema:e.Object({kind:e.Literal("upload"),paths:e.Array(e.String(),{description:"Absolute path(s) of the file(s) to upload. They must already exist on this machine.",minItems:1}),ref:e.Optional(e.String({description:"A visible file input to fill directly. Omit to answer the next file chooser a click opens."}))})},B={kind:"drag",schema:e.Object({kind:e.Literal("drag"),ref:e.Optional(e.String({description:"Element to drag \u2014 REQUIRED unless x/y are given (this pair is drag-only; click/hover still require ref or x/y)"})),name:e.Optional(e.String({description:"The ref's expected name \u2014 an identity check on ref, never a start of its own; a non-control (a list row) has no ref, so take its x/y off a screenshot"})),x:e.Optional(e.Number({description:"Drag start x (instead of ref)"})),y:e.Optional(e.Number({description:"Drag start y"})),toRef:e.Optional(e.String({description:"Element to drop onto \u2014 REQUIRED unless toX/toY are given"})),toX:e.Optional(e.Number({description:"Destination x (instead of toRef)"})),toY:e.Optional(e.Number({description:"Destination y"}))},{description:"Press, drag, release: move sliders, resize handles, reorder/sort items, drag into bins, or select text by dragging across it. Needs a start (ref or x/y) AND a destination (toRef or toX/toY)."})};async function U(o,t,i){const r=async(n,m,w)=>{if(typeof n!="string")return null;const d=await i(n,m);return d?(await d.scrollIntoViewIfNeeded({timeout:4e3}).catch(()=>{}),d):{ok:!1,note:`no element for ${w} ${n}`}},a=async(n,m,w)=>{const d=await n.boundingBox().catch(()=>null);return d?{x:d.x+d.width/2,y:d.y+d.height/2}:{ok:!1,note:`no element for ${m} ${String(w)}`}},f=typeof t.x=="number"&&typeof t.y=="number",l=typeof t.toX=="number"&&typeof t.toY=="number";if(!f&&typeof t.ref!="string")return{res:{ok:!1,note:`drag needs a start: ref or x/y${typeof t.name=="string"?" \u2014 name only checks a ref, it does not locate one; find() it for a ref, or read x/y off a screenshot for a non-control":""}`},point:null};if(!l&&typeof t.toRef!="string")return{res:{ok:!1,note:"drag needs a destination: toRef or toX/toY"},point:null};const c=f?null:await r(t.ref,t.name,"ref");if(c&&"ok"in c)return{res:c,point:null};const h=l?null:await r(t.toRef,void 0,"toRef");if(h&&"ok"in h)return{res:h,point:null};let s;if(f)s={x:t.x,y:t.y};else{const n=await a(c,"ref",t.ref);if("ok"in n)return{res:n,point:null};s=n}let u;if(l)u={x:t.toX,y:t.toY};else{const n=await a(h,"toRef",t.toRef);if("ok"in n)return{res:n,point:null};u=n}const p=await o.evaluate("({ w: innerWidth, h: innerHeight })").catch(()=>null),g=n=>!!p&&(n.x<0||n.y<0||n.x>p.w||n.y>p.h);if(g(s)||g(u))return{res:{ok:!1,note:`drag needs both ends on screen at once, and (${Math.round(s.x)},${Math.round(s.y)}) \u2192 (${Math.round(u.x)},${Math.round(u.y)}) is not \u2014 scroll until both are visible, then drag`},point:null};await o.mouse.move(s.x,s.y),await o.mouse.down(),await o.mouse.move(u.x,u.y,{steps:12}),await o.mouse.up();let b="";if(c){const n=await a(c,"ref",t.ref).catch(()=>null);if(n&&!("ok"in n)){const m=Math.round(Math.hypot(n.x-u.x,n.y-u.y));b=Math.round(Math.hypot(n.x-s.x,n.y-s.y))<2?` \u2014 ${String(t.ref)} did not move`:m<2?` \u2014 ${String(t.ref)} followed to the release point`:` \u2014 ${String(t.ref)} now sits at (${Math.round(n.x)},${Math.round(n.y)}), ${m}px short of the release point`}}return{res:{ok:!0,note:`dragged (${Math.round(s.x)},${Math.round(s.y)}) \u2192 (${Math.round(u.x)},${Math.round(u.y)})${b}`,journal:{fromX:Math.round(s.x),fromY:Math.round(s.y)}},point:u}}export{A as DIALOG_VERB,B as DRAG_VERB,j as UPLOAD_VERB,I as clickNamingCover,N as fileInputFor,U as performDrag,_ as performUploadToRef,D as refuseIfGated,L as refuseMissingFiles};
|
|
@@ -1,4 +1,8 @@
|
|
|
1
1
|
import type { Page } from "playwright-core";
|
|
2
2
|
import type { ActResult } from "../types.ts";
|
|
3
|
+
export declare const RUN_JS_TIMEOUT_MS = 10000;
|
|
4
|
+
export declare const RUN_JS_MAX_MS = 45000;
|
|
5
|
+
export declare const RUN_JS_MIN_MS = 1000;
|
|
6
|
+
export declare const RUN_JS_MS_DESC: string;
|
|
3
7
|
export declare const RUN_JS_CODE_DESC: string;
|
|
4
|
-
export declare function performRunJs(page: () => Page, code: unknown): Promise<ActResult>;
|
|
8
|
+
export declare function performRunJs(page: () => Page, code: unknown, ms?: unknown, budgetMs?: unknown): Promise<ActResult>;
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
const
|
|
2
|
-
`)[0]??"")}`}}finally{
|
|
1
|
+
const d=1e4,g=45e3,M=1e3,w="How long to wait for the value, in ms (default 10000, 1000\u201345000). Raise it only for code that is meant to run that long (a loop that waits for the page between steps); it spends the call's 60s, and a promise that never settles waits the whole time.";function p(t,n=2e3){return t.length<=n?t:`${t.slice(0,n)}\u2026 (+${t.length-n} chars)`}const f=200,y="JavaScript evaluated in the page, like typing into the DevTools console \u2014 an expression, or a function body using `return` (either form works). An expression returning a Promise is awaited. Only a JSON-safe value comes back \u2014 a string, number, array or plain object; a DOM node, a function or a circular structure does not survive, so map what you want out of it in the expression. READ, COMPUTE, DIAGNOSE with it: a whole table or list in one call, localStorage, the scroll position, a page variable, an API the page already exposes. To ACT \u2014 click, type, select \u2014 use the interaction verbs: element.click() from here skips the visibility and real-input checks and leaves no record that the page was changed. If you do change state with it, say so in your report.",m=/Illegal return statement/;async function N(t,n,o,i){const r=typeof n=="string"?n.trim():"";if(!r)return{ok:!1,note:"run_js needs code \u2014 the JavaScript to evaluate"};const c=typeof o=="number"&&Number.isFinite(o)?Math.min(Math.max(Math.trunc(o),1e3),45e3):1e4,S=typeof i=="number"&&Number.isFinite(i)?Math.max(1e3,Math.trunc(i)):1/0,a=Math.min(c,S),u=Symbol("run_js-timeout");let s;const h=e=>Promise.race([t().evaluate(e),new Promise(l=>{s=setTimeout(()=>l(u),a)})]);try{let e;try{e=await h(r)}catch(_){if(!m.test(_.message||""))throw _;e=await h(("((async()=>{("+(r)+")})())"))}if(e===u)return{ok:!1,note:`run_js gave no result within ${a/1e3}s \u2014 the code may still be running in the page; observe to see its effect. `+(a<c?"That was all this call had left of its 60s \u2014 run it first thing in a fresh call. ":a<45e3?`Code meant to run longer can pass ms (up to ${45e3/1e3}s). `:"")+"A promise waiting on a permission or a user gesture (navigator.clipboard.readText, Notification.requestPermission, geolocation) can wait forever in a tab nobody is looking at, however long ms is \u2014 to check what a button copied, replace navigator.clipboard.writeText with a recorder before the click and read what it was handed"};const l=e===void 0?"undefined":JSON.stringify(e)??String(e);return{ok:!0,value:e,note:`result: ${p(l,f)}`}}catch(e){return{ok:!0,note:`threw: ${p((e.message||String(e)).split(`
|
|
2
|
+
`)[0]??"")}`}}finally{s&&clearTimeout(s)}}export{y as RUN_JS_CODE_DESC,g as RUN_JS_MAX_MS,M as RUN_JS_MIN_MS,w as RUN_JS_MS_DESC,d as RUN_JS_TIMEOUT_MS,N as performRunJs};
|
|
@@ -4,4 +4,4 @@ This driver attaches to a running app and never starts one, and it needs that ap
|
|
|
4
4
|
An INSTALLED app opens no such port and cannot be driven here: the port is unauthenticated, so shipping one would hand every local process the app. To operate an installed app, use its own authenticated route (a CLI or API it ships).`}import{RUN_JS_VERB as x,UPLOAD_TO_REF_VERB as I}from"./verbs.js";class j extends u{previewTarget(){return!this.active||this.active.isClosed()?Promise.resolve(null):p(this.active,this.cdpUrl,"electron")}platform="electron";browser=null;context=null;active=null;windowIds=new WeakMap;nextWindowId=1;cdpUrl;constructor(t={}){super();const e=t.env??process.env;this.cdpUrl=t.cdpUrl?.trim()||e.COMPUTER_USE_ELECTRON_CDP_URL?.trim()||R}async snapshot(){const t=await super.snapshot(),e=await this.page().evaluate(()=>{const n=/\(KHTML, like Gecko\) (.+?)\/(\S+) Chrome\/.* Electron\//.exec(navigator.userAgent);return n?{name:n[1],version:n[2]}:void 0}).catch(()=>{});return t.target={transport:"cdp",endpoint:this.cdpUrl,windowId:this.windowId(this.page()),...e?{application:e}:{}},t}extraVerbs(){return[w,v,b,E,I,x]}staleRefCause(){return"it predates a navigation, a reload or a window switch"}promptFragment(){return"You are operating a DESKTOP APPLICATION, not a web page: there is no address bar, no back button and no reload \u2014 reach everything through the app's own UI. computer.surfaces() lists WINDOWS, not tabs. run_js is the app's DevTools console \u2014 a DEBUG channel for inspecting state; operating the product goes through the interaction verbs. Grounding: observations are text lists of interactive elements with refs, for the CURRENT viewport only. Act by ref from the latest observation. When what you want is NOT in the list: if you can name it, w.find('<name>') searches EVERY control in the window and hands back a handle \u2014 act on it directly, the click scrolls it into view, so do not scroll first. Scroll only to discover what you cannot name. Observations may show only changes since the previous one; refs stay valid across observations. Use text() to read the full window text."}async ensure(){if(this.browser?.isConnected()&&this.active&&!this.active.isClosed())return;if(this.browser?.isConnected()){if(!this.surfaces().length)throw new h(this.goneMessage());this.resetObservationState();return}this.browser=null,this.context=null,this.active=null;const t=await C(this.cdpUrl);if(t===null)throw new Error(l(this.cdpUrl,"nothing answers at /json/list"));const e=t?await U(this.cdpUrl,t):[];let n;try{n=await d.connectOverCDP(this.cdpUrl,{timeout:O}),await y(n)}catch(r){const a=r instanceof Error?r.message.split(`
|
|
5
5
|
`)[0]??"":String(r);throw t?new Error(`mirasim-cu: ${this.cdpUrl} answers \u2014 ${t.length} target(s): ${t.map(c=>c.url).join(", ")} \u2014 but attaching to it failed.
|
|
6
6
|
(${a})
|
|
7
|
-
Attaching initialises every target, so one renderer that never answers hangs the whole connection; a DevTools window is the usual one`+(e.length?` (${e.length} unresponsive one(s) were just closed: ${e.join(", ")})`:"")+". Close the app's DevTools windows and try again."):new Error(l(this.cdpUrl,a))}const o=n.contexts()[0];if(!o)throw await n.close().catch(()=>{}),new Error(`mirasim-cu: ${this.cdpUrl} answered but exposes no browser context.`);this.browser=n,this.context=o;const s=this.surfaces();if(!s.length)throw new Error(`mirasim-cu: attached to ${this.cdpUrl} but the app has no renderer window to drive (only its DevTools or an about:blank surface). Open the app's window and try again.`);this.active=s[s.length-1],this.resetObservationState()}surfaces(){return(this.context?.pages()??[]).filter(S)}goneMessage(){return"The application you were driving has closed \u2014 every window is gone. This driver attaches to a running app and never starts one, so there is nothing to reopen and nothing to re-observe. Stop and report what you had done when it closed."}page(){if(this.active&&!this.active.isClosed())return this.active;throw new h("The bound Electron window closed. List surfaces and explicitly bind the intended remaining window; no input was redirected.")}async listTargets(){return await this.ensure(),await Promise.all(this.surfaces().map(async t=>({id:this.windowId(t),kind:"window",title:await t.title().catch(()=>"")||t.url(),active:t===this.active})))}async focusTarget(t){await this.ensure();for(const e of this.surfaces())if(this.windowId(e)===t)return this.active=e,this.resetObservationState(),{ok:!0};return{ok:!1,note:`no window with id ${t}; call computer.surfaces() for current ids`}}windowId(t){let e=this.windowIds.get(t);return e||(e=`w${this.nextWindowId++}`,this.windowIds.set(t,e)),e}async perform(t,e){if(await this.ensure(),t==="upload"){const n=Array.isArray(e.paths)?e.paths:[],o=m(n);if(o)return o;const s=String(e.ref),r=e.name,a=await this.locate(s,r,!1);if(!a)return await this.missing(s,r);const c=await f(a);return c?await g(n,s,async()=>c,()=>this.missing(s,r)):{ok:!1,note:`${s} is not a file input and the UI around it holds no single one \u2014 pass the attach control itself; if that button opens the app's OWN file dialog, the window is native and cannot be answered`}}return t==="run_js"?await T(()=>this.page(),e.code):await this.dispatchPage(t,e)??{ok:!1,note:`unknown action kind ${t}`}}async close(){await this.browser?.close().catch(()=>{}),this.browser=null,this.context=null,this.active=null,this.resetObservationState(),this.resetPageState(),await this.disconnected()}}export{j as ElectronDriver};
|
|
7
|
+
Attaching initialises every target, so one renderer that never answers hangs the whole connection; a DevTools window is the usual one`+(e.length?` (${e.length} unresponsive one(s) were just closed: ${e.join(", ")})`:"")+". Close the app's DevTools windows and try again."):new Error(l(this.cdpUrl,a))}const o=n.contexts()[0];if(!o)throw await n.close().catch(()=>{}),new Error(`mirasim-cu: ${this.cdpUrl} answered but exposes no browser context.`);this.browser=n,this.context=o;const s=this.surfaces();if(!s.length)throw new Error(`mirasim-cu: attached to ${this.cdpUrl} but the app has no renderer window to drive (only its DevTools or an about:blank surface). Open the app's window and try again.`);this.active=s[s.length-1],this.resetObservationState()}surfaces(){return(this.context?.pages()??[]).filter(S)}goneMessage(){return"The application you were driving has closed \u2014 every window is gone. This driver attaches to a running app and never starts one, so there is nothing to reopen and nothing to re-observe. Stop and report what you had done when it closed."}page(){if(this.active&&!this.active.isClosed())return this.active;throw new h("The bound Electron window closed. List surfaces and explicitly bind the intended remaining window; no input was redirected.")}async listTargets(){return await this.ensure(),await Promise.all(this.surfaces().map(async t=>({id:this.windowId(t),kind:"window",title:await t.title().catch(()=>"")||t.url(),active:t===this.active})))}async focusTarget(t){await this.ensure();for(const e of this.surfaces())if(this.windowId(e)===t)return this.active=e,this.resetObservationState(),{ok:!0};return{ok:!1,note:`no window with id ${t}; call computer.surfaces() for current ids`}}windowId(t){let e=this.windowIds.get(t);return e||(e=`w${this.nextWindowId++}`,this.windowIds.set(t,e)),e}async perform(t,e){if(await this.ensure(),t==="upload"){const n=Array.isArray(e.paths)?e.paths:[],o=m(n);if(o)return o;const s=String(e.ref),r=e.name,a=await this.locate(s,r,!1);if(!a)return await this.missing(s,r);const c=await f(a);return c?await g(n,s,async()=>c,()=>this.missing(s,r)):{ok:!1,note:`${s} is not a file input and the UI around it holds no single one \u2014 pass the attach control itself; if that button opens the app's OWN file dialog, the window is native and cannot be answered`}}return t==="run_js"?await T(()=>this.page(),e.code,e.ms,e.budgetMs):await this.dispatchPage(t,e)??{ok:!1,note:`unknown action kind ${t}`}}async close(){await this.browser?.close().catch(()=>{}),this.browser=null,this.context=null,this.active=null,this.resetObservationState(),this.resetPageState(),await this.disconnected()}}export{j as ElectronDriver};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{Type as e}from"typebox";import{RUN_JS_CODE_DESC as t}from"../chromium/run-js.js";import"../types.js";const
|
|
1
|
+
import{Type as e}from"typebox";import{RUN_JS_CODE_DESC as t,RUN_JS_MAX_MS as i,RUN_JS_MIN_MS as n,RUN_JS_MS_DESC as a}from"../chromium/run-js.js";import"../types.js";const d={kind:"upload",schema:e.Object({kind:e.Literal("upload"),paths:e.Array(e.String(),{description:"Absolute path(s) of the file(s) to upload. They must already exist on this machine.",minItems:1}),ref:e.String({description:"REQUIRED here: the attach control you can SEE \u2014 the paperclip/add button, or a `file` element if the list shows one. The input behind such a button is hidden and never appears in an observation, so it is found from this ref instead. If the button opens the app's OWN file dialog rather than a hidden input, that window is native and cannot be answered \u2014 use another route (many apps also accept a typed or pasted path)."}),name:e.Optional(e.String({description:"Visible name of the element \u2014 identity check + fallback locator"}))})},p={kind:"run_js",readOnly:!0,schema:e.Object({kind:e.Literal("run_js"),code:e.String({description:`${t} This is YOUR OWN app under development, so the console is an ordinary part of debugging it \u2014 including the command bridges it exposes.`}),ms:e.Optional(e.Integer({minimum:n,maximum:i,description:a}))})};export{p as RUN_JS_VERB,d as UPLOAD_TO_REF_VERB};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{Type as h}from"typebox";const
|
|
1
|
+
import{Type as h}from"typebox";const m={kind:"type",schema:h.Object({kind:h.Literal("type"),text:h.String({description:"Text to type, character by character"})},{description:"Type text with key events at the app's keyboard focus \u2014 for a field that takes typed input only (a file being renamed, a search field that filters as you type). set_text fills a field by ref; type is for where that does not stick. Focus the field first (click, or select_text with caret)."})},l=250;async function u(s,c,f){const{typedKey:a,typedValue:e,typedRange:o}=f;if(!a||typeof e!="string"||!o)return{note:"unverified \u2014 the target field, original text or selection could not be read before input"};const{start:t,length:r}=o;if(!Number.isSafeInteger(t)||!Number.isSafeInteger(r)||t<0||r<0||t>e.length||r>e.length-t)return{note:"unverified \u2014 the original selection was outside the readable text"};const i=e.slice(0,t)+c+e.slice(t+r);try{const d=Date.now()+200;for(;;){const{value:n}=await s("textvalue",{key:a});if(typeof n!="string")break;if(Date.now()>=d)return n!==i&&n===c&&e.trim()&&!n.includes(e.trim())?{verified:!0,note:`matched \u2014 the field reads back exactly the typed text; what it read before input (${JSON.stringify(e.trim().slice(0,40))}) was an empty field's placeholder, which some web fields report as their value`}:n===i&&i===e?{note:"unverified \u2014 the expected text was already present before input, so read-back cannot establish whether the keys took effect"}:{verified:n===i,note:n===i?"matched \u2014 the same field reads back exactly the expected text after inserting or replacing the original selection":"content differs \u2014 the same field does not read back the expected text; automatic correction or formatting may change it. This does not establish where the keys landed; inspect it with text() before editing further, and do not automatically replay the input"};await new Promise(p=>setTimeout(p,Math.min(50,d-Date.now())))}}catch{}return{note:"unverified \u2014 the original target field could not be read after input; inspect it before editing further, and do not automatically replay the input"}}async function b(s,c,f,a,e=0){if(!a)return{ok:!1,note:"type needs text"};const o=e?Math.max(0,Date.now()-e):Number.POSITIVE_INFINITY,t=o<l?Math.ceil(l-o):0;t&&await new Promise(d=>setTimeout(d,t));const r=await s("type",{pid:c,text:a}),i=await u(s,a,r??{});return{ok:!0,...i,note:`posted ${r?.postedCount??a.length} characters at ${f}'s keyboard focus with key events`+(t?`, after ${t} ms for the key before it to finish moving the focus \u2014 a field that commits on Enter hands it on asynchronously, and a character posted into that gap is lost with no error`:"")+`; ${i.note}`}}export{l as TYPE_AFTER_KEY_MS,m as TYPE_VERB,b as typeVerb};
|
|
@@ -15,6 +15,7 @@ export interface RunLedger {
|
|
|
15
15
|
push(kind: string, target: string, ok: boolean, note: string, read?: boolean, outcome?: ActionOutcome): void;
|
|
16
16
|
attach(part: ToolResult["content"][number]): void;
|
|
17
17
|
round?: WriteRound;
|
|
18
|
+
remainingMs?(): number;
|
|
18
19
|
}
|
|
19
20
|
export interface WriteRound {
|
|
20
21
|
fields: number;
|