agent-device 0.2.3 → 0.2.5

package/README.md

@@ -33,9 +33,11 @@ npx agent-device open SampleApp
 
 ## Quick Start
 
+Use refs for agent-driven exploration and normal automation flows.
+
 ```bash
 agent-device open Contacts --platform ios # creates session on iOS Simulator
-agent-device snapshot                      
+agent-device snapshot
 agent-device click @e5
 agent-device fill @e6 "John"
 agent-device fill @e7 "Doe"
@@ -75,7 +77,7 @@ Coordinates:
 ## Command Index
 - `open`, `close`, `home`, `back`, `app-switcher`
 - `snapshot`, `find`, `get`
-- `click`, `focus`, `type`, `fill`, `press`, `long-press`, `scroll`, `scrollintoview`
+- `click`, `focus`, `type`, `fill`, `press`, `long-press`, `scroll`, `scrollintoview`, `is`
 - `alert`, `wait`, `screenshot`
 - `trace start`, `trace stop`
 - `settings wifi|airplane|location on|off`
@@ -99,7 +101,6 @@ Flags:
 - `--udid <udid>` (iOS)
 - `--serial <serial>` (Android)
 - `--activity <component>` (Android; package/Activity or package/.Activity)
-- `--out <path>` (screenshot)
 - `--session <name>`
 - `--verbose` for daemon and runner logs
 - `--json` for structured output
@@ -108,19 +109,55 @@ Flags:
 ## Skills
 Install the automation skills listed in [SKILL.md](skills/agent-device/SKILL.md).
 
+```bash
+npx skills add https://github.com/callstackincubator/agent-device --skill agent-device
+```
+
 Sessions:
 - `open` starts a session. Without args boots/activates the target device/simulator without launching an app.
 - All interaction commands require an open session.
 - If a session is already open, `open <app>` switches the active app and updates the session app bundle.
 - `close` stops the session and releases device resources. Pass an app to close it explicitly, or omit to just close the session.
 - Use `--session <name>` to manage multiple sessions.
-- Session logs are written to `~/.agent-device/sessions/<session>-<timestamp>.ad`.
+- Session scripts are written to `~/.agent-device/sessions/<session>-<timestamp>.ad` when recording is enabled with `--save-script`.
+- Deterministic replay is `.ad`-based; use `replay --update` (`-u`) to update selector drift and rewrite the replay file in place.
 
 Find (semantic):
 - `find <text> <action> [value]` finds by any text (label/value/identifier) using a scoped snapshot.
 - `find text|label|value|role|id <value> <action> [value]` for specific locators.
 - Actions: `click` (default), `fill`, `type`, `focus`, `get text`, `get attrs`, `wait [timeout]`, `exists`.
 
+Assertions:
+- `is` predicates: `visible`, `hidden`, `exists`, `editable`, `selected`, `text`.
+- `is text` uses exact equality.
+
+Replay update:
+- `replay <path>` runs deterministic replay from `.ad` scripts.
+- `replay -u <path>` attempts selector updates on failures and atomically rewrites the same file.
+- Refs are the default/core mechanism for interactive agent flows.
+- Update targets: `click`, `fill`, `get`, `is`, `wait`.
+- Selector matching is a replay-update internal: replay parses `.ad` lines into actions, tries them, snapshots on failure, resolves a better selector, then rewrites that failing line.
+
+Update examples:
+
+```sh
+# Before (stale selector)
+click "id=\"old_continue\" || label=\"Continue\""
+
+# After replay -u (rewritten in place)
+click "id=\"auth_continue\" || label=\"Continue\""
+```
+
+```sh
+# Before (ref-based action from discovery)
+snapshot -i -c -s "Continue"
+click @e13 "Continue"
+
+# After replay -u (upgraded to selector-based action)
+snapshot -i -c -s "Continue"
+click "id=\"auth_continue\" || label=\"Continue\""
+```
+
 Android fill reliability:
 - `fill` clears the current value, then enters text.
 - `type` enters text into the focused field without clearing.

package/dist/src/bin.js

@@ -2,7 +2,7 @@ import{node_path as e,fileURLToPath as t,asAppError as r,pathToFileURL as n,runC
 `)}function d(e){let t=e.details?`
 ${JSON.stringify(e.details,null,2)}`:"";process.stderr.write(`Error (${e.code}): ${e.message}${t}
 `)}function u(e,t,r){let n=p(e.type??"Element"),a=function(e,t){var r,n;let a=e.label?.trim(),i=e.value?.trim();if("text-field"===(r=t)||"text-view"===r||"search"===r){if(i)return i;if(a)return a}else if(a)return a;if(i)return i;let s=e.identifier?.trim();return!s||(n=s,/^[\w.]+:id\/[\w.-]+$/i.test(n)&&("group"===t||"image"===t||"list"===t||"collection"===t))?"":s}(e,n),i="  ".repeat(t),s=e.ref?`@${e.ref}`:"",o=[!1===e.enabled?"disabled":null].filter(Boolean).join(", "),l=o?` [${o}]`:"",c=a?` "${a}"`:"";return r?`${i}${s} [${n}]${l}`.trimEnd():`${i}${s} [${n}]${c}${l}`.trimEnd()}function p(e){let t=e.replace(/XCUIElementType/gi,"").toLowerCase(),r=e.includes(".")&&(e.startsWith("android.")||e.startsWith("androidx.")||e.startsWith("com."));switch(t.startsWith("ax")&&(t=t.replace(/^ax/,"")),t.includes(".")&&(t=t.replace(/^android\.widget\./,"").replace(/^android\.view\./,"").replace(/^android\.webkit\./,"").replace(/^androidx\./,"").replace(/^com\.google\.android\./,"").replace(/^com\.android\./,"")),t){case"application":return"application";case"navigationbar":return"navigation-bar";case"tabbar":return"tab-bar";case"button":case"imagebutton":return"button";case"link":return"link";case"cell":return"cell";case"statictext":case"checkedtextview":return"text";case"textfield":case"edittext":return"text-field";case"textview":return r?"text":"text-view";case"textarea":return"text-view";case"switch":return"switch";case"slider":return"slider";case"image":case"imageview":return"image";case"webview":return"webview";case"framelayout":case"linearlayout":case"relativelayout":case"constraintlayout":case"viewgroup":case"view":case"group":return"group";case"listview":case"recyclerview":return"list";case"collectionview":return"collection";case"searchfield":return"search";case"segmentedcontrol":return"segmented-control";case"window":return"window";case"checkbox":return"checkbox";case"radio":return"radio";case"menuitem":return"menu-item";case"toolbar":return"toolbar";case"scrollarea":case"scrollview":case"nestedscrollview":return"scroll-area";case"table":return"table";default:return t||"element"}}let f=e.join(s.homedir(),".agent-device"),m=e.join(f,"daemon.json"),h=function(){let e=process.env.AGENT_DEVICE_DAEMON_TIMEOUT_MS;if(!e)return 6e4;let t=Number(e);return Number.isFinite(t)?Math.max(1e3,Math.floor(t)):6e4}();async function w(e){let t=await y(),r={...e,token:t.token};return await x(t,r)}async function y(){let t=g(),r=function(){try{let t=$();return JSON.parse(i.readFileSync(e.join(t,"package.json"),"utf8")).version??"0.0.0"}catch{return"0.0.0"}}();if(t&&t.version===r&&await v(t))return t;t&&(t.version!==r||!await v(t))&&i.existsSync(m)&&i.unlinkSync(m),await b();let n=Date.now();for(;Date.now()-n<5e3;){let e=g();if(e&&await v(e))return e;await new Promise(e=>setTimeout(e,100))}throw new l("COMMAND_FAILED","Failed to start daemon",{infoPath:m,hint:"Run pnpm build, or delete ~/.agent-device/daemon.json if stale."})}function g(){if(!i.existsSync(m))return null;try{let e=JSON.parse(i.readFileSync(m,"utf8"));if(!e.port||!e.token)return null;return e}catch{return null}}async function v(e){return new Promise(t=>{let r=o.createConnection({host:"127.0.0.1",port:e.port},()=>{r.destroy(),t(!0)});r.on("error",()=>{t(!1)})})}async function b(){let t=$(),r=e.join(t,"dist","src","daemon.js"),n=e.join(t,"src","daemon.ts"),s=i.existsSync(r);if(!s&&!i.existsSync(n))throw new l("COMMAND_FAILED","Daemon entry not found",{distPath:r,srcPath:n});let o=s?[r]:["--experimental-strip-types",n];a(process.execPath,o)}async function x(e,t){return new Promise((r,n)=>{let a=o.createConnection({host:"127.0.0.1",port:e.port},()=>{a.write(`${JSON.stringify(t)}
-`)}),i=setTimeout(()=>{a.destroy(),n(new l("COMMAND_FAILED","Daemon request timed out",{timeoutMs:h}))},h),s="";a.setEncoding("utf8"),a.on("data",e=>{let t=(s+=e).indexOf("\n");if(-1===t)return;let o=s.slice(0,t).trim();if(o)try{let e=JSON.parse(o);a.end(),clearTimeout(i),r(e)}catch(e){clearTimeout(i),n(e)}}),a.on("error",e=>{clearTimeout(i),n(e)})})}function $(){let r=e.dirname(t(import.meta.url)),n=r;for(let t=0;t<6;t+=1){let t=e.join(n,"package.json");if(i.existsSync(t))return n;n=e.dirname(n)}return r}async function S(t){let n=function(e){let t={json:!1,help:!1},r=[];for(let n=0;n<e.length;n+=1){let a=e[n];if("--json"===a){t.json=!0;continue}if("--help"===a||"-h"===a){t.help=!0;continue}if("--verbose"===a||"-v"===a){t.verbose=!0;continue}if("-i"===a){t.snapshotInteractiveOnly=!0;continue}if("-c"===a){t.snapshotCompact=!0;continue}if("--raw"===a){t.snapshotRaw=!0;continue}if("--no-record"===a){t.noRecord=!0;continue}if("--record-json"===a){t.recordJson=!0;continue}if("--user-installed"===a){t.appsFilter="user-installed";continue}if("--all"===a){t.appsFilter="all";continue}if("--metadata"===a){t.appsMetadata=!0;continue}if(a.startsWith("--backend")){let r=a.includes("=")?a.split("=")[1]:e[n+1];if(a.includes("=")||(n+=1),"ax"!==r&&"xctest"!==r)throw new l("INVALID_ARGS",`Invalid backend: ${r}`);t.snapshotBackend=r;continue}if(a.startsWith("--")){let[r,i]=a.split("="),s=i??e[n+1];switch(!i&&(n+=1),r){case"--platform":if("ios"!==s&&"android"!==s)throw new l("INVALID_ARGS",`Invalid platform: ${s}`);t.platform=s;break;case"--depth":{let e=Number(s);if(!Number.isFinite(e)||e<0)throw new l("INVALID_ARGS",`Invalid depth: ${s}`);t.snapshotDepth=Math.floor(e);break}case"--scope":t.snapshotScope=s;break;case"--device":t.device=s;break;case"--udid":t.udid=s;break;case"--serial":t.serial=s;break;case"--out":t.out=s;break;case"--session":t.session=s;break;case"--activity":t.activity=s;break;default:throw new l("INVALID_ARGS",`Unknown flag: ${r}`)}continue}if("-d"===a){let r=e[n+1];n+=1;let a=Number(r);if(!Number.isFinite(a)||a<0)throw new l("INVALID_ARGS",`Invalid depth: ${r}`);t.snapshotDepth=Math.floor(a);continue}if("-s"===a){let r=e[n+1];n+=1,t.snapshotScope=r;continue}r.push(a)}return{command:r.shift()??null,positionals:r,flags:t}}(t);(n.flags.help||!n.command)&&(process.stdout.write(`agent-device <command> [args] [--json]
+`)}),i=setTimeout(()=>{a.destroy(),n(new l("COMMAND_FAILED","Daemon request timed out",{timeoutMs:h}))},h),s="";a.setEncoding("utf8"),a.on("data",e=>{let t=(s+=e).indexOf("\n");if(-1===t)return;let o=s.slice(0,t).trim();if(o)try{let e=JSON.parse(o);a.end(),clearTimeout(i),r(e)}catch(e){clearTimeout(i),n(e)}}),a.on("error",e=>{clearTimeout(i),n(e)})})}function $(){let r=e.dirname(t(import.meta.url)),n=r;for(let t=0;t<6;t+=1){let t=e.join(n,"package.json");if(i.existsSync(t))return n;n=e.dirname(n)}return r}async function S(t){let n=function(e){let t={json:!1,help:!1},r=[];for(let n=0;n<e.length;n+=1){let a=e[n];if("--json"===a){t.json=!0;continue}if("--help"===a||"-h"===a){t.help=!0;continue}if("--verbose"===a||"-v"===a){t.verbose=!0;continue}if("-i"===a){t.snapshotInteractiveOnly=!0;continue}if("-c"===a){t.snapshotCompact=!0;continue}if("--raw"===a){t.snapshotRaw=!0;continue}if("--no-record"===a){t.noRecord=!0;continue}if("--save-script"===a){t.saveScript=!0;continue}if("--update"===a||"-u"===a){t.replayUpdate=!0;continue}if("--user-installed"===a){t.appsFilter="user-installed";continue}if("--all"===a){t.appsFilter="all";continue}if("--metadata"===a){t.appsMetadata=!0;continue}if(a.startsWith("--backend")){let r=a.includes("=")?a.split("=")[1]:e[n+1];if(a.includes("=")||(n+=1),"ax"!==r&&"xctest"!==r)throw new l("INVALID_ARGS",`Invalid backend: ${r}`);t.snapshotBackend=r;continue}if(a.startsWith("--")){let[r,i]=a.split("="),s=i??e[n+1];switch(!i&&(n+=1),r){case"--platform":if("ios"!==s&&"android"!==s)throw new l("INVALID_ARGS",`Invalid platform: ${s}`);t.platform=s;break;case"--depth":{let e=Number(s);if(!Number.isFinite(e)||e<0)throw new l("INVALID_ARGS",`Invalid depth: ${s}`);t.snapshotDepth=Math.floor(e);break}case"--scope":t.snapshotScope=s;break;case"--device":t.device=s;break;case"--udid":t.udid=s;break;case"--serial":t.serial=s;break;case"--out":t.out=s;break;case"--session":t.session=s;break;case"--activity":t.activity=s;break;default:throw new l("INVALID_ARGS",`Unknown flag: ${r}`)}continue}if("-d"===a){let r=e[n+1];n+=1;let a=Number(r);if(!Number.isFinite(a)||a<0)throw new l("INVALID_ARGS",`Invalid depth: ${r}`);t.snapshotDepth=Math.floor(a);continue}if("-s"===a){let r=e[n+1];n+=1,t.snapshotScope=r;continue}r.push(a)}return{command:r.shift()??null,positionals:r,flags:t}}(t);(n.flags.help||!n.command)&&(process.stdout.write(`agent-device <command> [args] [--json]
 
 CLI to control iOS and Android devices for AI agents.
 
@@ -24,20 +24,22 @@ Commands:
   back                                      Navigate back (where supported)
   home                                      Go to home screen (where supported)
   app-switcher                              Open app switcher (where supported)
-  wait <ms>|text <text>|@ref [timeoutMs]     Wait for duration or text to appear
+  wait <ms>|text <text>|@ref|<selector> [timeoutMs]
+                                             Wait for duration, text, ref, or selector to appear
   alert [get|accept|dismiss|wait] [timeout] Inspect or handle alert (iOS simulator)
-  click <@ref>                               Click element by snapshot ref
-  get text <@ref>                            Return element text by ref
-  get attrs <@ref>                           Return element attributes by ref
-  replay <path>                              Replay a recorded session
+  click <@ref|selector>                      Click element by snapshot ref or selector
+  get text <@ref|selector>                   Return element text by ref or selector
+  get attrs <@ref|selector>                  Return element attributes by ref or selector
+  replay <path> [--update|-u]                Replay a recorded session
   press <x> <y>                              Tap at coordinates
   long-press <x> <y> [durationMs]            Long press (where supported)
   focus <x> <y>                              Focus input at coordinates
   type <text>                                Type text in focused field
-  fill <x> <y> <text> | fill <@ref> <text>   Tap then type
+  fill <x> <y> <text> | fill <@ref|selector> <text>
+                                             Tap then type
   scroll <direction> [amount]                Scroll in direction (0-1 amount)
   scrollintoview <text>                      Scroll until text appears (Android only)
-  screenshot [--out path]                    Capture screenshot
+  screenshot [path]                          Capture screenshot
   record start [path]                        Start screen recording
   record stop                                Stop screen recording
   trace start [path]                         Start trace log capture
@@ -48,6 +50,7 @@ Commands:
   find value <value> <action> [value]        Find by value
   find role <role> <action> [value]          Find by role/type
   find id <id> <action> [value]              Find by identifier/resource-id
+  is <predicate> <selector> [value]          Assert UI state (visible|hidden|exists|editable|selected|text)
   settings <wifi|airplane|location> <on|off> Toggle OS settings (simulators)
   session list                               List active sessions
 
@@ -57,12 +60,12 @@ Flags:
   --udid <udid>                              iOS device UDID
   --serial <serial>                          Android device serial
   --activity <component>                     Android activity to launch (package/Activity)
-  --out <path>                               Output path for screenshots
   --session <name>                           Named session
   --verbose                                  Stream daemon/runner logs
   --json                                     JSON output
+  --save-script                             Save session script (.ad) on close
   --no-record                                Do not record this action
-  --record-json                              Record JSON session log
+  --update, -u                               Replay: update selectors and rewrite replay file in place
   --user-installed                           Apps: list user-installed packages (Android only)
   --all                                      Apps: list all packages (Android only)
 
@@ -80,7 +83,8 @@ ${f.join("\n")}
 `),h&&h();return}}if("find"===a){let t=e.data;if("string"==typeof t?.text){process.stdout.write(`${t.text}
 `),h&&h();return}if("boolean"==typeof t?.found){process.stdout.write(`Found: ${t.found}
 `),h&&h();return}if(t?.node){process.stdout.write(`${JSON.stringify(t.node,null,2)}
-`),h&&h();return}}if("click"===a){let t=e.data?.ref??"",r=e.data?.x,n=e.data?.y;t&&"number"==typeof r&&"number"==typeof n&&process.stdout.write(`Clicked @${t} (${r}, ${n})
+`),h&&h();return}}if("is"===a){let t=e.data?.predicate??"assertion";process.stdout.write(`Passed: is ${t}
+`),h&&h();return}if("click"===a){let t=e.data?.ref??"",r=e.data?.x,n=e.data?.y;t&&"number"==typeof r&&"number"==typeof n&&process.stdout.write(`Clicked @${t} (${r}, ${n})
 `),h&&h();return}if(e.data&&"object"==typeof e.data){let t=e.data;if("devices"===a){let e=(Array.isArray(t.devices)?t.devices:[]).map(e=>{let t=e?.name??e?.id??"unknown",r=e?.platform??"unknown",n=e?.kind?` ${e.kind}`:"",a="boolean"==typeof e?.booted?` booted=${e.booted}`:"";return`${t} (${r}${n})${a}`});process.stdout.write(`${e.join("\n")}
 `),h&&h();return}if("apps"===a){let e=(Array.isArray(t.apps)?t.apps:[]).map(e=>{if("string"==typeof e)return e;if(e&&"object"==typeof e){let t=e.bundleId??e.package,r=e.name??e.label;return r&&t?`${r} (${t})`:t&&"boolean"==typeof e.launchable?`${t} (launchable=${e.launchable})`:t?String(t):JSON.stringify(e)}return String(e)});process.stdout.write(`${e.join("\n")}
 `),h&&h();return}if("appstate"===a){let e=t?.platform,r=t?.appBundleId,n=t?.appName,a=t?.source,i=t?.package,s=t?.activity;if("ios"===e){process.stdout.write(`Foreground app: ${n??r}