@petukhovart/agent-view 0.12.0 → 0.13.0

package/dist/server/tauri-dialog-shim.js

@@ -0,0 +1,55 @@
+/** What the arm script reports back from the page. */
+export var TauriShimResult;
+(function (TauriShimResult) {
+    TauriShimResult["Armed"] = "armed";
+    /** The window is not a Tauri webview — nothing to patch, and nothing is wrong. */
+    TauriShimResult["NoTauri"] = "no-tauri";
+})(TauriShimResult || (TauriShimResult = {}));
+const STORE_KEY = '__agentViewTauriDialog__';
+/**
+ * Tauri opens its file dialog in Rust, past the webview, so CDP cannot see it
+ * and cannot close it. The one seam is the JS side of the call:
+ * `@tauri-apps/plugin-dialog` resolves `window.__TAURI_INTERNALS__.invoke` at
+ * call time, so replacing that function answers the dialog before the Rust side
+ * ever runs. The patch survives repeated arming but not a navigation — a fresh
+ * document has a fresh `window`, and the window must be armed again.
+ */
+export function buildTauriArmScript(arm) {
+    return `(() => {
+  const internals = window.__TAURI_INTERNALS__;
+  if (!internals || typeof internals.invoke !== 'function') return ${JSON.stringify(TauriShimResult.NoTauri)};
+  let store = window[${JSON.stringify(STORE_KEY)}];
+  // Disarming a window that was never armed must not install the patch: a
+  // command whose whole purpose is to remove interference cannot add some.
+  if (!store && ${JSON.stringify(arm === null)}) return ${JSON.stringify(TauriShimResult.NoTauri)};
+  if (!store) {
+    store = window[${JSON.stringify(STORE_KEY)}] = { arm: null, fired: [] };
+    const original = internals.invoke.bind(internals);
+    internals.invoke = function (cmd, args, opts) {
+      const arm = store.arm;
+      if (arm && (cmd === 'plugin:dialog|open' || cmd === 'plugin:dialog|save')) {
+        store.arm = null;
+        store.fired.push({ ts: Date.now(), cmd: cmd, cancel: arm.kind === 'cancel' });
+        if (store.fired.length > 20) store.fired.shift();
+        if (arm.kind === 'cancel') return Promise.resolve(null);
+        const multiple = cmd === 'plugin:dialog|open' && !!(args && args.options && args.options.multiple);
+        return Promise.resolve(multiple ? arm.files : (arm.files[0] === undefined ? null : arm.files[0]));
+      }
+      return original(cmd, args, opts);
+    };
+  }
+  store.arm = ${JSON.stringify(arm)};
+  return ${JSON.stringify(TauriShimResult.Armed)};
+})()`;
+}
+export function buildTauriStatusScript() {
+    return `(() => {
+  const store = window[${JSON.stringify(STORE_KEY)}];
+  return {
+    patched: !!store,
+    armed: !!(store && store.arm),
+    fired: store ? store.fired : [],
+  };
+})()`;
+}
+//# sourceMappingURL=tauri-dialog-shim.js.map

package/dist/server/tauri-dialog-shim.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tauri-dialog-shim.js","sourceRoot":"","sources":["../../src/server/tauri-dialog-shim.ts"],"names":[],"mappings":"AAEA,sDAAsD;AACtD,MAAM,CAAN,IAAY,eAIX;AAJD,WAAY,eAAe;IACzB,kCAAe,CAAA;IACf,kFAAkF;IAClF,uCAAoB,CAAA;AACtB,CAAC,EAJW,eAAe,KAAf,eAAe,QAI1B;AAQD,MAAM,SAAS,GAAG,0BAA0B,CAAA;AAE5C;;;;;;;GAOG;AACH,MAAM,UAAU,mBAAmB,CAAC,GAA0B;IAC5D,OAAO;;qEAE4D,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,OAAO,CAAC;uBACrF,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC;;;kBAG9B,IAAI,CAAC,SAAS,CAAC,GAAG,KAAK,IAAI,CAAC,YAAY,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,OAAO,CAAC;;qBAE5E,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC;;;;;;;;;;;;;;;gBAe9B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC;WACxB,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,KAAK,CAAC;KAC3C,CAAA;AACL,CAAC;AAED,MAAM,UAAU,sBAAsB;IACpC,OAAO;yBACgB,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC;;;;;;KAM7C,CAAA;AACL,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@petukhovart/agent-view",
-  "version": "0.12.0",
+  "version": "0.13.0",
   "description": "CLI tool for visual verification of desktop apps (Electron/Tauri) via Chrome DevTools Protocol",
   "type": "module",
   "main": "dist/index.js",

package/skills/verify/SKILL.md

@@ -47,7 +47,8 @@ agent-view dom --diff                   # Lines changed since last dom call (+ a
 ### Interaction
 ```bash
 agent-view click <ref>                  # Click element by ref from dom output
-agent-view click --pos 100,200          # Click by coordinates (for canvas)
+agent-view click --filter "Save"        # Find element by text and click
+agent-view click --pos 100,200          # Click by coordinates — CANVAS ONLY, see below
 agent-view click <ref> --double         # Double-click (fires dblclick handlers); works with --filter / --pos too
 agent-view fill <ref> "text"            # Type into input field
 agent-view drag --from <ref> --to <ref>          # Drag element to another element by ref
@@ -61,6 +62,100 @@ ref and coordinate (e.g. `--from <ref> --to-pos 400,300`). For canvas/Pixi targe
 Refs are resolved fresh on each call, so window resizes between snapshots are tolerated.
 Increase `--steps` for handlers using `globalpointermove` so intermediate frames are not skipped.
 
+**Coordinates are a last resort.** `--pos` / `--from-pos` / `--to-pos` exist for canvas and WebGL,
+where no ref exists. On DOM the correct order is `dom --filter "<text>"` → take the `[ref=N]` →
+`click <ref>`, or `click --filter "<text>"` in one step. A coordinate pair breaks on any layout
+shift, scroll, zoom, or window resize, and it clicks whatever now sits at that point — silently.
+If you reach for `--pos` on a DOM element, first say why the ref was not usable.
+
+### Modals & file pickers (`dialog`, `upload`)
+
+A modal that agent-view cannot answer stops a run dead: the window looks frozen, every
+later command times out, and nothing says why. Two kinds, handled differently.
+
+**JS modals — `alert` / `confirm` / `prompt` / `beforeunload` — are answered for you.**
+Nothing to set up. While agent-view is attached these never block the page: the default
+standing answer is *dismiss*, and each one is recorded. `agent-view dialog` is the
+reliable place to read that record; the console feed carries the same line
+(`[agent-view] confirm auto-dismissed: <message>`) but only from the moment console
+attaches, so a modal answered before your first `console` call reaches `dialog` alone.
+
+```bash
+agent-view dialog                      # standing answer + every modal this window has seen
+agent-view dialog policy accept        # confirm() → true from now on
+agent-view dialog policy accept --text "name"   # prompt() → "name"
+agent-view dialog policy dismiss       # back to the default
+agent-view dialog dismiss              # answer one that is open right now
+agent-view dialog accept --text "x"    # …ditto, accepting
+```
+
+`dialog accept` / `dialog dismiss` exist for a modal that was already open **before**
+agent-view attached — that one produced no event, so no policy applied to it.
+
+**Native file pickers never open — you answer them in advance.** Which mechanism applies
+depends on how the app opens the picker, and `dialog arm` sets up all of them at once,
+so you do not have to know:
+
+```bash
+# The input already exists in the DOM (even hidden) — no picker at all, cheapest path
+agent-view upload --selector "#file-input" --file ./fixtures/a.png
+agent-view upload --selector "#imgs" --file ./a.png --file ./b.png   # multi-select
+agent-view upload --ref 12 --file ./a.png                            # if the AX tree exposes it
+
+# The input is created inside the click handler, or the app calls a native dialog API
+agent-view dialog arm --file ./fixtures/a.png    # then click the button that opens it
+agent-view dialog arm --cancel                   # act as if the user pressed Cancel
+agent-view dialog disarm                         # let real pickers open again
+```
+
+Rules that actually bite:
+
+- **Arm before the click.** A picker cannot be caught once it is open. `arm` is one-shot —
+  it is spent by the first picker and interception turns itself off, so a later click
+  opens a real OS dialog.
+- **Click through `agent-view click`, never `eval "el.click()"`.** Chromium refuses to
+  open a file picker without user activation, and an eval-driven click carries none: the
+  picker is silently dropped and never intercepted.
+- **Hidden inputs have no ref.** `display:none` / `v-show="false"` keeps them out of the
+  AX tree, so `dom` never prints one. Use `--selector`. `upload` has no `--filter` on
+  purpose: an accessible name lands on the label, not on the input behind it.
+- **`beforeunload` is always dismissed**, whatever the policy — accepting it navigates
+  away and loses the state you are checking.
+- Paths are resolved against your cwd and must exist — CDP accepts a bad path silently and
+  the app then reads an empty file.
+- `agent-view dialog` after the fact shows what was intercepted and what was answered.
+
+Known limits: `showOpenFilePicker()` (File System Access API) exposes no input to fill, so
+it can only be cancelled. Electron does not implement `window.prompt` at all. Native
+dialogs opened straight from an Electron **main** process (`dialog.showOpenDialog` behind
+an IPC channel) are out of reach — CDP does not see the main process.
+
+### Waiting (`wait`)
+
+```bash
+agent-view wait --filter "Saved"                    # until the text appears in the AX tree
+agent-view wait --filter "Saved" --timeout 20       # max wait in seconds (default 10)
+agent-view wait --filter "Row 5" --window "Main"    # specific window
+```
+
+Exits as soon as the element appears; exits non-zero on timeout — so `&&` after it is a real gate.
+
+**Never sleep for a fixed time.** No `sleep`, no `timeout`, no `ping -n N 127.0.0.1` as a delay
+(agents reach for `ping` when the harness blocks `sleep` — it is the same mistake in worse
+clothing). A fixed pause is either too short, and you assert against a half-rendered UI, or too
+long, and you burn wall-clock on every step. agent-view has a condition-based wait for every kind
+of signal:
+
+| You are waiting for… | Use |
+|---|---|
+| An element to render | `wait --filter "<text>"` |
+| A store/state value | `watch "<expr>" --until "<expr>"` |
+| A log line | `console --follow --until "<pattern>"` |
+| A request to fire | `network --follow --until "<url>"` |
+
+If none of these fits, poll: `dom --filter X --count` in a loop with an explicit attempt cap, and
+report how many attempts it took.
+
 ### Screenshots
 ```bash
 agent-view screenshot --scale 0.5              # Recommended: JPEG at half-res (~3× fewer vision tokens)
@@ -239,6 +334,7 @@ Verifications cost very different amounts. Pick the cheapest tool that can actua
 | Count of matching elements | `dom --filter X --count` | Single integer, no tree output, no ref mutations |
 | App state, store contents, computed values | `eval "expr"` | DOM doesn't expose JS state; reading the tree to infer it is wasteful and unreliable |
 | Does `window.X` / a globally-exposed API exist? | `eval "typeof window.X"` | DOM doesn't show JS globals; only authoritative check |
+| An element that has not rendered yet | `wait --filter "<text>"` | Exits on appearance and non-zero on timeout — a real gate, unlike a fixed pause |
 | State *trajectory* — what changed during/after an action | `watch "expr" --until …` or `--max-changes 1` | `eval` shows the final snapshot only; `watch` shows the diffs in order |
 | Worker logic (SharedWorker / ServiceWorker) | `eval --target <name>` | Workers have no DOM at all |
 | Did the last action throw or warn? | `console --clear` before, `console --level error,warn` after | Catches errors that don't surface in the DOM |
@@ -247,6 +343,9 @@ Verifications cost very different amounts. Pick the cheapest tool that can actua
 | Layout, spacing, full-window visual regression | `screenshot --scale 0.5` | The only tool that sees pixels — but expensive (~6k tokens), use last |
 | Canvas/WebGL scene contents | `scene --diff` | DOM is empty for canvas apps |
 | What DOM nodes changed after an interaction | `dom --diff` | Returns only `+`/`-` lines; much cheaper than re-reading the full tree |
+| Selecting a file for an input that exists in the DOM | `upload --selector` | No picker opens at all; works on hidden inputs, which have no ref |
+| Selecting a file when the input appears only mid-click | `dialog arm --file` then `click` | The only way — the input does not exist before the click and is gone after |
+| The window stopped responding after a click | `dialog` | Shows whether a modal was answered, and what the app was told |
 
 When two tools could answer the same question, prefer the one higher up the table. A common mistake is screenshotting to check "is the count = 5?" when `eval "store.counter"` returns the number directly for ~50 tokens.
 
@@ -256,6 +355,22 @@ When two tools could answer the same question, prefer the one higher up the tabl
 
 A run can produce one of three outcomes per step: **pass**, **fail**, **requires_visual_review**. There is no fourth bucket called "actually fine, here's why". A failed `Expected:` line is FAIL.
 
+**How to run the commands themselves:**
+
+- **Never discard output.** No `>/dev/null`, no `2>&1` to nowhere, no `| head -1` on a command whose
+  failure you have not yet read. Every agent-view command prints either the evidence or the reason
+  it failed; a suppressed `click` that matched nothing looks exactly like a successful one.
+- **Chain with `&&`, not newlines.** Newline-separated commands keep running after a failure, so a
+  broken first step is followed by three steps acting on the wrong state. `&&` stops at the first
+  non-zero exit.
+- **One action, then one check.** `click` is not evidence. The evidence is the `dom --diff`,
+  `dom --filter … --count`, `eval`, or `console --level error` you run after it. A block of three
+  clicks in a row with no check between them proves nothing about any of them.
+- **Call the `agent-view` binary.** Install it once in the target project
+  (`pnpm add -D @petukhovart/agent-view`) and call `agent-view …` or `pnpm exec agent-view …`.
+  Do not prefix every call with `npx <package>`: it re-resolves the package on each invocation and
+  falls outside this skill's `allowed-tools`, so each call needs a fresh permission prompt.
+
 These heuristics catch real bugs. Skipping them is how a run silently passes while the bug sits in plain sight in the same data:
 
 1. **A failed expectation is FAIL.** If output disagrees with what the step expected, mark `fail` and continue. Do not soften the expectation. Do not invent prose explanations inline ("label reuse", "convention", "arithmetic off"). Justifications belong in the bug report after the run, never in the per-step log.
@@ -321,7 +436,7 @@ Tolerance default: a designer's code-review level — flag what they'd notice, i
 ## Resilience
 
 - **Stale refs:** After HMR, navigation, or state change — re-run `dom` for fresh refs before interacting
-- **Element not found:** Wait 2s, retry once (render delay after HMR). If still missing — report FAIL
+- **Element not found:** `agent-view wait --filter "<text>" --timeout 5` (covers the render delay after HMR). If it times out — report FAIL. Do not insert a blind pause and retry.
 - **CDP disconnect:** Run `agent-view discover` to check. If no windows — `agent-view launch` (auto-starts Electron/Browser/Tauri). On `PORT_CONFLICT` — surface PID/process and ask user.
 - **`CDP_TIMEOUT` error:** the command hit the server-side deadline; cached CDP sessions for that port were dropped, so retry once. Repeated timeouts mean the app's DevTools endpoint is wedged — restart the app.
 - **Max retries per command:** 2. After that — SKIP scenario step with warning