@swmansion/argent 0.22.2-next.4 → 0.22.2-next.5

package/dist/cli-cmds.mjs

@@ -2815,6 +2815,10 @@ var FLAG_REGISTRY = [
     name: "disable-auto-screenshot",
     description: "Disable the automatic screenshot captured after interaction tools."
   },
+  {
+    name: "disable-auto-describe",
+    description: "Disable the accessibility element tree appended after interaction tools."
+  },
   {
     name: "argent-lens",
     description: "Argent Lens \u2014 the propose_variant / await_user_selection tools and the Electron preview window for staging UI design variants and letting a human pick among them. Off by default while the feature is in development."
@@ -7210,7 +7214,7 @@ var _CI_VENDOR_COUNT_FOR_TEST = vendors_default.length;
 var SESSION_ID2 = randomUUID5();
 function readCliVersion() {
   if (true) {
-    return "0.22.2-next.4";
+    return "0.22.2-next.5";
   }
   return "0.0.0";
 }

package/dist/installer.mjs

@@ -16428,7 +16428,7 @@ var _CI_VENDOR_COUNT_FOR_TEST = vendors_default.length;
 var SESSION_ID = randomUUID4();
 function readCliVersion() {
   if (true) {
-    return "0.22.2-next.4";
+    return "0.22.2-next.5";
   }
   return "0.0.0";
 }

package/dist/mcp-server.mjs

@@ -18491,7 +18491,7 @@ async function stepArtifactBlocks(artifacts, status, ctx, depth) {
   return blocks;
 }
 
-// ../argent-mcp/src/auto-screenshot.ts
+// ../argent-mcp/src/auto-capture.ts
 var AUTO_SCREENSHOT_TOOLS = /* @__PURE__ */ new Set([
   "gesture-tap",
   "gesture-swipe",
@@ -18532,6 +18532,30 @@ var DEFAULT_DELAY_MS = 1400;
 function autoScreenshotEnabled(options) {
   return !isFlagEnabled("disable-auto-screenshot", options);
 }
+var AUTO_DESCRIBE_TOOLS = /* @__PURE__ */ new Set([
+  "gesture-tap",
+  "gesture-swipe",
+  "gesture-scroll",
+  "gesture-drag",
+  "gesture-custom",
+  "gesture-pinch",
+  "gesture-rotate",
+  "button",
+  "keyboard",
+  "paste",
+  "rotate",
+  "launch-app",
+  "restart-app",
+  "open-url",
+  "run-sequence"
+]);
+function autoDescribeEnabled(options) {
+  return !isFlagEnabled("disable-auto-describe", options);
+}
+var AUTO_DESCRIBE_HEADER = "--- Elements after action (describe) ---";
+function shouldAutoDescribe(toolName) {
+  return AUTO_DESCRIBE_TOOLS.has(normalizeToolName(toolName));
+}
 var SECRET_PLACEHOLDER_MARKER = "{{secret:";
 function containsSecretPlaceholder(args) {
   try {
@@ -18628,6 +18652,7 @@ async function startMcpServer(options) {
     markFirstRunNoticeShown();
   }
   const autoScreenshotOn = autoScreenshotEnabled();
+  const autoDescribeOn = autoDescribeEnabled();
   let TOOLS_URL;
   let AUTH_TOKEN;
   const resolved = await getResolvedToolsUrl();
@@ -18715,7 +18740,7 @@ async function startMcpServer(options) {
     { name: "argent", version: getInstalledVersion() },
     {
       capabilities: { tools: {} },
-      instructions: "Argent \u2014 iOS Simulator, Android Emulator, and Chromium app control for interacting, testing, profiling and debugging mobile and Chromium applications. Always use discovery tools (describe / debugger-component-tree / screenshot) before tapping \u2014 never guess coordinates. On session end: call stop-all-simulator-servers with devices: [...] naming the devices this session used, and perform any necessary cleanup. One tool-server is shared by every agent using this argent install, so an unscoped call tears down their devices too \u2014 reserve it for a deliberate machine-wide cleanup. Full guidance is in the argent rule loaded from .claude/rules/argent.md."
+      instructions: "Argent \u2014 iOS Simulator, Android Emulator, and Chromium app control for interacting, testing, profiling and debugging mobile and Chromium applications. Interaction tools return the screen after the action: a screenshot plus the accessibility element tree with normalized tap frames. Take coordinates from that tree; call describe (or debugger-component-tree) only when no fresh tree is available \u2014 never guess coordinates from pixels. On session end: call stop-all-simulator-servers with devices: [...] naming the devices this session used, and perform any necessary cleanup. One tool-server is shared by every agent using this argent install, so an unscoped call tears down their devices too \u2014 reserve it for a deliberate machine-wide cleanup. Full guidance is in the argent rule loaded from .claude/rules/argent.md."
     }
   );
   server.setRequestHandler(ListToolsRequestSchema, async () => {
@@ -18767,15 +18792,17 @@ async function startMcpServer(options) {
         content = await toMcpContent(result, outputHint, ctx, params.arguments);
       }
       const udid = getUdidFromArgs(params.arguments);
-      if (autoScreenshotOn && udid && shouldAutoScreenshot(params.name) && containsSecretPlaceholder(params.arguments)) {
+      const wantScreenshot = autoScreenshotOn && shouldAutoScreenshot(params.name);
+      const wantTree = autoDescribeOn && shouldAutoDescribe(params.name);
+      if (udid && (wantScreenshot || wantTree) && containsSecretPlaceholder(params.arguments)) {
         content = [
           ...content,
           {
             type: "text",
-            text: "Auto-screenshot skipped: the input contains a {{secret:\u2026}} placeholder, and a screenshot of this screen could reveal the typed secret. The secret is already typed \u2014 do not send the typing step again, or the field will hold two copies of it. Submit or navigate away, then verify the resulting screen as usual. Only this call is covered: the next call is screenshotted normally, and captures the secret if the field is still on screen. To cover the submit as well, put the typing and the submit in ONE `run-sequence` the next time you type a secret."
+            text: "Auto-screenshot and element tree skipped: the input contains a {{secret:\u2026}} placeholder, and a capture of this screen could reveal the typed secret. The secret is already typed \u2014 do not send the typing step again, or the field will hold two copies of it. Submit or navigate away, then verify the resulting screen as usual. Only this call is covered: the next call is captured normally, and shows the secret if the field is still on screen. To cover the submit as well, put the typing and the submit in ONE `run-sequence` the next time you type a secret."
           }
         ];
-      } else if (autoScreenshotOn && udid && shouldAutoScreenshot(params.name)) {
+      } else if (udid && (wantScreenshot || wantTree)) {
         const maxWaitMs = getAutoScreenshotDelayMs(params.name);
         if (maxWaitMs > 0) {
           try {
@@ -18791,25 +18818,56 @@ async function startMcpServer(options) {
             await new Promise((r) => setTimeout(r, maxWaitMs));
           }
         }
-        try {
-          const screenshotResult = await callTool("screenshot", { udid });
-          const screenshotContent = await toMcpContent(screenshotResult.result, "image", {
-            toolsUrl: TOOLS_URL,
-            authToken: AUTH_TOKEN,
-            deviceId: udid
-          });
-          const hasImage = screenshotContent.some((b) => b.type === "image");
-          if (hasImage) {
-            content = [
-              ...content,
-              {
-                type: "text",
-                text: "--- Screen after action ---"
-              },
-              ...screenshotContent
-            ];
+        if (wantScreenshot) {
+          try {
+            const screenshotResult = await callTool("screenshot", { udid });
+            const screenshotContent = await toMcpContent(screenshotResult.result, "image", {
+              toolsUrl: TOOLS_URL,
+              authToken: AUTH_TOKEN,
+              deviceId: udid
+            });
+            const hasImage = screenshotContent.some((b) => b.type === "image");
+            if (hasImage) {
+              content = [
+                ...content,
+                {
+                  type: "text",
+                  text: "--- Screen after action ---"
+                },
+                ...screenshotContent
+              ];
+            }
+          } catch {
+          }
+        }
+        if (wantTree) {
+          const t1 = Date.now();
+          try {
+            const d = await callTool("describe", { udid });
+            const desc = d.result?.description;
+            if (typeof desc === "string" && desc.length > 0) {
+              content = [
+                ...content,
+                { type: "text", text: `${AUTO_DESCRIBE_HEADER}
+${desc}` }
+              ];
+            }
+            await spyLog({
+              ts: (/* @__PURE__ */ new Date()).toISOString(),
+              event: "auto_describe",
+              name: params.name,
+              durationMs: Date.now() - t1,
+              chars: typeof desc === "string" ? desc.length : 0
+            });
+          } catch (e) {
+            await spyLog({
+              ts: (/* @__PURE__ */ new Date()).toISOString(),
+              event: "auto_describe",
+              name: params.name,
+              durationMs: Date.now() - t1,
+              error: String(e instanceof Error ? e.message : e)
+            });
           }
-        } catch {
         }
       }
       if (note) {

package/dist/tool-server.cjs

@@ -89995,6 +89995,10 @@ var FLAG_REGISTRY = [
     name: "disable-auto-screenshot",
     description: "Disable the automatic screenshot captured after interaction tools."
   },
+  {
+    name: "disable-auto-describe",
+    description: "Disable the accessibility element tree appended after interaction tools."
+  },
   {
     name: "argent-lens",
     description: "Argent Lens \u2014 the propose_variant / await_user_selection tools and the Electron preview window for staging UI design variants and letting a human pick among them. Off by default while the feature is in development."
@@ -93886,7 +93890,7 @@ var _CI_VENDOR_COUNT_FOR_TEST = vendors_default.length;
 var SESSION_ID = (0, import_node_crypto3.randomUUID)();
 function readCliVersion() {
   if (true) {
-    return "0.22.2-next.4";
+    return "0.22.2-next.5";
   }
   return "0.0.0";
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@swmansion/argent",
-  "version": "0.22.2-next.4",
+  "version": "0.22.2-next.5",
   "mcpName": "io.github.software-mansion/argent",
   "description": "MCP server for iOS Simulator and Android Emulator control",
   "license": "Apache-2.0",

package/rules/argent.md

@@ -36,8 +36,10 @@ If argent is ABSENT, treat it as an expected state, not an error to retry. Do no
 > </availability_check>
 
 <tapping_rule>
-<important>**Never** derive tap coordinates from a screenshot</important>
-Before **every** tap, you MUST call a discovery tool and extract coordinates from the result. This is not optional. Preferred tools are, in order:
+<important>**Never** derive tap coordinates from screenshot pixels</important>
+Interaction tools (`gesture-tap`, `gesture-swipe`, `keyboard`, `button`, `launch-app`, `open-url`, `run-sequence`, …) return the screen **after** the action: a screenshot AND the accessibility element tree (`--- Elements after action (describe) ---`, the same format `describe` prints, with normalized 0–1 frames). Take tap coordinates from that tree — the centre of the element's frame (`x + width/2`, `y + height/2`). You do not need a separate discovery call before a tap when the last result already lists the target.
+
+Call a discovery tool yourself only when you have no fresh tree for the current screen — before the first action on a screen you have not touched yet, after waiting for something to load, or when the last tree did not list your target. Preferred tools are, in order:
 
 - `describe` - native app-level components and safely targetable foreground apps (iOS and Android).
 - `native-describe-screen` - accessibility screen description via injected native devtools (iOS only)
@@ -45,9 +47,7 @@ Before **every** tap, you MUST call a discovery tool and extract coordinates fro
 
 `native-user-interactable-view-at-point` / `native-view-at-point` are follow-up diagnostics once you already have a candidate point (iOS only).
 
-Whenever something changed YOU MUST first call `describe`, or another appropriate discovery tool so you do not hallucinate element positions. Do not guess coordinates if you can use discovery tool. Do not tap if you have not called a discovery tool in the current step. Screenshots alone are never sufficient for coordinates.
-
-If a **tap fails twice** at the same coordinates, **stop retrying**. Re-run the discovery tool.
+If a **tap fails twice** at the same coordinates, **stop retrying** and re-run a discovery tool.
 
 If `describe` fails, **read the exact error before reacting**, follow the recovery guidance in `argent-device-interact` to choose the correct next action.
 
@@ -73,8 +73,8 @@ Decision order:
 - All simulator/emulator interactions go through argent MCP tools — never use `xcrun simctl`,
   raw `curl` to simulator ports, or the simulator-server binary directly.
 - Before calling any gesture tool for the first time, use ToolSearch to load its schema.
-- Interaction tools (`gesture-tap`, `gesture-swipe`, `gesture-pinch`, `gesture-rotate`, `gesture-custom`, `launch-app`, etc.) return a screenshot automatically.
-  Call `screenshot` separately only for a baseline before any action or after a delay.
+- Interaction tools (`gesture-tap`, `gesture-swipe`, `gesture-pinch`, `gesture-rotate`, `gesture-custom`, `launch-app`, etc.) return a screenshot and the element tree automatically.
+  Call `screenshot` or `describe` separately only for a baseline before any action or after a delay.
 - Always open apps with `launch-app` or `open-url` — never tap home screen icons.
 - If a task can require a saved flow, choose `argent-create-flow` or `argent-qa-flows` before the first launch or in-app action. Start the recorder before walking the path; recording is not retroactive.
 - Always use `run-sequence` when performing multiple sequential device actions where you don't need to observe the screen between steps. More in `argent-device-interact` skill.

package/skills/argent-device-interact/SKILL.md

@@ -32,7 +32,7 @@ Use `list-devices` to get a target id. Results are tagged with `platform` (`ios`
 3. **Use `gesture-swipe` for lists/scrolling**, not `gesture-custom`, unless you need non-linear movement. On Chromium use `gesture-scroll` instead — `gesture-swipe` is touch-only. Consider whether you need multiple swipes, if yes - use `run-sequence`. Pass `momentum: false` when the swipe should decelerate before ending for a precise movement.
 4. **Tap a text field before typing**, then use `keyboard` to enter text.
 5. **Coordinates are normalized** — always 0.0–1.0, not pixels.
-6. **For app navigation, prefer `describe` first.** It works on any screen without app restart. Do not navigate from screenshots on regular in-app screens unless `describe` failed to expose a reliable target. Use `native-describe-screen` only when you need app-scoped UIKit properties.
+6. **For app navigation, use the element tree returned after each action** (`--- Elements after action (describe) ---`); call `describe` only when no fresh tree is available for the current screen. It works on any screen without app restart. Do not navigate from screenshot pixels on regular in-app screens unless the tree failed to expose a reliable target. Use `native-describe-screen` only when you need app-scoped UIKit properties.
 
 ## 3. Opening Apps