agent-device 0.3.2 → 0.3.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-device",
-  "version": "0.3.2",
+  "version": "0.3.4",
   "description": "Unified control plane for physical and virtual devices via an agent-driven CLI.",
   "license": "MIT",
   "author": "Callstack",

package/src/daemon/__tests__/selectors.test.ts

@@ -74,6 +74,97 @@ test('resolveSelectorChain falls back when first selector is ambiguous', () => {
   assert.equal(resolved.node.ref, 'e2');
 });
 
+test('resolveSelectorChain keeps strict ambiguity behavior by default', () => {
+  const chain = parseSelectorChain('label="Continue"');
+  const resolved = resolveSelectorChain(nodes, chain, {
+    platform: 'ios',
+    requireRect: true,
+    requireUnique: true,
+  });
+  assert.equal(resolved, null);
+});
+
+test('resolveSelectorChain disambiguates to deeper/smaller matching node when enabled', () => {
+  const disambiguationNodes: SnapshotState['nodes'] = [
+    {
+      ref: 'e1',
+      index: 0,
+      type: 'Other',
+      label: 'Press me',
+      rect: { x: 0, y: 0, width: 300, height: 300 },
+      depth: 1,
+      enabled: true,
+      hittable: true,
+    },
+    {
+      ref: 'e2',
+      index: 1,
+      type: 'Other',
+      label: 'Press me',
+      rect: { x: 10, y: 10, width: 100, height: 20 },
+      depth: 2,
+      enabled: true,
+      hittable: true,
+    },
+  ];
+  const chain = parseSelectorChain('role="other" label="Press me" || label="Press me"');
+  const resolved = resolveSelectorChain(disambiguationNodes, chain, {
+    platform: 'ios',
+    requireRect: true,
+    requireUnique: true,
+    disambiguateAmbiguous: true,
+  });
+  assert.ok(resolved);
+  assert.equal(resolved.node.ref, 'e2');
+  assert.equal(resolved.matches, 2);
+});
+
+test('resolveSelectorChain disambiguation tie falls back to next selector', () => {
+  const tieNodes: SnapshotState['nodes'] = [
+    {
+      ref: 'e1',
+      index: 0,
+      type: 'Other',
+      label: 'Press me',
+      rect: { x: 0, y: 0, width: 100, height: 20 },
+      depth: 2,
+      enabled: true,
+      hittable: true,
+    },
+    {
+      ref: 'e2',
+      index: 1,
+      type: 'Other',
+      label: 'Press me',
+      rect: { x: 0, y: 40, width: 100, height: 20 },
+      depth: 2,
+      enabled: true,
+      hittable: true,
+    },
+    {
+      ref: 'e3',
+      index: 2,
+      type: 'Other',
+      label: 'Press me',
+      identifier: 'press_me_unique',
+      rect: { x: 0, y: 80, width: 100, height: 20 },
+      depth: 2,
+      enabled: true,
+      hittable: true,
+    },
+  ];
+  const chain = parseSelectorChain('label="Press me" || id="press_me_unique"');
+  const resolved = resolveSelectorChain(tieNodes, chain, {
+    platform: 'ios',
+    requireRect: true,
+    requireUnique: true,
+    disambiguateAmbiguous: true,
+  });
+  assert.ok(resolved);
+  assert.equal(resolved.selectorIndex, 1);
+  assert.equal(resolved.node.ref, 'e3');
+});
+
 test('findSelectorChainMatch returns first matching selector for existence checks', () => {
   const chain = parseSelectorChain('label="Continue" || id=auth_continue');
   const match = findSelectorChainMatch(nodes, chain, {
@@ -91,12 +182,31 @@ test('splitSelectorFromArgs extracts selector prefix and trailing value', () =>
   assert.deepEqual(split.rest, ['qa@example.com']);
 });
 
+test('splitSelectorFromArgs prefers trailing token for value when requested', () => {
+  const split = splitSelectorFromArgs(['label="Filter"', 'visible=true'], { preferTrailingValue: true });
+  assert.ok(split);
+  assert.equal(split.selectorExpression, 'label="Filter"');
+  assert.deepEqual(split.rest, ['visible=true']);
+});
+
+test('splitSelectorFromArgs keeps full selector when trailing value preference is disabled', () => {
+  const split = splitSelectorFromArgs(['label="Filter"', 'visible=true']);
+  assert.ok(split);
+  assert.equal(split.selectorExpression, 'label="Filter" visible=true');
+  assert.deepEqual(split.rest, []);
+});
+
 test('parseSelectorChain rejects unknown keys and malformed quotes', () => {
   assert.throws(() => parseSelectorChain('foo=bar'), /Unknown selector key/i);
   assert.throws(() => parseSelectorChain('label="unclosed'), /Unclosed quote/i);
   assert.throws(() => parseSelectorChain(''), /cannot be empty/i);
 });
 
+test('parseSelectorChain handles quoted values ending in escaped backslashes', () => {
+  const chain = parseSelectorChain('label="path\\\\" || id=auth_continue');
+  assert.equal(chain.selectors.length, 2);
+});
+
 test('isSelectorToken only accepts known keys for key=value tokens', () => {
   assert.equal(isSelectorToken('id=foo'), true);
   assert.equal(isSelectorToken('editable=true'), true);
@@ -126,3 +236,26 @@ test('buildSelectorChainForNode prefers id and adds editable for fill action', (
   assert.ok(chain.some((entry) => entry.includes('id=')));
   assert.ok(chain.some((entry) => entry.includes('editable=true')));
 });
+
+test('role selector normalization matches Android class names by leaf type', () => {
+  const androidNodes: SnapshotState['nodes'] = [
+    {
+      ref: 'a1',
+      index: 0,
+      type: 'android.widget.Button',
+      label: 'Continue',
+      identifier: 'auth_continue',
+      rect: { x: 0, y: 0, width: 120, height: 44 },
+      enabled: true,
+      hittable: true,
+    },
+  ];
+  const chain = parseSelectorChain('role=button label="Continue"');
+  const resolved = resolveSelectorChain(androidNodes, chain, {
+    platform: 'android',
+    requireRect: true,
+    requireUnique: true,
+  });
+  assert.ok(resolved);
+  assert.equal(resolved.node.ref, 'a1');
+});

package/src/daemon/handlers/__tests__/replay-heal.test.ts

@@ -234,6 +234,65 @@ test('replay without --update does not heal or rewrite', async () => {
   assert.equal(fs.readFileSync(replayPath, 'utf8'), originalPayload);
 });
 
+test('replay --update skips malformed selector candidates and preserves replay error context', async () => {
+  const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-replay-malformed-candidate-'));
+  const sessionsDir = path.join(tempRoot, 'sessions');
+  const replayPath = path.join(tempRoot, 'replay.ad');
+  const sessionStore = new SessionStore(sessionsDir);
+  const sessionName = 'malformed-candidate-session';
+  sessionStore.set(sessionName, makeSession(sessionName));
+
+  writeReplayFile(replayPath, {
+    ts: Date.now(),
+    command: 'click',
+    positionals: ['id="old_continue" ||'],
+    flags: {},
+    result: {},
+  });
+
+  const dispatch = async (): Promise<Record<string, unknown> | void> => {
+    return {
+      nodes: [
+        {
+          index: 0,
+          type: 'XCUIElementTypeButton',
+          label: 'Continue',
+          identifier: 'auth_continue',
+          rect: { x: 10, y: 10, width: 100, height: 44 },
+          enabled: true,
+          hittable: true,
+        },
+      ],
+      truncated: false,
+      backend: 'xctest',
+    };
+  };
+
+  const response = await handleSessionCommands({
+    req: {
+      token: 't',
+      session: sessionName,
+      command: 'replay',
+      positionals: [replayPath],
+      flags: { replayUpdate: true },
+    },
+    sessionName,
+    logPath: path.join(tempRoot, 'daemon.log'),
+    sessionStore,
+    invoke: async () => ({ ok: false, error: { code: 'COMMAND_FAILED', message: 'selector stale' } }),
+    dispatch,
+  });
+
+  assert.ok(response);
+  assert.equal(response.ok, false);
+  if (!response.ok) {
+    assert.equal(response.error.code, 'COMMAND_FAILED');
+    assert.match(response.error.message, /Replay failed at step 1/);
+    assert.equal(response.error.details?.step, 1);
+    assert.equal(response.error.details?.action, 'click');
+  }
+});
+
 test('replay --update heals selector in is command', async () => {
   const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-replay-heal-is-'));
   const sessionsDir = path.join(tempRoot, 'sessions');
@@ -306,6 +365,87 @@ test('replay --update heals selector in is command', async () => {
   assert.ok(rewrittenSelector.includes('auth_continue'));
 });
 
+test('replay --update heals numeric get text drift when numeric candidate value is unique', async () => {
+  const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-replay-heal-get-numeric-'));
+  const sessionsDir = path.join(tempRoot, 'sessions');
+  const replayPath = path.join(tempRoot, 'replay.ad');
+  const sessionStore = new SessionStore(sessionsDir);
+  const sessionName = 'heal-get-numeric-session';
+  sessionStore.set(sessionName, makeSession(sessionName));
+
+  writeReplayFile(replayPath, {
+    ts: Date.now(),
+    command: 'get',
+    positionals: ['text', 'role="statictext" label="2" || label="2"'],
+    flags: {},
+    result: {},
+  });
+
+  const invokeCalls: string[] = [];
+  const invoke = async (request: DaemonRequest): Promise<DaemonResponse> => {
+    if (request.command !== 'get') {
+      return { ok: false, error: { code: 'INVALID_ARGS', message: `unexpected command ${request.command}` } };
+    }
+    const selector = request.positionals?.[1] ?? '';
+    invokeCalls.push(selector);
+    if (selector.includes('label="2"')) {
+      return { ok: false, error: { code: 'COMMAND_FAILED', message: 'selector stale' } };
+    }
+    if (selector.includes('label="20"')) {
+      return { ok: true, data: { text: '20' } };
+    }
+    return { ok: false, error: { code: 'COMMAND_FAILED', message: 'unexpected selector' } };
+  };
+
+  const dispatch = async (): Promise<Record<string, unknown> | void> => {
+    return {
+      nodes: [
+        {
+          index: 0,
+          type: 'XCUIElementTypeStaticText',
+          label: '20',
+          rect: { x: 0, y: 100, width: 100, height: 24 },
+          enabled: true,
+          hittable: true,
+        },
+        {
+          index: 1,
+          type: 'XCUIElementTypeStaticText',
+          label: 'Version: 0.84.0',
+          rect: { x: 0, y: 200, width: 220, height: 17 },
+          enabled: true,
+          hittable: true,
+        },
+      ],
+      truncated: false,
+      backend: 'xctest',
+    };
+  };
+
+  const response = await handleSessionCommands({
+    req: {
+      token: 't',
+      session: sessionName,
+      command: 'replay',
+      positionals: [replayPath],
+      flags: { replayUpdate: true },
+    },
+    sessionName,
+    logPath: path.join(tempRoot, 'daemon.log'),
+    sessionStore,
+    invoke,
+    dispatch,
+  });
+
+  assert.ok(response);
+  assert.equal(response.ok, true, JSON.stringify(response));
+  if (response.ok) {
+    assert.equal(response.data?.healed, 1);
+    assert.equal(response.data?.replayed, 1);
+  }
+  assert.equal(invokeCalls.length, 2);
+});
+
 test('replay rejects legacy JSON payload files', async () => {
   const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-replay-json-rejected-'));
   const sessionsDir = path.join(tempRoot, 'sessions');

package/src/daemon/handlers/interaction.ts

@@ -12,6 +12,7 @@ import {
   formatSelectorFailure,
   parseSelectorChain,
   resolveSelectorChain,
+  splitIsSelectorArgs,
   splitSelectorFromArgs,
 } from '../selectors.ts';
 
@@ -90,6 +91,7 @@ export async function handleInteractionCommands(params: {
       platform: session.device.platform,
       requireRect: true,
       requireUnique: true,
+      disambiguateAmbiguous: true,
     });
     if (!resolved || !resolved.node.rect) {
       return {
@@ -180,7 +182,7 @@ export async function handleInteractionCommands(params: {
         error: { code: 'SESSION_NOT_FOUND', message: 'No active session. Run open first.' },
       };
     }
-    const selectorArgs = splitSelectorFromArgs(req.positionals ?? []);
+    const selectorArgs = splitSelectorFromArgs(req.positionals ?? [], { preferTrailingValue: true });
     if (selectorArgs) {
       if (selectorArgs.rest.length === 0) {
         return { ok: false, error: { code: 'INVALID_ARGS', message: 'fill requires text after selector' } };
@@ -197,6 +199,7 @@ export async function handleInteractionCommands(params: {
         platform: session.device.platform,
         requireRect: true,
         requireUnique: true,
+        disambiguateAmbiguous: true,
       });
       if (!resolved || !resolved.node.rect) {
         return {
@@ -307,6 +310,7 @@ export async function handleInteractionCommands(params: {
       platform: session.device.platform,
       requireRect: false,
       requireUnique: true,
+      disambiguateAmbiguous: sub === 'text',
     });
     if (!resolved) {
       return {
@@ -367,8 +371,7 @@ export async function handleInteractionCommands(params: {
         error: { code: 'UNSUPPORTED_OPERATION', message: 'is is not supported on this device' },
       };
     }
-    const selectorArgs = req.positionals.slice(1);
-    const split = splitSelectorFromArgs(selectorArgs);
+    const { split } = splitIsSelectorArgs(req.positionals);
     if (!split) {
       return {
         ok: false,

package/src/daemon/handlers/session.ts

@@ -10,8 +10,14 @@ import { ensureDeviceReady } from '../device-ready.ts';
 import { resolveIosAppStateFromSnapshots } from '../app-state.ts';
 import { stopIosRunnerSession } from '../../platforms/ios/runner-client.ts';
 import { attachRefs, type RawSnapshotNode, type SnapshotState } from '../../utils/snapshot.ts';
-import { pruneGroupNodes } from '../snapshot-processing.ts';
-import { buildSelectorChainForNode, parseSelectorChain, resolveSelectorChain, splitSelectorFromArgs } from '../selectors.ts';
+import { extractNodeText, normalizeType, pruneGroupNodes } from '../snapshot-processing.ts';
+import {
+  buildSelectorChainForNode,
+  resolveSelectorChain,
+  splitIsSelectorArgs,
+  splitSelectorFromArgs,
+  tryParseSelectorChain,
+} from '../selectors.ts';
 import { inferFillText, uniqueStrings } from '../action-utils.ts';
 
 type ReinstallOps = {
@@ -511,14 +517,20 @@ async function healReplayAction(params: {
   const session = sessionStore.get(sessionName);
   if (!session) return null;
   const requiresRect = action.command === 'click' || action.command === 'fill';
+  const allowDisambiguation =
+    action.command === 'click' ||
+    action.command === 'fill' ||
+    (action.command === 'get' && action.positionals?.[0] === 'text');
   const snapshot = await captureSnapshotForReplay(session, action, logPath, requiresRect, dispatch, sessionStore);
   const selectorCandidates = collectReplaySelectorCandidates(action);
   for (const candidate of selectorCandidates) {
-    const chain = parseSelectorChain(candidate);
+    const chain = tryParseSelectorChain(candidate);
+    if (!chain) continue;
     const resolved = resolveSelectorChain(snapshot.nodes, chain, {
       platform: session.device.platform,
       requireRect: requiresRect,
       requireUnique: true,
+      disambiguateAmbiguous: allowDisambiguation,
     });
     if (!resolved) continue;
     const selectorChain = buildSelectorChainForNode(resolved.node, session.device.platform, {
@@ -548,9 +560,8 @@ async function healReplayAction(params: {
       };
     }
     if (action.command === 'is') {
-      const predicate = action.positionals?.[0];
+      const { predicate, split } = splitIsSelectorArgs(action.positionals);
       if (!predicate) continue;
-      const split = splitSelectorFromArgs(action.positionals.slice(1));
       const expectedText = split?.rest.join(' ').trim() ?? '';
       const nextPositionals = [predicate, selectorExpression];
       if (predicate === 'text' && expectedText.length > 0) {
@@ -573,6 +584,10 @@ async function healReplayAction(params: {
       };
     }
   }
+  const numericDriftHeal = healNumericGetTextDrift(action, snapshot, session);
+  if (numericDriftHeal) {
+    return numericDriftHeal;
+  }
   return null;
 }
 
@@ -641,7 +656,7 @@ function collectReplaySelectorCandidates(action: SessionAction): string[] {
     }
   }
   if (action.command === 'is') {
-    const split = splitSelectorFromArgs(action.positionals.slice(1));
+    const { split } = splitIsSelectorArgs(action.positionals);
     if (split) {
       result.push(split.selectorExpression);
     }
@@ -690,6 +705,56 @@ function parseSelectorWaitPositionals(positionals: string[]): {
   };
 }
 
+function healNumericGetTextDrift(
+  action: SessionAction,
+  snapshot: SnapshotState,
+  session: SessionState,
+): SessionAction | null {
+  if (action.command !== 'get') return null;
+  if (action.positionals?.[0] !== 'text') return null;
+  const selectorExpression = action.positionals?.[1];
+  if (!selectorExpression) return null;
+  const chain = tryParseSelectorChain(selectorExpression);
+  if (!chain) return null;
+
+  const roleFilters = new Set<string>();
+  let hasNumericTerm = false;
+  for (const selector of chain.selectors) {
+    for (const term of selector.terms) {
+      if (term.key === 'role' && typeof term.value === 'string') {
+        roleFilters.add(normalizeType(term.value));
+      }
+      if (
+        (term.key === 'text' || term.key === 'label' || term.key === 'value') &&
+        typeof term.value === 'string' &&
+        /^\d+$/.test(term.value.trim())
+      ) {
+        hasNumericTerm = true;
+      }
+    }
+  }
+  if (!hasNumericTerm) return null;
+
+  const numericNodes = snapshot.nodes.filter((node) => {
+    const text = extractNodeText(node).trim();
+    if (!/^\d+$/.test(text)) return false;
+    if (roleFilters.size === 0) return true;
+    return roleFilters.has(normalizeType(node.type ?? ''));
+  });
+  if (numericNodes.length === 0) return null;
+  const numericValues = uniqueStrings(numericNodes.map((node) => extractNodeText(node).trim()));
+  if (numericValues.length !== 1) return null;
+
+  const targetNode = numericNodes[0];
+  if (!targetNode) return null;
+  const selectorChain = buildSelectorChainForNode(targetNode, session.device.platform, { action: 'get' });
+  if (selectorChain.length === 0) return null;
+  return {
+    ...action,
+    positionals: ['text', selectorChain.join(' || ')],
+  };
+}
+
 function parseReplayScript(script: string): SessionAction[] {
   const actions: SessionAction[] = [];
   const lines = script.split(/\r?\n/);