@mobileaidev/ai-app-bridge 0.3.5 → 0.3.7

package/bin/executors/playwright-worker.js

@@ -0,0 +1,307 @@
+'use strict';
+
+const fs = require('node:fs');
+const path = require('node:path');
+const { randomUUID, createHash } = require('node:crypto');
+const { createRequire } = require('node:module');
+const { ReceiptJournal } = require('./receipt-journal');
+const { atomicJson } = require('./managed-runtime');
+
+const protocol = 'aab.playwright-worker/v1';
+const packageDirectory = process.argv[2];
+const sessionDirectory = process.argv[3];
+const identity = { sessionId: process.argv[4], runtimeEpoch: process.argv[5] };
+const playwright = createRequire(path.join(packageDirectory, 'package.json'))('playwright');
+const pages = new Map();
+const frames = new Map();
+const journals = new Map();
+const events = [];
+let eventSequence = 0;
+let eventBytes = 0;
+let browser;
+let browserContext;
+let closing = false;
+let active;
+let tail = Promise.resolve();
+const requests = new Map();
+
+function failure(code, message, details) { return Object.assign(new Error(message), { code, details }); }
+function emitEvent(type, data) {
+  const event = { sequence: ++eventSequence, observedAtMs: Date.now(), type, ...data };
+  const bytes = Buffer.byteLength(JSON.stringify(event));
+  events.push({ event, bytes }); eventBytes += bytes;
+  while (events.length > 1000 || eventBytes > 1024 * 1024) eventBytes -= events.shift().bytes;
+}
+function bindFrame(frame, targetId) {
+  let entry = frames.get(frame);
+  if (!entry) { entry = { frameId: randomUUID(), documentId: randomUUID(), targetId, frame }; frames.set(frame, entry); }
+  return entry;
+}
+function bindPage(page) {
+  if (pages.size >= 64) {
+    emitEvent('page-capacity', { maximum: 64 });
+    void page.close().catch(error => emitEvent('page-close-error', { message: error.message }));
+    return null;
+  }
+  const targetId = randomUUID();
+  pages.set(targetId, page);
+  for (const frame of page.frames()) bindFrame(frame, targetId);
+  page.on('frameattached', frame => bindFrame(frame, targetId));
+  page.on('framenavigated', frame => { bindFrame(frame, targetId).documentId = randomUUID(); });
+  page.on('framedetached', frame => frames.delete(frame));
+  page.on('close', () => { pages.delete(targetId); emitEvent('page-closed', { targetId }); for (const [frame, entry] of frames) if (entry.targetId === targetId) frames.delete(frame); });
+  page.on('console', message => emitEvent('console', { targetId, level: message.type(), text: message.text().slice(0, 8192) }));
+  page.on('pageerror', error => emitEvent('page-error', { targetId, message: error.message.slice(0, 8192) }));
+  page.on('requestfailed', request => emitEvent('request-failed', { targetId, url: request.url().slice(0, 8192), method: request.method(), failure: request.failure()?.errorText }));
+  page.on('response', response => emitEvent('response', { targetId, url: response.url().slice(0, 8192), status: response.status() }));
+  page.on('dialog', async dialog => {
+    const policy = active?.args.targetId === targetId ? active.args.dialog : null;
+    emitEvent('dialog', { targetId, dialogType: dialog.type(), message: dialog.message().slice(0, 8192), disposition: policy?.action || 'dismiss' });
+    try { if (policy?.action === 'accept') await dialog.accept(policy.promptText); else await dialog.dismiss(); }
+    catch (error) { emitEvent('dialog-error', { targetId, message: error.message.slice(0, 8192) }); }
+  });
+  emitEvent('page-opened', { targetId });
+  return targetId;
+}
+function pageFor(args) {
+  if (args.sessionId !== identity.sessionId || args.runtimeEpoch !== identity.runtimeEpoch) throw failure('executor_session_mismatch', 'Executor session identity changed.');
+  const page = pages.get(args.targetId);
+  if (!page || page.isClosed()) throw failure('executor_target_closed', 'The selected browser page is not open.');
+  return page;
+}
+function documentFor(args) {
+  pageFor(args);
+  const entry = [...frames.values()].find(value => value.frameId === args.frameId && value.targetId === args.targetId);
+  if (!entry || entry.frame.isDetached() || entry.documentId !== args.documentId) throw failure('reobserve_required', 'The selected frame/document changed; observe again.');
+  return entry;
+}
+async function locate(frame, selector, documentId) {
+  // The isolated selector world remembers the document bound by observe().
+  // Locator retries stay inside it; a new document can never acquire an old ID.
+  const root = frame.locator(`aabdocument=match:${documentId}`);
+  switch (selector.by) {
+    case 'role': return root.getByRole(selector.value, { name: selector.name, exact: true });
+    case 'testId': return root.getByTestId(selector.value);
+    case 'text': return root.getByText(selector.value, { exact: true });
+    case 'label': return root.getByLabel(selector.value, { exact: true });
+    case 'placeholder': return root.getByPlaceholder(selector.value, { exact: true });
+    case 'css': {
+      // Accept CSS syntax only. Playwright's selector chains could otherwise
+      // escape the observed document through an internal selector engine.
+      const valid = await frame.evaluate(value => {
+        try { document.querySelectorAll(value); return true; } catch { return false; }
+      }, selector.value);
+      if (!valid) throw failure('executor_selector_invalid', 'The css selector must use standard CSS syntax.');
+      return root.locator(`css=${selector.value}`);
+    }
+    default: throw failure('executor_selector_unsupported', 'Unknown selector kind.');
+  }
+}
+async function unique(frame, selector, documentId) {
+  const locator = await locate(frame, selector, documentId);
+  const count = await locator.count();
+  if (count !== 1) throw failure(count === 0 ? 'target_not_found' : 'ambiguous_target', 'Executor selectors must match exactly one element.', { matches: count });
+  return locator;
+}
+function assertDocument(entry, documentId) {
+  if (entry.documentId !== documentId || entry.frame.isDetached()) throw failure('reobserve_required', 'The selected document changed during preparation.');
+}
+function journalFor(targetId) {
+  let journal = journals.get(targetId);
+  if (!journal) { journal = new ReceiptJournal(path.join(sessionDirectory, 'receipts'), { ...identity, targetId }); journals.set(targetId, journal); }
+  return journal;
+}
+
+async function observe(args, options) {
+  const page = pageFor(args);
+  const documents = [];
+  const pageFrames = page.frames();
+  if (pageFrames.length > 64) throw failure('executor_frame_limit', 'This document exceeds the 64-frame observation limit.');
+  for (const frame of pageFrames) {
+    const entry = bindFrame(frame, args.targetId);
+    const documentId = entry.documentId;
+    await frame.locator(`aabdocument=bind:${documentId}`).count();
+    const snapshot = await frame.locator('body').ariaSnapshotJSON({ ...options, depth: 30 });
+    const controls = await frame.locator('input,textarea,select,button,a[href],[contenteditable="true"],[role],[data-testid]').evaluateAll((elements, limit) => elements.slice(0, limit).map(element => ({
+      tag: element.tagName.toLowerCase(), role: element.getAttribute('role'), testId: element.getAttribute('data-testid'),
+      id: element.id || null, text: typeof element.innerText === 'string' ? element.innerText.slice(0, 512) : null,
+      label: element.getAttribute('aria-label'), placeholder: element.getAttribute('placeholder'),
+      value: element.matches('input[type="password"]') ? null : ('value' in element ? String(element.value).slice(0, 4096) : null),
+      disabled: 'disabled' in element ? element.disabled : element.getAttribute('aria-disabled') === 'true',
+      visible: element.checkVisibility({ checkOpacity: true, checkVisibilityCSS: true }),
+    })), args.maxControls ?? 200);
+    assertDocument(entry, documentId);
+    documents.push({ frameId: entry.frameId, documentId, parentFrameId: frame.parentFrame() ? bindFrame(frame.parentFrame(), args.targetId).frameId : null,
+      url: frame.url(), name: frame.name(), snapshot, controls });
+  }
+  const result = { ok: true, ...identity, targetId: args.targetId, url: page.url(), title: await page.title(),
+    documents, pages: [...pages].filter(([, item]) => !item.isClosed()).map(([targetId, item]) => ({ targetId, url: item.url() })),
+    scope: 'browser-dom', engine: 'playwright', observedAtMs: Date.now() };
+  if (Buffer.byteLength(JSON.stringify(result)) > 4 * 1024 * 1024) throw failure('executor_observation_too_large', 'Browser observation exceeds 4 MiB.');
+  return result;
+}
+
+async function mutate(args, options) {
+  const actionId = args.actionId;
+  const journal = journalFor(args.targetId);
+  const begun = journal.begin(actionId, { operation: args.operation, targetId: args.targetId, frameId: args.frameId, documentId: args.documentId,
+    ...(args.action ? { action: args.action } : {}), ...(args.url ? { url: args.url } : {}), ...(args.dialog ? { dialog: args.dialog } : {}) });
+  if (!begun.fresh) {
+    if (begun.receipt.phase === 'completed') return { ...begun.receipt.result, replayed: true, executionReceipt: begun.receipt };
+    throw failure('executor_action_unresolved', 'This action was already started. Query its original receipt; do not replay it.');
+  }
+  let dispatched = false;
+  let result;
+  const startedAtMs = Date.now();
+  try {
+    const entry = documentFor(args);
+    if (args.operation === 'navigate') {
+      const page = pageFor(args);
+      if (entry.frame !== page.mainFrame()) throw failure('executor_navigation_scope', 'Navigation requires the observed main document.');
+      dispatched = true;
+      await page.goto(args.url, { ...options, waitUntil: 'domcontentloaded' });
+    } else {
+      const action = args.action;
+      const locator = await unique(entry.frame, action.selector, args.documentId);
+      const destination = action.type === 'drag' ? await unique(entry.frame, action.destination, args.documentId) : null;
+      assertDocument(entry, args.documentId);
+      if (options.signal.aborted) throw failure('executor_cancelled', 'Executor action was cancelled before dispatch.');
+      dispatched = true;
+      switch (action.type) {
+        case 'click': await locator.click(options); break;
+        case 'doubleClick': await locator.dblclick(options); break;
+        case 'hover': await locator.hover(options); break;
+        case 'fill': await locator.fill(action.text, options); break;
+        case 'type': await locator.pressSequentially(action.text, options); break;
+        case 'press': await locator.press(action.text, options); break;
+        case 'check': await locator.setChecked(action.checked, options); break;
+        case 'select': await locator.selectOption(action.values, options); break;
+        case 'scrollIntoView': await locator.scrollIntoViewIfNeeded(options); break;
+        case 'drag': await locator.dragTo(destination, options); break;
+        case 'upload': await locator.setInputFiles(action.files, options); break;
+        default: throw failure('executor_action_unsupported', 'Unknown browser action.');
+      }
+      if (action.type === 'fill') {
+        const value = await locator.evaluate(element => element.isContentEditable ? element.innerText : element.value);
+        if (value !== action.text) throw failure('executor_postcondition_failed', 'The editor did not retain the requested value.', { observedValue: value });
+      }
+    }
+    result = { ok: true, ...identity, targetId: args.targetId, actionId, dispatched: true, ambiguous: false,
+      mechanism: args.operation === 'navigate' ? 'browser-navigation' : `playwright-${args.action.type}` };
+  } catch (error) {
+    result = { ok: false, ...identity, targetId: args.targetId, actionId, error: error.code || 'executor_action_failed',
+      message: error.message, details: error.details, dispatched, ambiguous: dispatched };
+  }
+  result.timings = { executionMs: Date.now() - startedAtMs };
+  let receipt;
+  try { receipt = journal.finish(begun.receipt, result); }
+  catch (error) { error.dispatched = dispatched; error.ambiguous = dispatched; throw error; }
+  return { ...result, executionReceipt: receipt };
+}
+
+async function handle(args, controller) {
+  const options = { timeout: args.timeoutMs ?? 30000, signal: controller.signal };
+  if (args.operation === 'open') {
+    if (browser) throw failure('executor_already_open', 'Worker already owns a browser.');
+    await playwright.selectors.register('aabdocument', () => {
+      const identities = new WeakMap();
+      const query = (root, selector) => {
+        const document = root.nodeType === 9 ? root : root.ownerDocument;
+        const separator = selector.indexOf(':');
+        const operation = selector.slice(0, separator), id = selector.slice(separator + 1);
+        if (operation === 'bind') identities.set(document, id);
+        return identities.get(document) === id ? document.documentElement : null;
+      };
+      return { query, queryAll: (root, selector) => { const element = query(root, selector); return element ? [element] : []; } };
+    }, { contentScript: true });
+    browser = await playwright[args.browser || 'chromium'].launch({ headless: args.headless ?? true, timeout: options.timeout });
+    if (controller.signal.aborted) { await browser.close(); throw failure('executor_cancelled', 'Browser preparation was cancelled.'); }
+    browserContext = await browser.newContext({ viewport: args.viewport || { width: 1280, height: 900 } });
+    browserContext.on('page', bindPage);
+    const page = await browserContext.newPage();
+    const targetId = [...pages].find(([, item]) => item === page)[0];
+    await page.goto(args.url, { ...options, waitUntil: 'domcontentloaded' });
+    const descriptor = { schemaVersion: protocol, ...identity, targetId, engine: 'playwright', version: browser.version(),
+      browser: args.browser || 'chromium', openedAtMs: Date.now(), state: 'open' };
+    atomicJson(path.join(sessionDirectory, 'session.json'), descriptor);
+    return { ok: true, ...descriptor, capabilities: ['observe', 'act', 'navigate', 'wait', 'screenshot', 'events', 'receipt', 'close'],
+      limitations: ['closed-shadow-root', 'native-operating-system-ui'], dialogDefault: 'dismiss' };
+  }
+  if (args.operation === 'observe') return observe(args, options);
+  if (args.operation === 'act' || args.operation === 'navigate') return mutate(args, options);
+  if (args.operation === 'wait') {
+    const entry = documentFor(args), locator = await locate(entry.frame, args.selector, args.documentId);
+    await locator.waitFor({ ...options, state: args.state || 'visible' });
+    if (args.text !== undefined) {
+      const deadline = Date.now() + options.timeout;
+      while (await locator.textContent(options) !== args.text) {
+        assertDocument(entry, args.documentId);
+        if (controller.signal.aborted || Date.now() >= deadline) throw failure('executor_wait_timeout', 'Expected text was not observed.');
+        await new Promise(resolve => setTimeout(resolve, 25));
+      }
+    }
+    assertDocument(entry, args.documentId);
+    return { ok: true, ...identity, targetId: args.targetId, condition: 'matched', observedAtMs: Date.now() };
+  }
+  if (args.operation === 'screenshot') {
+    const page = pageFor(args);
+    const bytes = await page.screenshot({ ...options, fullPage: args.fullPage ?? false, type: 'png' });
+    fs.mkdirSync(path.dirname(args.outFile), { recursive: true });
+    fs.writeFileSync(args.outFile, bytes, { flag: 'wx', mode: 0o600 });
+    const sha256 = createHash('sha256').update(bytes).digest('hex');
+    return { ok: true, ...identity, targetId: args.targetId, outFile: args.outFile, artifact: { sha256, bytes: bytes.length },
+      refs: [{ stream: 'screenshot', screenshotId: args.outFile, sha256 }] };
+  }
+  if (args.operation === 'events') {
+    pageFor(args);
+    const after = args.afterSequence ?? 0;
+    const selected = events.filter(({ event }) => event.sequence > after).slice(0, args.limit ?? 100);
+    return { ok: true, ...identity, items: selected.map(({ event }) => event),
+      nextSequence: selected.at(-1)?.event.sequence ?? after, throughSequence: eventSequence,
+      gap: events.length > 0 && after < events[0].event.sequence - 1 };
+  }
+  if (args.operation === 'close') { pageFor(args); await shutdown(); return { ok: true, ...identity, closed: true }; }
+  throw failure('executor_operation_unsupported', 'Unknown worker operation.');
+}
+
+let shutdownPromise;
+function shutdown() {
+  if (shutdownPromise) return shutdownPromise;
+  closing = true;
+  active?.controller.abort();
+  shutdownPromise = (async () => {
+    await browser?.close();
+    atomicJson(path.join(sessionDirectory, 'closed.json'), { ...identity, closedAtMs: Date.now() });
+  })();
+  return shutdownPromise;
+}
+
+process.on('message', message => {
+  if (message?.protocol !== protocol) return;
+  if (message.type === 'cancel') { requests.get(message.id)?.controller.abort(); return; }
+  if (message.type !== 'request') return;
+  const controller = new AbortController();
+  const request = { id: message.id, controller, args: message.args };
+  requests.set(message.id, request);
+  const deadline = Date.now() + (message.args.timeoutMs ?? 30000);
+  const timer = setTimeout(() => controller.abort(), message.args.timeoutMs ?? 30000);
+  tail = tail.then(async () => {
+    active = request;
+    try {
+      if (closing) throw failure('executor_closed', 'Executor worker is closing.');
+      if (controller.signal.aborted || Date.now() >= deadline) throw failure('executor_cancelled', 'Executor request was cancelled while queued.');
+      const result = await handle({ ...message.args, timeoutMs: Math.max(1, deadline - Date.now()) }, controller);
+      if (process.connected) process.send({ protocol, id: message.id, result });
+      if (message.args.operation === 'close') process.disconnect();
+    } catch (error) {
+      const dispatched = error.dispatched ?? (message.args.operation === 'open' && Boolean(browser));
+      if (process.connected) process.send({ protocol, id: message.id, result: { ok: false, error: error.code || 'executor_failed', message: error.message,
+        details: error.details, dispatched, ambiguous: error.ambiguous ?? dispatched } });
+    } finally { clearTimeout(timer); requests.delete(message.id); active = null; }
+  }).catch(async error => {
+    if (process.connected) process.send({ protocol, id: message.id, result: { ok: false, error: error.code || 'executor_failed', message: error.message } });
+  });
+});
+process.on('disconnect', () => { shutdown().finally(() => process.exit(0)); });
+process.on('SIGTERM', () => { shutdown().finally(() => process.exit(0)); });
+process.send({ protocol, type: 'ready', ...identity });

package/bin/executors/receipt-journal.js

@@ -0,0 +1,56 @@
+'use strict';
+
+const fs = require('node:fs');
+const path = require('node:path');
+const { createHash } = require('node:crypto');
+const { CommandError } = require('../command-errors');
+const { atomicJson, readJson } = require('./managed-runtime');
+
+const hash = value => createHash('sha256').update(value).digest('hex');
+function canonical(value) {
+  if (Array.isArray(value)) return value.map(canonical);
+  if (value && typeof value === 'object') return Object.fromEntries(Object.keys(value).sort().map(key => [key, canonical(value[key])]));
+  return value;
+}
+
+class ReceiptJournal {
+  constructor(directory, identity, { maxActions = 4096 } = {}) {
+    this.directory = directory;
+    this.identity = identity;
+    this.maxActions = maxActions;
+    fs.mkdirSync(directory, { recursive: true, mode: 0o700 });
+  }
+
+  receipt(actionId) {
+    const value = readJson(path.join(this.directory, `${hash(actionId)}.json`));
+    if (value && (value.actionId !== actionId || JSON.stringify(value.identity) !== JSON.stringify(this.identity))) {
+      throw new CommandError('executor_receipt_identity_mismatch', 'Receipt does not belong to this executor session.');
+    }
+    return value;
+  }
+
+  begin(actionId, request) {
+    const digest = hash(JSON.stringify(canonical(request)));
+    const existing = this.receipt(actionId);
+    if (existing) {
+      if (existing.requestDigest !== digest) throw new CommandError('idempotency_conflict', 'actionId was already used with different executor arguments.');
+      return { fresh: false, receipt: existing };
+    }
+    if (fs.readdirSync(this.directory).filter(name => name.endsWith('.json')).length >= this.maxActions) {
+      throw new CommandError('executor_receipt_capacity', 'Close this session and open a new one; retained action receipts are full.');
+    }
+    const receipt = { schemaVersion: 'aab.executor-receipt/v1', identity: this.identity, actionId, requestDigest: digest,
+      phase: 'started', preparedAtMs: Date.now(), settled: false, dispatched: null };
+    atomicJson(path.join(this.directory, `${hash(actionId)}.json`), receipt);
+    return { fresh: true, receipt };
+  }
+
+  finish(receipt, result) {
+    const completed = { ...receipt, phase: 'completed', completedAtMs: Date.now(), settled: true,
+      dispatched: result.dispatched === true, result };
+    atomicJson(path.join(this.directory, `${hash(receipt.actionId)}.json`), completed);
+    return completed;
+  }
+}
+
+module.exports = { ReceiptJournal, canonical };

package/bin/feedback-probe.js

@@ -1,6 +1,31 @@
 'use strict';
 
-async function runWithFeedbackProbe({
+async function runWithFeedbackProbe(options = {}) {
+  const { command, args = {}, runner } = options;
+  if (!isFullFeedback(args.feedback) || !isMutationCommand(command)
+      || command === 'launch-app' || command === 'launch-activity') return runFeedback(options);
+  const observationCommand = String(command).startsWith('ios-') ? 'ios-ui-observation'
+    : String(command).startsWith('web-') ? 'web-ui-observation' : 'ui-observation';
+  const targetArgs = pick(args, ['serial', 'adb', 'packageName', 'port', 'deviceId', 'bundleId', 'runtimeUrl', 'iosHost', 'iosPort', 'sessionId', 'runtimeEpoch', 'targetId']);
+  if (observationCommand !== 'web-ui-observation') targetArgs.provider = String(command).includes('flutter') ? 'flutter' : 'native';
+  const started = await runner(observationCommand, { ...targetArgs, operation: 'start', durationMs: 5000 });
+  if (started?.ok !== true || started.active !== true || typeof started.leaseId !== 'string') {
+    return { result: { ok: false, error: 'ui_observation_unavailable', dispatched: false, ambiguous: false, details: started ?? null },
+      observation: { mode: 'full', inconclusive: true }, evidence: [] };
+  }
+  let output;
+  try {
+    output = await runFeedback(options);
+    return output;
+  } finally {
+    let cleanup;
+    try { cleanup = await runner(observationCommand, { ...targetArgs, operation: 'stop', leaseId: started.leaseId }); }
+    catch (error) { cleanup = { ok: false, error: error.code || 'ui_observation_cleanup_failed' }; }
+    if (output?.observation) output.observation.window = { leaseId: started.leaseId, maxDurationMs: 5000, cleanup };
+  }
+}
+
+async function runFeedback({
   command,
   args = {},
   runner,
@@ -130,6 +155,7 @@ function isFullFeedback(value) {
 function isMutationCommand(command) {
   const normalized = String(command || '').toLowerCase();
   const readOnly = new Set([
+    'ui-observation', 'ios-ui-observation', 'web-ui-observation',
     'status', 'tree', 'uia-tree', 'screenshot', 'logs', 'network', 'state', 'events',
     'logcat', 'keyboard-state', 'wait-text', 'flutter-tree', 'flutter-nodes', 'h5-dom',
     'flutter-h5-dom', 'webview-pages', 'webview-network', 'webview-console',

package/bin/ios-provider.js

@@ -95,6 +95,11 @@ class IOSBridgeProvider {
           return await this.launchApp(args, context);
         case 'ios-status':
           return await this.runtimeGet(args, '/v1/status');
+        case 'ios-ui-observation': {
+          const port = await this.runtimePort(args, context);
+          const result = await port.post(require('./ui-observation').path(args), require('./ui-observation').request(args));
+          return { ...result, endpoint: port.endpoint.baseUrl, device: port.endpoint.device, runtimeBinding: port.endpoint.runtimeBinding };
+        }
         case 'ios-tree':
           return await this.runtimeGet(args, '/v1/view/tree');
         case 'ios-logs':
@@ -468,7 +473,7 @@ class IOSBridgeProvider {
       const endpoint = port.endpoint;
       let response;
       if (method === 'POST') {
-        const status = await port.get('/v1/status');
+        const status = await port.get(endpointPath === '/v1/flutter/action' ? '/v1/flutter/snapshot' : '/v1/status');
         if (status.ok === false) return { ...status, dispatched: false, ambiguous: false };
         const kind = endpointPath === '/v1/h5/action' ? 'h5' : 'flutter';
         const target = { platform: 'ios', deviceId: endpoint.device.udid, bundleId: args.bundleId,
@@ -521,7 +526,7 @@ class IOSBridgeProvider {
   }
 
   async flutterTree(args = {}) {
-    const status = await this.runtimeGet(args, '/v1/status');
+    const status = await this.runtimeGet(args, '/v1/flutter/snapshot');
     if (status.ok === false) return status;
     return {
       ok: true,
@@ -552,7 +557,7 @@ class IOSBridgeProvider {
 
   async flutterControl(command, args, context = {}) {
     const port = await this.runtimePort(args, context);
-    const status = await port.get('/v1/status');
+    const status = await port.get('/v1/flutter/snapshot');
     if (status.ok !== true) return { ...status, dispatched: false, ambiguous: false };
     const action = flutterControls.get(command);
     let payload = { action };

package/bin/ios-runtime-binding.js

@@ -4,7 +4,7 @@ const { CommandError } = require('./command-errors');
 const { currentExecution } = require('./shared-kernel/execution-scope');
 
 const schemaVersion = 'aab.ios-runtime/v1';
-const sdkCommands = new Set(['ios-status', 'ios-tree', 'ios-logs', 'ios-network', 'ios-state', 'ios-events',
+const sdkCommands = new Set(['ios-ui-observation', 'ios-status', 'ios-tree', 'ios-logs', 'ios-network', 'ios-state', 'ios-events',
   'ios-h5-dom', 'ios-h5-eval', 'ios-h5-click', 'ios-h5-input', 'ios-h5-scroll', 'ios-flutter-tree', 'ios-flutter-nodes', 'ios-flutter-action',
   'ios-tap-flutter', 'ios-input-flutter-text', 'ios-scroll-flutter', 'ios-flutter-back', 'ios-flutter-hide-keyboard']);
 const fields = ['schemaVersion', 'bundleId', 'runtimeEpoch', 'processId', 'port'];

package/bin/runtime-directory.js

@@ -51,7 +51,7 @@ function runtimeLocation() {
 
 function runtimeIdentity(location = runtimeLocation()) {
   const names = ['AI_APP_BRIDGE_ADB_TIMEOUT_MS', 'AI_APP_BRIDGE_DEVICECTL', 'AI_APP_BRIDGE_FACT_CACHE',
-    'AI_APP_BRIDGE_IOS_TEAM_ID', 'AI_APP_BRIDGE_PYTHON', 'ANDROID_HOME', 'ANDROID_SDK_ROOT', 'DEVELOPMENT_TEAM', 'DEVELOPER_DIR', 'XCODEBUILD'];
+    'AI_APP_BRIDGE_IOS_TEAM_ID', 'AI_APP_BRIDGE_PYTHON', 'AI_APP_BRIDGE_EXECUTOR_HOME', 'ANDROID_HOME', 'ANDROID_SDK_ROOT', 'DEVELOPMENT_TEAM', 'DEVELOPER_DIR', 'XCODEBUILD'];
   const config = { facts: location.facts, profile: location.profile, ownership: canonicalPath(ownershipDirectory()),
     adb: executablePath(process.env.ADB || 'adb') ?? { unavailable: process.env.ADB || 'adb' },
     environment: Object.fromEntries(names.map(name => [name, process.env[name] ?? null])) };

package/bin/shared-kernel/device-ownership-recovery.js

@@ -19,6 +19,14 @@ async function deviceOwnership(args, { lease = getProcessDeviceMutationLease(),
   const recovered = await lease.reconcile(args.serial, async pending => {
     if (args.operation === 'cancel-install' && (pending.kind !== 'android-install' || pending.actionId !== args.actionId))
       return { settled: false, error: 'install_action_mismatch', actionId: pending.actionId };
+    if (pending.kind === 'android-test-executor') {
+      try { return await require('../executors/android-host').recoverAndroidExecutor(pending); }
+      catch (error) { return { settled: false, error: error.code || 'executor_completion_query_failed', message: error.message }; }
+    }
+    if (pending.kind === 'flutter-test-executor') {
+      try { return await require('../executors/flutter-host').recoverFlutterExecutor(pending); }
+      catch (error) { return { settled: false, error: error.code || 'executor_completion_query_failed', message: error.message }; }
+    }
     if (pending.kind === 'uia-node') {
       if (!uiaProtocol.validIdentity(pending) || pending.target.serial !== args.serial) return { settled: false, error: 'invalid_uia_execution_identity' };
       try {

package/bin/shared-kernel/native-target.js

@@ -2,9 +2,8 @@
 
 const { checkExecution } = require('./execution-scope');
 
-// WindowInspector retains other Activities during transitions, and focus can
-// lag the current Activity. Select its window group, including dialogs/popups
-// sharing its focus owner. Older trees keep their window-order semantics.
+// The SDK owns Android window topology and publishes its foreground decision.
+// Focus ownership is an input permission, not an Activity/window relationship.
 
 function visible(node) {
   return node && (node.effectiveVisible === true || node.visible === true)
@@ -21,12 +20,13 @@ function foregroundNativeWindow(rawTree) {
   if (!rawTree || typeof rawTree !== 'object') return null;
   const windows = Array.isArray(rawTree.windows) ? rawTree.windows : [];
   if (windows.length) {
-    const activity = windows.findLast(item => item?.activityDecor === true && Object.hasOwn(item, 'focusOwnerWindowId'));
-    const index = windows.findLastIndex(item => !explicitlyHidden(item?.root)
-      && (!activity || item === activity || !Object.hasOwn(item || {}, 'focusOwnerWindowId')
-        || item.focusOwnerWindowId !== null && item.focusOwnerWindowId === activity.focusOwnerWindowId));
-    if (index < 0) return null;
+    const id = rawTree.foregroundWindowId;
+    if (typeof id !== 'string' || !id.trim()) return null;
+    const matches = windows.map((window, index) => window?.windowId === id ? index : -1).filter(index => index >= 0);
+    if (matches.length !== 1) return null;
+    const [index] = matches;
     const window = windows[index];
+    if (explicitlyHidden(window?.root)) return null;
     return { index, root: window?.root, type: window?.type, windowId: window?.windowId,
       bounds: window && Object.hasOwn(window, 'bounds') ? window.bounds : window?.root?.bounds };
   }
@@ -45,6 +45,9 @@ function explicitlyHidden(node) {
 
 function selectNativeNode(rawTree, spec, editable) {
   const reject = (error) => ({ ok: false, error, dispatched: false });
+  if (rawTree?.windows?.length && (typeof rawTree.foregroundWindowId !== 'string' || !rawTree.foregroundWindowId.trim())) {
+    return reject('native_window_metadata_unavailable');
+  }
   const window = nativeWindow(rawTree);
   if (!window) return reject('visible_observed_window_required');
   const selector = spec.selector || (typeof spec.text === 'string' ? { text: spec.text } : null);

package/bin/shared-kernel/uia-runtime-port.js

@@ -205,6 +205,7 @@ function createUiaRuntimePort({ adb, serial, timeoutMs = 10000, root = protocol.
   }
 
   async function ensureLocked(rotate) {
+    await require('../executors/automation-owner').assertAvailable(serial);
     const asset = bundle();
     const previous = await readJson(`${root}/runtime.json`);
     let peer = previous === null ? null : descriptor(previous);
@@ -293,6 +294,40 @@ function createUiaRuntimePort({ adb, serial, timeoutMs = 10000, root = protocol.
   }
 
   return {
+    async withInstrumentation(descriptorFile, start) {
+      return withConnectionLock(async () => {
+        const automation = require('../executors/automation-owner');
+        await automation.assertAvailable(serial);
+        const value = await readJson(`${root}/runtime.json`);
+        if (value !== null) {
+          const peer = descriptor(value);
+          if (peer.running) {
+            const connection = await connect(peer);
+            const status = await statusOf(connection);
+            if (status.pending !== 0 || status.acknowledged !== status.count || status.activeActionId !== null)
+              throw failure('uia_runtime_pending_actions', 'Settle and acknowledge the original UIA actions before opening instrumentation.');
+            await stopRuntime(connection);
+          }
+        }
+        const claim = automation.claim(serial, descriptorFile);
+        try {
+          const result = await start();
+          if (result?.ok === false && result.dispatched === false && result.ambiguous === false) automation.release(serial, claim.sessionId);
+          return result;
+        } catch (error) {
+          if (error.dispatched === false && error.ambiguous === false) automation.release(serial, claim.sessionId);
+          throw error;
+        }
+      });
+    },
+    async releaseInstrumentation(sessionId) {
+      return withConnectionLock(async () => {
+        const automation = require('../executors/automation-owner');
+        const owner = automation.owner(serial);
+        if (owner && owner.sessionId !== sessionId) throw failure('executor_automation_owner_changed', 'Another test session owns UiAutomation.');
+        await automation.assertAvailable(serial);
+      });
+    },
     ensure, post,
     async observe() {
       const connection = await withConnectionLock(() => ensureLocked(true));
@@ -342,6 +377,7 @@ function createUiaRuntimePort({ adb, serial, timeoutMs = 10000, root = protocol.
       return withConnectionLock(async () => {
         const value = await readJson(`${root}/runtime.json`);
         if (operation === 'start') {
+          await require('../executors/automation-owner').assertAvailable(serial);
           const asset = bundle(), destination = await installAsset(asset);
           const ownerRaw = await shell(`CLASSPATH=${quote(destination)} app_process /system/bin ${mainClass} ${quote(root)} ${asset.manifest.sha256} owner-status`);
           let owner;

package/bin/ui-observation.js

@@ -0,0 +1,29 @@
+'use strict';
+
+const commands = new Set(['ui-observation', 'ios-ui-observation', 'web-ui-observation']);
+
+function schema(command, optionTypes) {
+  const target = command === 'web-ui-observation' ? ['sessionId', 'runtimeEpoch', 'targetId', 'timeoutMs']
+    : command === 'ios-ui-observation' ? ['deviceId', 'bundleId', 'runtimeUrl', 'iosHost', 'iosPort', 'timeoutMs', 'devicectl']
+      : ['serial', 'packageName', 'port', 'adb', 'timeoutMs'];
+  const properties = Object.fromEntries(target.map(name => [name, optionTypes[name]]));
+  Object.assign(properties, { operation: { enum: ['start', 'status', 'stop'] },
+    durationMs: { type: 'integer', minimum: 100, maximum: 5000 }, leaseId: { type: 'string', minLength: 1, maxLength: 256 } });
+  if (command !== 'web-ui-observation') properties.provider = { enum: ['native', 'flutter'], default: 'native' };
+  return { type: 'object', additionalProperties: false, properties,
+    required: ['operation', ...(command === 'web-ui-observation' ? ['sessionId', 'runtimeEpoch'] : command === 'ios-ui-observation' ? ['deviceId', 'bundleId'] : ['packageName'])],
+    oneOf: [
+      { properties: { operation: { const: 'start' }, leaseId: false }, required: ['durationMs'] },
+      { properties: { operation: { const: 'stop' }, durationMs: false }, required: ['leaseId'] },
+      { properties: { operation: { const: 'status' }, durationMs: false, leaseId: false } },
+    ] };
+}
+
+function request(args) {
+  return { operation: args.operation,
+    ...(args.operation === 'start' ? { durationMs: args.durationMs } : {}),
+    ...(args.operation === 'stop' ? { leaseId: args.leaseId } : {}) };
+}
+function path(args) { return args.provider === 'flutter' ? '/v1/flutter/observation' : '/v1/ui/observation'; }
+
+module.exports = { commands, schema, request, path };

package/bin/web-provider.js

@@ -43,6 +43,9 @@ class WebBridgeProvider {
           return { ok: true, session: this.sessionSummary(session), ownership: this.owner().status(sessionKey(args.sessionId)) };
         }
         case 'web-execution': return await this.executionControl(args);
+        case 'web-ui-observation': return await this.request(this.connected(args), 'read', {
+          name: 'uiObservation', args: require('./ui-observation').request(args),
+        }, args.timeoutMs ?? 5000);
         case 'web-dom': return args.history === true ? await this.captureResponse(args, 'dom') : await this.dom(args);
         case 'web-logs': return await this.captureResponse(args, 'logs');
         case 'web-network': return await this.captureResponse(args, 'network');
@@ -256,7 +259,9 @@ class WebBridgeProvider {
       this.pendingCommands.set(requestId, { socket, binding, type, payload, fail, resolve: result => end(null, result) });
       scope?.signal.addEventListener('abort', abort, { once: true });
       if (scope?.signal.aborted) { abort(); return; }
-      sent = true; markExecutionDispatched();
+      sent = true;
+      // Acquiring evidence must not mark the enclosing UI action as dispatched.
+      if (!(type === 'read' && payload.name === 'uiObservation')) markExecutionDispatched();
       try { socket.send(wire, error => { if (error && this.pendingCommands.has(requestId)) fail('web_command_transport_lost'); }); }
       catch { fail('web_command_transport_lost'); }
     });