@mobileaidev/ai-app-bridge 0.2.8 → 0.2.10

package/README.md

@@ -66,13 +66,15 @@ steps as skipped without mixing results from different commands:
 }
 ```
 
-For dynamic screens, MCP agents should follow the `ai-app-bridge-use` cadence:
-thaw immediately before reading app content or performing an app action, freeze
-immediately after evidence or action results are captured, keep the app frozen
-while reasoning/planning, and thaw once more before finishing the overall task.
-For visible state changes such as panels, dialogs, page transitions, tabs, or
-button-triggered content, verify with both `screenshot` and `tree`/`uia-tree`;
-do not conclude success from UI tree alone.
+For dynamic or transient screens, MCP agents can use `freeze-app`/`thaw-app` as
+an optional stabilization control: thaw before reads, actions, waits, or
+captures; freeze after evidence capture only when a changing UI would make
+reasoning unreliable; and thaw before the next app operation or before
+finishing so the app is not left frozen. Static screens and ordinary form
+flows usually do not need freezing.
+For visible state changes such as panels, dialogs, page transitions, tabs, or
+button-triggered content, verify with both `screenshot` and `tree`/`uia-tree`;
+do not conclude success from UI tree alone.
 
 WebView network and console capture use Android WebView DevTools/CDP when the
 target app is debuggable and WebView debugging is enabled.

package/bin/mcp-server.js

@@ -1,12 +1,13 @@
 #!/usr/bin/env node
 
-const { spawn } = require('child_process');
-const path = require('path');
-
-const packageInfo = require('../package.json');
-const bridgeDir = __dirname;
-const cliScript = path.join(bridgeDir, 'ai-app-bridge.js');
-const nodeBinary = process.env.AI_APP_BRIDGE_NODE || process.execPath;
+const { spawn } = require('child_process');
+const path = require('path');
+
+const packageInfo = require('../package.json');
+const { WebBridgeProvider } = require('./web-provider');
+const bridgeDir = __dirname;
+const cliScript = path.join(bridgeDir, 'ai-app-bridge.js');
+const nodeBinary = process.env.AI_APP_BRIDGE_NODE || process.execPath;
 const supportedProtocolVersions = ['2025-06-18', '2024-11-05'];
 const defaultProtocolVersion = supportedProtocolVersions[0];
 const mcpSurface = (process.env.AI_APP_BRIDGE_MCP_SURFACE || 'compact').toLowerCase();
@@ -14,8 +15,9 @@ const serverInstructions = [
   'AI App Bridge observes and controls Android apps for agent workflows. Prefer these tools over raw adb when inspecting UI, text, WebView, logs, network, app install, data reset, launch, and permissions.',
   'Default surface is compact: call capabilities to discover domains, then call run with a command and arguments.',
   'Always pass packageName for app-specific commands, or pass an explicit port. Do not rely on a sample/default package in MCP sessions.',
-  'When freezing is available, keep the target app frozen while the model thinks or plans. Thaw only immediately before reading app content or performing an app action, freeze again as soon as that evidence/action result is captured, and thaw once more before finishing the overall task so the app is not left frozen.',
-].join(' ');
+  'Use freeze-app/thaw-app only as an optional stabilization control for dynamic or transient screens: thaw before reads/actions/captures, freeze after evidence capture only when it helps reasoning, and thaw before the next operation or before finishing so the app is not left frozen.',
+].join(' ');
+const webProvider = new WebBridgeProvider();
 
 let buffer = Buffer.alloc(0);
 let responseFormat = null;
@@ -257,10 +259,10 @@ function fullToolDefinitions() {
       deltaY: { type: 'number', description: 'Window scroll delta Y when no selector/text is supplied.' },
     }),
     bridgeTool('logs', 'Read generic in-app log records.'),
-    bridgeTool('freeze_app', 'Stop the target app processes with SIGSTOP after app content is captured, preventing playback or animation from changing the observed evidence.', {
+    bridgeTool('freeze_app', 'Optionally stop target app processes with SIGSTOP when a dynamic or transient screen needs stable evidence for review.', {
       pid: { type: 'string', description: 'Optional explicit process id. Defaults to all processes named packageName or packageName:*.' },
     }, ['packageName']),
-    bridgeTool('thaw_app', 'Resume the target app processes with SIGCONT before reading app content or dispatching actions, so bridge endpoints can answer.', {
+    bridgeTool('thaw_app', 'Resume target app processes with SIGCONT before reads, waits, captures, or actions, and before finishing any task that used freeze-app.', {
       pid: { type: 'string', description: 'Optional explicit process id. Defaults to all processes named packageName or packageName:*.' },
     }, ['packageName']),
     bridgeTool('logcat', 'Read Android logcat through ADB with optional pid/tag/level/grep filters.', {
@@ -458,8 +460,8 @@ const commandDefinitions = [
   { command: 'logcat', domain: 'diagnostics', summary: 'Read Android logcat with optional app pid, tag, level, and grep filters.', options: ['serial', 'packageName', 'pid', 'appPid', 'tag', 'level', 'grep', 'lines', 'since', 'follow', 'durationSec', 'clear'] },
   { command: 'install-apk', domain: 'app', summary: 'Install an APK and assist device-side installer confirmation screens.', options: ['serial', 'packageName', 'apkPath', 'allowDowngrade', 'streaming', 'installTimeoutMs', 'installerTimeoutMs', 'intervalMs'] },
   { command: 'clear-app-data', domain: 'app', summary: 'Clear target app local data through the bridge runtime.', targetApp: true, options: ['serial', 'packageName'] },
-  { command: 'freeze-app', domain: 'app', summary: 'Stop target app processes with SIGSTOP after content capture to keep observed evidence stable.', targetApp: true, options: ['serial', 'packageName', 'pid'] },
-  { command: 'thaw-app', domain: 'app', summary: 'Resume target app processes with SIGCONT before content capture or actions so bridge data can be read.', targetApp: true, options: ['serial', 'packageName', 'pid'] },
+  { command: 'freeze-app', domain: 'app', summary: 'Optionally stop target app processes with SIGSTOP when dynamic UI needs stable evidence.', targetApp: true, options: ['serial', 'packageName', 'pid'] },
+  { command: 'thaw-app', domain: 'app', summary: 'Resume target app processes with SIGCONT before reads, waits, captures, actions, or final handoff.', targetApp: true, options: ['serial', 'packageName', 'pid'] },
   { command: 'launch-app', domain: 'app', summary: 'Launch the target package LAUNCHER Activity and report launcher candidates.', targetApp: true, options: ['serial', 'packageName', 'activity', 'component', 'action', 'category', 'data', 'extra'] },
   { command: 'launch-activity', domain: 'app', summary: 'Launch an explicit Android Activity component with optional string extras.', targetApp: true, options: ['serial', 'packageName', 'activity', 'component', 'action', 'category', 'data', 'extra'] },
   { command: 'launch-native-test', domain: 'app', summary: 'Launch the debug native bridge test Activity.', targetApp: true, options: ['serial', 'packageName'] },
@@ -498,8 +500,23 @@ const commandDefinitions = [
   { command: 'flutter-h5-scroll', domain: 'webview', summary: 'Scroll Flutter H5 content or a DOM element into view.', targetApp: true, options: ['serial', 'packageName', 'selector', 'targetText', 'deltaX', 'deltaY'] },
   { command: 'webview-pages', domain: 'webview', summary: 'List attachable Android WebView DevTools/CDP pages.', targetApp: true, options: ['serial', 'packageName', 'webviewPort', 'socketName', 'targetId', 'pageUrlFilter', 'keepForward'] },
   { command: 'webview-network', domain: 'webview', summary: 'Capture WebView Network events through CDP.', targetApp: true, options: ['serial', 'packageName', 'webviewPort', 'socketName', 'targetId', 'pageUrlFilter', 'urlFilter', 'durationMs', 'script', 'includeResponseBody', 'bodyMaxBytes', 'maxEvents'] },
-  { command: 'webview-console', domain: 'webview', summary: 'Capture WebView console/log events through CDP.', targetApp: true, options: ['serial', 'packageName', 'webviewPort', 'socketName', 'targetId', 'pageUrlFilter', 'durationMs', 'script', 'maxEvents'] },
-  { command: 'forward', domain: 'advanced', summary: 'Create the ADB port forward for the bridge.', targetApp: true, options: ['serial', 'packageName', 'port'] },
+  { command: 'webview-console', domain: 'webview', summary: 'Capture WebView console/log events through CDP.', targetApp: true, options: ['serial', 'packageName', 'webviewPort', 'socketName', 'targetId', 'pageUrlFilter', 'durationMs', 'script', 'maxEvents'] },
+  { command: 'web-provider-status', domain: 'web', summary: 'Read desktop Web Bridge provider status.', options: [] },
+  { command: 'web-session-start', domain: 'web', summary: 'Start the desktop Web Bridge WebSocket session server.', options: ['host', 'webPort', 'path', 'token'] },
+  { command: 'web-connect-info', domain: 'web', summary: 'Read the Web Bridge endpoint and token for SDK clients.', options: [] },
+  { command: 'web-sessions', domain: 'web', summary: 'List connected Web Bridge SDK sessions.', options: [] },
+  { command: 'web-status', domain: 'web', summary: 'Read status and capture counts for a Web Bridge session.', targetKind: 'web-target', options: ['sessionId'] },
+  { command: 'web-dom', domain: 'web', summary: 'Read or refresh a Web Bridge DOM snapshot.', targetKind: 'web-target', options: ['sessionId', 'targetId', 'selector', 'refresh', 'timeoutMs'] },
+  { command: 'web-logs', domain: 'web', summary: 'Read Web Bridge log records.', targetKind: 'web-target', options: ['sessionId', 'sinceId', 'sinceMs', 'limit'] },
+  { command: 'web-network', domain: 'web', summary: 'Read Web Bridge network records.', targetKind: 'web-target', options: ['sessionId', 'sinceId', 'sinceMs', 'limit'] },
+  { command: 'web-state', domain: 'web', summary: 'Read Web Bridge state records.', targetKind: 'web-target', options: ['sessionId', 'sinceId', 'sinceMs', 'limit'] },
+  { command: 'web-events', domain: 'web', summary: 'Read Web Bridge event records.', targetKind: 'web-target', options: ['sessionId', 'sinceId', 'sinceMs', 'limit'] },
+  { command: 'web-command', domain: 'web', summary: 'Run a whitelisted command in a connected Web Bridge SDK session.', targetKind: 'web-target', options: ['sessionId', 'targetId', 'name', 'arguments', 'timeoutMs'] },
+  { command: 'web-click', domain: 'web', summary: 'Click a DOM element through the Web Bridge SDK command path.', targetKind: 'web-target', options: ['sessionId', 'targetId', 'selector', 'targetText', 'timeoutMs'] },
+  { command: 'web-input', domain: 'web', summary: 'Set text in a DOM input through the Web Bridge SDK command path.', targetKind: 'web-target', options: ['sessionId', 'targetId', 'selector', 'value', 'timeoutMs'] },
+  { command: 'web-wait', domain: 'web', summary: 'Wait for text or selector through the Web Bridge SDK command path.', targetKind: 'web-target', options: ['sessionId', 'targetId', 'selector', 'targetText', 'timeoutMs'] },
+  { command: 'web-scroll', domain: 'web', summary: 'Scroll a Web Bridge DOM target.', targetKind: 'web-target', options: ['sessionId', 'targetId', 'selector', 'deltaX', 'deltaY', 'timeoutMs'] },
+  { command: 'forward', domain: 'advanced', summary: 'Create the ADB port forward for the bridge.', targetApp: true, options: ['serial', 'packageName', 'port'] },
   { command: 'remove-forward', domain: 'advanced', summary: 'Remove the ADB port forward for the bridge.', options: ['serial', 'port'] },
   { command: 'batch', domain: 'advanced', summary: 'Run multiple AI App Bridge commands serially in one MCP call.', options: ['defaults', 'steps', 'stopOnError', 'includeRaw', 'maxRawChars'] },
   { command: 'smoke', domain: 'diagnostics', summary: 'Run the native sample smoke test.', options: ['serial', 'packageName', 'outFile', 'artifactDir', 'skipFlutterLaunch'] },
@@ -589,14 +606,15 @@ function capabilityPayload(args = {}) {
   };
 }
 
-function shapeCommandDefinition(definition, includeOptions) {
-  return {
-    command: definition.command,
-    summary: definition.summary,
-    targetApp: Boolean(definition.targetApp),
-    ...(includeOptions ? { options: definition.options || [] } : {}),
-  };
-}
+function shapeCommandDefinition(definition, includeOptions) {
+  return {
+    command: definition.command,
+    summary: definition.summary,
+    targetApp: Boolean(definition.targetApp),
+    targetKind: definition.targetKind || (definition.targetApp ? 'android-app' : 'none'),
+    ...(includeOptions ? { options: definition.options || [] } : {}),
+  };
+}
 
 async function runGeneric(args = {}) {
   const command = normalizeCommandName(args.command);
@@ -606,10 +624,10 @@ async function runGeneric(args = {}) {
   const commandArgs = {
     ...(args.arguments && typeof args.arguments === 'object' ? args.arguments : {}),
   };
-  for (const key of ['adb', 'serial', 'port', 'packageName']) {
-    if (args[key] !== undefined && commandArgs[key] === undefined) {
-      commandArgs[key] = args[key];
-    }
+  for (const key of ['adb', 'serial', 'port', 'packageName', 'sessionId', 'targetId', 'webPort']) {
+    if (args[key] !== undefined && commandArgs[key] === undefined) {
+      commandArgs[key] = args[key];
+    }
   }
   if (command === 'batch') {
     return runBatch(commandArgs);
@@ -621,19 +639,27 @@ function normalizeCommandName(value) {
   return String(value || '').trim().replace(/_/g, '-');
 }
 
-function runBridgeChecked(command, args = {}) {
-  if (command === 'clear-app-data' && !args.packageName) {
-    return toolText('clear-app-data: packageName is required in MCP mode so the command cannot clear a default package.', true);
-  }
+function runBridgeChecked(command, args = {}) {
+  const definition = commandByName.get(command);
+  if (definition?.domain === 'web') {
+    return runWebChecked(command, args);
+  }
+  if (command === 'clear-app-data' && !args.packageName) {
+    return toolText('clear-app-data: packageName is required in MCP mode so the command cannot clear a default package.', true);
+  }
   if ((command === 'freeze-app' || command === 'thaw-app') && !args.packageName) {
     return toolText(`${command}: packageName is required in MCP mode so the command cannot signal a default package.`, true);
   }
-  const definition = commandByName.get(command);
-  if (definition?.targetApp && !args.packageName && !args.port) {
-    return toolText(`${command}: packageName or explicit port is required in MCP mode so the command cannot fall back to a default package.`, true);
-  }
-  return runBridge(command, args);
-}
+  if (definition?.targetApp && !args.packageName && !args.port) {
+    return toolText(`${command}: packageName or explicit port is required in MCP mode so the command cannot fall back to a default package.`, true);
+  }
+  return runBridge(command, args);
+}
+
+async function runWebChecked(command, args = {}) {
+  const result = await webProvider.run(command, args);
+  return toolJson(result, result?.ok === false);
+}
 
 async function runBatch(args = {}, runner = runBridgeChecked) {
   const startedAtMs = Date.now();
@@ -654,10 +680,10 @@ async function runBatch(args = {}, runner = runBridgeChecked) {
   }
 
   const defaults = args.defaults && typeof args.defaults === 'object' ? { ...args.defaults } : {};
-  for (const key of ['adb', 'serial', 'port', 'packageName', 'artifactDir']) {
-    if (args[key] !== undefined && defaults[key] === undefined) {
-      defaults[key] = args[key];
-    }
+  for (const key of ['adb', 'serial', 'port', 'packageName', 'artifactDir', 'sessionId', 'targetId', 'webPort']) {
+    if (args[key] !== undefined && defaults[key] === undefined) {
+      defaults[key] = args[key];
+    }
   }
 
   const normalizedSteps = [];
@@ -706,10 +732,10 @@ async function runBatch(args = {}, runner = runBridgeChecked) {
       ...defaults,
       ...(step.arguments && typeof step.arguments === 'object' ? step.arguments : {}),
     };
-    for (const key of ['adb', 'serial', 'port', 'packageName']) {
-      if (step[key] !== undefined) {
-        stepArgs[key] = step[key];
-      }
+    for (const key of ['adb', 'serial', 'port', 'packageName', 'sessionId', 'targetId', 'webPort']) {
+      if (step[key] !== undefined) {
+        stepArgs[key] = step[key];
+      }
     }
     try {
       const toolResult = await runner(step.command, stepArgs);
@@ -717,11 +743,13 @@ async function runBatch(args = {}, runner = runBridgeChecked) {
       const passed = !parsed.isError && parsed.payload?.ok !== false;
       const stepResult = {
         id: step.id,
-        command: step.command,
-        status: passed ? 'passed' : 'failed',
-        ok: passed,
-        packageName: stepArgs.packageName,
-        port: stepArgs.port,
+        command: step.command,
+        status: passed ? 'passed' : 'failed',
+        ok: passed,
+        packageName: stepArgs.packageName,
+        sessionId: stepArgs.sessionId,
+        targetId: stepArgs.targetId,
+        port: stepArgs.port,
         durationMs: Date.now() - stepStartedAtMs,
         summary: summarizeToolPayload(parsed),
       };
@@ -801,8 +829,11 @@ function summarizeToolPayload(parsed) {
     ok: payload.ok,
     error: payload.error || null,
   };
-  if (payload.packageName) summary.packageName = payload.packageName;
-  if (payload.app?.packageName) summary.app = payload.app.packageName;
+  if (payload.packageName) summary.packageName = payload.packageName;
+  if (payload.sessionId) summary.sessionId = payload.sessionId;
+  if (payload.targetId) summary.targetId = payload.targetId;
+  if (payload.session?.sessionId) summary.sessionId = payload.session.sessionId;
+  if (payload.app?.packageName) summary.app = payload.app.packageName;
   if (payload.activity) summary.activity = payload.activity;
   if (payload.component) summary.component = payload.component;
   if (payload.transport) summary.transport = payload.transport;
@@ -814,8 +845,9 @@ function summarizeToolPayload(parsed) {
       port: payload.debugBridge.port,
     };
   }
-  if (payload.count !== undefined) summary.count = payload.count;
-  if (payload.nodeCount !== undefined) summary.nodeCount = payload.nodeCount;
+  if (payload.count !== undefined) summary.count = payload.count;
+  if (payload.sessionCount !== undefined) summary.sessionCount = payload.sessionCount;
+  if (payload.nodeCount !== undefined) summary.nodeCount = payload.nodeCount;
   if (Array.isArray(payload.items)) summary.items = payload.items.length;
   if (payload.values && typeof payload.values === 'object') {
     summary.values = Object.keys(payload.values).length;