@aarwitz/tapp 0.17.0-rc.11 → 0.17.0-rc.13

package/mcp-server/src/web-explorer.js

@@ -23,6 +23,7 @@ import { execFileSync } from "child_process";
 const CLICK_SETTLE_MS = 700;
 const NAV_TIMEOUT_MS = 15_000;
 const BUTTONS_PER_PAGE = 4;
+const WATCH_ACTION_DELAY_MS = 350;
 const ERROR_TEXT_RE = /\b(something went wrong|internal server error|an error occurred|failed to load|unhandled exception)\b/i;
 const STANDALONE_ERROR_TEXT_RE = /^(something went wrong|internal server error|an error occurred|failed to load|unhandled exception)(?:[.!:]|\s|$)/i;
 
@@ -249,13 +250,14 @@ export async function loadPlaywright() {
   );
 }
 
-export async function submitWebLogin(page) {
+export async function submitWebLogin(page, beforeClick = null) {
   const candidates = [
     page.locator("button[type=submit], input[type=submit], form button").first(),
     page.getByRole("button", { name: /sign ?in|log ?in|continue/i }).first(),
   ];
   for (const candidate of candidates) {
     if (await candidate.isVisible().catch(() => false)) {
+      if (beforeClick) await beforeClick(candidate);
       await candidate.click({ timeout: 3000 });
       return true;
     }
@@ -279,13 +281,14 @@ export function webTransitionOrigin(pendingNavigation, currentScreen) {
   return pendingNavigation?.fromScreen || currentScreen || null;
 }
 
-export function webBrowserLaunchOptions(environment = process.env) {
+export function webBrowserLaunchOptions(environment = process.env, { watch = false } = {}) {
   const browserProxy = String(environment.TAPP_BROWSER_PROXY_SERVER || "").trim();
   if (environment.TAPP_ENFORCE_PUBLIC_EGRESS === "1" && !/^http:\/\/127\.0\.0\.1:\d+$/.test(browserProxy)) {
     throw new Error("public egress policy proxy is required");
   }
   return {
-    headless: true,
+    headless: !watch,
+    ...(watch ? { slowMo: 200 } : {}),
     ...(browserProxy ? { proxy: { server: browserProxy, bypass: "<-loopback>" } } : {}),
     args: browserProxy ? [
       "--disable-quic",
@@ -296,6 +299,102 @@ export function webBrowserLaunchOptions(environment = process.env) {
   };
 }
 
+// A headed Playwright browser does not move the host OS pointer when locator.click() runs. In
+// explicit watch mode, draw a pointer inside the controlled page so a human can follow Tapp's
+// real actions. The UI lives in a closed shadow root, ignores pointer events, and is hidden from
+// evidence screenshots; it therefore cannot become an app control or alter detector input.
+async function installWebWatchUi(context) {
+  await context.addInitScript(() => {
+    const stateKey = Symbol.for("tapp.watchUi");
+    const ensure = () => {
+      if (window[stateKey]?.host?.isConnected) return window[stateKey];
+      const host = document.createElement("div");
+      host.setAttribute("data-tapp-watch-ui", "");
+      host.setAttribute("aria-hidden", "true");
+      Object.assign(host.style, {
+        position: "fixed",
+        inset: "0",
+        zIndex: "2147483647",
+        pointerEvents: "none",
+      });
+      const shadow = host.attachShadow({ mode: "closed" });
+      const style = document.createElement("style");
+      style.textContent = `
+        .cursor { position: fixed; left: 24px; top: 72px; width: 22px; height: 28px;
+          filter: drop-shadow(0 2px 2px rgba(0,0,0,.45)); transition: left 260ms ease, top 260ms ease;
+          transform: rotate(-8deg); }
+        .cursor::before { content: ""; display: block; width: 100%; height: 100%; background: #111827;
+          clip-path: polygon(0 0, 0 88%, 25% 67%, 39% 100%, 53% 93%, 39% 61%, 70% 61%); }
+        .cursor::after { content: ""; position: absolute; inset: 2px 3px 4px 2px; background: white;
+          clip-path: polygon(0 0, 0 79%, 25% 59%, 40% 91%, 46% 88%, 32% 55%, 60% 55%); }
+        .hud { position: fixed; top: 14px; right: 14px; max-width: min(420px, calc(100vw - 28px));
+          box-sizing: border-box; padding: 9px 12px; border-radius: 10px; color: white;
+          background: rgba(17,24,39,.92); box-shadow: 0 5px 18px rgba(0,0,0,.24);
+          font: 600 13px/1.35 -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; }
+        .brand { color: #93c5fd; margin-right: 6px; }
+      `;
+      const cursor = document.createElement("div");
+      cursor.className = "cursor";
+      const hud = document.createElement("div");
+      hud.className = "hud";
+      shadow.append(style, cursor, hud);
+      (document.documentElement || document).appendChild(host);
+      const state = { host, cursor, hud };
+      Object.defineProperty(window, stateKey, { value: state, configurable: true });
+      return state;
+    };
+    Object.defineProperty(window, "__tappShowWatchAction", {
+      configurable: true,
+      value: ({ x, y, action, target }) => {
+        const state = ensure();
+        state.host.style.display = "block";
+        if (Number.isFinite(x) && Number.isFinite(y)) {
+          state.cursor.style.left = `${Math.max(4, Math.min(window.innerWidth - 26, x))}px`;
+          state.cursor.style.top = `${Math.max(4, Math.min(window.innerHeight - 32, y))}px`;
+        }
+        state.hud.replaceChildren();
+        const brand = document.createElement("span");
+        brand.className = "brand";
+        brand.textContent = "Tapp";
+        state.hud.append(brand, document.createTextNode(`${action}${target ? ` · ${target}` : ""}`));
+      },
+    });
+    Object.defineProperty(window, "__tappSetWatchUiVisible", {
+      configurable: true,
+      value: (visible) => {
+        if (window[stateKey]?.host) window[stateKey].host.style.display = visible ? "block" : "none";
+      },
+    });
+  });
+}
+
+async function showWebWatchAction(page, { locator = null, action = "Exploring", target = "" } = {}) {
+  let x = 28;
+  let y = 76;
+  if (locator) {
+    await locator.scrollIntoViewIfNeeded().catch(() => {});
+    const box = await locator.boundingBox().catch(() => null);
+    if (box) {
+      x = box.x + box.width / 2;
+      y = box.y + box.height / 2;
+    }
+  }
+  await page.evaluate(({ x, y, action, target }) => {
+    window.__tappShowWatchAction?.({ x, y, action, target });
+  }, { x, y, action, target }).catch(() => {});
+  await page.waitForTimeout(WATCH_ACTION_DELAY_MS).catch(() => {});
+}
+
+async function screenshotWithoutWebWatchUi(page, options, watch) {
+  if (!watch) return page.screenshot(options);
+  await page.evaluate(() => window.__tappSetWatchUiVisible?.(false)).catch(() => {});
+  try {
+    return await page.screenshot(options);
+  } finally {
+    await page.evaluate(() => window.__tappSetWatchUiVisible?.(true)).catch(() => {});
+  }
+}
+
 // Focused one-screen inspection for the agent-facing `tapp open <url>` and `tapp tree <url>`
 // commands. This deliberately does no exploration or judgment; it opens exactly one page,
 // captures the visible semantic controls, and optionally takes one screenshot.
@@ -425,7 +524,7 @@ export function normalizeWebSeedTargets(seedTargets = [], limit = 5) {
   return result;
 }
 
-export async function exploreWeb({ url, maxActions = 40, timeoutSec = 300, outDir, testEmail = "", testPassword = "", seedRoutes = [], seedTargets = [], onProgress }) {
+export async function exploreWeb({ url, maxActions = 40, timeoutSec = 300, outDir, testEmail = "", testPassword = "", seedRoutes = [], seedTargets = [], watch = false, onProgress }) {
   const start = new URL(url);
   if (!/^https?:$/.test(start.protocol)) throw new Error("url must be http(s)");
   fs.mkdirSync(outDir, { recursive: true });
@@ -434,9 +533,10 @@ export async function exploreWeb({ url, maxActions = 40, timeoutSec = 300, outDi
   const emit = (kind, payload) => fs.writeSync(markersFd, `OCQA_${kind}:${JSON.stringify(payload)}\n`);
 
   const { chromium } = await loadPlaywright();
-  const browser = await chromium.launch(webBrowserLaunchOptions());
+  const browser = await chromium.launch(webBrowserLaunchOptions(process.env, { watch }));
   const context = await browser.newContext({ viewport: { width: 1280, height: 900 } });
   await installWebListenerTracking(context);
+  if (watch) await installWebWatchUi(context);
   const page = await context.newPage();
   page.setDefaultTimeout(NAV_TIMEOUT_MS);
 
@@ -604,7 +704,7 @@ export async function exploreWeb({ url, maxActions = 40, timeoutSec = 300, outDi
       if (existingKey && existingKey !== evidenceKey) screenshotFor.delete(existingKey);
       screenshotFor.set(evidenceKey, { path: screenshotPath, busy: info.busy, route: key });
       screenCount = screenshotFor.size;
-      await page.screenshot({ path: screenshotPath }).catch(() => {});
+      await screenshotWithoutWebWatchUi(page, { path: screenshotPath }, watch).catch(() => {});
       // Deterministic per-page detectors run once per distinct screen.
       if (webPageAppearsBlank(info)) issue("blank_screen", "high", "Page rendered no visible content", screen);
       else {
@@ -628,12 +728,19 @@ export async function exploreWeb({ url, maxActions = 40, timeoutSec = 300, outDi
     if (!(await pw.isVisible().catch(() => false))) return null;
     loginTried = true;
     const emailSel = "input[type=email], input[name*=mail i], input[name*=user i], input[id*=mail i], input[id*=user i]";
-    if (testEmail) await page.locator(emailSel).first().fill(testEmail).catch(() => {});
+    if (testEmail) {
+      const email = page.locator(emailSel).first();
+      if (watch) await showWebWatchAction(page, { locator: email, action: "Type", target: "Email" });
+      await email.fill(testEmail).catch(() => {});
+    }
+    if (watch) await showWebWatchAction(page, { locator: pw, action: "Type", target: "Password" });
     await pw.fill(testPassword).catch(() => {});
     lastActionTarget = "Sign in";
     emit("ACTION", { type: "login", target: "Sign in", screen, narrative: "Filled and submitted the sign-in form with the provided test credentials" });
     actions += 1;
-    await submitWebLogin(page).catch(() => false);
+    await submitWebLogin(page, watch
+      ? (locator) => showWebWatchAction(page, { locator, action: "Click", target: "Sign in" })
+      : null).catch(() => false);
     await waitForWebStability(page, { timeoutMs: Math.min(5_000, CLICK_SETTLE_MS * 6) });
     // Still on the login form after a submit = the sign-in failed — full stop. (A quiet
     // credential rejection often shows NO other symptom, so this must not be coupled to
@@ -688,6 +795,7 @@ export async function exploreWeb({ url, maxActions = 40, timeoutSec = 300, outDi
         progress();
         continue;
       }
+      if (watch) await showWebWatchAction(page, { action: "Open", target });
       await waitForWebStability(page);
       if (nav && typeof nav.status === "function" && nav.status() === 404) {
         issue("broken_link", "medium", `Broken link: ${target} → 404`, target);
@@ -708,6 +816,7 @@ export async function exploreWeb({ url, maxActions = 40, timeoutSec = 300, outDi
           emit("ACTION", { type: action.type, target: action.target, screen: beforeScreen, reason: "pr_ui_map_path", narrative: `Following observed UI Map path: ${action.type} ${action.target}` });
           let acted = false;
           if (action.type === "back") {
+            if (watch) await showWebWatchAction(page, { action: "Back", target: beforeScreen });
             await page.goBack({ waitUntil: "domcontentloaded", timeout: step.wait?.timeoutMs || NAV_TIMEOUT_MS }).catch(() => {});
             acted = true;
           } else {
@@ -721,6 +830,7 @@ export async function exploreWeb({ url, maxActions = 40, timeoutSec = 300, outDi
               else continue;
               if (await locator.isVisible().catch(() => false)) {
                 try {
+                  if (watch) await showWebWatchAction(page, { locator, action: "Click", target: action.target });
                   await locator.click({ timeout: Math.min(step.wait?.timeoutMs || NAV_TIMEOUT_MS, NAV_TIMEOUT_MS) });
                   acted = true;
                   break;
@@ -785,6 +895,7 @@ export async function exploreWeb({ url, maxActions = 40, timeoutSec = 300, outDi
         emit("ACTION", { type: "tap", target: label, screen: webActionScreen(ob), narrative: `Tapped "${label}"` });
         let clickSucceeded = false;
         try {
+          if (watch) await showWebWatchAction(page, { locator: b, action: "Click", target: label });
           await b.click({ timeout: 3000 });
           clickSucceeded = true;
         } catch {}
@@ -795,6 +906,7 @@ export async function exploreWeb({ url, maxActions = 40, timeoutSec = 300, outDi
         await waitForWebStability(page);
         if (page.url() !== beforeState.url) {
           await observe();
+          if (watch) await showWebWatchAction(page, { action: "Back", target: ob.screen });
           await page.goBack({ waitUntil: "domcontentloaded" }).catch(() => {});
           await waitForWebStability(page);
         } else {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aarwitz/tapp",
-  "version": "0.17.0-rc.11",
+  "version": "0.17.0-rc.13",
   "mcpName": "io.github.aarwitz/tapp",
   "description": "Release contracts, autonomous QA, and evidence-backed CI gates for iOS, Android, and web.",
   "license": "MIT",
@@ -41,6 +41,8 @@
     "Harness/OCQAHarnessUITests/",
     "Harness/OCQAHarness.xcodeproj/",
     "Harness/generate-harness-xcodeproj.rb",
+    "skills/",
+    ".claude-plugin/",
     "AGENTS.md"
   ],
   "dependencies": {
@@ -87,7 +89,7 @@
     "mobile"
   ],
   "scripts": {
-    "test": "node --test tests/report.test.js tests/regression.test.js tests/engine.test.js tests/project-config.test.js tests/application-model.test.js tests/ui-map.test.js tests/task-runtime.test.js tests/release-contract.test.js tests/pr-selection.test.js tests/flow-runtime.test.js tests/web-explorer.test.js tests/web-session.test.js tests/web-flow.test.js tests/scenario-runtime.test.js tests/android-driver.test.js tests/android-explorer.test.js tests/android-flow.test.js tests/android-primitives-protocol.test.js tests/managed-web.test.js tests/product-operations.test.js tests/browser-product.test.js tests/browser-onboarding.test.js tests/managed-operation.test.js tests/cloud-runner.test.js tests/ci-setup.test.js tests/ci-install.test.js tests/cli.test.js tests/action.test.js tests/package-surface.test.js tests/landing-brand.test.js tests/ci-gate.test.js tests/ci-report.test.js tests/desktop-protocol.test.js tests/ios-flow-protocol.test.js",
+    "test": "node --test tests/report.test.js tests/regression.test.js tests/engine.test.js tests/project-config.test.js tests/application-model.test.js tests/ui-map.test.js tests/task-runtime.test.js tests/release-contract.test.js tests/pr-selection.test.js tests/flow-runtime.test.js tests/web-explorer.test.js tests/web-session.test.js tests/web-flow.test.js tests/scenario-runtime.test.js tests/android-driver.test.js tests/android-explorer.test.js tests/android-flow.test.js tests/android-primitives-protocol.test.js tests/managed-web.test.js tests/product-operations.test.js tests/browser-product.test.js tests/browser-onboarding.test.js tests/managed-operation.test.js tests/cloud-runner.test.js tests/ci-setup.test.js tests/ci-install.test.js tests/cli.test.js tests/mcp-workspace.test.js tests/action.test.js tests/package-surface.test.js tests/agent-surface.test.js tests/landing-brand.test.js tests/ci-gate.test.js tests/ci-report.test.js tests/desktop-protocol.test.js tests/ios-flow-protocol.test.js vscode-extension/test/bridge.test.js",
     "test:browser-journey": "node --test tests/browser-journey.test.js",
     "test:browser-native": "TAPP_RUN_NATIVE_BROWSER=1 node --test tests/browser-native-journey.test.js"
   }

package/skills/tapp/SKILL.md

@@ -0,0 +1,74 @@
+---
+name: tapp
+description: Use Tapp to see, drive, explore, and verify real application surfaces on iOS simulators, Android emulators/devices, or the web. Use when a user asks an agent to test an app or UI change, find bugs, inspect or screenshot a screen, exercise a journey, create a replayable flow, gather release evidence, or run the deterministic Tapp gate. Also use when the user mentions Tapp, @aarwitz/tapp, tapp_* tools, .tapp artifacts, or asks whether agent-authored UI actually works.
+---
+
+# Tapp
+
+Use Tapp as the app's hands and eyes. Work on the real UI surface and show evidence; do not claim a
+screen or journey works from source inspection alone.
+
+## Choose the smallest operation
+
+| Intent | Operation |
+|---|---|
+| See or screenshot one screen | `open` / `tapp_open_app` |
+| Inspect controls on the current screen | `tree` / `tapp_ui_tree` |
+| Drive a specific journey | MCP session start → act → end |
+| Find bugs autonomously | `explore` / `tapp_explore` |
+| Preserve a journey | record and save a Flow; replay it deterministically |
+| Decide whether a merge passes policy | `ci`; exploration never decides this |
+
+Prefer connected `tapp_*` MCP tools when available: they keep interactive sessions alive and return
+screenshots inline. Otherwise run `npx -y @aarwitz/tapp@latest` from the app repository. Do not require MCP,
+an account, an API key, or a global install for the core workflow.
+
+## Start source-connected
+
+When the user asks for a general first test of a repository:
+
+1. If `.tapp/application-model.json` exists, run `npx -y @aarwitz/tapp@latest explore`.
+2. Otherwise run `npx -y @aarwitz/tapp@latest init . --explore` or call `tapp_init` with
+   `{operation:"explore", projectDir:"."}`.
+3. If Tapp returns `target-selection-required`, present its actual choices and ask the user to pick.
+   Never guess among multiple targets. Re-run with the selected platform/target exactly as Tapp
+   instructs.
+4. If a prerequisite is missing, run `tapp doctor`, apply only the stated remediation that is in
+   scope, and retry once.
+
+For a focused request, use the requested target directly rather than forcing repository onboarding.
+Targets may be a repository path, Xcode container, `.app`, iOS bundle id, APK plus Android app id,
+or owned HTTP(S) URL. Never explore a third-party web property without authorization: exploration
+clicks and types.
+
+## Observe honestly
+
+Exploration returns findings, coverage, evidence, and `inconclusive`; it does not return a score or
+ship verdict. Report:
+
+- target and platform;
+- screens/actions and whether coverage was conclusive;
+- deterministic versus advisory finding counts;
+- each important finding and its evidence/report path;
+- what Tapp explicitly did not check.
+
+If `inconclusive: true`, explain the blocker. A login wall or missing test data is not a pass. Ask for
+credentials or launch configuration instead of rerunning blindly. Do not infer content accuracy,
+privacy, brand consistency, or business guarantees from a generic crawl; those require a reviewed
+Flow, Scenario, contract, verifier, or human review.
+
+When a screenshot path is printed, open it with the client's image-reading tool before describing
+the screen. For web, use `--watch` when the human wants to follow Tapp's controlled browser. For iOS,
+point the human to the report's exploration recording when available.
+
+## Drive safely
+
+For an interactive MCP session, read returned `elements[]` before every action, target accessibility
+ids or visible labels, check `hittable`, tap a field before typing, and wait for navigation or async
+content. Use coordinates only as a last resort. End the session when finished.
+
+Do not edit the app merely because testing found a defect unless the user also asked for a fix. State
+what the evidence proves and what remains untested.
+
+Read [references/commands.md](references/commands.md) only when exact CLI/MCP syntax, Flow replay,
+credentials, or platform prerequisites are needed.

package/skills/tapp/agents/openai.yaml

@@ -0,0 +1,4 @@
+interface:
+  display_name: "Tapp"
+  short_description: "Test real iOS, Android, and web apps"
+  default_prompt: "Use $tapp to test this app on its real UI surface and report evidence-backed findings."

package/skills/tapp/references/commands.md

@@ -0,0 +1,102 @@
+# Tapp command reference
+
+## Plain CLI
+
+Run from the application repository. `[target]` is optional when Tapp can read the repository model
+or detect one unambiguous target.
+
+```bash
+npx -y @aarwitz/tapp@latest init . --explore
+npx -y @aarwitz/tapp@latest explore [target]
+npx -y @aarwitz/tapp@latest open [target]
+npx -y @aarwitz/tapp@latest tree [target] --json
+npx -y @aarwitz/tapp@latest shot
+npx -y @aarwitz/tapp@latest report latest
+npx -y @aarwitz/tapp@latest doctor
+```
+
+Platform examples:
+
+```bash
+# Web: Tapp may build/start/stop a source target; --watch is human-visible.
+npx -y @aarwitz/tapp@latest explore --platform web --target website --watch
+npx -y @aarwitz/tapp@latest explore https://staging.example.com
+
+# iOS: source repo, .app, or bundle id.
+npx -y @aarwitz/tapp@latest explore MyApp.xcodeproj --platform ios
+npx -y @aarwitz/tapp@latest open com.example.MyApp --platform ios
+
+# Android: app id is required; APK is optional if already installed.
+npx -y @aarwitz/tapp@latest explore app-debug.apk --platform android --app-id com.example.app
+```
+
+Focused web evidence can perform one semantic interaction and wait for async content:
+
+```bash
+npx -y @aarwitz/tapp@latest open https://example.com --tap "Not now" --wait-for "Dashboard"
+```
+
+## MCP mapping
+
+- `tapp_init`: inspect or initialize a source repository; `operation:"explore"` prepares and explores.
+- `tapp_build`: build and install an iOS app without needing its bundle id first.
+- `tapp_open_app`: launch and return a screen summary plus inline screenshot.
+- `tapp_ui_tree` / `tapp_screenshot`: inspect the current real surface.
+- `tapp_session_start` → `tapp_session_act` → `tapp_session_end`: drive one persistent journey.
+- `tapp_explore`: autonomous iOS, Android, or web exploration; observation only.
+- `tapp_flow_save` / `tapp_flow_run`: save a driven journey and replay it deterministically.
+- `tapp_release_contract`: validate, compile, or run a reviewed business guarantee.
+- `tapp_ci_setup`: create a target-scoped baseline or reviewable CI installation.
+
+An iOS no-bundle-id MCP path is `tapp_build {projectDir:"."}` followed by `tapp_explore` with the
+returned `bundleId`. Android uses `androidAppId` and optional `apkPath`; web uses `url` or
+source-connected `tapp_init`.
+
+## Interactive session loop
+
+```text
+tapp_session_start {appBundleId:"com.example.app"}
+tapp_session_act   {action:"tap", id:"Email"}
+tapp_session_act   {action:"type", text:"qa@example.com"}
+tapp_session_act   {action:"tap", id:"Password"}
+tapp_session_act   {action:"type", text:"..."}
+tapp_session_act   {action:"tap", id:"Sign In"}
+tapp_session_act   {action:"wait", text:"Home", timeoutMs:10000}
+tapp_screenshot
+tapp_session_end
+```
+
+Android sessions use `androidAppId`, optional `apkPath`, and the same action loop.
+
+## Credentials and test configuration
+
+For autonomous exploration, pass test-only values when authorized:
+
+- CLI: `--email`, `--password`, repeated `--launch-arg`, and JSON `--launch-env`.
+- MCP: `testEmail`, `testPassword`, `inputOverrides`, `appLaunchArgs`, `appLaunchEnv`, or explicit
+  `loginSteps`.
+
+Do not persist secrets in `.tapp/`. If the result reports input fields and no values were supplied,
+ask the user rather than pretending the explored surface was complete.
+
+## Replay and gating
+
+Flow YAML belongs under `.tapp/flows/` and can replay without a model or API key:
+
+```bash
+npx -y @aarwitz/tapp@latest flow run .tapp/flows/smoke.yml
+npx -y @aarwitz/tapp@latest ci
+```
+
+`tapp explore` observes. `tapp ci` applies versioned deterministic policy to evidence, selected
+Flows/Scenarios/contracts, coverage, and any target-scoped baseline. Its outcomes are `pass`, `fail`,
+or `inconclusive`; both `fail` and `inconclusive` block a merge.
+
+## Platform prerequisites
+
+- iOS: macOS, Xcode, and a booted simulator. First use builds a cached harness under `~/.tapp`.
+- Android: `adb` and a connected authorized emulator/device.
+- Web: Playwright and Chromium. If Tapp reports the browser missing, run
+  `npx playwright install chromium` and retry.
+
+Captures and screenshots are stored under `~/.tapp/captures/` and `~/.tapp/shots/`.