@aarwitz/tapp 0.16.4 → 0.17.0-rc.1

package/mcp-server/src/report.js

@@ -96,14 +96,41 @@ export const ISSUE_CATEGORY = {
   explore_timeout: "performance_timeout",
 };
 export const CRITICAL_ISSUE_TYPES = new Set(["crash"]);
+export const WEB_SAMPLED_ISSUE_TYPES = new Set(["unresponsive_element"]);
 
 export function severityRank(s) {
   return { critical: 0, high: 1, medium: 2, low: 3 }[s] ?? 4;
 }
 
-export function verdictBadge(report) {
-  if (report?.platform === "web" && report?.verdict === "ready") return "🟢 AUTOMATED CHECKS PASSED";
-  return { ready: "🟢 SHIP-READY", caution: "🟡 CAUTION", blocked: "🔴 BLOCKED" }[report?.verdict] || report?.verdict;
+export function findingEvaluationTier(finding, platform = "ios") {
+  return platform === "web" && WEB_SAMPLED_ISSUE_TYPES.has(finding?.type) ? "sampled" : "deterministic";
+}
+
+// The deterministic block-by-findings rule, expressed over verdict-tier finding COUNTS — not the
+// score scalar and not the `verdict` label — so the CI gate survives removal of verdict/releaseScore
+// from exploration output. It encodes exactly what `verdict === "blocked"` used to: a critical
+// finding always blocks; otherwise, on a CONCLUSIVE run, a risk threshold blocks. The risk threshold
+// is kept as an explicit, chosen rule (ADR-0005) — `riskFromCounts` is the single source of that
+// formula, shared with buildQaReport's verdict label. Inconclusive runs are a separate gate outcome,
+// never a findings-block, so a thin run reports `inconclusive`, not `fail`.
+export function riskFromCounts({ critical = 0, high = 0, medium = 0 } = {}) {
+  return Math.max(0, Math.min(100, 100 - critical * 25 - high * 10 - medium * 3));
+}
+export function findingsBlock(deterministicFindingCounts = {}, { inconclusive = false } = {}) {
+  if ((deterministicFindingCounts.critical || 0) > 0) return true;
+  if (inconclusive) return false;
+  return riskFromCounts(deterministicFindingCounts) < 50;
+}
+
+// Exploration OBSERVES; it never renders a ship verdict or score (ADR-0005). These label the
+// observation honestly. The release decision (pass/fail/inconclusive) is the gate's, shown separately.
+export function observationBadge(report) {
+  return report?.inconclusive ? "🟡 INCONCLUSIVE (exploration)" : "🔭 EXPLORED";
+}
+
+export function observationSummary(report) {
+  const n = report?.findingCounts?.total || 0;
+  return `${report?.screensExplored || 0} screens · ${report?.actionsPerformed || 0} actions · ${n} finding(s) · observation only`;
 }
 
 // Turn a capture's OCQA markers into the same ship/no-ship report Tapp produces:
@@ -186,7 +213,15 @@ export function buildQaReport(markersFilePath, { platform = "ios" } = {}) {
     const key = `${i.type}|${i.screen}|${i.target ?? ""}`;
     if (seen.has(key)) continue;
     seen.add(key);
-    findings.push({ ...i, category: ISSUE_CATEGORY[i.type] || i.type });
+    findings.push({
+      ...i,
+      category: ISSUE_CATEGORY[i.type] || i.type,
+      // Structural evidence authority (ADR-0005): marker-derived findings are deterministic. The
+      // default gate consumes only deterministic-authority evidence; model-observed findings
+      // (vision/assert_ai) carry authority:"model-observed" and are advisory, never gate fails.
+      authority: "deterministic",
+      ...(platform === "web" ? { evaluationTier: findingEvaluationTier(i, platform) } : {}),
+    });
   }
   findings.sort((a, b) => severityRank(a.severity) - severityRank(b.severity));
 
@@ -197,28 +232,27 @@ export function buildQaReport(markersFilePath, { platform = "ios" } = {}) {
   const high = findings.filter((f) => f.severity === "high").length;
   const med = findings.filter((f) => f.severity === "medium").length;
   const low = findings.filter((f) => f.severity === "low").length;
+  const verdictFindings = platform === "web"
+    ? findings.filter((finding) => finding.evaluationTier !== "sampled")
+    : findings;
+  const verdictCrit = verdictFindings.filter((f) => f.severity === "critical").length;
+  const verdictHigh = verdictFindings.filter((f) => f.severity === "high").length;
+  const verdictMed = verdictFindings.filter((f) => f.severity === "medium").length;
+  const verdictLow = verdictFindings.filter((f) => f.severity === "low").length;
+  const sampledFindings = platform === "web" ? findings.filter((finding) => finding.evaluationTier === "sampled") : [];
 
-  // Coverage floor: a verdict is only trustworthy if the app was actually exercised.
+  // Coverage floor: exploration is inconclusive if the app wasn't actually exercised. Exploration
+  // OBSERVES — it does not render a ship verdict or score (ADR-0005). Judgment (pass/fail/
+  // inconclusive) is the gate's job (evaluateGate), computed from these findings + coverage + policy.
   const inconclusive = screensExplored < 2 || actionsPerformed < 3;
-  let confidence = Math.max(0, Math.min(100, 100 - crit * 25 - high * 10 - med * 3));
-  if (inconclusive) confidence = Math.min(confidence, 40);
-
-  let verdict;
-  if (crit > 0) verdict = "blocked";
-  else if (inconclusive) verdict = "caution";
-  else if (confidence < 50) verdict = "blocked";
-  else if (high > 0 || confidence < 80) verdict = "caution";
-  else verdict = "ready";
 
   const headline = inconclusive
     ? `Inconclusive — only ${screensExplored} screen(s) / ${actionsPerformed} action(s) explored. The app may have crashed on launch, be stuck behind a sign-in wall, or otherwise prevent exploration. Absence of issues is NOT a pass.`
-    : verdict === "ready"
+    : findings.length === 0
     ? platform === "web"
-      ? "Automated web checks passed — no release-blocking technical issues found in the exercised surfaces. This is not a content, privacy, brand, or business-claim review."
-      : "Ship-ready — no release-blocking issues found."
-    : verdict === "caution"
-    ? `Proceed with caution — ${findings.length} issue(s) to review.`
-    : `Not ready — ${findings.length} issue(s): ${crit} critical, ${high} high, ${med} medium, ${low} low.`;
+      ? "Automated web checks completed — no deterministic findings in the exercised surfaces. Sampled control probes are advisory. An observation, not a release decision, and not a content, privacy, brand, or business-claim review."
+      : "No issues surfaced in the exercised surfaces. An observation, not a release decision."
+    : `${findings.length} issue(s) surfaced for review (${crit} critical, ${high} high, ${med} medium, ${low} low). An observation, not a release decision.`;
 
   // The verdict's own honesty label: exactly which defect classes this run checked, which
   // it structurally could NOT check, and which conditions never came up — so "checked" is
@@ -230,7 +264,7 @@ export function buildQaReport(markersFilePath, { platform = "ios" } = {}) {
   if (platform === "web") {
     checkedFor = [
       "page errors (uncaught exceptions)", "failed/5xx requests", "broken links (404)",
-      "placeholder links with no destination", "dead buttons", "error text on pages", "load timeouts",
+      "placeholder links with no destination", "sampled dead-button probes (advisory)", "error text on pages", "load timeouts",
     ];
     notChecked = [
       "app-specific business logic (cover with Flows: record or generate, then assert)",
@@ -271,14 +305,21 @@ export function buildQaReport(markersFilePath, { platform = "ios" } = {}) {
   else conditionsNotReached.push("sign-in (no login form encountered this run)");
 
   return {
-    verdict,
-    // `releaseScore` is the honest name: a heuristic quality score from fixed deductions,
-    // NOT calibrated statistical confidence. `confidence` is kept as an alias for
-    // compatibility (baselines, desktop app, existing consumers).
-    confidence,
-    releaseScore: confidence,
+    // An ExplorationRun observation: findings + coverage + evidence, NO ship verdict or score
+    // (ADR-0005). The gate (evaluateGate) turns this into a pass/fail/inconclusive release outcome.
+    kind: "tapp-exploration-run",
+    schemaVersion: 1,
+    // Complete ExplorationRun contract (ADR-0005 §4). runStatus/stopReason describe HOW the run
+    // ended; coverage/evidence/uiMap/comparison are the structured observation. uiMap and comparison
+    // are populated by consumers that build the map / diff a baseline (null in the bare observation).
+    runStatus: inconclusive ? "limited" : "completed",
+    stopReason: inconclusive ? "coverage-floor-not-met" : "completed",
     headline,
     inconclusive,
+    coverage: { screensExplored, actionsPerformed, screens: Array.from(screens) },
+    evidence: { markers: base.relativeMarkersFilePath },
+    uiMap: null,
+    comparison: null,
     checkedFor,
     notChecked,
     conditionsNotReached,
@@ -286,6 +327,20 @@ export function buildQaReport(markersFilePath, { platform = "ios" } = {}) {
     screensExplored,
     actionsPerformed,
     findingCounts: { critical: crit, high, medium: med, low, total: findings.length },
+    deterministicFindingCounts: {
+      critical: verdictCrit,
+      high: verdictHigh,
+      medium: verdictMed,
+      low: verdictLow,
+      total: verdictFindings.length,
+    },
+    sampledFindingCounts: {
+      critical: sampledFindings.filter((f) => f.severity === "critical").length,
+      high: sampledFindings.filter((f) => f.severity === "high").length,
+      medium: sampledFindings.filter((f) => f.severity === "medium").length,
+      low: sampledFindings.filter((f) => f.severity === "low").length,
+      total: sampledFindings.length,
+    },
     findings,
     screens: Array.from(screens),
     screenElementCounts,
@@ -314,6 +369,7 @@ export function computeContentCollapse(currentCounts, baselineCounts) {
         type: "content_collapse",
         severity: "high",
         category: "content_collapse",
+        authority: "deterministic",
         title: `Screen lost most of its content (${base} → ${cur} elements)`,
         screen,
         step: null,
@@ -342,6 +398,7 @@ export function computeReachabilityLoss(current, baseline) {
       type: "screen_unreachable",
       severity: "high",
       category: "navigation_dead_end",
+      authority: "deterministic",
       title: "Screen explored in the baseline was never reached this run",
       screen,
       step: null,
@@ -380,15 +437,84 @@ export function computeRegression(current, baseline) {
   const newFindings = current.filter((f) => !currentMatches(f));
   const persisting = current.filter((f) => currentMatches(f));
   const resolved = baseline.filter((b) => !baselineMatched(b));
-  const newCritical = newFindings.filter((f) => f.severity === "critical").length;
-  const newHigh = newFindings.filter((f) => f.severity === "high").length;
 
+  // Comparison ONLY — no gate/pass/fail signal (ADR-0005). Exploration surfaces this diff; the merge
+  // decision is the gate's job. evaluateGate derives its regression fail from `newFindings` severities.
   return {
     hadBaseline: true,
     counts: { new: newFindings.length, persisting: persisting.length, resolved: resolved.length },
     newFindings,
     resolved,
-    // CI gate: fail the build when this run introduced new high/critical findings vs. the baseline.
-    gate: { newCritical, newHigh, failed: newCritical + newHigh > 0 },
+  };
+}
+
+// The gate's public outcome model (ADR-0005). A merge gate is ultimately block / don't-block, but
+// callers need to distinguish WHY: a deterministic violation is not the same as "we couldn't get
+// the evidence." Exit codes are the CI contract; precedence is fail > inconclusive > pass.
+export const GATE_EXIT = { pass: 0, fail: 1, error: 2, inconclusive: 3 };
+// Bump when the gate's decision semantics change (NOT the npm version). Recorded on every GateRun.
+export const GATE_POLICY_VERSION = "1";
+
+// Pure gate evaluator: frozen evidence + policy → a GateRun decision. Extracted verbatim from the
+// former inline logic in ci-report.js so the `[char]` characterization tests keep passing — the
+// merge decision (block/don't-block) is unchanged; this only classifies each reason as a
+// deterministic `fail` or an evidence-absent `inconclusive` and folds them by precedence. Reason
+// MESSAGES are preserved exactly (several are asserted by tests).
+//
+// DECOUPLED FROM THE SCORE (ADR-0005): the block-by-findings decision reads `deterministicFindingCounts` +
+// `inconclusive` via `findingsBlock`, NOT the score scalar or the `verdict` label. `verdict`/
+// `releaseScore` can therefore be removed from exploration output without changing any merge
+// decision. The `riskScore < 50` threshold is retained deliberately (kept explicit, inside
+// `findingsBlock`) and locked by the `[char]` risk-threshold test.
+export function evaluateGate({ report, regression = null, flows = [], scenarios = [], contracts = [], prPlan = null, baseline = null, failOn = "gate" } = {}) {
+  const reasons = []; // { kind: "fail" | "inconclusive", message }
+  const fail = (message) => reasons.push({ kind: "fail", message });
+  const inconclusive = (message) => reasons.push({ kind: "inconclusive", message });
+
+  // Classify each replayed suite by evidence authority (ADR-0005): a DETERMINISTIC step failure (or
+  // a non-model failure like an aborted/missing run) is a real fail; a suite with no deterministic
+  // failure that carries a model-observed (assert_ai) assertion cannot be decided by the default
+  // deterministic gate → inconclusive/needs-review (it must not silently pass, and a model verdict
+  // must not masquerade as a deterministic fail). No `--policy probabilistic` opt-in in 0.17.
+  const deterministicFail = (s) => s.deterministicFailed === true || (s.passed === false && !s.modelObserved);
+  const classify = (s) => (deterministicFail(s) ? "fail" : s.modelObserved ? "needs-review" : "pass");
+  for (const [label, suites] of [["flow", flows], ["multi-actor scenario", scenarios], ["release contract", contracts]]) {
+    const failed = suites.filter((s) => classify(s) === "fail");
+    if (failed.length) fail(`${failed.length} ${label}(s) failed`);
+    const needsReview = suites.filter((s) => classify(s) === "needs-review");
+    if (needsReview.length) inconclusive(`${needsReview.length} ${label}(s) contain assert_ai (model-observed); the deterministic gate cannot decide them — review, or add an explicit probabilistic policy`);
+  }
+  // Selected-but-unexecuted work is missing evidence, not an observed violation → inconclusive.
+  if (prPlan?.execution?.notRun) inconclusive(`${prPlan.execution.notRun} selected release contract(s) did not run`);
+  if (prPlan?.execution?.explorationFailed) inconclusive(`${prPlan.execution.explorationFailed} planned PR exploration target(s) failed or were not reached`);
+
+  if (failOn === "any") {
+    if (report.findingCounts.total > 0) fail(`${report.findingCounts.total} finding(s) (fail-on: any)`);
+    // "any" is the strictest policy — an inconclusive run (evidence not obtained) must never pass it.
+    if (report.inconclusive) inconclusive("run was inconclusive (coverage floor not met)");
+  } else if (failOn === "absolute" || (failOn === "gate" && !regression)) {
+    if (findingsBlock(report.deterministicFindingCounts, { inconclusive: report.inconclusive })) fail("blocking deterministic finding(s)");
+    if (report.inconclusive) inconclusive("run was inconclusive (coverage floor not met)");
+  } else {
+    if (regression?.newFindings?.length) {
+      const newCritical = regression.newFindings.filter((f) => f.severity === "critical").length;
+      const newHigh = regression.newFindings.filter((f) => f.severity === "high").length;
+      if (newCritical + newHigh > 0) fail(`${newCritical} new critical + ${newHigh} new high vs. baseline`);
+    }
+    if (findingsBlock(report.deterministicFindingCounts, { inconclusive: report.inconclusive })) fail("blocking deterministic finding(s)");
+    if (report.inconclusive && !baseline?.inconclusive) inconclusive("run became inconclusive vs. baseline (app may no longer launch/explore)");
+  }
+
+  const outcome = reasons.some((r) => r.kind === "fail") ? "fail"
+    : reasons.some((r) => r.kind === "inconclusive") ? "inconclusive"
+    : "pass";
+  return {
+    policy: failOn,
+    outcome,
+    exitCode: GATE_EXIT[outcome],
+    failed: outcome !== "pass", // retained for markdown/JSON consumers during migration
+    reasons: reasons.map((r) => r.message),
+    reasonDetails: reasons,
+    policyVersion: GATE_POLICY_VERSION,
   };
 }

package/mcp-server/src/task-runtime.js

@@ -109,7 +109,7 @@ export function loadTaskRegistry({ sourcePath, projectDir = "", taskFiles = [] }
   // Draft contracts generated under `.tapp/proposals/contracts` may compile
   // against sibling untrusted Task drafts. Ordinary committed contracts never
   // see this directory, so a proposal cannot silently enter the release gate.
-  const proposalSource = [".tapp", ".autotap"].some((directory) => String(path.resolve(sourcePath || "")).includes(`${path.sep}${directory}${path.sep}proposals${path.sep}`));
+  const proposalSource = String(path.resolve(sourcePath || "")).includes(`${path.sep}.tapp${path.sep}proposals${path.sep}`);
   const proposalDir = proposalSource && tappDir ? path.join(tappDir, "proposals", "tasks") : "";
   const proposed = proposalDir && fs.existsSync(proposalDir)
     ? fs.readdirSync(proposalDir).filter((name) => /\.ya?ml$|\.json$/i.test(name)).map((name) => path.join(proposalDir, name))

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aarwitz/tapp",
-  "version": "0.16.4",
+  "version": "0.17.0-rc.1",
   "mcpName": "io.github.aarwitz/tapp",
   "description": "Release contracts, autonomous QA, and evidence-backed CI gates for iOS, Android, and web.",
   "license": "MIT",
@@ -87,7 +87,7 @@
     "mobile"
   ],
   "scripts": {
-    "test": "node --test tests/report.test.js tests/regression.test.js tests/engine.test.js tests/project-config.test.js tests/application-model.test.js tests/ui-map.test.js tests/task-runtime.test.js tests/release-contract.test.js tests/pr-selection.test.js tests/flow-runtime.test.js tests/web-explorer.test.js tests/web-session.test.js tests/web-flow.test.js tests/scenario-runtime.test.js tests/android-driver.test.js tests/android-explorer.test.js tests/android-flow.test.js tests/managed-web.test.js tests/product-operations.test.js tests/browser-product.test.js tests/browser-onboarding.test.js tests/managed-operation.test.js tests/ci-setup.test.js tests/ci-install.test.js tests/cli.test.js tests/action.test.js tests/package-surface.test.js tests/ci-gate.test.js tests/ci-report.test.js tests/desktop-protocol.test.js tests/ios-flow-protocol.test.js",
+    "test": "node --test tests/report.test.js tests/regression.test.js tests/engine.test.js tests/project-config.test.js tests/application-model.test.js tests/ui-map.test.js tests/task-runtime.test.js tests/release-contract.test.js tests/pr-selection.test.js tests/flow-runtime.test.js tests/web-explorer.test.js tests/web-session.test.js tests/web-flow.test.js tests/scenario-runtime.test.js tests/android-driver.test.js tests/android-explorer.test.js tests/android-flow.test.js tests/managed-web.test.js tests/product-operations.test.js tests/browser-product.test.js tests/browser-onboarding.test.js tests/managed-operation.test.js tests/cloud-runner.test.js tests/ci-setup.test.js tests/ci-install.test.js tests/cli.test.js tests/action.test.js tests/package-surface.test.js tests/ci-gate.test.js tests/ci-report.test.js tests/desktop-protocol.test.js tests/ios-flow-protocol.test.js",
     "test:browser-journey": "node --test tests/browser-journey.test.js",
     "test:browser-native": "TAPP_RUN_NATIVE_BROWSER=1 node --test tests/browser-native-journey.test.js"
   }

package/scripts/ci-gate.sh

@@ -25,7 +25,7 @@
 #                      [--pr-plan-out <file.json>] # persist the reviewable selection plan
 #                      [--baseline <file.json>]   # prior report to diff against (skipped if absent)
 #                      [--target-key <stable-id>] # isolates target-specific baselines in monorepos
-#                      [--fail-on gate|blocked|any]  # gate policy (default gate; see ci-report.js)
+#                      [--fail-on gate|absolute|any]  # gate policy (default gate; see ci-report.js)
 #                      [--json-out <file.json>]   # write the full report (use as the next baseline)
 #                      [--md-out <file.md>]       # write the rendered markdown report (for a PR comment)
 #                      [--device <name>]          # simulator device to boot if none is (default "iPhone 16 Pro")
@@ -74,7 +74,7 @@ done
 [[ "$PLATFORM" == "ios" || "$PLATFORM" == "android" || "$PLATFORM" == "web" ]] || { echo "❌ --platform must be ios|android|web" >&2; exit 2; }
 [[ "$ACTIONS" =~ ^[1-9][0-9]*$ ]] || { echo "❌ --actions must be a positive integer" >&2; exit 2; }
 [[ "$TIMEOUT" =~ ^[1-9][0-9]*$ ]] || { echo "❌ --timeout must be a positive integer" >&2; exit 2; }
-[[ "$FAIL_ON" == "gate" || "$FAIL_ON" == "blocked" || "$FAIL_ON" == "any" ]] || { echo "❌ --fail-on must be gate|blocked|any" >&2; exit 2; }
+[[ "$FAIL_ON" == "gate" || "$FAIL_ON" == "absolute" || "$FAIL_ON" == "any" ]] || { echo "❌ --fail-on must be gate|absolute|any" >&2; exit 2; }
 if [[ -n "$PROJECT_DIR" ]]; then
   [[ -d "$PROJECT_DIR" ]] || { echo "❌ Project directory not found: $PROJECT_DIR" >&2; exit 2; }
   PROJECT_DIR="$(cd "$PROJECT_DIR" && pwd)"
@@ -274,7 +274,7 @@ xcrun simctl install "$UDID" "$APP_PATH" || { echo "❌ simctl install failed
 # ── Autonomous exploration (quick-capture builds the harness itself if needed).
 step "Explore ($ACTIONS actions, ${TIMEOUT}s watchdog)"
 set +e
-CAPTURE_ROOT="${TAPP_HOME:-${AUTOTAP_HOME:-$ROOT}}/captures"
+CAPTURE_ROOT="${TAPP_HOME:-$ROOT}/captures"
 mkdir -p "$CAPTURE_ROOT"
 CAPTURE_DIR="$(mktemp -d "$CAPTURE_ROOT/ci.XXXXXX")"
 TAPP_CAPTURE_DIR="$CAPTURE_DIR" OCQA_PR_TARGET_JSON="$IOS_PR_TARGET_JSON" "$ROOT/scripts/quick-capture.sh" explore "$BUNDLE_ID" --actions "$ACTIONS" --timeout "$TIMEOUT"

package/scripts/quick-capture.sh

@@ -19,7 +19,7 @@ PROJECT_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
 # accidentally reuse stale markers). TAPP_HOME redirects all writable output otherwise —
 # set by the `tapp` CLI when running as an installed npm package, where the package dir
 # must stay read-only. Unset (repo dev flow), everything lands in the repo as before.
-TAPP_RUNTIME_HOME="${TAPP_HOME:-${AUTOTAP_HOME:-}}"
+TAPP_RUNTIME_HOME="${TAPP_HOME:-}"
 if [[ -n "${TAPP_CAPTURE_DIR:-}" ]]; then
   CAPTURE_DIR="$TAPP_CAPTURE_DIR"
 elif [[ -n "$TAPP_RUNTIME_HOME" ]]; then

package/scripts/run-flow.sh

@@ -26,7 +26,7 @@ UDID="$(xcrun simctl list devices booted -j 2>/dev/null | python3 -c 'import sys
 [ -z "$UDID" ] && { echo "❌ No booted simulator."; exit 2; }
 # When run through the installed `tapp` CLI, the harness cache lives under TAPP_HOME.
 XCTR=""
-TAPP_RUNTIME_HOME="${TAPP_HOME:-${AUTOTAP_HOME:-}}"
+TAPP_RUNTIME_HOME="${TAPP_HOME:-}"
 [ -n "$TAPP_RUNTIME_HOME" ] && XCTR="$(find "$TAPP_RUNTIME_HOME/harness-derived/Build/Products" -name '*.xctestrun' 2>/dev/null | head -1)"
 [ -z "$XCTR" ] && XCTR="$(find "$HOME/Library/Developer/Xcode/DerivedData/OCQAHarness-"*/Build/Products -name '*.xctestrun' 2>/dev/null | head -1)"
 [ -z "$XCTR" ] && XCTR="$(find /tmp/tapp-harness-derived/Build/Products -name '*.xctestrun' 2>/dev/null | head -1)"