@aarwitz/tapp 0.16.5 → 0.17.0-rc.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,32 +1,20 @@
1
- import fs from "node:fs";
2
1
  import path from "node:path";
3
2
 
4
3
  export const TAPP_DIRECTORY = ".tapp";
5
- export const LEGACY_TAPP_DIRECTORY = ".autotap";
6
4
  export const TAPP_CONFIG = ".tapp.yml";
7
- export const LEGACY_TAPP_CONFIG = ".autotap.yml";
8
5
 
9
- export function projectArtifactDirectory(projectDir, requested = TAPP_DIRECTORY) {
10
- const root = path.resolve(projectDir);
11
- if (requested !== TAPP_DIRECTORY) return requested;
12
- const canonical = path.join(root, TAPP_DIRECTORY);
13
- const legacy = path.join(root, LEGACY_TAPP_DIRECTORY);
14
- if (!fs.existsSync(canonical) && fs.existsSync(legacy)) return LEGACY_TAPP_DIRECTORY;
15
- return TAPP_DIRECTORY;
6
+ export function projectArtifactDirectory(_projectDir, requested = TAPP_DIRECTORY) {
7
+ return requested;
16
8
  }
17
9
 
18
10
  export function projectArtifactPath(projectDir, ...parts) {
19
- return path.join(path.resolve(projectDir), projectArtifactDirectory(projectDir), ...parts);
11
+ return path.join(path.resolve(projectDir), TAPP_DIRECTORY, ...parts);
20
12
  }
21
13
 
22
14
  export function existingProjectArtifactPath(projectDir, ...parts) {
23
- const root = path.resolve(projectDir);
24
- const canonical = path.join(root, TAPP_DIRECTORY, ...parts);
25
- if (fs.existsSync(canonical)) return canonical;
26
- const legacy = path.join(root, LEGACY_TAPP_DIRECTORY, ...parts);
27
- return fs.existsSync(legacy) ? legacy : canonical;
15
+ return path.join(path.resolve(projectDir), TAPP_DIRECTORY, ...parts);
28
16
  }
29
17
 
30
18
  export function isProjectArtifactDirectory(name) {
31
- return name === TAPP_DIRECTORY || name === LEGACY_TAPP_DIRECTORY;
19
+ return name === TAPP_DIRECTORY;
32
20
  }
@@ -11,8 +11,8 @@ import { semanticUiKey } from "./ui-map.js";
11
11
 
12
12
  const PLATFORMS = new Set(["ios", "android", "web"]);
13
13
  const CRITICALITIES = new Set(["low", "medium", "high", "critical"]);
14
- const CONTRACT_AUTHORING_SPECIFIERS = new Set(["@aarwitz/tapp/contracts", "runtapp/contracts", "tapp-mcp/contracts"]);
15
- const CONTRACT_AUTHORING_IMPORT = /(["'])(?:@aarwitz\/tapp|runtapp|tapp-mcp)\/contracts\1/g;
14
+ const CONTRACT_AUTHORING_SPECIFIERS = new Set(["@aarwitz/tapp/contracts"]);
15
+ const CONTRACT_AUTHORING_IMPORT = /(["'])@aarwitz\/tapp\/contracts\1/g;
16
16
  const authoringUrl = pathToFileURL(path.resolve(path.dirname(fileURLToPath(import.meta.url)), "contract-authoring.js")).href;
17
17
 
18
18
  function expectationAction(expectation) {
@@ -86,7 +86,7 @@ function rewriteAuthoringImport(source, contractPath) {
86
86
  }
87
87
  const imports = [...source.matchAll(/(?:from\s*|import\s*)["']([^"']+)["']/g)].map((match) => match[1]);
88
88
  const unsupported = imports.filter((specifier) => !CONTRACT_AUTHORING_SPECIFIERS.has(specifier));
89
- if (unsupported.length) throw new Error(`Release contract imports are limited to @aarwitz/tapp/contracts (legacy runtapp/contracts and tapp-mcp/contracts are also accepted; found ${unsupported.join(", ")})`);
89
+ if (unsupported.length) throw new Error(`Release contract imports are limited to @aarwitz/tapp/contracts (found ${unsupported.join(", ")})`);
90
90
  const output = ts.transpileModule(source, {
91
91
  fileName: contractPath,
92
92
  compilerOptions: { module: ts.ModuleKind.ES2022, target: ts.ScriptTarget.ES2022, verbatimModuleSyntax: true },
@@ -1,6 +1,6 @@
1
1
  // Pure report/gate logic shared by the MCP server (index.js) and the CI gate CLI
2
- // (ci-report.js). Turns a capture's OCQA markers into the same ship/no-ship report the Tapp
3
- // app produces, and diffs two runs' findings into the CI regression gate. No shell, no server —
2
+ // (ci-report.js). Turns a capture's OCQA markers into a scoreless exploration observation,
3
+ // and applies explicit policy separately in the CI gate. No shell, no server —
4
4
  // keep it dependency-free so the CI path stays importable and testable.
5
5
  import fs from "fs";
6
6
  import path from "path";
@@ -106,21 +106,36 @@ export function findingEvaluationTier(finding, platform = "ios") {
106
106
  return platform === "web" && WEB_SAMPLED_ISSUE_TYPES.has(finding?.type) ? "sampled" : "deterministic";
107
107
  }
108
108
 
109
- export function verdictBadge(report) {
110
- if (report?.platform === "web" && report?.verdict === "ready") return "🔵 AUTOMATED CHECKS COMPLETE";
111
- return { ready: "🟢 SHIP-READY", caution: "🟡 CAUTION", blocked: "🔴 BLOCKED" }[report?.verdict] || report?.verdict;
109
+ // The deterministic block-by-findings rule, expressed over verdict-tier finding COUNTS — not the
110
+ // score scalar and not the `verdict` label so the CI gate survives removal of verdict/releaseScore
111
+ // from exploration output. It encodes exactly what `verdict === "blocked"` used to: a critical
112
+ // finding always blocks; otherwise, on a CONCLUSIVE run, a risk threshold blocks. The risk threshold
113
+ // is kept as an explicit, chosen rule (ADR-0005) — `riskFromCounts` is the single source of that
114
+ // formula, shared with buildQaReport's verdict label. Inconclusive runs are a separate gate outcome,
115
+ // never a findings-block, so a thin run reports `inconclusive`, not `fail`.
116
+ export function riskFromCounts({ critical = 0, high = 0, medium = 0 } = {}) {
117
+ return Math.max(0, Math.min(100, 100 - critical * 25 - high * 10 - medium * 3));
118
+ }
119
+ export function findingsBlock(deterministicFindingCounts = {}, { inconclusive = false } = {}) {
120
+ if ((deterministicFindingCounts.critical || 0) > 0) return true;
121
+ if (inconclusive) return false;
122
+ return riskFromCounts(deterministicFindingCounts) < 50;
123
+ }
124
+
125
+ // Exploration OBSERVES; it never renders a ship verdict or score (ADR-0005). These label the
126
+ // observation honestly. The release decision (pass/fail/inconclusive) is the gate's, shown separately.
127
+ export function observationBadge(report) {
128
+ return report?.inconclusive ? "🟡 INCONCLUSIVE (exploration)" : "🔭 EXPLORED";
112
129
  }
113
130
 
114
- export function qaScoreLabel(report) {
115
- const score = report?.releaseScore ?? report?.confidence;
116
- if (Number.isFinite(score)) return `release score ${score}/100`;
117
- if (report?.platform === "web") return "exploratory web · no scalar score";
118
- return "score unavailable";
131
+ export function observationSummary(report) {
132
+ const n = report?.findingCounts?.total || 0;
133
+ return `${report?.screensExplored || 0} screens · ${report?.actionsPerformed || 0} actions · ${n} finding(s) · observation only`;
119
134
  }
120
135
 
121
- // Turn a capture's OCQA markers into the same ship/no-ship report Tapp produces:
122
- // deduped findings + a trustworthy verdict with a coverage floor (mirrors OrchestratorService).
123
- export function buildQaReport(markersFilePath, { platform = "ios" } = {}) {
136
+ // Turn a capture's OCQA markers into a scoreless observation with deduped findings and an
137
+ // explicit coverage floor. Release judgment is applied later by evaluateGate.
138
+ export function buildQaReport(markersFilePath, { platform = "ios", target = null } = {}) {
124
139
  const base = parseOcqaMarkers(markersFilePath);
125
140
  if (!base) return null;
126
141
 
@@ -130,6 +145,7 @@ export function buildQaReport(markersFilePath, { platform = "ios" } = {}) {
130
145
  const inputsByScreen = new Map();
131
146
  const screenElementCounts = {}; // screen -> max elements observed (content-collapse detection)
132
147
  let anySecure = false;
148
+ let loginAttempted = false;
133
149
  let actions = 0;
134
150
 
135
151
  for (const line of raw.split(/\r?\n/)) {
@@ -148,6 +164,17 @@ export function buildQaReport(markersFilePath, { platform = "ios" } = {}) {
148
164
  }
149
165
  } else if (t.startsWith("OCQA_ACTION:")) {
150
166
  actions += 1;
167
+ try {
168
+ const action = JSON.parse(t.slice("OCQA_ACTION:".length));
169
+ if (action?.type === "login" || String(action?.type || "").startsWith("login_")) loginAttempted = true;
170
+ } catch {
171
+ /* ignore malformed */
172
+ }
173
+ } else if (
174
+ t === "OCQA_STATE:login_preamble_submitted" ||
175
+ t === "OCQA_STATE:login_preamble_two_step_submitted"
176
+ ) {
177
+ loginAttempted = true;
151
178
  } else if (t.startsWith("OCQA_STATE:{")) {
152
179
  try {
153
180
  const s = JSON.parse(t.slice("OCQA_STATE:".length));
@@ -187,7 +214,12 @@ export function buildQaReport(markersFilePath, { platform = "ios" } = {}) {
187
214
  const missingResources = new Set(normalizedIssues
188
215
  .filter((issue) => issue.type === "missing_asset" && issue.target)
189
216
  .map((issue) => issue.target));
217
+ // Older native captures represented the caller's wall-clock budget as a high-severity app
218
+ // finding. A timeout makes the evidence partial/inconclusive; it does not prove an app
219
+ // performance defect (the harness has a separate app_hang detector for that).
220
+ const timeBudgetExhausted = base.complete?.timedOut === true || normalizedIssues.some((issue) => issue.type === "explore_timeout");
190
221
  const reportIssues = normalizedIssues.filter((issue) =>
222
+ issue.type !== "explore_timeout" &&
191
223
  !(issue.type === "network_error" && issue.target && missingResources.has(issue.target) && /^Request failed:/i.test(String(issue.title || ""))));
192
224
 
193
225
  // Dedup by stable signature (type|screen|target) so repeated detections count once —
@@ -201,14 +233,23 @@ export function buildQaReport(markersFilePath, { platform = "ios" } = {}) {
201
233
  findings.push({
202
234
  ...i,
203
235
  category: ISSUE_CATEGORY[i.type] || i.type,
236
+ // Structural evidence authority (ADR-0005): marker-derived findings are deterministic. The
237
+ // default gate consumes only deterministic-authority evidence; model-observed findings
238
+ // (vision/assert_ai) carry authority:"model-observed" and are advisory, never gate fails.
239
+ authority: "deterministic",
204
240
  ...(platform === "web" ? { evaluationTier: findingEvaluationTier(i, platform) } : {}),
205
241
  });
206
242
  }
207
243
  findings.sort((a, b) => severityRank(a.severity) - severityRank(b.severity));
208
244
 
209
245
  const screensExplored = screens.size || base.uniqueScreens.length;
210
- const actionsPerformed =
211
- actions || (base.complete && typeof base.complete === "object" ? base.complete.actions || 0 : 0);
246
+ // Some native recovery operations are counted by the harness budget but intentionally do not
247
+ // emit a public action narrative. Preserve the larger authoritative completion count instead of
248
+ // understating coverage whenever at least one narrated action exists.
249
+ const actionsPerformed = Math.max(
250
+ actions,
251
+ base.complete && typeof base.complete === "object" ? base.complete.actions || 0 : 0,
252
+ );
212
253
  const crit = findings.filter((f) => f.severity === "critical").length;
213
254
  const high = findings.filter((f) => f.severity === "high").length;
214
255
  const med = findings.filter((f) => f.severity === "medium").length;
@@ -222,29 +263,33 @@ export function buildQaReport(markersFilePath, { platform = "ios" } = {}) {
222
263
  const verdictLow = verdictFindings.filter((f) => f.severity === "low").length;
223
264
  const sampledFindings = platform === "web" ? findings.filter((finding) => finding.evaluationTier === "sampled") : [];
224
265
 
225
- // Coverage floor: a verdict is only trustworthy if the app was actually exercised.
226
- const inconclusive = screensExplored < 2 || actionsPerformed < 3;
227
- let riskScore = Math.max(0, Math.min(100, 100 - verdictCrit * 25 - verdictHigh * 10 - verdictMed * 3));
228
- if (inconclusive) riskScore = Math.min(riskScore, 40);
229
-
230
- let verdict;
231
- if (verdictCrit > 0) verdict = "blocked";
232
- else if (inconclusive) verdict = "caution";
233
- else if (riskScore < 50) verdict = "blocked";
234
- else if (verdictHigh > 0 || riskScore < 80) verdict = "caution";
235
- else verdict = "ready";
236
-
237
- const headline = inconclusive
266
+ // Coverage floor: exploration is inconclusive if the app wasn't actually exercised. Exploration
267
+ // OBSERVES it does not render a ship verdict or score (ADR-0005). Judgment (pass/fail/
268
+ // inconclusive) is the gate's job (evaluateGate), computed from these findings + coverage + policy.
269
+ // A one-page web target can still be swept exhaustively: page errors, requests, links, assets,
270
+ // placeholder anchors, and visible controls do not require a second route. Native exploration
271
+ // retains the stronger multi-screen/action floor. A credentialless single-screen login remains
272
+ // inconclusive so a login wall can never turn into a clean pass.
273
+ const coverageFloorMet = platform === "web"
274
+ ? screensExplored >= 1 && actionsPerformed >= 1
275
+ : screensExplored >= 2 && actionsPerformed >= 3;
276
+ const credentiallessLoginWall = anySecure && !loginAttempted && screensExplored <= 1;
277
+ const inconclusive = !coverageFloorMet || credentiallessLoginWall || timeBudgetExhausted;
278
+ const stopReason = credentiallessLoginWall ? "login-wall-no-credentials"
279
+ : timeBudgetExhausted ? "time-budget-exhausted"
280
+ : coverageFloorMet ? "completed" : "coverage-floor-not-met";
281
+
282
+ const headline = timeBudgetExhausted
283
+ ? `Inconclusive — exploration reached its ${base.complete?.timeoutSeconds || "configured"}s time budget after ${actionsPerformed} action(s) across ${screensExplored} screen(s). Findings are partial; this is not an app performance finding. Increase --timeout or request fewer actions.`
284
+ : inconclusive
238
285
  ? `Inconclusive — only ${screensExplored} screen(s) / ${actionsPerformed} action(s) explored. The app may have crashed on launch, be stuck behind a sign-in wall, or otherwise prevent exploration. Absence of issues is NOT a pass.`
239
- : verdict === "ready"
286
+ : findings.length === 0
240
287
  ? platform === "web"
241
- ? "Automated web checks completed — no release-blocking deterministic findings in the exercised surfaces. Sampled control probes are advisory. This is not a content, privacy, brand, or business-claim review."
242
- : "Ship-ready no release-blocking issues found."
243
- : verdict === "caution"
244
- ? `Proceed with caution — ${findings.length} issue(s) to review.`
245
- : `Not ready — ${findings.length} issue(s): ${crit} critical, ${high} high, ${med} medium, ${low} low.`;
288
+ ? "Automated web checks completed — no deterministic findings in the exercised surfaces. Sampled control probes are advisory. An observation, not a release decision, and not a content, privacy, brand, or business-claim review."
289
+ : "No issues surfaced in the exercised surfaces. An observation, not a release decision."
290
+ : `${findings.length} issue(s) surfaced for review (${crit} critical, ${high} high, ${med} medium, ${low} low). An observation, not a release decision.`;
246
291
 
247
- // The verdict's own honesty label: exactly which defect classes this run checked, which
292
+ // The observation's honesty label: exactly which defect classes this run checked, which
248
293
  // it structurally could NOT check, and which conditions never came up — so "checked" is
249
294
  // never claimed for a state the run didn't reach. Platform-aware: a web run doesn't
250
295
  // inherit iOS keyboard assertions and vice versa.
@@ -290,30 +335,38 @@ export function buildQaReport(markersFilePath, { platform = "ios" } = {}) {
290
335
  "content & reachability regressions require a baseline" ,
291
336
  ];
292
337
  }
293
- // "Failed sign-ins" is only a claim when a sign-in surface was actually encountered.
294
- if (anySecure) checkedFor.splice(2, 0, "failed sign-ins");
338
+ // "Failed sign-ins" is only a claim when credentials were actually submitted. Merely seeing a
339
+ // password field proves that a login surface was reached, not that authentication was exercised.
340
+ if (loginAttempted) checkedFor.splice(2, 0, "failed sign-ins");
341
+ else if (anySecure) notChecked.push("sign-in behavior (login form reached, no test credentials supplied)");
295
342
  else conditionsNotReached.push("sign-in (no login form encountered this run)");
343
+ if (timeBudgetExhausted) notChecked.push("the full requested action budget (run reached its wall-clock timeout)");
296
344
 
297
345
  return {
298
- verdict,
299
- // Exploratory web QA deliberately has no scalar. Its verdict derives from deterministic
300
- // checks on exercised pages; budget-capped control probes remain visible but advisory.
301
- // Native keeps the legacy heuristic score until it has an equivalent tier split.
302
- confidence: platform === "web" ? null : riskScore,
303
- releaseScore: platform === "web" ? null : riskScore,
304
- scoreUnavailableReason: platform === "web"
305
- ? "Exploratory web runs report deterministic findings, advisory sampled probes, and coverage instead of a scalar release score."
306
- : null,
346
+ // An ExplorationRun observation: findings + coverage + evidence, NO ship verdict or score
347
+ // (ADR-0005). The gate (evaluateGate) turns this into a pass/fail/inconclusive release outcome.
348
+ kind: "tapp-exploration-run",
349
+ schemaVersion: 1,
350
+ // Complete ExplorationRun contract (ADR-0005 §4). runStatus/stopReason describe HOW the run
351
+ // ended; coverage/evidence/uiMap/comparison are the structured observation. uiMap and comparison
352
+ // are populated by consumers that build the map / diff a baseline (null in the bare observation).
353
+ runStatus: inconclusive ? "limited" : "completed",
354
+ stopReason,
307
355
  headline,
308
356
  inconclusive,
357
+ coverage: { screensExplored, actionsPerformed, screens: Array.from(screens) },
358
+ evidence: { markers: base.relativeMarkersFilePath },
359
+ uiMap: null,
360
+ comparison: null,
309
361
  checkedFor,
310
362
  notChecked,
311
363
  conditionsNotReached,
312
364
  platform,
365
+ target: typeof target === "string" && target.trim() ? target.trim() : null,
313
366
  screensExplored,
314
367
  actionsPerformed,
315
368
  findingCounts: { critical: crit, high, medium: med, low, total: findings.length },
316
- verdictFindingCounts: {
369
+ deterministicFindingCounts: {
317
370
  critical: verdictCrit,
318
371
  high: verdictHigh,
319
372
  medium: verdictMed,
@@ -355,6 +408,7 @@ export function computeContentCollapse(currentCounts, baselineCounts) {
355
408
  type: "content_collapse",
356
409
  severity: "high",
357
410
  category: "content_collapse",
411
+ authority: "deterministic",
358
412
  title: `Screen lost most of its content (${base} → ${cur} elements)`,
359
413
  screen,
360
414
  step: null,
@@ -383,6 +437,7 @@ export function computeReachabilityLoss(current, baseline) {
383
437
  type: "screen_unreachable",
384
438
  severity: "high",
385
439
  category: "navigation_dead_end",
440
+ authority: "deterministic",
386
441
  title: "Screen explored in the baseline was never reached this run",
387
442
  screen,
388
443
  step: null,
@@ -421,15 +476,94 @@ export function computeRegression(current, baseline) {
421
476
  const newFindings = current.filter((f) => !currentMatches(f));
422
477
  const persisting = current.filter((f) => currentMatches(f));
423
478
  const resolved = baseline.filter((b) => !baselineMatched(b));
424
- const newCritical = newFindings.filter((f) => f.severity === "critical").length;
425
- const newHigh = newFindings.filter((f) => f.severity === "high").length;
426
479
 
480
+ // Comparison ONLY — no gate/pass/fail signal (ADR-0005). Exploration surfaces this diff; the merge
481
+ // decision is the gate's job. evaluateGate derives its regression fail from `newFindings` severities.
427
482
  return {
428
483
  hadBaseline: true,
429
484
  counts: { new: newFindings.length, persisting: persisting.length, resolved: resolved.length },
430
485
  newFindings,
431
486
  resolved,
432
- // CI gate: fail the build when this run introduced new high/critical findings vs. the baseline.
433
- gate: { newCritical, newHigh, failed: newCritical + newHigh > 0 },
487
+ };
488
+ }
489
+
490
+ // The gate's public outcome model (ADR-0005). A merge gate is ultimately block / don't-block, but
491
+ // callers need to distinguish WHY: a deterministic violation is not the same as "we couldn't get
492
+ // the evidence." Exit codes are the CI contract; precedence is fail > inconclusive > pass.
493
+ export const GATE_EXIT = { pass: 0, fail: 1, error: 2, inconclusive: 3 };
494
+ // Bump when the gate's decision semantics change (NOT the npm version). Recorded on every GateRun.
495
+ export const GATE_POLICY_VERSION = "2";
496
+
497
+ // Pure gate evaluator: frozen evidence + policy → a GateRun decision. Extracted verbatim from the
498
+ // former inline logic in ci-report.js so the `[char]` characterization tests keep passing — the
499
+ // merge decision (block/don't-block) is unchanged; this only classifies each reason as a
500
+ // deterministic `fail` or an evidence-absent `inconclusive` and folds them by precedence. Reason
501
+ // MESSAGES are preserved exactly (several are asserted by tests).
502
+ //
503
+ // DECOUPLED FROM THE SCORE (ADR-0005): the block-by-findings decision reads `deterministicFindingCounts` +
504
+ // `inconclusive` via `findingsBlock`, NOT the score scalar or the `verdict` label. `verdict`/
505
+ // `releaseScore` can therefore be removed from exploration output without changing any merge
506
+ // decision. The `riskScore < 50` threshold is retained deliberately (kept explicit, inside
507
+ // `findingsBlock`) and locked by the `[char]` risk-threshold test.
508
+ export function evaluateGate({ report, regression = null, flows = [], scenarios = [], contracts = [], prPlan = null, baseline = null, failOn = "gate" } = {}) {
509
+ const reasons = []; // { kind: "fail" | "inconclusive", message }
510
+ const fail = (message) => reasons.push({ kind: "fail", message });
511
+ const inconclusive = (message) => reasons.push({ kind: "inconclusive", message });
512
+
513
+ // Classify each replayed suite by evidence authority (ADR-0005): a DETERMINISTIC step failure (or
514
+ // a non-model failure like an aborted/missing run) is a real fail; a suite with no deterministic
515
+ // failure that carries a model-observed (assert_ai) assertion cannot be decided by the default
516
+ // deterministic gate → inconclusive/needs-review (it must not silently pass, and a model verdict
517
+ // must not masquerade as a deterministic fail). No `--policy probabilistic` opt-in in 0.17.
518
+ const deterministicFail = (s) => s.deterministicFailed === true || (s.passed === false && !s.modelObserved);
519
+ const classify = (s) => (deterministicFail(s) ? "fail" : s.modelObserved ? "needs-review" : "pass");
520
+ for (const [label, suites] of [["flow", flows], ["multi-actor scenario", scenarios], ["release contract", contracts]]) {
521
+ const failed = suites.filter((s) => classify(s) === "fail");
522
+ if (failed.length) fail(`${failed.length} ${label}(s) failed`);
523
+ const needsReview = suites.filter((s) => classify(s) === "needs-review");
524
+ if (needsReview.length) inconclusive(`${needsReview.length} ${label}(s) contain assert_ai (model-observed); the deterministic gate cannot decide them — review, or add an explicit probabilistic policy`);
525
+ }
526
+ // A submitted sign-in that remains on the login surface is an explicit exercised guarantee,
527
+ // not ordinary pre-existing UI debt. Letting it pass without a baseline would produce the
528
+ // contradictory public result "failed sign-in detected" + gate PASS. Keep sampled probes out,
529
+ // but fail every deterministic auth failure under every gate policy (including with a baseline).
530
+ const authFailures = (report.findings || []).filter((finding) =>
531
+ finding?.type === "auth_failed" &&
532
+ finding?.authority !== "model-observed" &&
533
+ finding?.evaluationTier !== "sampled"
534
+ );
535
+ if (authFailures.length) fail(`${authFailures.length} deterministic sign-in attempt(s) failed`);
536
+ // Selected-but-unexecuted work is missing evidence, not an observed violation → inconclusive.
537
+ if (prPlan?.execution?.notRun) inconclusive(`${prPlan.execution.notRun} selected release contract(s) did not run`);
538
+ if (prPlan?.execution?.explorationFailed) inconclusive(`${prPlan.execution.explorationFailed} planned PR exploration target(s) failed or were not reached`);
539
+
540
+ if (failOn === "any") {
541
+ if (report.findingCounts.total > 0) fail(`${report.findingCounts.total} finding(s) (fail-on: any)`);
542
+ // "any" is the strictest policy — an inconclusive run (evidence not obtained) must never pass it.
543
+ if (report.inconclusive) inconclusive("run was inconclusive (coverage floor not met)");
544
+ } else if (failOn === "absolute" || (failOn === "gate" && !regression)) {
545
+ if (findingsBlock(report.deterministicFindingCounts, { inconclusive: report.inconclusive })) fail("blocking deterministic finding(s)");
546
+ if (report.inconclusive) inconclusive("run was inconclusive (coverage floor not met)");
547
+ } else {
548
+ if (regression?.newFindings?.length) {
549
+ const newCritical = regression.newFindings.filter((f) => f.severity === "critical").length;
550
+ const newHigh = regression.newFindings.filter((f) => f.severity === "high").length;
551
+ if (newCritical + newHigh > 0) fail(`${newCritical} new critical + ${newHigh} new high vs. baseline`);
552
+ }
553
+ if (findingsBlock(report.deterministicFindingCounts, { inconclusive: report.inconclusive })) fail("blocking deterministic finding(s)");
554
+ if (report.inconclusive && !baseline?.inconclusive) inconclusive("run became inconclusive vs. baseline (app may no longer launch/explore)");
555
+ }
556
+
557
+ const outcome = reasons.some((r) => r.kind === "fail") ? "fail"
558
+ : reasons.some((r) => r.kind === "inconclusive") ? "inconclusive"
559
+ : "pass";
560
+ return {
561
+ policy: failOn,
562
+ outcome,
563
+ exitCode: GATE_EXIT[outcome],
564
+ failed: outcome !== "pass", // retained for markdown/JSON consumers during migration
565
+ reasons: reasons.map((r) => r.message),
566
+ reasonDetails: reasons,
567
+ policyVersion: GATE_POLICY_VERSION,
434
568
  };
435
569
  }
@@ -109,7 +109,7 @@ export function loadTaskRegistry({ sourcePath, projectDir = "", taskFiles = [] }
109
109
  // Draft contracts generated under `.tapp/proposals/contracts` may compile
110
110
  // against sibling untrusted Task drafts. Ordinary committed contracts never
111
111
  // see this directory, so a proposal cannot silently enter the release gate.
112
- const proposalSource = [".tapp", ".autotap"].some((directory) => String(path.resolve(sourcePath || "")).includes(`${path.sep}${directory}${path.sep}proposals${path.sep}`));
112
+ const proposalSource = String(path.resolve(sourcePath || "")).includes(`${path.sep}.tapp${path.sep}proposals${path.sep}`);
113
113
  const proposalDir = proposalSource && tappDir ? path.join(tappDir, "proposals", "tasks") : "";
114
114
  const proposed = proposalDir && fs.existsSync(proposalDir)
115
115
  ? fs.readdirSync(proposalDir).filter((name) => /\.ya?ml$|\.json$/i.test(name)).map((name) => path.join(proposalDir, name))
@@ -45,7 +45,7 @@ export function webPlaceholderLinkFindings(links = []) {
45
45
  const seen = new Set();
46
46
  for (const link of links || []) {
47
47
  const rawHref = String(link?.rawHref || "").trim().toLowerCase();
48
- const placeholder = rawHref === "#" || /^javascript:(?:void\(0\);?|;?)$/.test(rawHref);
48
+ const placeholder = rawHref === "" || rawHref === "#" || /^javascript:(?:void\(0\);?|;?)$/.test(rawHref);
49
49
  if (!placeholder || link?.handlerHint) continue;
50
50
  const label = String(link?.label || "").replace(/\s+/g, " ").trim().slice(0, 100);
51
51
  const fingerprint = String(link?.fingerprint || "link").replace(/\s+/g, " ").trim().slice(0, 100) || "link";
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@aarwitz/tapp",
3
- "version": "0.16.5",
3
+ "version": "0.17.0-rc.2",
4
4
  "mcpName": "io.github.aarwitz/tapp",
5
5
  "description": "Release contracts, autonomous QA, and evidence-backed CI gates for iOS, Android, and web.",
6
6
  "license": "MIT",
@@ -87,7 +87,7 @@
87
87
  "mobile"
88
88
  ],
89
89
  "scripts": {
90
- "test": "node --test tests/report.test.js tests/regression.test.js tests/engine.test.js tests/project-config.test.js tests/application-model.test.js tests/ui-map.test.js tests/task-runtime.test.js tests/release-contract.test.js tests/pr-selection.test.js tests/flow-runtime.test.js tests/web-explorer.test.js tests/web-session.test.js tests/web-flow.test.js tests/scenario-runtime.test.js tests/android-driver.test.js tests/android-explorer.test.js tests/android-flow.test.js tests/managed-web.test.js tests/product-operations.test.js tests/browser-product.test.js tests/browser-onboarding.test.js tests/managed-operation.test.js tests/ci-setup.test.js tests/ci-install.test.js tests/cli.test.js tests/action.test.js tests/package-surface.test.js tests/ci-gate.test.js tests/ci-report.test.js tests/desktop-protocol.test.js tests/ios-flow-protocol.test.js",
90
+ "test": "node --test tests/report.test.js tests/regression.test.js tests/engine.test.js tests/project-config.test.js tests/application-model.test.js tests/ui-map.test.js tests/task-runtime.test.js tests/release-contract.test.js tests/pr-selection.test.js tests/flow-runtime.test.js tests/web-explorer.test.js tests/web-session.test.js tests/web-flow.test.js tests/scenario-runtime.test.js tests/android-driver.test.js tests/android-explorer.test.js tests/android-flow.test.js tests/android-primitives-protocol.test.js tests/managed-web.test.js tests/product-operations.test.js tests/browser-product.test.js tests/browser-onboarding.test.js tests/managed-operation.test.js tests/cloud-runner.test.js tests/ci-setup.test.js tests/ci-install.test.js tests/cli.test.js tests/action.test.js tests/package-surface.test.js tests/ci-gate.test.js tests/ci-report.test.js tests/desktop-protocol.test.js tests/ios-flow-protocol.test.js",
91
91
  "test:browser-journey": "node --test tests/browser-journey.test.js",
92
92
  "test:browser-native": "TAPP_RUN_NATIVE_BROWSER=1 node --test tests/browser-native-journey.test.js"
93
93
  }
@@ -3,9 +3,11 @@
3
3
  #
4
4
  # Boots a simulator if needed, installs the app build under test, runs the autonomous exploration
5
5
  # harness, replays every committed Flow (deterministic E2E tests), diffs the findings against a
6
- # stored baseline, writes a GitHub Actions step summary, and exits non-zero when the gate fails
7
- # (new high/critical findings vs. baseline, or any failed Flow). Wrapped by ../action.yml for
8
- # GitHub Actions; equally usable from any other CI or locally.
6
+ # stored baseline, writes a GitHub Actions step summary, and exits with the public outcome contract:
7
+ # pass 0, deterministic fail 1, infrastructure/usage error 2, inconclusive evidence 3. Absolute
8
+ # blockers and reviewed Flow/Scenario/Contract failures are enforced even without a baseline;
9
+ # baseline comparisons additionally catch new high/critical regressions. Wrapped by ../action.yml
10
+ # for GitHub Actions; equally usable from any other CI or locally.
9
11
  #
10
12
  # Usage:
11
13
  # scripts/ci-gate.sh [--platform ios] --app <path/to/App.app> [--bundle-id <com.example.app>]
@@ -25,7 +27,7 @@
25
27
  # [--pr-plan-out <file.json>] # persist the reviewable selection plan
26
28
  # [--baseline <file.json>] # prior report to diff against (skipped if absent)
27
29
  # [--target-key <stable-id>] # isolates target-specific baselines in monorepos
28
- # [--fail-on gate|blocked|any] # gate policy (default gate; see ci-report.js)
30
+ # [--fail-on gate|absolute|any] # gate policy (default gate; see ci-report.js)
29
31
  # [--json-out <file.json>] # write the full report (use as the next baseline)
30
32
  # [--md-out <file.md>] # write the rendered markdown report (for a PR comment)
31
33
  # [--device <name>] # simulator device to boot if none is (default "iPhone 16 Pro")
@@ -35,7 +37,7 @@ set -uo pipefail
35
37
  ROOT="$(cd "$(dirname "$0")/.." && pwd)"
36
38
 
37
39
  usage() {
38
- sed -n '2,30p' "$0" | sed 's/^# \{0,1\}//'
40
+ sed -n '2,/^set -uo pipefail$/p' "$0" | sed '$d; s/^# \{0,1\}//'
39
41
  }
40
42
 
41
43
  PLATFORM="ios" APP_PATH="" BUNDLE_ID="" APK_PATH="" APP_ID="" URL="" WEB_TARGET="" TARGET_KEY="" SERIAL="" ACTIONS=40 TIMEOUT=600 FLOWS="" SCENARIOS="" CONTRACTS="" PROJECT_DIR="" BASELINE="" FAIL_ON="gate" JSON_OUT="" MD_OUT="" DEVICE="iPhone 16 Pro" PR_BASE="" PR_HEAD="HEAD" CHANGED_FILES_FILE="" PR_PLAN_OUT=""
@@ -74,7 +76,7 @@ done
74
76
  [[ "$PLATFORM" == "ios" || "$PLATFORM" == "android" || "$PLATFORM" == "web" ]] || { echo "❌ --platform must be ios|android|web" >&2; exit 2; }
75
77
  [[ "$ACTIONS" =~ ^[1-9][0-9]*$ ]] || { echo "❌ --actions must be a positive integer" >&2; exit 2; }
76
78
  [[ "$TIMEOUT" =~ ^[1-9][0-9]*$ ]] || { echo "❌ --timeout must be a positive integer" >&2; exit 2; }
77
- [[ "$FAIL_ON" == "gate" || "$FAIL_ON" == "blocked" || "$FAIL_ON" == "any" ]] || { echo "❌ --fail-on must be gate|blocked|any" >&2; exit 2; }
79
+ [[ "$FAIL_ON" == "gate" || "$FAIL_ON" == "absolute" || "$FAIL_ON" == "any" ]] || { echo "❌ --fail-on must be gate|absolute|any" >&2; exit 2; }
78
80
  if [[ -n "$PROJECT_DIR" ]]; then
79
81
  [[ -d "$PROJECT_DIR" ]] || { echo "❌ Project directory not found: $PROJECT_DIR" >&2; exit 2; }
80
82
  PROJECT_DIR="$(cd "$PROJECT_DIR" && pwd)"
@@ -82,7 +84,6 @@ fi
82
84
  TAPP_PROJECT_ARTIFACTS=""
83
85
  if [[ -n "$PROJECT_DIR" ]]; then
84
86
  [[ -d "$PROJECT_DIR/.tapp" ]] && TAPP_PROJECT_ARTIFACTS="$PROJECT_DIR/.tapp"
85
- [[ -z "$TAPP_PROJECT_ARTIFACTS" && -d "$PROJECT_DIR/.autotap" ]] && TAPP_PROJECT_ARTIFACTS="$PROJECT_DIR/.autotap"
86
87
  fi
87
88
  [[ -z "$FLOWS" && -n "$TAPP_PROJECT_ARTIFACTS" && -d "$TAPP_PROJECT_ARTIFACTS/flows" ]] && FLOWS="$TAPP_PROJECT_ARTIFACTS/flows/*.yml"
88
89
  [[ "$PLATFORM" == "web" && -z "$SCENARIOS" && -n "$TAPP_PROJECT_ARTIFACTS" && -d "$TAPP_PROJECT_ARTIFACTS/scenarios" ]] && SCENARIOS="$TAPP_PROJECT_ARTIFACTS/scenarios/*.yml"
@@ -274,7 +275,7 @@ xcrun simctl install "$UDID" "$APP_PATH" || { echo "❌ simctl install failed
274
275
  # ── Autonomous exploration (quick-capture builds the harness itself if needed).
275
276
  step "Explore ($ACTIONS actions, ${TIMEOUT}s watchdog)"
276
277
  set +e
277
- CAPTURE_ROOT="${TAPP_HOME:-${AUTOTAP_HOME:-$ROOT}}/captures"
278
+ CAPTURE_ROOT="${TAPP_HOME:-$ROOT}/captures"
278
279
  mkdir -p "$CAPTURE_ROOT"
279
280
  CAPTURE_DIR="$(mktemp -d "$CAPTURE_ROOT/ci.XXXXXX")"
280
281
  TAPP_CAPTURE_DIR="$CAPTURE_DIR" OCQA_PR_TARGET_JSON="$IOS_PR_TARGET_JSON" "$ROOT/scripts/quick-capture.sh" explore "$BUNDLE_ID" --actions "$ACTIONS" --timeout "$TIMEOUT"
@@ -37,7 +37,10 @@ for (let i = 2; i < process.argv.length; i += 1) {
37
37
  else if (key === "--json-out") args.jsonOut = value;
38
38
  else if (key === "--md-out") args.mdOut = value;
39
39
  else if (key === "--pr-plan") args.prPlan = value;
40
- else throw new Error(`Unknown argument: ${key}`);
40
+ else {
41
+ console.error(`Unknown argument: ${key}`);
42
+ process.exit(2);
43
+ }
41
44
  }
42
45
  if (!["web", "android"].includes(args.platform)) throw new Error("--platform must be web|android");
43
46
  if (args.platform === "web" && !args.url && !args.projectDir) throw new Error("Web gate requires --url or --project-dir for managed build/start");
@@ -94,7 +97,7 @@ if (args.prPlan) {
94
97
  }
95
98
 
96
99
  let managedRuntime = null;
97
- let exitCode = 1;
100
+ let exitCode = 2;
98
101
  try {
99
102
  if (args.platform === "web" && !args.url) {
100
103
  const started = await startManagedWebTarget({
@@ -106,7 +109,7 @@ try {
106
109
  if (started.error) {
107
110
  console.error(`❌ ${started.error}`);
108
111
  if (started.details?.remediation) console.error(` ${started.details.remediation}`);
109
- process.exitCode = 1;
112
+ exitCode = 2;
110
113
  } else {
111
114
  managedRuntime = started;
112
115
  args.url = started.url;
@@ -114,7 +117,7 @@ try {
114
117
  }
115
118
  }
116
119
  if (args.platform === "web" && !args.url) {
117
- exitCode = 1;
120
+ exitCode = 2;
118
121
  } else {
119
122
  const qa = args.platform === "web"
120
123
  ? await runQaWeb({ url: args.url, maxActions: args.actions, timeout: args.timeout, testEmail: process.env.OCQA_TEST_EMAIL, testPassword: process.env.OCQA_TEST_PASSWORD, seedTargets: prExplorationTargets })
@@ -122,7 +125,7 @@ try {
122
125
  testEmail: process.env.OCQA_TEST_EMAIL, testPassword: process.env.OCQA_TEST_PASSWORD, seedTargets: prExplorationTargets });
123
126
  if (qa.error) {
124
127
  console.error(`❌ ${qa.error}`);
125
- exitCode = 1;
128
+ exitCode = 2;
126
129
  } else {
127
130
  const captureDir = qa.structured.capture.path;
128
131
  const markers = path.join(captureDir, "ocqa-markers.txt");
@@ -177,6 +180,9 @@ try {
177
180
  exitCode = report.status ?? 1;
178
181
  }
179
182
  }
183
+ } catch (error) {
184
+ console.error(`❌ ${error?.message || String(error)}`);
185
+ exitCode = 2;
180
186
  } finally {
181
187
  if (managedRuntime) {
182
188
  await stopManagedWebTarget(managedRuntime);