@aarwitz/tapp 0.16.5 → 0.17.0-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +26 -21
- package/Harness/OCQAHarnessUITests/ExplorerTests.swift +62 -22
- package/README.md +64 -67
- package/bin/tapp.js +168 -45
- package/browser/app.js +12 -6
- package/docs/BROWSER-PRODUCT.md +75 -0
- package/docs/PRODUCT-ENGINE.md +107 -0
- package/docs/application-model.md +271 -0
- package/docs/scenarios.md +95 -0
- package/mcp-server/src/android-driver.js +15 -2
- package/mcp-server/src/android-explorer.js +54 -12
- package/mcp-server/src/application-model.js +9 -8
- package/mcp-server/src/browser-product.js +1 -1
- package/mcp-server/src/ci-report.js +84 -62
- package/mcp-server/src/ci-setup.js +35 -5
- package/mcp-server/src/enrich.js +1 -1
- package/mcp-server/src/html-report.js +20 -6
- package/mcp-server/src/index.js +182 -67
- package/mcp-server/src/pr-selection.js +4 -3
- package/mcp-server/src/product-execution.js +1 -1
- package/mcp-server/src/product-operations.js +2 -2
- package/mcp-server/src/project-config.js +1 -2
- package/mcp-server/src/project-paths.js +5 -17
- package/mcp-server/src/release-contract.js +3 -3
- package/mcp-server/src/report.js +185 -51
- package/mcp-server/src/task-runtime.js +1 -1
- package/mcp-server/src/web-explorer.js +1 -1
- package/package.json +2 -2
- package/scripts/ci-gate.sh +9 -8
- package/scripts/platform-gate.js +11 -5
- package/scripts/quick-capture.sh +72 -38
- package/scripts/run-flow.sh +1 -1
|
@@ -1,32 +1,20 @@
|
|
|
1
|
-
import fs from "node:fs";
|
|
2
1
|
import path from "node:path";
|
|
3
2
|
|
|
4
3
|
export const TAPP_DIRECTORY = ".tapp";
|
|
5
|
-
export const LEGACY_TAPP_DIRECTORY = ".autotap";
|
|
6
4
|
export const TAPP_CONFIG = ".tapp.yml";
|
|
7
|
-
export const LEGACY_TAPP_CONFIG = ".autotap.yml";
|
|
8
5
|
|
|
9
|
-
export function projectArtifactDirectory(
|
|
10
|
-
|
|
11
|
-
if (requested !== TAPP_DIRECTORY) return requested;
|
|
12
|
-
const canonical = path.join(root, TAPP_DIRECTORY);
|
|
13
|
-
const legacy = path.join(root, LEGACY_TAPP_DIRECTORY);
|
|
14
|
-
if (!fs.existsSync(canonical) && fs.existsSync(legacy)) return LEGACY_TAPP_DIRECTORY;
|
|
15
|
-
return TAPP_DIRECTORY;
|
|
6
|
+
export function projectArtifactDirectory(_projectDir, requested = TAPP_DIRECTORY) {
|
|
7
|
+
return requested;
|
|
16
8
|
}
|
|
17
9
|
|
|
18
10
|
export function projectArtifactPath(projectDir, ...parts) {
|
|
19
|
-
return path.join(path.resolve(projectDir),
|
|
11
|
+
return path.join(path.resolve(projectDir), TAPP_DIRECTORY, ...parts);
|
|
20
12
|
}
|
|
21
13
|
|
|
22
14
|
export function existingProjectArtifactPath(projectDir, ...parts) {
|
|
23
|
-
|
|
24
|
-
const canonical = path.join(root, TAPP_DIRECTORY, ...parts);
|
|
25
|
-
if (fs.existsSync(canonical)) return canonical;
|
|
26
|
-
const legacy = path.join(root, LEGACY_TAPP_DIRECTORY, ...parts);
|
|
27
|
-
return fs.existsSync(legacy) ? legacy : canonical;
|
|
15
|
+
return path.join(path.resolve(projectDir), TAPP_DIRECTORY, ...parts);
|
|
28
16
|
}
|
|
29
17
|
|
|
30
18
|
export function isProjectArtifactDirectory(name) {
|
|
31
|
-
return name === TAPP_DIRECTORY
|
|
19
|
+
return name === TAPP_DIRECTORY;
|
|
32
20
|
}
|
|
@@ -11,8 +11,8 @@ import { semanticUiKey } from "./ui-map.js";
|
|
|
11
11
|
|
|
12
12
|
const PLATFORMS = new Set(["ios", "android", "web"]);
|
|
13
13
|
const CRITICALITIES = new Set(["low", "medium", "high", "critical"]);
|
|
14
|
-
const CONTRACT_AUTHORING_SPECIFIERS = new Set(["@aarwitz/tapp/contracts"
|
|
15
|
-
const CONTRACT_AUTHORING_IMPORT = /(["'])
|
|
14
|
+
const CONTRACT_AUTHORING_SPECIFIERS = new Set(["@aarwitz/tapp/contracts"]);
|
|
15
|
+
const CONTRACT_AUTHORING_IMPORT = /(["'])@aarwitz\/tapp\/contracts\1/g;
|
|
16
16
|
const authoringUrl = pathToFileURL(path.resolve(path.dirname(fileURLToPath(import.meta.url)), "contract-authoring.js")).href;
|
|
17
17
|
|
|
18
18
|
function expectationAction(expectation) {
|
|
@@ -86,7 +86,7 @@ function rewriteAuthoringImport(source, contractPath) {
|
|
|
86
86
|
}
|
|
87
87
|
const imports = [...source.matchAll(/(?:from\s*|import\s*)["']([^"']+)["']/g)].map((match) => match[1]);
|
|
88
88
|
const unsupported = imports.filter((specifier) => !CONTRACT_AUTHORING_SPECIFIERS.has(specifier));
|
|
89
|
-
if (unsupported.length) throw new Error(`Release contract imports are limited to @aarwitz/tapp/contracts (
|
|
89
|
+
if (unsupported.length) throw new Error(`Release contract imports are limited to @aarwitz/tapp/contracts (found ${unsupported.join(", ")})`);
|
|
90
90
|
const output = ts.transpileModule(source, {
|
|
91
91
|
fileName: contractPath,
|
|
92
92
|
compilerOptions: { module: ts.ModuleKind.ES2022, target: ts.ScriptTarget.ES2022, verbatimModuleSyntax: true },
|
package/mcp-server/src/report.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// Pure report/gate logic shared by the MCP server (index.js) and the CI gate CLI
|
|
2
|
-
// (ci-report.js). Turns a capture's OCQA markers into
|
|
3
|
-
//
|
|
2
|
+
// (ci-report.js). Turns a capture's OCQA markers into a scoreless exploration observation,
|
|
3
|
+
// and applies explicit policy separately in the CI gate. No shell, no server —
|
|
4
4
|
// keep it dependency-free so the CI path stays importable and testable.
|
|
5
5
|
import fs from "fs";
|
|
6
6
|
import path from "path";
|
|
@@ -106,21 +106,36 @@ export function findingEvaluationTier(finding, platform = "ios") {
|
|
|
106
106
|
return platform === "web" && WEB_SAMPLED_ISSUE_TYPES.has(finding?.type) ? "sampled" : "deterministic";
|
|
107
107
|
}
|
|
108
108
|
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
109
|
+
// The deterministic block-by-findings rule, expressed over verdict-tier finding COUNTS — not the
|
|
110
|
+
// score scalar and not the `verdict` label — so the CI gate survives removal of verdict/releaseScore
|
|
111
|
+
// from exploration output. It encodes exactly what `verdict === "blocked"` used to: a critical
|
|
112
|
+
// finding always blocks; otherwise, on a CONCLUSIVE run, a risk threshold blocks. The risk threshold
|
|
113
|
+
// is kept as an explicit, chosen rule (ADR-0005) — `riskFromCounts` is the single source of that
|
|
114
|
+
// formula, shared with buildQaReport's verdict label. Inconclusive runs are a separate gate outcome,
|
|
115
|
+
// never a findings-block, so a thin run reports `inconclusive`, not `fail`.
|
|
116
|
+
export function riskFromCounts({ critical = 0, high = 0, medium = 0 } = {}) {
|
|
117
|
+
return Math.max(0, Math.min(100, 100 - critical * 25 - high * 10 - medium * 3));
|
|
118
|
+
}
|
|
119
|
+
export function findingsBlock(deterministicFindingCounts = {}, { inconclusive = false } = {}) {
|
|
120
|
+
if ((deterministicFindingCounts.critical || 0) > 0) return true;
|
|
121
|
+
if (inconclusive) return false;
|
|
122
|
+
return riskFromCounts(deterministicFindingCounts) < 50;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
// Exploration OBSERVES; it never renders a ship verdict or score (ADR-0005). These label the
|
|
126
|
+
// observation honestly. The release decision (pass/fail/inconclusive) is the gate's, shown separately.
|
|
127
|
+
export function observationBadge(report) {
|
|
128
|
+
return report?.inconclusive ? "🟡 INCONCLUSIVE (exploration)" : "🔭 EXPLORED";
|
|
112
129
|
}
|
|
113
130
|
|
|
114
|
-
export function
|
|
115
|
-
const
|
|
116
|
-
|
|
117
|
-
if (report?.platform === "web") return "exploratory web · no scalar score";
|
|
118
|
-
return "score unavailable";
|
|
131
|
+
export function observationSummary(report) {
|
|
132
|
+
const n = report?.findingCounts?.total || 0;
|
|
133
|
+
return `${report?.screensExplored || 0} screens · ${report?.actionsPerformed || 0} actions · ${n} finding(s) · observation only`;
|
|
119
134
|
}
|
|
120
135
|
|
|
121
|
-
// Turn a capture's OCQA markers into
|
|
122
|
-
//
|
|
123
|
-
export function buildQaReport(markersFilePath, { platform = "ios" } = {}) {
|
|
136
|
+
// Turn a capture's OCQA markers into a scoreless observation with deduped findings and an
|
|
137
|
+
// explicit coverage floor. Release judgment is applied later by evaluateGate.
|
|
138
|
+
export function buildQaReport(markersFilePath, { platform = "ios", target = null } = {}) {
|
|
124
139
|
const base = parseOcqaMarkers(markersFilePath);
|
|
125
140
|
if (!base) return null;
|
|
126
141
|
|
|
@@ -130,6 +145,7 @@ export function buildQaReport(markersFilePath, { platform = "ios" } = {}) {
|
|
|
130
145
|
const inputsByScreen = new Map();
|
|
131
146
|
const screenElementCounts = {}; // screen -> max elements observed (content-collapse detection)
|
|
132
147
|
let anySecure = false;
|
|
148
|
+
let loginAttempted = false;
|
|
133
149
|
let actions = 0;
|
|
134
150
|
|
|
135
151
|
for (const line of raw.split(/\r?\n/)) {
|
|
@@ -148,6 +164,17 @@ export function buildQaReport(markersFilePath, { platform = "ios" } = {}) {
|
|
|
148
164
|
}
|
|
149
165
|
} else if (t.startsWith("OCQA_ACTION:")) {
|
|
150
166
|
actions += 1;
|
|
167
|
+
try {
|
|
168
|
+
const action = JSON.parse(t.slice("OCQA_ACTION:".length));
|
|
169
|
+
if (action?.type === "login" || String(action?.type || "").startsWith("login_")) loginAttempted = true;
|
|
170
|
+
} catch {
|
|
171
|
+
/* ignore malformed */
|
|
172
|
+
}
|
|
173
|
+
} else if (
|
|
174
|
+
t === "OCQA_STATE:login_preamble_submitted" ||
|
|
175
|
+
t === "OCQA_STATE:login_preamble_two_step_submitted"
|
|
176
|
+
) {
|
|
177
|
+
loginAttempted = true;
|
|
151
178
|
} else if (t.startsWith("OCQA_STATE:{")) {
|
|
152
179
|
try {
|
|
153
180
|
const s = JSON.parse(t.slice("OCQA_STATE:".length));
|
|
@@ -187,7 +214,12 @@ export function buildQaReport(markersFilePath, { platform = "ios" } = {}) {
|
|
|
187
214
|
const missingResources = new Set(normalizedIssues
|
|
188
215
|
.filter((issue) => issue.type === "missing_asset" && issue.target)
|
|
189
216
|
.map((issue) => issue.target));
|
|
217
|
+
// Older native captures represented the caller's wall-clock budget as a high-severity app
|
|
218
|
+
// finding. A timeout makes the evidence partial/inconclusive; it does not prove an app
|
|
219
|
+
// performance defect (the harness has a separate app_hang detector for that).
|
|
220
|
+
const timeBudgetExhausted = base.complete?.timedOut === true || normalizedIssues.some((issue) => issue.type === "explore_timeout");
|
|
190
221
|
const reportIssues = normalizedIssues.filter((issue) =>
|
|
222
|
+
issue.type !== "explore_timeout" &&
|
|
191
223
|
!(issue.type === "network_error" && issue.target && missingResources.has(issue.target) && /^Request failed:/i.test(String(issue.title || ""))));
|
|
192
224
|
|
|
193
225
|
// Dedup by stable signature (type|screen|target) so repeated detections count once —
|
|
@@ -201,14 +233,23 @@ export function buildQaReport(markersFilePath, { platform = "ios" } = {}) {
|
|
|
201
233
|
findings.push({
|
|
202
234
|
...i,
|
|
203
235
|
category: ISSUE_CATEGORY[i.type] || i.type,
|
|
236
|
+
// Structural evidence authority (ADR-0005): marker-derived findings are deterministic. The
|
|
237
|
+
// default gate consumes only deterministic-authority evidence; model-observed findings
|
|
238
|
+
// (vision/assert_ai) carry authority:"model-observed" and are advisory, never gate fails.
|
|
239
|
+
authority: "deterministic",
|
|
204
240
|
...(platform === "web" ? { evaluationTier: findingEvaluationTier(i, platform) } : {}),
|
|
205
241
|
});
|
|
206
242
|
}
|
|
207
243
|
findings.sort((a, b) => severityRank(a.severity) - severityRank(b.severity));
|
|
208
244
|
|
|
209
245
|
const screensExplored = screens.size || base.uniqueScreens.length;
|
|
210
|
-
|
|
211
|
-
|
|
246
|
+
// Some native recovery operations are counted by the harness budget but intentionally do not
|
|
247
|
+
// emit a public action narrative. Preserve the larger authoritative completion count instead of
|
|
248
|
+
// understating coverage whenever at least one narrated action exists.
|
|
249
|
+
const actionsPerformed = Math.max(
|
|
250
|
+
actions,
|
|
251
|
+
base.complete && typeof base.complete === "object" ? base.complete.actions || 0 : 0,
|
|
252
|
+
);
|
|
212
253
|
const crit = findings.filter((f) => f.severity === "critical").length;
|
|
213
254
|
const high = findings.filter((f) => f.severity === "high").length;
|
|
214
255
|
const med = findings.filter((f) => f.severity === "medium").length;
|
|
@@ -222,29 +263,33 @@ export function buildQaReport(markersFilePath, { platform = "ios" } = {}) {
|
|
|
222
263
|
const verdictLow = verdictFindings.filter((f) => f.severity === "low").length;
|
|
223
264
|
const sampledFindings = platform === "web" ? findings.filter((finding) => finding.evaluationTier === "sampled") : [];
|
|
224
265
|
|
|
225
|
-
// Coverage floor:
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
const
|
|
266
|
+
// Coverage floor: exploration is inconclusive if the app wasn't actually exercised. Exploration
|
|
267
|
+
// OBSERVES — it does not render a ship verdict or score (ADR-0005). Judgment (pass/fail/
|
|
268
|
+
// inconclusive) is the gate's job (evaluateGate), computed from these findings + coverage + policy.
|
|
269
|
+
// A one-page web target can still be swept exhaustively: page errors, requests, links, assets,
|
|
270
|
+
// placeholder anchors, and visible controls do not require a second route. Native exploration
|
|
271
|
+
// retains the stronger multi-screen/action floor. A credentialless single-screen login remains
|
|
272
|
+
// inconclusive so a login wall can never turn into a clean pass.
|
|
273
|
+
const coverageFloorMet = platform === "web"
|
|
274
|
+
? screensExplored >= 1 && actionsPerformed >= 1
|
|
275
|
+
: screensExplored >= 2 && actionsPerformed >= 3;
|
|
276
|
+
const credentiallessLoginWall = anySecure && !loginAttempted && screensExplored <= 1;
|
|
277
|
+
const inconclusive = !coverageFloorMet || credentiallessLoginWall || timeBudgetExhausted;
|
|
278
|
+
const stopReason = credentiallessLoginWall ? "login-wall-no-credentials"
|
|
279
|
+
: timeBudgetExhausted ? "time-budget-exhausted"
|
|
280
|
+
: coverageFloorMet ? "completed" : "coverage-floor-not-met";
|
|
281
|
+
|
|
282
|
+
const headline = timeBudgetExhausted
|
|
283
|
+
? `Inconclusive — exploration reached its ${base.complete?.timeoutSeconds || "configured"}s time budget after ${actionsPerformed} action(s) across ${screensExplored} screen(s). Findings are partial; this is not an app performance finding. Increase --timeout or request fewer actions.`
|
|
284
|
+
: inconclusive
|
|
238
285
|
? `Inconclusive — only ${screensExplored} screen(s) / ${actionsPerformed} action(s) explored. The app may have crashed on launch, be stuck behind a sign-in wall, or otherwise prevent exploration. Absence of issues is NOT a pass.`
|
|
239
|
-
:
|
|
286
|
+
: findings.length === 0
|
|
240
287
|
? platform === "web"
|
|
241
|
-
? "Automated web checks completed — no
|
|
242
|
-
: "
|
|
243
|
-
:
|
|
244
|
-
? `Proceed with caution — ${findings.length} issue(s) to review.`
|
|
245
|
-
: `Not ready — ${findings.length} issue(s): ${crit} critical, ${high} high, ${med} medium, ${low} low.`;
|
|
288
|
+
? "Automated web checks completed — no deterministic findings in the exercised surfaces. Sampled control probes are advisory. An observation, not a release decision, and not a content, privacy, brand, or business-claim review."
|
|
289
|
+
: "No issues surfaced in the exercised surfaces. An observation, not a release decision."
|
|
290
|
+
: `${findings.length} issue(s) surfaced for review (${crit} critical, ${high} high, ${med} medium, ${low} low). An observation, not a release decision.`;
|
|
246
291
|
|
|
247
|
-
// The
|
|
292
|
+
// The observation's honesty label: exactly which defect classes this run checked, which
|
|
248
293
|
// it structurally could NOT check, and which conditions never came up — so "checked" is
|
|
249
294
|
// never claimed for a state the run didn't reach. Platform-aware: a web run doesn't
|
|
250
295
|
// inherit iOS keyboard assertions and vice versa.
|
|
@@ -290,30 +335,38 @@ export function buildQaReport(markersFilePath, { platform = "ios" } = {}) {
|
|
|
290
335
|
"content & reachability regressions require a baseline" ,
|
|
291
336
|
];
|
|
292
337
|
}
|
|
293
|
-
// "Failed sign-ins" is only a claim when
|
|
294
|
-
|
|
338
|
+
// "Failed sign-ins" is only a claim when credentials were actually submitted. Merely seeing a
|
|
339
|
+
// password field proves that a login surface was reached, not that authentication was exercised.
|
|
340
|
+
if (loginAttempted) checkedFor.splice(2, 0, "failed sign-ins");
|
|
341
|
+
else if (anySecure) notChecked.push("sign-in behavior (login form reached, no test credentials supplied)");
|
|
295
342
|
else conditionsNotReached.push("sign-in (no login form encountered this run)");
|
|
343
|
+
if (timeBudgetExhausted) notChecked.push("the full requested action budget (run reached its wall-clock timeout)");
|
|
296
344
|
|
|
297
345
|
return {
|
|
298
|
-
verdict
|
|
299
|
-
//
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
346
|
+
// An ExplorationRun observation: findings + coverage + evidence, NO ship verdict or score
|
|
347
|
+
// (ADR-0005). The gate (evaluateGate) turns this into a pass/fail/inconclusive release outcome.
|
|
348
|
+
kind: "tapp-exploration-run",
|
|
349
|
+
schemaVersion: 1,
|
|
350
|
+
// Complete ExplorationRun contract (ADR-0005 §4). runStatus/stopReason describe HOW the run
|
|
351
|
+
// ended; coverage/evidence/uiMap/comparison are the structured observation. uiMap and comparison
|
|
352
|
+
// are populated by consumers that build the map / diff a baseline (null in the bare observation).
|
|
353
|
+
runStatus: inconclusive ? "limited" : "completed",
|
|
354
|
+
stopReason,
|
|
307
355
|
headline,
|
|
308
356
|
inconclusive,
|
|
357
|
+
coverage: { screensExplored, actionsPerformed, screens: Array.from(screens) },
|
|
358
|
+
evidence: { markers: base.relativeMarkersFilePath },
|
|
359
|
+
uiMap: null,
|
|
360
|
+
comparison: null,
|
|
309
361
|
checkedFor,
|
|
310
362
|
notChecked,
|
|
311
363
|
conditionsNotReached,
|
|
312
364
|
platform,
|
|
365
|
+
target: typeof target === "string" && target.trim() ? target.trim() : null,
|
|
313
366
|
screensExplored,
|
|
314
367
|
actionsPerformed,
|
|
315
368
|
findingCounts: { critical: crit, high, medium: med, low, total: findings.length },
|
|
316
|
-
|
|
369
|
+
deterministicFindingCounts: {
|
|
317
370
|
critical: verdictCrit,
|
|
318
371
|
high: verdictHigh,
|
|
319
372
|
medium: verdictMed,
|
|
@@ -355,6 +408,7 @@ export function computeContentCollapse(currentCounts, baselineCounts) {
|
|
|
355
408
|
type: "content_collapse",
|
|
356
409
|
severity: "high",
|
|
357
410
|
category: "content_collapse",
|
|
411
|
+
authority: "deterministic",
|
|
358
412
|
title: `Screen lost most of its content (${base} → ${cur} elements)`,
|
|
359
413
|
screen,
|
|
360
414
|
step: null,
|
|
@@ -383,6 +437,7 @@ export function computeReachabilityLoss(current, baseline) {
|
|
|
383
437
|
type: "screen_unreachable",
|
|
384
438
|
severity: "high",
|
|
385
439
|
category: "navigation_dead_end",
|
|
440
|
+
authority: "deterministic",
|
|
386
441
|
title: "Screen explored in the baseline was never reached this run",
|
|
387
442
|
screen,
|
|
388
443
|
step: null,
|
|
@@ -421,15 +476,94 @@ export function computeRegression(current, baseline) {
|
|
|
421
476
|
const newFindings = current.filter((f) => !currentMatches(f));
|
|
422
477
|
const persisting = current.filter((f) => currentMatches(f));
|
|
423
478
|
const resolved = baseline.filter((b) => !baselineMatched(b));
|
|
424
|
-
const newCritical = newFindings.filter((f) => f.severity === "critical").length;
|
|
425
|
-
const newHigh = newFindings.filter((f) => f.severity === "high").length;
|
|
426
479
|
|
|
480
|
+
// Comparison ONLY — no gate/pass/fail signal (ADR-0005). Exploration surfaces this diff; the merge
|
|
481
|
+
// decision is the gate's job. evaluateGate derives its regression fail from `newFindings` severities.
|
|
427
482
|
return {
|
|
428
483
|
hadBaseline: true,
|
|
429
484
|
counts: { new: newFindings.length, persisting: persisting.length, resolved: resolved.length },
|
|
430
485
|
newFindings,
|
|
431
486
|
resolved,
|
|
432
|
-
|
|
433
|
-
|
|
487
|
+
};
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
// The gate's public outcome model (ADR-0005). A merge gate is ultimately block / don't-block, but
|
|
491
|
+
// callers need to distinguish WHY: a deterministic violation is not the same as "we couldn't get
|
|
492
|
+
// the evidence." Exit codes are the CI contract; precedence is fail > inconclusive > pass.
|
|
493
|
+
export const GATE_EXIT = { pass: 0, fail: 1, error: 2, inconclusive: 3 };
|
|
494
|
+
// Bump when the gate's decision semantics change (NOT the npm version). Recorded on every GateRun.
|
|
495
|
+
export const GATE_POLICY_VERSION = "2";
|
|
496
|
+
|
|
497
|
+
// Pure gate evaluator: frozen evidence + policy → a GateRun decision. Extracted verbatim from the
|
|
498
|
+
// former inline logic in ci-report.js so the `[char]` characterization tests keep passing — the
|
|
499
|
+
// merge decision (block/don't-block) is unchanged; this only classifies each reason as a
|
|
500
|
+
// deterministic `fail` or an evidence-absent `inconclusive` and folds them by precedence. Reason
|
|
501
|
+
// MESSAGES are preserved exactly (several are asserted by tests).
|
|
502
|
+
//
|
|
503
|
+
// DECOUPLED FROM THE SCORE (ADR-0005): the block-by-findings decision reads `deterministicFindingCounts` +
|
|
504
|
+
// `inconclusive` via `findingsBlock`, NOT the score scalar or the `verdict` label. `verdict`/
|
|
505
|
+
// `releaseScore` can therefore be removed from exploration output without changing any merge
|
|
506
|
+
// decision. The `riskScore < 50` threshold is retained deliberately (kept explicit, inside
|
|
507
|
+
// `findingsBlock`) and locked by the `[char]` risk-threshold test.
|
|
508
|
+
export function evaluateGate({ report, regression = null, flows = [], scenarios = [], contracts = [], prPlan = null, baseline = null, failOn = "gate" } = {}) {
|
|
509
|
+
const reasons = []; // { kind: "fail" | "inconclusive", message }
|
|
510
|
+
const fail = (message) => reasons.push({ kind: "fail", message });
|
|
511
|
+
const inconclusive = (message) => reasons.push({ kind: "inconclusive", message });
|
|
512
|
+
|
|
513
|
+
// Classify each replayed suite by evidence authority (ADR-0005): a DETERMINISTIC step failure (or
|
|
514
|
+
// a non-model failure like an aborted/missing run) is a real fail; a suite with no deterministic
|
|
515
|
+
// failure that carries a model-observed (assert_ai) assertion cannot be decided by the default
|
|
516
|
+
// deterministic gate → inconclusive/needs-review (it must not silently pass, and a model verdict
|
|
517
|
+
// must not masquerade as a deterministic fail). No `--policy probabilistic` opt-in in 0.17.
|
|
518
|
+
const deterministicFail = (s) => s.deterministicFailed === true || (s.passed === false && !s.modelObserved);
|
|
519
|
+
const classify = (s) => (deterministicFail(s) ? "fail" : s.modelObserved ? "needs-review" : "pass");
|
|
520
|
+
for (const [label, suites] of [["flow", flows], ["multi-actor scenario", scenarios], ["release contract", contracts]]) {
|
|
521
|
+
const failed = suites.filter((s) => classify(s) === "fail");
|
|
522
|
+
if (failed.length) fail(`${failed.length} ${label}(s) failed`);
|
|
523
|
+
const needsReview = suites.filter((s) => classify(s) === "needs-review");
|
|
524
|
+
if (needsReview.length) inconclusive(`${needsReview.length} ${label}(s) contain assert_ai (model-observed); the deterministic gate cannot decide them — review, or add an explicit probabilistic policy`);
|
|
525
|
+
}
|
|
526
|
+
// A submitted sign-in that remains on the login surface is an explicit exercised guarantee,
|
|
527
|
+
// not ordinary pre-existing UI debt. Letting it pass without a baseline would produce the
|
|
528
|
+
// contradictory public result "failed sign-in detected" + gate PASS. Keep sampled probes out,
|
|
529
|
+
// but fail every deterministic auth failure under every gate policy (including with a baseline).
|
|
530
|
+
const authFailures = (report.findings || []).filter((finding) =>
|
|
531
|
+
finding?.type === "auth_failed" &&
|
|
532
|
+
finding?.authority !== "model-observed" &&
|
|
533
|
+
finding?.evaluationTier !== "sampled"
|
|
534
|
+
);
|
|
535
|
+
if (authFailures.length) fail(`${authFailures.length} deterministic sign-in attempt(s) failed`);
|
|
536
|
+
// Selected-but-unexecuted work is missing evidence, not an observed violation → inconclusive.
|
|
537
|
+
if (prPlan?.execution?.notRun) inconclusive(`${prPlan.execution.notRun} selected release contract(s) did not run`);
|
|
538
|
+
if (prPlan?.execution?.explorationFailed) inconclusive(`${prPlan.execution.explorationFailed} planned PR exploration target(s) failed or were not reached`);
|
|
539
|
+
|
|
540
|
+
if (failOn === "any") {
|
|
541
|
+
if (report.findingCounts.total > 0) fail(`${report.findingCounts.total} finding(s) (fail-on: any)`);
|
|
542
|
+
// "any" is the strictest policy — an inconclusive run (evidence not obtained) must never pass it.
|
|
543
|
+
if (report.inconclusive) inconclusive("run was inconclusive (coverage floor not met)");
|
|
544
|
+
} else if (failOn === "absolute" || (failOn === "gate" && !regression)) {
|
|
545
|
+
if (findingsBlock(report.deterministicFindingCounts, { inconclusive: report.inconclusive })) fail("blocking deterministic finding(s)");
|
|
546
|
+
if (report.inconclusive) inconclusive("run was inconclusive (coverage floor not met)");
|
|
547
|
+
} else {
|
|
548
|
+
if (regression?.newFindings?.length) {
|
|
549
|
+
const newCritical = regression.newFindings.filter((f) => f.severity === "critical").length;
|
|
550
|
+
const newHigh = regression.newFindings.filter((f) => f.severity === "high").length;
|
|
551
|
+
if (newCritical + newHigh > 0) fail(`${newCritical} new critical + ${newHigh} new high vs. baseline`);
|
|
552
|
+
}
|
|
553
|
+
if (findingsBlock(report.deterministicFindingCounts, { inconclusive: report.inconclusive })) fail("blocking deterministic finding(s)");
|
|
554
|
+
if (report.inconclusive && !baseline?.inconclusive) inconclusive("run became inconclusive vs. baseline (app may no longer launch/explore)");
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
const outcome = reasons.some((r) => r.kind === "fail") ? "fail"
|
|
558
|
+
: reasons.some((r) => r.kind === "inconclusive") ? "inconclusive"
|
|
559
|
+
: "pass";
|
|
560
|
+
return {
|
|
561
|
+
policy: failOn,
|
|
562
|
+
outcome,
|
|
563
|
+
exitCode: GATE_EXIT[outcome],
|
|
564
|
+
failed: outcome !== "pass", // retained for markdown/JSON consumers during migration
|
|
565
|
+
reasons: reasons.map((r) => r.message),
|
|
566
|
+
reasonDetails: reasons,
|
|
567
|
+
policyVersion: GATE_POLICY_VERSION,
|
|
434
568
|
};
|
|
435
569
|
}
|
|
@@ -109,7 +109,7 @@ export function loadTaskRegistry({ sourcePath, projectDir = "", taskFiles = [] }
|
|
|
109
109
|
// Draft contracts generated under `.tapp/proposals/contracts` may compile
|
|
110
110
|
// against sibling untrusted Task drafts. Ordinary committed contracts never
|
|
111
111
|
// see this directory, so a proposal cannot silently enter the release gate.
|
|
112
|
-
const proposalSource =
|
|
112
|
+
const proposalSource = String(path.resolve(sourcePath || "")).includes(`${path.sep}.tapp${path.sep}proposals${path.sep}`);
|
|
113
113
|
const proposalDir = proposalSource && tappDir ? path.join(tappDir, "proposals", "tasks") : "";
|
|
114
114
|
const proposed = proposalDir && fs.existsSync(proposalDir)
|
|
115
115
|
? fs.readdirSync(proposalDir).filter((name) => /\.ya?ml$|\.json$/i.test(name)).map((name) => path.join(proposalDir, name))
|
|
@@ -45,7 +45,7 @@ export function webPlaceholderLinkFindings(links = []) {
|
|
|
45
45
|
const seen = new Set();
|
|
46
46
|
for (const link of links || []) {
|
|
47
47
|
const rawHref = String(link?.rawHref || "").trim().toLowerCase();
|
|
48
|
-
const placeholder = rawHref === "#" || /^javascript:(?:void\(0\);?|;?)$/.test(rawHref);
|
|
48
|
+
const placeholder = rawHref === "" || rawHref === "#" || /^javascript:(?:void\(0\);?|;?)$/.test(rawHref);
|
|
49
49
|
if (!placeholder || link?.handlerHint) continue;
|
|
50
50
|
const label = String(link?.label || "").replace(/\s+/g, " ").trim().slice(0, 100);
|
|
51
51
|
const fingerprint = String(link?.fingerprint || "link").replace(/\s+/g, " ").trim().slice(0, 100) || "link";
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aarwitz/tapp",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.17.0-rc.2",
|
|
4
4
|
"mcpName": "io.github.aarwitz/tapp",
|
|
5
5
|
"description": "Release contracts, autonomous QA, and evidence-backed CI gates for iOS, Android, and web.",
|
|
6
6
|
"license": "MIT",
|
|
@@ -87,7 +87,7 @@
|
|
|
87
87
|
"mobile"
|
|
88
88
|
],
|
|
89
89
|
"scripts": {
|
|
90
|
-
"test": "node --test tests/report.test.js tests/regression.test.js tests/engine.test.js tests/project-config.test.js tests/application-model.test.js tests/ui-map.test.js tests/task-runtime.test.js tests/release-contract.test.js tests/pr-selection.test.js tests/flow-runtime.test.js tests/web-explorer.test.js tests/web-session.test.js tests/web-flow.test.js tests/scenario-runtime.test.js tests/android-driver.test.js tests/android-explorer.test.js tests/android-flow.test.js tests/managed-web.test.js tests/product-operations.test.js tests/browser-product.test.js tests/browser-onboarding.test.js tests/managed-operation.test.js tests/ci-setup.test.js tests/ci-install.test.js tests/cli.test.js tests/action.test.js tests/package-surface.test.js tests/ci-gate.test.js tests/ci-report.test.js tests/desktop-protocol.test.js tests/ios-flow-protocol.test.js",
|
|
90
|
+
"test": "node --test tests/report.test.js tests/regression.test.js tests/engine.test.js tests/project-config.test.js tests/application-model.test.js tests/ui-map.test.js tests/task-runtime.test.js tests/release-contract.test.js tests/pr-selection.test.js tests/flow-runtime.test.js tests/web-explorer.test.js tests/web-session.test.js tests/web-flow.test.js tests/scenario-runtime.test.js tests/android-driver.test.js tests/android-explorer.test.js tests/android-flow.test.js tests/android-primitives-protocol.test.js tests/managed-web.test.js tests/product-operations.test.js tests/browser-product.test.js tests/browser-onboarding.test.js tests/managed-operation.test.js tests/cloud-runner.test.js tests/ci-setup.test.js tests/ci-install.test.js tests/cli.test.js tests/action.test.js tests/package-surface.test.js tests/ci-gate.test.js tests/ci-report.test.js tests/desktop-protocol.test.js tests/ios-flow-protocol.test.js",
|
|
91
91
|
"test:browser-journey": "node --test tests/browser-journey.test.js",
|
|
92
92
|
"test:browser-native": "TAPP_RUN_NATIVE_BROWSER=1 node --test tests/browser-native-journey.test.js"
|
|
93
93
|
}
|
package/scripts/ci-gate.sh
CHANGED
|
@@ -3,9 +3,11 @@
|
|
|
3
3
|
#
|
|
4
4
|
# Boots a simulator if needed, installs the app build under test, runs the autonomous exploration
|
|
5
5
|
# harness, replays every committed Flow (deterministic E2E tests), diffs the findings against a
|
|
6
|
-
# stored baseline, writes a GitHub Actions step summary, and exits
|
|
7
|
-
#
|
|
8
|
-
#
|
|
6
|
+
# stored baseline, writes a GitHub Actions step summary, and exits with the public outcome contract:
|
|
7
|
+
# pass 0, deterministic fail 1, infrastructure/usage error 2, inconclusive evidence 3. Absolute
|
|
8
|
+
# blockers and reviewed Flow/Scenario/Contract failures are enforced even without a baseline;
|
|
9
|
+
# baseline comparisons additionally catch new high/critical regressions. Wrapped by ../action.yml
|
|
10
|
+
# for GitHub Actions; equally usable from any other CI or locally.
|
|
9
11
|
#
|
|
10
12
|
# Usage:
|
|
11
13
|
# scripts/ci-gate.sh [--platform ios] --app <path/to/App.app> [--bundle-id <com.example.app>]
|
|
@@ -25,7 +27,7 @@
|
|
|
25
27
|
# [--pr-plan-out <file.json>] # persist the reviewable selection plan
|
|
26
28
|
# [--baseline <file.json>] # prior report to diff against (skipped if absent)
|
|
27
29
|
# [--target-key <stable-id>] # isolates target-specific baselines in monorepos
|
|
28
|
-
# [--fail-on gate|
|
|
30
|
+
# [--fail-on gate|absolute|any] # gate policy (default gate; see ci-report.js)
|
|
29
31
|
# [--json-out <file.json>] # write the full report (use as the next baseline)
|
|
30
32
|
# [--md-out <file.md>] # write the rendered markdown report (for a PR comment)
|
|
31
33
|
# [--device <name>] # simulator device to boot if none is (default "iPhone 16 Pro")
|
|
@@ -35,7 +37,7 @@ set -uo pipefail
|
|
|
35
37
|
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
|
36
38
|
|
|
37
39
|
usage() {
|
|
38
|
-
sed -n '2
|
|
40
|
+
sed -n '2,/^set -uo pipefail$/p' "$0" | sed '$d; s/^# \{0,1\}//'
|
|
39
41
|
}
|
|
40
42
|
|
|
41
43
|
PLATFORM="ios" APP_PATH="" BUNDLE_ID="" APK_PATH="" APP_ID="" URL="" WEB_TARGET="" TARGET_KEY="" SERIAL="" ACTIONS=40 TIMEOUT=600 FLOWS="" SCENARIOS="" CONTRACTS="" PROJECT_DIR="" BASELINE="" FAIL_ON="gate" JSON_OUT="" MD_OUT="" DEVICE="iPhone 16 Pro" PR_BASE="" PR_HEAD="HEAD" CHANGED_FILES_FILE="" PR_PLAN_OUT=""
|
|
@@ -74,7 +76,7 @@ done
|
|
|
74
76
|
[[ "$PLATFORM" == "ios" || "$PLATFORM" == "android" || "$PLATFORM" == "web" ]] || { echo "❌ --platform must be ios|android|web" >&2; exit 2; }
|
|
75
77
|
[[ "$ACTIONS" =~ ^[1-9][0-9]*$ ]] || { echo "❌ --actions must be a positive integer" >&2; exit 2; }
|
|
76
78
|
[[ "$TIMEOUT" =~ ^[1-9][0-9]*$ ]] || { echo "❌ --timeout must be a positive integer" >&2; exit 2; }
|
|
77
|
-
[[ "$FAIL_ON" == "gate" || "$FAIL_ON" == "
|
|
79
|
+
[[ "$FAIL_ON" == "gate" || "$FAIL_ON" == "absolute" || "$FAIL_ON" == "any" ]] || { echo "❌ --fail-on must be gate|absolute|any" >&2; exit 2; }
|
|
78
80
|
if [[ -n "$PROJECT_DIR" ]]; then
|
|
79
81
|
[[ -d "$PROJECT_DIR" ]] || { echo "❌ Project directory not found: $PROJECT_DIR" >&2; exit 2; }
|
|
80
82
|
PROJECT_DIR="$(cd "$PROJECT_DIR" && pwd)"
|
|
@@ -82,7 +84,6 @@ fi
|
|
|
82
84
|
TAPP_PROJECT_ARTIFACTS=""
|
|
83
85
|
if [[ -n "$PROJECT_DIR" ]]; then
|
|
84
86
|
[[ -d "$PROJECT_DIR/.tapp" ]] && TAPP_PROJECT_ARTIFACTS="$PROJECT_DIR/.tapp"
|
|
85
|
-
[[ -z "$TAPP_PROJECT_ARTIFACTS" && -d "$PROJECT_DIR/.autotap" ]] && TAPP_PROJECT_ARTIFACTS="$PROJECT_DIR/.autotap"
|
|
86
87
|
fi
|
|
87
88
|
[[ -z "$FLOWS" && -n "$TAPP_PROJECT_ARTIFACTS" && -d "$TAPP_PROJECT_ARTIFACTS/flows" ]] && FLOWS="$TAPP_PROJECT_ARTIFACTS/flows/*.yml"
|
|
88
89
|
[[ "$PLATFORM" == "web" && -z "$SCENARIOS" && -n "$TAPP_PROJECT_ARTIFACTS" && -d "$TAPP_PROJECT_ARTIFACTS/scenarios" ]] && SCENARIOS="$TAPP_PROJECT_ARTIFACTS/scenarios/*.yml"
|
|
@@ -274,7 +275,7 @@ xcrun simctl install "$UDID" "$APP_PATH" || { echo "❌ simctl install failed
|
|
|
274
275
|
# ── Autonomous exploration (quick-capture builds the harness itself if needed).
|
|
275
276
|
step "Explore ($ACTIONS actions, ${TIMEOUT}s watchdog)"
|
|
276
277
|
set +e
|
|
277
|
-
CAPTURE_ROOT="${TAPP_HOME:-$
|
|
278
|
+
CAPTURE_ROOT="${TAPP_HOME:-$ROOT}/captures"
|
|
278
279
|
mkdir -p "$CAPTURE_ROOT"
|
|
279
280
|
CAPTURE_DIR="$(mktemp -d "$CAPTURE_ROOT/ci.XXXXXX")"
|
|
280
281
|
TAPP_CAPTURE_DIR="$CAPTURE_DIR" OCQA_PR_TARGET_JSON="$IOS_PR_TARGET_JSON" "$ROOT/scripts/quick-capture.sh" explore "$BUNDLE_ID" --actions "$ACTIONS" --timeout "$TIMEOUT"
|
package/scripts/platform-gate.js
CHANGED
|
@@ -37,7 +37,10 @@ for (let i = 2; i < process.argv.length; i += 1) {
|
|
|
37
37
|
else if (key === "--json-out") args.jsonOut = value;
|
|
38
38
|
else if (key === "--md-out") args.mdOut = value;
|
|
39
39
|
else if (key === "--pr-plan") args.prPlan = value;
|
|
40
|
-
else
|
|
40
|
+
else {
|
|
41
|
+
console.error(`Unknown argument: ${key}`);
|
|
42
|
+
process.exit(2);
|
|
43
|
+
}
|
|
41
44
|
}
|
|
42
45
|
if (!["web", "android"].includes(args.platform)) throw new Error("--platform must be web|android");
|
|
43
46
|
if (args.platform === "web" && !args.url && !args.projectDir) throw new Error("Web gate requires --url or --project-dir for managed build/start");
|
|
@@ -94,7 +97,7 @@ if (args.prPlan) {
|
|
|
94
97
|
}
|
|
95
98
|
|
|
96
99
|
let managedRuntime = null;
|
|
97
|
-
let exitCode =
|
|
100
|
+
let exitCode = 2;
|
|
98
101
|
try {
|
|
99
102
|
if (args.platform === "web" && !args.url) {
|
|
100
103
|
const started = await startManagedWebTarget({
|
|
@@ -106,7 +109,7 @@ try {
|
|
|
106
109
|
if (started.error) {
|
|
107
110
|
console.error(`❌ ${started.error}`);
|
|
108
111
|
if (started.details?.remediation) console.error(` ${started.details.remediation}`);
|
|
109
|
-
|
|
112
|
+
exitCode = 2;
|
|
110
113
|
} else {
|
|
111
114
|
managedRuntime = started;
|
|
112
115
|
args.url = started.url;
|
|
@@ -114,7 +117,7 @@ try {
|
|
|
114
117
|
}
|
|
115
118
|
}
|
|
116
119
|
if (args.platform === "web" && !args.url) {
|
|
117
|
-
exitCode =
|
|
120
|
+
exitCode = 2;
|
|
118
121
|
} else {
|
|
119
122
|
const qa = args.platform === "web"
|
|
120
123
|
? await runQaWeb({ url: args.url, maxActions: args.actions, timeout: args.timeout, testEmail: process.env.OCQA_TEST_EMAIL, testPassword: process.env.OCQA_TEST_PASSWORD, seedTargets: prExplorationTargets })
|
|
@@ -122,7 +125,7 @@ try {
|
|
|
122
125
|
testEmail: process.env.OCQA_TEST_EMAIL, testPassword: process.env.OCQA_TEST_PASSWORD, seedTargets: prExplorationTargets });
|
|
123
126
|
if (qa.error) {
|
|
124
127
|
console.error(`❌ ${qa.error}`);
|
|
125
|
-
exitCode =
|
|
128
|
+
exitCode = 2;
|
|
126
129
|
} else {
|
|
127
130
|
const captureDir = qa.structured.capture.path;
|
|
128
131
|
const markers = path.join(captureDir, "ocqa-markers.txt");
|
|
@@ -177,6 +180,9 @@ try {
|
|
|
177
180
|
exitCode = report.status ?? 1;
|
|
178
181
|
}
|
|
179
182
|
}
|
|
183
|
+
} catch (error) {
|
|
184
|
+
console.error(`❌ ${error?.message || String(error)}`);
|
|
185
|
+
exitCode = 2;
|
|
180
186
|
} finally {
|
|
181
187
|
if (managedRuntime) {
|
|
182
188
|
await stopManagedWebTarget(managedRuntime);
|