@aarwitz/tapp 0.16.5 → 0.17.0-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +26 -21
- package/Harness/OCQAHarnessUITests/ExplorerTests.swift +62 -22
- package/README.md +64 -67
- package/bin/tapp.js +168 -45
- package/browser/app.js +12 -6
- package/docs/BROWSER-PRODUCT.md +75 -0
- package/docs/PRODUCT-ENGINE.md +107 -0
- package/docs/application-model.md +271 -0
- package/docs/scenarios.md +95 -0
- package/mcp-server/src/android-driver.js +15 -2
- package/mcp-server/src/android-explorer.js +54 -12
- package/mcp-server/src/application-model.js +9 -8
- package/mcp-server/src/browser-product.js +1 -1
- package/mcp-server/src/ci-report.js +84 -62
- package/mcp-server/src/ci-setup.js +35 -5
- package/mcp-server/src/enrich.js +1 -1
- package/mcp-server/src/html-report.js +20 -6
- package/mcp-server/src/index.js +182 -67
- package/mcp-server/src/pr-selection.js +4 -3
- package/mcp-server/src/product-execution.js +1 -1
- package/mcp-server/src/product-operations.js +2 -2
- package/mcp-server/src/project-config.js +1 -2
- package/mcp-server/src/project-paths.js +5 -17
- package/mcp-server/src/release-contract.js +3 -3
- package/mcp-server/src/report.js +185 -51
- package/mcp-server/src/task-runtime.js +1 -1
- package/mcp-server/src/web-explorer.js +1 -1
- package/package.json +2 -2
- package/scripts/ci-gate.sh +9 -8
- package/scripts/platform-gate.js +11 -5
- package/scripts/quick-capture.sh +72 -38
- package/scripts/run-flow.sh +1 -1
|
@@ -16,17 +16,19 @@
|
|
|
16
16
|
// [--pr-plan <plan.json>] # selected PR contract execution manifest
|
|
17
17
|
// [--project-dir <repo> --maintenance-url <url>]
|
|
18
18
|
// # optional disposable web patch replay
|
|
19
|
-
// [--fail-on <gate|
|
|
19
|
+
// [--fail-on <gate|absolute|any>] # default: gate
|
|
20
20
|
//
|
|
21
21
|
// Gate policy (--fail-on):
|
|
22
22
|
// gate fail when the run introduced NEW high/critical findings vs. the baseline
|
|
23
|
-
// (no baseline ⇒ falls back to `
|
|
24
|
-
// pre-existing debt doesn't block, regressions and broken
|
|
25
|
-
//
|
|
26
|
-
//
|
|
23
|
+
// (no baseline ⇒ falls back to `absolute`), or when any suite failed. The default:
|
|
24
|
+
// pre-existing debt doesn't block, regressions and broken suites do.
|
|
25
|
+
// absolute fail on any current-run deterministic findings-block (critical / risk threshold) or an
|
|
26
|
+
// inconclusive run, or any failed suite — no baseline needed.
|
|
27
|
+
// any fail on any finding at all, or any suite failure. Strictest.
|
|
27
28
|
import fs from "fs";
|
|
28
29
|
import path from "node:path";
|
|
29
|
-
import {
|
|
30
|
+
import { execSync } from "node:child_process";
|
|
31
|
+
import { buildQaReport, computeRegression, computeContentCollapse, computeReachabilityLoss, evaluateGate, GATE_EXIT } from "./report.js";
|
|
30
32
|
import { writeHtmlReport } from "./html-report.js";
|
|
31
33
|
import { buildUiMapFromMarkers, writeUiMap } from "./ui-map.js";
|
|
32
34
|
import { proposeSelectorMaintenance, validateWebMaintenanceProposal } from "./maintenance-proposal.js";
|
|
@@ -59,13 +61,31 @@ function parseArgs(argv) {
|
|
|
59
61
|
console.error("Required: --markers <ocqa-markers.txt>");
|
|
60
62
|
process.exit(2);
|
|
61
63
|
}
|
|
62
|
-
if (!["gate", "
|
|
63
|
-
console.error(`--fail-on must be gate|
|
|
64
|
+
if (!["gate", "absolute", "any"].includes(args.failOn)) {
|
|
65
|
+
console.error(`--fail-on must be gate|absolute|any, got: ${args.failOn}`);
|
|
64
66
|
process.exit(2);
|
|
65
67
|
}
|
|
66
68
|
return args;
|
|
67
69
|
}
|
|
68
70
|
|
|
71
|
+
// A GateRun records the revision it judged. A commit SHA alone doesn't represent a dirty local
|
|
72
|
+
// checkout, so record both. Best-effort: CI env → git → nulls (never throws / fails the gate).
|
|
73
|
+
function gitRevision(repoDir) {
|
|
74
|
+
// Inspect the TARGET repository (--project-dir) when given, not the process cwd, so a gate run
|
|
75
|
+
// launched from elsewhere doesn't record the wrong repo's SHA. GITHUB_SHA still wins in CI.
|
|
76
|
+
const opts = { stdio: ["ignore", "pipe", "ignore"], ...(repoDir ? { cwd: repoDir } : {}) };
|
|
77
|
+
const sh = (cmd) => execSync(cmd, opts).toString().trim();
|
|
78
|
+
let sha = process.env.GITHUB_SHA || null;
|
|
79
|
+
let dirty = false;
|
|
80
|
+
try {
|
|
81
|
+
if (!sha) sha = sh("git rev-parse HEAD") || null;
|
|
82
|
+
dirty = sh("git status --porcelain").length > 0;
|
|
83
|
+
} catch {
|
|
84
|
+
/* not a git checkout / git unavailable */
|
|
85
|
+
}
|
|
86
|
+
return { sha, dirty };
|
|
87
|
+
}
|
|
88
|
+
|
|
69
89
|
async function validateMaintenancePlan(plan, args) {
|
|
70
90
|
if (!plan || plan.platform !== "web" || !args.projectDir || !args.maintenanceUrl) return plan;
|
|
71
91
|
let remaining = 3;
|
|
@@ -101,7 +121,7 @@ async function validateMaintenancePlan(plan, args) {
|
|
|
101
121
|
// Port of flow_lib.py report() / FlowRunnerService.parseReport — kept in sync deliberately.
|
|
102
122
|
function parseFlowLog(logPath) {
|
|
103
123
|
const name = logPath.split("/").pop().replace(/\.log$/, "");
|
|
104
|
-
if (!fs.existsSync(logPath)) return { name, passed: false, total: 0, failed: 0, steps: [], missing: true };
|
|
124
|
+
if (!fs.existsSync(logPath)) return { name, passed: false, total: 0, failed: 0, steps: [], missing: true, modelObserved: false, deterministicFailed: false };
|
|
105
125
|
const steps = [];
|
|
106
126
|
let total = 0, executed = 0, failed = 0, passed = false, sawResult = false, flowName = null, kind = "flow", contract = "", criticality = "";
|
|
107
127
|
for (const raw of fs.readFileSync(logPath, "utf8").split(/\r?\n/)) {
|
|
@@ -132,7 +152,12 @@ function parseFlowLog(logPath) {
|
|
|
132
152
|
failed = steps.filter((s) => s.status === "fail").length;
|
|
133
153
|
passed = steps.length > 0 && failed === 0;
|
|
134
154
|
}
|
|
135
|
-
|
|
155
|
+
// Structural evidence authority (ADR-0005): assert_ai steps are model-observed. A deterministic
|
|
156
|
+
// step failure is real; a suite that only carries a model assertion cannot be decided by the
|
|
157
|
+
// default deterministic gate. evaluateGate reads these flags, never the raw action string.
|
|
158
|
+
const modelObserved = steps.some((s) => s.action === "assert_ai");
|
|
159
|
+
const deterministicFailed = steps.some((s) => s.action !== "assert_ai" && s.status === "fail");
|
|
160
|
+
return { name: flowName || name, kind, ...(contract ? { contract, criticality } : {}), passed, total, executed, failed, steps, modelObserved, deterministicFailed };
|
|
136
161
|
}
|
|
137
162
|
|
|
138
163
|
function loadBaseline(baselinePath) {
|
|
@@ -338,16 +363,19 @@ function enrichPrPlan(plan, contracts, currentUiMap = null, markersPath = "", pr
|
|
|
338
363
|
}
|
|
339
364
|
|
|
340
365
|
const SEV_ICON = { critical: "🟥", high: "🟧", medium: "🟨", low: "🟩" };
|
|
366
|
+
const GATE_BADGE = { pass: "🟢 PASS", fail: "🔴 FAIL", inconclusive: "🟡 INCONCLUSIVE" };
|
|
341
367
|
|
|
342
368
|
function renderMarkdown(report, regression, flows, scenarios, contracts, prPlan, gate) {
|
|
343
369
|
const lines = [];
|
|
344
|
-
|
|
370
|
+
// The header shows the GATE decision (pass/fail/inconclusive), not a ship verdict — exploration
|
|
371
|
+
// only observes; the gate judges (ADR-0005).
|
|
372
|
+
lines.push(`## tapp release check — ${GATE_BADGE[gate.outcome] || gate.outcome}`);
|
|
345
373
|
lines.push("");
|
|
346
374
|
lines.push(report.headline);
|
|
347
375
|
lines.push("");
|
|
348
|
-
lines.push(
|
|
376
|
+
lines.push(`${report.screensExplored} screens · ${report.actionsPerformed} actions · ${report.findingCounts.total} finding(s)`);
|
|
349
377
|
if (report.platform === "web") {
|
|
350
|
-
lines.push(`**
|
|
378
|
+
lines.push(`**Deterministic basis:** ${report.deterministicFindingCounts?.total || 0} deterministic finding(s); ${report.sampledFindingCounts?.total || 0} sampled probe finding(s) are advisory.`);
|
|
351
379
|
}
|
|
352
380
|
if (report.uiMap) lines.push(`**UI Map:** ${report.uiMap.nodeCount} states · ${report.uiMap.edgeCount} transitions · ${report.uiMap.controlCount} semantic controls`);
|
|
353
381
|
if (report.findings.length) {
|
|
@@ -359,18 +387,22 @@ function renderMarkdown(report, regression, flows, scenarios, contracts, prPlan,
|
|
|
359
387
|
}
|
|
360
388
|
}
|
|
361
389
|
if (regression) {
|
|
362
|
-
|
|
390
|
+
// This is the GATE's report, so it may judge the regression. computeRegression is comparison-only,
|
|
391
|
+
// so derive the new-high/critical count here (mirrors evaluateGate).
|
|
392
|
+
const newCritical = regression.newFindings.filter((f) => f.severity === "critical").length;
|
|
393
|
+
const newHigh = regression.newFindings.filter((f) => f.severity === "high").length;
|
|
394
|
+
const regFailed = newCritical + newHigh > 0;
|
|
363
395
|
lines.push("");
|
|
364
|
-
lines.push(`### Since baseline — ${
|
|
396
|
+
lines.push(`### Since baseline — ${regFailed ? "🔴 regression gate FAILED" : "🟢 regression gate passed"}`);
|
|
365
397
|
lines.push(`+${regression.counts.new} new · ${regression.counts.persisting} persisting · ${regression.counts.resolved} resolved` +
|
|
366
|
-
(
|
|
398
|
+
(regFailed ? ` — **${newCritical} new critical, ${newHigh} new high**` : ""));
|
|
367
399
|
for (const f of regression.newFindings) {
|
|
368
400
|
lines.push(`- NEW ${SEV_ICON[f.severity] || ""} ${f.severity}: ${f.title} (${f.screen ?? "—"})`);
|
|
369
401
|
}
|
|
370
402
|
} else if (gate.policy === "gate") {
|
|
371
403
|
lines.push("");
|
|
372
404
|
lines.push("### Baseline — 🟡 not active yet");
|
|
373
|
-
lines.push("No baseline was supplied
|
|
405
|
+
lines.push("No baseline was supplied. This gate still enforces absolute blockers and reviewed suite failures, but it cannot identify new regressions until a baseline is saved.");
|
|
374
406
|
}
|
|
375
407
|
if (prPlan) {
|
|
376
408
|
lines.push("");
|
|
@@ -444,15 +476,23 @@ function renderMarkdown(report, regression, flows, scenarios, contracts, prPlan,
|
|
|
444
476
|
}
|
|
445
477
|
}
|
|
446
478
|
lines.push("");
|
|
447
|
-
|
|
479
|
+
const badge = GATE_BADGE[gate.outcome] || (gate.failed ? "🔴 FAIL" : "🟢 PASS");
|
|
480
|
+
lines.push(`**Gate (${gate.policy}): ${badge}**${gate.reasons.length ? " — " + gate.reasons.join("; ") : ""}`);
|
|
481
|
+
// The gate is only authoritative about what it actually ran — record the scope explicitly.
|
|
482
|
+
const rev = gate.revision?.sha ? `${String(gate.revision.sha).slice(0, 12)}${gate.revision.dirty ? "-dirty" : ""}` : "unknown";
|
|
483
|
+
lines.push(`_target: ${gate.target || "—"} · revision: ${rev} · policy: ${gate.policy} v${gate.policyVersion || "?"}_`);
|
|
484
|
+
if (Array.isArray(gate.checked) && gate.checked.length) lines.push(`_Checked: ${gate.checked.join(" · ")}_`);
|
|
485
|
+
if (Array.isArray(gate.notChecked) && gate.notChecked.length) lines.push(`_Not checked: ${gate.notChecked.join(" · ")}_`);
|
|
448
486
|
return lines.join("\n");
|
|
449
487
|
}
|
|
450
488
|
|
|
451
489
|
const args = parseArgs(process.argv.slice(2));
|
|
452
|
-
const report = buildQaReport(args.markers, { platform: args.platform || "ios" });
|
|
490
|
+
const report = buildQaReport(args.markers, { platform: args.platform || "ios", target: args.label || null });
|
|
453
491
|
if (!report) {
|
|
454
|
-
|
|
455
|
-
|
|
492
|
+
// Required evidence could not be obtained — this is inconclusive (fails closed), not a gate FAIL
|
|
493
|
+
// and not a usage error. See the outcome model in report.js (GATE_EXIT).
|
|
494
|
+
console.error(`No OCQA markers found at ${args.markers} — the exploration did not run (inconclusive).`);
|
|
495
|
+
process.exit(GATE_EXIT.inconclusive);
|
|
456
496
|
}
|
|
457
497
|
let currentUiMap = null;
|
|
458
498
|
if (args.htmlDir) {
|
|
@@ -491,18 +531,12 @@ if (collapsed.length) {
|
|
|
491
531
|
report.findings.push(...collapsed);
|
|
492
532
|
report.findingCounts.high += collapsed.length;
|
|
493
533
|
report.findingCounts.total += collapsed.length;
|
|
494
|
-
report.
|
|
495
|
-
report.
|
|
496
|
-
//
|
|
497
|
-
// baseline
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
report.releaseScore = report.confidence;
|
|
501
|
-
}
|
|
502
|
-
if (report.verdict === "ready") {
|
|
503
|
-
report.verdict = Number.isFinite(report.confidence) && report.confidence < 50 ? "blocked" : "caution";
|
|
504
|
-
}
|
|
505
|
-
report.headline = `Proceed with caution — ${collapsed.length} screen(s) regressed vs. baseline (content collapsed or became unreachable).`;
|
|
534
|
+
report.deterministicFindingCounts.high += collapsed.length;
|
|
535
|
+
report.deterministicFindingCounts.total += collapsed.length;
|
|
536
|
+
// Collapsed/unreachable screens are deterministic regressions: they raise the deterministic finding
|
|
537
|
+
// counts (so findingsBlock sees them) and count as new-vs-baseline (so the gate fails on them).
|
|
538
|
+
// No score/verdict to mutate — exploration is scoreless; the gate renders the outcome.
|
|
539
|
+
report.headline = `${collapsed.length} screen(s) regressed vs. baseline (content collapsed or became unreachable).`;
|
|
506
540
|
}
|
|
507
541
|
const regression = computeRegression(report.findings, baseline?.findings ?? null);
|
|
508
542
|
const runs = args.flowLogs.map(parseFlowLog);
|
|
@@ -519,34 +553,19 @@ try {
|
|
|
519
553
|
process.exit(2);
|
|
520
554
|
}
|
|
521
555
|
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
const
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
if (report.inconclusive) reasons.push("run was inconclusive (coverage floor not met)");
|
|
536
|
-
} else {
|
|
537
|
-
if (regression?.gate.failed) {
|
|
538
|
-
reasons.push(`${regression.gate.newCritical} new critical + ${regression.gate.newHigh} new high vs. baseline`);
|
|
539
|
-
}
|
|
540
|
-
// A regression gate must also catch regressions in EXPLORABILITY, not just in findings:
|
|
541
|
-
// a change that makes the app crash at launch (or reintroduces a login wall) produces an
|
|
542
|
-
// inconclusive run with zero new findings — that must never pass. (Found via corpus
|
|
543
|
-
// bug-seeding: a seeded crash-at-startup sailed through on the findings diff alone.)
|
|
544
|
-
if (report.verdict === "blocked") reasons.push("verdict is blocked");
|
|
545
|
-
if (report.inconclusive && !baseline.inconclusive) {
|
|
546
|
-
reasons.push("run became inconclusive vs. baseline (app may no longer launch/explore)");
|
|
547
|
-
}
|
|
548
|
-
}
|
|
549
|
-
const gate = { policy: args.failOn, failed: reasons.length > 0, reasons };
|
|
556
|
+
// The gate decision now lives in a pure, unit-tested evaluator (report.js). A regression gate
|
|
557
|
+
// must catch regressions in EXPLORABILITY, not just findings: a change that makes the app crash at
|
|
558
|
+
// launch (or reintroduces a login wall) produces an inconclusive run with zero new findings — that
|
|
559
|
+
// must never pass (found via corpus bug-seeding). evaluateGate encodes that as an `inconclusive`
|
|
560
|
+
// outcome (exit 3), distinct from a deterministic `fail` (exit 1).
|
|
561
|
+
const decision = evaluateGate({ report, regression, flows, scenarios, contracts, prPlan, baseline, failOn: args.failOn });
|
|
562
|
+
const gate = {
|
|
563
|
+
...decision,
|
|
564
|
+
target: args.targetKey || report.target || null,
|
|
565
|
+
revision: gitRevision(args.projectDir),
|
|
566
|
+
checked: report.checkedFor,
|
|
567
|
+
notChecked: report.notChecked,
|
|
568
|
+
};
|
|
550
569
|
|
|
551
570
|
const md = renderMarkdown(report, regression, flows, scenarios, contracts, prPlan, gate);
|
|
552
571
|
console.log(md);
|
|
@@ -563,4 +582,7 @@ if (args.htmlDir) {
|
|
|
563
582
|
const html = writeHtmlReport(args.htmlDir, { report, label: args.label || "CI run" });
|
|
564
583
|
if (html) console.log(`\nEvidence report: ${html}`);
|
|
565
584
|
}
|
|
566
|
-
|
|
585
|
+
// Outcome → exit code (ADR-0005): pass 0 · fail 1 · error 2 · inconclusive 3. fail and inconclusive
|
|
586
|
+
// both block a merge; distinct codes let CI tell "a regression was observed" from "we couldn't be
|
|
587
|
+
// sure" and keep infra/usage errors (2) separate.
|
|
588
|
+
process.exit(gate.exitCode);
|
|
@@ -15,7 +15,7 @@ export function targetSlug(value) {
|
|
|
15
15
|
return slug || "target";
|
|
16
16
|
}
|
|
17
17
|
|
|
18
|
-
export function selectApplicationTarget(model, { platform = "", target = "" } = {}) {
|
|
18
|
+
export function selectApplicationTarget(model, { platform = "", target = "", useDefault = false } = {}) {
|
|
19
19
|
if (model?.kind !== "tapp-application-model" || !Array.isArray(model.targets)) {
|
|
20
20
|
throw new Error("Expected a Tapp application model; run tapp init first");
|
|
21
21
|
}
|
|
@@ -26,6 +26,14 @@ export function selectApplicationTarget(model, { platform = "", target = "" } =
|
|
|
26
26
|
const normalized = requested.replaceAll("\\", "/").replace(/^\.\//, "");
|
|
27
27
|
candidates = candidates.filter((item) => [item.id, item.name, item.sourcePath].some((value) => String(value || "").replaceAll("\\", "/") === normalized));
|
|
28
28
|
}
|
|
29
|
+
// Target-resolution ladder (ADR-0005 §5): explicit narrowing (above) → exactly one candidate →
|
|
30
|
+
// the model's recorded default target (opt-in: `explore` uses it, but the gate/baseline stay
|
|
31
|
+
// strict so CI never silently picks a target) → otherwise list the choices and stop.
|
|
32
|
+
if (candidates.length > 1 && useDefault && !requested) {
|
|
33
|
+
const def = String(model.application?.defaultTargetId || "").trim();
|
|
34
|
+
const chosen = def && candidates.find((item) => item.id === def);
|
|
35
|
+
if (chosen) return chosen;
|
|
36
|
+
}
|
|
29
37
|
if (candidates.length !== 1) {
|
|
30
38
|
const summary = candidates.length ? candidates : model.targets.filter((item) => !selectedPlatform || item.platform === selectedPlatform);
|
|
31
39
|
throw new Error(candidates.length
|
|
@@ -35,6 +43,22 @@ export function selectApplicationTarget(model, { platform = "", target = "" } =
|
|
|
35
43
|
return candidates[0];
|
|
36
44
|
}
|
|
37
45
|
|
|
46
|
+
// For a bare `tapp explore` in a repo: if the model's default target is a web target with a recorded
|
|
47
|
+
// owned URL, return that URL so exploration hits it directly (the `init --url X` → `explore` path).
|
|
48
|
+
// Targets that must be built/started from source need the prepare pipeline (wired separately), so
|
|
49
|
+
// this returns null and the caller falls back to its normal target resolution.
|
|
50
|
+
export function defaultWebExploreUrl(model) {
|
|
51
|
+
let target;
|
|
52
|
+
try {
|
|
53
|
+
target = selectApplicationTarget(model, { useDefault: true });
|
|
54
|
+
} catch {
|
|
55
|
+
return null; // no model, or ambiguous with no recorded default
|
|
56
|
+
}
|
|
57
|
+
if (target?.platform !== "web") return null;
|
|
58
|
+
const url = String(target.runtime?.ownedUrl || "");
|
|
59
|
+
return /^https?:\/\//i.test(url) ? url : null;
|
|
60
|
+
}
|
|
61
|
+
|
|
38
62
|
export function baselinePathForTarget(projectDir, target) {
|
|
39
63
|
const root = fs.realpathSync(path.resolve(projectDir));
|
|
40
64
|
return path.join(root, ".tapp", "baselines", target.platform, `${targetSlug(target.id)}.json`);
|
|
@@ -52,9 +76,15 @@ export function validateBaselineReport(report, { platform, targetId } = {}) {
|
|
|
52
76
|
const reportTarget = String(report.targetKey || report.baselineIdentity?.targetId || "").trim();
|
|
53
77
|
if (!reportTarget) throw new Error("Baseline source is missing its targetKey; Tapp will not guess which same-platform application produced the evidence");
|
|
54
78
|
if (reportTarget !== targetId) throw new Error(`Baseline target '${reportTarget}' does not match application-model target '${targetId}'`);
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
79
|
+
// A trusted baseline must be a clean PASS. The gate outcome is the single authoritative signal
|
|
80
|
+
// (ADR-0005) — reject fail, inconclusive, error, or a missing/unknown outcome, naming which.
|
|
81
|
+
const outcome = report.gate?.outcome;
|
|
82
|
+
if (outcome !== "pass") {
|
|
83
|
+
const why = outcome === "inconclusive" ? "An inconclusive run"
|
|
84
|
+
: outcome === "fail" ? "A failing run"
|
|
85
|
+
: `A non-passing run (${outcome || "no gate outcome"})`;
|
|
86
|
+
throw new Error(`${why} cannot become a trusted baseline; only a passing gate run can`);
|
|
87
|
+
}
|
|
58
88
|
for (const collection of ["flows", "scenarios", "contracts"]) {
|
|
59
89
|
const failed = (report[collection] || []).filter((item) => item.passed !== true);
|
|
60
90
|
if (failed.length) throw new Error(`Baseline source contains ${failed.length} failed ${collection}`);
|
|
@@ -64,7 +94,7 @@ export function validateBaselineReport(report, { platform, targetId } = {}) {
|
|
|
64
94
|
platform,
|
|
65
95
|
targetId,
|
|
66
96
|
conclusive: true,
|
|
67
|
-
|
|
97
|
+
outcome: report.gate?.outcome ?? null,
|
|
68
98
|
screensExplored: Number(report.screensExplored || report.screens.length),
|
|
69
99
|
actionsPerformed: Number(report.actionsPerformed || 0),
|
|
70
100
|
suite: {
|
package/mcp-server/src/enrich.js
CHANGED
|
@@ -40,7 +40,7 @@ export async function enrichFindings(findings, { backend, callModel, screens = [
|
|
|
40
40
|
`Title: ${f.title}`;
|
|
41
41
|
try {
|
|
42
42
|
const res = await Promise.race([
|
|
43
|
-
callModel(backend, { system, userText, model: process.env.TAPP_FINDING_MODEL ||
|
|
43
|
+
callModel(backend, { system, userText, model: process.env.TAPP_FINDING_MODEL || "claude-haiku-4-5-20251001", maxTokens: 300 }),
|
|
44
44
|
new Promise((r) => setTimeout(() => r({ error: "timeout" }), TIMEOUT_MS)),
|
|
45
45
|
]);
|
|
46
46
|
const parsed = res && !res.error ? parseEnrichment(res.text) : null;
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
|
|
9
9
|
import fs from "fs";
|
|
10
10
|
import path from "path";
|
|
11
|
-
import { buildQaReport,
|
|
11
|
+
import { buildQaReport, observationBadge, observationSummary } from "./report.js";
|
|
12
12
|
|
|
13
13
|
const SEV_COLOR = { critical: "#cf222e", high: "#bc4c00", medium: "#9a6700", low: "#57606a" };
|
|
14
14
|
|
|
@@ -49,7 +49,7 @@ function collectShots(captureDir) {
|
|
|
49
49
|
}
|
|
50
50
|
}
|
|
51
51
|
|
|
52
|
-
export function writeHtmlReport(captureDir, { report, label = "" } = {}) {
|
|
52
|
+
export function writeHtmlReport(captureDir, { report, label = "", recordingWarning = "" } = {}) {
|
|
53
53
|
const r = report || buildQaReport(path.join(captureDir, "ocqa-markers.txt"));
|
|
54
54
|
if (!r) return null;
|
|
55
55
|
|
|
@@ -78,7 +78,16 @@ export function writeHtmlReport(captureDir, { report, label = "" } = {}) {
|
|
|
78
78
|
|
|
79
79
|
const videoHtml = video
|
|
80
80
|
? `<h2>Recording — the full exploration</h2>\n<video controls preload="metadata" style="width:100%;border:1px solid #d0d7de;border-radius:8px" src="${esc(video)}"></video>`
|
|
81
|
-
:
|
|
81
|
+
: recordingWarning
|
|
82
|
+
? `<h2>Recording</h2>\n<p class="warning">Unavailable: ${esc(recordingWarning)}. Screenshots were still captured.</p>`
|
|
83
|
+
: "";
|
|
84
|
+
|
|
85
|
+
const scopeList = (items) => (items || []).map((item) => `<li>${esc(item)}</li>`).join("\n");
|
|
86
|
+
const scopeHtml = `<section class="scope">
|
|
87
|
+
<div><h2>Checked this run</h2><ul>${scopeList(r.checkedFor) || "<li class='dim'>No automated checks completed.</li>"}</ul></div>
|
|
88
|
+
<div><h2>Not checked this run</h2><ul>${scopeList(r.notChecked) || "<li class='dim'>No additional limitations recorded.</li>"}</ul></div>
|
|
89
|
+
${r.conditionsNotReached?.length ? `<div><h2>Conditions not reached</h2><ul>${scopeList(r.conditionsNotReached)}</ul></div>` : ""}
|
|
90
|
+
</section>`;
|
|
82
91
|
|
|
83
92
|
const html = `<!doctype html>
|
|
84
93
|
<html lang="en">
|
|
@@ -90,6 +99,10 @@ export function writeHtmlReport(captureDir, { report, label = "" } = {}) {
|
|
|
90
99
|
h1 { font-size: 1.6rem; margin-bottom: 0.2rem; }
|
|
91
100
|
.meta { color: #57606a; margin-bottom: 1.2rem; }
|
|
92
101
|
.headline { background: #f6f8fa; border-radius: 8px; padding: 0.9rem 1.1rem; margin: 1rem 0; }
|
|
102
|
+
.warning { background: #fff8c5; border: 1px solid #d4a72c; border-radius: 8px; padding: 0.75rem 1rem; }
|
|
103
|
+
.scope { display: grid; grid-template-columns: repeat(auto-fit, minmax(260px, 1fr)); gap: 0.8rem 1.5rem; }
|
|
104
|
+
.scope h2 { font-size: 1.05rem; margin-bottom: 0.35rem; }
|
|
105
|
+
.scope ul { margin-top: 0; padding-left: 1.2rem; }
|
|
93
106
|
.sev { color: #fff; border-radius: 4px; padding: 0.05rem 0.45rem; font-size: 0.78rem; font-weight: 600; margin-right: 0.4rem; }
|
|
94
107
|
ul.findings { padding-left: 1.1rem; } ul.findings li { margin-bottom: 0.6rem; }
|
|
95
108
|
.ai { color: #57606a; font-size: 0.88rem; margin: 0.15rem 0 0 0.2rem; }
|
|
@@ -101,10 +114,11 @@ export function writeHtmlReport(captureDir, { report, label = "" } = {}) {
|
|
|
101
114
|
</style>
|
|
102
115
|
</head>
|
|
103
116
|
<body>
|
|
104
|
-
<h1>${esc(
|
|
105
|
-
<div class="meta">${esc(label)} ·
|
|
106
|
-
${r.platform === "web" ? `<div class="meta">
|
|
117
|
+
<h1>${esc(observationBadge(r))} <span class="dim">· ${esc(observationSummary(r))}</span></h1>
|
|
118
|
+
<div class="meta">${esc(label)} · evidence page (observation, not a release decision)</div>
|
|
119
|
+
${r.platform === "web" ? `<div class="meta">Deterministic basis: ${r.deterministicFindingCounts?.total || 0} deterministic finding(s); ${r.sampledFindingCounts?.total || 0} sampled probe finding(s) are advisory.</div>` : ""}
|
|
107
120
|
<div class="headline">${esc(r.headline)}</div>
|
|
121
|
+
${scopeHtml}
|
|
108
122
|
<h2>Findings</h2>
|
|
109
123
|
<ul class="findings">
|
|
110
124
|
${findingsHtml}
|