create-cmp-cli 0.20.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/packages/harness/package.json +1 -1
- package/packages/harness/src/lib/affected-tests.mjs +9 -1
- package/packages/harness/src/lib/device-provider.mjs +153 -0
- package/packages/harness/src/lib/e2e-coverage.mjs +60 -0
- package/packages/harness/src/lib/evidence-badge.mjs +11 -0
- package/packages/harness/src/lib/evidence-level.mjs +34 -2
- package/packages/harness/src/lib/feature-brief.mjs +16 -3
- package/packages/harness/src/lib/harness-lock.mjs +6 -1
- package/packages/harness/src/lib/inputs-hash.mjs +27 -0
- package/packages/harness/src/lib/lane-runner.mjs +7 -0
- package/packages/harness/src/lib/spec-coverage.mjs +27 -3
- package/packages/harness/src/lib/step-outcomes.mjs +104 -0
- package/packages/harness/src/lib/steps-cmp.mjs +133 -39
- package/packages/harness/src/receipt-check.mjs +25 -0
- package/packages/harness/src/scaffold-feature.mjs +55 -0
- package/packages/harness/src/verify.mjs +18 -2
- package/packages/receipts/src/inputs-hash.mjs +27 -0
- package/src/commands/upgrade.mjs +31 -2
- package/src/lib/tabs.mjs +18 -3
- package/src/lib/upgrade.mjs +50 -0
- package/template/.github/workflows/verify.yml +9 -0
- package/template/gitignore +9 -0
- package/template/qa/e2e/README.md +23 -4
- package/template/qa/e2e/smoke.yaml +9 -1
- package/template/qa/evidence/schema.json +6 -1
- package/template/qa/lib/affected-tests.mjs +9 -1
- package/template/qa/lib/device-provider.mjs +153 -0
- package/template/qa/lib/e2e-coverage.mjs +60 -0
- package/template/qa/lib/evidence-badge.mjs +11 -0
- package/template/qa/lib/evidence-level.mjs +34 -2
- package/template/qa/lib/feature-brief.mjs +16 -3
- package/template/qa/lib/harness-lock.mjs +6 -1
- package/template/qa/lib/inputs-hash.mjs +27 -0
- package/template/qa/lib/lane-runner.mjs +7 -0
- package/template/qa/lib/spec-coverage.mjs +27 -3
- package/template/qa/lib/step-outcomes.mjs +104 -0
- package/template/qa/lib/steps-cmp.mjs +133 -39
- package/template/qa/receipt-check.mjs +25 -0
- package/template/qa/scaffold-feature.mjs +55 -0
- package/template/qa/verify.mjs +18 -2
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "create-cmp-cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.22.0",
|
|
4
4
|
"description": "Create production mobile apps (Android + iOS, one Kotlin codebase) with AI — the delivery harness for Compose Multiplatform, the current generation of cross-platform (Google-backed KMP, iOS stable since May 2025). A deterministic, non-interactive generator that scaffolds a green-building app in minutes, then holds AI-driven changes to a machine-enforced verify lane with a committed evidence receipt. Every app carries a device-free UI preview loop (real screens rendered headlessly on save; changed-screen attribution and compile-error surfacing for coding agents, a live gallery for humans) plus agent-first docs (CLAUDE.md + AGENTS.md). Installs the `create-cmp` command.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@create-cmp/harness",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.17.0",
|
|
4
4
|
"description": "The create-cmp verify lane — the machine-owned harness code every stamped app carries byte-identical: evidence receipts, spec coverage, approvals, conformance reporting, golden trees, a11y, and the preview/inspector libs. Dependency-free ESM, vendored into each generated project so the lane runs offline with no install step, and content-hashed so a receipt can name the exact lane that issued it.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/verify.mjs",
|
|
@@ -35,7 +35,15 @@ import path from "node:path";
|
|
|
35
35
|
* principle as inputs-hash.mjs's EXCLUDED_PREFIXES: lane outputs are not
|
|
36
36
|
* verdict inputs).
|
|
37
37
|
*/
|
|
38
|
-
|
|
38
|
+
// qa/flight-recorder.jsonl is a lane output in the strictest sense: the lane
|
|
39
|
+
// appends one line to it AFTER the receipt is written, on every run. It is
|
|
40
|
+
// committed (the journal is the cost record), so after the first run it sits
|
|
41
|
+
// in the changed set as a modified tracked file under qa/ — and qa/** is the
|
|
42
|
+
// "harness itself" escape hatch. Uncounted here, every --fast run after the
|
|
43
|
+
// first fell open to the full suite, visible only in one parenthetical.
|
|
44
|
+
// Found by payment-blueprint's spine adoption (2026-09-03), where the same
|
|
45
|
+
// line also landed in their locked region.
|
|
46
|
+
export const LANE_OUTPUT_PREFIXES = ["qa/evidence", "qa-artifacts", "qa/flight-recorder.jsonl"];
|
|
39
47
|
|
|
40
48
|
function isLaneOutput(p) {
|
|
41
49
|
return LANE_OUTPUT_PREFIXES.some((prefix) => p === prefix || p.startsWith(`${prefix}/`));
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
// device-provider.mjs — the lane provisions its own device.
|
|
2
|
+
//
|
|
3
|
+
// Until 2026-09-03 every device-tier step (tokenDrift, e2eSmoke, androidChecks,
|
|
4
|
+
// releaseSmoke) SKIPped with "no Android device/emulator attached" and the
|
|
5
|
+
// receipt earned L1 with three visible gaps. Visible, and universally ignored:
|
|
6
|
+
// the showcase's latest receipt had e2eSmoke SKIP, four hand-written flows had
|
|
7
|
+
// never been executed by any gate, and 0.21.0 was published on a fleet check
|
|
8
|
+
// with the whole tier SKIPped. Karel: "they can be run headlessly and should
|
|
9
|
+
// not take too long." Measured: emulator boot 36 s, e2eSmoke 109 s,
|
|
10
|
+
// androidChecks 64 s. So the full lane boots a headless emulator itself when
|
|
11
|
+
// nothing is attached, drives it, and shuts it down when the lane exits.
|
|
12
|
+
//
|
|
13
|
+
// Rules, each bounded (PRINCIPLES #5 — never wait on nothing):
|
|
14
|
+
// - an attached device (adb `device` state) is used as-is, never rebooted;
|
|
15
|
+
// - CMP_DEVICE=none is the ONE explicit opt-out (CI runners without KVM);
|
|
16
|
+
// the step rows say so, and qa/receipt-check.mjs refuses such a receipt
|
|
17
|
+
// as done-evidence at the change stage — an opt-out is visible, never done;
|
|
18
|
+
// - the AVD is CMP_AVD, else the doctor's `cmp_pixel`, else the only AVD;
|
|
19
|
+
// several AVDs and no way to choose is a refusal naming them (per-app AVD
|
|
20
|
+
// isolation: a lane must never guess another app's emulator);
|
|
21
|
+
// - boot is bounded (BOOT_BOUND_MS); past it the emulator is killed and the
|
|
22
|
+
// step rows read ERROR "could not provision a device", which FAILs the
|
|
23
|
+
// lane — a device that never comes up is a failure to test, not a SKIP.
|
|
24
|
+
//
|
|
25
|
+
// Pure where it can be: every subprocess goes through the injected `sh`
|
|
26
|
+
// (the lane's own, so step deadlines apply) and the emulator spawn through
|
|
27
|
+
// `spawnImpl`, so the whole decision tree is unit-tested without an SDK.
|
|
28
|
+
|
|
29
|
+
import { spawn } from "node:child_process";
|
|
30
|
+
import fs from "node:fs";
|
|
31
|
+
import os from "node:os";
|
|
32
|
+
import path from "node:path";
|
|
33
|
+
|
|
34
|
+
/** How long a headless boot may take before it is killed and reported. */
|
|
35
|
+
export const BOOT_BOUND_MS = 240_000;
|
|
36
|
+
/** The AVD cmp-doctor creates (src/bootstrap/checks.mjs); preferred when present. */
|
|
37
|
+
export const PREFERRED_AVD = "cmp_pixel";
|
|
38
|
+
/** Headless, deterministic, no snapshot: every lane boots the same cold device. */
|
|
39
|
+
export const HEADLESS_ARGS = Object.freeze(["-no-window", "-no-audio", "-no-boot-anim", "-no-snapshot", "-gpu", "swiftshader_indirect"]);
|
|
40
|
+
const POLL_MS = 3000;
|
|
41
|
+
|
|
42
|
+
function sleepSync(ms) {
|
|
43
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** Serials in `device` state from `adb devices` output. */
|
|
47
|
+
export function parseAdbDevices(out) {
|
|
48
|
+
return String(out ?? "")
|
|
49
|
+
.split("\n")
|
|
50
|
+
.slice(1)
|
|
51
|
+
.map((l) => l.trim())
|
|
52
|
+
.filter(Boolean)
|
|
53
|
+
.map((l) => l.split(/\s+/))
|
|
54
|
+
.filter(([, state]) => state === "device")
|
|
55
|
+
.map(([serial]) => serial);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** The emulator binary: SDK roots first, then PATH. */
|
|
59
|
+
export function emulatorBinary({ env = process.env, exists = fs.existsSync, home = os.homedir() } = {}) {
|
|
60
|
+
const roots = [env.ANDROID_HOME, env.ANDROID_SDK_ROOT, path.join(home, "Library", "Android", "sdk"), path.join(home, "Android", "Sdk")].filter(Boolean);
|
|
61
|
+
for (const root of roots) {
|
|
62
|
+
const bin = path.join(root, "emulator", "emulator");
|
|
63
|
+
if (exists(bin)) return bin;
|
|
64
|
+
}
|
|
65
|
+
return "emulator";
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Which AVD to boot. Never a guess between several: another app's emulator
|
|
70
|
+
* carries another app's state, and a lane driving it crosses sessions.
|
|
71
|
+
* @param {string[]} listed `emulator -list-avds` lines
|
|
72
|
+
* @param {NodeJS.ProcessEnv} env
|
|
73
|
+
* @returns {{ok: true, avd: string} | {ok: false, reason: string}}
|
|
74
|
+
*/
|
|
75
|
+
export function chooseAvd(listed, env = process.env) {
|
|
76
|
+
const avds = listed.map((l) => l.trim()).filter((l) => l && !l.startsWith("INFO") && !l.startsWith("WARNING"));
|
|
77
|
+
if (env.CMP_AVD) {
|
|
78
|
+
if (avds.includes(env.CMP_AVD)) return { ok: true, avd: env.CMP_AVD };
|
|
79
|
+
return { ok: false, reason: `CMP_AVD=${env.CMP_AVD} is not an AVD on this machine (have: ${avds.join(", ") || "none"})` };
|
|
80
|
+
}
|
|
81
|
+
if (avds.includes(PREFERRED_AVD)) return { ok: true, avd: PREFERRED_AVD };
|
|
82
|
+
if (avds.length === 1) return { ok: true, avd: avds[0] };
|
|
83
|
+
if (avds.length === 0) return { ok: false, reason: `no AVD on this machine — run the cmp-doctor skill (it creates ${PREFERRED_AVD}) or set CMP_AVD` };
|
|
84
|
+
return { ok: false, reason: `${avds.length} AVDs and no ${PREFERRED_AVD} (${avds.join(", ")}) — set CMP_AVD to the one this app owns; the lane will not guess between apps' emulators` };
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Ensure a device is attached, booting a headless emulator when none is.
|
|
89
|
+
* @param {{
|
|
90
|
+
* sh: (cmd: string, opts?: object) => {ok: boolean, out: string},
|
|
91
|
+
* env?: NodeJS.ProcessEnv, spawnImpl?: typeof spawn, sleep?: (ms: number) => void,
|
|
92
|
+
* now?: () => number, log?: (line: string) => void, bootBoundMs?: number,
|
|
93
|
+
* exists?: (p: string) => boolean, kill?: (pid: number) => void,
|
|
94
|
+
* }} deps
|
|
95
|
+
* @returns {{ok: true, serial: string, booted: boolean, avd?: string, pid?: number, bootMs?: number}
|
|
96
|
+
* | {ok: false, optOut?: boolean, reason: string}}
|
|
97
|
+
*/
|
|
98
|
+
export function ensureDevice({ sh, env = process.env, spawnImpl = spawn, sleep = sleepSync, now = Date.now, log = () => {}, bootBoundMs = BOOT_BOUND_MS, exists = fs.existsSync, kill = (pid) => process.kill(pid) } = {}) {
|
|
99
|
+
if (env.CMP_DEVICE === "none") {
|
|
100
|
+
return { ok: false, optOut: true, reason: "device tier disabled by CMP_DEVICE=none (the one explicit opt-out; a receipt carrying it is never done-evidence)" };
|
|
101
|
+
}
|
|
102
|
+
const attached = parseAdbDevices(sh("adb devices", { timeout: 10_000 }).out);
|
|
103
|
+
if (attached.length > 0) return { ok: true, serial: attached[0], booted: false };
|
|
104
|
+
|
|
105
|
+
const bin = emulatorBinary({ env, exists });
|
|
106
|
+
const listing = sh(`"${bin}" -list-avds`, { timeout: 30_000 });
|
|
107
|
+
if (!listing.ok) {
|
|
108
|
+
return { ok: false, reason: `no device attached and the emulator binary could not list AVDs (${bin}) — install the Android emulator (cmp-doctor) or attach a device` };
|
|
109
|
+
}
|
|
110
|
+
const choice = chooseAvd(listing.out.split("\n"), env);
|
|
111
|
+
if (!choice.ok) return { ok: false, reason: `no device attached and ${choice.reason}` };
|
|
112
|
+
|
|
113
|
+
log(`no device attached — booting ${choice.avd} headless (bound ${Math.round(bootBoundMs / 1000)} s)`);
|
|
114
|
+
const started = now();
|
|
115
|
+
let child;
|
|
116
|
+
try {
|
|
117
|
+
child = spawnImpl(bin, ["-avd", choice.avd, ...HEADLESS_ARGS], { detached: true, stdio: "ignore" });
|
|
118
|
+
if (typeof child.unref === "function") child.unref();
|
|
119
|
+
} catch (err) {
|
|
120
|
+
return { ok: false, reason: `could not start the emulator (${err && err.message ? err.message : String(err)})` };
|
|
121
|
+
}
|
|
122
|
+
let serial = null;
|
|
123
|
+
while (now() - started < bootBoundMs) {
|
|
124
|
+
sleep(POLL_MS);
|
|
125
|
+
const serials = parseAdbDevices(sh("adb devices", { timeout: 10_000 }).out);
|
|
126
|
+
if (serials.length === 0) continue;
|
|
127
|
+
serial = serials[0];
|
|
128
|
+
const booted = sh(`adb -s ${serial} shell getprop sys.boot_completed`, { timeout: 10_000 });
|
|
129
|
+
if (booted.ok && booted.out.trim() === "1") {
|
|
130
|
+
return { ok: true, serial, booted: true, avd: choice.avd, pid: child.pid, bootMs: now() - started };
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
// Past the bound: kill what we started, and say so. A device that never
|
|
134
|
+
// comes up is a failure to test — the step rows read ERROR, the lane FAILs.
|
|
135
|
+
if (serial) sh(`adb -s ${serial} emu kill`, { timeout: 15_000 });
|
|
136
|
+
try {
|
|
137
|
+
if (child.pid) kill(child.pid);
|
|
138
|
+
} catch {
|
|
139
|
+
/* already gone */
|
|
140
|
+
}
|
|
141
|
+
return { ok: false, reason: `emulator ${choice.avd} did not reach boot_completed within ${Math.round(bootBoundMs / 1000)} s (killed) — boot it by hand once to see why, or set CMP_AVD to a lighter AVD` };
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* Shut down the emulator THIS lane booted; an attached device is left alone.
|
|
146
|
+
* CMP_KEEP_DEVICE=1 keeps a booted one up for the next run (saves the boot).
|
|
147
|
+
*/
|
|
148
|
+
export function releaseDevice(handle, { sh, env = process.env } = {}) {
|
|
149
|
+
if (!handle || !handle.ok || !handle.booted) return { shutdown: false };
|
|
150
|
+
if (env.CMP_KEEP_DEVICE === "1") return { shutdown: false, kept: true };
|
|
151
|
+
sh(`adb -s ${handle.serial} emu kill`, { timeout: 15_000 });
|
|
152
|
+
return { shutdown: true };
|
|
153
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
// e2e-coverage.mjs — every real feature with a screen has a device journey.
|
|
2
|
+
//
|
|
3
|
+
// The question (Karel, 2026-09-03): "is anything forcing e2e maestro tests to be
|
|
4
|
+
// written when we implement a feature?" Before this gate: no. A feature's flow
|
|
5
|
+
// was stamped as a skeleton that cites nothing, every clause could be proven by
|
|
6
|
+
// JVM tests alone, and a UI feature reached "done" with zero device evidence.
|
|
7
|
+
//
|
|
8
|
+
// The rule: a feature is REAL when it has a screen (presentation/<f>/*Screen.kt
|
|
9
|
+
// — reachability's definition) AND a spec (specs/<f>.spec.md). A real feature
|
|
10
|
+
// must have at least one live clause cited from a flow under qa/e2e — a flow
|
|
11
|
+
// that the lane RUNS (spec-coverage's listFlowFiles is the one list). A
|
|
12
|
+
// placeholder screen without a spec is not yet real (CHANGE-FLOW-DESIGN:
|
|
13
|
+
// "placeholder tabs earn a brief only when they become real") — reported, not
|
|
14
|
+
// failed. A screen declared `{ "unrouted": true }` in its brief has no journey
|
|
15
|
+
// to walk — exempt, by the same declare-not-gate mechanism reachability uses.
|
|
16
|
+
//
|
|
17
|
+
// Pure Node, milliseconds. FAILs by name with the file to write.
|
|
18
|
+
|
|
19
|
+
import fs from "node:fs";
|
|
20
|
+
import path from "node:path";
|
|
21
|
+
import { evaluateReachability } from "./reachability.mjs";
|
|
22
|
+
import { E2E_FLOW_DIR, scanCitations, scanSpecClauses } from "./spec-coverage.mjs";
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* @param {string} root
|
|
26
|
+
* @returns {{verdict: "PASS"|"FAIL"|"SKIP", reason?: string, details: {features: Array<{name: string, spec: string|null, liveClauses: number, e2eCited: string[], status: "covered"|"uncovered"|"unspecified"|"unrouted"}>}}}
|
|
27
|
+
*/
|
|
28
|
+
export function evaluateE2eCoverage(root) {
|
|
29
|
+
if (!fs.existsSync(path.join(root, E2E_FLOW_DIR))) {
|
|
30
|
+
return { verdict: "SKIP", reason: `e2e harness not included in this project (no ${E2E_FLOW_DIR}/)`, details: { features: [] } };
|
|
31
|
+
}
|
|
32
|
+
const reach = evaluateReachability(root);
|
|
33
|
+
const screenFeatures = reach.details && Array.isArray(reach.details.features) ? reach.details.features : [];
|
|
34
|
+
if (screenFeatures.length === 0) {
|
|
35
|
+
return { verdict: "SKIP", reason: reach.reason ?? "no presentation/<feature> directory has a *Screen.kt file — nothing to cover", details: { features: [] } };
|
|
36
|
+
}
|
|
37
|
+
const clauses = scanSpecClauses(root);
|
|
38
|
+
const e2eTags = scanCitations(root).filter((t) => t.tier === "e2e");
|
|
39
|
+
const features = screenFeatures.map((f) => {
|
|
40
|
+
const specRel = `specs/${f.name}.spec.md`;
|
|
41
|
+
const spec = fs.existsSync(path.join(root, specRel)) ? specRel : null;
|
|
42
|
+
if (f.unrouted) return { name: f.name, spec, liveClauses: 0, e2eCited: [], status: "unrouted" };
|
|
43
|
+
if (!spec) return { name: f.name, spec: null, liveClauses: 0, e2eCited: [], status: "unspecified" };
|
|
44
|
+
const live = [...clauses.entries()].filter(([, c]) => c.file.split(path.sep).join("/") === specRel && !c.withdrawn).map(([id]) => id);
|
|
45
|
+
const e2eCited = [...new Set(e2eTags.filter((t) => live.includes(t.id)).map((t) => t.id))].sort();
|
|
46
|
+
return { name: f.name, spec, liveClauses: live.length, e2eCited, status: e2eCited.length ? "covered" : "uncovered" };
|
|
47
|
+
});
|
|
48
|
+
const uncovered = features.filter((f) => f.status === "uncovered");
|
|
49
|
+
if (uncovered.length === 0) return { verdict: "PASS", details: { features } };
|
|
50
|
+
const lines = [
|
|
51
|
+
`${uncovered.length} feature${uncovered.length === 1 ? " has" : "s have"} a screen and a spec but no device journey — no flow under ${E2E_FLOW_DIR}/ cites any of their clauses:`,
|
|
52
|
+
...uncovered.map((f) =>
|
|
53
|
+
f.liveClauses === 0
|
|
54
|
+
? ` [${f.name}] ${f.spec} has no live clauses — promise the behaviour there first, then prove one clause in ${E2E_FLOW_DIR}/${f.name}.yaml (# SPEC: <ID> above the steps)`
|
|
55
|
+
: ` [${f.name}] ${f.spec} has ${f.liveClauses} live clause${f.liveClauses === 1 ? "" : "s"} — write the journey in ${E2E_FLOW_DIR}/${f.name}.yaml and cite the clause(s) it proves (# SPEC: <ID> above the steps)`,
|
|
56
|
+
),
|
|
57
|
+
"A UI feature is proven on a device, not only on the JVM. Declare { \"unrouted\": true } in its brief only if the screen is intentionally not reachable yet.",
|
|
58
|
+
];
|
|
59
|
+
return { verdict: "FAIL", reason: lines.join("\n"), details: { features } };
|
|
60
|
+
}
|
|
@@ -146,6 +146,17 @@ export function updateReadmeBadge(root) {
|
|
|
146
146
|
if (receipt && receipt.mode === "fast") {
|
|
147
147
|
return { changed: false, reason: "fast run — the inner loop bears no evidence, so the badge is left as it stands" };
|
|
148
148
|
}
|
|
149
|
+
// The same rule for the two other receipts qa/receipt-check.mjs refuses as
|
|
150
|
+
// done-evidence: smoke (Rule 0 — proves the framework, never the change) and
|
|
151
|
+
// nightly (proves the harness and the tree's invariants). Both derive no
|
|
152
|
+
// rung, and a smoke run — scripts/framework-check.mjs runs one on every
|
|
153
|
+
// scaffold — was rewriting a true L1 badge to "rung unrecorded". Found on
|
|
154
|
+
// 2026-09-03 by deriving the affected filter on a fresh app: README.md was
|
|
155
|
+
// the dirty file. Receipts predating `stage` are read by profile.
|
|
156
|
+
const stage = receipt && (typeof receipt.stage === "string" ? receipt.stage : receipt.profile);
|
|
157
|
+
if (stage === "smoke" || stage === "nightly") {
|
|
158
|
+
return { changed: false, reason: `${stage} run — refused as done-evidence, so the badge is left as it stands` };
|
|
159
|
+
}
|
|
149
160
|
|
|
150
161
|
const body = `${renderEvidenceBadge(receipt)}\n`;
|
|
151
162
|
const next = readme.replace(
|
|
@@ -41,6 +41,7 @@ const SCAFFOLD_CORE = [
|
|
|
41
41
|
"approvals",
|
|
42
42
|
"componentStories",
|
|
43
43
|
"reachability",
|
|
44
|
+
"e2eCoverage",
|
|
44
45
|
"archDoc",
|
|
45
46
|
"schemaHistory",
|
|
46
47
|
"build",
|
|
@@ -65,6 +66,27 @@ const RELEASE_EXECUTION = "releaseSmoke";
|
|
|
65
66
|
|
|
66
67
|
const RUNG_NAMES = { L0: "scaffold", L1: "desktop", L2: "device", L3: "release" };
|
|
67
68
|
|
|
69
|
+
/**
|
|
70
|
+
* The Compose Multiplatform pack's ladder — the step names above, as one
|
|
71
|
+
* object a pack hands to the spine. THE LADDER IS THE PACK'S, NOT THE SPINE'S
|
|
72
|
+
* (2026-09-03): vendored into a Kotlin backend, these names graded its
|
|
73
|
+
* strongest run — detekt, Konsist, gitleaks — as L0 "scaffold" and
|
|
74
|
+
* made L1 unreachable by construction. A fixed-amount understatement is not
|
|
75
|
+
* conservative, it is wrong, and receipts are where labels get quoted. So a
|
|
76
|
+
* pack declares its ladder (`evidenceLadder` on createXSteps' return); a pack
|
|
77
|
+
* that declares none earns no rung at all, which is the honest grade for a
|
|
78
|
+
* ladder nobody has calibrated. Every field is a list of step names except
|
|
79
|
+
* `release`, one name. `names` maps rung → label.
|
|
80
|
+
*/
|
|
81
|
+
export const CMP_LADDER = Object.freeze({
|
|
82
|
+
scaffoldCore: Object.freeze(SCAFFOLD_CORE),
|
|
83
|
+
l0Required: Object.freeze(L0_REQUIRED),
|
|
84
|
+
l1Required: Object.freeze(L1_REQUIRED),
|
|
85
|
+
deviceExecution: Object.freeze(DEVICE_EXECUTION),
|
|
86
|
+
release: RELEASE_EXECUTION,
|
|
87
|
+
names: Object.freeze(RUNG_NAMES),
|
|
88
|
+
});
|
|
89
|
+
|
|
68
90
|
/**
|
|
69
91
|
* Derive the receipt's evidence rung from the lane's step results.
|
|
70
92
|
*
|
|
@@ -81,8 +103,18 @@ const RUNG_NAMES = { L0: "scaffold", L1: "desktop", L2: "device", L3: "release"
|
|
|
81
103
|
* was not earned. `satisfiedBy` lists the PASSed steps the rung counts as
|
|
82
104
|
* its evidence, in lane order.
|
|
83
105
|
*/
|
|
84
|
-
export function evidenceLevel(stepResults, profile, { mode } = {}) { // eslint-disable-line no-unused-vars
|
|
106
|
+
export function evidenceLevel(stepResults, profile, { mode, ladder } = {}) { // eslint-disable-line no-unused-vars
|
|
85
107
|
if (mode === "fast") return null; // the inner loop derives no rung — ever
|
|
108
|
+
// `ladder` absent → the Compose ladder (every caller before packs declared
|
|
109
|
+
// one). `ladder: null` → the pack declares none: no rung, by decision.
|
|
110
|
+
if (ladder === null) return null;
|
|
111
|
+
const L = ladder ?? CMP_LADDER;
|
|
112
|
+
const SCAFFOLD_CORE = L.scaffoldCore ?? [];
|
|
113
|
+
const L0_REQUIRED = L.l0Required ?? [];
|
|
114
|
+
const L1_REQUIRED = L.l1Required ?? [];
|
|
115
|
+
const DEVICE_EXECUTION = L.deviceExecution ?? [];
|
|
116
|
+
const RELEASE_EXECUTION = L.release ?? null;
|
|
117
|
+
const RUNG_NAMES = L.names ?? CMP_LADDER.names;
|
|
86
118
|
const steps = Array.isArray(stepResults) ? stepResults.filter((s) => s && typeof s.name === "string") : [];
|
|
87
119
|
// A failed lane has no rung — and a lane with a step that could not run
|
|
88
120
|
// (ERROR) has none either: a rung is evidence, and "could not check" is not.
|
|
@@ -108,7 +140,7 @@ export function evidenceLevel(stepResults, profile, { mode } = {}) { // eslint-d
|
|
|
108
140
|
|
|
109
141
|
// Only a PASSed releaseSmoke lifts to L3 — a SKIP (unsigned keystore,
|
|
110
142
|
// no device) never does.
|
|
111
|
-
if (passed.has(RELEASE_EXECUTION)) {
|
|
143
|
+
if (RELEASE_EXECUTION && passed.has(RELEASE_EXECUTION)) {
|
|
112
144
|
rung = "L3";
|
|
113
145
|
counted.add(RELEASE_EXECUTION);
|
|
114
146
|
}
|
|
@@ -334,15 +334,24 @@ export function deriveFeatureStatus(root, brief, pre = {}) {
|
|
|
334
334
|
const specRels = specNames.map((n) => `specs/${n}.spec.md`);
|
|
335
335
|
const specExists = specRels.every((rel) => fs.existsSync(path.join(root, rel)));
|
|
336
336
|
const specRel = specRels.join(" + ");
|
|
337
|
-
const
|
|
337
|
+
const citations = pre.citations ?? scanCitations(root);
|
|
338
|
+
const citedIds = new Set(citations.map((t) => t.id));
|
|
339
|
+
// Which clauses a DEVICE journey proves: citations from qa/e2e flows (tier
|
|
340
|
+
// "e2e" — spec-coverage's tierForFile). A UI feature (screens: true) is not
|
|
341
|
+
// done until at least one of its live clauses is cited from a flow: JVM
|
|
342
|
+
// tests prove logic and structure, the flow proves the journey on a device.
|
|
343
|
+
const e2eCitedIds = new Set(citations.filter((t) => t.tier === "e2e").map((t) => t.id));
|
|
338
344
|
const clauses = specRels
|
|
339
345
|
.flatMap((rel) => clausesOfSpec(root, rel))
|
|
340
|
-
.map((c) => ({ ...c, cited: citedIds.has(c.id) }));
|
|
346
|
+
.map((c) => ({ ...c, cited: citedIds.has(c.id), e2eCited: e2eCitedIds.has(c.id) }));
|
|
341
347
|
const live = clauses.filter((c) => !c.withdrawn);
|
|
342
348
|
const covered = live.filter((c) => c.cited).length;
|
|
349
|
+
const e2eCovered = live.filter((c) => c.e2eCited).length;
|
|
350
|
+
const needsJourney = block.screens === true && block.unrouted !== true;
|
|
343
351
|
|
|
344
352
|
const receipt = pre.receipt ?? receiptAttestation(root);
|
|
345
|
-
const provenDone =
|
|
353
|
+
const provenDone =
|
|
354
|
+
live.length > 0 && covered === live.length && (!needsJourney || e2eCovered > 0) && receipt.verdict === "PASS" && receipt.attestsTree;
|
|
346
355
|
|
|
347
356
|
return {
|
|
348
357
|
name: brief.name,
|
|
@@ -361,6 +370,8 @@ export function deriveFeatureStatus(root, brief, pre = {}) {
|
|
|
361
370
|
specExists,
|
|
362
371
|
clauses,
|
|
363
372
|
covered,
|
|
373
|
+
e2eCovered,
|
|
374
|
+
needsJourney,
|
|
364
375
|
total: live.length,
|
|
365
376
|
receipt,
|
|
366
377
|
provenDone,
|
|
@@ -372,6 +383,8 @@ export function deriveFeatureStatus(root, brief, pre = {}) {
|
|
|
372
383
|
? `no spec yet (${specRel}) — behavior starts as clauses there`
|
|
373
384
|
: live.length === 0
|
|
374
385
|
? `${specRel} has no live clauses — nothing is promised yet`
|
|
386
|
+
: needsJourney && covered === live.length && e2eCovered === 0
|
|
387
|
+
? `${covered}/${live.length} clauses cited, but none from a qa/e2e flow — a UI feature is proven on a device: write the journey in qa/e2e/${brief.name}.yaml and cite the clause it proves`
|
|
375
388
|
: covered < live.length
|
|
376
389
|
? `${covered}/${live.length} clauses cited — ${live.length - covered} promise(s) have no citing test`
|
|
377
390
|
: !receipt.present
|
|
@@ -134,7 +134,12 @@ export function checkHarnessIntegrity(root) {
|
|
|
134
134
|
*/
|
|
135
135
|
export function describeIntegrity(r) {
|
|
136
136
|
if (r.status === "intact") {
|
|
137
|
-
|
|
137
|
+
// The region digest rides beside the version: two lanes can carry the same
|
|
138
|
+
// package version with different content (create-cmp-showcase, 2026-09-03 —
|
|
139
|
+
// seven files changed, "0.16.0" on both receipts), and a receipt must name
|
|
140
|
+
// WHICH lane produced it in words a human reads, not only in the lock file.
|
|
141
|
+
const region = typeof r.sha256 === "string" && r.sha256 ? ` (region ${r.sha256.slice(0, 8)})` : "";
|
|
142
|
+
return `${r.name ?? "harness"} ${r.version ?? "?"}${region} — ${r.fileCount} files verified`;
|
|
138
143
|
}
|
|
139
144
|
if (r.status === "unlocked") {
|
|
140
145
|
return `no ${LOCK_PATH} — this app's lane version is unrecorded`;
|
|
@@ -232,6 +232,33 @@ function resolveSurfaceFiles(root, VERIFIED_SURFACE) {
|
|
|
232
232
|
return collected.filter((relPath) => !isExcluded(relPath));
|
|
233
233
|
}
|
|
234
234
|
|
|
235
|
+
/**
|
|
236
|
+
* Top-level entries (first path segment of every file git would commit) that
|
|
237
|
+
* NO surface entry covers — the files the receipt does not attest. An
|
|
238
|
+
* allowlist is silent about what it omits: a new top-level directory is simply
|
|
239
|
+
* unmatched, no error, unattested (payment-blueprint, 2026-09-03). This names
|
|
240
|
+
* the omission so the receipt can carry it and a reader can decide whether
|
|
241
|
+
* it belongs in qa/verified-surface.json. Sorted; [] when git is unavailable
|
|
242
|
+
* (the walk fallback has no notion of "what git sees") or everything is
|
|
243
|
+
* covered. Lane outputs (EXCLUDED_PREFIXES) are not "undeclared" — they are
|
|
244
|
+
* excluded by decision.
|
|
245
|
+
* @param {string} root
|
|
246
|
+
* @param {string[]} [surface] defaults to resolveVerifiedSurface(root)
|
|
247
|
+
* @returns {string[]}
|
|
248
|
+
*/
|
|
249
|
+
export function undeclaredTopLevel(root, surface = resolveVerifiedSurface(root)) {
|
|
250
|
+
const gitFiles = tryGitLsFiles(root);
|
|
251
|
+
if (!gitFiles) return [];
|
|
252
|
+
const covered = (relPath) => surface.some((entry) => relPath === entry || relPath.startsWith(`${entry}/`)) || isExcluded(relPath);
|
|
253
|
+
const out = new Set();
|
|
254
|
+
for (const raw of gitFiles) {
|
|
255
|
+
const relPath = raw.split(path.sep).join("/");
|
|
256
|
+
if (covered(relPath)) continue;
|
|
257
|
+
out.add(relPath.includes("/") ? relPath.slice(0, relPath.indexOf("/")) : relPath);
|
|
258
|
+
}
|
|
259
|
+
return [...out].sort();
|
|
260
|
+
}
|
|
261
|
+
|
|
235
262
|
/**
|
|
236
263
|
* Compute the sha256 hash of the verified surface for the project rooted at `root`.
|
|
237
264
|
* Deterministic: same tree (same file paths + same file bytes) → same hash.
|
|
@@ -145,6 +145,13 @@ export function runLane(ctx) {
|
|
|
145
145
|
} catch (err) {
|
|
146
146
|
result = stepErrorResult(name, err, Date.now() - stepStarted);
|
|
147
147
|
}
|
|
148
|
+
// Layer tag: a pack may mark a step function with the layer of the
|
|
149
|
+
// stack it proves (`fn.layer = "backend"`). The runner stamps it onto
|
|
150
|
+
// the row so the receipt carries it and the console can group by it —
|
|
151
|
+
// a step that set its own `layer` in the result keeps its word.
|
|
152
|
+
if (result && typeof result === "object" && typeof step.layer === "string" && step.layer && typeof result.layer !== "string") {
|
|
153
|
+
result.layer = step.layer;
|
|
154
|
+
}
|
|
148
155
|
results.push(result);
|
|
149
156
|
if (print) {
|
|
150
157
|
print(
|
|
@@ -58,7 +58,29 @@ const TYPE_DECL_RE = /^(?:@\w+\s+)*(?:public\s+|internal\s+|private\s+|abstract\
|
|
|
58
58
|
|
|
59
59
|
// A YAML flow's own shape counts as its test: a Maestro file IS the test, so a
|
|
60
60
|
// tag in one binds to the flow rather than to a declaration inside it.
|
|
61
|
-
const FLOW_EXTS = [".yaml", ".yml"];
|
|
61
|
+
export const FLOW_EXTS = [".yaml", ".yml"];
|
|
62
|
+
/** Where Maestro flows live. The lane runs this DIRECTORY (every top-level flow). */
|
|
63
|
+
export const E2E_FLOW_DIR = "qa/e2e";
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* The flows the lane executes: top-level `*.yaml`/`*.yml` under qa/e2e, sorted,
|
|
67
|
+
* root-relative. ONE list serves two readers — the e2eSmoke step (what runs)
|
|
68
|
+
* and scanCitations (what may count as coverage) — so a citation can only
|
|
69
|
+
* ever come from a flow that executes. Before 2026-09-03 the step ran ONE
|
|
70
|
+
* file by name (smoke.yaml) while the scan walked the whole directory: four
|
|
71
|
+
* hand-written flows on the showcase satisfied clauses without ever running.
|
|
72
|
+
* @param {string} root
|
|
73
|
+
* @returns {string[]}
|
|
74
|
+
*/
|
|
75
|
+
export function listFlowFiles(root) {
|
|
76
|
+
const dir = path.join(root, E2E_FLOW_DIR);
|
|
77
|
+
if (!fs.existsSync(dir)) return [];
|
|
78
|
+
return fs
|
|
79
|
+
.readdirSync(dir, { withFileTypes: true })
|
|
80
|
+
.filter((e) => e.isFile() && FLOW_EXTS.some((ext) => e.name.endsWith(ext)))
|
|
81
|
+
.map((e) => `${E2E_FLOW_DIR}/${e.name}`)
|
|
82
|
+
.sort();
|
|
83
|
+
}
|
|
62
84
|
|
|
63
85
|
/**
|
|
64
86
|
* Does a test declaration follow `index` within BINDING_WINDOW non-blank lines,
|
|
@@ -170,8 +192,10 @@ export const DESKTOP_TIERS = Object.freeze(["commonTest", "desktopTest"]);
|
|
|
170
192
|
*/
|
|
171
193
|
export function scanCitations(root) {
|
|
172
194
|
const tags = [];
|
|
173
|
-
|
|
174
|
-
|
|
195
|
+
// Kotlin sources anywhere under composeApp/src; flows ONLY from the list the
|
|
196
|
+
// lane runs (listFlowFiles) — a flow in a subfolder, or a yaml under
|
|
197
|
+
// composeApp/src, is not executed by e2eSmoke and therefore proves nothing.
|
|
198
|
+
const files = [...walkFiles(path.join(root, "composeApp/src"), [".kt", ".kts"]), ...listFlowFiles(root).map((rel) => path.join(root, rel))];
|
|
175
199
|
for (const f of files) {
|
|
176
200
|
const rel = path.relative(root, f);
|
|
177
201
|
const tier = tierForFile(rel);
|
|
@@ -121,3 +121,107 @@ export function stepErrorResult(name, err, durationMs) {
|
|
|
121
121
|
`Nothing here is a claim about your change.`;
|
|
122
122
|
return { name, verdict: "ERROR", reason, durationMs, details: { executed: false, kind: timeout ? "deadline" : "threw" } };
|
|
123
123
|
}
|
|
124
|
+
|
|
125
|
+
// ── Maestro directory run → per-flow outcome ────────────────────────────────
|
|
126
|
+
/**
|
|
127
|
+
* Parse Maestro's JUnit report (`maestro test <dir> --format junit --output f`)
|
|
128
|
+
* into one row per flow. Tolerant: a report that is missing or unparsable
|
|
129
|
+
* returns null and the caller falls back to the exit code — never a fabricated
|
|
130
|
+
* per-flow list.
|
|
131
|
+
* @param {string|null} xml
|
|
132
|
+
* @returns {Array<{flow: string, ok: boolean, message?: string}>|null}
|
|
133
|
+
*/
|
|
134
|
+
export function parseMaestroJunit(xml) {
|
|
135
|
+
if (typeof xml !== "string" || !/<testcase\b/.test(xml)) return null;
|
|
136
|
+
const rows = [];
|
|
137
|
+
const caseRe = /<testcase\b([^>]*?)(?:\/>|>([\s\S]*?)<\/testcase>)/g;
|
|
138
|
+
let m;
|
|
139
|
+
while ((m = caseRe.exec(xml))) {
|
|
140
|
+
const attrs = m[1] || "";
|
|
141
|
+
const body = m[2] || "";
|
|
142
|
+
const name = (attrs.match(/\bname="([^"]*)"/) || [])[1] ?? (attrs.match(/\bid="([^"]*)"/) || [])[1] ?? "?";
|
|
143
|
+
const status = (attrs.match(/\bstatus="([^"]*)"/) || [])[1];
|
|
144
|
+
const failed = /<(failure|error)\b/.test(body) || (status && !/^(SUCCESS|PASSED?|OK)$/i.test(status));
|
|
145
|
+
const message = failed ? ((body.match(/<(?:failure|error)\b[^>]*message="([^"]*)"/) || [])[1] ?? body.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim().slice(0, 300)) : undefined;
|
|
146
|
+
rows.push(failed ? { flow: name, ok: false, message } : { flow: name, ok: true });
|
|
147
|
+
}
|
|
148
|
+
return rows.length ? rows : null;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* The e2e step's verdict from the Maestro run: exit code + per-flow report +
|
|
153
|
+
* the list of flows the directory held. FAIL names every failing flow; a run
|
|
154
|
+
* whose report lists fewer flows than the directory holds is ERROR — the lane
|
|
155
|
+
* cannot claim flows it has no row for.
|
|
156
|
+
* @param {{ok: boolean, out: string}} res
|
|
157
|
+
* @param {Array<{flow: string, ok: boolean, message?: string}>|null} perFlow
|
|
158
|
+
* @param {string[]} flows root-relative flow files the directory holds
|
|
159
|
+
* @returns {{verdict: "PASS"|"FAIL"|"ERROR", reason?: string, details: object}}
|
|
160
|
+
*/
|
|
161
|
+
export function maestroOutcome(res, perFlow, flows) {
|
|
162
|
+
const details = { flows, results: perFlow ?? undefined };
|
|
163
|
+
if (perFlow) {
|
|
164
|
+
const failed = perFlow.filter((r) => !r.ok);
|
|
165
|
+
if (failed.length) {
|
|
166
|
+
return {
|
|
167
|
+
verdict: "FAIL",
|
|
168
|
+
reason: `Maestro: ${failed.length} of ${perFlow.length} flow${perFlow.length === 1 ? "" : "s"} failed — ${failed.map((r) => `${r.flow}${r.message ? ` (${r.message.split("\n")[0].slice(0, 120)})` : ""}`).join("; ")}`,
|
|
169
|
+
details,
|
|
170
|
+
};
|
|
171
|
+
}
|
|
172
|
+
if (perFlow.length < flows.length) {
|
|
173
|
+
return {
|
|
174
|
+
verdict: "ERROR",
|
|
175
|
+
reason: `Maestro reported ${perFlow.length} flow${perFlow.length === 1 ? "" : "s"} but ${E2E_DIR_LABEL} holds ${flows.length} — the run did not cover every flow, so no verdict can be claimed for the rest`,
|
|
176
|
+
details,
|
|
177
|
+
};
|
|
178
|
+
}
|
|
179
|
+
if (!res.ok) {
|
|
180
|
+
return { verdict: "FAIL", reason: `Maestro exited non-zero with every flow reported green — treat as a run failure:\n${String(res.out).split("\n").slice(-10).join("\n")}`, details };
|
|
181
|
+
}
|
|
182
|
+
return { verdict: "PASS", details };
|
|
183
|
+
}
|
|
184
|
+
if (!res.ok) {
|
|
185
|
+
return { verdict: "FAIL", reason: `Maestro failed (no per-flow report was written):\n${String(res.out).split("\n").slice(-15).join("\n")}`, details };
|
|
186
|
+
}
|
|
187
|
+
return { verdict: "PASS", reason: "Maestro exited 0 but wrote no per-flow report — verdict from the exit code only", details };
|
|
188
|
+
}
|
|
189
|
+
const E2E_DIR_LABEL = "qa/e2e";
|
|
190
|
+
|
|
191
|
+
// ── Device-log incidents, scoped to the app under test ──────────────────────
|
|
192
|
+
/**
|
|
193
|
+
* ANR / fatal-exception lines from `adb logcat -d -b system,crash,main` that
|
|
194
|
+
* belong to one of `appIds` (an app's process may be `pkg` or `pkg:remote`).
|
|
195
|
+
* An emulator carries other apps — on 2026-09-03 the first self-booted lane
|
|
196
|
+
* went red on `ANR in com.karel.bratometer` while driving com.fleet.check —
|
|
197
|
+
* so an incident in another package is NOT this lane's failure. With no
|
|
198
|
+
* appIds known the sweep stays unscoped (every incident counts) and says so.
|
|
199
|
+
* @param {string} log
|
|
200
|
+
* @param {string[]} appIds
|
|
201
|
+
* @returns {{lines: string[], scoped: boolean}}
|
|
202
|
+
*/
|
|
203
|
+
export function deviceLogIncidents(log, appIds = []) {
|
|
204
|
+
const lines = String(log ?? "").split("\n");
|
|
205
|
+
const ids = appIds.filter((id) => typeof id === "string" && id.trim() && id !== "__PACKAGE__");
|
|
206
|
+
const scoped = ids.length > 0;
|
|
207
|
+
const ours = (proc) => !scoped || ids.some((id) => proc === id || proc.startsWith(`${id}:`));
|
|
208
|
+
const out = [];
|
|
209
|
+
lines.forEach((line, i) => {
|
|
210
|
+
const anr = line.match(/ANR in (\S+?)(?:\s|,|$)/);
|
|
211
|
+
if (anr) {
|
|
212
|
+
if (ours(anr[1].replace(/[,)]$/, ""))) out.push(line);
|
|
213
|
+
return;
|
|
214
|
+
}
|
|
215
|
+
if (/FATAL EXCEPTION/i.test(line)) {
|
|
216
|
+
if (!scoped) {
|
|
217
|
+
out.push(line);
|
|
218
|
+
return;
|
|
219
|
+
}
|
|
220
|
+
// The crash buffer names the process on one of the next few lines.
|
|
221
|
+
const window = lines.slice(i, i + 8).join("\n");
|
|
222
|
+
const proc = window.match(/Process:\s*(\S+?)(?:,|\s|$)/);
|
|
223
|
+
if (proc && ours(proc[1])) out.push(line);
|
|
224
|
+
}
|
|
225
|
+
});
|
|
226
|
+
return { lines: out, scoped };
|
|
227
|
+
}
|