create-cmp-cli 0.18.0 → 0.20.0

package/packages/harness/src/lib/spec-coverage.mjs

@@ -14,8 +14,94 @@ import path from "node:path";
 
 /** `- **HOME-01** — …` (live) or `- ~~**HOME-01**~~ — …` (withdrawn). */
 export const CLAUSE_LINE_RE = /^-\s+(~~)?\*\*([A-Z][A-Z0-9]*-\d{2,})\*\*/;
+// An OPTIONAL tier requirement on the clause line itself:
+//
+//   - **MOTION-13** [tier: device] — Given a cold start, When … Then …
+//
+// The clause declares what it takes to OBSERVE it, which is a property of the
+// promise, not of whatever test happened to cite it. Note this attaches to the
+// clause line, not to `[enforced: …]` — that tags docs/ARCHITECTURE.md prose and
+// is a different grammar entirely.
+const CLAUSE_TIER_RE = /\[tier:\s*(device|e2e)\]/i;
+/** Which citing tiers satisfy a declared requirement. */
+export const TIERS_SATISFYING = Object.freeze({
+  device: ["androidInstrumentedTest", "e2e"],
+  e2e: ["e2e"],
+});
 
 const TAG_LINE_RE = /^(?:\/\/|#)\s*SPEC:/;
+
+// A citation is a claim that a TEST covers a clause, so it has to sit on one.
+// Counting the tag wherever it appears makes a red specCoverage curable with a
+// comment and zero assertions — the one escape this gate exists to close. It is
+// not hypothetical: payment-blueprint hit a citation that had drifted onto a
+// class declaration, where it counted for the whole file while testing nothing.
+//
+// So a tag counts only when a test declaration follows it within
+// BINDING_WINDOW non-blank lines. The window is small enough that the tag must
+// be attached to the test, and loose enough for the @DisplayName / annotation
+// stack that idiomatically sits between them.
+export const BINDING_WINDOW = 5;
+
+// Kotlin @Test, a backticked test function, and the node:test / Maestro-adjacent
+// `test(` / `it(` call forms. Deliberately syntactic: a citation's binding must
+// be readable without compiling anything.
+const TEST_DECL_RE = /@Test\b|\bfun\s+`[^`]+`\s*\(|\b(?:test|it)\s*\(/;
+
+// A tag whose first meaningful line declares a TYPE is documenting that type,
+// not claiming a test — and it must be refused structurally rather than by
+// distance, because a short class body puts a real @Test inside the window and
+// would otherwise launder the citation. This is exactly payment-blueprint's
+// drift: `// SPEC: PP-07` sat on `class PaymentWorkerTest`, three properties
+// above a genuine @Test, and vouched for the whole file.
+const TYPE_DECL_RE = /^(?:@\w+\s+)*(?:public\s+|internal\s+|private\s+|abstract\s+|open\s+|sealed\s+|data\s+|enum\s+)*(?:class|object|interface)\b/;
+
+// A YAML flow's own shape counts as its test: a Maestro file IS the test, so a
+// tag in one binds to the flow rather than to a declaration inside it.
+const FLOW_EXTS = [".yaml", ".yml"];
+
+/**
+ * Does a test declaration follow `index` within BINDING_WINDOW non-blank lines,
+ * skipping block-comment bodies (a tag inside one is documentation, not a claim)?
+ * @param {string[]} lines
+ * @param {number} index line the tag sits on
+ * @returns {boolean}
+ */
+export function citationIsBound(lines, index) {
+  let seen = 0;
+  let inBlockComment = false;
+  for (let i = index + 1; i < lines.length && seen < BINDING_WINDOW; i += 1) {
+    const line = lines[i].trim();
+    if (line === "") continue;
+    if (inBlockComment) {
+      if (line.includes("*/")) inBlockComment = false;
+      continue;
+    }
+    if (line.startsWith("/*")) {
+      if (!line.includes("*/")) inBlockComment = true;
+      continue;
+    }
+    if (line.startsWith("//") || line.startsWith("*")) continue;
+    seen += 1;
+    // The FIRST meaningful line decides whether this tag is on a test at all.
+    if (seen === 1 && TYPE_DECL_RE.test(line)) return false;
+    if (TEST_DECL_RE.test(line)) return true;
+  }
+  return false;
+}
+
+/** Is this tag inside a block comment that began earlier in the file? */
+function insideBlockComment(lines, index) {
+  let open = false;
+  for (let i = 0; i < index; i += 1) {
+    const line = lines[i];
+    for (let c = 0; c < line.length - 1; c += 1) {
+      if (!open && line[c] === "/" && line[c + 1] === "*") open = true;
+      else if (open && line[c] === "*" && line[c + 1] === "/") open = false;
+    }
+  }
+  return open;
+}
 const TAG_IDS_RE = /SPEC:\s*([A-Z0-9,\s-]+)/;
 const CLAUSE_ID_RE = /^[A-Z][A-Z0-9]*-\d{2,}$/;
 
@@ -45,7 +131,12 @@ export function scanSpecClauses(root) {
     for (const line of fs.readFileSync(abs, "utf8").split("\n")) {
       const m = line.match(CLAUSE_LINE_RE);
       if (!m) continue;
-      clauses.set(m[2], { file: path.relative(root, abs), withdrawn: Boolean(m[1]) });
+      const tierMatch = line.match(CLAUSE_TIER_RE);
+      clauses.set(m[2], {
+        file: path.relative(root, abs),
+        withdrawn: Boolean(m[1]),
+        requiredTier: tierMatch ? tierMatch[1].toLowerCase() : null,
+      });
     }
   }
   return clauses;
@@ -86,11 +177,14 @@ export function scanCitations(root) {
     const tier = tierForFile(rel);
     fs.readFileSync(f, "utf8")
       .split("\n")
-      .forEach((line, i) => {
+      .forEach((line, i, lines) => {
         const trimmed = line.trim();
         if (!TAG_LINE_RE.test(trimmed)) return;
         const m = trimmed.match(TAG_IDS_RE);
         if (!m) return;
+        // A flow file IS its test; anything else must have a test under the tag.
+        const isFlow = FLOW_EXTS.some((ext) => rel.endsWith(ext));
+        if (!isFlow && (insideBlockComment(lines, i) || !citationIsBound(lines, i))) return;
         const ids = m[1]
           .split(/[,\s]+/)
           .map((s) => s.trim())
@@ -108,6 +202,9 @@ export function scanCitations(root) {
  * (commonTest/desktopTest) — behavior claims no device-tier evidence backs.
  * `summaryLine` is the one line the lane's specCoverage step (and any other
  * consumer) can print verbatim; null when nothing is desktop-only.
+ * `unmetTier` is the PRESCRIPTIVE half — clauses that declared `[tier: …]` and
+ * have no citation from a tier that could observe them. specCoverage FAILS on it:
+ * "instrument before you police" was the right first move, and this is the second.
  * @param {Map<string, {file: string, withdrawn: boolean}>} clauses from scanSpecClauses
  * @param {Array<{id: string, tier: string}>} tags from scanCitations
  * @returns {{tiersByClause: Record<string, string[]>, desktopOnly: string[], summaryLine: string|null}}
@@ -117,6 +214,17 @@ export function clauseTierCoverage(clauses, tags) {
   for (const t of tags) {
     (tiersByClause[t.id] ??= []).includes(t.tier) || tiersByClause[t.id].push(t.tier);
   }
+  // The gate input. A clause that DECLARED the tier it needs and has no citation
+  // from that tier is not covered — it is cited by tests structurally incapable
+  // of observing it, which is the exact hole `desktopOnly` below could only ever
+  // describe. MOTION-13 promised an animation "plays once per process start" and
+  // was cited by a desktop Compose test, a tier with no process lifecycle at all:
+  // the citation existed, the gate went green, and nothing ever observed the
+  // promise. Declared requirements are checked; undeclared clauses are unchanged.
+  const unmetTier = [...clauses.entries()]
+    .filter(([, c]) => !c.withdrawn && c.requiredTier)
+    .map(([id, c]) => ({ id, requiredTier: c.requiredTier, tiers: tiersByClause[id] ?? [], file: c.file }))
+    .filter((u) => !(TIERS_SATISFYING[u.requiredTier] ?? []).some((t) => u.tiers.includes(t)));
   const desktopOnly = [...clauses.entries()]
     .filter(([, c]) => !c.withdrawn)
     .map(([id]) => id)
@@ -127,5 +235,5 @@ export function clauseTierCoverage(clauses, tags) {
   const summaryLine = desktopOnly.length
     ? `${desktopOnly.length} clause${desktopOnly.length === 1 ? "" : "s"} cited only from desktop-tier tests (${desktopOnly.join(", ")})`
     : null;
-  return { tiersByClause, desktopOnly, summaryLine };
+  return { tiersByClause, desktopOnly, unmetTier, summaryLine };
 }

package/packages/harness/src/lib/step-cache.mjs

@@ -138,7 +138,7 @@ export function loadStepCache(root) {
 export function lookupCachedPass(root, stepName, inputsHash) {
   const entry = loadStepCache(root).steps[stepName];
   if (!entry || typeof entry !== "object") return null;
-  if (entry.verdict !== "PASS") return null; // FAIL/SKIP are never reused
+  if (entry.verdict !== "PASS") return null; // FAIL/SKIP/ERROR are never reused
   if (typeof inputsHash !== "string" || entry.inputsHash !== inputsHash) return null;
   if (typeof entry.at !== "string") return null;
   return entry;

package/packages/harness/src/lib/step-outcomes.mjs

@@ -0,0 +1,123 @@
+// step-outcomes.mjs — a step's VERDICT, separated from its INVOCATION.
+//
+// A step that ran zero tests knows nothing about behaviour and must not speak
+// as though it does. Observed 2026-09-02 (create-cmp-showcase): a concurrent
+// adb session collided with androidChecks, Gradle exited non-zero having
+// executed no tests, and the step reported "an on-device behavior claim is
+// broken. Fix the behavior, not the test." The identical task passed 8 tests
+// moments later. Believed, that sends the reader hunting a defect that does not
+// exist; disbelieved once, it teaches them to discount every future red from
+// the step — a gate that misattributes its own failures corrodes the gates that
+// are right.
+//
+// Pure, so the wording and the rule are testable without Gradle or a device.
+// (docs/proposals/evidence-economics.md C3, S4.)
+//
+// FOUR VERDICTS. PASS / FAIL / SKIP had no way to say "I could not run": a
+// step whose infrastructure broke reported a behaviour failure. ERROR is that
+// fourth word — zero tests executed, a deadline passed, a tool vanished, a
+// step threw. An ERROR never accuses the change, never counts as evidence
+// (evidence-level derives no rung over it; the plausibility check does not
+// count it as executed), is visibly distinct from FAIL (⊘, not ✗), and is
+// never silently retried. It still makes the lane FAIL — "could not check" is
+// not green. This is JUnit's error-vs-failure, Bazel's FAILED_TO_BUILD /
+// TIMEOUT vs FAILED, pytest's error vs failed — the distinction every mature
+// runner makes and this one did not.
+
+/**
+ * The androidChecks outcome from Gradle's exit and the JUnit summary.
+ *
+ * @param {{ok: boolean, out: string}} res the Gradle invocation
+ * @param {{tests: number, failures: number, errors: number}|null} summary parsed JUnit
+ *   results, or null when none were written
+ * @param {{gradlew?: string}} [opts]
+ * @returns {{verdict: "PASS"|"FAIL"|"ERROR", executed: boolean, reason?: string}}
+ */
+export function androidChecksOutcome(res, summary, { gradlew = "./gradlew" } = {}) {
+  const executed = Boolean(summary && summary.tests > 0);
+  if (res.ok) return { verdict: "PASS", executed };
+  const tail = String(res.out ?? "")
+    .split("\n")
+    .filter((l) => /FAILED|error:|failed/i.test(l))
+    .slice(0, 12)
+    .join("\n");
+  if (executed) {
+    return {
+      verdict: "FAIL",
+      executed,
+      reason:
+        `connectedDebugAndroidTest failed (${summary.failures + summary.errors} of ${summary.tests} tests) — ` +
+        `an on-device behavior claim is broken. Fix the behavior, not the test:\n${tail}`,
+    };
+  }
+  // ERROR, not FAIL: the step could not execute. A device tier that could not
+  // run is not evidence (the lane still FAILs), and going green would be the
+  // worse lie — but "your behaviour is broken" is withdrawn, and the receipt
+  // can tell a red that measured something from a red that measured nothing.
+  return {
+    verdict: "ERROR",
+    executed,
+    reason:
+      "connectedDebugAndroidTest DID NOT EXECUTE — the run reported no tests at all, so this step has observed " +
+      "nothing about your change and is not accusing it. Usual cause: another adb/Gradle session touching the same " +
+      "device (a manual `adb` command, a second lane, a running preview), or an install that never landed. " +
+      `Re-run this step alone with nothing else on the device before suspecting the code:\n  ${gradlew} :composeApp:connectedDebugAndroidTest --rerun\n${tail}`,
+  };
+}
+
+/** Thrown by the lane's subprocess helper when a step's deadline passes. */
+export class StepTimeout extends Error {
+  constructor(cmd, deadlineMs) {
+    super(`deadline of ${Math.round(deadlineMs / 60000)} min passed: ${cmd}`);
+    this.name = "StepTimeout";
+    this.cmd = cmd;
+    this.deadlineMs = deadlineMs;
+  }
+}
+
+/**
+ * Did a spawnSync result hit its deadline? Node reports ETIMEDOUT on
+ * `error.code` and the kill signal on `signal`; either alone is enough — an
+ * older Node sets only one of them.
+ * @param {{error?: {code?: string}, signal?: string|null}} res
+ * @returns {boolean}
+ */
+export function spawnTimedOut(res) {
+  if (!res) return false;
+  if (res.error && res.error.code === "ETIMEDOUT") return true;
+  return res.signal === "SIGTERM" && (res.status === null || res.status === undefined);
+}
+
+/**
+ * A step's own deadline, from the journal's last measured duration for it:
+ * three times what it usually takes, never under five minutes (a cold Gradle
+ * daemon is slow, not wedged), never over thirty (past that it IS wedged).
+ * Unknown steps get the ceiling — a first run is never cut short.
+ * @param {number|null|undefined} expectedMs
+ * @returns {number}
+ */
+export function stepDeadlineMs(expectedMs, { floorMs = 5 * 60_000, ceilingMs = 30 * 60_000 } = {}) {
+  if (!(expectedMs > 0)) return ceilingMs;
+  return Math.min(ceilingMs, Math.max(floorMs, Math.round(expectedMs * 3)));
+}
+
+/**
+ * The step result for a step that could not run — a deadline, or any throw
+ * out of the step's own body (which used to crash the whole lane; now it is
+ * one ERROR row and the lane keeps going, because the other steps' verdicts
+ * are still worth having).
+ * @param {string} name the step's display name
+ * @param {unknown} err
+ * @param {number} durationMs
+ * @returns {{name: string, verdict: "ERROR", reason: string, durationMs: number, details: {executed: false, kind: string}}}
+ */
+export function stepErrorResult(name, err, durationMs) {
+  const timeout = err instanceof StepTimeout;
+  const reason = timeout
+    ? `DID NOT COMPLETE — no result within its deadline (${Math.round(err.deadlineMs / 60000)} min). This step has observed nothing about your change and is not accusing it. ` +
+      `A wedged Gradle daemon or a device that stopped answering are the usual causes; check \`./gradlew --status\` and \`adb devices\`, then re-run the step alone.
+  ${err.cmd}`
+    : `DID NOT RUN — the step threw before producing a verdict: ${err && err.message ? err.message : String(err)}. ` +
+      `Nothing here is a claim about your change.`;
+  return { name, verdict: "ERROR", reason, durationMs, details: { executed: false, kind: timeout ? "deadline" : "threw" } };
+}