create-cmp-cli 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/create-cmp.mjs +3 -0
- package/package.json +1 -1
- package/src/commands/upgrade.mjs +287 -0
- package/src/lib/harness-upgrade.mjs +364 -0
- package/template/CLAUDE.md +4 -1
- package/template/README.md +4 -0
- package/template/qa/lib/audit-cadence.mjs +290 -0
- package/template/qa/lib/determinism.mjs +179 -0
- package/template/qa/lib/evidence-badge.mjs +158 -0
- package/template/qa/lib/flight-recorder.mjs +332 -0
- package/template/qa/lib/inputs-hash.mjs +16 -1
- package/template/qa/record-audit.mjs +83 -0
- package/template/qa/retrospective.mjs +51 -0
- package/template/qa/verify.mjs +305 -9
- package/template/qa/watch.mjs +2 -2
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
// determinism.mjs — the comparison half of the lane's determinism probe.
|
|
2
|
+
//
|
|
3
|
+
// ARCH-13 statically bans ambient time reads (Clock.System, LocalDate.now,
|
|
4
|
+
// TimeZone.currentSystemDefault) in APP code — but a library the app calls
|
|
5
|
+
// can still read the wall clock, and a golden test can still depend on the
|
|
6
|
+
// machine's timezone through a seam the static net cannot see. This project
|
|
7
|
+
// family has already been bitten: a golden tree green at 23:00 and red by
|
|
8
|
+
// morning, because a ViewModel was constructed without its injected clock.
|
|
9
|
+
//
|
|
10
|
+
// The probe (verify.mjs stepDeterminism) runs the JVM test tier TWICE under
|
|
11
|
+
// maximally-shifted timezones and fails iff the two runs' OUTCOMES differ.
|
|
12
|
+
// This module owns the two judgments that make that comparison honest:
|
|
13
|
+
//
|
|
14
|
+
// - WHAT COUNTS AS AN OUTCOME: a test's verdict (pass/fail/error/skip)
|
|
15
|
+
// and its failure output — never its duration. Durations are not parsed
|
|
16
|
+
// at all, so a timing wobble is structurally incapable of tripping the
|
|
17
|
+
// probe (the brief-level rule "duration is not a difference" is enforced
|
|
18
|
+
// by construction, not by filtering).
|
|
19
|
+
//
|
|
20
|
+
// - WHAT THE FAILURE MESSAGE MUST SAY: which test, which lane step owns
|
|
21
|
+
// it, and the observable difference between the two runs — never a bare
|
|
22
|
+
// "nondeterministic". A probe whose red is unactionable just teaches
|
|
23
|
+
// people to turn it off.
|
|
24
|
+
|
|
25
|
+
import fs from "node:fs";
|
|
26
|
+
import path from "node:path";
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* The two probe timezones — chosen so the two legs NEVER share a calendar
|
|
30
|
+
* date, at any instant:
|
|
31
|
+
*
|
|
32
|
+
* Etc/GMT+12 = UTC-12 (POSIX sign convention: Etc/GMT+N means UTC-N)
|
|
33
|
+
* Etc/GMT-14 = UTC+14 (the highest real-world offset, Line Islands)
|
|
34
|
+
*
|
|
35
|
+
* The offsets are 26 hours apart — more than a full day — so the two legs'
|
|
36
|
+
* local dates differ at EVERY moment of every day, and any date-derived
|
|
37
|
+
* value (a "today" default, a day-boundary bucket, a formatted date in a
|
|
38
|
+
* golden tree) is guaranteed to differ between the legs. A UTC-vs-UTC+14
|
|
39
|
+
* pair would NOT have this property: those legs share a date for ten hours
|
|
40
|
+
* of every day, so the probe's power would depend on what time you ran it —
|
|
41
|
+
* the exact class of flakiness it exists to hunt.
|
|
42
|
+
*/
|
|
43
|
+
export const DETERMINISM_TIMEZONES = [
|
|
44
|
+
{ tz: "Etc/GMT+12", label: "UTC-12" },
|
|
45
|
+
{ tz: "Etc/GMT-14", label: "UTC+14" },
|
|
46
|
+
];
|
|
47
|
+
|
|
48
|
+
const XML_ENTITIES = { "<": "<", ">": ">", """: '"', "'": "'", "&": "&" };
|
|
49
|
+
|
|
50
|
+
function unescapeXml(s) {
|
|
51
|
+
return s
|
|
52
|
+
.replace(/&#x([0-9a-fA-F]+);/g, (_, hex) => String.fromCodePoint(parseInt(hex, 16)))
|
|
53
|
+
.replace(/&#(\d+);/g, (_, dec) => String.fromCodePoint(Number(dec)))
|
|
54
|
+
.replace(/&(lt|gt|quot|apos|amp);/g, (m) => XML_ENTITIES[m]);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function attr(attrs, name) {
|
|
58
|
+
const m = attrs.match(new RegExp(`${name}="([^"]*)"`));
|
|
59
|
+
return m ? unescapeXml(m[1]) : null;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Parse one Gradle JUnit results directory into per-test outcomes.
|
|
64
|
+
* DELIBERATELY parses only verdict-bearing content: testcase identity,
|
|
65
|
+
* status, and failure/error text. `time="…"` attributes are never read, so
|
|
66
|
+
* two runs that differ only in duration produce identical outcome maps.
|
|
67
|
+
*
|
|
68
|
+
* @param {string} dir a test-results directory (TEST-*.xml files, flat)
|
|
69
|
+
* @returns {Record<string, {status: "pass"|"fail"|"error"|"skip", messages: string[]}>}
|
|
70
|
+
* keyed by `classname.name`; empty object when the directory is absent
|
|
71
|
+
* (the caller decides what an empty leg means — this parser never guesses)
|
|
72
|
+
*/
|
|
73
|
+
export function parseJUnitOutcomes(dir) {
|
|
74
|
+
const outcomes = {};
|
|
75
|
+
if (!fs.existsSync(dir)) return outcomes;
|
|
76
|
+
for (const entry of fs.readdirSync(dir)) {
|
|
77
|
+
if (!entry.startsWith("TEST-") || !entry.endsWith(".xml")) continue;
|
|
78
|
+
const xml = fs.readFileSync(path.join(dir, entry), "utf8");
|
|
79
|
+
const caseRe = /<testcase\b([^>]*?)(?:\/>|>([\s\S]*?)<\/testcase>)/g;
|
|
80
|
+
for (const m of xml.matchAll(caseRe)) {
|
|
81
|
+
const attrs = m[1];
|
|
82
|
+
const body = m[2] ?? "";
|
|
83
|
+
const classname = attr(attrs, "classname") ?? "";
|
|
84
|
+
const name = attr(attrs, "name") ?? "";
|
|
85
|
+
if (!classname && !name) continue;
|
|
86
|
+
let status = "pass";
|
|
87
|
+
const messages = [];
|
|
88
|
+
const childRe = /<(failure|error)\b([^>]*?)(?:\/>|>([\s\S]*?)<\/\1>)/g;
|
|
89
|
+
for (const c of body.matchAll(childRe)) {
|
|
90
|
+
status = c[1] === "error" ? "error" : "fail";
|
|
91
|
+
const message = attr(c[2], "message");
|
|
92
|
+
const text = c[3] ? unescapeXml(c[3]).trim() : "";
|
|
93
|
+
messages.push(message ?? text.split("\n")[0] ?? "");
|
|
94
|
+
}
|
|
95
|
+
if (status === "pass" && /<skipped\b/.test(body)) status = "skip";
|
|
96
|
+
outcomes[`${classname}.${name}`] = { status, messages };
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
return outcomes;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Which lane step owns a test class — so the probe's failure message names
|
|
104
|
+
* the step a reader would re-run, not just a class name. The patterns are
|
|
105
|
+
* the same filters the lane's own gradleTestStep calls use.
|
|
106
|
+
* @param {string} classname fully-qualified test class
|
|
107
|
+
* @returns {"goldenTrees"|"conformance"|"a11y"|"unitTests"}
|
|
108
|
+
*/
|
|
109
|
+
export function laneStepForTestClass(classname) {
|
|
110
|
+
if (/GoldenTreeTest$/.test(classname)) return "goldenTrees";
|
|
111
|
+
if (/ArchitectureConformanceTest$/.test(classname)) return "conformance";
|
|
112
|
+
if (/A11yConformanceTest$/.test(classname)) return "a11y";
|
|
113
|
+
return "unitTests";
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function classnameOf(testId) {
|
|
117
|
+
// testId is `classname.name`; the class is everything before the last dot
|
|
118
|
+
// segment that starts the (possibly backticked, space-bearing) test name.
|
|
119
|
+
// Kotlin test names contain dots rarely but spaces often — the classname
|
|
120
|
+
// never contains a space, so split at the first segment containing one,
|
|
121
|
+
// falling back to the last dot.
|
|
122
|
+
const spaceIdx = testId.indexOf(" ");
|
|
123
|
+
const scope = spaceIdx === -1 ? testId : testId.slice(0, spaceIdx);
|
|
124
|
+
const lastDot = scope.lastIndexOf(".");
|
|
125
|
+
return lastDot === -1 ? testId : testId.slice(0, lastDot);
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Compare two legs' outcomes. Returns one entry per observable difference,
|
|
130
|
+
* each carrying everything the failure message must name: the test, the
|
|
131
|
+
* owning lane step, and what differed between the legs.
|
|
132
|
+
*
|
|
133
|
+
* Kinds:
|
|
134
|
+
* verdict-flip different status (pass/fail/error/skip)
|
|
135
|
+
* only-in-one-leg the test executed in one leg only
|
|
136
|
+
* failure-text-changed failed in BOTH legs, but with different output —
|
|
137
|
+
* a date-dependent assertion message is still a
|
|
138
|
+
* timezone leak even when both legs are red
|
|
139
|
+
*
|
|
140
|
+
* @param {Record<string, {status: string, messages: string[]}>} a leg A outcomes
|
|
141
|
+
* @param {Record<string, {status: string, messages: string[]}>} b leg B outcomes
|
|
142
|
+
* @param {string} labelA human label for leg A (e.g. "TZ=Etc/GMT+12 (UTC-12)")
|
|
143
|
+
* @param {string} labelB human label for leg B
|
|
144
|
+
* @returns {Array<{test: string, step: string, kind: string, detail: string}>}
|
|
145
|
+
*/
|
|
146
|
+
export function compareOutcomes(a, b, labelA, labelB) {
|
|
147
|
+
const diffs = [];
|
|
148
|
+
const ids = [...new Set([...Object.keys(a), ...Object.keys(b)])].sort();
|
|
149
|
+
for (const id of ids) {
|
|
150
|
+
const step = laneStepForTestClass(classnameOf(id));
|
|
151
|
+
const inA = a[id];
|
|
152
|
+
const inB = b[id];
|
|
153
|
+
if (!inA || !inB) {
|
|
154
|
+
const where = inA ? labelA : labelB;
|
|
155
|
+
const missing = inA ? labelB : labelA;
|
|
156
|
+
diffs.push({ test: id, step, kind: "only-in-one-leg", detail: `executed under ${where} but produced no result under ${missing}` });
|
|
157
|
+
continue;
|
|
158
|
+
}
|
|
159
|
+
if (inA.status !== inB.status) {
|
|
160
|
+
const firstLine = (inA.status === "pass" ? inB : inA).messages[0]?.split("\n")[0] ?? "";
|
|
161
|
+
diffs.push({
|
|
162
|
+
test: id,
|
|
163
|
+
step,
|
|
164
|
+
kind: "verdict-flip",
|
|
165
|
+
detail: `${inA.status.toUpperCase()} under ${labelA}, ${inB.status.toUpperCase()} under ${labelB}${firstLine ? `: ${firstLine}` : ""}`,
|
|
166
|
+
});
|
|
167
|
+
continue;
|
|
168
|
+
}
|
|
169
|
+
if (inA.status !== "pass" && inA.messages.join("\n") !== inB.messages.join("\n")) {
|
|
170
|
+
diffs.push({
|
|
171
|
+
test: id,
|
|
172
|
+
step,
|
|
173
|
+
kind: "failure-text-changed",
|
|
174
|
+
detail: `failed under both, with different output — ${labelA}: "${inA.messages[0]?.split("\n")[0] ?? ""}" vs ${labelB}: "${inB.messages[0]?.split("\n")[0] ?? ""}"`,
|
|
175
|
+
});
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
return diffs;
|
|
179
|
+
}
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
// The README's evidence badge — the evidence ladder, rendered where a human
|
|
2
|
+
// actually looks (roadmap §10 item 2: "render the rung in the console and
|
|
3
|
+
// README badge").
|
|
4
|
+
//
|
|
5
|
+
// The console already shows the rung (rail foot, Evidence section, receipt
|
|
6
|
+
// timeline). The README is the surface a human meets FIRST, and the one that
|
|
7
|
+
// travels — into a GitHub repo page, a PR, a screenshot in a deck. That makes
|
|
8
|
+
// it the surface where an overclaim does the most damage, so the badge obeys
|
|
9
|
+
// one rule above all others:
|
|
10
|
+
//
|
|
11
|
+
// **The badge is a statement about a specific commit, never about "now".**
|
|
12
|
+
//
|
|
13
|
+
// A badge that says "L2 device" says nothing about whether the code has moved
|
|
14
|
+
// since. So it never renders a bare rung: it renders the rung AND the commit
|
|
15
|
+
// it was attested against AND the date. That sentence stays true forever — a
|
|
16
|
+
// reader can see at a glance whether the sha still matches what they are
|
|
17
|
+
// looking at. Everything else follows from the same rule: no receipt says so,
|
|
18
|
+
// a FAIL says so, and a --fast run (which the ladder deliberately grants no
|
|
19
|
+
// rung) says so rather than borrowing the last good one.
|
|
20
|
+
//
|
|
21
|
+
// Written by the lane AFTER the receipt (it is an output derived from the
|
|
22
|
+
// receipt, never a gate), and committed alongside it.
|
|
23
|
+
|
|
24
|
+
import fs from "node:fs";
|
|
25
|
+
import path from "node:path";
|
|
26
|
+
|
|
27
|
+
export const README_REL_PATH = "README.md";
|
|
28
|
+
export const BADGE_SECTION_ID = "evidence";
|
|
29
|
+
|
|
30
|
+
const MARKER_RE = new RegExp(
|
|
31
|
+
`<!-- cmp:generated ${BADGE_SECTION_ID} -->\\n([\\s\\S]*?)<!-- /cmp:generated -->`
|
|
32
|
+
);
|
|
33
|
+
|
|
34
|
+
/** Shields.io colours, one per rung — the ladder read at a glance. */
|
|
35
|
+
const RUNG_COLOR = {
|
|
36
|
+
L0: "9E9E9E", // scaffold — grey: a green build, nothing proven about behavior
|
|
37
|
+
L1: "42A5F5", // desktop — blue
|
|
38
|
+
L2: "26A69A", // device — teal
|
|
39
|
+
L3: "43A047", // release — green: the strongest rung this harness can attest
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
/** shields.io escaping: `-` → `--`, `_` → `__`, space → `_`. */
|
|
43
|
+
function shieldEscape(s) {
|
|
44
|
+
return String(s).replace(/-/g, "--").replace(/_/g, "__").replace(/ /g, "_");
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* The badge body for a receipt — Markdown, no trailing newline handling (the
|
|
49
|
+
* caller frames it). Pure: every degraded state has its own honest rendering
|
|
50
|
+
* and NONE of them fall back to a rung.
|
|
51
|
+
*
|
|
52
|
+
* @param {object|null} receipt parsed qa/evidence/latest.json, or null
|
|
53
|
+
* @returns {string} Markdown
|
|
54
|
+
*/
|
|
55
|
+
export function renderEvidenceBadge(receipt) {
|
|
56
|
+
const link = "https://github.com/kvdm-co-pilot/create-cmp";
|
|
57
|
+
const badge = (label, message, color, title) =>
|
|
58
|
+
`[}-${shieldEscape(message)}-${color})](${link})`;
|
|
59
|
+
|
|
60
|
+
if (!receipt || typeof receipt !== "object") {
|
|
61
|
+
return `${badge("evidence", "none yet", "9E9E9E", "No evidence receipt")} — no verify receipt yet. Run \`node qa/verify.mjs\`.`;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
const verdict = typeof receipt.verdict === "string" ? receipt.verdict : "?";
|
|
65
|
+
const mode = receipt.mode === "fast" ? "fast" : "full";
|
|
66
|
+
const sha = typeof receipt.commit?.sha === "string" ? receipt.commit.sha.slice(0, 7) : null;
|
|
67
|
+
const dirty = Array.isArray(receipt.commit?.dirty) ? receipt.commit.dirty.length : 0;
|
|
68
|
+
const when = typeof receipt.generatedAt === "string" ? receipt.generatedAt.slice(0, 10) : null;
|
|
69
|
+
|
|
70
|
+
// Provenance is not decoration — it is what keeps the sentence true later.
|
|
71
|
+
const at = sha ? ` at \`${sha}\`` : "";
|
|
72
|
+
const on = when ? ` on ${when}` : "";
|
|
73
|
+
const uncommitted =
|
|
74
|
+
dirty > 0
|
|
75
|
+
? ` The tree had ${dirty} uncommitted file${dirty === 1 ? "" : "s"} at attestation, so this describes that run, not that commit.`
|
|
76
|
+
: "";
|
|
77
|
+
|
|
78
|
+
if (verdict !== "PASS") {
|
|
79
|
+
return `${badge("evidence", `lane ${verdict}`, "E53935", `Verify lane ${verdict}`)} — the last lane run${at}${on} did not pass. No rung is earned by a failed lane.${uncommitted}`;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
if (mode === "fast") {
|
|
83
|
+
// The inner loop is a signal, never evidence. Borrowing the previous
|
|
84
|
+
// full run's rung here is exactly the lie the ladder exists to prevent.
|
|
85
|
+
return `${badge("evidence", "fast run, no rung", "9E9E9E", "Fast run — no evidence rung")} — the last run${at}${on} was \`--fast\`: the device and release tiers were skipped, so it earns no rung.${uncommitted} Run \`node qa/verify.mjs\` for evidence.`;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
const level = receipt.evidenceLevel;
|
|
89
|
+
if (!level || typeof level.rung !== "string" || typeof level.name !== "string") {
|
|
90
|
+
return `${badge("evidence", `PASS, rung unrecorded`, "9E9E9E", "Lane PASS, no rung recorded")} — the lane passed${at}${on} but the receipt records no evidence rung.`;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const color = RUNG_COLOR[level.rung] || "9E9E9E";
|
|
94
|
+
const satisfied = Array.isArray(level.satisfiedBy) && level.satisfiedBy.length
|
|
95
|
+
? ` Earned by: ${level.satisfiedBy.map((s) => `\`${s}\``).join(", ")}.`
|
|
96
|
+
: "";
|
|
97
|
+
return (
|
|
98
|
+
`${badge("evidence", `${level.rung} ${level.name}`, color, `Evidence ${level.rung} — ${level.name}`)}` +
|
|
99
|
+
` — the verify lane passed${at}${on} at rung **${level.rung} · ${level.name}**.` +
|
|
100
|
+
`${satisfied}${uncommitted}` +
|
|
101
|
+
` The rung describes that run; it says nothing about changes made since.`
|
|
102
|
+
);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Read the project's receipt and rewrite README.md's `cmp:generated evidence`
|
|
107
|
+
* block from it. Never creates the marker — a project that removed the block
|
|
108
|
+
* has opted out, and that is honoured silently.
|
|
109
|
+
*
|
|
110
|
+
* NOTE the asymmetry with renderEvidenceBadge above: the RENDERER is total —
|
|
111
|
+
* every receipt, including a `--fast` one, has an honest rendering. The WRITER
|
|
112
|
+
* is selective: a fast receipt is not written to the README at all. Two
|
|
113
|
+
* reasons, and the second is the load-bearing one:
|
|
114
|
+
* 1. The badge reports EVIDENCE. A fast run produces none, so it has nothing
|
|
115
|
+
* to say — and overwriting a true statement about a real full-lane run
|
|
116
|
+
* with "no rung" loses information rather than adding honesty.
|
|
117
|
+
* 2. `qa/watch.mjs` runs the fast lane on every save. A writer that fired
|
|
118
|
+
* there would rewrite README.md on every keystroke-to-save cycle, putting
|
|
119
|
+
* a permanently-dirty file in the inner loop. A recorder must not disturb
|
|
120
|
+
* what it records.
|
|
121
|
+
* The badge therefore always describes the last run that could BEAR evidence,
|
|
122
|
+
* and says so by naming that run's commit and date.
|
|
123
|
+
*
|
|
124
|
+
* @param {string} root project root
|
|
125
|
+
* @returns {{changed: boolean, reason?: string}}
|
|
126
|
+
*/
|
|
127
|
+
export function updateReadmeBadge(root) {
|
|
128
|
+
const readmePath = path.join(root, README_REL_PATH);
|
|
129
|
+
let readme;
|
|
130
|
+
try {
|
|
131
|
+
readme = fs.readFileSync(readmePath, "utf8");
|
|
132
|
+
} catch {
|
|
133
|
+
return { changed: false, reason: `${README_REL_PATH} not found` };
|
|
134
|
+
}
|
|
135
|
+
if (!MARKER_RE.test(readme)) {
|
|
136
|
+
return { changed: false, reason: `${README_REL_PATH} has no cmp:generated ${BADGE_SECTION_ID} block` };
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
let receipt = null;
|
|
140
|
+
try {
|
|
141
|
+
receipt = JSON.parse(fs.readFileSync(path.join(root, "qa", "evidence", "latest.json"), "utf8"));
|
|
142
|
+
} catch {
|
|
143
|
+
receipt = null; // no receipt / unreadable → the "none yet" rendering, never a guess
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
if (receipt && receipt.mode === "fast") {
|
|
147
|
+
return { changed: false, reason: "fast run — the inner loop bears no evidence, so the badge is left as it stands" };
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
const body = `${renderEvidenceBadge(receipt)}\n`;
|
|
151
|
+
const next = readme.replace(
|
|
152
|
+
MARKER_RE,
|
|
153
|
+
() => `<!-- cmp:generated ${BADGE_SECTION_ID} -->\n${body}<!-- /cmp:generated -->`
|
|
154
|
+
);
|
|
155
|
+
if (next === readme) return { changed: false };
|
|
156
|
+
fs.writeFileSync(readmePath, next);
|
|
157
|
+
return { changed: true };
|
|
158
|
+
}
|
|
@@ -0,0 +1,332 @@
|
|
|
1
|
+
// flight-recorder.mjs — the lane's append-only journal of its own runs.
|
|
2
|
+
//
|
|
3
|
+
// The retrospective that reshaped this harness was only possible because
|
|
4
|
+
// session transcripts happened to exist. This module removes the "happened
|
|
5
|
+
// to": every verify-lane run appends one JSON line — profile, mode, verdict,
|
|
6
|
+
// evidence rung, per-step verdicts, every SKIP reason VERBATIM, and every
|
|
7
|
+
// degraded-path activation — so "did this project drift from its tooling?"
|
|
8
|
+
// is answerable mechanically (qa/retrospective.mjs), from the repo alone.
|
|
9
|
+
// The SKIP reasons are the signal that matters: they are where a harness
|
|
10
|
+
// quietly stops being used ("no device attached" forever, "maestro not
|
|
11
|
+
// installed" forever) without any single run ever failing.
|
|
12
|
+
//
|
|
13
|
+
// GROUND RULES, each load-bearing:
|
|
14
|
+
//
|
|
15
|
+
// - IN-REPO ONLY, NO PHONE-HOME. That is a product constraint, not a
|
|
16
|
+
// preference: the journal is the app's own artifact, and nothing here
|
|
17
|
+
// records a machine, a hostname, or a user beyond what git itself
|
|
18
|
+
// already records (the commit sha).
|
|
19
|
+
//
|
|
20
|
+
// - THE RECORDER MUST NEVER BREAK THE LANE. A recorder that fails the
|
|
21
|
+
// thing it observes is worse than no recorder: every write is wrapped,
|
|
22
|
+
// every failure degrades to {ok: false, reason} for the lane to NOTE in
|
|
23
|
+
// its own output — never to throw, never to change a verdict.
|
|
24
|
+
//
|
|
25
|
+
// - THE JOURNAL IS COMMITTED, NOT GITIGNORED — deliberately. The console's
|
|
26
|
+
// Evidence timeline reconstructs history from the git log of committed
|
|
27
|
+
// receipts; a gitignored journal could answer nothing about the past on
|
|
28
|
+
// a fresh clone, which is exactly the question this file exists to
|
|
29
|
+
// answer. It follows qa/evidence/latest.json's precedent: a lane output
|
|
30
|
+
// that is committed with the change and EXCLUDED from the receipt's
|
|
31
|
+
// hashed input surface (qa/lib/inputs-hash.mjs), because a lane output
|
|
32
|
+
// inside the hash would invalidate the very receipt that produced it.
|
|
33
|
+
//
|
|
34
|
+
// - THE READER STATES ONLY WHAT THE JOURNAL RECORDED. Summaries are
|
|
35
|
+
// counts and dates, never extrapolation: a short journal says it is
|
|
36
|
+
// short, a single full run yields no "stretch" arithmetic, and nothing
|
|
37
|
+
// here editorializes about the developer — the journal records lane
|
|
38
|
+
// runs, not people.
|
|
39
|
+
//
|
|
40
|
+
// ONE DELIBERATE GAP, stated because a silent one would be a lie: qa/watch.mjs
|
|
41
|
+
// passes `--no-journal`, so save-triggered fast runs are never journaled. The
|
|
42
|
+
// rule it follows is the same one the README evidence badge follows — THE INNER
|
|
43
|
+
// LOOP DOES NOT WRITE TO COMMITTED FILES. A watcher that appended on every save
|
|
44
|
+
// would add hundreds of lines a day to a committed file, turn the app's history
|
|
45
|
+
// into keystroke noise, and leave a permanently-dirty tree inside the very loop
|
|
46
|
+
// this recorder exists to observe. What survives is every full lane and every
|
|
47
|
+
// deliberate fast run — which is what the retrospective's questions actually
|
|
48
|
+
// rest on (SKIP reasons, degraded paths, the longest stretch with no full
|
|
49
|
+
// lane). renderFlightReport states the gap in its own output.
|
|
50
|
+
|
|
51
|
+
import fs from "node:fs";
|
|
52
|
+
import path from "node:path";
|
|
53
|
+
|
|
54
|
+
export const FLIGHT_JOURNAL_REL_PATH = "qa/flight-recorder.jsonl";
|
|
55
|
+
export const FLIGHT_SCHEMA = "cmp-flight/1";
|
|
56
|
+
|
|
57
|
+
// Below this many entries the report carries an explicit shortness note —
|
|
58
|
+
// two entries are two facts, not a trend, and the report must say so rather
|
|
59
|
+
// than let a reader infer a pattern from a journal that cannot support one.
|
|
60
|
+
const SHORT_JOURNAL_FLOOR = 5;
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Shape one lane run into a journal entry. Pure — verify.mjs passes what it
|
|
64
|
+
* already computed for the receipt, so the journal can never disagree with
|
|
65
|
+
* the receipt about the same run.
|
|
66
|
+
*
|
|
67
|
+
* @param {object} run
|
|
68
|
+
* @param {string} run.profile lane profile (scaffold | local | ci | release)
|
|
69
|
+
* @param {string} run.mode "full" | "fast"
|
|
70
|
+
* @param {string} run.verdict "PASS" | "FAIL"
|
|
71
|
+
* @param {{rung: string}|null} run.evidenceLevel the derived rung (or null —
|
|
72
|
+
* fast runs and FAILed lanes carry none, and the journal records that
|
|
73
|
+
* honestly rather than borrowing a rung from elsewhere)
|
|
74
|
+
* @param {Array<{name: string, verdict: string, reason?: string}>} run.steps
|
|
75
|
+
* the lane's step results, verbatim
|
|
76
|
+
* @param {string|null} run.sha parent HEAD at run time (null before git init)
|
|
77
|
+
* @param {number} run.durationMs wall time of the step loop
|
|
78
|
+
* @param {string[]} run.onDeviceSteps device-tier steps that actually PASSed
|
|
79
|
+
* (verify.mjs's own strength derivation — reused, not recomputed, so the
|
|
80
|
+
* two can never drift)
|
|
81
|
+
* @param {string[]} run.degraded degraded-path activations the lane observed
|
|
82
|
+
* (self-heals, fallbacks) — each a short verbatim description
|
|
83
|
+
* @returns {object} one journal entry (JSON-serializable)
|
|
84
|
+
*/
|
|
85
|
+
export function buildFlightEntry({ profile, mode, verdict, evidenceLevel, steps, sha, durationMs, onDeviceSteps, degraded }) {
|
|
86
|
+
const stepList = Array.isArray(steps) ? steps.filter((s) => s && typeof s.name === "string") : [];
|
|
87
|
+
return {
|
|
88
|
+
schema: FLIGHT_SCHEMA,
|
|
89
|
+
at: new Date().toISOString(),
|
|
90
|
+
commit: sha ?? null,
|
|
91
|
+
profile,
|
|
92
|
+
mode,
|
|
93
|
+
verdict,
|
|
94
|
+
evidenceRung: evidenceLevel?.rung ?? null,
|
|
95
|
+
durationMs,
|
|
96
|
+
steps: stepList.map((s) => ({ name: s.name, verdict: s.verdict })),
|
|
97
|
+
// SKIP reasons verbatim — the journal's core signal (see file header).
|
|
98
|
+
skips: stepList.filter((s) => s.verdict === "SKIP").map((s) => ({ step: s.name, reason: s.reason ?? "" })),
|
|
99
|
+
deviceSteps: Array.isArray(onDeviceSteps) ? onDeviceSteps : [],
|
|
100
|
+
degraded: Array.isArray(degraded) ? degraded : [],
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Append one entry to the journal. NEVER throws — a recorder that breaks the
|
|
106
|
+
* lane is worse than no recorder (see ground rules). The caller is expected
|
|
107
|
+
* to surface a failed append in the lane's own output.
|
|
108
|
+
* @param {string} root project root (absolute)
|
|
109
|
+
* @param {object} entry a buildFlightEntry() result
|
|
110
|
+
* @returns {{ok: true}|{ok: false, reason: string}}
|
|
111
|
+
*/
|
|
112
|
+
export function appendFlightRecord(root, entry) {
|
|
113
|
+
try {
|
|
114
|
+
const p = path.join(root, FLIGHT_JOURNAL_REL_PATH);
|
|
115
|
+
fs.mkdirSync(path.dirname(p), { recursive: true });
|
|
116
|
+
fs.appendFileSync(p, `${JSON.stringify(entry)}\n`);
|
|
117
|
+
return { ok: true };
|
|
118
|
+
} catch (err) {
|
|
119
|
+
return { ok: false, reason: err?.message ?? String(err) };
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* Read the journal. Absent is not an error — it is the honest "no flight
|
|
125
|
+
* data recorded yet" state. Unparsable lines are counted, never silently
|
|
126
|
+
* dropped into the totals: the report must be able to say "N lines could
|
|
127
|
+
* not be read" instead of quietly under-counting.
|
|
128
|
+
* @param {string} root project root (absolute)
|
|
129
|
+
* @returns {{exists: boolean, entries: object[], malformed: number, error?: string}}
|
|
130
|
+
*/
|
|
131
|
+
export function readFlightJournal(root) {
|
|
132
|
+
const p = path.join(root, FLIGHT_JOURNAL_REL_PATH);
|
|
133
|
+
if (!fs.existsSync(p)) return { exists: false, entries: [], malformed: 0 };
|
|
134
|
+
let raw;
|
|
135
|
+
try {
|
|
136
|
+
raw = fs.readFileSync(p, "utf8");
|
|
137
|
+
} catch (err) {
|
|
138
|
+
return { exists: true, entries: [], malformed: 0, error: err?.message ?? String(err) };
|
|
139
|
+
}
|
|
140
|
+
const entries = [];
|
|
141
|
+
let malformed = 0;
|
|
142
|
+
for (const line of raw.split("\n")) {
|
|
143
|
+
if (!line.trim()) continue;
|
|
144
|
+
try {
|
|
145
|
+
const parsed = JSON.parse(line);
|
|
146
|
+
if (parsed && typeof parsed === "object") entries.push(parsed);
|
|
147
|
+
else malformed += 1;
|
|
148
|
+
} catch {
|
|
149
|
+
malformed += 1;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
return { exists: true, entries, malformed };
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
function countBy(entries, keyFn) {
|
|
156
|
+
const out = new Map();
|
|
157
|
+
for (const e of entries) {
|
|
158
|
+
const k = keyFn(e);
|
|
159
|
+
if (k === undefined || k === null) continue;
|
|
160
|
+
out.set(k, (out.get(k) ?? 0) + 1);
|
|
161
|
+
}
|
|
162
|
+
return out;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
function fmtDay(iso) {
|
|
166
|
+
return typeof iso === "string" ? iso.slice(0, 10) : "unknown";
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
function fmtGap(ms) {
|
|
170
|
+
const days = Math.floor(ms / 86_400_000);
|
|
171
|
+
const hours = Math.round((ms % 86_400_000) / 3_600_000);
|
|
172
|
+
if (days > 0) return `${days}d ${hours}h`;
|
|
173
|
+
const mins = Math.round((ms % 3_600_000) / 60_000);
|
|
174
|
+
return hours > 0 ? `${hours}h ${mins}m` : `${mins}m`;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* Summarize journal entries into the facts the retrospective report prints.
|
|
179
|
+
* Pure arithmetic over recorded entries — no clock reads except the caller-
|
|
180
|
+
* supplied `now` (used only for the clearly-labeled "as of" distance to the
|
|
181
|
+
* last full run), no filesystem, no extrapolation.
|
|
182
|
+
* @param {object[]} entries parsed journal entries, in append (chronological) order
|
|
183
|
+
* @param {{now?: Date}} [opts]
|
|
184
|
+
* @returns {object} summary — see the field-by-field construction below
|
|
185
|
+
*/
|
|
186
|
+
export function summarizeFlightJournal(entries, { now = new Date() } = {}) {
|
|
187
|
+
const runs = entries.filter((e) => e && typeof e === "object");
|
|
188
|
+
const byMode = countBy(runs, (e) => e.mode ?? "unknown");
|
|
189
|
+
const byProfile = countBy(runs, (e) => e.profile ?? "unknown");
|
|
190
|
+
const byVerdict = countBy(runs, (e) => e.verdict ?? "unknown");
|
|
191
|
+
|
|
192
|
+
// SKIP reasons, grouped VERBATIM — the reason string is the key on purpose:
|
|
193
|
+
// paraphrasing or normalizing would erase exactly the signal the journal
|
|
194
|
+
// exists to keep (two different reasons are two different problems).
|
|
195
|
+
const skipGroups = new Map();
|
|
196
|
+
for (const e of runs) {
|
|
197
|
+
for (const s of Array.isArray(e.skips) ? e.skips : []) {
|
|
198
|
+
// JSON-array key: reasons are arbitrary text, so a delimiter-joined
|
|
199
|
+
// string key would be ambiguous — and ambiguity here merges two
|
|
200
|
+
// different problems into one count.
|
|
201
|
+
const key = JSON.stringify([s.step ?? "?", s.reason ?? ""]);
|
|
202
|
+
skipGroups.set(key, (skipGroups.get(key) ?? 0) + 1);
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
const skipReasons = [...skipGroups.entries()]
|
|
206
|
+
.map(([key, count]) => {
|
|
207
|
+
const [step, reason] = JSON.parse(key);
|
|
208
|
+
return { step, reason, count };
|
|
209
|
+
})
|
|
210
|
+
.sort((a, b) => b.count - a.count || a.step.localeCompare(b.step));
|
|
211
|
+
|
|
212
|
+
const degradedGroups = countBy(
|
|
213
|
+
runs.flatMap((e) => (Array.isArray(e.degraded) ? e.degraded : [])).map((d) => ({ d })),
|
|
214
|
+
(x) => x.d,
|
|
215
|
+
);
|
|
216
|
+
|
|
217
|
+
const fullRuns = runs.filter((e) => e.mode === "full");
|
|
218
|
+
const deviceReached = runs.filter((e) => Array.isArray(e.deviceSteps) && e.deviceSteps.length > 0);
|
|
219
|
+
const rungOrder = { L0: 0, L1: 1, L2: 2, L3: 3 };
|
|
220
|
+
const highestRung = runs
|
|
221
|
+
.map((e) => e.evidenceRung)
|
|
222
|
+
.filter((r) => typeof r === "string" && r in rungOrder)
|
|
223
|
+
.sort((a, b) => rungOrder[b] - rungOrder[a])[0] ?? null;
|
|
224
|
+
|
|
225
|
+
// Longest stretch with no full lane — only computable BETWEEN two recorded
|
|
226
|
+
// full runs. One full run is a date, not a stretch; the report says so
|
|
227
|
+
// instead of inventing a gap against "now" or the journal's edges.
|
|
228
|
+
let longestFullGap = null;
|
|
229
|
+
for (let i = 1; i < fullRuns.length; i += 1) {
|
|
230
|
+
const a = Date.parse(fullRuns[i - 1].at);
|
|
231
|
+
const b = Date.parse(fullRuns[i].at);
|
|
232
|
+
if (Number.isNaN(a) || Number.isNaN(b)) continue;
|
|
233
|
+
const gap = b - a;
|
|
234
|
+
if (!longestFullGap || gap > longestFullGap.ms) {
|
|
235
|
+
longestFullGap = { ms: gap, from: fullRuns[i - 1].at, to: fullRuns[i].at };
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
const lastFull = fullRuns.length ? fullRuns[fullRuns.length - 1].at : null;
|
|
240
|
+
const lastFullAgoMs = lastFull && !Number.isNaN(Date.parse(lastFull)) ? now.getTime() - Date.parse(lastFull) : null;
|
|
241
|
+
|
|
242
|
+
return {
|
|
243
|
+
total: runs.length,
|
|
244
|
+
span: runs.length ? { from: runs[0].at, to: runs[runs.length - 1].at } : null,
|
|
245
|
+
short: runs.length > 0 && runs.length < SHORT_JOURNAL_FLOOR,
|
|
246
|
+
byMode: Object.fromEntries(byMode),
|
|
247
|
+
byProfile: Object.fromEntries(byProfile),
|
|
248
|
+
byVerdict: Object.fromEntries(byVerdict),
|
|
249
|
+
skipReasons,
|
|
250
|
+
degraded: [...degradedGroups.entries()].map(([what, count]) => ({ what, count })),
|
|
251
|
+
fullRuns: {
|
|
252
|
+
count: fullRuns.length,
|
|
253
|
+
last: lastFull,
|
|
254
|
+
lastAgoMs: lastFullAgoMs,
|
|
255
|
+
longestGap: longestFullGap,
|
|
256
|
+
},
|
|
257
|
+
device: { reachedRuns: deviceReached.length, highestRung },
|
|
258
|
+
};
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/**
|
|
262
|
+
* Render the summary as the plain-text report a human reads in ten seconds.
|
|
263
|
+
* Every line is a recorded fact; the honesty notes (short journal, single
|
|
264
|
+
* full run, malformed lines) are part of the report, not caveats around it.
|
|
265
|
+
* @param {object} summary a summarizeFlightJournal() result
|
|
266
|
+
* @param {{malformed?: number}} [opts]
|
|
267
|
+
* @returns {string[]} report lines
|
|
268
|
+
*/
|
|
269
|
+
export function renderFlightReport(summary, { malformed = 0 } = {}) {
|
|
270
|
+
const lines = [];
|
|
271
|
+
if (summary.total === 0) {
|
|
272
|
+
lines.push("flight recorder: journal exists but holds no readable entries");
|
|
273
|
+
if (malformed > 0) lines.push(` ${malformed} line(s) could not be parsed`);
|
|
274
|
+
return lines;
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
lines.push(`flight recorder — ${summary.total} lane run(s) recorded (${fmtDay(summary.span.from)} → ${fmtDay(summary.span.to)})`);
|
|
278
|
+
if (malformed > 0) lines.push(` ${malformed} line(s) could not be parsed and are not counted`);
|
|
279
|
+
if (summary.short) {
|
|
280
|
+
lines.push(` only ${summary.total} run(s) recorded — the counts below are individual facts, not a trend`);
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
const modeBits = ["full", "fast"].map((m) => `${summary.byMode[m] ?? 0} ${m}`).join(" · ");
|
|
284
|
+
lines.push(`modes: ${modeBits}`);
|
|
285
|
+
// Disclosure, not a footnote: qa/watch.mjs passes --no-journal, so
|
|
286
|
+
// save-triggered fast runs are deliberately absent (a committed journal must
|
|
287
|
+
// not grow by hundreds of lines a day, and the inner loop must not leave the
|
|
288
|
+
// tree dirty). The fast count is therefore DELIBERATE fast runs only, and
|
|
289
|
+
// saying so here keeps the ratio from being read as a complete census.
|
|
290
|
+
lines.push(" (fast = deliberate runs only; qa/watch.mjs save-triggered runs are not journaled)");
|
|
291
|
+
lines.push(`verdicts: ${Object.entries(summary.byVerdict).map(([v, n]) => `${n} ${v}`).join(" · ")}`);
|
|
292
|
+
lines.push(`profiles: ${Object.entries(summary.byProfile).map(([p, n]) => `${p} ${n}`).join(" · ")}`);
|
|
293
|
+
|
|
294
|
+
if (summary.device.reachedRuns > 0) {
|
|
295
|
+
lines.push(
|
|
296
|
+
`device tier: reached in ${summary.device.reachedRuns} of ${summary.total} run(s)${summary.device.highestRung ? ` (highest evidence rung recorded: ${summary.device.highestRung})` : ""}`,
|
|
297
|
+
);
|
|
298
|
+
} else {
|
|
299
|
+
lines.push("device tier: never reached in any recorded run (no device-tier step ever PASSed)");
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
if (summary.fullRuns.count === 0) {
|
|
303
|
+
lines.push("full lane: never recorded — every recorded run was --fast (the inner loop; no run earned evidence)");
|
|
304
|
+
} else {
|
|
305
|
+
if (summary.fullRuns.longestGap) {
|
|
306
|
+
lines.push(
|
|
307
|
+
`full lane: longest recorded stretch with no full run: ${fmtGap(summary.fullRuns.longestGap.ms)} (${fmtDay(summary.fullRuns.longestGap.from)} → ${fmtDay(summary.fullRuns.longestGap.to)})`,
|
|
308
|
+
);
|
|
309
|
+
} else {
|
|
310
|
+
lines.push(`full lane: one full run recorded (${fmtDay(summary.fullRuns.last)}) — no stretch to measure between full runs`);
|
|
311
|
+
}
|
|
312
|
+
if (summary.fullRuns.lastAgoMs !== null) {
|
|
313
|
+
lines.push(` last full lane: ${fmtDay(summary.fullRuns.last)} (${fmtGap(summary.fullRuns.lastAgoMs)} before this report)`);
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
if (summary.skipReasons.length > 0) {
|
|
318
|
+
lines.push("skip reasons (verbatim, grouped):");
|
|
319
|
+
for (const s of summary.skipReasons) {
|
|
320
|
+
lines.push(` ${s.count}× [${s.step}] ${s.reason.split("\n")[0]}`);
|
|
321
|
+
}
|
|
322
|
+
} else {
|
|
323
|
+
lines.push("skip reasons: none recorded");
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
if (summary.degraded.length > 0) {
|
|
327
|
+
lines.push("degraded paths activated:");
|
|
328
|
+
for (const d of summary.degraded) lines.push(` ${d.count}× ${d.what}`);
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
return lines;
|
|
332
|
+
}
|