create-cmp-cli 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/create-cmp.mjs +3 -0
- package/package.json +1 -1
- package/src/commands/upgrade.mjs +287 -0
- package/src/lib/harness-upgrade.mjs +364 -0
- package/template/CLAUDE.md +4 -1
- package/template/README.md +4 -0
- package/template/qa/lib/audit-cadence.mjs +290 -0
- package/template/qa/lib/determinism.mjs +179 -0
- package/template/qa/lib/evidence-badge.mjs +158 -0
- package/template/qa/lib/flight-recorder.mjs +332 -0
- package/template/qa/lib/inputs-hash.mjs +16 -1
- package/template/qa/record-audit.mjs +83 -0
- package/template/qa/retrospective.mjs +51 -0
- package/template/qa/verify.mjs +305 -9
- package/template/qa/watch.mjs +2 -2
|
@@ -40,7 +40,22 @@ export const VERIFIED_SURFACE = [
|
|
|
40
40
|
// snapshot (qa/approvals.json) already carries as state — no lane step reads
|
|
41
41
|
// it, so recording who/why must never invalidate a receipt for a tree whose
|
|
42
42
|
// code did not change (the exact failure FI-8 killed for acceptance).
|
|
43
|
-
|
|
43
|
+
// qa/flight-recorder.jsonl is a lane OUTPUT in the strictest sense: the lane
|
|
44
|
+
// appends one line to it on every run, after the receipt is written — hashing
|
|
45
|
+
// it would make every run invalidate its own receipt. qa/audits.jsonl (the
|
|
46
|
+
// cmp-audit ledger) is read by exactly one lane step (auditCadence), which is
|
|
47
|
+
// a REPORT and can never change the verdict — and recording an audit is
|
|
48
|
+
// bookkeeping about a commit that already happened, so appending a record
|
|
49
|
+
// must never invalidate a receipt for a tree whose code did not change
|
|
50
|
+
// (approvals.log.jsonl's principle, applied to audits).
|
|
51
|
+
const EXCLUDED_PREFIXES = [
|
|
52
|
+
"qa/evidence",
|
|
53
|
+
"qa-artifacts",
|
|
54
|
+
"qa/comments.json",
|
|
55
|
+
"qa/approvals.log.jsonl",
|
|
56
|
+
"qa/flight-recorder.jsonl",
|
|
57
|
+
"qa/audits.jsonl",
|
|
58
|
+
];
|
|
44
59
|
|
|
45
60
|
// qa/approvals.json is hashed by PROJECTION, not raw bytes. The approvals gate's
|
|
46
61
|
// verdict depends on exactly three row fields (artifact, status, hash) plus the
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Record that a cmp-audit of one androidMain subsystem happened — the
|
|
3
|
+
// write half of the audit-cadence report (qa/lib/audit-cadence.mjs).
|
|
4
|
+
//
|
|
5
|
+
// node qa/record-audit.mjs <subsystem> [--by <who-or-what>]
|
|
6
|
+
// node qa/record-audit.mjs --list
|
|
7
|
+
//
|
|
8
|
+
// Recording is a CLAIM — "this subsystem, as of this commit, was audited" —
|
|
9
|
+
// so the entry's sha is derived from HEAD by the library, never passed in,
|
|
10
|
+
// and the write is REFUSED when there is no git history, when the subsystem
|
|
11
|
+
// is not one this app actually has (derived from the tree, printed on
|
|
12
|
+
// refusal), or when the subsystem's files differ from HEAD (the record
|
|
13
|
+
// would name a commit the audited bytes did not match — commit first).
|
|
14
|
+
// Refusal over fabrication, the same stance as qa/approve.mjs.
|
|
15
|
+
//
|
|
16
|
+
// This CLI exists so the audit loop closes mechanically: the release
|
|
17
|
+
// profile's receipt nudges "changed since last audit → cmp-audit <name>",
|
|
18
|
+
// and the auditor's last act is this one command.
|
|
19
|
+
|
|
20
|
+
import path from "node:path";
|
|
21
|
+
import { fileURLToPath } from "node:url";
|
|
22
|
+
|
|
23
|
+
import { AUDITS_REL_PATH, ROOT_SUBSYSTEM, androidMainPackageRoot, evaluateAuditCadence, listSubsystems, recordAudit } from "./lib/audit-cadence.mjs";
|
|
24
|
+
|
|
25
|
+
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
|
26
|
+
|
|
27
|
+
const USAGE = `node qa/record-audit.mjs <subsystem> [--by <who-or-what>]
|
|
28
|
+
|
|
29
|
+
Appends one audit record (subsystem, HEAD sha, ISO timestamp, recorder) to
|
|
30
|
+
${AUDITS_REL_PATH}. The verify lane's release profile reports which
|
|
31
|
+
subsystems changed since their last record. Subsystems are derived from the
|
|
32
|
+
tree: the immediate package directories under the androidMain Kotlin source
|
|
33
|
+
root ("${ROOT_SUBSYSTEM}" for files directly at the package root).
|
|
34
|
+
|
|
35
|
+
--list print the derived subsystems and their audit status
|
|
36
|
+
--by <name> who/what recorded this (default: git user.name)
|
|
37
|
+
--help, -h this usage
|
|
38
|
+
`;
|
|
39
|
+
|
|
40
|
+
const args = process.argv.slice(2);
|
|
41
|
+
|
|
42
|
+
if (args.includes("--help") || args.includes("-h") || args.length === 0) {
|
|
43
|
+
console.log(USAGE);
|
|
44
|
+
process.exit(args.length === 0 ? 2 : 0);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
if (args.includes("--list")) {
|
|
48
|
+
const report = evaluateAuditCadence(ROOT);
|
|
49
|
+
if (!report.ok) {
|
|
50
|
+
console.log(report.reason);
|
|
51
|
+
process.exit(0);
|
|
52
|
+
}
|
|
53
|
+
console.log(`androidMain subsystems under ${report.packageRoot} (${report.summary}):`);
|
|
54
|
+
for (const s of report.subsystems) {
|
|
55
|
+
const when = s.audit?.at ? ` — last audit ${s.audit.at.slice(0, 10)} (${s.audit.sha.slice(0, 7)}, by ${s.audit.by ?? "unknown"})` : "";
|
|
56
|
+
console.log(` ${s.name}: ${s.status}${when}`);
|
|
57
|
+
}
|
|
58
|
+
process.exit(0);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
const byIdx = args.indexOf("--by");
|
|
62
|
+
const by = byIdx >= 0 ? args[byIdx + 1] : undefined;
|
|
63
|
+
if (byIdx >= 0 && !by) {
|
|
64
|
+
console.error("--by needs a value");
|
|
65
|
+
process.exit(2);
|
|
66
|
+
}
|
|
67
|
+
const positional = args.filter((a, i) => !(byIdx >= 0 && (i === byIdx || i === byIdx + 1)));
|
|
68
|
+
if (positional.length !== 1 || positional[0].startsWith("--")) {
|
|
69
|
+
console.error(`expected exactly one subsystem name — run node qa/record-audit.mjs --help`);
|
|
70
|
+
process.exit(2);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const res = recordAudit(ROOT, { subsystem: positional[0], by });
|
|
74
|
+
if (!res.ok) {
|
|
75
|
+
console.error(`refused: ${res.reason}`);
|
|
76
|
+
const pkgRoot = androidMainPackageRoot(ROOT);
|
|
77
|
+
if (pkgRoot.ok) {
|
|
78
|
+
console.error(`derived subsystems: ${listSubsystems(ROOT, pkgRoot.rel).join(", ")}`);
|
|
79
|
+
}
|
|
80
|
+
process.exit(1);
|
|
81
|
+
}
|
|
82
|
+
console.log(`recorded: audit of ${res.entry.subsystem} against ${res.sha.slice(0, 7)} (by ${res.entry.by}) → ${AUDITS_REL_PATH}`);
|
|
83
|
+
console.log("commit the ledger with your change — the release profile's receipt reads it.");
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// The flight recorder's reader — "did this project drift from its tooling?"
|
|
3
|
+
// answered mechanically, from qa/flight-recorder.jsonl alone.
|
|
4
|
+
//
|
|
5
|
+
// node qa/retrospective.mjs [--json]
|
|
6
|
+
//
|
|
7
|
+
// The verify lane appends one journal line per run (qa/lib/flight-recorder.mjs);
|
|
8
|
+
// this CLI turns those lines into the ten-second report a retrospective
|
|
9
|
+
// starts from: fast vs full ratio, which steps SKIP most and why (verbatim
|
|
10
|
+
// reasons, grouped), whether the device tier is ever actually reached, the
|
|
11
|
+
// longest recorded stretch with no full lane, and which degraded paths fired.
|
|
12
|
+
//
|
|
13
|
+
// HONESTY RULES — the reader is only as good as its refusals:
|
|
14
|
+
// - it states only what the journal recorded; a missing journal is
|
|
15
|
+
// "no flight data recorded yet" and exit 0, never an error, never a
|
|
16
|
+
// fabricated baseline;
|
|
17
|
+
// - a short journal says it is short instead of letting two entries read
|
|
18
|
+
// as a trend;
|
|
19
|
+
// - it never editorializes about the developer — every line is a count or
|
|
20
|
+
// a date about lane runs, and the human draws the conclusions.
|
|
21
|
+
|
|
22
|
+
import path from "node:path";
|
|
23
|
+
import { fileURLToPath } from "node:url";
|
|
24
|
+
|
|
25
|
+
import { readFlightJournal, renderFlightReport, summarizeFlightJournal } from "./lib/flight-recorder.mjs";
|
|
26
|
+
|
|
27
|
+
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
|
28
|
+
const asJson = process.argv.includes("--json");
|
|
29
|
+
|
|
30
|
+
const journal = readFlightJournal(ROOT);
|
|
31
|
+
|
|
32
|
+
if (!journal.exists) {
|
|
33
|
+
if (asJson) console.log(JSON.stringify({ recorded: false }));
|
|
34
|
+
else console.log("no flight data recorded yet — the journal (qa/flight-recorder.jsonl) appears after the first verify-lane run");
|
|
35
|
+
process.exit(0);
|
|
36
|
+
}
|
|
37
|
+
if (journal.error) {
|
|
38
|
+
console.error(`qa/flight-recorder.jsonl exists but could not be read: ${journal.error}`);
|
|
39
|
+
process.exit(1);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
const summary = summarizeFlightJournal(journal.entries);
|
|
43
|
+
|
|
44
|
+
if (asJson) {
|
|
45
|
+
console.log(JSON.stringify({ recorded: true, malformed: journal.malformed, summary }, null, 2));
|
|
46
|
+
} else {
|
|
47
|
+
for (const line of renderFlightReport(summary, { malformed: journal.malformed })) {
|
|
48
|
+
console.log(line);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
process.exit(0);
|
package/template/qa/verify.mjs
CHANGED
|
@@ -37,10 +37,14 @@ import { clauseTierCoverage, scanCitations, scanSpecClauses, walkFiles } from ".
|
|
|
37
37
|
import { evaluateComponentStoryParity } from "./lib/component-stories.mjs";
|
|
38
38
|
import { evaluateReachability } from "./lib/reachability.mjs";
|
|
39
39
|
import { evidenceLevel } from "./lib/evidence-level.mjs";
|
|
40
|
+
import { updateReadmeBadge, README_REL_PATH } from "./lib/evidence-badge.mjs";
|
|
40
41
|
import { memoizeStep } from "./lib/step-cache.mjs";
|
|
41
42
|
import { changedWorkingTreePaths, deriveAffectedFilter } from "./lib/affected-tests.mjs";
|
|
42
43
|
import { acquireDeviceLease, releaseDeviceLease, formatHolder } from "./lib/device-lease.mjs";
|
|
43
44
|
import { ARCH_DOC_REL_PATH, SECTION_IDS, regenerateArchDoc } from "./lib/arch-doc.mjs";
|
|
45
|
+
import { DETERMINISM_TIMEZONES, compareOutcomes, parseJUnitOutcomes } from "./lib/determinism.mjs";
|
|
46
|
+
import { evaluateAuditCadence } from "./lib/audit-cadence.mjs";
|
|
47
|
+
import { appendFlightRecord, buildFlightEntry } from "./lib/flight-recorder.mjs";
|
|
44
48
|
|
|
45
49
|
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
|
46
50
|
const EVIDENCE_DIR = path.join(ROOT, "qa", "evidence");
|
|
@@ -76,6 +80,18 @@ Flags:
|
|
|
76
80
|
evidence rung, and can NEVER satisfy the
|
|
77
81
|
done-gate — run the full lane once before
|
|
78
82
|
you call it done
|
|
83
|
+
--determinism run the timezone determinism probe: the JVM
|
|
84
|
+
test tier (unit + golden + the other
|
|
85
|
+
desktop suites) executes TWICE, under
|
|
86
|
+
TZ=Etc/GMT+12 (UTC-12) and TZ=Etc/GMT-14
|
|
87
|
+
(UTC+14), and the probe FAILs naming every
|
|
88
|
+
test whose verdict or failure output
|
|
89
|
+
differs — a nondeterminism leak ARCH-13's
|
|
90
|
+
static net missed. Bare (no --profile) it
|
|
91
|
+
runs JUST the probe and writes no receipt;
|
|
92
|
+
with --profile ci (or release) it runs
|
|
93
|
+
inside the lane and lands on the receipt.
|
|
94
|
+
Never combinable with --fast
|
|
79
95
|
--json print the receipt as JSON instead of the
|
|
80
96
|
human-readable step-by-step log
|
|
81
97
|
--help, -h print this usage and exit 0 without
|
|
@@ -99,7 +115,7 @@ if (rawArgs.includes("--help") || rawArgs.includes("-h")) {
|
|
|
99
115
|
process.exit(0);
|
|
100
116
|
}
|
|
101
117
|
|
|
102
|
-
const RECOGNIZED_FLAGS = new Set(["--profile", "--json", "--fast"]);
|
|
118
|
+
const RECOGNIZED_FLAGS = new Set(["--profile", "--json", "--fast", "--determinism"]);
|
|
103
119
|
for (let i = 0; i < rawArgs.length; i += 1) {
|
|
104
120
|
const arg = rawArgs[i];
|
|
105
121
|
if (arg === "--profile") {
|
|
@@ -115,8 +131,36 @@ const args = rawArgs;
|
|
|
115
131
|
const profile = args.includes("--profile") ? args[args.indexOf("--profile") + 1] : "local";
|
|
116
132
|
const asJson = args.includes("--json");
|
|
117
133
|
const fast = args.includes("--fast");
|
|
134
|
+
// --no-journal suppresses the flight-recorder append (qa/watch.mjs passes it).
|
|
135
|
+
// See the append site below for why the inner loop must not write here.
|
|
136
|
+
const noJournal = args.includes("--no-journal");
|
|
118
137
|
const mode = fast ? "fast" : "full";
|
|
119
138
|
|
|
139
|
+
// ── --determinism: the timezone double-run probe (roadmap §10 item 8) ───────
|
|
140
|
+
// Refusals up front, by name (same stance as unknown arguments above):
|
|
141
|
+
// - never with --fast: the probe deliberately runs the JVM test tier twice,
|
|
142
|
+
// and --fast is the inner loop that exists to not pay such costs — the
|
|
143
|
+
// combination is a contradiction, so it is refused rather than silently
|
|
144
|
+
// resolved either way.
|
|
145
|
+
// - only the ci profile (and release, which inherits ci) carries the probe's
|
|
146
|
+
// lane row; asking for it in local/scaffold is refused with the two ways
|
|
147
|
+
// that DO work, instead of silently running a step the requested profile
|
|
148
|
+
// does not own.
|
|
149
|
+
const determinism = args.includes("--determinism");
|
|
150
|
+
const profileExplicit = args.includes("--profile");
|
|
151
|
+
if (determinism && fast) {
|
|
152
|
+
console.error(
|
|
153
|
+
"--determinism cannot be combined with --fast: the probe runs the JVM test tier twice by design, and --fast is the inner loop. Run it alone (node qa/verify.mjs --determinism) or inside a full ci/release lane (--profile ci --determinism).",
|
|
154
|
+
);
|
|
155
|
+
process.exit(2);
|
|
156
|
+
}
|
|
157
|
+
if (determinism && profileExplicit && profile !== "ci" && profile !== "release") {
|
|
158
|
+
console.error(
|
|
159
|
+
`--determinism belongs to the ci profile (release inherits it), not "${profile}" — run --profile ci --determinism, or bare --determinism to run the probe alone.`,
|
|
160
|
+
);
|
|
161
|
+
process.exit(2);
|
|
162
|
+
}
|
|
163
|
+
|
|
120
164
|
const GRADLEW = process.platform === "win32" ? "gradlew.bat" : "./gradlew";
|
|
121
165
|
|
|
122
166
|
// ── `--rerun` is scoped to FULL mode ────────────────────────────────────────
|
|
@@ -158,6 +202,14 @@ function sh(cmd, opts = {}) {
|
|
|
158
202
|
const LANE_MARKER = path.join(ROOT, "composeApp", "build", ".cmp-lane-in-progress");
|
|
159
203
|
const KSP_COLLISION_RE = /Storage for \[[^\]]*\] is already registered/;
|
|
160
204
|
|
|
205
|
+
// Degraded-path activations observed during this run — self-heals and
|
|
206
|
+
// fallbacks that kept the lane moving without failing it. Collected for the
|
|
207
|
+
// flight recorder (qa/lib/flight-recorder.mjs): a degradation that fires
|
|
208
|
+
// once is a shrug, one that fires every run for a month is the tooling
|
|
209
|
+
// quietly rotting under a green lane — and only a journal can tell those
|
|
210
|
+
// two apart.
|
|
211
|
+
const DEGRADED_PATHS = [];
|
|
212
|
+
|
|
161
213
|
// The daemon's half of defense 2 above — pid + ISO timestamp, mirroring
|
|
162
214
|
// LANE_MARKER's own content shape (see where LANE_MARKER is stamped, below).
|
|
163
215
|
const RENDER_MARKER = path.join(ROOT, "composeApp", "build", ".cmp-render-in-progress");
|
|
@@ -197,6 +249,7 @@ function shGradle(cmd, opts = {}) {
|
|
|
197
249
|
const retry = sh(cmd, opts);
|
|
198
250
|
retry.durationMs += first.durationMs;
|
|
199
251
|
retry.selfHealed = "ksp-cache-collision";
|
|
252
|
+
DEGRADED_PATHS.push("ksp-cache-collision: cleared kspCaches and retried the Gradle step");
|
|
200
253
|
return retry;
|
|
201
254
|
}
|
|
202
255
|
|
|
@@ -696,6 +749,7 @@ function stepUnitTests() {
|
|
|
696
749
|
res = retry;
|
|
697
750
|
note = "full suite — the affected-test filter matched no tests (fell back)";
|
|
698
751
|
affected = null;
|
|
752
|
+
DEGRADED_PATHS.push("affected-test filter matched no tests — fell back to the full desktopTest suite");
|
|
699
753
|
}
|
|
700
754
|
const summary = junitSummary(path.join(ROOT, "composeApp/build/test-results/desktopTest"));
|
|
701
755
|
let details = summary ?? undefined;
|
|
@@ -728,6 +782,115 @@ const stepA11y = gradleTestStep(
|
|
|
728
782
|
"A11y gate failed (SHELL-04): interactive nodes must expose a testTag, text, or contentDescription:",
|
|
729
783
|
);
|
|
730
784
|
|
|
785
|
+
// ── Determinism probe (roadmap §10 item 8) — opt-in, ci-profile ────────────
|
|
786
|
+
// ARCH-13 statically bans ambient time reads — in APP code. A library the
|
|
787
|
+
// app calls can still read the wall clock, and a golden can still depend on
|
|
788
|
+
// the machine's timezone through a seam the static net cannot see (a
|
|
789
|
+
// ViewModel constructed without its injected clock already caused one
|
|
790
|
+
// overnight golden-tree drift). This probe closes that gap DYNAMICALLY: it
|
|
791
|
+
// runs the JVM test tier twice, under two timezones whose local calendar
|
|
792
|
+
// dates never agree — the offsets are 26 hours apart, so any date-derived
|
|
793
|
+
// value differs between the legs at every instant (see DETERMINISM_TIMEZONES
|
|
794
|
+
// in qa/lib/determinism.mjs for why UTC-12/UTC+14 and not UTC/UTC+14) — and
|
|
795
|
+
// FAILs naming every test whose outcome differs between the legs.
|
|
796
|
+
//
|
|
797
|
+
// Mechanics that carry the honesty:
|
|
798
|
+
// - TZ reaches the test JVM through the environment: Gradle forwards the
|
|
799
|
+
// client's environment to the daemon on every build, and test workers
|
|
800
|
+
// fork from the daemon — so the child env below is inherited all the way
|
|
801
|
+
// down to the JVM whose default timezone the tests see.
|
|
802
|
+
// - BOTH legs force --rerun. Without it Gradle would mark the second leg
|
|
803
|
+
// up-to-date (TZ is not a declared build input) and replay the first
|
|
804
|
+
// leg's results — the probe would then compare a run against its own
|
|
805
|
+
// echo and certify a determinism it never tested (the build-cache-replay
|
|
806
|
+
// lesson, again). The legs use the mode-scoped RERUN like every other
|
|
807
|
+
// desktopTest invocation — and because --determinism is refused alongside
|
|
808
|
+
// --fast up front, RERUN is always " --rerun" by the time a leg runs.
|
|
809
|
+
// - Only verdicts and failure output are compared; durations are never even
|
|
810
|
+
// parsed (qa/lib/determinism.mjs), so a timing wobble is structurally
|
|
811
|
+
// unable to trip the probe.
|
|
812
|
+
function stepDeterminism() {
|
|
813
|
+
const started = Date.now();
|
|
814
|
+
const elapsed = () => Date.now() - started;
|
|
815
|
+
if (!determinism) {
|
|
816
|
+
return {
|
|
817
|
+
name: "determinism",
|
|
818
|
+
verdict: "SKIP",
|
|
819
|
+
reason: "determinism probe is opt-in (it runs the JVM test tier twice) — add --determinism to this lane, or run the probe alone: node qa/verify.mjs --determinism",
|
|
820
|
+
durationMs: elapsed(),
|
|
821
|
+
};
|
|
822
|
+
}
|
|
823
|
+
|
|
824
|
+
const resultsDir = path.join(ROOT, "composeApp/build/test-results/desktopTest");
|
|
825
|
+
const legs = [];
|
|
826
|
+
for (const { tz, label } of DETERMINISM_TIMEZONES) {
|
|
827
|
+
const res = shGradle(`${GRADLEW} :composeApp:desktopTest${RERUN} --console=plain`, { env: { ...process.env, TZ: tz } });
|
|
828
|
+
// Parsed NOW, before the next leg overwrites the same results directory.
|
|
829
|
+
const outcomes = parseJUnitOutcomes(resultsDir);
|
|
830
|
+
legs.push({ tz, label, ok: res.ok, outcomes, tail: res.out.split("\n").slice(-8).join("\n") });
|
|
831
|
+
}
|
|
832
|
+
const [a, b] = legs;
|
|
833
|
+
const labelA = `TZ=${a.tz} (${a.label})`;
|
|
834
|
+
const labelB = `TZ=${b.tz} (${b.label})`;
|
|
835
|
+
const countA = Object.keys(a.outcomes).length;
|
|
836
|
+
const countB = Object.keys(b.outcomes).length;
|
|
837
|
+
|
|
838
|
+
if (countA === 0 && countB === 0) {
|
|
839
|
+
// Neither leg produced a single test result: the suite failed before
|
|
840
|
+
// running anything (build error). The probe measured nothing — that is
|
|
841
|
+
// a FAIL that says so, never a PASS by absence of differences.
|
|
842
|
+
return {
|
|
843
|
+
name: "determinism",
|
|
844
|
+
verdict: "FAIL",
|
|
845
|
+
reason: `determinism probe could not execute: desktopTest produced no test results under either timezone — the suite fails before running (fix the build first; this is not a timezone difference):\n${a.tail}`,
|
|
846
|
+
durationMs: elapsed(),
|
|
847
|
+
};
|
|
848
|
+
}
|
|
849
|
+
if (countA === 0 || countB === 0) {
|
|
850
|
+
const ran = countA > 0 ? { label: labelA, n: countA } : { label: labelB, n: countB };
|
|
851
|
+
const empty = countA > 0 ? b : a;
|
|
852
|
+
return {
|
|
853
|
+
name: "determinism",
|
|
854
|
+
verdict: "FAIL",
|
|
855
|
+
reason: `Nondeterminism under timezone shift: desktopTest ran ${ran.n} test(s) under ${ran.label} but produced no results at all under TZ=${empty.tz} (${empty.label}) — the suite itself dies under that zone:\n${empty.tail}`,
|
|
856
|
+
durationMs: elapsed(),
|
|
857
|
+
details: { timezones: DETERMINISM_TIMEZONES.map((t) => t.tz) },
|
|
858
|
+
};
|
|
859
|
+
}
|
|
860
|
+
|
|
861
|
+
const diffs = compareOutcomes(a.outcomes, b.outcomes, labelA, labelB);
|
|
862
|
+
if (diffs.length > 0) {
|
|
863
|
+
const lines = [
|
|
864
|
+
`Nondeterminism under timezone shift — the same tree produced different outcomes under ${labelA} vs ${labelB}. Something reads ambient time or zone past the ARCH-13 net (a library default, an uninjected clock, a golden that captures "today"):`,
|
|
865
|
+
];
|
|
866
|
+
for (const d of diffs.slice(0, 20)) lines.push(` [${d.step}] ${d.test} — ${d.detail}`);
|
|
867
|
+
if (diffs.length > 20) lines.push(` … and ${diffs.length - 20} more differing test(s)`);
|
|
868
|
+
return {
|
|
869
|
+
name: "determinism",
|
|
870
|
+
verdict: "FAIL",
|
|
871
|
+
reason: lines.join("\n"),
|
|
872
|
+
durationMs: elapsed(),
|
|
873
|
+
details: { timezones: DETERMINISM_TIMEZONES.map((t) => t.tz), diffs: diffs.slice(0, 50) },
|
|
874
|
+
};
|
|
875
|
+
}
|
|
876
|
+
|
|
877
|
+
const failedIdentically = Object.values(a.outcomes).filter((o) => o.status !== "pass" && o.status !== "skip").length;
|
|
878
|
+
return {
|
|
879
|
+
name: "determinism",
|
|
880
|
+
verdict: "PASS",
|
|
881
|
+
// Identical red is DETERMINISTIC red: the probe's claim ("no timezone
|
|
882
|
+
// dependence") holds, and the failing tests already belong to
|
|
883
|
+
// unitTests/goldenTrees, which fail the lane on their own merits — a
|
|
884
|
+
// second FAIL here would report the same defect twice under a wrong name.
|
|
885
|
+
note:
|
|
886
|
+
failedIdentically > 0
|
|
887
|
+
? `${failedIdentically} test(s) failed identically under both timezones — deterministic, but red (the owning test steps report it)`
|
|
888
|
+
: undefined,
|
|
889
|
+
durationMs: elapsed(),
|
|
890
|
+
details: { timezones: DETERMINISM_TIMEZONES.map((t) => t.tz), testsCompared: countA },
|
|
891
|
+
};
|
|
892
|
+
}
|
|
893
|
+
|
|
731
894
|
// Live tokenDrift tier (harness M4-D): when a debug app + device are available,
|
|
732
895
|
// fetches the declared catalog and the live semantics tree off the debug-only
|
|
733
896
|
// inspector server (127.0.0.1:9500, see composeApp/src/androidDebug/.../
|
|
@@ -1052,6 +1215,43 @@ function stepReleaseSmoke() {
|
|
|
1052
1215
|
return runMaestroSmoke("releaseSmoke", install.durationMs);
|
|
1053
1216
|
}
|
|
1054
1217
|
|
|
1218
|
+
// ── Audit cadence (roadmap §10 item 9) — a REPORT, never a gate ────────────
|
|
1219
|
+
// cmp-audit (the adversarial platform-semantics audit) found six latent
|
|
1220
|
+
// defects the first time a human happened to ask for it — which is exactly
|
|
1221
|
+
// why it must not depend on someone remembering to ask. This step is the
|
|
1222
|
+
// cheapest honest replacement for that memory: at ship time (release
|
|
1223
|
+
// profile) the receipt lists which androidMain subsystems changed since
|
|
1224
|
+
// their last RECORDED audit (qa/audits.jsonl, appended by
|
|
1225
|
+
// node qa/record-audit.mjs). The derivation lives in
|
|
1226
|
+
// qa/lib/audit-cadence.mjs; this step adds only the bookkeeping every step
|
|
1227
|
+
// carries — and by construction it maps every outcome to PASS or SKIP,
|
|
1228
|
+
// never FAIL: audit debt is a judgment call (a rename is not six latent
|
|
1229
|
+
// defects), and a gate here would teach people to game the ledger, which
|
|
1230
|
+
// would destroy the only value it has.
|
|
1231
|
+
function stepAuditCadence() {
|
|
1232
|
+
const started = Date.now();
|
|
1233
|
+
const report = evaluateAuditCadence(ROOT);
|
|
1234
|
+
if (!report.ok) {
|
|
1235
|
+
return { name: "auditCadence", verdict: "SKIP", reason: report.reason, durationMs: Date.now() - started };
|
|
1236
|
+
}
|
|
1237
|
+
return {
|
|
1238
|
+
name: "auditCadence",
|
|
1239
|
+
verdict: "PASS",
|
|
1240
|
+
note: report.summary,
|
|
1241
|
+
durationMs: Date.now() - started,
|
|
1242
|
+
details: {
|
|
1243
|
+
packageRoot: report.packageRoot,
|
|
1244
|
+
subsystems: report.subsystems.map((s) => ({
|
|
1245
|
+
name: s.name,
|
|
1246
|
+
status: s.status,
|
|
1247
|
+
changedFiles: s.changedFiles,
|
|
1248
|
+
lastAudit: s.audit ? { sha: s.audit.sha, at: s.audit.at, by: s.audit.by } : null,
|
|
1249
|
+
})),
|
|
1250
|
+
lines: report.lines,
|
|
1251
|
+
},
|
|
1252
|
+
};
|
|
1253
|
+
}
|
|
1254
|
+
|
|
1055
1255
|
// ── Lane ───────────────────────────────────────────────────────────────────
|
|
1056
1256
|
|
|
1057
1257
|
// Device-dependent steps, in lane order. Used twice: receipt STRENGTH (which
|
|
@@ -1137,19 +1337,57 @@ const stepsForProfile = {
|
|
|
1137
1337
|
stepAndroidChecks,
|
|
1138
1338
|
],
|
|
1139
1339
|
};
|
|
1140
|
-
|
|
1141
|
-
//
|
|
1142
|
-
//
|
|
1143
|
-
//
|
|
1144
|
-
//
|
|
1145
|
-
//
|
|
1146
|
-
|
|
1340
|
+
// ci = local + the determinism probe's row — the first place ci diverges
|
|
1341
|
+
// from local. The probe is OPT-IN (the step SKIPs unless --determinism was
|
|
1342
|
+
// passed: it doubles the JVM test tier's cost), but its row lives in the ci
|
|
1343
|
+
// profile so a ci receipt always records whether the probe ran — an honest,
|
|
1344
|
+
// visible gap beats an invisible one ("SKIPs are recorded so the pipeline
|
|
1345
|
+
// stays honest", per the profile's own contract). local deliberately does
|
|
1346
|
+
// NOT carry the row: the per-change developer profile is not where a
|
|
1347
|
+
// deliberate double-run belongs.
|
|
1348
|
+
stepsForProfile.ci = [...stepsForProfile.local, stepDeterminism];
|
|
1349
|
+
// release = everything ci proves PLUS the audit-cadence report and the
|
|
1350
|
+
// release-APK behavior smoke. The expensive proofs are profile-tiered by
|
|
1351
|
+
// decision: per-change stays fast (local/ci pay for the release COMPILE via
|
|
1352
|
+
// releaseBuild, already in the set), and the release-variant *behavior* cost
|
|
1353
|
+
// lands once, at ship time. auditCadence (a report, never a gate) also
|
|
1354
|
+
// belongs to ship time — "what moved in androidMain since its last
|
|
1355
|
+
// adversarial audit?" is the question asked before shipping, not per edit.
|
|
1356
|
+
// releaseSmoke runs last so the device ends the run holding the exact build
|
|
1357
|
+
// that was proven.
|
|
1358
|
+
stepsForProfile.release = [...stepsForProfile.ci, stepAuditCadence, stepReleaseSmoke];
|
|
1147
1359
|
|
|
1148
1360
|
if (!stepsForProfile[profile]) {
|
|
1149
1361
|
console.error(`Unknown profile "${profile}" — use scaffold | local | ci | release.`);
|
|
1150
1362
|
process.exit(2);
|
|
1151
1363
|
}
|
|
1152
1364
|
|
|
1365
|
+
// ── Bare --determinism: the probe, nothing else, and NO receipt ─────────────
|
|
1366
|
+
// "Run it alone" means alone: no other steps, and deliberately no
|
|
1367
|
+
// qa/evidence/latest.json. The done-gate (qa/receipt-check.mjs) validates a
|
|
1368
|
+
// receipt by verdict + content hash — a receipt whose steps are one probe
|
|
1369
|
+
// would satisfy it while attesting almost nothing, so a probe-only run must
|
|
1370
|
+
// never mint one. The lane marker IS still stamped: the probe runs Gradle
|
|
1371
|
+
// and owes the preview daemon the same coexistence courtesy as the lane.
|
|
1372
|
+
if (determinism && !profileExplicit) {
|
|
1373
|
+
fs.mkdirSync(path.dirname(LANE_MARKER), { recursive: true });
|
|
1374
|
+
fs.writeFileSync(LANE_MARKER, `${process.pid} ${new Date().toISOString()}\n`);
|
|
1375
|
+
let probe;
|
|
1376
|
+
try {
|
|
1377
|
+
probe = stepDeterminism();
|
|
1378
|
+
} finally {
|
|
1379
|
+
fs.rmSync(LANE_MARKER, { force: true });
|
|
1380
|
+
}
|
|
1381
|
+
if (asJson) {
|
|
1382
|
+
console.log(JSON.stringify(probe, null, 2));
|
|
1383
|
+
} else {
|
|
1384
|
+
const mark = probe.verdict === "PASS" ? "✓" : "✗";
|
|
1385
|
+
console.log(`${mark} determinism: ${probe.verdict}${probe.note ? ` (${probe.note})` : ""}${probe.reason ? ` — ${probe.reason}` : ""}`);
|
|
1386
|
+
console.log("\n(probe-only run — no receipt written; the full lane is where evidence is earned)");
|
|
1387
|
+
}
|
|
1388
|
+
process.exit(probe.verdict === "FAIL" ? 1 : 0);
|
|
1389
|
+
}
|
|
1390
|
+
|
|
1153
1391
|
// ── --fast: the inner loop, mechanically unable to claim done ───────────────
|
|
1154
1392
|
// The genuinely slow tier is device/release work — every DEVICE_STEPS entry
|
|
1155
1393
|
// (Gradle install + emulator + Maestro + instrumented runner) plus
|
|
@@ -1202,6 +1440,7 @@ if (fast) {
|
|
|
1202
1440
|
// always removed, even on a failing step, so the eyes only ever defer briefly.
|
|
1203
1441
|
fs.mkdirSync(path.dirname(LANE_MARKER), { recursive: true });
|
|
1204
1442
|
fs.writeFileSync(LANE_MARKER, `${process.pid} ${new Date().toISOString()}\n`);
|
|
1443
|
+
const laneStartedAt = Date.now(); // for the flight-recorder entry's durationMs
|
|
1205
1444
|
const steps = [];
|
|
1206
1445
|
try {
|
|
1207
1446
|
for (const step of laneSteps) {
|
|
@@ -1301,6 +1540,51 @@ fs.writeFileSync(path.join(EVIDENCE_DIR, "latest.json"), `${JSON.stringify(recei
|
|
|
1301
1540
|
// studio console's Evidence audit trail reconstructs the full history from the
|
|
1302
1541
|
// git log of this file — every commit is one verified, attributed state.
|
|
1303
1542
|
|
|
1543
|
+
// The README's evidence badge is DERIVED from the receipt just written — an
|
|
1544
|
+
// output, never a gate, so it runs after the verdict and cannot change it. It
|
|
1545
|
+
// renders the rung together with the commit it was attested against, so the
|
|
1546
|
+
// sentence stays true as the tree moves on (qa/lib/evidence-badge.mjs).
|
|
1547
|
+
const badge = updateReadmeBadge(ROOT);
|
|
1548
|
+
|
|
1549
|
+
// ── Flight recorder (roadmap §10 item 5) — the lane journals its own run ────
|
|
1550
|
+
// One JSON line per run into qa/flight-recorder.jsonl (committed, and
|
|
1551
|
+
// excluded from the receipt's hashed surface — qa/lib/flight-recorder.mjs
|
|
1552
|
+
// carries the whole rationale). Appended AFTER the receipt so the entry
|
|
1553
|
+
// records the final verdict and rung. A failed append must never fail the
|
|
1554
|
+
// lane — a recorder that breaks the thing it observes is worse than no
|
|
1555
|
+
// recorder — so the failure degrades to a note in the lane's own output,
|
|
1556
|
+
// which is itself the honest record of the degradation.
|
|
1557
|
+
//
|
|
1558
|
+
// --no-journal is the ONE exemption, and qa/watch.mjs passes it on every
|
|
1559
|
+
// save-triggered run. Same rule the README badge obeys, for the same reason:
|
|
1560
|
+
// THE INNER LOOP DOES NOT WRITE TO COMMITTED FILES. A watcher journaling every
|
|
1561
|
+
// save would add hundreds of lines a day to a committed file — turning the
|
|
1562
|
+
// app's history into keystroke noise and leaving a permanently-dirty tree in
|
|
1563
|
+
// the loop the recorder exists to observe. What survives is every full lane
|
|
1564
|
+
// and every DELIBERATE fast run, which is what the retrospective's questions
|
|
1565
|
+
// actually rest on (SKIP reasons, degraded paths, the longest stretch with no
|
|
1566
|
+
// full lane). qa/retrospective.mjs discloses the exemption in its own output
|
|
1567
|
+
// so the fast-vs-full ratio is never read as a complete census.
|
|
1568
|
+
const flight = noJournal
|
|
1569
|
+
? { ok: true, skipped: true }
|
|
1570
|
+
: appendFlightRecord(
|
|
1571
|
+
ROOT,
|
|
1572
|
+
buildFlightEntry({
|
|
1573
|
+
profile,
|
|
1574
|
+
mode,
|
|
1575
|
+
verdict,
|
|
1576
|
+
evidenceLevel: level,
|
|
1577
|
+
steps,
|
|
1578
|
+
sha: receipt.commit.sha,
|
|
1579
|
+
durationMs: Date.now() - laneStartedAt,
|
|
1580
|
+
onDeviceSteps,
|
|
1581
|
+
degraded: DEGRADED_PATHS,
|
|
1582
|
+
}),
|
|
1583
|
+
);
|
|
1584
|
+
if (!flight.ok) {
|
|
1585
|
+
console.error(`· flight recorder: journal append failed (${flight.reason}) — the lane verdict is unaffected, but this run is missing from qa/flight-recorder.jsonl`);
|
|
1586
|
+
}
|
|
1587
|
+
|
|
1304
1588
|
if (asJson) {
|
|
1305
1589
|
console.log(JSON.stringify(receipt, null, 2));
|
|
1306
1590
|
if (fast) {
|
|
@@ -1313,7 +1597,19 @@ if (asJson) {
|
|
|
1313
1597
|
`\n${verdict === "PASS" ? "⚡⚡" : "❌"} verify lane [FAST — INNER LOOP ONLY, NOT DONE]: ${verdict} (skipped device/release tier: ${fastExcluded.join(", ") || "none"}) — this fast receipt satisfies no done-gate; run the full lane (node qa/verify.mjs) once before you finish`,
|
|
1314
1598
|
);
|
|
1315
1599
|
} else {
|
|
1316
|
-
console.log(`\n${verdict === "PASS" ? "✅" : "❌"} verify lane: ${verdict}${level ? ` · ${level.rung} ${level.name}` : ""} (${strengthLabel}) — receipt written to qa/evidence/latest.json (commit it with your change)`);
|
|
1600
|
+
console.log(`\n${verdict === "PASS" ? "✅" : "❌"} verify lane: ${verdict}${level ? ` · ${level.rung} ${level.name}` : ""} (${strengthLabel}) — receipt written to qa/evidence/latest.json${badge.changed ? ` and ${README_REL_PATH}'s evidence badge refreshed` : ""} (commit ${badge.changed ? "them" : "it"} with your change)`);
|
|
1601
|
+
}
|
|
1602
|
+
|
|
1603
|
+
// The audit-cadence nudges print in the human path, not only inside the
|
|
1604
|
+
// receipt JSON — a ship-time report that lives only in a JSON field is a
|
|
1605
|
+
// report nobody reads at ship time. Nudges only; a gate this is not.
|
|
1606
|
+
if (!asJson) {
|
|
1607
|
+
const auditStep = steps.find((s) => s.name === "auditCadence");
|
|
1608
|
+
const auditLines = auditStep?.details?.lines ?? [];
|
|
1609
|
+
if (auditLines.length > 0) {
|
|
1610
|
+
console.log("\naudit cadence (report, never a gate):");
|
|
1611
|
+
for (const l of auditLines) console.log(` ${l}`);
|
|
1612
|
+
}
|
|
1317
1613
|
}
|
|
1318
1614
|
|
|
1319
1615
|
process.exit(verdict === "PASS" ? 0 : 1);
|
package/template/qa/watch.mjs
CHANGED
|
@@ -358,8 +358,8 @@ function main() {
|
|
|
358
358
|
const n = runCounter;
|
|
359
359
|
const startedAtIso = new Date().toISOString();
|
|
360
360
|
const started = Date.now();
|
|
361
|
-
say(`── watch run #${n} starting (node qa/verify.mjs --fast) …`);
|
|
362
|
-
const child = spawn(process.execPath, [path.join(ROOT, "qa", "verify.mjs"), "--fast", "--json"], {
|
|
361
|
+
say(`── watch run #${n} starting (node qa/verify.mjs --fast --no-journal) …`);
|
|
362
|
+
const child = spawn(process.execPath, [path.join(ROOT, "qa", "verify.mjs"), "--fast", "--json", "--no-journal"], {
|
|
363
363
|
cwd: ROOT,
|
|
364
364
|
stdio: ["ignore", "pipe", "pipe"],
|
|
365
365
|
// Its own process GROUP: verify spawns Gradle through a shell, and a
|