opencode-agent-harness 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/LICENSE +21 -0
- package/README.md +590 -2
- package/bin/agent-harness.js +3059 -0
- package/contrib/global-plugins/README.md +49 -0
- package/contrib/global-plugins/harness-freshness.ts +355 -0
- package/package.json +48 -4
- package/templates/.agents/AGENTS.md +204 -0
- package/templates/.agents/features/INDEX.md +10 -0
- package/templates/.agents/handoffs/README.md +87 -0
- package/templates/.agents/memory/dev-loop.md +59 -0
- package/templates/.agents/memory/domain-map.md +29 -0
- package/templates/.agents/memory/environments.md +35 -0
- package/templates/.agents/memory/flow-map.md +23 -0
- package/templates/.agents/memory/history.md +7 -0
- package/templates/.agents/memory/locations.md +14 -0
- package/templates/.agents/memory/model-routing.md +261 -0
- package/templates/.agents/memory/orchestration.md +183 -0
- package/templates/.agents/memory/owned-files.md +19 -0
- package/templates/.agents/memory/product-principles.md +37 -0
- package/templates/.agents/memory/quality-parameters.md +37 -0
- package/templates/.agents/memory/stack-versions.md +35 -0
- package/templates/.agents/memory/test-design-parameters.md +36 -0
- package/templates/.agents/nightly.json +17 -0
- package/templates/.agents/qa-plans/README.md +70 -0
- package/templates/.agents/qa-runs/README.md +21 -0
- package/templates/.agents/reviews/README.md +47 -0
- package/templates/.agents/rules/00-operating.md +199 -0
- package/templates/.agents/skills/data-engineer/SKILL.md +31 -0
- package/templates/.agents/skills/devops-engineer/SKILL.md +50 -0
- package/templates/.agents/skills/diagnostics-expert/SKILL.md +44 -0
- package/templates/.agents/skills/docs/SKILL.md +140 -0
- package/templates/.agents/skills/features/SKILL.md +187 -0
- package/templates/.agents/skills/frontend-engineer/SKILL.md +81 -0
- package/templates/.agents/skills/handoff/SKILL.md +85 -0
- package/templates/.agents/skills/mobile-engineer/SKILL.md +101 -0
- package/templates/.agents/skills/planner/SKILL.md +112 -0
- package/templates/.agents/skills/product-manager/SKILL.md +115 -0
- package/templates/.agents/skills/qa-architect/SKILL.md +224 -0
- package/templates/.agents/skills/qa-runner/SKILL.md +64 -0
- package/templates/.agents/skills/security-engineer/SKILL.md +53 -0
- package/templates/.agents/skills/system-architect/SKILL.md +42 -0
- package/templates/.agents/skills/ui-designer/SKILL.md +71 -0
- package/templates/.github/workflows/ci.yml +148 -0
- package/templates/.github/workflows/deploy-uat.yml +95 -0
- package/templates/.github/workflows/nightly.yml +83 -0
- package/templates/.github/workflows/uat-health.yml +41 -0
- package/templates/.husky/pre-push +23 -0
- package/templates/.ignore +28 -0
- package/templates/.opencode/agents/data-engineer.md +37 -0
- package/templates/.opencode/agents/devops-engineer.md +37 -0
- package/templates/.opencode/agents/diagnostics-expert.md +37 -0
- package/templates/.opencode/agents/docs.md +37 -0
- package/templates/.opencode/agents/features.md +21 -0
- package/templates/.opencode/agents/frontend-engineer.md +37 -0
- package/templates/.opencode/agents/handoff.md +21 -0
- package/templates/.opencode/agents/mobile-engineer.md +37 -0
- package/templates/.opencode/agents/planner.md +34 -0
- package/templates/.opencode/agents/product-manager.md +29 -0
- package/templates/.opencode/agents/qa-architect.md +27 -0
- package/templates/.opencode/agents/qa-runner.md +17 -0
- package/templates/.opencode/agents/security-engineer.md +37 -0
- package/templates/.opencode/agents/system-architect.md +37 -0
- package/templates/.opencode/agents/ui-designer.md +37 -0
- package/templates/.opencode/commands/board.md +15 -0
- package/templates/.opencode/commands/checkpoint.md +25 -0
- package/templates/.opencode/commands/feature.md +56 -0
- package/templates/.opencode/commands/finish.md +36 -0
- package/templates/.opencode/commands/next.md +49 -0
- package/templates/.opencode/commands/report.md +55 -0
- package/templates/.opencode/commands/resume.md +39 -0
- package/templates/.opencode/commands/switch-model.md +34 -0
- package/templates/.opencode/commands/tidy.md +49 -0
- package/templates/.opencode/plugins/backlog-notify.ts +57 -0
- package/templates/.opencode/plugins/backlog.ts +272 -0
- package/templates/.opencode/plugins/checkpoint.ts +160 -0
- package/templates/.opencode/plugins/context-guard.ts +1164 -0
- package/templates/.opencode/plugins/diff.ts +85 -0
- package/templates/.opencode/plugins/gates.ts +71 -0
- package/templates/.opencode/plugins/job-notify.ts +241 -0
- package/templates/.opencode/plugins/orient.ts +67 -0
- package/templates/.opencode/plugins/session.ts +89 -0
- package/templates/.opencode/plugins/wait.ts +101 -0
- package/templates/.opencode/plugins/where.ts +70 -0
- package/templates/.opencode/skills/bounded-explore/SKILL.md +38 -0
- package/templates/.opencode/skills/dev-loop/SKILL.md +87 -0
- package/templates/.vercelignore +16 -0
- package/templates/AGENTS.md +27 -0
- package/templates/BACKLOG.md +95 -0
- package/templates/docs/ci-cd-pipeline.md +81 -0
- package/templates/docs/gates.md +128 -0
- package/templates/docs/mcp-integrations.md +97 -0
- package/templates/docs/mobile-testing.md +555 -0
- package/templates/docs/nightly-testing.md +106 -0
- package/templates/docs/wsl-optimizations.md +290 -0
- package/templates/opencode.json +188 -0
- package/templates/scripts/agent-report.js +541 -0
- package/templates/scripts/agent-search.sh +117 -0
- package/templates/scripts/backlog.js +1864 -0
- package/templates/scripts/board-service.sh +207 -0
- package/templates/scripts/checkpoint.js +290 -0
- package/templates/scripts/finalize.js +277 -0
- package/templates/scripts/nightly.js +913 -0
- package/templates/scripts/orient.js +514 -0
- package/templates/scripts/qa/board-client.js +260 -0
- package/templates/scripts/qa/board-lib.js +96 -0
- package/templates/scripts/qa/board-sync.js +469 -0
- package/templates/scripts/qa/capacity-lib.js +916 -0
- package/templates/scripts/qa/check-backlog.js +427 -0
- package/templates/scripts/qa/check-capacity.js +85 -0
- package/templates/scripts/qa/check-ci-policy.js +164 -0
- package/templates/scripts/qa/check-concurrency.js +344 -0
- package/templates/scripts/qa/check-dispatch-config.js +296 -0
- package/templates/scripts/qa/check-docs.js +284 -0
- package/templates/scripts/qa/check-integrate.js +98 -0
- package/templates/scripts/qa/check-linger.js +160 -0
- package/templates/scripts/qa/check-prefix-stability.js +183 -0
- package/templates/scripts/qa/check-product.js +177 -0
- package/templates/scripts/qa/check-qa-plan.js +378 -0
- package/templates/scripts/qa/check-qa-scripts.js +85 -0
- package/templates/scripts/qa/check-quality.js +249 -0
- package/templates/scripts/qa/check-test-wiring.js +253 -0
- package/templates/scripts/qa/ci-triage.js +346 -0
- package/templates/scripts/qa/ci-usage-lib.js +185 -0
- package/templates/scripts/qa/ci-wait-lib.js +294 -0
- package/templates/scripts/qa/command-classifier.js +552 -0
- package/templates/scripts/qa/context-guard-core.js +1618 -0
- package/templates/scripts/qa/diff.js +174 -0
- package/templates/scripts/qa/feature-scaffold.js +224 -0
- package/templates/scripts/qa/fixtures/backlog.md +104 -0
- package/templates/scripts/qa/governance.js +174 -0
- package/templates/scripts/qa/hash-file.js +32 -0
- package/templates/scripts/qa/lease-lib.js +263 -0
- package/templates/scripts/qa/merge-backlog.js +324 -0
- package/templates/scripts/qa/mobile-e2e/analyze-timings.js +300 -0
- package/templates/scripts/qa/mobile-e2e/apk-cache.js +420 -0
- package/templates/scripts/qa/mobile-e2e/backend-build-cache.js +203 -0
- package/templates/scripts/qa/mobile-e2e/boot-emulator.sh +432 -0
- package/templates/scripts/qa/mobile-e2e/build-apk.sh +210 -0
- package/templates/scripts/qa/mobile-e2e/check-flows.js +453 -0
- package/templates/scripts/qa/mobile-e2e/config.yaml +18 -0
- package/templates/scripts/qa/mobile-e2e/device-down.sh +88 -0
- package/templates/scripts/qa/mobile-e2e/device-up.sh +258 -0
- package/templates/scripts/qa/mobile-e2e/ios-simulator-down.sh +72 -0
- package/templates/scripts/qa/mobile-e2e/ios-simulator-up.sh +153 -0
- package/templates/scripts/qa/mobile-e2e/job-events.js +181 -0
- package/templates/scripts/qa/mobile-e2e/prewarm.sh +56 -0
- package/templates/scripts/qa/mobile-e2e/run-detached.sh +222 -0
- package/templates/scripts/qa/mobile-e2e/run-maestro.js +1543 -0
- package/templates/scripts/qa/mobile-e2e/ui-dump.js +327 -0
- package/templates/scripts/qa/models.allowlist.txt +32 -0
- package/templates/scripts/qa/provider-stats.js +878 -0
- package/templates/scripts/qa/run-clean.js +83 -0
- package/templates/scripts/qa/run-gates.js +212 -0
- package/templates/scripts/qa/run-loop.js +220 -0
- package/templates/scripts/qa/session-recon.js +512 -0
- package/templates/scripts/qa/store-lib.js +324 -0
- package/templates/scripts/qa/tool-lib.js +134 -0
- package/templates/scripts/qa/where.js +214 -0
- package/templates/scripts/qa/wslconfig-merge.js +157 -0
- package/templates/scripts/serve.js +1328 -0
- package/templates/scripts/setup-nightly.sh +185 -0
- package/templates/scripts/setup-tmpfs.sh +374 -0
- package/templates/scripts/ship.js +586 -0
- package/templates/scripts/switch-reasoning-model.ps1 +101 -0
- package/templates/scripts/switch-reasoning-model.sh +104 -0
- package/templates/scripts/wait-for.js +342 -0
- package/templates/scripts/wsl-gpu.sh +180 -0
- package/templates/scripts/wsl-tune.sh +261 -0
|
@@ -0,0 +1,913 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
"use strict";
|
|
3
|
+
/*
|
|
4
|
+
* nightly — unattended full-E2E cycle for the day's landed code.
|
|
5
|
+
*
|
|
6
|
+
* The day loop lands features on `main` behind cheap gates; the device/Maestro
|
|
7
|
+
* tier is deliberately NOT run during the day (it is slow and needs the emulator
|
|
8
|
+
* + GPU). This script is the off-hours half: at ~02:00 a systemd timer runs the
|
|
9
|
+
* full battery against a dedicated `nightly` worktree fast-forwarded to
|
|
10
|
+
* `origin/main`, writes a machine-readable report, and reconciles the board:
|
|
11
|
+
*
|
|
12
|
+
* - a journey that fails and has no row → backlog add [nightly:flow=<j>]
|
|
13
|
+
* - a shipped row (archived, pending validation)
|
|
14
|
+
* whose journey now passes → mark nightly-validated
|
|
15
|
+
* whose journey still fails (journey-level) → backlog reopen the same row
|
|
16
|
+
* - a journey failure that already has a row → annotate, never duplicate
|
|
17
|
+
*
|
|
18
|
+
* The decision core (`decideReconcile`) is pure and unit-tested; this file owns
|
|
19
|
+
* only IO. Reports live under `.agents/qa-runs/nightly/<stamp>/`; the loop state
|
|
20
|
+
* (reopen counts, first-seen) lives in `.agents/qa-runs/nightly/state.json`.
|
|
21
|
+
*
|
|
22
|
+
* Usage:
|
|
23
|
+
* node scripts/nightly.js run [--root <dir>] [--dry-run] [--phase <name>] [--json]
|
|
24
|
+
* node scripts/nightly.js report [--root <dir>] [--json]
|
|
25
|
+
* node scripts/nightly.js status [--root <dir>]
|
|
26
|
+
*
|
|
27
|
+
* Config: `.agents/nightly.json` (project-owned, create-if-missing). See
|
|
28
|
+
* docs/nightly-testing.md.
|
|
29
|
+
*/
|
|
30
|
+
|
|
31
|
+
const fs = require("fs");
|
|
32
|
+
const path = require("path");
|
|
33
|
+
const { spawnSync } = require("child_process");
|
|
34
|
+
|
|
35
|
+
const SCHEMA = "nightly/1";
|
|
36
|
+
|
|
37
|
+
/* Default phase battery. `gate` phases map to package.json scripts (a missing
|
|
38
|
+
* script is SKIPPED, never failed — same semantics as run-gates.js). `heavy`
|
|
39
|
+
* phases are run capped through `serve.js run`. `device-e2e` is the built-in
|
|
40
|
+
* mobile sequence (device-up → Maestro matrix → device-down), included only when
|
|
41
|
+
* the project ships the mobile-e2e tier. */
|
|
42
|
+
const DEFAULT_CONFIG = {
|
|
43
|
+
schema: SCHEMA,
|
|
44
|
+
worktree: null,
|
|
45
|
+
seedCmd: null,
|
|
46
|
+
waitForQuietMin: 30,
|
|
47
|
+
stalePendingNights: 3,
|
|
48
|
+
keepDays: 30,
|
|
49
|
+
fileBacklog: true,
|
|
50
|
+
notify: "",
|
|
51
|
+
phases: [
|
|
52
|
+
{ name: "static-unit", kind: "gate", script: "test:quick", heavy: false, timeoutMin: 20 },
|
|
53
|
+
{ name: "api", kind: "gate", script: "test:api", heavy: false, timeoutMin: 30 },
|
|
54
|
+
{ name: "routes", kind: "gate", script: "test:routes", heavy: false, timeoutMin: 20 },
|
|
55
|
+
{ name: "web-e2e", kind: "gate", script: "test:e2e", heavy: true, timeoutMin: 30 },
|
|
56
|
+
{ name: "device-e2e", kind: "device-e2e", heavy: true, timeoutMin: 90 },
|
|
57
|
+
],
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
// ---------------------------------------------------------------- pure core
|
|
61
|
+
|
|
62
|
+
/* The journey a nightly row tracks, from its Task cell marker. */
|
|
63
|
+
function nightlyFlow(text) {
|
|
64
|
+
const m = String(text || "").match(/\[nightly:flow=([A-Za-z0-9._-]+)\]/i);
|
|
65
|
+
return m ? m[1] : null;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/* A row is validated once its note records the confirming nightly run. */
|
|
69
|
+
function isValidated(text) {
|
|
70
|
+
return /nightly-validated/i.test(String(text || ""));
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/* Normalize an error line into a stable-ish signature (paths/numbers elided).
|
|
74
|
+
* Used for the report and for spotting a changed signature on the same journey,
|
|
75
|
+
* NOT for row matching (which is journey-level). */
|
|
76
|
+
function normalizeSignature(text) {
|
|
77
|
+
return String(text || "")
|
|
78
|
+
.toLowerCase()
|
|
79
|
+
.replace(/0x[0-9a-f]+/g, "0xn")
|
|
80
|
+
.replace(/\d+/g, "n")
|
|
81
|
+
.replace(/(?:\/[\w.\-]+){2,}/g, "/path")
|
|
82
|
+
.replace(/\s+/g, " ")
|
|
83
|
+
.replace(/[^a-z0-9 <>_:/.\-]+/g, "")
|
|
84
|
+
.trim()
|
|
85
|
+
.slice(0, 160);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function shortHash(s) {
|
|
89
|
+
let h = 5381;
|
|
90
|
+
const str = String(s || "");
|
|
91
|
+
for (let i = 0; i < str.length; i++) h = ((h << 5) + h + str.charCodeAt(i)) >>> 0;
|
|
92
|
+
return h.toString(16).padStart(8, "0").slice(0, 8);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/* Lines the nightly itself prints (progress, advisories, teardown hints). They
|
|
96
|
+
* are never a failure reason: the 2026-10-16 field shape recorded the harness's
|
|
97
|
+
* own `run-detached: --wait blocks this shell …` advisory as the device-e2e
|
|
98
|
+
* reason and a stack-dump fragment (`operator: '==', diff: 'simple' }`) as the
|
|
99
|
+
* static-unit reason, hiding the real failures. Pure. */
|
|
100
|
+
const HARNESS_NOISE = [
|
|
101
|
+
/^run-detached:/i,
|
|
102
|
+
/^in an agent session prefer detached/i,
|
|
103
|
+
/^use --wait only headless/i,
|
|
104
|
+
/^progress every \d+s below\.?$/i,
|
|
105
|
+
/^[!…]?\s*wsl gpu passthrough/i,
|
|
106
|
+
/^enable d3d12 acceleration/i,
|
|
107
|
+
/^…?\s*invocation \(\d+\/\d+\)/,
|
|
108
|
+
/^mobile e2e progress:/i,
|
|
109
|
+
/^continue f-\d+: classify each failure/i,
|
|
110
|
+
/^flow (pass|fail):/i,
|
|
111
|
+
/^serve run: .* (started|exited)/i,
|
|
112
|
+
/^job:\s|^record:\s|^events:\s|^log:\s|^tail:\s|^exit:\s/i,
|
|
113
|
+
];
|
|
114
|
+
|
|
115
|
+
/* The one line that names the actual failure. Prefers a test/flow marker over
|
|
116
|
+
* raw tail order, and ignores harness noise so a real failure is never masked by
|
|
117
|
+
* the runner's own output. */
|
|
118
|
+
function failureReason(output) {
|
|
119
|
+
const lines = String(output || "")
|
|
120
|
+
.split(/\r?\n/)
|
|
121
|
+
.map((l) => l.trimEnd())
|
|
122
|
+
.filter((l) => l.trim() !== "" && !HARNESS_NOISE.some((re) => re.test(l.trim())));
|
|
123
|
+
const pick = (re) => {
|
|
124
|
+
for (let i = lines.length - 1; i >= 0; i--) if (re.test(lines[i].trim())) return lines[i].trim();
|
|
125
|
+
return null;
|
|
126
|
+
};
|
|
127
|
+
const reason =
|
|
128
|
+
pick(/^(✖|×|not ok\b|FAIL\b|FAILED\b)/i) ||
|
|
129
|
+
pick(/^-\s+[\w.\-]+\.ya?ml:\s+\S/) ||
|
|
130
|
+
pick(/(assertionerror|error|exception):\s*\S/i) ||
|
|
131
|
+
pick(/^assertion is false:/i) ||
|
|
132
|
+
pick(/(command not found|no such file or directory|not found on path|ENOENT|EACCES)/i) ||
|
|
133
|
+
(lines.length ? lines[lines.length - 1].trim() : "");
|
|
134
|
+
return reason.slice(0, 240);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/* Harness-side or product-side? A missing tool, an environment gap, a capacity
|
|
138
|
+
* refusal or a runner queued-but-unfinished is the harness's problem, and must
|
|
139
|
+
* NOT reopen a product row (the F-331 churn: two harness fixes, still re-opened
|
|
140
|
+
* P1 every night on its own advisory text). Pure. */
|
|
141
|
+
function classifyFailure(reason) {
|
|
142
|
+
const r = String(reason || "");
|
|
143
|
+
if (
|
|
144
|
+
/\b(command not found|not found on path|no such file or directory|permission denied|ENOENT|EACCES|ANDROID_HOME|JAVA_HOME|maestro|adb)\b/i.test(r) ||
|
|
145
|
+
/run-detached|--wait blocks|llvmpipe|WSL GPU|capacity|envelope|device-up|emulator|timeout|timed out|signal|killed/i.test(r)
|
|
146
|
+
) {
|
|
147
|
+
return "harness";
|
|
148
|
+
}
|
|
149
|
+
return "product";
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/* Resolve the configured phases against what this project actually wires.
|
|
153
|
+
* Returns { phases, skipped }: missing `gate` scripts and the device phase (when
|
|
154
|
+
* the tier is absent) are reported as skipped, never run. Pure. */
|
|
155
|
+
function resolvePhases(config, scripts, hasDeviceE2e) {
|
|
156
|
+
const phases = [];
|
|
157
|
+
const skipped = [];
|
|
158
|
+
for (const p of (config && config.phases) || []) {
|
|
159
|
+
if (!p || !p.name) continue;
|
|
160
|
+
if (p.kind === "gate") {
|
|
161
|
+
if (!p.script || !(p.script in (scripts || {}))) {
|
|
162
|
+
skipped.push({ name: p.name, reason: `gate "${p.script || "?"}" not wired in package.json` });
|
|
163
|
+
continue;
|
|
164
|
+
}
|
|
165
|
+
phases.push({ ...p, cmd: `npm run ${p.script}` });
|
|
166
|
+
} else if (p.kind === "cmd") {
|
|
167
|
+
if (!p.cmd) {
|
|
168
|
+
skipped.push({ name: p.name, reason: "cmd phase has no cmd" });
|
|
169
|
+
continue;
|
|
170
|
+
}
|
|
171
|
+
phases.push({ ...p });
|
|
172
|
+
} else if (p.kind === "device-e2e") {
|
|
173
|
+
if (!hasDeviceE2e) {
|
|
174
|
+
skipped.push({ name: p.name, reason: "mobile-e2e tier not present" });
|
|
175
|
+
continue;
|
|
176
|
+
}
|
|
177
|
+
phases.push({ ...p });
|
|
178
|
+
} else {
|
|
179
|
+
skipped.push({ name: p.name, reason: `unknown kind "${p.kind}"` });
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
return { phases, skipped };
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
function daysBetween(a, b) {
|
|
186
|
+
const t1 = Date.parse(a);
|
|
187
|
+
const t2 = Date.parse(b);
|
|
188
|
+
if (!Number.isFinite(t1) || !Number.isFinite(t2)) return 0;
|
|
189
|
+
return Math.max(0, Math.round((t2 - t1) / 86400000));
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/*
|
|
193
|
+
* Decide the board reconciliation. Pure.
|
|
194
|
+
*
|
|
195
|
+
* input.failures [{ flow, phase, signature, reason }] — failed journeys/phases
|
|
196
|
+
* input.exercised [flow, ...] — flows this run reached
|
|
197
|
+
* input.rows [{ id, section, task, closed }] — whole-board snapshot
|
|
198
|
+
* input.state { flows: { <flow>: { reopenCount, firstSeen, lastSeen,
|
|
199
|
+
* lastSignature, validatedAt } } }
|
|
200
|
+
* input.now ISO timestamp
|
|
201
|
+
*
|
|
202
|
+
* Journey-level matching: one row per failing journey, reopened until green.
|
|
203
|
+
* actions: { kind: reopen|validate|add|annotate|pending, ... }
|
|
204
|
+
*/
|
|
205
|
+
function decideReconcile(input) {
|
|
206
|
+
const failures = Array.isArray(input.failures) ? input.failures : [];
|
|
207
|
+
const exercised = new Set((input.exercised || []).map((f) => String(f).toLowerCase()));
|
|
208
|
+
const rows = Array.isArray(input.rows) ? input.rows : [];
|
|
209
|
+
const now = input.now || new Date().toISOString();
|
|
210
|
+
const staleDays = Number.isFinite(input.stalePendingDays) ? input.stalePendingDays : 3;
|
|
211
|
+
|
|
212
|
+
const failedFlows = new Map();
|
|
213
|
+
for (const f of failures) {
|
|
214
|
+
if (f && f.flow) failedFlows.set(String(f.flow).toLowerCase(), f);
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
const byFlow = new Map();
|
|
218
|
+
for (const r of rows) {
|
|
219
|
+
const flow = nightlyFlow(r.task);
|
|
220
|
+
if (!flow) continue;
|
|
221
|
+
const k = flow.toLowerCase();
|
|
222
|
+
if (!byFlow.has(k)) byFlow.set(k, []);
|
|
223
|
+
byFlow.get(k).push(r);
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
const actions = [];
|
|
227
|
+
const flows = { ...((input.state && input.state.flows) || {}) };
|
|
228
|
+
const ensured = (k) => {
|
|
229
|
+
if (!flows[k]) flows[k] = { reopenCount: 0, firstSeen: now, lastSeen: now };
|
|
230
|
+
return flows[k];
|
|
231
|
+
};
|
|
232
|
+
|
|
233
|
+
// Direction A — archived rows awaiting validation that did NOT fail.
|
|
234
|
+
for (const [k, list] of byFlow) {
|
|
235
|
+
if (failedFlows.has(k)) continue; // the failure pass handles it
|
|
236
|
+
for (const r of list) {
|
|
237
|
+
if (r.section !== "archive" || isValidated(r.closed)) continue;
|
|
238
|
+
const st = ensured(k);
|
|
239
|
+
if (exercised.has(k)) {
|
|
240
|
+
flows[k] = { ...st, lastSeen: now, validatedAt: now };
|
|
241
|
+
actions.push({ kind: "validate", id: r.id, flow: k });
|
|
242
|
+
} else {
|
|
243
|
+
const stale = st.firstSeen ? daysBetween(st.firstSeen, now) >= staleDays : false;
|
|
244
|
+
actions.push({ kind: "pending", id: r.id, flow: k, reason: "not exercised this run", stale });
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
// Direction B — failures.
|
|
250
|
+
for (const [k, f] of failedFlows) {
|
|
251
|
+
const list = byFlow.get(k) || [];
|
|
252
|
+
const openish = list.find((r) => r.section !== "archive");
|
|
253
|
+
const archived = list.find((r) => r.section === "archive");
|
|
254
|
+
|
|
255
|
+
// A HARNESS-side failure (missing tool, env gap, capacity refusal, runner
|
|
256
|
+
// timeout) is not evidence about the product. Re-opening a product row on it
|
|
257
|
+
// is the F-331 churn: a P1 product row re-opened every night by the runner's
|
|
258
|
+
// own advisory. Surface it as a run-level action instead, and leave the row
|
|
259
|
+
// alone. A product failure keeps the original journey-level behaviour.
|
|
260
|
+
if (f.side === "harness") {
|
|
261
|
+
flows[k] = { ...ensured(k), lastSeen: now, lastSignature: f.signature || null };
|
|
262
|
+
actions.push({
|
|
263
|
+
kind: "harness",
|
|
264
|
+
flow: k,
|
|
265
|
+
phase: f.phase || null,
|
|
266
|
+
signature: f.signature || null,
|
|
267
|
+
reason: f.reason || "",
|
|
268
|
+
trackedId: (openish || archived || {}).id || null,
|
|
269
|
+
});
|
|
270
|
+
continue;
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
if (openish) {
|
|
274
|
+
actions.push({ kind: "annotate", id: openish.id, flow: k, reason: "already tracked (open)" });
|
|
275
|
+
continue;
|
|
276
|
+
}
|
|
277
|
+
if (archived) {
|
|
278
|
+
const st = ensured(k);
|
|
279
|
+
const reopenCount = (st.reopenCount || 0) + 1;
|
|
280
|
+
flows[k] = { ...st, reopenCount, lastSeen: now, lastSignature: f.signature || st.lastSignature };
|
|
281
|
+
actions.push({
|
|
282
|
+
kind: "reopen",
|
|
283
|
+
id: archived.id,
|
|
284
|
+
flow: k,
|
|
285
|
+
reopenCount,
|
|
286
|
+
priority: reopenCount >= 2 ? "P1" : "P2",
|
|
287
|
+
signature: f.signature || null,
|
|
288
|
+
});
|
|
289
|
+
continue;
|
|
290
|
+
}
|
|
291
|
+
flows[k] = { ...ensured(k), lastSeen: now, lastSignature: f.signature || null };
|
|
292
|
+
actions.push({ kind: "add", flow: k, phase: f.phase || null, signature: f.signature || null, reason: f.reason || "" });
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
return { actions, state: { flows } };
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
// ---------------------------------------------------------------- IO helpers
|
|
299
|
+
|
|
300
|
+
function findRoot(start) {
|
|
301
|
+
let dir = path.resolve(start || process.cwd());
|
|
302
|
+
for (;;) {
|
|
303
|
+
if (fs.existsSync(path.join(dir, "BACKLOG.md"))) return dir;
|
|
304
|
+
const parent = path.dirname(dir);
|
|
305
|
+
if (parent === dir) return null;
|
|
306
|
+
dir = parent;
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
function readJson(p, fallback) {
|
|
311
|
+
try {
|
|
312
|
+
return JSON.parse(fs.readFileSync(p, "utf8"));
|
|
313
|
+
} catch (_) {
|
|
314
|
+
return fallback;
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
function loadConfig(root) {
|
|
319
|
+
const p = path.join(root, ".agents", "nightly.json");
|
|
320
|
+
const raw = readJson(p, null);
|
|
321
|
+
if (!raw) return { ...DEFAULT_CONFIG, _missing: true, _path: p };
|
|
322
|
+
return { ...DEFAULT_CONFIG, ...raw, phases: Array.isArray(raw.phases) ? raw.phases : DEFAULT_CONFIG.phases, _path: p };
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
function readScripts(root) {
|
|
326
|
+
const pkg = readJson(path.join(root, "package.json"), null);
|
|
327
|
+
return pkg && pkg.scripts ? pkg.scripts : {};
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
function reportsDir(root) {
|
|
331
|
+
return path.join(root, ".agents", "qa-runs", "nightly");
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
function latestPath(root) {
|
|
335
|
+
return path.join(reportsDir(root), "latest.json");
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
function statePath(root) {
|
|
339
|
+
return path.join(reportsDir(root), "state.json");
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
function stamp() {
|
|
343
|
+
const d = new Date();
|
|
344
|
+
const pad = (n) => String(n).padStart(2, "0");
|
|
345
|
+
return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())}T${pad(d.getHours())}${pad(d.getMinutes())}`;
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
function tail(text, lines = 15) {
|
|
349
|
+
const arr = String(text || "").split(/\r?\n/).filter(Boolean);
|
|
350
|
+
return arr.slice(-lines).join("\n");
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
function run(root, cmd, timeoutMs) {
|
|
354
|
+
const r = spawnSync(cmd, {
|
|
355
|
+
cwd: root,
|
|
356
|
+
shell: true,
|
|
357
|
+
encoding: "utf8",
|
|
358
|
+
timeout: timeoutMs,
|
|
359
|
+
maxBuffer: 32 * 1024 * 1024,
|
|
360
|
+
});
|
|
361
|
+
return {
|
|
362
|
+
code: r.status == null ? (r.signal ? 124 : 1) : r.status,
|
|
363
|
+
signal: r.signal || null,
|
|
364
|
+
output: (r.stdout || "") + (r.stderr || ""),
|
|
365
|
+
};
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
function runNode(root, args, timeoutMs) {
|
|
369
|
+
const r = spawnSync(process.execPath, args, {
|
|
370
|
+
cwd: root,
|
|
371
|
+
encoding: "utf8",
|
|
372
|
+
timeout: timeoutMs,
|
|
373
|
+
maxBuffer: 32 * 1024 * 1024,
|
|
374
|
+
});
|
|
375
|
+
return { code: r.status == null ? 1 : r.status, output: (r.stdout || "") + (r.stderr || "") };
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
function git(root, args) {
|
|
379
|
+
const r = spawnSync("git", ["-C", root, ...args], { encoding: "utf8", timeout: 120000 });
|
|
380
|
+
return { code: r.status == null ? 1 : r.status, out: (r.stdout || "").trim(), err: (r.stderr || "").trim() };
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
// ---------------------------------------------------------------- phases
|
|
384
|
+
|
|
385
|
+
function runPhase(root, phase, opts) {
|
|
386
|
+
const timeoutMs = (phase.timeoutMin || 30) * 60000;
|
|
387
|
+
if (phase.kind === "device-e2e") return runDeviceE2e(root, phase, opts);
|
|
388
|
+
const cmd = phase.cmd;
|
|
389
|
+
if (opts.dryRun) return { name: phase.name, status: "pass", dryRun: true, cmd, output: "" };
|
|
390
|
+
let res;
|
|
391
|
+
if (phase.heavy) {
|
|
392
|
+
const args = ["scripts/serve.js", "run", "--cmd", cmd, "--name", `nightly-${phase.name}`];
|
|
393
|
+
if (phase.memMax) args.push("--mem-max", phase.memMax);
|
|
394
|
+
if (phase.cpuMax) args.push("--cpu-max", phase.cpuMax);
|
|
395
|
+
res = runNode(root, args, timeoutMs);
|
|
396
|
+
} else {
|
|
397
|
+
res = run(root, cmd, timeoutMs);
|
|
398
|
+
}
|
|
399
|
+
return {
|
|
400
|
+
name: phase.name,
|
|
401
|
+
status: res.code === 0 ? "pass" : "fail",
|
|
402
|
+
code: res.code,
|
|
403
|
+
output: tail(res.output, 15),
|
|
404
|
+
};
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
function runDeviceE2e(root, phase, opts) {
|
|
408
|
+
const me = path.join("scripts", "qa", "mobile-e2e");
|
|
409
|
+
const reportFile = path.join(root, ".serve", "mobile-e2e-last-run.json");
|
|
410
|
+
const flows = [];
|
|
411
|
+
const steps = [];
|
|
412
|
+
if (opts.dryRun) {
|
|
413
|
+
return { name: phase.name, status: "pass", dryRun: true, cmd: "device-e2e", output: "", flows };
|
|
414
|
+
}
|
|
415
|
+
const step = (label, cmd, timeoutMin) => {
|
|
416
|
+
const r = run(root, cmd, (timeoutMin || phase.timeoutMin || 90) * 60000);
|
|
417
|
+
steps.push({ label, code: r.code, output: tail(r.output, 8) });
|
|
418
|
+
return r;
|
|
419
|
+
};
|
|
420
|
+
|
|
421
|
+
// Assert the external toolchain BEFORE device-up / the matrix. A missing tool
|
|
422
|
+
// must name itself in seconds, not surface as a bare exit code after the run
|
|
423
|
+
// (2026-10-16: `device-e2e code 127` after 23 min, on a passing matrix). A
|
|
424
|
+
// preflight failure is harness-side — it must not reopen a product row.
|
|
425
|
+
const pre = step("device-preflight", `node ${me}/run-maestro.js --flows all --preflight`, 3);
|
|
426
|
+
if (pre.code !== 0) {
|
|
427
|
+
return {
|
|
428
|
+
name: phase.name,
|
|
429
|
+
status: "fail",
|
|
430
|
+
code: pre.code,
|
|
431
|
+
harness: true,
|
|
432
|
+
output: tail(pre.output, 20),
|
|
433
|
+
flows,
|
|
434
|
+
steps,
|
|
435
|
+
};
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
const up = step("device-up", `bash ${me}/device-up.sh`, phase.timeoutMin);
|
|
439
|
+
if (up.code !== 0) {
|
|
440
|
+
try {
|
|
441
|
+
run(root, `bash ${me}/device-down.sh --emulator`, 120000);
|
|
442
|
+
} catch (_) {
|
|
443
|
+
/* best-effort teardown */
|
|
444
|
+
}
|
|
445
|
+
return { name: phase.name, status: "fail", code: up.code, harness: true, output: tail(up.output, 20), flows, steps };
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
const matrix = step("matrix", `bash ${me}/run-detached.sh --flows all --reuse-session --wait`, phase.timeoutMin);
|
|
449
|
+
const report = readJson(reportFile, null);
|
|
450
|
+
if (report && Array.isArray(report.flows)) {
|
|
451
|
+
for (const f of report.flows) {
|
|
452
|
+
flows.push({ name: f.name, status: f.status, reason: f.reason || "", durationMs: f.durationMs });
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
step("device-down", `bash ${me}/device-down.sh --emulator`, 5);
|
|
457
|
+
if (fs.existsSync(path.join(root, "package.json")) && readScripts(root)["mobile-e2e:reset"]) {
|
|
458
|
+
run(root, "npm run mobile-e2e:reset", 300000);
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
const failed = flows.filter((f) => f.status !== "pass");
|
|
462
|
+
const ok = matrix.code === 0 && failed.length === 0;
|
|
463
|
+
// A non-zero matrix that produced NO product flow failure is the runner itself
|
|
464
|
+
// (aborted, tool missing, timeout) — harness-side, so it never reopens a row.
|
|
465
|
+
const harness = !ok && failed.length === 0;
|
|
466
|
+
return {
|
|
467
|
+
name: phase.name,
|
|
468
|
+
status: ok ? "pass" : "fail",
|
|
469
|
+
code: matrix.code,
|
|
470
|
+
harness,
|
|
471
|
+
output: tail(matrix.output, 20),
|
|
472
|
+
flows,
|
|
473
|
+
steps,
|
|
474
|
+
};
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
// ---------------------------------------------------------------- prepare
|
|
478
|
+
|
|
479
|
+
/* Fast-forward (or create) the dedicated nightly worktree to origin/main so the
|
|
480
|
+
* device build compiles exactly the day's landed code, never the live tree. */
|
|
481
|
+
function prepareWorktree(root, config, opts) {
|
|
482
|
+
if (opts.noWorktree) return { root, note: "worktree disabled (--no-worktree)" };
|
|
483
|
+
const repo = path.basename(root);
|
|
484
|
+
const dir = config.worktree
|
|
485
|
+
? path.resolve(root, config.worktree)
|
|
486
|
+
: path.resolve(root, "..", ".worktrees", repo, "nightly");
|
|
487
|
+
|
|
488
|
+
if (opts.dryRun) return { root: dir, note: `[dry-run] would sync worktree ${dir}` };
|
|
489
|
+
|
|
490
|
+
const list = git(root, ["worktree", "list", "--porcelain"]);
|
|
491
|
+
const exists = list.out.split(/\r?\n/).some((l) => l.startsWith("worktree ") && path.resolve(l.slice(9).trim()) === dir);
|
|
492
|
+
|
|
493
|
+
if (!exists) {
|
|
494
|
+
fs.mkdirSync(path.dirname(dir), { recursive: true });
|
|
495
|
+
git(root, ["fetch", "origin", "--quiet"]);
|
|
496
|
+
const base = git(root, ["rev-parse", "--verify", "origin/main"]).code === 0 ? "origin/main" : "HEAD";
|
|
497
|
+
// A leftover `nightly` branch (its worktree was removed) must be reused, not
|
|
498
|
+
// recreated — `git worktree add -b nightly` fails when the branch exists.
|
|
499
|
+
const hasBranch = git(root, ["rev-parse", "--verify", "nightly"]).code === 0;
|
|
500
|
+
const add = hasBranch
|
|
501
|
+
? git(root, ["worktree", "add", "--no-track", dir, "nightly"])
|
|
502
|
+
: git(root, ["worktree", "add", "--no-track", "-b", "nightly", dir, base]);
|
|
503
|
+
if (add.code !== 0) return { root, error: `could not create nightly worktree: ${add.err || add.out}` };
|
|
504
|
+
// Bootstrap the fresh checkout exactly as `backlog worktree` does (project
|
|
505
|
+
// convention script): node_modules / generated client / .env / hooks. A
|
|
506
|
+
// reused worktree keeps its bootstrap, so only the create path runs it.
|
|
507
|
+
const setup = [path.join(dir, "scripts", "worktree-setup.js"), path.join(root, "scripts", "worktree-setup.js")].find((p) =>
|
|
508
|
+
fs.existsSync(p)
|
|
509
|
+
);
|
|
510
|
+
if (setup) run(dir, `node ${setup} ${dir}`, 20 * 60000);
|
|
511
|
+
} else {
|
|
512
|
+
git(dir, ["fetch", "origin", "--quiet"]);
|
|
513
|
+
if (git(dir, ["rev-parse", "--verify", "origin/main"]).code === 0) {
|
|
514
|
+
git(dir, ["checkout", "nightly"]);
|
|
515
|
+
const reset = git(dir, ["reset", "--hard", "origin/main"]);
|
|
516
|
+
if (reset.code !== 0) return { root, error: `could not fast-forward nightly worktree: ${reset.err || reset.out}` };
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
return { root: dir, note: `nightly worktree ${dir} @ ${git(dir, ["rev-parse", "--short", "HEAD"]).out || "?"}` };
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
/* Wait (bounded) for live human sessions to end so nightly never fights a live
|
|
523
|
+
* agent for the single heavy-tier budget. Returns { quiet, waitedMin, live }. */
|
|
524
|
+
function waitForQuiet(root, config, opts) {
|
|
525
|
+
if (opts.dryRun) return { quiet: true, waitedMin: 0, live: 0 };
|
|
526
|
+
let leases = null;
|
|
527
|
+
try {
|
|
528
|
+
leases = require(path.join(root, "scripts", "qa", "lease-lib.js"));
|
|
529
|
+
} catch (_) {
|
|
530
|
+
return { quiet: true, waitedMin: 0, live: 0, note: "lease-lib unavailable — skipping quiesce check" };
|
|
531
|
+
}
|
|
532
|
+
const deadline = Date.now() + (config.waitForQuietMin || 0) * 60000;
|
|
533
|
+
let live = 0;
|
|
534
|
+
for (;;) {
|
|
535
|
+
try {
|
|
536
|
+
live = leases.liveLeases(root).length;
|
|
537
|
+
} catch (_) {
|
|
538
|
+
live = 0;
|
|
539
|
+
}
|
|
540
|
+
if (live === 0) return { quiet: true, waitedMin: Math.round((Date.now() - (deadline - (config.waitForQuietMin || 0) * 60000)) / 60000), live: 0 };
|
|
541
|
+
if (Date.now() >= deadline) return { quiet: false, waitedMin: config.waitForQuietMin || 0, live };
|
|
542
|
+
spawnSync(process.execPath, ["-e", "setTimeout(()=>{},5000)"]);
|
|
543
|
+
}
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
// ---------------------------------------------------------------- reconcile
|
|
547
|
+
|
|
548
|
+
function boardRows(root) {
|
|
549
|
+
const r = runNode(root, ["scripts/backlog.js", "list", "--all", "--json"], 30000);
|
|
550
|
+
if (r.code !== 0) return null;
|
|
551
|
+
try {
|
|
552
|
+
return JSON.parse(r.output);
|
|
553
|
+
} catch (_) {
|
|
554
|
+
return null;
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
/* Execute the reconcile actions through the canonical `backlog.js` CLI (so the
|
|
559
|
+
* board transaction commits + publishes each change). Board ops run in the MAIN
|
|
560
|
+
* checkout, never the disposable nightly worktree. */
|
|
561
|
+
function applyReconcile(boardRoot, actions, ctx) {
|
|
562
|
+
const applied = [];
|
|
563
|
+
const evidence = (flow) =>
|
|
564
|
+
`nightly ${ctx.date} flow=${flow} sig=${ctx.signature(flow) || "?"} report=${ctx.reportRel}`;
|
|
565
|
+
|
|
566
|
+
for (const a of actions) {
|
|
567
|
+
if (a.kind === "harness") {
|
|
568
|
+
// Not a product signal — no board mutation. Recorded so the operator (and
|
|
569
|
+
// the report) can see the run failed on harness plumbing, not the product.
|
|
570
|
+
applied.push({
|
|
571
|
+
...a,
|
|
572
|
+
ok: true,
|
|
573
|
+
out: a.trackedId
|
|
574
|
+
? `${a.trackedId} left untouched (harness-side failure)`
|
|
575
|
+
: "harness-side failure (no product row)",
|
|
576
|
+
});
|
|
577
|
+
} else if (a.kind === "add") {
|
|
578
|
+
const task = `[nightly:flow=${a.flow}] ${a.flow} e2e failed`;
|
|
579
|
+
const r = runNode(
|
|
580
|
+
boardRoot,
|
|
581
|
+
["scripts/backlog.js", "add", task, "--p", "P2", "--evidence", evidence(a.flow)],
|
|
582
|
+
30000
|
|
583
|
+
);
|
|
584
|
+
applied.push({ ...a, ok: r.code === 0, out: tail(r.output, 3) });
|
|
585
|
+
} else if (a.kind === "reopen") {
|
|
586
|
+
const reason = `nightly ${ctx.date}: still failing — ${a.flow}${a.signature ? ` (sig ${a.signature})` : ""} [nightly:reopen=${a.reopenCount}]`;
|
|
587
|
+
const r = runNode(
|
|
588
|
+
boardRoot,
|
|
589
|
+
["scripts/backlog.js", "reopen", a.id, "--reason", reason, "--p", a.priority || "P2"],
|
|
590
|
+
30000
|
|
591
|
+
);
|
|
592
|
+
applied.push({ ...a, ok: r.code === 0, out: tail(r.output, 3) });
|
|
593
|
+
} else if (a.kind === "validate") {
|
|
594
|
+
const r = runNode(
|
|
595
|
+
boardRoot,
|
|
596
|
+
["scripts/backlog.js", "archive", a.id, "--closed", `nightly-validated ${ctx.date} (${ctx.reportRel})`],
|
|
597
|
+
30000
|
|
598
|
+
);
|
|
599
|
+
applied.push({ ...a, ok: r.code === 0, out: tail(r.output, 3) });
|
|
600
|
+
} else {
|
|
601
|
+
applied.push({ ...a, ok: true, out: "no board change" });
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
return applied;
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
// ---------------------------------------------------------------- report
|
|
608
|
+
|
|
609
|
+
function buildFailureList(phaseResults) {
|
|
610
|
+
const failures = [];
|
|
611
|
+
const exercised = [];
|
|
612
|
+
for (const p of phaseResults) {
|
|
613
|
+
// A phase that ran without per-journey results (a gate/cmd phase) is
|
|
614
|
+
// "exercised" by its own name, so a flow=<phase> row can be validated on a
|
|
615
|
+
// green night. A matrix phase reports journeys instead (below).
|
|
616
|
+
if ((p.status === "pass" || p.status === "fail") && !(p.flows && p.flows.length)) exercised.push(p.name);
|
|
617
|
+
if (p.flows && p.flows.length) {
|
|
618
|
+
for (const f of p.flows) {
|
|
619
|
+
exercised.push(f.name);
|
|
620
|
+
if (f.status !== "pass") {
|
|
621
|
+
const reason = f.reason || "";
|
|
622
|
+
failures.push({
|
|
623
|
+
flow: f.name,
|
|
624
|
+
phase: p.name,
|
|
625
|
+
signature: shortHash(normalizeSignature(reason || f.name)),
|
|
626
|
+
reason,
|
|
627
|
+
side: classifyFailure(reason),
|
|
628
|
+
});
|
|
629
|
+
}
|
|
630
|
+
}
|
|
631
|
+
} else if (p.status === "fail") {
|
|
632
|
+
// Prefer a flow-level reason from the failed phase over its raw tail.
|
|
633
|
+
const fromFlow = (p.flows || []).find((f) => f.status !== "pass" && f.reason);
|
|
634
|
+
const reason = fromFlow ? fromFlow.reason : failureReason(p.output);
|
|
635
|
+
failures.push({
|
|
636
|
+
flow: p.name,
|
|
637
|
+
phase: p.name,
|
|
638
|
+
signature: shortHash(normalizeSignature(reason || p.name)),
|
|
639
|
+
reason,
|
|
640
|
+
side: p.side || (p.harness ? "harness" : null) || classifyFailure(reason),
|
|
641
|
+
});
|
|
642
|
+
}
|
|
643
|
+
}
|
|
644
|
+
return { failures, exercised };
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
function writeReport(root, report) {
|
|
648
|
+
const dir = path.join(reportsDir(root), report.stamp);
|
|
649
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
650
|
+
const jsonPath = path.join(dir, "report.json");
|
|
651
|
+
fs.writeFileSync(jsonPath, JSON.stringify(report, null, 2) + "\n", "utf8");
|
|
652
|
+
|
|
653
|
+
const lines = [];
|
|
654
|
+
lines.push(`# Nightly E2E — ${report.verdict === "pass" ? "PASS" : "FAIL"} (${report.stamp})`);
|
|
655
|
+
lines.push("");
|
|
656
|
+
lines.push(`- target: ${report.target} @ ${report.head || "?"}`);
|
|
657
|
+
lines.push(`- wall: ${(report.durationMs / 60000).toFixed(1)} min`);
|
|
658
|
+
lines.push(`- phases: ${report.phases.map((p) => `${p.name}=${p.status}`).join(", ")}`);
|
|
659
|
+
if (report.failures.length) {
|
|
660
|
+
const product = report.failures.filter((f) => f.side !== "harness");
|
|
661
|
+
const harness = report.failures.filter((f) => f.side === "harness");
|
|
662
|
+
if (product.length) {
|
|
663
|
+
lines.push("");
|
|
664
|
+
lines.push("## Failures");
|
|
665
|
+
for (const f of product) lines.push(`- **${f.flow}** (${f.phase}): ${f.reason || "see report"}`);
|
|
666
|
+
}
|
|
667
|
+
if (harness.length) {
|
|
668
|
+
lines.push("");
|
|
669
|
+
lines.push("## Harness-side (not a product failure — no board row touched)");
|
|
670
|
+
for (const f of harness) lines.push(`- **${f.flow}** (${f.phase}): ${f.reason || "see report"}`);
|
|
671
|
+
}
|
|
672
|
+
}
|
|
673
|
+
if (report.reconcile && report.reconcile.length) {
|
|
674
|
+
lines.push("");
|
|
675
|
+
lines.push("## Board reconciliation");
|
|
676
|
+
for (const a of report.reconcile) lines.push(`- ${a.kind} ${a.id || a.flow}${a.flow && a.id ? ` (${a.flow})` : ""}${a.ok === false ? " — FAILED" : ""}`);
|
|
677
|
+
}
|
|
678
|
+
lines.push("");
|
|
679
|
+
lines.push("Full report: " + path.relative(root, jsonPath));
|
|
680
|
+
fs.writeFileSync(path.join(dir, "report.md"), lines.join("\n") + "\n", "utf8");
|
|
681
|
+
fs.writeFileSync(latestPath(root), JSON.stringify(report, null, 2) + "\n", "utf8");
|
|
682
|
+
return { jsonPath, mdPath: path.join(dir, "report.md") };
|
|
683
|
+
}
|
|
684
|
+
|
|
685
|
+
function pruneOldReports(root, keepDays) {
|
|
686
|
+
const base = reportsDir(root);
|
|
687
|
+
if (!fs.existsSync(base)) return;
|
|
688
|
+
const cutoff = Date.now() - (keepDays || 30) * 86400000;
|
|
689
|
+
for (const name of fs.readdirSync(base)) {
|
|
690
|
+
const p = path.join(base, name);
|
|
691
|
+
let st;
|
|
692
|
+
try {
|
|
693
|
+
st = fs.statSync(p);
|
|
694
|
+
} catch (_) {
|
|
695
|
+
continue;
|
|
696
|
+
}
|
|
697
|
+
if (st.isDirectory() && /^\d{4}-\d{2}-\d{2}T/.test(name) && st.mtimeMs < cutoff) {
|
|
698
|
+
try {
|
|
699
|
+
fs.rmSync(p, { recursive: true, force: true });
|
|
700
|
+
} catch (_) {
|
|
701
|
+
/* best-effort */
|
|
702
|
+
}
|
|
703
|
+
}
|
|
704
|
+
}
|
|
705
|
+
}
|
|
706
|
+
|
|
707
|
+
// ---------------------------------------------------------------- commands
|
|
708
|
+
|
|
709
|
+
function cmdRun(opts) {
|
|
710
|
+
const root = path.resolve(opts.root || process.cwd());
|
|
711
|
+
const config = loadConfig(root);
|
|
712
|
+
const scripts = readScripts(root);
|
|
713
|
+
const hasDeviceE2e = fs.existsSync(path.join(root, "scripts", "qa", "mobile-e2e", "device-up.sh"));
|
|
714
|
+
const { phases, skipped } = resolvePhases(config, scripts, hasDeviceE2e);
|
|
715
|
+
const selected = opts.phase ? phases.filter((p) => p.name === opts.phase) : phases;
|
|
716
|
+
|
|
717
|
+
const started = Date.now();
|
|
718
|
+
const runStamp = stamp();
|
|
719
|
+
const phaseResults = [];
|
|
720
|
+
const notes = [];
|
|
721
|
+
|
|
722
|
+
console.log(`nightly — ${root} (${selected.length} phase(s)${opts.dryRun ? ", dry-run" : ""})`);
|
|
723
|
+
if (config._missing) notes.push(`no .agents/nightly.json — using defaults (${config._path})`);
|
|
724
|
+
|
|
725
|
+
const wt = prepareWorktree(root, config, opts);
|
|
726
|
+
if (wt.error) {
|
|
727
|
+
notes.push(wt.error);
|
|
728
|
+
} else if (wt.note) {
|
|
729
|
+
notes.push(wt.note);
|
|
730
|
+
}
|
|
731
|
+
const testRoot = wt.error ? root : wt.root;
|
|
732
|
+
|
|
733
|
+
const quiet = waitForQuiet(root, config, opts);
|
|
734
|
+
if (!quiet.quiet) notes.push(`still busy after ${quiet.waitedMin} min (${quiet.live} live session(s)) — proceeding`);
|
|
735
|
+
|
|
736
|
+
if (!opts.dryRun && config.seedCmd) {
|
|
737
|
+
const s = run(testRoot, config.seedCmd, 300000);
|
|
738
|
+
notes.push(`seed: ${s.code === 0 ? "ok" : "FAILED"}`);
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
for (const phase of selected) {
|
|
742
|
+
const t0 = Date.now();
|
|
743
|
+
const res = runPhase(testRoot, phase, opts);
|
|
744
|
+
res.ms = Date.now() - t0;
|
|
745
|
+
phaseResults.push(res);
|
|
746
|
+
console.log(` ${res.status === "pass" ? "\u2713" : "\u2717"} ${res.name} (${(res.ms / 1000).toFixed(1)}s)`);
|
|
747
|
+
}
|
|
748
|
+
|
|
749
|
+
const { failures, exercised } = buildFailureList(phaseResults);
|
|
750
|
+
const rows = boardRows(root) || [];
|
|
751
|
+
const flatRows = [];
|
|
752
|
+
for (const section of ["inprogress", "queued", "open", "archive"]) {
|
|
753
|
+
const list = section === "inprogress" ? rows.inprogress : section === "archive" ? rows.archive : rows[section];
|
|
754
|
+
for (const r of list || []) {
|
|
755
|
+
flatRows.push({ id: r.id, section: section === "inprogress" ? "inprogress" : section, task: r.task || "", closed: r.closed || r.handoff || "" });
|
|
756
|
+
}
|
|
757
|
+
}
|
|
758
|
+
|
|
759
|
+
const priorState = readJson(statePath(root), { flows: {} });
|
|
760
|
+
const decision = decideReconcile({
|
|
761
|
+
failures,
|
|
762
|
+
exercised,
|
|
763
|
+
rows: flatRows,
|
|
764
|
+
state: priorState,
|
|
765
|
+
stalePendingDays: config.stalePendingNights,
|
|
766
|
+
now: new Date().toISOString(),
|
|
767
|
+
});
|
|
768
|
+
|
|
769
|
+
const reportRel = path.join(".agents", "qa-runs", "nightly", runStamp, "report.json");
|
|
770
|
+
let reconcile = decision.actions.map((a) => ({ ...a, ok: true, out: "not executed" }));
|
|
771
|
+
if (!opts.dryRun && config.fileBacklog !== false && !opts.phase) {
|
|
772
|
+
const ctx = {
|
|
773
|
+
date: new Date().toISOString().slice(0, 10),
|
|
774
|
+
reportRel,
|
|
775
|
+
signature: (flow) => {
|
|
776
|
+
const f = failures.find((x) => x.flow.toLowerCase() === flow.toLowerCase());
|
|
777
|
+
return f ? f.signature : "";
|
|
778
|
+
},
|
|
779
|
+
};
|
|
780
|
+
reconcile = applyReconcile(root, decision.actions, ctx);
|
|
781
|
+
fs.mkdirSync(reportsDir(root), { recursive: true });
|
|
782
|
+
fs.writeFileSync(statePath(root), JSON.stringify(decision.state, null, 2) + "\n", "utf8");
|
|
783
|
+
}
|
|
784
|
+
|
|
785
|
+
const verdict = failures.length === 0 && reconcile.every((a) => a.ok !== false) ? "pass" : "fail";
|
|
786
|
+
const report = {
|
|
787
|
+
schema: "nightly/1",
|
|
788
|
+
stamp: runStamp,
|
|
789
|
+
target: testRoot,
|
|
790
|
+
boardRoot: root,
|
|
791
|
+
head: git(testRoot, ["rev-parse", "--short", "HEAD"]).out || null,
|
|
792
|
+
startedAt: new Date(started).toISOString(),
|
|
793
|
+
finishedAt: new Date().toISOString(),
|
|
794
|
+
durationMs: Date.now() - started,
|
|
795
|
+
verdict,
|
|
796
|
+
phases: phaseResults.map((p) => ({ name: p.name, status: p.status, ms: p.ms, code: p.code || 0, output: p.output || "" })),
|
|
797
|
+
skipped,
|
|
798
|
+
failures,
|
|
799
|
+
exercised,
|
|
800
|
+
reconcile,
|
|
801
|
+
notes,
|
|
802
|
+
};
|
|
803
|
+
|
|
804
|
+
if (!opts.dryRun) {
|
|
805
|
+
writeReport(root, report);
|
|
806
|
+
pruneOldReports(root, config.keepDays);
|
|
807
|
+
if (config.notify) {
|
|
808
|
+
const n = run(root, `${config.notify} ${path.join(reportsDir(root), runStamp, "report.json")}`, 30000);
|
|
809
|
+
if (n.code !== 0) notes.push("notify command failed");
|
|
810
|
+
}
|
|
811
|
+
}
|
|
812
|
+
|
|
813
|
+
console.log(`\nRESULT: ${verdict.toUpperCase()} — ${phaseResults.filter((p) => p.status === "pass").length}/${phaseResults.length} phases passed, ${failures.length} failure(s)`);
|
|
814
|
+
for (const n of notes) console.log(` note: ${n}`);
|
|
815
|
+
if (opts.json) console.log(JSON.stringify(report, null, 2));
|
|
816
|
+
process.exitCode = verdict === "pass" ? 0 : 1;
|
|
817
|
+
}
|
|
818
|
+
|
|
819
|
+
function cmdReport(opts) {
|
|
820
|
+
const root = path.resolve(opts.root || process.cwd());
|
|
821
|
+
const latest = readJson(latestPath(root), null);
|
|
822
|
+
if (!latest) {
|
|
823
|
+
console.log("nightly: no report yet — run `node scripts/nightly.js run`.");
|
|
824
|
+
return;
|
|
825
|
+
}
|
|
826
|
+
if (opts.json) return console.log(JSON.stringify(latest, null, 2));
|
|
827
|
+
console.log(`nightly ${latest.verdict.toUpperCase()} — ${latest.stamp} (${(latest.durationMs / 60000).toFixed(1)} min) @ ${latest.target}`);
|
|
828
|
+
console.log(` phases: ${latest.phases.map((p) => `${p.name}=${p.status}`).join(", ")}`);
|
|
829
|
+
if (latest.failures.length) {
|
|
830
|
+
console.log(" failures:");
|
|
831
|
+
for (const f of latest.failures) console.log(` - ${f.flow} (${f.phase}): ${(f.reason || "").split(/\r?\n/)[0]}`);
|
|
832
|
+
}
|
|
833
|
+
if (latest.reconcile.length) {
|
|
834
|
+
console.log(" board:");
|
|
835
|
+
for (const a of latest.reconcile) console.log(` - ${a.kind} ${a.id || a.flow}${a.ok === false ? " FAILED" : ""}`);
|
|
836
|
+
}
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
function cmdStatus(opts) {
|
|
840
|
+
const root = path.resolve(opts.root || process.cwd());
|
|
841
|
+
const latest = readJson(latestPath(root), null);
|
|
842
|
+
const state = readJson(statePath(root), { flows: {} });
|
|
843
|
+
const pending = [];
|
|
844
|
+
const rows = boardRows(root);
|
|
845
|
+
if (rows) {
|
|
846
|
+
for (const r of rows.archive || []) {
|
|
847
|
+
const flow = nightlyFlow(r.task);
|
|
848
|
+
if (flow && !isValidated(r.closed) && !state.flows[flow.toLowerCase()]?.validatedAt) {
|
|
849
|
+
pending.push({ id: r.id, flow });
|
|
850
|
+
}
|
|
851
|
+
}
|
|
852
|
+
}
|
|
853
|
+
console.log(latest ? `last nightly: ${latest.verdict.toUpperCase()} @ ${latest.stamp}` : "last nightly: none");
|
|
854
|
+
console.log(`awaiting nightly validation: ${pending.length}${pending.length ? " — " + pending.map((p) => p.id).join(", ") : ""}`);
|
|
855
|
+
if (opts.json) console.log(JSON.stringify({ latest, state, pending }, null, 2));
|
|
856
|
+
}
|
|
857
|
+
|
|
858
|
+
function parseArgs(argv) {
|
|
859
|
+
const out = { cmd: "run", root: process.cwd(), dryRun: false, phase: null, json: false, noWorktree: false };
|
|
860
|
+
for (let i = 0; i < argv.length; i++) {
|
|
861
|
+
const a = argv[i];
|
|
862
|
+
if (a === "run" || a === "report" || a === "status") out.cmd = a;
|
|
863
|
+
else if (a === "--dry-run") out.dryRun = true;
|
|
864
|
+
else if (a === "--json") out.json = true;
|
|
865
|
+
else if (a === "--no-worktree") out.noWorktree = true;
|
|
866
|
+
else if (a === "--phase") out.phase = argv[++i];
|
|
867
|
+
else if (a.startsWith("--phase=")) out.phase = a.slice("--phase=".length);
|
|
868
|
+
else if (a === "--root") out.root = argv[++i];
|
|
869
|
+
else if (a.startsWith("--root=")) out.root = a.slice("--root=".length);
|
|
870
|
+
else if (a === "--help" || a === "-h") out.help = true;
|
|
871
|
+
}
|
|
872
|
+
return out;
|
|
873
|
+
}
|
|
874
|
+
|
|
875
|
+
function main() {
|
|
876
|
+
const opts = parseArgs(process.argv.slice(2));
|
|
877
|
+
if (opts.help) {
|
|
878
|
+
console.log(
|
|
879
|
+
"Usage: node scripts/nightly.js [run|report|status] [--root <dir>] [--dry-run]\n" +
|
|
880
|
+
" [--phase <name>] [--no-worktree] [--json]"
|
|
881
|
+
);
|
|
882
|
+
return;
|
|
883
|
+
}
|
|
884
|
+
if (opts.cmd === "report") return cmdReport(opts);
|
|
885
|
+
if (opts.cmd === "status") return cmdStatus(opts);
|
|
886
|
+
return cmdRun(opts);
|
|
887
|
+
}
|
|
888
|
+
|
|
889
|
+
if (require.main === module) {
|
|
890
|
+
try {
|
|
891
|
+
main();
|
|
892
|
+
} catch (e) {
|
|
893
|
+
console.error("nightly: " + (e && e.message ? e.message : e));
|
|
894
|
+
process.exit(1);
|
|
895
|
+
}
|
|
896
|
+
}
|
|
897
|
+
|
|
898
|
+
module.exports = {
|
|
899
|
+
SCHEMA,
|
|
900
|
+
DEFAULT_CONFIG,
|
|
901
|
+
nightlyFlow,
|
|
902
|
+
isValidated,
|
|
903
|
+
normalizeSignature,
|
|
904
|
+
shortHash,
|
|
905
|
+
failureReason,
|
|
906
|
+
classifyFailure,
|
|
907
|
+
HARNESS_NOISE,
|
|
908
|
+
resolvePhases,
|
|
909
|
+
decideReconcile,
|
|
910
|
+
loadConfig,
|
|
911
|
+
buildFailureList,
|
|
912
|
+
parseArgs,
|
|
913
|
+
};
|