opencode-agent-harness 0.0.0-stage → 0.3.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +217 -0
- package/LICENSE +21 -0
- package/README.md +636 -2
- package/bin/agent-harness.js +3182 -0
- package/contrib/global-plugins/README.md +49 -0
- package/contrib/global-plugins/harness-freshness.ts +355 -0
- package/package.json +86 -4
- package/templates/.agents/AGENTS.md +148 -0
- package/templates/.agents/features/INDEX.md +10 -0
- package/templates/.agents/handoffs/README.md +87 -0
- package/templates/.agents/memory/dev-loop.md +59 -0
- package/templates/.agents/memory/domain-map.md +29 -0
- package/templates/.agents/memory/environments.md +35 -0
- package/templates/.agents/memory/flow-map.md +23 -0
- package/templates/.agents/memory/history.md +7 -0
- package/templates/.agents/memory/locations.md +14 -0
- package/templates/.agents/memory/model-routing.md +261 -0
- package/templates/.agents/memory/orchestration.md +183 -0
- package/templates/.agents/memory/owned-files.md +19 -0
- package/templates/.agents/memory/product-principles.md +37 -0
- package/templates/.agents/memory/quality-parameters.md +37 -0
- package/templates/.agents/memory/stack-versions.md +35 -0
- package/templates/.agents/memory/test-design-parameters.md +36 -0
- package/templates/.agents/nightly.json +17 -0
- package/templates/.agents/qa-plans/README.md +70 -0
- package/templates/.agents/qa-runs/README.md +21 -0
- package/templates/.agents/reviews/README.md +47 -0
- package/templates/.agents/rules/00-operating.md +199 -0
- package/templates/.agents/skills/data-engineer/SKILL.md +31 -0
- package/templates/.agents/skills/devops-engineer/SKILL.md +50 -0
- package/templates/.agents/skills/diagnostics-expert/SKILL.md +44 -0
- package/templates/.agents/skills/docs/SKILL.md +140 -0
- package/templates/.agents/skills/features/SKILL.md +187 -0
- package/templates/.agents/skills/frontend-engineer/SKILL.md +81 -0
- package/templates/.agents/skills/handoff/SKILL.md +85 -0
- package/templates/.agents/skills/mobile-engineer/SKILL.md +101 -0
- package/templates/.agents/skills/planner/SKILL.md +112 -0
- package/templates/.agents/skills/product-manager/SKILL.md +115 -0
- package/templates/.agents/skills/qa-architect/SKILL.md +224 -0
- package/templates/.agents/skills/qa-runner/SKILL.md +64 -0
- package/templates/.agents/skills/security-engineer/SKILL.md +53 -0
- package/templates/.agents/skills/system-architect/SKILL.md +42 -0
- package/templates/.agents/skills/ui-designer/SKILL.md +71 -0
- package/templates/.github/workflows/ci.yml +148 -0
- package/templates/.github/workflows/deploy-uat.yml +95 -0
- package/templates/.github/workflows/nightly.yml +83 -0
- package/templates/.github/workflows/uat-health.yml +41 -0
- package/templates/.husky/pre-push +23 -0
- package/templates/.ignore +28 -0
- package/templates/.opencode/agents/data-engineer.md +37 -0
- package/templates/.opencode/agents/devops-engineer.md +37 -0
- package/templates/.opencode/agents/diagnostics-expert.md +37 -0
- package/templates/.opencode/agents/docs.md +37 -0
- package/templates/.opencode/agents/features.md +21 -0
- package/templates/.opencode/agents/frontend-engineer.md +37 -0
- package/templates/.opencode/agents/handoff.md +21 -0
- package/templates/.opencode/agents/mobile-engineer.md +37 -0
- package/templates/.opencode/agents/planner.md +34 -0
- package/templates/.opencode/agents/product-manager.md +29 -0
- package/templates/.opencode/agents/qa-architect.md +27 -0
- package/templates/.opencode/agents/qa-runner.md +17 -0
- package/templates/.opencode/agents/security-engineer.md +37 -0
- package/templates/.opencode/agents/system-architect.md +37 -0
- package/templates/.opencode/agents/ui-designer.md +37 -0
- package/templates/.opencode/commands/board.md +15 -0
- package/templates/.opencode/commands/checkpoint.md +25 -0
- package/templates/.opencode/commands/feature.md +56 -0
- package/templates/.opencode/commands/finish.md +36 -0
- package/templates/.opencode/commands/next.md +49 -0
- package/templates/.opencode/commands/report.md +55 -0
- package/templates/.opencode/commands/resume.md +39 -0
- package/templates/.opencode/commands/switch-model.md +34 -0
- package/templates/.opencode/commands/tidy.md +49 -0
- package/templates/.opencode/plugins/backlog-notify.ts +57 -0
- package/templates/.opencode/plugins/backlog.ts +272 -0
- package/templates/.opencode/plugins/checkpoint.ts +160 -0
- package/templates/.opencode/plugins/context-guard.ts +1164 -0
- package/templates/.opencode/plugins/diff.ts +85 -0
- package/templates/.opencode/plugins/gates.ts +71 -0
- package/templates/.opencode/plugins/job-notify.ts +241 -0
- package/templates/.opencode/plugins/orient.ts +67 -0
- package/templates/.opencode/plugins/session.ts +89 -0
- package/templates/.opencode/plugins/wait.ts +101 -0
- package/templates/.opencode/plugins/where.ts +70 -0
- package/templates/.opencode/skills/bounded-explore/SKILL.md +38 -0
- package/templates/.opencode/skills/dev-loop/SKILL.md +87 -0
- package/templates/.vercelignore +16 -0
- package/templates/AGENTS.md +27 -0
- package/templates/BACKLOG.md +95 -0
- package/templates/docs/ci-cd-pipeline.md +81 -0
- package/templates/docs/gates.md +128 -0
- package/templates/docs/mcp-integrations.md +97 -0
- package/templates/docs/mobile-testing.md +555 -0
- package/templates/docs/nightly-testing.md +106 -0
- package/templates/docs/wsl-optimizations.md +290 -0
- package/templates/opencode.json +188 -0
- package/templates/scripts/agent-report.js +541 -0
- package/templates/scripts/agent-search.sh +117 -0
- package/templates/scripts/backlog.js +1924 -0
- package/templates/scripts/board-service.sh +214 -0
- package/templates/scripts/checkpoint.js +290 -0
- package/templates/scripts/finalize.js +416 -0
- package/templates/scripts/nightly.js +913 -0
- package/templates/scripts/orient.js +514 -0
- package/templates/scripts/qa/board-client.js +323 -0
- package/templates/scripts/qa/board-lib.js +96 -0
- package/templates/scripts/qa/board-sync.js +499 -0
- package/templates/scripts/qa/capacity-lib.js +916 -0
- package/templates/scripts/qa/check-backlog.js +434 -0
- package/templates/scripts/qa/check-capacity.js +85 -0
- package/templates/scripts/qa/check-ci-policy.js +164 -0
- package/templates/scripts/qa/check-concurrency.js +344 -0
- package/templates/scripts/qa/check-dispatch-config.js +296 -0
- package/templates/scripts/qa/check-docs.js +288 -0
- package/templates/scripts/qa/check-integrate.js +98 -0
- package/templates/scripts/qa/check-linger.js +160 -0
- package/templates/scripts/qa/check-prefix-stability.js +187 -0
- package/templates/scripts/qa/check-product.js +177 -0
- package/templates/scripts/qa/check-qa-plan.js +378 -0
- package/templates/scripts/qa/check-qa-scripts.js +85 -0
- package/templates/scripts/qa/check-quality.js +249 -0
- package/templates/scripts/qa/check-test-wiring.js +253 -0
- package/templates/scripts/qa/check-worktrees.js +93 -0
- package/templates/scripts/qa/ci-triage.js +346 -0
- package/templates/scripts/qa/ci-usage-lib.js +185 -0
- package/templates/scripts/qa/ci-wait-lib.js +294 -0
- package/templates/scripts/qa/command-classifier.js +552 -0
- package/templates/scripts/qa/context-guard-core.js +1618 -0
- package/templates/scripts/qa/diff.js +174 -0
- package/templates/scripts/qa/feature-scaffold.js +224 -0
- package/templates/scripts/qa/fixtures/backlog.md +104 -0
- package/templates/scripts/qa/governance.js +174 -0
- package/templates/scripts/qa/hash-file.js +32 -0
- package/templates/scripts/qa/lease-lib.js +263 -0
- package/templates/scripts/qa/merge-backlog.js +324 -0
- package/templates/scripts/qa/mobile-e2e/analyze-timings.js +300 -0
- package/templates/scripts/qa/mobile-e2e/apk-cache.js +420 -0
- package/templates/scripts/qa/mobile-e2e/backend-build-cache.js +203 -0
- package/templates/scripts/qa/mobile-e2e/boot-emulator.sh +432 -0
- package/templates/scripts/qa/mobile-e2e/build-apk.sh +210 -0
- package/templates/scripts/qa/mobile-e2e/check-flows.js +568 -0
- package/templates/scripts/qa/mobile-e2e/config.yaml +18 -0
- package/templates/scripts/qa/mobile-e2e/device-down.sh +88 -0
- package/templates/scripts/qa/mobile-e2e/device-up.sh +258 -0
- package/templates/scripts/qa/mobile-e2e/ios-simulator-down.sh +72 -0
- package/templates/scripts/qa/mobile-e2e/ios-simulator-up.sh +153 -0
- package/templates/scripts/qa/mobile-e2e/job-events.js +181 -0
- package/templates/scripts/qa/mobile-e2e/prewarm.sh +56 -0
- package/templates/scripts/qa/mobile-e2e/run-detached.sh +222 -0
- package/templates/scripts/qa/mobile-e2e/run-maestro.js +1543 -0
- package/templates/scripts/qa/mobile-e2e/ui-dump.js +327 -0
- package/templates/scripts/qa/models.allowlist.txt +32 -0
- package/templates/scripts/qa/provider-stats.js +878 -0
- package/templates/scripts/qa/run-clean.js +83 -0
- package/templates/scripts/qa/run-gates.js +213 -0
- package/templates/scripts/qa/run-loop.js +220 -0
- package/templates/scripts/qa/session-recon.js +512 -0
- package/templates/scripts/qa/store-lib.js +324 -0
- package/templates/scripts/qa/tool-lib.js +134 -0
- package/templates/scripts/qa/where.js +214 -0
- package/templates/scripts/qa/worktree-hygiene.js +217 -0
- package/templates/scripts/qa/wslconfig-merge.js +157 -0
- package/templates/scripts/serve.js +1328 -0
- package/templates/scripts/setup-nightly.sh +185 -0
- package/templates/scripts/setup-tmpfs.sh +374 -0
- package/templates/scripts/ship.js +586 -0
- package/templates/scripts/switch-reasoning-model.ps1 +101 -0
- package/templates/scripts/switch-reasoning-model.sh +104 -0
- package/templates/scripts/wait-for.js +342 -0
- package/templates/scripts/wsl-gpu.sh +180 -0
- package/templates/scripts/wsl-tune.sh +319 -0
|
@@ -0,0 +1,346 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
"use strict";
|
|
3
|
+
/*
|
|
4
|
+
* ci-triage — name a red GitHub Actions run for what it is, and price a
|
|
5
|
+
* dispatch before it runs.
|
|
6
|
+
*
|
|
7
|
+
* The 2026-10 incident: the account had no payment method, exhausted its
|
|
8
|
+
* included Actions minutes, and GitHub answered every dispatch with a job that
|
|
9
|
+
* provisioned NO runner (`runner_name` empty, `steps: 0`, ~4-6s) and a generic
|
|
10
|
+
* `failure`. Every automated reader called that "tests failed" — so a nine-day
|
|
11
|
+
* account-wide outage looked like a product regression and blocked landing.
|
|
12
|
+
*
|
|
13
|
+
* `ci-usage-lib.js` owns the cost model; `ci-wait-lib.js` owns the in-watch
|
|
14
|
+
* detection. This is the on-demand reader an operator/agent calls to ask
|
|
15
|
+
* "why is CI red?" and get one of three named verdicts:
|
|
16
|
+
*
|
|
17
|
+
* green — the run succeeded (or was skipped/neutral)
|
|
18
|
+
* infra-block — no runner was ever provisioned: NOT a test failure
|
|
19
|
+
* failed — a runner ran and the code/tests failed
|
|
20
|
+
*
|
|
21
|
+
* It also has a `--preflight` mode: before you dispatch an expensive workflow,
|
|
22
|
+
* print the runner rates it uses, what recent runs actually cost, and the
|
|
23
|
+
* budget impact — so the burn is visible before the dispatch, not after.
|
|
24
|
+
*
|
|
25
|
+
* Usage:
|
|
26
|
+
* node scripts/qa/ci-triage.js [--limit 10] [--workflow <name>] [--repo <dir>]
|
|
27
|
+
* node scripts/qa/ci-triage.js --run <databaseId>
|
|
28
|
+
* node scripts/qa/ci-triage.js --preflight .github/workflows/ci.yml
|
|
29
|
+
* node scripts/qa/ci-triage.js ... --json
|
|
30
|
+
*
|
|
31
|
+
* Exit: 0 = green/ok, 1 = a real failure, 2 = infra-block (or a budget overrun
|
|
32
|
+
* on preflight). A distinct code lets a caller/script branch without parsing the
|
|
33
|
+
* prose. Dependency-light: `gh` is the only spawn, and every path is
|
|
34
|
+
* overridable so the classification is unit-testable offline.
|
|
35
|
+
*/
|
|
36
|
+
|
|
37
|
+
const fs = require("fs");
|
|
38
|
+
const path = require("path");
|
|
39
|
+
|
|
40
|
+
const usage = require("./ci-usage-lib.js");
|
|
41
|
+
const { ghJson } = require("./ci-wait-lib.js");
|
|
42
|
+
|
|
43
|
+
/* ---------------------------------------------------------------- verdicts */
|
|
44
|
+
|
|
45
|
+
/* Classify ONE run (with its jobs when available) into a named verdict plus a
|
|
46
|
+
* short reason and the fix hint. Never says "tests failed" for a blocked run. */
|
|
47
|
+
function triageRun(run, jobs, opts = {}) {
|
|
48
|
+
if (!run) return { verdict: "unknown", reason: "no run", fix: null };
|
|
49
|
+
const id = run.databaseId || run.id || null;
|
|
50
|
+
const name = run.name || run.workflowName || null;
|
|
51
|
+
const sha = run.headSha || run.head_sha || null;
|
|
52
|
+
const url = run.url || null;
|
|
53
|
+
const base = { id, name, sha, url };
|
|
54
|
+
const verdict = usage.classifyRun(run, jobs);
|
|
55
|
+
if (verdict === "green") {
|
|
56
|
+
return { ...base, verdict: "green", reason: `concluded ${run.conclusion || "success"}`, fix: null };
|
|
57
|
+
}
|
|
58
|
+
if (verdict === "incomplete") {
|
|
59
|
+
return { ...base, verdict: "incomplete", reason: `still ${run.status || "running"}`, fix: null };
|
|
60
|
+
}
|
|
61
|
+
if (verdict === "infra-block") {
|
|
62
|
+
return {
|
|
63
|
+
...base,
|
|
64
|
+
verdict: "infra-block",
|
|
65
|
+
reason: "no runner was provisioned (0 steps, no runner_name) — NOT a test failure",
|
|
66
|
+
fix: "add/repair the account payment method or raise the spending limit, then re-dispatch",
|
|
67
|
+
hint: "gh auth refresh -h github.com -s user # makes the billing state readable",
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
const failedJobs = (Array.isArray(jobs) ? jobs : [])
|
|
71
|
+
.filter((j) => j && !["success", "skipped", "neutral"].includes(String(j.conclusion || "")))
|
|
72
|
+
.map((j) => j.name)
|
|
73
|
+
.filter(Boolean);
|
|
74
|
+
const allowance = Number(opts.marginMinutes) || 0;
|
|
75
|
+
return {
|
|
76
|
+
...base,
|
|
77
|
+
verdict: "failed",
|
|
78
|
+
reason: failedJobs.length ? `ran and failed: ${failedJobs.join(", ")}` : "ran and failed",
|
|
79
|
+
failedJobs,
|
|
80
|
+
fix: id ? `gh run view ${id} --log-failed` : null,
|
|
81
|
+
allowance,
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/* Triage a list of runs. `jobsByRun` maps run id -> jobs[]. */
|
|
86
|
+
function triageRuns(runs, jobsByRun) {
|
|
87
|
+
const jobs = jobsByRun || {};
|
|
88
|
+
return (runs || []).map((run) => {
|
|
89
|
+
const id = run && (run.databaseId || run.id);
|
|
90
|
+
return triageRun(run, id && jobs[id] ? jobs[id] : run && run.jobs ? run.jobs : null);
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/* The worst verdict in a triage, in the order a caller should care:
|
|
95
|
+
* infra-block (a blind account) > failed (real work) > incomplete > green. */
|
|
96
|
+
function worstVerdict(rows) {
|
|
97
|
+
const rank = { "infra-block": 3, failed: 2, incomplete: 1, unknown: 1, green: 0 };
|
|
98
|
+
let worst = "green";
|
|
99
|
+
for (const r of rows || []) {
|
|
100
|
+
if ((rank[r.verdict] ?? 0) > (rank[worst] ?? 0)) worst = r.verdict;
|
|
101
|
+
}
|
|
102
|
+
return worst;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/* Exit code for a verdict: green/ok -> 0, failed -> 1, infra-block -> 2. */
|
|
106
|
+
function exitCodeFor(verdict) {
|
|
107
|
+
if (verdict === "failed") return 1;
|
|
108
|
+
if (verdict === "infra-block") return 2;
|
|
109
|
+
return 0;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/* --------------------------------------------------------------- preflight */
|
|
113
|
+
|
|
114
|
+
/* Parse the runner labels a workflow file asks for. Handles scalar
|
|
115
|
+
* `runs-on: ubuntu-latest`, block scalars, and matrix values
|
|
116
|
+
* (`runs-on: ${{ matrix.os }}` + `os: [ubuntu-latest, macos-14]`). Best-effort
|
|
117
|
+
* text scan — a workflow is not a full YAML parse here, only a cost signal. */
|
|
118
|
+
function workflowRunnerLabels(src) {
|
|
119
|
+
const labels = new Set();
|
|
120
|
+
const text = String(src || "");
|
|
121
|
+
for (const m of text.matchAll(/runs-on:\s*(.+)$/gm)) {
|
|
122
|
+
const raw = m[1].trim();
|
|
123
|
+
if (raw.startsWith("${{")) continue; // resolved from a matrix below
|
|
124
|
+
for (const part of raw.replace(/[\[\]"]/g, "").split(",")) {
|
|
125
|
+
const v = part.trim();
|
|
126
|
+
if (v) labels.add(v);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
// matrix os/runner lists: `os: [ubuntu-latest, macos-14]`
|
|
130
|
+
for (const m of text.matchAll(/^\s{2,}(?:os|runner|runs-on):\s*\[([^\]]+)\]/gm)) {
|
|
131
|
+
for (const part of m[1].split(",")) {
|
|
132
|
+
const v = part.trim().replace(/["']/g, "");
|
|
133
|
+
if (v) labels.add(v);
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
return [...labels];
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/* The schedule intervals a workflow declares (a `cron:` of every 30 minutes
|
|
140
|
+
* yields 30). Feeds `jobsPerMonth` so a sub-minute probe on a tight cron is
|
|
141
|
+
* visible as a monthly burn instead of "a free minute". */
|
|
142
|
+
function scheduleMinutes(src) {
|
|
143
|
+
const out = [];
|
|
144
|
+
for (const m of String(src || "").matchAll(/cron:\s*["']?([^"'\n]+)["']?/g)) {
|
|
145
|
+
const step = m[1].trim().split(/\s+/)[0];
|
|
146
|
+
const everyN = /^\*\/(\d+)$/.exec(step);
|
|
147
|
+
if (everyN) out.push(Number(everyN[1]));
|
|
148
|
+
else if (step === "*") out.push(1);
|
|
149
|
+
else if (/^\d+$/.test(step)) out.push(Number(step));
|
|
150
|
+
}
|
|
151
|
+
return out;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/* Price a dispatch of one workflow: the rates it can hit, what its recent runs
|
|
155
|
+
* cost (median billable minutes), and the budget impact of one more run. */
|
|
156
|
+
function preflightWorkflow(src, recentRuns, opts = {}) {
|
|
157
|
+
const labels = workflowRunnerLabels(src);
|
|
158
|
+
const rates = labels.map((l) => ({ label: l, ...usage.rateForLabel(l) }));
|
|
159
|
+
const larger = rates.filter((r) => r.isLarger).map((r) => r.label);
|
|
160
|
+
const history = usage.estimateUsage(recentRuns || []);
|
|
161
|
+
const perRunMinutes = (recentRuns || []).length
|
|
162
|
+
? Math.round((history.minutes / recentRuns.length) * 10) / 10
|
|
163
|
+
: null;
|
|
164
|
+
const dominant = rates.find((r) => !r.isLarger) || rates[0] || usage.rateForLabel("");
|
|
165
|
+
const estMinutes = perRunMinutes == null ? 1 : perRunMinutes;
|
|
166
|
+
const estUsd = Math.round(estMinutes * dominant.rate * 10000) / 10000;
|
|
167
|
+
const extraMinutes = scheduleMinutes(src).reduce((acc, n) => acc + usage.jobsPerMonth(n), 0);
|
|
168
|
+
const budget = usage.budgetState(
|
|
169
|
+
{ minutes: (Number(opts.usedMinutes) || 0) + estMinutes, usd: (Number(opts.usedUsd) || 0) + estUsd },
|
|
170
|
+
opts
|
|
171
|
+
);
|
|
172
|
+
return {
|
|
173
|
+
labels,
|
|
174
|
+
rates,
|
|
175
|
+
largerRunners: larger,
|
|
176
|
+
samples: (recentRuns || []).length,
|
|
177
|
+
perRunMinutes,
|
|
178
|
+
estMinutes,
|
|
179
|
+
estUsd,
|
|
180
|
+
scheduledJobsPerMonth: extraMinutes,
|
|
181
|
+
budget,
|
|
182
|
+
risky: larger.length > 0 || extraMinutes > Number(opts.scheduledJobsPerMonthMax || 200),
|
|
183
|
+
};
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/* --------------------------------------------------------------------- CLI */
|
|
187
|
+
|
|
188
|
+
function parseArgs(argv) {
|
|
189
|
+
const out = { limit: 10, json: false };
|
|
190
|
+
for (let i = 0; i < argv.length; i++) {
|
|
191
|
+
const a = argv[i];
|
|
192
|
+
if (a === "--run") out.run = argv[++i];
|
|
193
|
+
else if (a.startsWith("--run=")) out.run = a.slice("--run=".length);
|
|
194
|
+
else if (a === "--workflow") out.workflow = argv[++i];
|
|
195
|
+
else if (a.startsWith("--workflow=")) out.workflow = a.slice("--workflow=".length);
|
|
196
|
+
else if (a === "--sha") out.sha = argv[++i];
|
|
197
|
+
else if (a === "--limit") out.limit = Number(argv[++i]);
|
|
198
|
+
else if (a.startsWith("--limit=")) out.limit = Number(a.slice("--limit=".length));
|
|
199
|
+
else if (a === "--repo" || a === "--root") out.repo = argv[++i];
|
|
200
|
+
else if (a.startsWith("--repo=")) out.repo = a.slice("--repo=".length);
|
|
201
|
+
else if (a === "--preflight") out.preflight = argv[++i];
|
|
202
|
+
else if (a.startsWith("--preflight=")) out.preflight = a.slice("--preflight=".length);
|
|
203
|
+
else if (a === "--runs-json") out.runsJson = argv[++i];
|
|
204
|
+
else if (a === "--jobs-json") out.jobsJson = argv[++i];
|
|
205
|
+
else if (a === "--json") out.json = true;
|
|
206
|
+
else if (a === "--help" || a === "-h") out.help = true;
|
|
207
|
+
}
|
|
208
|
+
return out;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/* Recent runs for a repo (optionally a workflow), via `gh`. Returns
|
|
212
|
+
* { ok, runs, error }. */
|
|
213
|
+
function fetchRuns(cwd, limit, workflow) {
|
|
214
|
+
const args = [
|
|
215
|
+
"run",
|
|
216
|
+
"list",
|
|
217
|
+
"--limit",
|
|
218
|
+
String(limit || 10),
|
|
219
|
+
"--json",
|
|
220
|
+
"databaseId,name,status,conclusion,headSha,url,createdAt,updatedAt",
|
|
221
|
+
];
|
|
222
|
+
if (workflow) args.splice(1, 0, "--workflow", workflow);
|
|
223
|
+
const res = ghJson(cwd, args);
|
|
224
|
+
return res.ok ? { ok: true, runs: res.data || [], error: null } : { ok: false, runs: [], error: res.error };
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
/* Jobs for one run, via `gh api`. */
|
|
228
|
+
function fetchJobs(cwd, runId) {
|
|
229
|
+
const res = ghJson(cwd, ["api", `/repos/{owner}/{repo}/actions/runs/${runId}/jobs`]);
|
|
230
|
+
return res.ok ? { ok: true, jobs: (res.data && res.data.jobs) || [], error: null } : { ok: false, jobs: [], error: res.error };
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
function readJson(file) {
|
|
234
|
+
return JSON.parse(fs.readFileSync(file, "utf8"));
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
function printRows(rows) {
|
|
238
|
+
for (const r of rows) {
|
|
239
|
+
const tag = r.verdict === "green" ? "OK " : r.verdict === "infra-block" ? "BLOCK" : r.verdict === "failed" ? "FAIL" : "…";
|
|
240
|
+
console.log(` [${tag}] ${r.name || "?"}${r.sha ? " @ " + String(r.sha).slice(0, 7) : ""} — ${r.reason}`);
|
|
241
|
+
if (r.verdict !== "green" && r.fix) console.log(` fix: ${r.fix}`);
|
|
242
|
+
if (r.hint && r.verdict === "infra-block" && process.env.CI_TRIAGE_VERBOSE) console.log(` ${r.hint}`);
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
function main() {
|
|
247
|
+
const args = parseArgs(process.argv.slice(2));
|
|
248
|
+
if (args.help) {
|
|
249
|
+
console.log(
|
|
250
|
+
"Usage: node scripts/qa/ci-triage.js [--limit N] [--workflow <name>] [--repo <dir>] [--sha <sha>]\n" +
|
|
251
|
+
" node scripts/qa/ci-triage.js --run <databaseId>\n" +
|
|
252
|
+
" node scripts/qa/ci-triage.js --preflight .github/workflows/<file>.yml\n" +
|
|
253
|
+
" [--runs-json <file>] [--jobs-json <file>] [--json]"
|
|
254
|
+
);
|
|
255
|
+
return;
|
|
256
|
+
}
|
|
257
|
+
const root = path.resolve(args.repo || process.cwd());
|
|
258
|
+
|
|
259
|
+
if (args.preflight) {
|
|
260
|
+
const file = path.resolve(root, args.preflight);
|
|
261
|
+
const src = fs.readFileSync(file, "utf8");
|
|
262
|
+
let recent = [];
|
|
263
|
+
if (args.runsJson) recent = readJson(args.runsJson);
|
|
264
|
+
else {
|
|
265
|
+
const got = fetchRuns(root, args.limit, args.workflow);
|
|
266
|
+
if (got.ok) recent = got.runs;
|
|
267
|
+
}
|
|
268
|
+
const pf = preflightWorkflow(src, recent);
|
|
269
|
+
if (args.json) {
|
|
270
|
+
console.log(JSON.stringify({ schema: "ci-preflight/1", workflow: args.preflight, ...pf }, null, 2));
|
|
271
|
+
} else {
|
|
272
|
+
console.log(`ci preflight — ${args.preflight}`);
|
|
273
|
+
console.log(` runners: ${pf.labels.join(", ") || "(none detected)"}`);
|
|
274
|
+
for (const r of pf.rates) {
|
|
275
|
+
console.log(` ${r.label}: $${r.rate}/min${r.isLarger ? " (LARGER runner — not covered by included minutes)" : ""}`);
|
|
276
|
+
}
|
|
277
|
+
if (pf.largerRunners.length) {
|
|
278
|
+
console.log(` ! larger runners present (${pf.largerRunners.join(", ")}) — always billed, never free-tier`);
|
|
279
|
+
}
|
|
280
|
+
console.log(
|
|
281
|
+
` history: ${pf.samples} run(s)${pf.perRunMinutes != null ? `, ~${pf.perRunMinutes} min each` : " (no history — assuming 1 min)"}`
|
|
282
|
+
);
|
|
283
|
+
console.log(` one dispatch ≈ ${pf.estMinutes} min ≈ $${pf.estUsd}`);
|
|
284
|
+
if (pf.scheduledJobsPerMonth) console.log(` schedule burn ≈ ${pf.scheduledJobsPerMonth} jobs/mo from this file`);
|
|
285
|
+
console.log(
|
|
286
|
+
` budget: ${pf.budget.status} — ${pf.budget.minutes}/${pf.budget.allowanceMinutes} min (${pf.budget.pct}%)`
|
|
287
|
+
);
|
|
288
|
+
}
|
|
289
|
+
const bad = pf.budget.status === "fail" || pf.budget.status === "warn";
|
|
290
|
+
process.exit(bad ? 2 : 0);
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
let rows;
|
|
294
|
+
if (args.runsJson) {
|
|
295
|
+
const runs = readJson(args.runsJson);
|
|
296
|
+
const jobsById = args.jobsJson ? readJson(args.jobsJson) : {};
|
|
297
|
+
rows = triageRuns(runs, jobsById);
|
|
298
|
+
} else if (args.run) {
|
|
299
|
+
const got = fetchJobs(root, args.run);
|
|
300
|
+
const run = {
|
|
301
|
+
databaseId: args.run,
|
|
302
|
+
name: args.workflow || null,
|
|
303
|
+
conclusion: got.ok ? "failure" : null,
|
|
304
|
+
};
|
|
305
|
+
rows = [triageRun(run, got.ok ? got.jobs : null)];
|
|
306
|
+
} else {
|
|
307
|
+
const got = fetchRuns(root, args.limit, args.workflow);
|
|
308
|
+
if (!got.ok) {
|
|
309
|
+
console.error(`ci-triage: could not read runs (${got.error})`);
|
|
310
|
+
process.exit(2);
|
|
311
|
+
}
|
|
312
|
+
rows = triageRuns(
|
|
313
|
+
got.runs.map((r) => (args.sha ? r : r)),
|
|
314
|
+
{}
|
|
315
|
+
);
|
|
316
|
+
if (args.sha) rows = rows.filter((r) => String(r.sha || "").startsWith(args.sha));
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
if (args.json) {
|
|
320
|
+
console.log(JSON.stringify({ schema: "ci-triage/1", rows, worst: worstVerdict(rows) }, null, 2));
|
|
321
|
+
} else {
|
|
322
|
+
printRows(rows);
|
|
323
|
+
console.log(`\nverdict: ${worstVerdict(rows)}`);
|
|
324
|
+
}
|
|
325
|
+
process.exit(exitCodeFor(worstVerdict(rows)));
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
if (require.main === module) {
|
|
329
|
+
try {
|
|
330
|
+
main();
|
|
331
|
+
} catch (e) {
|
|
332
|
+
console.error("ci-triage: " + (e && e.message ? e.message : e));
|
|
333
|
+
process.exit(1);
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
module.exports = {
|
|
338
|
+
triageRun,
|
|
339
|
+
triageRuns,
|
|
340
|
+
worstVerdict,
|
|
341
|
+
exitCodeFor,
|
|
342
|
+
workflowRunnerLabels,
|
|
343
|
+
scheduleMinutes,
|
|
344
|
+
preflightWorkflow,
|
|
345
|
+
parseArgs,
|
|
346
|
+
};
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
'use strict';
|
|
3
|
+
/*
|
|
4
|
+
* ci-usage-lib — GitHub Actions cost model + billing-block classification.
|
|
5
|
+
*
|
|
6
|
+
* The 2026-10 incident: the account had no payment method, exhausted its
|
|
7
|
+
* included Actions quota, and GitHub answered every dispatch with a job that
|
|
8
|
+
* provisions NO runner (`runner_name` empty, `steps: 0`, ~4-6s) and a generic
|
|
9
|
+
* `failure`. Every automated reader treated that as a real test failure — so a
|
|
10
|
+
* nine-day, account-wide outage looked like a product regression, filed board
|
|
11
|
+
* rows, and blocked the landing path.
|
|
12
|
+
*
|
|
13
|
+
* This module makes both halves mechanical:
|
|
14
|
+
* 1. `classifyJob`/`classifyRun` turn the zero-step signature into a distinct
|
|
15
|
+
* `infra-block` state, so callers can name it instead of "tests failed".
|
|
16
|
+
* 2. `billableMinutes`/`estimateUsage` price a run list with the vendor rate
|
|
17
|
+
* card (per-job runtime rounded UP to the next whole minute), so the budget
|
|
18
|
+
* is visible without a `user`-scoped billing token.
|
|
19
|
+
*
|
|
20
|
+
* Dependency-free and pure given its inputs (jobs/runs as parsed JSON), so the
|
|
21
|
+
* model is unit-testable without a network or `gh`.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
/* Vendor per-minute rates (USD), GitHub-hosted standard runners.
|
|
25
|
+
* Source: docs.github.com/en/billing/concepts/product-billing/github-actions */
|
|
26
|
+
const RATES = {
|
|
27
|
+
linux_slim: 0.002,
|
|
28
|
+
linux: 0.006,
|
|
29
|
+
linux_arm: 0.005,
|
|
30
|
+
windows: 0.01,
|
|
31
|
+
windows_arm: 0.01,
|
|
32
|
+
macos: 0.062,
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
/* Included minutes per billing cycle for private repos on the Free plan. Public
|
|
36
|
+
* repos and self-hosted runners are free and are not modelled here. */
|
|
37
|
+
const FREE_MINUTES = { free: 2000, pro: 3000, team: 3000 };
|
|
38
|
+
|
|
39
|
+
/* A scheduled run every N minutes burns ~1440/N jobs/day. This is the single
|
|
40
|
+
* most common way a "free" account silently leaves the free tier: a sub-minute
|
|
41
|
+
* probe on a 30-minute cron is still a billed job EVERY time it runs. */
|
|
42
|
+
function jobsPerMonth(cronMinutes) {
|
|
43
|
+
if (!Number.isFinite(cronMinutes) || cronMinutes <= 0) return 0;
|
|
44
|
+
return Math.round(((60 / cronMinutes) * 24 * 30));
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/* Map a GitHub runner label (`runs-on`) to a rate key. Larger macOS/Windows
|
|
48
|
+
* runners are always charged and are NOT covered by included minutes — the
|
|
49
|
+
* caller must treat `isLarger` results as paywalled rather than free-tier. */
|
|
50
|
+
function rateForLabel(label) {
|
|
51
|
+
const l = String(label || '').toLowerCase();
|
|
52
|
+
if (!l) return { key: 'linux', rate: RATES.linux, isLarger: false };
|
|
53
|
+
if (l.includes('macos') || l.includes('macos-')) {
|
|
54
|
+
// macos-13/14/15/26 are standard; a `-large`/`-xlarge`/`-m2` suffix is larger.
|
|
55
|
+
const larger = /(-large|-xlarge|-xxlarge|\d+-core|m2|m4|pro)/.test(l) && !/^macos-\d+$/.test(l.split(' ')[0]);
|
|
56
|
+
return { key: 'macos', rate: RATES.macos, isLarger: larger };
|
|
57
|
+
}
|
|
58
|
+
if (l.includes('windows')) {
|
|
59
|
+
const larger = /(-large|-xlarge)/.test(l);
|
|
60
|
+
return { key: 'windows', rate: RATES.windows, isLarger: larger };
|
|
61
|
+
}
|
|
62
|
+
if (l.includes('arm')) return { key: 'linux_arm', rate: RATES.linux_arm, isLarger: false };
|
|
63
|
+
return { key: 'linux', rate: RATES.linux, isLarger: false };
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/* A job's billable minutes: wall-clock from start to completion, rounded UP to
|
|
67
|
+
* the next whole minute (a 6-second job is one billed minute). Returns 0 when a
|
|
68
|
+
* job never provisioned a runner, so a blocked job costs nothing. */
|
|
69
|
+
function billableMinutes(job) {
|
|
70
|
+
if (!job) return 0;
|
|
71
|
+
const steps = Array.isArray(job.steps) ? job.steps.length : 0;
|
|
72
|
+
const runner = job.runner_name == null ? '' : String(job.runner_name).trim();
|
|
73
|
+
const started = Date.parse(job.started_at || job.startedAt || '');
|
|
74
|
+
const done = Date.parse(job.completed_at || job.completedAt || '');
|
|
75
|
+
if (steps === 0 && !runner) return 0;
|
|
76
|
+
if (!Number.isFinite(started) || !Number.isFinite(done) || done < started) return 1;
|
|
77
|
+
const secs = (done - started) / 1000;
|
|
78
|
+
if (secs < 30 && steps === 0 && !runner) return 0;
|
|
79
|
+
return Math.max(1, Math.ceil(secs / 60));
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/* The billing-block signature on ONE job: it completed with zero steps and no
|
|
83
|
+
* runner was ever provisioned. Distinct from a job that ran and failed. */
|
|
84
|
+
function isBlockedJob(job) {
|
|
85
|
+
if (!job) return false;
|
|
86
|
+
const steps = Array.isArray(job.steps) ? job.steps.length : 0;
|
|
87
|
+
const runner = job.runner_name == null ? '' : String(job.runner_name).trim();
|
|
88
|
+
return steps === 0 && runner === '';
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/* Classify a run from its jobs (or, when jobs are unavailable, from the run's
|
|
92
|
+
* own shape). `infra-block` means no runner was provisioned for ANY job while
|
|
93
|
+
* the run is not green — the caller must NOT report this as a test failure. */
|
|
94
|
+
function classifyRun(run, jobs) {
|
|
95
|
+
if (!run) return 'unknown';
|
|
96
|
+
const conclusion = String(run.conclusion || '');
|
|
97
|
+
const green = ['success', 'skipped', 'neutral'].includes(conclusion);
|
|
98
|
+
if (!conclusion) return 'incomplete';
|
|
99
|
+
if (green) return 'green';
|
|
100
|
+
if (Array.isArray(jobs) && jobs.length > 0) {
|
|
101
|
+
if (jobs.every(isBlockedJob)) return 'infra-block';
|
|
102
|
+
return 'failed';
|
|
103
|
+
}
|
|
104
|
+
// No job detail: only the cheap run-list shape. A red run that completed in
|
|
105
|
+
// under ~30s never provisioned a runner (a real gate cannot run that fast).
|
|
106
|
+
const created = Date.parse(run.createdAt || run.created_at || '');
|
|
107
|
+
const updated = Date.parse(run.updatedAt || run.updated_at || '');
|
|
108
|
+
if (Number.isFinite(created) && Number.isFinite(updated) && updated - created < 30000) return 'infra-block';
|
|
109
|
+
return 'failed';
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/* Price a set of runs. `runs` is `[{ jobs: [...] }]` for accuracy, or run-list
|
|
113
|
+
* rows (priced as one Linux minute each — an explicit, conservative floor).
|
|
114
|
+
* Returns { minutes, usd, byRate, blockedJobs, billedJobs }. */
|
|
115
|
+
function estimateUsage(runs) {
|
|
116
|
+
const byRate = {};
|
|
117
|
+
let minutes = 0;
|
|
118
|
+
let usd = 0;
|
|
119
|
+
let blockedJobs = 0;
|
|
120
|
+
let billedJobs = 0;
|
|
121
|
+
for (const run of runs || []) {
|
|
122
|
+
const jobs = Array.isArray(run && run.jobs) ? run.jobs : null;
|
|
123
|
+
if (jobs) {
|
|
124
|
+
for (const job of jobs) {
|
|
125
|
+
if (isBlockedJob(job)) {
|
|
126
|
+
blockedJobs += 1;
|
|
127
|
+
continue;
|
|
128
|
+
}
|
|
129
|
+
const labels = Array.isArray(job.labels) && job.labels.length ? job.labels : ['ubuntu-latest'];
|
|
130
|
+
const { key, rate } = rateForLabel(labels[0]);
|
|
131
|
+
const m = billableMinutes(job);
|
|
132
|
+
if (m <= 0) continue;
|
|
133
|
+
billedJobs += 1;
|
|
134
|
+
minutes += m;
|
|
135
|
+
usd += m * rate;
|
|
136
|
+
byRate[key] = byRate[key] || { minutes: 0, usd: 0 };
|
|
137
|
+
byRate[key].minutes += m;
|
|
138
|
+
byRate[key].usd += m * rate;
|
|
139
|
+
}
|
|
140
|
+
} else if (run && run.conclusion) {
|
|
141
|
+
// Coarse fallback: one Linux minute per completed run.
|
|
142
|
+
minutes += 1;
|
|
143
|
+
usd += RATES.linux;
|
|
144
|
+
billedJobs += 1;
|
|
145
|
+
byRate.linux = byRate.linux || { minutes: 0, usd: 0 };
|
|
146
|
+
byRate.linux.minutes += 1;
|
|
147
|
+
byRate.linux.usd += RATES.linux;
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
return { minutes, usd, byRate, blockedJobs, billedJobs };
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/* Compare estimated usage against an allowance. `allowanceMinutes` defaults to
|
|
154
|
+
* the Free plan. `spendingLimitUsd` is the budget the operator set; the honest
|
|
155
|
+
* state is `unknown` when usage could not be read — never `ok`. */
|
|
156
|
+
function budgetState(usage, opts = {}) {
|
|
157
|
+
const allowanceMinutes = Number(opts.allowanceMinutes) || FREE_MINUTES.free;
|
|
158
|
+
const warnAtPct = Number(opts.warnAtPct) || 80;
|
|
159
|
+
if (!usage || !Number.isFinite(Number(usage.minutes))) return { status: 'unknown', reason: 'usage unreadable' };
|
|
160
|
+
const minutes = Number(usage.minutes);
|
|
161
|
+
const pct = allowanceMinutes > 0 ? (minutes / allowanceMinutes) * 100 : 0;
|
|
162
|
+
let status = 'ok';
|
|
163
|
+
if (pct >= 100) status = 'fail';
|
|
164
|
+
else if (pct >= warnAtPct) status = 'warn';
|
|
165
|
+
return {
|
|
166
|
+
status,
|
|
167
|
+
minutes,
|
|
168
|
+
allowanceMinutes,
|
|
169
|
+
pct: Math.round(pct),
|
|
170
|
+
usd: Number(usage.usd || 0),
|
|
171
|
+
remainingMinutes: Math.max(0, allowanceMinutes - minutes),
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
module.exports = {
|
|
176
|
+
RATES,
|
|
177
|
+
FREE_MINUTES,
|
|
178
|
+
jobsPerMonth,
|
|
179
|
+
rateForLabel,
|
|
180
|
+
billableMinutes,
|
|
181
|
+
isBlockedJob,
|
|
182
|
+
classifyRun,
|
|
183
|
+
estimateUsage,
|
|
184
|
+
budgetState,
|
|
185
|
+
};
|