@mmerterden/multi-agent-pipeline 20.27.1 → 20.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +38 -0
- package/README.md +6 -5
- package/README.tr.md +6 -5
- package/docs/adr/0008-installer-modularization-and-secret-leak-defense.md +0 -1
- package/docs/architecture.md +4 -4
- package/docs/ecosystem.md +5 -5
- package/docs/facts.json +14 -12
- package/docs/features.md +2 -0
- package/install/_dev-only-files.mjs +1 -0
- package/install/catalog-history.json +1 -1
- package/install/index.mjs +0 -2
- package/manifest.json +74 -51
- package/package.json +4 -1
- package/pipeline/agents/code-reviewer.md +1 -0
- package/pipeline/agents/test-gen-analyst.md +98 -0
- package/pipeline/commands/multi-agent/sync/SKILL.md +3 -3
- package/pipeline/commands/multi-agent/test-gen/SKILL.md +124 -0
- package/pipeline/multi-agent-refs/cross-cli-contract.md +11 -5
- package/pipeline/multi-agent-refs/features/doctor.md +13 -0
- package/pipeline/multi-agent-refs/features/test-classification.md +76 -0
- package/pipeline/multi-agent-refs/features/usage-reporting.md +17 -0
- package/pipeline/multi-agent-refs/help/en.md +1 -0
- package/pipeline/multi-agent-refs/help/tr.md +1 -0
- package/pipeline/multi-agent-refs/phases/operations.md +2 -2
- package/pipeline/multi-agent-refs/phases/phase-2-dev.md +6 -7
- package/pipeline/multi-agent-refs/phases/phase-3-review.md +3 -3
- package/pipeline/multi-agent-refs/phases/phase-5-report.md +6 -5
- package/pipeline/multi-agent-refs/store-ready/gate-3-policy.md +6 -2
- package/pipeline/multi-agent-refs/test-gen/stages.md +227 -0
- package/pipeline/preferences-template.json +3 -0
- package/pipeline/rules/app-store-guidelines.md +29 -41
- package/pipeline/schemas/prefs.schema.json +14 -0
- package/pipeline/schemas/test-gen-discovery.schema.json +73 -0
- package/pipeline/schemas/test-gen-gaps.schema.json +122 -0
- package/pipeline/schemas/test-gen-inventory.schema.json +59 -0
- package/pipeline/schemas/test-gen-results.schema.json +44 -0
- package/pipeline/scripts/_handle-patterns.mjs +166 -0
- package/pipeline/scripts/_http-post.mjs +141 -0
- package/pipeline/scripts/_stack-routing.mjs +4 -0
- package/pipeline/scripts/_test-gaming.mjs +227 -0
- package/pipeline/scripts/_unified-diff.mjs +104 -0
- package/pipeline/scripts/_worktree-snapshot.mjs +113 -0
- package/pipeline/scripts/doctor.mjs +51 -0
- package/pipeline/scripts/evidence-gate.mjs +74 -3
- package/pipeline/scripts/handle-diff-audit.mjs +188 -0
- package/pipeline/scripts/routine-registry.mjs +5 -1
- package/pipeline/scripts/run-ui-tests.sh +27 -3
- package/pipeline/scripts/test-classify.mjs +296 -0
- package/pipeline/scripts/test-gen-gate.mjs +409 -0
- package/pipeline/scripts/test-gen-state.mjs +313 -0
- package/pipeline/scripts/test-integrity-gate.mjs +76 -11
- package/pipeline/scripts/uninstall.mjs +1 -0
- package/pipeline/scripts/usage-register-beacon.mjs +239 -0
- package/pipeline/scripts/usage-register.mjs +128 -33
- package/pipeline/scripts/usage-report.mjs +43 -33
- package/pipeline/skills/.skill-manifest.json +25 -5
- package/pipeline/skills/.skills-index.json +57 -2
- package/pipeline/skills/shared/README.md +11 -6
- package/pipeline/skills/shared/core/apple-archive-compliance/SKILL.md +5 -4
- package/pipeline/skills/shared/core/apple-archive-compliance/references/rules.yml +8 -8
- package/pipeline/skills/shared/core/multi-agent-sync/SKILL.md +3 -3
- package/pipeline/skills/shared/core/multi-agent-test-gen/SKILL.md +120 -0
- package/pipeline/skills/shared/external/app-store-review/SKILL.md +69 -8
- package/pipeline/skills/shared/external/app-store-review/evals/evals.json +12 -0
- package/pipeline/skills/shared/external/app-store-review/references/review-checklists.md +10 -1
- package/pipeline/skills/shared/external/fix-review-verdicts/SKILL.md +133 -0
- package/pipeline/skills/shared/external/handle-request/SKILL.md +97 -0
- package/pipeline/skills/shared/external/scenario-gap-analysis/SKILL.md +129 -0
- package/pipeline/skills/shared/external/scenario-gap-analysis/references/scenario-schema.md +60 -0
- package/pipeline/skills/shared/external/test-result-triage/SKILL.md +160 -0
- package/pipeline/skills/skills-index.md +7 -2
- package/install/_telemetry.mjs +0 -61
|
@@ -0,0 +1,313 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* @file test-gen-state.mjs - project-level state of /multi-agent:test-gen.
|
|
4
|
+
*
|
|
5
|
+
* One JSON file per project (the command keeps it at
|
|
6
|
+
* ~/.claude/knowledge/<project>/test-gen-state.json) holding three things:
|
|
7
|
+
*
|
|
8
|
+
* base the commit the last test-gen run finished at, with the run id
|
|
9
|
+
* and a reason. The reason names the run ("test-gen run X
|
|
10
|
+
* finished"): it says which tests were written against which
|
|
11
|
+
* code, never that the app is understood.
|
|
12
|
+
* scans[] one row per impact scan of base..HEAD: the changed files, and
|
|
13
|
+
* what graph-affected.mjs found behind them. A scan that could
|
|
14
|
+
* not be made still writes a row, `measured: false` with the
|
|
15
|
+
* reason, so "no impact" and "not measured" never look alike.
|
|
16
|
+
* revisions[] the stale-test revisions applied, each with its run and time.
|
|
17
|
+
* Autopilot counts them per run AND per calendar day (UTC)
|
|
18
|
+
* against prefs.global.testGen.revisionCap (default 20): a cap
|
|
19
|
+
* counted per run alone is passed by starting many small runs.
|
|
20
|
+
*
|
|
21
|
+
* Subcommands:
|
|
22
|
+
* base-record --state <f> --repo <path> --run <id> [--reason <text>] [--now <iso>]
|
|
23
|
+
* base-scan --state <f> --repo <path> [--graph <code-graph.json>] [--now <iso>]
|
|
24
|
+
* revise --state <f> --run <id> --test <id> --verdict <v> [--autopilot]
|
|
25
|
+
* [--cap <n>] [--prefs <path>] [--now <iso>]
|
|
26
|
+
*
|
|
27
|
+
* revise answers with an action:
|
|
28
|
+
* apply a stale-test revision under the cap; it is recorded
|
|
29
|
+
* park autopilot stops the run: the cap is reached, or the verdict is
|
|
30
|
+
* product-bug, which is never applied without a person
|
|
31
|
+
* ask attended product-bug: the person decides
|
|
32
|
+
* none new-scenario, valid, ambiguous: nothing to revise here
|
|
33
|
+
*
|
|
34
|
+
* Exit: 0 done (base-scan: measured; revise: apply/ask/none), 1 base-scan not
|
|
35
|
+
* measured, 2 usage, 3 revise parks the run, 4 the state (or the prefs) cannot
|
|
36
|
+
* be read on an attended run, and nothing was recorded, 5 internal error (a
|
|
37
|
+
* crash, never a verdict)
|
|
38
|
+
*
|
|
39
|
+
* The state file is read and replaced under a lock (lib/json-file-lock.mjs),
|
|
40
|
+
* so two revisions at once cannot both count from the same total. A state or
|
|
41
|
+
* prefs file that exists but does not parse is never read as empty: autopilot
|
|
42
|
+
* parks, an attended run stops with exit 4.
|
|
43
|
+
*
|
|
44
|
+
* @module pipeline/scripts/test-gen-state
|
|
45
|
+
*/
|
|
46
|
+
|
|
47
|
+
import { spawnSync } from "node:child_process";
|
|
48
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
49
|
+
import { homedir } from "node:os";
|
|
50
|
+
import { dirname, join } from "node:path";
|
|
51
|
+
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
52
|
+
|
|
53
|
+
import { withFileLockSync, writeJsonAtomicSync } from "../lib/json-file-lock.mjs";
|
|
54
|
+
|
|
55
|
+
export const VERDICTS = ["product-bug", "stale-test", "new-scenario", "valid", "ambiguous"];
|
|
56
|
+
export const DEFAULT_REVISION_CAP = 20;
|
|
57
|
+
const MAX_REVISION_CAP = 500;
|
|
58
|
+
const MAX_SCANS = 50;
|
|
59
|
+
const SHA = /^[0-9a-f]{7,64}$/;
|
|
60
|
+
const GRAPH_AFFECTED = join(dirname(fileURLToPath(import.meta.url)), "graph-affected.mjs");
|
|
61
|
+
|
|
62
|
+
class Unreadable extends Error {}
|
|
63
|
+
|
|
64
|
+
function usage(msg) {
|
|
65
|
+
process.stderr.write(`test-gen-state: ${msg}\n`);
|
|
66
|
+
process.exit(2);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function parseArgs(argv) {
|
|
70
|
+
const out = {};
|
|
71
|
+
for (let i = 0; i < argv.length; i++) {
|
|
72
|
+
const a = argv[i];
|
|
73
|
+
if (!a.startsWith("--")) usage(`unexpected argument ${a}`);
|
|
74
|
+
const next = argv[i + 1];
|
|
75
|
+
if (next === undefined || next.startsWith("-")) out[a.slice(2)] = true;
|
|
76
|
+
else {
|
|
77
|
+
out[a.slice(2)] = next;
|
|
78
|
+
i++;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
return out;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function need(args, key) {
|
|
85
|
+
if (typeof args[key] !== "string" || !args[key].trim()) usage(`--${key} is required`);
|
|
86
|
+
return args[key];
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** A JSON document; {} when the file is absent, Unreadable when it exists and does not parse. */
|
|
90
|
+
export function loadState(path) {
|
|
91
|
+
if (!existsSync(path)) return {};
|
|
92
|
+
try {
|
|
93
|
+
const doc = JSON.parse(readFileSync(path, "utf8"));
|
|
94
|
+
if (doc && typeof doc === "object" && !Array.isArray(doc)) return doc;
|
|
95
|
+
throw new Error("not a JSON object");
|
|
96
|
+
} catch (err) {
|
|
97
|
+
throw new Unreadable(`${path} is unreadable: ${err.message}`);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function saveState(path, doc) {
|
|
102
|
+
writeJsonAtomicSync(path, { ...doc, schemaVersion: 1 });
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** Run `fn` holding the state file's lock; the directory is created first. */
|
|
106
|
+
function locked(path, fn) {
|
|
107
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
108
|
+
return withFileLockSync(path, fn);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
function git(repo, args) {
|
|
112
|
+
const r = spawnSync("git", ["-C", repo, ...args], {
|
|
113
|
+
encoding: "utf8",
|
|
114
|
+
maxBuffer: 64 * 1024 * 1024,
|
|
115
|
+
timeout: 60000,
|
|
116
|
+
});
|
|
117
|
+
return { ok: r.status === 0, out: (r.stdout || "").trim(), err: (r.stderr || "").trim() };
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
function baseRecord(args) {
|
|
121
|
+
const statePath = need(args, "state");
|
|
122
|
+
const repo = need(args, "repo");
|
|
123
|
+
const runId = need(args, "run");
|
|
124
|
+
const reason = typeof args.reason === "string" ? args.reason : `test-gen run ${runId} finished`;
|
|
125
|
+
if (!reason.includes(runId)) {
|
|
126
|
+
usage(`--reason must name the run (${runId}): a base records which run wrote the tests`);
|
|
127
|
+
}
|
|
128
|
+
const head = git(repo, ["rev-parse", "--verify", "--quiet", "HEAD"]);
|
|
129
|
+
if (!head.ok) usage(`cannot read HEAD in ${repo}: ${head.err}`);
|
|
130
|
+
return locked(statePath, () => {
|
|
131
|
+
const doc = loadState(statePath);
|
|
132
|
+
doc.base = { sha: head.out, runId, reason, recordedAt: args.now || new Date().toISOString() };
|
|
133
|
+
saveState(statePath, doc);
|
|
134
|
+
return [doc.base, 0];
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
function graphImpact(repo, range, graphPath) {
|
|
139
|
+
const argv = [GRAPH_AFFECTED, "--diff", range, "--root", repo, "--json"];
|
|
140
|
+
if (graphPath) argv.push("--graph", graphPath);
|
|
141
|
+
const r = spawnSync(process.execPath, argv, { encoding: "utf8", timeout: 120000 });
|
|
142
|
+
if (r.status !== 0) {
|
|
143
|
+
const why = (r.stderr || "").trim().split("\n")[0] || `exit ${r.status}`;
|
|
144
|
+
return { measured: false, reason: `graph-affected: ${why}` };
|
|
145
|
+
}
|
|
146
|
+
try {
|
|
147
|
+
const j = JSON.parse(r.stdout);
|
|
148
|
+
const files = [...new Set((j.affected || []).map((a) => a.path).filter(Boolean))];
|
|
149
|
+
return {
|
|
150
|
+
measured: true,
|
|
151
|
+
changedSymbols: (j.changedSymbols || []).length,
|
|
152
|
+
affectedFiles: files,
|
|
153
|
+
notIndexed: j.notIndexed || [],
|
|
154
|
+
stale: Boolean(j.freshness?.stale),
|
|
155
|
+
};
|
|
156
|
+
} catch (err) {
|
|
157
|
+
return { measured: false, reason: `graph-affected output unreadable: ${err.message}` };
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function scanRow(repo, base, at, graphPath) {
|
|
162
|
+
if (!base?.sha) {
|
|
163
|
+
return { at, measured: false, reason: "no recorded base: no test-gen run has finished here" };
|
|
164
|
+
}
|
|
165
|
+
if (!SHA.test(String(base.sha))) {
|
|
166
|
+
return { at, measured: false, reason: `recorded base is not a commit sha: ${base.sha}` };
|
|
167
|
+
}
|
|
168
|
+
const head = git(repo, ["rev-parse", "--verify", "--quiet", "HEAD"]);
|
|
169
|
+
if (!head.ok)
|
|
170
|
+
return { at, base: base.sha, measured: false, reason: `cannot read HEAD: ${head.err}` };
|
|
171
|
+
const range = `${base.sha}..${head.out}`;
|
|
172
|
+
const diff = git(repo, ["diff", "--name-only", "--no-renames", "--end-of-options", range]);
|
|
173
|
+
if (!diff.ok) {
|
|
174
|
+
return {
|
|
175
|
+
at,
|
|
176
|
+
base: base.sha,
|
|
177
|
+
head: head.out,
|
|
178
|
+
measured: false,
|
|
179
|
+
reason: `git diff ${range} failed: ${diff.err.split("\n")[0]}`,
|
|
180
|
+
};
|
|
181
|
+
}
|
|
182
|
+
return {
|
|
183
|
+
at,
|
|
184
|
+
base: base.sha,
|
|
185
|
+
head: head.out,
|
|
186
|
+
measured: true,
|
|
187
|
+
files: diff.out ? diff.out.split("\n") : [],
|
|
188
|
+
graph: graphImpact(repo, range, graphPath),
|
|
189
|
+
};
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
function baseScan(args) {
|
|
193
|
+
const statePath = need(args, "state");
|
|
194
|
+
const repo = need(args, "repo");
|
|
195
|
+
const at = args.now || new Date().toISOString();
|
|
196
|
+
const graphPath = typeof args.graph === "string" ? args.graph : null;
|
|
197
|
+
return locked(statePath, () => {
|
|
198
|
+
const doc = loadState(statePath);
|
|
199
|
+
const row = scanRow(repo, doc.base, at, graphPath);
|
|
200
|
+
doc.scans = [...(Array.isArray(doc.scans) ? doc.scans : []), row].slice(-MAX_SCANS);
|
|
201
|
+
saveState(statePath, doc);
|
|
202
|
+
return [row, row.measured ? 0 : 1];
|
|
203
|
+
});
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
/** The cap and where it came from. An unreadable prefs file throws Unreadable. */
|
|
207
|
+
function revisionCap(args) {
|
|
208
|
+
if (typeof args.cap === "string") {
|
|
209
|
+
const n = Number(args.cap);
|
|
210
|
+
if (!Number.isInteger(n) || n < 0 || n > MAX_REVISION_CAP) {
|
|
211
|
+
usage(`--cap must be an integer from 0 to ${MAX_REVISION_CAP}`);
|
|
212
|
+
}
|
|
213
|
+
return { cap: n, capSource: "--cap" };
|
|
214
|
+
}
|
|
215
|
+
const prefsPath =
|
|
216
|
+
typeof args.prefs === "string"
|
|
217
|
+
? args.prefs
|
|
218
|
+
: join(homedir(), ".claude", "multi-agent-preferences.json");
|
|
219
|
+
const cap = loadState(prefsPath)?.global?.testGen?.revisionCap;
|
|
220
|
+
if (cap === undefined) return { cap: DEFAULT_REVISION_CAP, capSource: "default" };
|
|
221
|
+
if (Number.isInteger(cap) && cap >= 0 && cap <= MAX_REVISION_CAP) {
|
|
222
|
+
return { cap, capSource: "prefs.global.testGen.revisionCap" };
|
|
223
|
+
}
|
|
224
|
+
return {
|
|
225
|
+
cap: DEFAULT_REVISION_CAP,
|
|
226
|
+
capSource: `default (prefs value ${JSON.stringify(cap)} is outside 0..${MAX_REVISION_CAP})`,
|
|
227
|
+
};
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
function decide({ verdict, testId, autopilot, counts, cap }) {
|
|
231
|
+
if (verdict === "product-bug") {
|
|
232
|
+
return autopilot
|
|
233
|
+
? [
|
|
234
|
+
{
|
|
235
|
+
action: "park",
|
|
236
|
+
reason: `product-bug verdict on ${testId}: a product change is never applied without a person`,
|
|
237
|
+
},
|
|
238
|
+
3,
|
|
239
|
+
]
|
|
240
|
+
: [{ action: "ask", reason: `product-bug verdict on ${testId}: the person decides` }, 0];
|
|
241
|
+
}
|
|
242
|
+
if (verdict !== "stale-test")
|
|
243
|
+
return [{ action: "none", reason: `${verdict}: nothing to revise` }, 0];
|
|
244
|
+
if (autopilot && (counts.run >= cap || counts.day >= cap)) {
|
|
245
|
+
const which = counts.run >= cap ? `this run has ${counts.run}` : `today has ${counts.day}`;
|
|
246
|
+
return [
|
|
247
|
+
{
|
|
248
|
+
action: "park",
|
|
249
|
+
reason: `stale-test revision cap reached: ${which} of ${cap} (prefs.global.testGen.revisionCap)`,
|
|
250
|
+
},
|
|
251
|
+
3,
|
|
252
|
+
];
|
|
253
|
+
}
|
|
254
|
+
return null;
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
function revise(args) {
|
|
258
|
+
const statePath = need(args, "state");
|
|
259
|
+
const runId = need(args, "run");
|
|
260
|
+
const testId = need(args, "test");
|
|
261
|
+
const verdict = need(args, "verdict");
|
|
262
|
+
if (!VERDICTS.includes(verdict)) usage(`--verdict must be one of ${VERDICTS.join(", ")}`);
|
|
263
|
+
const autopilot = args.autopilot === true;
|
|
264
|
+
const now = args.now || new Date().toISOString();
|
|
265
|
+
try {
|
|
266
|
+
const { cap, capSource } = revisionCap(args);
|
|
267
|
+
return locked(statePath, () => {
|
|
268
|
+
const doc = loadState(statePath);
|
|
269
|
+
const revisions = Array.isArray(doc.revisions) ? doc.revisions : [];
|
|
270
|
+
const day = now.slice(0, 10);
|
|
271
|
+
const counts = {
|
|
272
|
+
run: revisions.filter((r) => r.runId === runId).length,
|
|
273
|
+
day: revisions.filter((r) => String(r.at).slice(0, 10) === day).length,
|
|
274
|
+
cap,
|
|
275
|
+
capSource,
|
|
276
|
+
};
|
|
277
|
+
const early = decide({ verdict, testId, autopilot, counts, cap });
|
|
278
|
+
if (early) return [{ ...early[0], counts }, early[1]];
|
|
279
|
+
doc.revisions = [...revisions, { runId, testId, at: now }];
|
|
280
|
+
saveState(statePath, doc);
|
|
281
|
+
return [
|
|
282
|
+
{
|
|
283
|
+
action: "apply",
|
|
284
|
+
reason: `stale-test revision of ${testId} recorded`,
|
|
285
|
+
counts: { ...counts, run: counts.run + 1, day: counts.day + 1 },
|
|
286
|
+
},
|
|
287
|
+
0,
|
|
288
|
+
];
|
|
289
|
+
});
|
|
290
|
+
} catch (err) {
|
|
291
|
+
if (!(err instanceof Unreadable)) throw err;
|
|
292
|
+
return autopilot
|
|
293
|
+
? [{ action: "park", reason: `${err.message}; autopilot does not count from zero` }, 3]
|
|
294
|
+
: [{ action: "error", reason: `${err.message}; nothing recorded` }, 4];
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
function main() {
|
|
299
|
+
const [cmd, ...rest] = process.argv.slice(2);
|
|
300
|
+
const args = parseArgs(rest);
|
|
301
|
+
const commands = { "base-record": baseRecord, "base-scan": baseScan, revise };
|
|
302
|
+
if (!commands[cmd]) usage("first argument is base-record, base-scan or revise");
|
|
303
|
+
let result;
|
|
304
|
+
try {
|
|
305
|
+
result = commands[cmd](args);
|
|
306
|
+
} catch (err) {
|
|
307
|
+
result = [{ ok: false, reason: err.message }, err instanceof Unreadable ? 4 : 5];
|
|
308
|
+
}
|
|
309
|
+
writeFileSync(1, `${JSON.stringify(result[0], null, 2)}\n`);
|
|
310
|
+
process.exit(result[1]);
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) main();
|
|
@@ -20,27 +20,59 @@
|
|
|
20
20
|
* IMPORTANT: feed it the FULL diff-risk report (run diff-risk-score.mjs WITHOUT
|
|
21
21
|
* --top), so a shrinking test file beyond the top-N priority list is not missed.
|
|
22
22
|
*
|
|
23
|
+
* With --diff (a saved unified diff) or --diff-base <ref> [--repo <path>] it
|
|
24
|
+
* also scans the run's change for test gaming (_test-gaming.mjs): product code
|
|
25
|
+
* that branches on the test environment, swallowed errors, inflated timeouts.
|
|
26
|
+
* --diff-base reads the working tree, untracked files included, against the
|
|
27
|
+
* merge base of HEAD and <ref>, so commits that landed on the base branch
|
|
28
|
+
* after the run started are not read as the run's. Those findings carry the
|
|
29
|
+
* same tag with `signal: "test_gaming"`. A diff that cannot be read (an
|
|
30
|
+
* unknown ref, a failed git call, a missing file) is reported as
|
|
31
|
+
* `gaming.scanned: false` with the reason, never as a clean scan.
|
|
32
|
+
*
|
|
23
33
|
* Usage:
|
|
24
34
|
* node diff-risk-score.mjs --base B --head H | node test-integrity-gate.mjs
|
|
25
|
-
* node test-integrity-gate.mjs --file risk.json [--
|
|
35
|
+
* node test-integrity-gate.mjs --file risk.json [--diff <unified.diff> | --diff-base <ref> [--repo <path>]]
|
|
36
|
+
* [--json] [--out <path>]
|
|
26
37
|
*
|
|
27
38
|
* --out writes the JSON to <path> (parent directories created) instead of
|
|
28
39
|
* stdout, the empty result included, so Phase 3 always has a file to merge and
|
|
29
40
|
* the step needs no shell fallback.
|
|
30
41
|
*
|
|
31
|
-
* Output (stdout, JSON): { schemaVersion, count, findings: [ ... ] }
|
|
42
|
+
* Output (stdout, JSON): { schemaVersion, count, findings: [ ... ], gaming?: { scanned, count, reason? } }
|
|
32
43
|
* Exit 0 always (detection tool; severity is adjudicated downstream). Exit 1
|
|
33
|
-
* only on a setup error
|
|
44
|
+
* only on a setup error: an option given without a value, or a value that
|
|
45
|
+
* starts with "-" (unreadable or invalid risk input is treated as "no
|
|
46
|
+
* findings").
|
|
34
47
|
*/
|
|
35
48
|
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
36
49
|
import { dirname } from "node:path";
|
|
37
50
|
import { runMain } from "../lib/fatal.mjs";
|
|
51
|
+
import { scanTestGaming } from "./_test-gaming.mjs";
|
|
52
|
+
import { GitError, diffFromMergeBase } from "./_worktree-snapshot.mjs";
|
|
38
53
|
|
|
39
54
|
const args = process.argv.slice(2);
|
|
40
|
-
|
|
55
|
+
|
|
56
|
+
/** The value after `flag`, or null when the flag is absent. A missing or option-shaped value is a setup error. */
|
|
57
|
+
function flagValue(flag) {
|
|
58
|
+
const i = args.indexOf(flag);
|
|
59
|
+
if (i === -1) return null;
|
|
60
|
+
const v = args[i + 1];
|
|
61
|
+
if (v === undefined || v === "" || v.startsWith("-")) {
|
|
62
|
+
process.stderr.write(
|
|
63
|
+
`test-integrity-gate: ${flag} needs a value that does not start with "-"\n`,
|
|
64
|
+
);
|
|
65
|
+
process.exit(1);
|
|
66
|
+
}
|
|
67
|
+
return v;
|
|
68
|
+
}
|
|
69
|
+
|
|
41
70
|
const PRETTY = args.includes("--json");
|
|
42
|
-
const
|
|
43
|
-
const OUT =
|
|
71
|
+
const FILE = flagValue("--file");
|
|
72
|
+
const OUT = flagValue("--out");
|
|
73
|
+
const DIFF = flagValue("--diff");
|
|
74
|
+
const DIFF_BASE = flagValue("--diff-base");
|
|
75
|
+
const REPO = flagValue("--repo") ?? process.cwd();
|
|
44
76
|
|
|
45
77
|
function emit(text) {
|
|
46
78
|
if (!OUT) {
|
|
@@ -53,23 +85,55 @@ function emit(text) {
|
|
|
53
85
|
|
|
54
86
|
function readInput() {
|
|
55
87
|
try {
|
|
56
|
-
if (
|
|
57
|
-
return readFileSync(args[fileFlag + 1], "utf8");
|
|
58
|
-
}
|
|
88
|
+
if (FILE !== null) return readFileSync(FILE, "utf8");
|
|
59
89
|
return readFileSync(0, "utf8"); // stdin
|
|
60
90
|
} catch {
|
|
61
91
|
return "";
|
|
62
92
|
}
|
|
63
93
|
}
|
|
64
94
|
|
|
95
|
+
/** The diff to scan for test gaming: { text } or the { reason } it could not be read. */
|
|
96
|
+
function readDiff() {
|
|
97
|
+
if (DIFF !== null) {
|
|
98
|
+
try {
|
|
99
|
+
return { text: readFileSync(DIFF, "utf8") };
|
|
100
|
+
} catch (err) {
|
|
101
|
+
return { reason: `cannot read --diff: ${err.message}` };
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
try {
|
|
105
|
+
return { text: diffFromMergeBase(REPO, DIFF_BASE) };
|
|
106
|
+
} catch (err) {
|
|
107
|
+
if (err instanceof GitError) return { reason: err.message };
|
|
108
|
+
throw err;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/** Test-gaming findings, and whether the diff was read at all. */
|
|
113
|
+
function gamingScan() {
|
|
114
|
+
if (DIFF === null && DIFF_BASE === null) return { findings: [], gaming: undefined };
|
|
115
|
+
const { text, reason } = readDiff();
|
|
116
|
+
if (text === undefined) return { findings: [], gaming: { scanned: false, count: 0, reason } };
|
|
117
|
+
const findings = scanTestGaming(text);
|
|
118
|
+
return { findings, gaming: { scanned: true, count: findings.length } };
|
|
119
|
+
}
|
|
120
|
+
|
|
65
121
|
function main() {
|
|
66
122
|
const raw = readInput();
|
|
123
|
+
const { findings: gamingFindings, gaming } = gamingScan();
|
|
67
124
|
let report;
|
|
68
125
|
try {
|
|
69
126
|
report = JSON.parse(raw);
|
|
70
127
|
} catch {
|
|
71
128
|
// No/invalid input -> fail OPEN (no findings). A gate bug must not wedge Phase 4.
|
|
72
|
-
emit(
|
|
129
|
+
emit(
|
|
130
|
+
JSON.stringify({
|
|
131
|
+
schemaVersion: "1.0.0",
|
|
132
|
+
count: gamingFindings.length,
|
|
133
|
+
findings: gamingFindings,
|
|
134
|
+
...(gaming ? { gaming } : {}),
|
|
135
|
+
}),
|
|
136
|
+
);
|
|
73
137
|
return;
|
|
74
138
|
}
|
|
75
139
|
|
|
@@ -100,9 +164,10 @@ function main() {
|
|
|
100
164
|
});
|
|
101
165
|
}
|
|
102
166
|
|
|
167
|
+
findings.push(...gamingFindings);
|
|
103
168
|
emit(
|
|
104
169
|
JSON.stringify(
|
|
105
|
-
{ schemaVersion: "1.0.0", count: findings.length, findings },
|
|
170
|
+
{ schemaVersion: "1.0.0", count: findings.length, findings, ...(gaming ? { gaming } : {}) },
|
|
106
171
|
null,
|
|
107
172
|
PRETTY ? 2 : 0,
|
|
108
173
|
),
|
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* usage-register-beacon.mjs - tell the reporting site that registration failed.
|
|
4
|
+
*
|
|
5
|
+
* A machine whose registration fails sends nothing else, so whoever runs the
|
|
6
|
+
* reporting site cannot see that the person tried at all. This sends one small
|
|
7
|
+
* report to `<register URL>/failure`: { u, v, reason, os }. `u` is the login
|
|
8
|
+
* registration used (proven or declared), `v` the installed pipeline version,
|
|
9
|
+
* `reason` one word from REASONS, `os` darwin when known. Nothing else: no
|
|
10
|
+
* token, no header beyond Content-Type, no host name, no failure text.
|
|
11
|
+
*
|
|
12
|
+
* usage-register.mjs starts it detached once a registration ends in a failure
|
|
13
|
+
* the person did not choose, so it never blocks or fails the caller. It sends
|
|
14
|
+
* at most once per (reason, version) per day; a report that was not delivered
|
|
15
|
+
* is tried again on the next failed registration. The state lives in
|
|
16
|
+
* `~/.claude/logs/multi-agent/usage-register-beacon.json` and holds no secret.
|
|
17
|
+
*
|
|
18
|
+
* Usage (internal): MULTI_AGENT_REGISTER_BEACON='<json>' node usage-register-beacon.mjs
|
|
19
|
+
*
|
|
20
|
+
* @module pipeline/scripts/usage-register-beacon
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
|
24
|
+
import { homedir } from "node:os";
|
|
25
|
+
import { join } from "node:path";
|
|
26
|
+
import { pathToFileURL } from "node:url";
|
|
27
|
+
import { postJson } from "./_http-post.mjs";
|
|
28
|
+
|
|
29
|
+
export const BEACON_FILE = "usage-register-beacon.json";
|
|
30
|
+
export const BEACON_TIMEOUT_MS = 3000;
|
|
31
|
+
export const BEACON_ENV = "MULTI_AGENT_REGISTER_BEACON";
|
|
32
|
+
|
|
33
|
+
export const REASONS = Object.freeze([
|
|
34
|
+
"endpoint-unreachable",
|
|
35
|
+
"tls-untrusted",
|
|
36
|
+
"proxy-refused",
|
|
37
|
+
"timeout",
|
|
38
|
+
"redirected",
|
|
39
|
+
"not-collaborator",
|
|
40
|
+
"gh-account-mismatch",
|
|
41
|
+
"proof-rejected",
|
|
42
|
+
"proof-required",
|
|
43
|
+
"no-proof",
|
|
44
|
+
"rate-limited",
|
|
45
|
+
"server-error",
|
|
46
|
+
"credential-store",
|
|
47
|
+
"prefs-write",
|
|
48
|
+
"other",
|
|
49
|
+
]);
|
|
50
|
+
|
|
51
|
+
const LOGIN_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,38}$/;
|
|
52
|
+
const VERSION_RE = /^v?\d{1,5}\.\d{1,5}\.\d{1,5}(-[0-9A-Za-z.-]{1,24})?$/;
|
|
53
|
+
|
|
54
|
+
const TLS_FETCH =
|
|
55
|
+
/CERT|SELF_SIGNED|UNABLE_TO_GET_ISSUER|UNABLE_TO_VERIFY|DEPTH_ZERO|ERR_TLS|ERR_SSL|EPROTO/;
|
|
56
|
+
const TLS_CURL = new Set([35, 51, 53, 54, 58, 59, 60, 64, 66, 77, 80, 82, 83, 90, 91]);
|
|
57
|
+
const PROXY_CURL = new Set([5, 56, 97]);
|
|
58
|
+
|
|
59
|
+
function curlExit(failure) {
|
|
60
|
+
const m = /^curl-exit-(\d+)$/.exec(String(failure ?? ""));
|
|
61
|
+
return m ? Number(m[1]) : null;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* The reason for a request that reached no HTTP answer, from the fetch cause
|
|
66
|
+
* and curl's exit code (_http-post.mjs). curl is asked second and honours the
|
|
67
|
+
* proxy and the system trust store, so its verdict is read first.
|
|
68
|
+
*/
|
|
69
|
+
export function transportReason(fetchFailure, curlFailure) {
|
|
70
|
+
const exit = curlExit(curlFailure);
|
|
71
|
+
if (exit !== null && PROXY_CURL.has(exit)) return "proxy-refused";
|
|
72
|
+
if (exit !== null && TLS_CURL.has(exit)) return "tls-untrusted";
|
|
73
|
+
if (exit === 28 || fetchFailure === "timeout") return "timeout";
|
|
74
|
+
if (TLS_FETCH.test(String(fetchFailure ?? ""))) return "tls-untrusted";
|
|
75
|
+
return "endpoint-unreachable";
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** The reason for an HTTP answer that issued no token. */
|
|
79
|
+
export function httpReason(status, code) {
|
|
80
|
+
if (code === "login_mismatch") return "gh-account-mismatch";
|
|
81
|
+
if (code === "not_collaborator") return "not-collaborator";
|
|
82
|
+
if (code === "proof_required") return "proof-required";
|
|
83
|
+
if (status >= 300 && status < 400) return "redirected";
|
|
84
|
+
if (status === 401) return "proof-rejected";
|
|
85
|
+
if (status === 403) return "not-collaborator";
|
|
86
|
+
if (status === 407) return "proxy-refused";
|
|
87
|
+
if (status === 429) return "rate-limited";
|
|
88
|
+
if (status >= 500) return "server-error";
|
|
89
|
+
return "other";
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** The failure URL next to the register URL. */
|
|
93
|
+
export function beaconUrl(registerUrl) {
|
|
94
|
+
return `${String(registerUrl).replace(/\/+$/, "")}/failure`;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** The installed pipeline version: the host's stamp, else the package next to this script. */
|
|
98
|
+
export function pipelineVersion(home = homedir()) {
|
|
99
|
+
for (const root of [".claude", ".copilot", ".codex"]) {
|
|
100
|
+
try {
|
|
101
|
+
const v = readFileSync(join(home, root, ".pipeline-version"), "utf8").trim();
|
|
102
|
+
if (VERSION_RE.test(v)) return v.replace(/^v/, "");
|
|
103
|
+
} catch {
|
|
104
|
+
/* no stamp for this host */
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
try {
|
|
108
|
+
const v = JSON.parse(
|
|
109
|
+
readFileSync(join(import.meta.dirname, "..", "..", "package.json"), "utf8"),
|
|
110
|
+
).version;
|
|
111
|
+
if (typeof v === "string" && VERSION_RE.test(v)) return v.replace(/^v/, "");
|
|
112
|
+
} catch {
|
|
113
|
+
/* not inside a package */
|
|
114
|
+
}
|
|
115
|
+
return null;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** The exact body sent, or null when any field would be refused. */
|
|
119
|
+
export function beaconBody({ login, version, reason, os = process.platform }) {
|
|
120
|
+
if (typeof login !== "string" || !LOGIN_RE.test(login)) return null;
|
|
121
|
+
if (typeof version !== "string" || !VERSION_RE.test(version)) return null;
|
|
122
|
+
if (!REASONS.includes(reason)) return null;
|
|
123
|
+
const body = { u: login, v: version.replace(/^v/, ""), reason };
|
|
124
|
+
if (os === "darwin") body.os = os;
|
|
125
|
+
return body;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
export function beaconStatePath(home = homedir()) {
|
|
129
|
+
return join(home, ".claude", "logs", "multi-agent", BEACON_FILE);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
function today(now) {
|
|
133
|
+
return new Date(now).toISOString().slice(0, 10);
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
export function readBeaconState(home = homedir()) {
|
|
137
|
+
try {
|
|
138
|
+
const j = JSON.parse(readFileSync(beaconStatePath(home), "utf8"));
|
|
139
|
+
return j && typeof j === "object" && !Array.isArray(j) ? j : {};
|
|
140
|
+
} catch {
|
|
141
|
+
return {};
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
function writeBeaconState(home, state) {
|
|
146
|
+
const target = beaconStatePath(home);
|
|
147
|
+
const tmp = `${target}.${process.pid}.tmp`;
|
|
148
|
+
try {
|
|
149
|
+
if (!existsSync(join(home, ".claude"))) return;
|
|
150
|
+
mkdirSync(join(home, ".claude", "logs", "multi-agent"), { recursive: true });
|
|
151
|
+
writeFileSync(tmp, JSON.stringify(state) + "\n", { mode: 0o600 });
|
|
152
|
+
renameSync(tmp, target);
|
|
153
|
+
} catch {
|
|
154
|
+
/* logs dir unwritable - never break the caller */
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/** Was this (reason, version) already delivered today? */
|
|
159
|
+
export function alreadySent(state, body, now = Date.now()) {
|
|
160
|
+
return state?.sent?.[`${body.reason}@${body.v}`] === today(now);
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* Send one beacon unless the same (reason, version) was delivered today.
|
|
165
|
+
* Resolves to { outcome: "sent" | "failed" | "duplicate" | "invalid", status? };
|
|
166
|
+
* never throws.
|
|
167
|
+
*/
|
|
168
|
+
export async function sendRegisterBeacon({
|
|
169
|
+
home = homedir(),
|
|
170
|
+
url,
|
|
171
|
+
login,
|
|
172
|
+
version = pipelineVersion(home),
|
|
173
|
+
reason,
|
|
174
|
+
os = process.platform,
|
|
175
|
+
now = Date.now(),
|
|
176
|
+
post = postJson,
|
|
177
|
+
timeoutMs = BEACON_TIMEOUT_MS,
|
|
178
|
+
}) {
|
|
179
|
+
const body = beaconBody({ login, version, reason, os });
|
|
180
|
+
if (!body || typeof url !== "string" || !url) return { outcome: "invalid" };
|
|
181
|
+
const state = readBeaconState(home);
|
|
182
|
+
if (alreadySent(state, body, now)) return { outcome: "duplicate" };
|
|
183
|
+
let r;
|
|
184
|
+
try {
|
|
185
|
+
r = await post({ url, headers: {}, body: JSON.stringify(body), timeoutMs });
|
|
186
|
+
} catch {
|
|
187
|
+
r = { failure: "error" };
|
|
188
|
+
}
|
|
189
|
+
const delivered = !r.failure && r.status >= 200 && r.status < 300;
|
|
190
|
+
const day = today(now);
|
|
191
|
+
const sent = Object.fromEntries(
|
|
192
|
+
Object.entries(state.sent && typeof state.sent === "object" ? state.sent : {}).filter(
|
|
193
|
+
([, d]) => d === day,
|
|
194
|
+
),
|
|
195
|
+
);
|
|
196
|
+
if (delivered) sent[`${body.reason}@${body.v}`] = day;
|
|
197
|
+
const last = {
|
|
198
|
+
t: new Date(now).toISOString(),
|
|
199
|
+
reason: body.reason,
|
|
200
|
+
version: body.v,
|
|
201
|
+
outcome: delivered ? "sent" : "failed",
|
|
202
|
+
};
|
|
203
|
+
if (r.failure) last.cause = String(r.failure).slice(0, 40);
|
|
204
|
+
else last.status = r.status;
|
|
205
|
+
writeBeaconState(home, { sent, last });
|
|
206
|
+
return delivered ? { outcome: "sent", status: r.status } : { outcome: "failed", ...last };
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
function isEntrypoint() {
|
|
210
|
+
const arg = process.argv[1];
|
|
211
|
+
if (!arg) return false;
|
|
212
|
+
try {
|
|
213
|
+
return import.meta.url === pathToFileURL(arg).href;
|
|
214
|
+
} catch {
|
|
215
|
+
return false;
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
function payloadFromEnv() {
|
|
220
|
+
try {
|
|
221
|
+
return JSON.parse(process.env[BEACON_ENV] || "null");
|
|
222
|
+
} catch {
|
|
223
|
+
return null;
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
if (isEntrypoint()) {
|
|
228
|
+
const payload = payloadFromEnv();
|
|
229
|
+
if (payload && typeof payload === "object" && existsSync(payload.home || homedir())) {
|
|
230
|
+
sendRegisterBeacon({
|
|
231
|
+
home: payload.home || homedir(),
|
|
232
|
+
url: payload.url,
|
|
233
|
+
login: payload.login,
|
|
234
|
+
reason: payload.reason,
|
|
235
|
+
}).finally(() => {
|
|
236
|
+
process.exitCode = 0;
|
|
237
|
+
});
|
|
238
|
+
}
|
|
239
|
+
}
|