@mmerterden/multi-agent-pipeline 20.27.2 → 20.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +41 -0
- package/README.md +20 -6
- package/README.tr.md +6 -5
- package/docs/adr/0008-installer-modularization-and-secret-leak-defense.md +0 -1
- package/docs/architecture.md +4 -4
- package/docs/ecosystem.md +5 -5
- package/docs/facts.json +14 -12
- package/docs/features.md +2 -0
- package/install/_dev-only-files.mjs +1 -0
- package/install/catalog-history.json +1 -1
- package/install/claude.mjs +112 -0
- package/install/index.mjs +0 -2
- package/install/templates/claude-hooks.json +30 -1
- package/manifest.json +101 -67
- package/package.json +4 -1
- package/pipeline/agents/code-reviewer.md +1 -0
- package/pipeline/agents/test-gen-analyst.md +98 -0
- package/pipeline/commands/multi-agent/analysis/SKILL.md +3 -1
- package/pipeline/commands/multi-agent/resume/SKILL.md +2 -1
- package/pipeline/commands/multi-agent/sync/SKILL.md +3 -3
- package/pipeline/commands/multi-agent/test-gen/SKILL.md +124 -0
- package/pipeline/contract/types/index.d.ts +1 -1
- package/pipeline/lib/hook-registration.mjs +252 -0
- package/pipeline/lib/prefs-value.mjs +44 -0
- package/pipeline/lib/run-completion.mjs +148 -0
- package/pipeline/multi-agent-refs/analysis/render.md +4 -4
- package/pipeline/multi-agent-refs/cross-cli-contract.md +11 -5
- package/pipeline/multi-agent-refs/features/doctor.md +24 -0
- package/pipeline/multi-agent-refs/features/test-classification.md +76 -0
- package/pipeline/multi-agent-refs/features/usage-reporting.md +17 -0
- package/pipeline/multi-agent-refs/help/en.md +1 -0
- package/pipeline/multi-agent-refs/help/tr.md +1 -0
- package/pipeline/multi-agent-refs/phases/operations.md +19 -2
- package/pipeline/multi-agent-refs/phases/phase-0-init.md +6 -2
- package/pipeline/multi-agent-refs/phases/phase-1-plan.md +2 -2
- package/pipeline/multi-agent-refs/phases/phase-2-dev.md +7 -8
- package/pipeline/multi-agent-refs/phases/phase-3-review.md +5 -5
- package/pipeline/multi-agent-refs/phases/phase-4-commit.md +7 -6
- package/pipeline/multi-agent-refs/phases/phase-5-report.md +10 -8
- package/pipeline/multi-agent-refs/test-gen/stages.md +227 -0
- package/pipeline/preferences-template.json +10 -0
- package/pipeline/rules/figma-pipeline.md +2 -0
- package/pipeline/schemas/agent-state.schema.json +20 -5
- package/pipeline/schemas/phases.json +3 -2
- package/pipeline/schemas/phases.schema.json +5 -1
- package/pipeline/schemas/prefs.schema.json +51 -0
- package/pipeline/schemas/runs-index.schema.json +1 -1
- package/pipeline/schemas/test-gen-discovery.schema.json +73 -0
- package/pipeline/schemas/test-gen-gaps.schema.json +122 -0
- package/pipeline/schemas/test-gen-inventory.schema.json +59 -0
- package/pipeline/schemas/test-gen-results.schema.json +44 -0
- package/pipeline/scripts/_active-run.mjs +158 -0
- package/pipeline/scripts/_autopilot-outcomes.mjs +1 -0
- package/pipeline/scripts/_guard_rules.py +605 -0
- package/pipeline/scripts/_handle-patterns.mjs +166 -0
- package/pipeline/scripts/_http-post.mjs +141 -0
- package/pipeline/scripts/_stack-routing.mjs +4 -0
- package/pipeline/scripts/_test-gaming.mjs +227 -0
- package/pipeline/scripts/_unified-diff.mjs +104 -0
- package/pipeline/scripts/_worktree-snapshot.mjs +113 -0
- package/pipeline/scripts/agent-guard.py +118 -7
- package/pipeline/scripts/agent-guard.sh +50 -3
- package/pipeline/scripts/argv-secret-scan.mjs +114 -0
- package/pipeline/scripts/autopilot-runner.mjs +8 -2
- package/pipeline/scripts/confirm-command.mjs +70 -0
- package/pipeline/scripts/doctor.mjs +105 -0
- package/pipeline/scripts/evidence-gate.mjs +82 -7
- package/pipeline/scripts/gate-ledger.mjs +163 -7
- package/pipeline/scripts/handle-diff-audit.mjs +188 -0
- package/pipeline/scripts/plan-coverage-gate.mjs +9 -8
- package/pipeline/scripts/pre-commit-check.sh +12 -0
- package/pipeline/scripts/routine-registry.mjs +5 -1
- package/pipeline/scripts/run-scope-guard.mjs +188 -0
- package/pipeline/scripts/run-scope-guard.sh +23 -0
- package/pipeline/scripts/run-stop-guard.mjs +246 -0
- package/pipeline/scripts/run-stop-guard.sh +25 -0
- package/pipeline/scripts/run-ui-tests.sh +27 -3
- package/pipeline/scripts/test-classify.mjs +296 -0
- package/pipeline/scripts/test-gen-gate.mjs +409 -0
- package/pipeline/scripts/test-gen-state.mjs +313 -0
- package/pipeline/scripts/test-integrity-gate.mjs +76 -11
- package/pipeline/scripts/uninstall.mjs +2 -0
- package/pipeline/scripts/usage-register-beacon.mjs +239 -0
- package/pipeline/scripts/usage-register.mjs +128 -33
- package/pipeline/scripts/usage-report.mjs +43 -33
- package/pipeline/scripts/validate-state.mjs +26 -0
- package/pipeline/scripts/write-state.mjs +114 -0
- package/pipeline/skills/.skill-manifest.json +23 -3
- package/pipeline/skills/.skills-index.json +57 -2
- package/pipeline/skills/shared/README.md +10 -5
- package/pipeline/skills/shared/core/multi-agent-sync/SKILL.md +3 -3
- package/pipeline/skills/shared/core/multi-agent-test-gen/SKILL.md +120 -0
- package/pipeline/skills/shared/external/fix-review-verdicts/SKILL.md +133 -0
- package/pipeline/skills/shared/external/handle-request/SKILL.md +97 -0
- package/pipeline/skills/shared/external/scenario-gap-analysis/SKILL.md +129 -0
- package/pipeline/skills/shared/external/scenario-gap-analysis/references/scenario-schema.md +60 -0
- package/pipeline/skills/shared/external/test-result-triage/SKILL.md +160 -0
- package/pipeline/skills/skills-index.md +7 -2
- package/install/_telemetry.mjs +0 -61
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file _handle-patterns.mjs - what an automation handle looks like, per platform.
|
|
3
|
+
*
|
|
4
|
+
* A handle is the stable identifier a UI test finds an element by: an iOS
|
|
5
|
+
* `accessibilityIdentifier`, a Compose `testTag` (or `testTagsAsResourceId` in
|
|
6
|
+
* a semantics block), a web `data-testid`. Each file is judged only by its own
|
|
7
|
+
* platform's pattern, chosen from the extension, so a Swift string that
|
|
8
|
+
* happens to contain "data-testid" is not an iOS handle and a Flutter `Key`
|
|
9
|
+
* is not one at all.
|
|
10
|
+
*
|
|
11
|
+
* Shared by handle-diff-audit.mjs (did the diff add only handles) and
|
|
12
|
+
* test-gen-gate.mjs (did the handle stage add any).
|
|
13
|
+
*
|
|
14
|
+
* @module pipeline/scripts/_handle-patterns
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { parseUnifiedDiff } from "./_unified-diff.mjs";
|
|
18
|
+
|
|
19
|
+
export { parseUnifiedDiff };
|
|
20
|
+
|
|
21
|
+
const STR = String.raw`"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'`;
|
|
22
|
+
const REF = String.raw`[A-Za-z_$][\w$.]*(?:\([^()]*\))?`;
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* `handle` decides whether a line carries a handle; `strip` removes the handle
|
|
26
|
+
* construct so what is left can be compared with the removed lines.
|
|
27
|
+
*/
|
|
28
|
+
export const PLATFORMS = Object.freeze({
|
|
29
|
+
ios: {
|
|
30
|
+
ext: /\.(swift|m|mm)$/i,
|
|
31
|
+
handle: /\baccessibilityIdentifier\b/,
|
|
32
|
+
strip: [
|
|
33
|
+
new RegExp(String.raw`\.accessibilityIdentifier\(\s*(?:${STR}|${REF})\s*\)`, "g"),
|
|
34
|
+
new RegExp(
|
|
35
|
+
String.raw`(?:\b[\w$.]*\.)?accessibilityIdentifier\s*=\s*(?:${STR}|${REF})\s*;?`,
|
|
36
|
+
"g",
|
|
37
|
+
),
|
|
38
|
+
],
|
|
39
|
+
},
|
|
40
|
+
compose: {
|
|
41
|
+
ext: /\.kts?$/i,
|
|
42
|
+
handle: /\btestTag\b|\btestTagsAsResourceId\b/,
|
|
43
|
+
strip: [
|
|
44
|
+
new RegExp(
|
|
45
|
+
String.raw`(?:\bmodifier\s*=\s*)?\bModifier\s*\.\s*testTag\(\s*(?:${STR}|${REF})\s*\)\s*,?`,
|
|
46
|
+
"g",
|
|
47
|
+
),
|
|
48
|
+
new RegExp(String.raw`\.testTag\(\s*(?:${STR}|${REF})\s*\)`, "g"),
|
|
49
|
+
new RegExp(
|
|
50
|
+
String.raw`\.semantics\s*(?:\([^()]*\))?\s*\{\s*(?:testTag\s*=\s*(?:${STR}|${REF})|testTagsAsResourceId\s*=\s*true)\s*\}`,
|
|
51
|
+
"g",
|
|
52
|
+
),
|
|
53
|
+
new RegExp(String.raw`\btestTag\s*=\s*(?:${STR}|${REF})`, "g"),
|
|
54
|
+
/\btestTagsAsResourceId\s*=\s*true/g,
|
|
55
|
+
],
|
|
56
|
+
},
|
|
57
|
+
web: {
|
|
58
|
+
ext: /\.(html?|vue|svelte|astro|[cm]?[jt]sx?)$/i,
|
|
59
|
+
handle: /\bdata-testid\b/,
|
|
60
|
+
strip: [
|
|
61
|
+
new RegExp(String.raw`:?\bdata-testid\s*=\s*(?:${STR}|\{[^{}]*\})`, "g"),
|
|
62
|
+
new RegExp(String.raw`(["'])data-testid\1\s*:\s*(?:${STR}|${REF})\s*,?`, "g"),
|
|
63
|
+
],
|
|
64
|
+
},
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
const TEST_PATH =
|
|
68
|
+
/(^|\/)(tests?|__tests__|e2e|specs?|androidTest|[\w-]*UITests?)\/|\.(test|spec)\.[cm]?[jt]sx?$|Tests?\.(swift|kt)$/;
|
|
69
|
+
const SNAPSHOT_PATH = /(^|\/)__snapshots__\/|\.snap$/;
|
|
70
|
+
|
|
71
|
+
/** @param {string} path @returns {"ios"|"compose"|"web"|null} */
|
|
72
|
+
export function platformOf(path) {
|
|
73
|
+
for (const [id, p] of Object.entries(PLATFORMS)) if (p.ext.test(path)) return id;
|
|
74
|
+
return null;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** @param {string} path */
|
|
78
|
+
export function isTestPath(path) {
|
|
79
|
+
return TEST_PATH.test(path);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** @param {string} path */
|
|
83
|
+
export function isSnapshotPath(path) {
|
|
84
|
+
return SNAPSHOT_PATH.test(path);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** Does this line of this file carry a handle, by that file's own platform? */
|
|
88
|
+
export function isHandleLine(path, line) {
|
|
89
|
+
const id = platformOf(path);
|
|
90
|
+
return id !== null && PLATFORMS[id].handle.test(line);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** The line with its handle constructs removed. */
|
|
94
|
+
export function stripHandles(path, line) {
|
|
95
|
+
const id = platformOf(path);
|
|
96
|
+
if (id === null) return line;
|
|
97
|
+
return PLATFORMS[id].strip.reduce((s, re) => s.replace(re, " "), line);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* The code a line carries, as an ordered token list: string literals whole,
|
|
102
|
+
* identifiers, numbers, multi-character operators and every other punctuation
|
|
103
|
+
* character. Whitespace and line breaks are dropped, so re-wrapping a tag over
|
|
104
|
+
* several lines or moving a modifier onto its own line changes nothing, while
|
|
105
|
+
* a flipped comparison, a dropped `!` or swapped arguments do. A comma left
|
|
106
|
+
* dangling before a closing bracket by a stripped handle is dropped too.
|
|
107
|
+
*
|
|
108
|
+
* @param {string} text
|
|
109
|
+
* @returns {string[]}
|
|
110
|
+
*/
|
|
111
|
+
export function codeTokens(text) {
|
|
112
|
+
const out =
|
|
113
|
+
String(text).match(
|
|
114
|
+
/"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'|`(?:[^`\\]|\\.)*`|[A-Za-z_$][\w$]*|\d[\w.]*|===|!==|==|!=|<=|>=|&&|\|\||\?\?|\?\.|->|=>|::|\.\.\.|\.\.<|\S/g,
|
|
115
|
+
) || [];
|
|
116
|
+
return out.filter((t, i) => !(t === "," && /^[)\]}>]$/.test(out[i + 1] || "")));
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
const STR_LIT = String.raw`(?:"(?:[^"\\$]|\\.)*"|'(?:[^'\\$]|\\.)*'|` + "`[^`$]*`)";
|
|
120
|
+
const IDENTIFIER_FILE_LINE = [
|
|
121
|
+
/^$/,
|
|
122
|
+
/^(\/\/|\/\*|\*|#|<!--)/,
|
|
123
|
+
/^(import|package)\s/,
|
|
124
|
+
/^@file:/,
|
|
125
|
+
/^(?:(?:public|internal|private|fileprivate|export|default)\s+)*(?:enum|struct|object|class|extension|namespace)\s+[\w.]+(?:\s*:\s*[\w., ]+)?\s*\{?$/,
|
|
126
|
+
/^(?:export\s+)?(?:const|let|var|val)\s+\w+(?:\s*:\s*[\w.<>? ]+)?\s*=\s*\{$/,
|
|
127
|
+
new RegExp(
|
|
128
|
+
String.raw`^(?:(?:public|internal|private|fileprivate|export)\s+)*(?:static\s+)?(?:let|var|const\s+val|val|const)\s+\w+(?:\s*:\s*\w+)?\s*=\s*${STR_LIT}\s*[;,]?$`,
|
|
129
|
+
),
|
|
130
|
+
new RegExp(String.raw`^case\s+\w+(?:\s*=\s*${STR_LIT})?$`),
|
|
131
|
+
new RegExp(String.raw`^(?:\w+|${STR_LIT})\s*:\s*${STR_LIT}\s*,?$`),
|
|
132
|
+
/^[}\])]+(?:\s*as\s+const)?\s*[;,]?$/,
|
|
133
|
+
];
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Does a new file hold handle constants and nothing else? Declarations of
|
|
137
|
+
* string constants, the enums or objects that group them, imports, comments
|
|
138
|
+
* and closing brackets. A function, a computed value or an interpolated
|
|
139
|
+
* string is behaviour, whatever the file is called.
|
|
140
|
+
*
|
|
141
|
+
* @param {string[]} lines
|
|
142
|
+
*/
|
|
143
|
+
export function isHandleConstantsFile(lines) {
|
|
144
|
+
return lines.every((l) => IDENTIFIER_FILE_LINE.some((re) => re.test(l.trim())));
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* Added handle lines in a diff, product files only: a selector string in a
|
|
149
|
+
* test file names a handle, it does not add one.
|
|
150
|
+
*
|
|
151
|
+
* @param {string} text
|
|
152
|
+
* @returns {{total: number, byFile: Record<string, number>}}
|
|
153
|
+
*/
|
|
154
|
+
export function countAddedHandles(text) {
|
|
155
|
+
const byFile = {};
|
|
156
|
+
let total = 0;
|
|
157
|
+
for (const [file, f] of parseUnifiedDiff(text)) {
|
|
158
|
+
if (isTestPath(file) || isSnapshotPath(file)) continue;
|
|
159
|
+
const n = f.added.filter((l) => isHandleLine(file, l)).length;
|
|
160
|
+
if (n > 0) {
|
|
161
|
+
byFile[file] = n;
|
|
162
|
+
total += n;
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
return { total, byFile };
|
|
166
|
+
}
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
// _http-post.mjs - POST a JSON body, falling back to curl when Node's fetch
|
|
2
|
+
// cannot reach the host at all.
|
|
3
|
+
//
|
|
4
|
+
// Node's built-in fetch ignores HTTPS_PROXY and trusts only its bundled CA
|
|
5
|
+
// list, so behind a corporate proxy or a TLS-inspecting gateway it fails with
|
|
6
|
+
// ECONNREFUSED, ENOTFOUND or SELF_SIGNED_CERT_IN_CHAIN while gh, git and curl on
|
|
7
|
+
// the same machine work. curl honours the proxy variables and the system trust
|
|
8
|
+
// store, so a transport failure (never an HTTP status) is retried through it.
|
|
9
|
+
//
|
|
10
|
+
// Secret headers never reach argv: curl reads them from a 0600 file that is
|
|
11
|
+
// removed before this returns. Redirects are never followed by either path.
|
|
12
|
+
|
|
13
|
+
import { spawn as spawnChild } from "node:child_process";
|
|
14
|
+
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
|
15
|
+
import { tmpdir } from "node:os";
|
|
16
|
+
import { join } from "node:path";
|
|
17
|
+
|
|
18
|
+
/** The low-level reason a fetch failed, e.g. ECONNREFUSED or SELF_SIGNED_CERT_IN_CHAIN. */
|
|
19
|
+
export function failureCode(err, aborted = false) {
|
|
20
|
+
if (aborted) return "timeout";
|
|
21
|
+
return String(err?.cause?.code ?? err?.code ?? err?.name ?? "error").slice(0, 40);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function run(spawn, cmd, args, input, timeoutMs) {
|
|
25
|
+
return new Promise((resolve) => {
|
|
26
|
+
let stdout = "";
|
|
27
|
+
let child;
|
|
28
|
+
try {
|
|
29
|
+
child = spawn(cmd, args, { stdio: ["pipe", "pipe", "ignore"] });
|
|
30
|
+
} catch (error) {
|
|
31
|
+
resolve({ error });
|
|
32
|
+
return;
|
|
33
|
+
}
|
|
34
|
+
const timer = setTimeout(() => child.kill("SIGKILL"), timeoutMs);
|
|
35
|
+
child.stdout.on("data", (c) => (stdout += c));
|
|
36
|
+
child.on("error", (error) => {
|
|
37
|
+
clearTimeout(timer);
|
|
38
|
+
resolve({ error });
|
|
39
|
+
});
|
|
40
|
+
child.on("close", (status) => {
|
|
41
|
+
clearTimeout(timer);
|
|
42
|
+
resolve({ status, stdout });
|
|
43
|
+
});
|
|
44
|
+
child.stdin.end(input);
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* One POST through curl. Returns { status, body, location, transport: "curl" }
|
|
50
|
+
* or { failure, transport: "curl" } when curl is missing or cannot connect.
|
|
51
|
+
*/
|
|
52
|
+
export async function curlPost({ url, headers = {}, body, timeoutMs, spawn = spawnChild }) {
|
|
53
|
+
const dir = mkdtempSync(join(tmpdir(), "ma-http-"));
|
|
54
|
+
const headerFile = join(dir, "headers");
|
|
55
|
+
try {
|
|
56
|
+
const lines = Object.entries(headers).map(([k, v]) => `${k}: ${v}`);
|
|
57
|
+
writeFileSync(headerFile, lines.join("\n") + "\n", { mode: 0o600 });
|
|
58
|
+
const seconds = String(Math.max(1, Math.ceil(timeoutMs / 1000)));
|
|
59
|
+
const r = await run(
|
|
60
|
+
spawn,
|
|
61
|
+
"curl",
|
|
62
|
+
[
|
|
63
|
+
"-sS",
|
|
64
|
+
"--max-time",
|
|
65
|
+
seconds,
|
|
66
|
+
"-X",
|
|
67
|
+
"POST",
|
|
68
|
+
"-H",
|
|
69
|
+
`@${headerFile}`,
|
|
70
|
+
"--data-binary",
|
|
71
|
+
"@-",
|
|
72
|
+
"-w",
|
|
73
|
+
"\n%{http_code} %{redirect_url}",
|
|
74
|
+
url,
|
|
75
|
+
],
|
|
76
|
+
body,
|
|
77
|
+
timeoutMs + 2000,
|
|
78
|
+
);
|
|
79
|
+
if (r.error) return { failure: failureCode(r.error), transport: "curl" };
|
|
80
|
+
if (r.status !== 0) {
|
|
81
|
+
return { failure: `curl-exit-${r.status}`, transport: "curl" };
|
|
82
|
+
}
|
|
83
|
+
const out = String(r.stdout ?? "");
|
|
84
|
+
const cut = out.lastIndexOf("\n");
|
|
85
|
+
const [code, location = ""] = out.slice(cut + 1).split(" ");
|
|
86
|
+
return {
|
|
87
|
+
status: Number(code) || 0,
|
|
88
|
+
body: out.slice(0, cut),
|
|
89
|
+
location: location || null,
|
|
90
|
+
transport: "curl",
|
|
91
|
+
};
|
|
92
|
+
} finally {
|
|
93
|
+
rmSync(dir, { recursive: true, force: true });
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* fetch first; on a transport failure, curl. Returns { status, body, location,
|
|
99
|
+
* transport } on any HTTP answer, or { failure, fetchFailure, transport } when
|
|
100
|
+
* neither path reached the host. `fetchFailure` keeps the original cause.
|
|
101
|
+
*/
|
|
102
|
+
export async function postJson({
|
|
103
|
+
url,
|
|
104
|
+
headers = {},
|
|
105
|
+
body,
|
|
106
|
+
timeoutMs,
|
|
107
|
+
fetchImpl = fetch,
|
|
108
|
+
curl = curlPost,
|
|
109
|
+
}) {
|
|
110
|
+
const ctrl = new AbortController();
|
|
111
|
+
const timer = setTimeout(() => ctrl.abort(), timeoutMs);
|
|
112
|
+
let fetchFailure;
|
|
113
|
+
try {
|
|
114
|
+
const res = await fetchImpl(url, {
|
|
115
|
+
method: "POST",
|
|
116
|
+
headers: { "Content-Type": "application/json", ...headers },
|
|
117
|
+
body,
|
|
118
|
+
redirect: "manual",
|
|
119
|
+
signal: ctrl.signal,
|
|
120
|
+
});
|
|
121
|
+
const text = await res.text().catch(() => "");
|
|
122
|
+
return {
|
|
123
|
+
status: res.status,
|
|
124
|
+
body: text,
|
|
125
|
+
location: res.headers.get("location"),
|
|
126
|
+
transport: "fetch",
|
|
127
|
+
};
|
|
128
|
+
} catch (err) {
|
|
129
|
+
fetchFailure = failureCode(err, ctrl.signal.aborted);
|
|
130
|
+
} finally {
|
|
131
|
+
clearTimeout(timer);
|
|
132
|
+
}
|
|
133
|
+
const viaCurl = await curl({
|
|
134
|
+
url,
|
|
135
|
+
headers: { "Content-Type": "application/json", ...headers },
|
|
136
|
+
body,
|
|
137
|
+
timeoutMs,
|
|
138
|
+
});
|
|
139
|
+
if (viaCurl.failure) return { failure: viaCurl.failure, fetchFailure, transport: "curl" };
|
|
140
|
+
return { ...viaCurl, fetchFailure };
|
|
141
|
+
}
|
|
@@ -31,6 +31,10 @@ export const COMMON_SKILLS = [
|
|
|
31
31
|
"security-review",
|
|
32
32
|
"valven-rules",
|
|
33
33
|
"e2e-testing",
|
|
34
|
+
"scenario-gap-analysis",
|
|
35
|
+
"test-result-triage",
|
|
36
|
+
"handle-request",
|
|
37
|
+
"fix-review-verdicts",
|
|
34
38
|
];
|
|
35
39
|
|
|
36
40
|
// The analyst plugin is the second always-enabled toolkit. It carries outside
|
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file _test-gaming.mjs - added lines that make a suite green without fixing the code.
|
|
3
|
+
*
|
|
4
|
+
* Three shapes, read from the added lines of a unified diff:
|
|
5
|
+
*
|
|
6
|
+
* environment-branch product code that asks whether it is under test
|
|
7
|
+
* (`navigator.webdriver`, `process.env.CI`, ProcessInfo
|
|
8
|
+
* and XCTest, `isRunningTests`, a test-harness check) and
|
|
9
|
+
* does something else when it is. The test then measures
|
|
10
|
+
* a path users never take. Blocking. Test files are
|
|
11
|
+
* exempt: a test asking about its own runner is normal.
|
|
12
|
+
* swallowed-error an empty catch, `.catch(() => {})`, `except: pass`, a
|
|
13
|
+
* statement-level Swift `try?`, a Kotlin `runCatching`
|
|
14
|
+
* whose failure nothing reads. The failure the test
|
|
15
|
+
* should have seen is discarded. One finding per
|
|
16
|
+
* occurrence.
|
|
17
|
+
* inflated-timeout a timeout raised to at least twice the removed value
|
|
18
|
+
* it replaces (the nearest removed timeout in the same
|
|
19
|
+
* hunk), or a new wait of 30s or more in a test file. A
|
|
20
|
+
* slow or racy path is waited out instead of fixed.
|
|
21
|
+
* Units follow the language: JavaScript and TypeScript
|
|
22
|
+
* timeouts are milliseconds, Swift and Python seconds,
|
|
23
|
+
* Kotlin and Java milliseconds; a Ms / Millis / Seconds
|
|
24
|
+
* suffix overrides.
|
|
25
|
+
*
|
|
26
|
+
* Only source files are read, and comment lines are skipped. Configuration
|
|
27
|
+
* (`*.config.*`, CI workflows, YAML) and documentation are out of scope:
|
|
28
|
+
* `retries: process.env.CI ? 2 : 0` in a Playwright config tunes the runner,
|
|
29
|
+
* it does not change what the product does.
|
|
30
|
+
*
|
|
31
|
+
* A grep finds the obvious forms only; the Phase 3 reviewers judge the rest
|
|
32
|
+
* under the same name. Every finding is reviewer-shaped (`test_integrity`
|
|
33
|
+
* tag, `test_gaming` signal) so triage adjudicates it like the other
|
|
34
|
+
* test-integrity findings.
|
|
35
|
+
*
|
|
36
|
+
* @module pipeline/scripts/_test-gaming
|
|
37
|
+
*/
|
|
38
|
+
|
|
39
|
+
import { isTestPath } from "./_handle-patterns.mjs";
|
|
40
|
+
import { parseUnifiedDiff } from "./_unified-diff.mjs";
|
|
41
|
+
|
|
42
|
+
const SOURCE = /\.(swift|m|mm|kt|kts|java|[cm]?[jt]sx?|vue|svelte|py|go|rb|dart|cs)$/i;
|
|
43
|
+
const CONFIG = /(^|\/)[^/]*\.config\.[^/]+$|(^|\/)\.github\//;
|
|
44
|
+
const COMMENT = /^\s*(\/\/|\/\*|\*|#|<!--|--)/;
|
|
45
|
+
|
|
46
|
+
const ENV_BRANCH = [
|
|
47
|
+
/\bnavigator\s*\.\s*webdriver\b/,
|
|
48
|
+
/\bprocess\.env(?:\.CI\b|\[\s*["']CI["']\s*\])/,
|
|
49
|
+
/\bprocess\.env(?:\.|\[\s*["'])(?:JEST_WORKER_ID|VITEST|CYPRESS|PLAYWRIGHT\w*)\b/,
|
|
50
|
+
/\bwindow\s*\.\s*(?:Cypress|__playwright\w*)\b/,
|
|
51
|
+
/\bProcessInfo\b.*\bXCTest/,
|
|
52
|
+
/\bXCTestConfigurationFilePath\b/,
|
|
53
|
+
/NSClassFromString\(\s*"XCTest/,
|
|
54
|
+
/\bisRunning(?:Unit|UI)?Tests?\b/,
|
|
55
|
+
/\bisRunningInTestHarness\b/,
|
|
56
|
+
/\bBuild\s*\.\s*FINGERPRINT\b.*robolectric/i,
|
|
57
|
+
];
|
|
58
|
+
|
|
59
|
+
const SWALLOWED = [
|
|
60
|
+
/\bcatch\s*(?:\([^()]*\))?\s*\{\s*\}/g,
|
|
61
|
+
/\.catch\(\s*(?:\([^()]*\)|[A-Za-z_$][\w$]*)?\s*=>\s*(?:\{\s*\}|null|undefined|void 0)\s*\)/g,
|
|
62
|
+
/\bexcept\b[^:\n]*:\s*pass\b/g,
|
|
63
|
+
];
|
|
64
|
+
const SWIFT_TRY = /^\s*(?:_\s*=\s*)?try\?\s/;
|
|
65
|
+
const KOTLIN_RUN_CATCHING = /\brunCatching\s*\{/;
|
|
66
|
+
const RESULT_READ =
|
|
67
|
+
/\.(onFailure|getOrThrow|getOrElse|getOrDefault|exceptionOrNull|fold|isFailure|isSuccess|recover)\b|=\s*runCatching|return\s+runCatching/;
|
|
68
|
+
|
|
69
|
+
const LONG_TEST_WAIT_SECONDS = 30;
|
|
70
|
+
const RAISE_FACTOR = 2;
|
|
71
|
+
|
|
72
|
+
const TIMEOUT_VALUE = [
|
|
73
|
+
/\b(?:set|waitFor)?[Tt]imeout(Interval|Ms|Millis|Milliseconds|Seconds)?\b\s*(?:[:=]|\()\s*(\d[\d_]*(?:\.\d+)?)/g,
|
|
74
|
+
/\bsetTimeout\(\s*[^,()]+(?:\([^()]*\))?\s*,\s*()(\d[\d_]*)/g,
|
|
75
|
+
/\bwithTimeout\(\s*()(\d[\d_]*)/g,
|
|
76
|
+
];
|
|
77
|
+
|
|
78
|
+
/** Seconds or milliseconds for a bare timeout number, by the file's language. */
|
|
79
|
+
function bareUnitIsMs(file) {
|
|
80
|
+
return /\.([cm]?[jt]sx?|vue|svelte|kt|kts|java|dart|cs)$/i.test(file);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** Timeout values on a line, in seconds. */
|
|
84
|
+
export function timeoutsOn(line, file = "") {
|
|
85
|
+
const out = [];
|
|
86
|
+
for (const re of TIMEOUT_VALUE) {
|
|
87
|
+
re.lastIndex = 0;
|
|
88
|
+
for (const m of line.matchAll(re)) {
|
|
89
|
+
const raw = Number(m[2].replace(/_/g, ""));
|
|
90
|
+
if (!Number.isFinite(raw)) continue;
|
|
91
|
+
const unit = m[1] || "";
|
|
92
|
+
const ms = unit === "Seconds" ? false : /^M/.test(unit) || bareUnitIsMs(file);
|
|
93
|
+
out.push(ms ? raw / 1000 : raw);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
return out;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** Runs of consecutive added lines, so a pattern may span a line break. */
|
|
100
|
+
function segments(addedAt) {
|
|
101
|
+
const runs = [];
|
|
102
|
+
for (const a of addedAt) {
|
|
103
|
+
const last = runs[runs.length - 1];
|
|
104
|
+
if (last && last[last.length - 1].line + 1 === a.line) last.push(a);
|
|
105
|
+
else runs.push([a]);
|
|
106
|
+
}
|
|
107
|
+
return runs;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function finding(file, line, kind, severity, evidence, message) {
|
|
111
|
+
return {
|
|
112
|
+
file,
|
|
113
|
+
line,
|
|
114
|
+
severity,
|
|
115
|
+
tag: "test_integrity",
|
|
116
|
+
signal: "test_gaming",
|
|
117
|
+
kind,
|
|
118
|
+
evidence: evidence.trim().slice(0, 200),
|
|
119
|
+
message,
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
const swallowedMessage = (file, line) =>
|
|
124
|
+
`An error is caught and discarded at ${file}:${line}. Whatever the test should have seen fail is hidden; handle it or let it propagate.`;
|
|
125
|
+
|
|
126
|
+
/** Swallowed errors in a run of consecutive added lines, one per occurrence. */
|
|
127
|
+
function swallowedIn(file, run) {
|
|
128
|
+
const out = [];
|
|
129
|
+
const joined = run.map((a) => (COMMENT.test(a.text) ? "" : a.text)).join("\n");
|
|
130
|
+
const lineAt = (index) => run[joined.slice(0, index).split("\n").length - 1].line;
|
|
131
|
+
for (const re of SWALLOWED) {
|
|
132
|
+
re.lastIndex = 0;
|
|
133
|
+
for (const m of joined.matchAll(re)) {
|
|
134
|
+
out.push({ line: lineAt(m.index), evidence: m[0] });
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
for (const a of run) {
|
|
138
|
+
if (COMMENT.test(a.text)) continue;
|
|
139
|
+
if (/\.swift$/i.test(file) && SWIFT_TRY.test(a.text))
|
|
140
|
+
out.push({ line: a.line, evidence: a.text });
|
|
141
|
+
if (/\.kts?$/i.test(file) && KOTLIN_RUN_CATCHING.test(a.text) && !RESULT_READ.test(a.text)) {
|
|
142
|
+
out.push({ line: a.line, evidence: a.text });
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
return out
|
|
146
|
+
.sort((x, y) => x.line - y.line)
|
|
147
|
+
.map((s) =>
|
|
148
|
+
finding(
|
|
149
|
+
file,
|
|
150
|
+
s.line,
|
|
151
|
+
"swallowed-error",
|
|
152
|
+
"important",
|
|
153
|
+
s.evidence,
|
|
154
|
+
swallowedMessage(file, s.line),
|
|
155
|
+
),
|
|
156
|
+
);
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/** Inflated timeouts in one hunk: each added value against the nearest removed one. */
|
|
160
|
+
function timeoutsIn(file, hunk, testFile) {
|
|
161
|
+
const out = [];
|
|
162
|
+
const removed = hunk.removed
|
|
163
|
+
.filter((d) => !COMMENT.test(d.text))
|
|
164
|
+
.flatMap((d) => timeoutsOn(d.text, file).map((s) => ({ line: d.line, s })));
|
|
165
|
+
hunk.added.forEach((a, i) => {
|
|
166
|
+
if (COMMENT.test(a.text)) return;
|
|
167
|
+
const values = timeoutsOn(a.text, file);
|
|
168
|
+
if (!values.length) return;
|
|
169
|
+
let near = null;
|
|
170
|
+
if (removed.length) {
|
|
171
|
+
const anchor = hunk.removed[Math.min(i, hunk.removed.length - 1)]?.line ?? a.line;
|
|
172
|
+
near = removed.reduce((best, r) =>
|
|
173
|
+
Math.abs(r.line - anchor) < Math.abs(best.line - anchor) ? r : best,
|
|
174
|
+
);
|
|
175
|
+
}
|
|
176
|
+
for (const seconds of values) {
|
|
177
|
+
const raised = near !== null && near.s > 0 && seconds >= near.s * RAISE_FACTOR;
|
|
178
|
+
const longNew = near === null && testFile && seconds >= LONG_TEST_WAIT_SECONDS;
|
|
179
|
+
if (!raised && !longNew) continue;
|
|
180
|
+
out.push(
|
|
181
|
+
finding(
|
|
182
|
+
file,
|
|
183
|
+
a.line,
|
|
184
|
+
"inflated-timeout",
|
|
185
|
+
"important",
|
|
186
|
+
a.text,
|
|
187
|
+
raised
|
|
188
|
+
? `Timeout at ${file}:${a.line} raised from ${near.s}s to ${seconds}s. Waiting longer hides a slow or racy path; find why it is slow.`
|
|
189
|
+
: `A ${seconds}s wait was added to a test at ${file}:${a.line}. Waiting longer hides a slow or racy path; wait for a condition instead.`,
|
|
190
|
+
),
|
|
191
|
+
);
|
|
192
|
+
break;
|
|
193
|
+
}
|
|
194
|
+
});
|
|
195
|
+
return out;
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
/**
|
|
199
|
+
* @param {string} diffText unified diff
|
|
200
|
+
* @returns {object[]} reviewer-shaped findings
|
|
201
|
+
*/
|
|
202
|
+
export function scanTestGaming(diffText) {
|
|
203
|
+
const findings = [];
|
|
204
|
+
for (const [file, f] of parseUnifiedDiff(diffText)) {
|
|
205
|
+
if (!SOURCE.test(file) || CONFIG.test(file)) continue;
|
|
206
|
+
const testFile = isTestPath(file);
|
|
207
|
+
|
|
208
|
+
if (!testFile) {
|
|
209
|
+
for (const { line, text } of f.addedAt) {
|
|
210
|
+
if (COMMENT.test(text) || !ENV_BRANCH.some((re) => re.test(text))) continue;
|
|
211
|
+
findings.push(
|
|
212
|
+
finding(
|
|
213
|
+
file,
|
|
214
|
+
line,
|
|
215
|
+
"environment-branch",
|
|
216
|
+
"blocking",
|
|
217
|
+
text,
|
|
218
|
+
`Product code at ${file}:${line} checks whether it runs under test. A test that passes through this branch measures a path users never take; fix the code the test exercises instead.`,
|
|
219
|
+
),
|
|
220
|
+
);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
for (const run of segments(f.addedAt)) findings.push(...swallowedIn(file, run));
|
|
224
|
+
for (const hunk of f.hunks) findings.push(...timeoutsIn(file, hunk, testFile));
|
|
225
|
+
}
|
|
226
|
+
return findings;
|
|
227
|
+
}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file _unified-diff.mjs - a unified diff split into files, hunks and lines.
|
|
3
|
+
*
|
|
4
|
+
* Hunk bodies are read by the line counts in their `@@ -a,b +c,d @@` header,
|
|
5
|
+
* so a removed line that starts with "--" or an added line that starts with
|
|
6
|
+
* "++" is never mistaken for the next file's header.
|
|
7
|
+
*
|
|
8
|
+
* @module pipeline/scripts/_unified-diff
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* @typedef {{line: number, text: string}} DiffLine
|
|
13
|
+
* @typedef {{removed: DiffLine[], added: DiffLine[]}} Hunk
|
|
14
|
+
* @typedef {{added: string[], addedAt: DiffLine[], removed: string[], removedAt: DiffLine[],
|
|
15
|
+
* hunks: Hunk[], isNew: boolean, isDeleted: boolean, binary: boolean}} DiffFile
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
const HUNK = /^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@/;
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Split a unified diff into files. Works with any context size; -U0 is what
|
|
22
|
+
* the callers produce. Line numbers come from the hunk headers: `addedAt` in
|
|
23
|
+
* the new file, `removedAt` in the old one.
|
|
24
|
+
*
|
|
25
|
+
* @param {string} text
|
|
26
|
+
* @returns {Map<string, DiffFile>}
|
|
27
|
+
*/
|
|
28
|
+
export function parseUnifiedDiff(text) {
|
|
29
|
+
const files = new Map();
|
|
30
|
+
let cur = null;
|
|
31
|
+
const open = (name) => {
|
|
32
|
+
if (!files.has(name)) {
|
|
33
|
+
files.set(name, {
|
|
34
|
+
added: [],
|
|
35
|
+
addedAt: [],
|
|
36
|
+
removed: [],
|
|
37
|
+
removedAt: [],
|
|
38
|
+
hunks: [],
|
|
39
|
+
isNew: false,
|
|
40
|
+
isDeleted: false,
|
|
41
|
+
binary: false,
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
return files.get(name);
|
|
45
|
+
};
|
|
46
|
+
const lines = String(text).split("\n");
|
|
47
|
+
for (let i = 0; i < lines.length; i++) {
|
|
48
|
+
const line = lines[i];
|
|
49
|
+
const header = /^diff --git a\/(.+?) b\/(.+)$/.exec(line);
|
|
50
|
+
if (header) {
|
|
51
|
+
cur = open(header[2]);
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
54
|
+
if (line.startsWith("+++ ") || line.startsWith("--- ")) {
|
|
55
|
+
const side = line.slice(4).replace(/\t.*$/, "");
|
|
56
|
+
if (line.startsWith("+++ ") && side !== "/dev/null") {
|
|
57
|
+
const name = side.replace(/^b\//, "");
|
|
58
|
+
if (!cur || !files.has(name)) cur = open(name);
|
|
59
|
+
}
|
|
60
|
+
if (side === "/dev/null" && cur) {
|
|
61
|
+
if (line.startsWith("--- ")) cur.isNew = true;
|
|
62
|
+
else cur.isDeleted = true;
|
|
63
|
+
}
|
|
64
|
+
continue;
|
|
65
|
+
}
|
|
66
|
+
if (!cur) continue;
|
|
67
|
+
if (line.startsWith("new file mode")) cur.isNew = true;
|
|
68
|
+
else if (line.startsWith("deleted file mode")) cur.isDeleted = true;
|
|
69
|
+
else if (line.startsWith("Binary files")) cur.binary = true;
|
|
70
|
+
const h = HUNK.exec(line);
|
|
71
|
+
if (!h) continue;
|
|
72
|
+
let oldLeft = h[2] === undefined ? 1 : Number(h[2]);
|
|
73
|
+
let newLeft = h[4] === undefined ? 1 : Number(h[4]);
|
|
74
|
+
let oldNo = Number(h[1]);
|
|
75
|
+
let newNo = Number(h[3]);
|
|
76
|
+
const hunk = { removed: [], added: [] };
|
|
77
|
+
cur.hunks.push(hunk);
|
|
78
|
+
while ((oldLeft > 0 || newLeft > 0) && i + 1 < lines.length) {
|
|
79
|
+
const body = lines[++i];
|
|
80
|
+
if (body.startsWith("\\")) continue;
|
|
81
|
+
const mark = body[0];
|
|
82
|
+
const rest = body.slice(1);
|
|
83
|
+
if (mark === "-" && oldLeft > 0) {
|
|
84
|
+
const d = { line: oldNo++, text: rest };
|
|
85
|
+
hunk.removed.push(d);
|
|
86
|
+
cur.removed.push(rest);
|
|
87
|
+
cur.removedAt.push(d);
|
|
88
|
+
oldLeft--;
|
|
89
|
+
} else if (mark === "+" && newLeft > 0) {
|
|
90
|
+
const d = { line: newNo++, text: rest };
|
|
91
|
+
hunk.added.push(d);
|
|
92
|
+
cur.added.push(rest);
|
|
93
|
+
cur.addedAt.push(d);
|
|
94
|
+
newLeft--;
|
|
95
|
+
} else {
|
|
96
|
+
oldNo++;
|
|
97
|
+
newNo++;
|
|
98
|
+
oldLeft--;
|
|
99
|
+
newLeft--;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
return files;
|
|
104
|
+
}
|