@guilz-dev/sdlc-gh 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/CODEOWNERS +5 -0
- package/.github/ISSUE_TEMPLATE/bug_report.yml +68 -0
- package/.github/ISSUE_TEMPLATE/config.yml +1 -0
- package/.github/ISSUE_TEMPLATE/feature_request.yml +39 -0
- package/.github/ISSUE_TEMPLATE/support.yml +56 -0
- package/.github/ISSUE_TEMPLATE/task.yml +89 -0
- package/.github/agents/implementer.agent.md +17 -0
- package/.github/agents/reviewer.agent.md +18 -0
- package/.github/agents/triager.agent.md +13 -0
- package/.github/aw/actions-lock.json +9 -0
- package/.github/copilot-instructions.md +35 -0
- package/.github/hooks/hooks.json +12 -0
- package/.github/instructions/core.instructions.md +11 -0
- package/.github/instructions/profiles/go.instructions.md +10 -0
- package/.github/instructions/profiles/php.instructions.md +11 -0
- package/.github/instructions/profiles/python.instructions.md +11 -0
- package/.github/instructions/profiles/ruby.instructions.md +11 -0
- package/.github/instructions/profiles/typescript.instructions.md +11 -0
- package/.github/labels.yml +55 -0
- package/.github/pull_request_template.md +33 -0
- package/.github/ruleset.example.json +33 -0
- package/.github/ruleset.harness-eval.example.json +29 -0
- package/.github/skills/quality-loop/SKILL.md +23 -0
- package/.github/workflows/agent-retry-orchestrator.yml +161 -0
- package/.github/workflows/copilot-setup-steps.yml +64 -0
- package/.github/workflows/eval-ci.yml +169 -0
- package/.github/workflows/eval-drift.yml +75 -0
- package/.github/workflows/gh-aw-dogfood-ci.yml +73 -0
- package/.github/workflows/harness-ci.yml +244 -0
- package/.github/workflows/harness-sync.yml +28 -0
- package/.github/workflows/l1-readiness-check.yml +45 -0
- package/.github/workflows/labels-sync.yml +24 -0
- package/.github/workflows/nightly-harness-review.lock.yml +1643 -0
- package/.github/workflows/nightly-harness-review.md +87 -0
- package/.github/workflows/nightly-harness-review.yml +63 -0
- package/.github/workflows/npm-publish.yml +49 -0
- package/.github/workflows/pr-context-comment.yml +138 -0
- package/.github/workflows/product-ci-go.yml +33 -0
- package/.github/workflows/product-ci-php.yml +39 -0
- package/.github/workflows/product-ci-python.yml +34 -0
- package/.github/workflows/product-ci-ruby.yml +35 -0
- package/.github/workflows/product-ci-ts.yml +37 -0
- package/.github/workflows/task-issue-label-sync.yml +50 -0
- package/.github/workflows/weekly-redteam.lock.yml +1571 -0
- package/.github/workflows/weekly-redteam.md +76 -0
- package/.github/zizmor.yml +11 -0
- package/AGENTS.md +54 -0
- package/LICENSE +21 -0
- package/README.md +366 -0
- package/config/stacks.json +55 -0
- package/docs/adoption.md +126 -0
- package/docs/arch.md +535 -0
- package/docs/auth-boundaries.md +16 -0
- package/docs/coding-agent-l1.md +152 -0
- package/docs/exceptions/README.md +25 -0
- package/docs/exceptions/TEMPLATE.md +8 -0
- package/docs/failure-taxonomy.md +23 -0
- package/docs/gh-aw-dogfood.md +109 -0
- package/docs/kpi-baseline.md +9 -0
- package/docs/nightly-harness-review.md +94 -0
- package/docs/operations.md +108 -0
- package/docs/publishing.md +79 -0
- package/docs/revert-playbook.md +44 -0
- package/docs/shared-config.md +30 -0
- package/docs/telemetry-artifacts.md +78 -0
- package/docs/telemetry-schema.md +60 -0
- package/evals/.score-baseline.json +6 -0
- package/evals/e2e-bench/README.md +28 -0
- package/evals/e2e-bench/manifest.json +16 -0
- package/evals/e2e-bench/tasks/e2e-001.yml +10 -0
- package/evals/e2e-bench/tasks/e2e-002.yml +11 -0
- package/evals/e2e-bench/tasks/e2e-003.yml +10 -0
- package/evals/e2e-bench/tasks/e2e-004.yml +14 -0
- package/evals/e2e-bench/tasks/e2e-005.yml +11 -0
- package/evals/e2e-bench/tasks/e2e-006.yml +10 -0
- package/evals/e2e-bench/tasks/e2e-007.yml +10 -0
- package/evals/e2e-bench/tasks/e2e-008.yml +10 -0
- package/evals/e2e-bench/tasks/e2e-009.yml +10 -0
- package/evals/trajectories/rubric.md +12 -0
- package/evals/trajectories/test_harness_conventions.py +271 -0
- package/infra/README.md +49 -0
- package/infra/langfuse/docker-compose.yml +25 -0
- package/infra/otel/collector-config.yml +24 -0
- package/infra/samples/gh-aw-dogfood-report.json +44 -0
- package/infra/samples/harness-review-routing-plan.json +19 -0
- package/infra/samples/harness-review-summary.json +61 -0
- package/infra/samples/telemetry-artifact.json +29 -0
- package/infra/samples/telemetry-payload.json +19 -0
- package/package.json +85 -0
- package/prompts/triager-classify.prompt.yml +10 -0
- package/sample/go/add.go +5 -0
- package/sample/go/add_test.go +9 -0
- package/sample/go/go.mod +3 -0
- package/sample/php/composer.json +26 -0
- package/sample/php/composer.lock +1881 -0
- package/sample/php/phpunit.xml +8 -0
- package/sample/php/src/Add.php +13 -0
- package/sample/php/tests/AddTest.php +16 -0
- package/sample/python/requirements-dev.txt +2 -0
- package/sample/python/src/__init__.py +0 -0
- package/sample/python/src/greet.py +3 -0
- package/sample/python/tests/conftest.py +4 -0
- package/sample/python/tests/test_greet.py +5 -0
- package/sample/ruby/.rubocop.yml +10 -0
- package/sample/ruby/Gemfile +6 -0
- package/sample/ruby/Gemfile.lock +58 -0
- package/sample/ruby/lib/add.rb +9 -0
- package/sample/ruby/spec/add_spec.rb +11 -0
- package/sample/ts/biome.json +6 -0
- package/sample/ts/package-lock.json +1763 -0
- package/sample/ts/package.json +15 -0
- package/sample/ts/src/add.ts +3 -0
- package/sample/ts/tests/add.test.ts +8 -0
- package/sample/ts/tsconfig.json +12 -0
- package/scripts/aggregate-harness-review.mjs +48 -0
- package/scripts/bootstrap-harness.sh +411 -0
- package/scripts/check-diff-size.mjs +46 -0
- package/scripts/check-e2e-manifest.mjs +35 -0
- package/scripts/check-eval-score-drift.mjs +31 -0
- package/scripts/check-gh-aw-dogfood-scope.mjs +51 -0
- package/scripts/check-issue-spec.mjs +215 -0
- package/scripts/check-l1-readiness.mjs +82 -0
- package/scripts/check-open-pr-limit.mjs +34 -0
- package/scripts/doctor.mjs +177 -0
- package/scripts/emit-gh-aw-dogfood-report.mjs +112 -0
- package/scripts/emit-telemetry-artifact.mjs +99 -0
- package/scripts/fetch-telemetry-artifacts.mjs +176 -0
- package/scripts/harness-drift-report.mjs +99 -0
- package/scripts/lib/bootstrap-copy.mjs +123 -0
- package/scripts/lib/ccsd-contract.mjs +212 -0
- package/scripts/lib/diff-size.mjs +103 -0
- package/scripts/lib/doctor-local.mjs +179 -0
- package/scripts/lib/e2e-manifest.mjs +76 -0
- package/scripts/lib/gh-aw-dogfood.mjs +293 -0
- package/scripts/lib/github-config.mjs +94 -0
- package/scripts/lib/harness-ci-fragments.mjs +98 -0
- package/scripts/lib/harness-review-routing.mjs +244 -0
- package/scripts/lib/harness-review.mjs +388 -0
- package/scripts/lib/issue-form-label-sync.mjs +56 -0
- package/scripts/lib/l1-readiness.mjs +258 -0
- package/scripts/lib/merge-harness-package.mjs +36 -0
- package/scripts/lib/npm-package.mjs +129 -0
- package/scripts/lib/setup-wizard.mjs +224 -0
- package/scripts/lib/stacks.mjs +138 -0
- package/scripts/lib/telemetry-artifact.mjs +253 -0
- package/scripts/lib/template-root.mjs +39 -0
- package/scripts/merge-harness-package.mjs +14 -0
- package/scripts/route-harness-review.mjs +168 -0
- package/scripts/run-e2e-bench.mjs +216 -0
- package/scripts/sdlc-gh-cli.mjs +91 -0
- package/scripts/select-eval-jobs.mjs +41 -0
- package/scripts/setup-github.mjs +242 -0
- package/scripts/setup-github.sh +4 -0
- package/scripts/setup-wizard.mjs +426 -0
- package/scripts/test-bootstrap-guidance-scenarios.mjs +94 -0
- package/scripts/test-diff-size-scenarios.mjs +88 -0
- package/scripts/test-doctor-scenarios.mjs +70 -0
- package/scripts/test-e2e-manifest-scenarios.mjs +65 -0
- package/scripts/test-gh-aw-dogfood-scenarios.mjs +74 -0
- package/scripts/test-harness-review-routing-scenarios.mjs +130 -0
- package/scripts/test-harness-review-scenarios.mjs +92 -0
- package/scripts/test-hooks-scenarios.mjs +44 -0
- package/scripts/test-issue-form-label-sync-scenarios.mjs +48 -0
- package/scripts/test-issue-spec-scenarios.mjs +258 -0
- package/scripts/test-l1-readiness-scenarios.mjs +204 -0
- package/scripts/test-merge-harness-package-scenarios.mjs +53 -0
- package/scripts/test-npm-package-scenarios.mjs +31 -0
- package/scripts/test-sdlc-gh-cli-scenarios.mjs +54 -0
- package/scripts/test-setup-github-scenarios.mjs +103 -0
- package/scripts/test-setup-wizard-scenarios.mjs +114 -0
- package/scripts/test-telemetry-artifact-scenarios.mjs +69 -0
- package/scripts/trim-harness-ci.mjs +18 -0
- package/scripts/validate-gh-aw-compile.mjs +64 -0
- package/scripts/validate-harness.mjs +199 -0
- package/scripts/validate-telemetry.mjs +21 -0
- package/scripts/verify-bootstrap-stacks.sh +192 -0
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Canonical CC-SD contract — single source of truth for field names.
|
|
3
|
+
* Used by CI validation, regression tests, and template alignment checks.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
/** @type {readonly string[]} */
|
|
7
|
+
export const CCSD_REQUIRED_FIELDS = [
|
|
8
|
+
"Goal",
|
|
9
|
+
"Non-goals",
|
|
10
|
+
"Constraints",
|
|
11
|
+
"Acceptance criteria",
|
|
12
|
+
"Rollback hints",
|
|
13
|
+
];
|
|
14
|
+
|
|
15
|
+
/** @type {readonly string[]} */
|
|
16
|
+
export const CCSD_OPTIONAL_FIELDS = ["Additional context"];
|
|
17
|
+
|
|
18
|
+
/** @type {readonly string[]} */
|
|
19
|
+
export const CCSD_ALL_FIELDS = [...CCSD_REQUIRED_FIELDS, ...CCSD_OPTIONAL_FIELDS];
|
|
20
|
+
|
|
21
|
+
/** L1 task classes that require a complete CC-SD contract in v1. */
|
|
22
|
+
export const CCSD_ENFORCED_TASK_CLASSES = ["docs", "test-fix"];
|
|
23
|
+
|
|
24
|
+
/** PR template summary fields that mirror the Issue contract. */
|
|
25
|
+
export const CCSD_PR_SUMMARY_FIELDS = [
|
|
26
|
+
"Goal implemented",
|
|
27
|
+
"Non-goals preserved",
|
|
28
|
+
"Constraints handled",
|
|
29
|
+
"Acceptance criteria",
|
|
30
|
+
"Rollback",
|
|
31
|
+
];
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Placeholder text from the Issue template — treated as missing content.
|
|
35
|
+
* @type {readonly string[]}
|
|
36
|
+
*/
|
|
37
|
+
export const CCSD_PLACEHOLDER_SNIPPETS = [
|
|
38
|
+
"One short paragraph describing what this task achieves.",
|
|
39
|
+
"- Item the task must not do or change",
|
|
40
|
+
"- Technical or policy limits (stack, paths, time)",
|
|
41
|
+
"- [ ] Criterion 1",
|
|
42
|
+
"- [ ] Criterion 2",
|
|
43
|
+
"How to revert this change immediately if needed.",
|
|
44
|
+
];
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Parse GitHub Issue form body sections (### Label headers).
|
|
48
|
+
* @param {string} body
|
|
49
|
+
* @returns {Record<string, string>}
|
|
50
|
+
*/
|
|
51
|
+
export function parseIssueSections(body) {
|
|
52
|
+
const sections = {};
|
|
53
|
+
if (!body?.trim()) return sections;
|
|
54
|
+
|
|
55
|
+
const parts = body.split(/^### /m);
|
|
56
|
+
for (const part of parts.slice(1)) {
|
|
57
|
+
const newline = part.indexOf("\n");
|
|
58
|
+
if (newline === -1) continue;
|
|
59
|
+
const title = part.slice(0, newline).trim();
|
|
60
|
+
const content = part.slice(newline + 1).trim();
|
|
61
|
+
sections[title] = content;
|
|
62
|
+
}
|
|
63
|
+
return sections;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function normalizeLine(line) {
|
|
67
|
+
return line.replace(/\s+/g, " ").trim();
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
function isPlaceholderLine(line) {
|
|
71
|
+
const normalized = normalizeLine(line);
|
|
72
|
+
if (!normalized) return true;
|
|
73
|
+
return CCSD_PLACEHOLDER_SNIPPETS.some((snippet) => normalized === snippet);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* @param {string} content
|
|
78
|
+
* @returns {boolean}
|
|
79
|
+
*/
|
|
80
|
+
export function isPlaceholderContent(content) {
|
|
81
|
+
if (!content?.trim()) return true;
|
|
82
|
+
|
|
83
|
+
const stripped = content.replace(/<!--[\s\S]*?-->/g, "").trim();
|
|
84
|
+
if (!stripped) return true;
|
|
85
|
+
|
|
86
|
+
const lines = stripped
|
|
87
|
+
.split("\n")
|
|
88
|
+
.map((line) => line.trim())
|
|
89
|
+
.filter((line) => line && !/^\-\s*\[\s*\]\s*$/.test(line));
|
|
90
|
+
|
|
91
|
+
if (lines.length === 0) return true;
|
|
92
|
+
|
|
93
|
+
// Single paragraph (Goal, Rollback hints): exact match only — extra text is valid.
|
|
94
|
+
if (lines.length === 1 && !lines[0].startsWith("-")) {
|
|
95
|
+
return isPlaceholderLine(lines[0]);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// Bullet lists: placeholder-only when every remaining line is a template snippet.
|
|
99
|
+
return lines.every((line) => isPlaceholderLine(line));
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Validate CC-SD fields in an Issue body.
|
|
104
|
+
* @param {string} issueBody
|
|
105
|
+
* @returns {{ ok: boolean, missing: string[], placeholder: string[] }}
|
|
106
|
+
*/
|
|
107
|
+
export function validateCcsdFields(issueBody) {
|
|
108
|
+
const sections = parseIssueSections(issueBody);
|
|
109
|
+
const missing = [];
|
|
110
|
+
const placeholder = [];
|
|
111
|
+
|
|
112
|
+
for (const field of CCSD_REQUIRED_FIELDS) {
|
|
113
|
+
const content = sections[field];
|
|
114
|
+
if (!content?.trim()) {
|
|
115
|
+
missing.push(field);
|
|
116
|
+
} else if (isPlaceholderContent(content)) {
|
|
117
|
+
placeholder.push(field);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
return {
|
|
122
|
+
ok: missing.length === 0 && placeholder.length === 0,
|
|
123
|
+
missing,
|
|
124
|
+
placeholder,
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Whether CC-SD enforcement applies for the given labels.
|
|
130
|
+
* @param {string[]} labels
|
|
131
|
+
* @returns {boolean}
|
|
132
|
+
*/
|
|
133
|
+
export function shouldEnforceCcsd(labels) {
|
|
134
|
+
const normalized = labels.map((l) => l.trim());
|
|
135
|
+
const taskLabels = normalized
|
|
136
|
+
.filter((l) => l.startsWith("task:"))
|
|
137
|
+
.map((l) => l.replace(/^task:/, ""));
|
|
138
|
+
if (taskLabels.length !== 1) return false;
|
|
139
|
+
|
|
140
|
+
const isL1 = normalized.some((l) => l === "autonomy:L1");
|
|
141
|
+
return isL1 && CCSD_ENFORCED_TASK_CLASSES.includes(taskLabels[0]);
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* @param {string[]} labels
|
|
146
|
+
* @returns {{ ok: true } | { ok: false, message: string }}
|
|
147
|
+
*/
|
|
148
|
+
export function validateLabelShape(labels) {
|
|
149
|
+
const normalized = labels.map((l) => l.trim());
|
|
150
|
+
const taskLabels = normalized.filter((l) => l.startsWith("task:"));
|
|
151
|
+
const autonomyLabels = normalized.filter((l) => l.startsWith("autonomy:"));
|
|
152
|
+
|
|
153
|
+
if (taskLabels.length > 1) {
|
|
154
|
+
return {
|
|
155
|
+
ok: false,
|
|
156
|
+
message: `Issue has multiple task:* labels (${taskLabels.join(", ")}); keep exactly one`,
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
if (autonomyLabels.length > 1) {
|
|
160
|
+
return {
|
|
161
|
+
ok: false,
|
|
162
|
+
message: `Issue has multiple autonomy:* labels (${autonomyLabels.join(", ")}); keep exactly one`,
|
|
163
|
+
};
|
|
164
|
+
}
|
|
165
|
+
return { ok: true };
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/**
|
|
169
|
+
* When linked Issues cannot be fetched, fail only if proxy labels indicate L1 docs/test-fix.
|
|
170
|
+
* @param {string[]} proxyLabels - Issue labels, or PR labels as fallback
|
|
171
|
+
* @returns {"fail" | "warn_skip"}
|
|
172
|
+
*/
|
|
173
|
+
export function resolveFetchFailureAction(proxyLabels) {
|
|
174
|
+
return shouldEnforceCcsd(proxyLabels) ? "fail" : "warn_skip";
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* Extract issue numbers from PR closing keywords (fixes #N, closes #N, …).
|
|
179
|
+
* Bare #N mentions are ignored to avoid matching unrelated references.
|
|
180
|
+
* @param {string} body
|
|
181
|
+
* @returns {number[]}
|
|
182
|
+
*/
|
|
183
|
+
export function extractClosingIssueNumbers(body) {
|
|
184
|
+
if (!body) return [];
|
|
185
|
+
const numbers = new Set();
|
|
186
|
+
const pattern = /(?:close[sd]?|fix(?:e[sd])?|resolve[sd]?)\s+#(\d+)/gi;
|
|
187
|
+
for (const match of body.matchAll(pattern)) {
|
|
188
|
+
numbers.add(Number(match[1]));
|
|
189
|
+
}
|
|
190
|
+
return [...numbers];
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* Pick the Issue to validate when a PR links multiple Issues.
|
|
195
|
+
* Prefers the single L1 docs/test-fix candidate; flags ambiguity when several match.
|
|
196
|
+
* @param {{ body: string, labels: string[], issueNumber: number }[]} issues
|
|
197
|
+
* @returns {{ kind: "none" } | { kind: "ambiguous", issueNumbers: number[] } | { kind: "issue", body: string, labels: string[], issueNumber: number }}
|
|
198
|
+
*/
|
|
199
|
+
export function pickLinkedIssue(issues) {
|
|
200
|
+
if (!issues?.length) return { kind: "none" };
|
|
201
|
+
|
|
202
|
+
const enforced = issues.filter((issue) => shouldEnforceCcsd(issue.labels));
|
|
203
|
+
if (enforced.length > 1) {
|
|
204
|
+
return {
|
|
205
|
+
kind: "ambiguous",
|
|
206
|
+
issueNumbers: enforced.map((issue) => issue.issueNumber),
|
|
207
|
+
};
|
|
208
|
+
}
|
|
209
|
+
if (enforced.length === 1) return { kind: "issue", ...enforced[0] };
|
|
210
|
+
|
|
211
|
+
return { kind: "issue", ...issues[0] };
|
|
212
|
+
}
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
/** Pure helpers for diff-size / autonomy gate (testable without git). */
|
|
2
|
+
|
|
3
|
+
export const LIMITS = {
|
|
4
|
+
L1: { loc: 300, files: 8 },
|
|
5
|
+
L2: { loc: 120, files: 4 },
|
|
6
|
+
L3: { loc: 60, files: 2 },
|
|
7
|
+
};
|
|
8
|
+
|
|
9
|
+
export const SENSITIVE_PATH_PREFIXES = [".github/workflows/", "infra/"];
|
|
10
|
+
|
|
11
|
+
export function parseLabelInput(input) {
|
|
12
|
+
return String(input || "")
|
|
13
|
+
.split(",")
|
|
14
|
+
.map((s) => s.trim())
|
|
15
|
+
.filter(Boolean);
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export function resolveAutonomyLevel(labels) {
|
|
19
|
+
if (labels.some((l) => l.includes("autonomy:L3"))) return "L3";
|
|
20
|
+
if (labels.some((l) => l.includes("autonomy:L2"))) return "L2";
|
|
21
|
+
if (labels.some((l) => l.includes("autonomy:L0"))) return "L0";
|
|
22
|
+
return "L1";
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export function resolveLimits(level) {
|
|
26
|
+
if (level === "L0") return null;
|
|
27
|
+
return LIMITS[level] ?? LIMITS.L1;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export function resolveEnforcementMode(level, { l1HardFail = false } = {}) {
|
|
31
|
+
if (level === "L0") return "proposal-only";
|
|
32
|
+
if (level === "L2" || level === "L3") return "hard-fail";
|
|
33
|
+
if (level === "L1" && l1HardFail) return "hard-fail";
|
|
34
|
+
return "warn";
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export function aggregateDiffStats(numstatText) {
|
|
38
|
+
let add = 0;
|
|
39
|
+
let del = 0;
|
|
40
|
+
let files = 0;
|
|
41
|
+
|
|
42
|
+
for (const line of String(numstatText).split("\n").filter(Boolean)) {
|
|
43
|
+
const [a, d] = line.split("\t");
|
|
44
|
+
if (a === "-" && d === "-") continue;
|
|
45
|
+
add += Number(a) || 0;
|
|
46
|
+
del += Number(d) || 0;
|
|
47
|
+
files += 1;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
return { add, del, loc: add + del, files };
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export function isOverLimit(stats, limits) {
|
|
54
|
+
if (!limits) return false;
|
|
55
|
+
return stats.loc > limits.loc || stats.files > limits.files;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export function formatSummary(level, stats, limits) {
|
|
59
|
+
if (!limits) {
|
|
60
|
+
return `Autonomy: ${level} | LOC: ${stats.loc} | Files: ${stats.files} (proposal only)`;
|
|
61
|
+
}
|
|
62
|
+
return `Autonomy: ${level} | LOC: ${stats.loc}/${limits.loc} | Files: ${stats.files}/${limits.files}`;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export function formatOverLimitMessage(level) {
|
|
66
|
+
return `Change size exceeds ${level} limits`;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export function findSensitivePathWarnings(diffFiles, labels) {
|
|
70
|
+
const hasInfra = labels.some((l) => l.includes("task:infra"));
|
|
71
|
+
const warnings = [];
|
|
72
|
+
for (const file of diffFiles) {
|
|
73
|
+
if (SENSITIVE_PATH_PREFIXES.some((prefix) => file.startsWith(prefix)) && !hasInfra) {
|
|
74
|
+
warnings.push(`${file} changed without task:infra label`);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
return warnings;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export function evaluateDiffSize({
|
|
81
|
+
labels,
|
|
82
|
+
numstatText,
|
|
83
|
+
diffFiles = [],
|
|
84
|
+
l1HardFail = false,
|
|
85
|
+
}) {
|
|
86
|
+
const level = resolveAutonomyLevel(labels);
|
|
87
|
+
const limits = resolveLimits(level);
|
|
88
|
+
const mode = resolveEnforcementMode(level, { l1HardFail });
|
|
89
|
+
const stats = aggregateDiffStats(numstatText);
|
|
90
|
+
const overLimit = isOverLimit(stats, limits);
|
|
91
|
+
const sensitiveWarnings = findSensitivePathWarnings(diffFiles, labels);
|
|
92
|
+
|
|
93
|
+
return {
|
|
94
|
+
level,
|
|
95
|
+
limits,
|
|
96
|
+
mode,
|
|
97
|
+
stats,
|
|
98
|
+
overLimit,
|
|
99
|
+
sensitiveWarnings,
|
|
100
|
+
summary: formatSummary(level, stats, limits),
|
|
101
|
+
overLimitMessage: overLimit ? formatOverLimitMessage(level) : null,
|
|
102
|
+
};
|
|
103
|
+
}
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
import { existsSync, readdirSync, readFileSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { getStack } from "./stacks.mjs";
|
|
4
|
+
import { CODEOWNERS_PLACEHOLDER } from "./setup-wizard.mjs";
|
|
5
|
+
|
|
6
|
+
export function result(status, label, detail, fix = "") {
|
|
7
|
+
return { status, label, detail, fix };
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
/** @param {string} repoRoot */
|
|
11
|
+
export function listProductCiWorkflows(repoRoot) {
|
|
12
|
+
const workflowsDir = join(repoRoot, ".github/workflows");
|
|
13
|
+
if (!existsSync(workflowsDir)) return [];
|
|
14
|
+
return readdirSync(workflowsDir).filter((name) => /^product-ci-([^.]+)\.yml$/.test(name));
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/** @param {string} repoRoot */
|
|
18
|
+
export function inferStackFromProductCi(repoRoot) {
|
|
19
|
+
const workflows = listProductCiWorkflows(repoRoot);
|
|
20
|
+
if (workflows.length !== 1) return "";
|
|
21
|
+
const match = workflows[0].match(/^product-ci-([^.]+)\.yml$/);
|
|
22
|
+
return match?.[1] ?? "";
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** @param {string} repoRoot */
|
|
26
|
+
export function resolveStackId(repoRoot) {
|
|
27
|
+
const stackFile = join(repoRoot, ".harness-stack");
|
|
28
|
+
if (existsSync(stackFile)) {
|
|
29
|
+
const stackId = readFileSync(stackFile, "utf8").trim();
|
|
30
|
+
try {
|
|
31
|
+
getStack(stackId);
|
|
32
|
+
return stackId;
|
|
33
|
+
} catch {
|
|
34
|
+
return "";
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const inferred = inferStackFromProductCi(repoRoot);
|
|
39
|
+
if (!inferred) return "";
|
|
40
|
+
try {
|
|
41
|
+
getStack(inferred);
|
|
42
|
+
return inferred;
|
|
43
|
+
} catch {
|
|
44
|
+
return "";
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export function localChecks(repoRoot, { nodeVersion = process.versions.node, templateMode = false } = {}) {
|
|
49
|
+
const entries = [];
|
|
50
|
+
let stackId = "";
|
|
51
|
+
const stackFile = join(repoRoot, ".harness-stack");
|
|
52
|
+
|
|
53
|
+
if (!existsSync(stackFile)) {
|
|
54
|
+
const inferred = inferStackFromProductCi(repoRoot);
|
|
55
|
+
if (inferred) {
|
|
56
|
+
try {
|
|
57
|
+
getStack(inferred);
|
|
58
|
+
stackId = inferred;
|
|
59
|
+
entries.push(
|
|
60
|
+
result(
|
|
61
|
+
"PASS",
|
|
62
|
+
".harness-stack",
|
|
63
|
+
`inferred ${inferred} from product-ci-${inferred}.yml (local file is gitignored; optional: run setup-wizard to write it)`,
|
|
64
|
+
),
|
|
65
|
+
);
|
|
66
|
+
} catch {
|
|
67
|
+
entries.push(
|
|
68
|
+
result(
|
|
69
|
+
"FAIL",
|
|
70
|
+
".harness-stack",
|
|
71
|
+
`missing and product-ci-${inferred}.yml maps to unknown stack`,
|
|
72
|
+
"Run `./scripts/setup-wizard.mjs` or `./scripts/bootstrap-harness.sh`.",
|
|
73
|
+
),
|
|
74
|
+
);
|
|
75
|
+
}
|
|
76
|
+
} else {
|
|
77
|
+
entries.push(
|
|
78
|
+
result(
|
|
79
|
+
"FAIL",
|
|
80
|
+
".harness-stack",
|
|
81
|
+
"missing",
|
|
82
|
+
"Run `./scripts/setup-wizard.mjs` or `./scripts/bootstrap-harness.sh`.",
|
|
83
|
+
),
|
|
84
|
+
);
|
|
85
|
+
}
|
|
86
|
+
} else {
|
|
87
|
+
stackId = readFileSync(stackFile, "utf8").trim();
|
|
88
|
+
try {
|
|
89
|
+
getStack(stackId);
|
|
90
|
+
entries.push(result("PASS", ".harness-stack", `stack is ${stackId}`));
|
|
91
|
+
} catch {
|
|
92
|
+
entries.push(
|
|
93
|
+
result(
|
|
94
|
+
"FAIL",
|
|
95
|
+
".harness-stack",
|
|
96
|
+
`unknown stack value: ${stackId}`,
|
|
97
|
+
"Set .harness-stack to a supported stack id.",
|
|
98
|
+
),
|
|
99
|
+
);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
const workflowsDir = join(repoRoot, ".github/workflows");
|
|
104
|
+
const productWorkflows = existsSync(workflowsDir)
|
|
105
|
+
? readdirSync(workflowsDir).filter((name) => /^product-ci-.*\.yml$/.test(name))
|
|
106
|
+
: [];
|
|
107
|
+
if (productWorkflows.length === 1) {
|
|
108
|
+
entries.push(result("PASS", "product-ci workflow", `found ${productWorkflows[0]}`));
|
|
109
|
+
} else if (templateMode && productWorkflows.length > 1) {
|
|
110
|
+
entries.push(
|
|
111
|
+
result(
|
|
112
|
+
"PASS",
|
|
113
|
+
"product-ci workflow",
|
|
114
|
+
`template repo: ${productWorkflows.length} workflows (${productWorkflows.join(", ")})`,
|
|
115
|
+
),
|
|
116
|
+
);
|
|
117
|
+
} else {
|
|
118
|
+
entries.push(
|
|
119
|
+
result(
|
|
120
|
+
"FAIL",
|
|
121
|
+
"product-ci workflow",
|
|
122
|
+
`expected exactly 1 product-ci workflow, found ${productWorkflows.length}`,
|
|
123
|
+
"Re-run `./scripts/bootstrap-harness.sh` or `./scripts/setup-wizard.mjs --template`.",
|
|
124
|
+
),
|
|
125
|
+
);
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
const codeownersFile = join(repoRoot, ".github/CODEOWNERS");
|
|
129
|
+
if (!existsSync(codeownersFile)) {
|
|
130
|
+
entries.push(result("FAIL", "CODEOWNERS", "missing", "Re-run `./scripts/bootstrap-harness.sh`."));
|
|
131
|
+
} else {
|
|
132
|
+
const hasPlaceholder = readFileSync(codeownersFile, "utf8").includes(CODEOWNERS_PLACEHOLDER);
|
|
133
|
+
if (templateMode) {
|
|
134
|
+
if (hasPlaceholder) {
|
|
135
|
+
entries.push(result("PASS", "CODEOWNERS", "template placeholder preserved"));
|
|
136
|
+
} else {
|
|
137
|
+
entries.push(
|
|
138
|
+
result(
|
|
139
|
+
"FAIL",
|
|
140
|
+
"CODEOWNERS",
|
|
141
|
+
"template repo must keep placeholder owners",
|
|
142
|
+
`Restore ${CODEOWNERS_PLACEHOLDER} in .github/CODEOWNERS (do not commit personal or org-specific owners here).`,
|
|
143
|
+
),
|
|
144
|
+
);
|
|
145
|
+
}
|
|
146
|
+
} else if (hasPlaceholder) {
|
|
147
|
+
entries.push(
|
|
148
|
+
result(
|
|
149
|
+
"FAIL",
|
|
150
|
+
"CODEOWNERS",
|
|
151
|
+
"placeholder team still present",
|
|
152
|
+
"Run `./scripts/setup-wizard.mjs` or `./scripts/bootstrap-harness.sh --codeowners-team @org/team`.",
|
|
153
|
+
),
|
|
154
|
+
);
|
|
155
|
+
} else {
|
|
156
|
+
entries.push(result("PASS", "CODEOWNERS", "team placeholder replaced"));
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
const nodeMajor = Number.parseInt(nodeVersion.split(".")[0], 10);
|
|
161
|
+
if (nodeMajor >= 22) {
|
|
162
|
+
entries.push(result("PASS", "Node.js", `version ${nodeVersion}`));
|
|
163
|
+
} else {
|
|
164
|
+
entries.push(
|
|
165
|
+
result(
|
|
166
|
+
"FAIL",
|
|
167
|
+
"Node.js",
|
|
168
|
+
`version ${nodeVersion} is below 22`,
|
|
169
|
+
"Install Node.js 22+ for local parity with CI.",
|
|
170
|
+
),
|
|
171
|
+
);
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
if (!stackId) {
|
|
175
|
+
stackId = resolveStackId(repoRoot);
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
return { entries, stackId };
|
|
179
|
+
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/** Pure validation for e2e-bench manifest (testable without filesystem). */
|
|
2
|
+
|
|
3
|
+
export const SUPPORTED_CLASSES = [
|
|
4
|
+
"docs",
|
|
5
|
+
"test-fix",
|
|
6
|
+
"refactor",
|
|
7
|
+
"feature-small",
|
|
8
|
+
"dependency-bump",
|
|
9
|
+
"infra",
|
|
10
|
+
"security-sensitive",
|
|
11
|
+
];
|
|
12
|
+
|
|
13
|
+
const QUARTER_MS = 90 * 24 * 60 * 60 * 1000;
|
|
14
|
+
|
|
15
|
+
export function validateLastRotated(lastRotated) {
|
|
16
|
+
if (!lastRotated) {
|
|
17
|
+
return { valid: false, message: "manifest missing last_rotated" };
|
|
18
|
+
}
|
|
19
|
+
const parsed = new Date(lastRotated);
|
|
20
|
+
if (Number.isNaN(parsed.getTime())) {
|
|
21
|
+
return { valid: false, message: `invalid last_rotated: ${lastRotated}` };
|
|
22
|
+
}
|
|
23
|
+
const stale = Date.now() - parsed.getTime() > QUARTER_MS;
|
|
24
|
+
return { valid: true, stale, parsed };
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function validateManifest(manifest, taskFileIds = []) {
|
|
28
|
+
const errors = [];
|
|
29
|
+
const warnings = [];
|
|
30
|
+
const tasks = manifest.tasks || [];
|
|
31
|
+
const minTasks = manifest.min_tasks ?? 5;
|
|
32
|
+
const fileIdSet = new Set(taskFileIds);
|
|
33
|
+
|
|
34
|
+
if (tasks.length < minTasks) {
|
|
35
|
+
errors.push(`Need at least ${minTasks} e2e tasks (found ${tasks.length})`);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const rotation = validateLastRotated(manifest.last_rotated);
|
|
39
|
+
if (!rotation.valid) {
|
|
40
|
+
errors.push(rotation.message);
|
|
41
|
+
} else if (rotation.stale) {
|
|
42
|
+
warnings.push("E2E bench not rotated in 90 days — review manifest");
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
const seenIds = new Set();
|
|
46
|
+
const manifestIds = new Set();
|
|
47
|
+
|
|
48
|
+
for (const entry of tasks) {
|
|
49
|
+
const id = entry?.id;
|
|
50
|
+
if (!id) {
|
|
51
|
+
errors.push("Manifest task missing id");
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
54
|
+
if (seenIds.has(id)) {
|
|
55
|
+
errors.push(`Duplicate manifest task id: ${id}`);
|
|
56
|
+
}
|
|
57
|
+
seenIds.add(id);
|
|
58
|
+
manifestIds.add(id);
|
|
59
|
+
|
|
60
|
+
if (!fileIdSet.has(id)) {
|
|
61
|
+
errors.push(`Missing task file: evals/e2e-bench/tasks/${id}.yml`);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
if (entry.class && !SUPPORTED_CLASSES.includes(entry.class)) {
|
|
65
|
+
errors.push(`Unsupported task class for ${id}: ${entry.class}`);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
for (const fileId of taskFileIds) {
|
|
70
|
+
if (!manifestIds.has(fileId)) {
|
|
71
|
+
errors.push(`Orphan task file not listed in manifest: ${fileId}.yml`);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
return { errors, warnings, taskCount: tasks.length, minTasks };
|
|
76
|
+
}
|