@guilz-dev/sdlc-gh 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/CODEOWNERS +5 -0
- package/.github/ISSUE_TEMPLATE/bug_report.yml +68 -0
- package/.github/ISSUE_TEMPLATE/config.yml +1 -0
- package/.github/ISSUE_TEMPLATE/feature_request.yml +39 -0
- package/.github/ISSUE_TEMPLATE/support.yml +56 -0
- package/.github/ISSUE_TEMPLATE/task.yml +89 -0
- package/.github/agents/implementer.agent.md +17 -0
- package/.github/agents/reviewer.agent.md +18 -0
- package/.github/agents/triager.agent.md +13 -0
- package/.github/aw/actions-lock.json +9 -0
- package/.github/copilot-instructions.md +35 -0
- package/.github/hooks/hooks.json +12 -0
- package/.github/instructions/core.instructions.md +11 -0
- package/.github/instructions/profiles/go.instructions.md +10 -0
- package/.github/instructions/profiles/php.instructions.md +11 -0
- package/.github/instructions/profiles/python.instructions.md +11 -0
- package/.github/instructions/profiles/ruby.instructions.md +11 -0
- package/.github/instructions/profiles/typescript.instructions.md +11 -0
- package/.github/labels.yml +55 -0
- package/.github/pull_request_template.md +33 -0
- package/.github/ruleset.example.json +33 -0
- package/.github/ruleset.harness-eval.example.json +29 -0
- package/.github/skills/quality-loop/SKILL.md +23 -0
- package/.github/workflows/agent-retry-orchestrator.yml +161 -0
- package/.github/workflows/copilot-setup-steps.yml +64 -0
- package/.github/workflows/eval-ci.yml +169 -0
- package/.github/workflows/eval-drift.yml +75 -0
- package/.github/workflows/gh-aw-dogfood-ci.yml +73 -0
- package/.github/workflows/harness-ci.yml +244 -0
- package/.github/workflows/harness-sync.yml +28 -0
- package/.github/workflows/l1-readiness-check.yml +45 -0
- package/.github/workflows/labels-sync.yml +24 -0
- package/.github/workflows/nightly-harness-review.lock.yml +1643 -0
- package/.github/workflows/nightly-harness-review.md +87 -0
- package/.github/workflows/nightly-harness-review.yml +63 -0
- package/.github/workflows/npm-publish.yml +49 -0
- package/.github/workflows/pr-context-comment.yml +138 -0
- package/.github/workflows/product-ci-go.yml +33 -0
- package/.github/workflows/product-ci-php.yml +39 -0
- package/.github/workflows/product-ci-python.yml +34 -0
- package/.github/workflows/product-ci-ruby.yml +35 -0
- package/.github/workflows/product-ci-ts.yml +37 -0
- package/.github/workflows/task-issue-label-sync.yml +50 -0
- package/.github/workflows/weekly-redteam.lock.yml +1571 -0
- package/.github/workflows/weekly-redteam.md +76 -0
- package/.github/zizmor.yml +11 -0
- package/AGENTS.md +54 -0
- package/LICENSE +21 -0
- package/README.md +366 -0
- package/config/stacks.json +55 -0
- package/docs/adoption.md +126 -0
- package/docs/arch.md +535 -0
- package/docs/auth-boundaries.md +16 -0
- package/docs/coding-agent-l1.md +152 -0
- package/docs/exceptions/README.md +25 -0
- package/docs/exceptions/TEMPLATE.md +8 -0
- package/docs/failure-taxonomy.md +23 -0
- package/docs/gh-aw-dogfood.md +109 -0
- package/docs/kpi-baseline.md +9 -0
- package/docs/nightly-harness-review.md +94 -0
- package/docs/operations.md +108 -0
- package/docs/publishing.md +79 -0
- package/docs/revert-playbook.md +44 -0
- package/docs/shared-config.md +30 -0
- package/docs/telemetry-artifacts.md +78 -0
- package/docs/telemetry-schema.md +60 -0
- package/evals/.score-baseline.json +6 -0
- package/evals/e2e-bench/README.md +28 -0
- package/evals/e2e-bench/manifest.json +16 -0
- package/evals/e2e-bench/tasks/e2e-001.yml +10 -0
- package/evals/e2e-bench/tasks/e2e-002.yml +11 -0
- package/evals/e2e-bench/tasks/e2e-003.yml +10 -0
- package/evals/e2e-bench/tasks/e2e-004.yml +14 -0
- package/evals/e2e-bench/tasks/e2e-005.yml +11 -0
- package/evals/e2e-bench/tasks/e2e-006.yml +10 -0
- package/evals/e2e-bench/tasks/e2e-007.yml +10 -0
- package/evals/e2e-bench/tasks/e2e-008.yml +10 -0
- package/evals/e2e-bench/tasks/e2e-009.yml +10 -0
- package/evals/trajectories/rubric.md +12 -0
- package/evals/trajectories/test_harness_conventions.py +271 -0
- package/infra/README.md +49 -0
- package/infra/langfuse/docker-compose.yml +25 -0
- package/infra/otel/collector-config.yml +24 -0
- package/infra/samples/gh-aw-dogfood-report.json +44 -0
- package/infra/samples/harness-review-routing-plan.json +19 -0
- package/infra/samples/harness-review-summary.json +61 -0
- package/infra/samples/telemetry-artifact.json +29 -0
- package/infra/samples/telemetry-payload.json +19 -0
- package/package.json +85 -0
- package/prompts/triager-classify.prompt.yml +10 -0
- package/sample/go/add.go +5 -0
- package/sample/go/add_test.go +9 -0
- package/sample/go/go.mod +3 -0
- package/sample/php/composer.json +26 -0
- package/sample/php/composer.lock +1881 -0
- package/sample/php/phpunit.xml +8 -0
- package/sample/php/src/Add.php +13 -0
- package/sample/php/tests/AddTest.php +16 -0
- package/sample/python/requirements-dev.txt +2 -0
- package/sample/python/src/__init__.py +0 -0
- package/sample/python/src/greet.py +3 -0
- package/sample/python/tests/conftest.py +4 -0
- package/sample/python/tests/test_greet.py +5 -0
- package/sample/ruby/.rubocop.yml +10 -0
- package/sample/ruby/Gemfile +6 -0
- package/sample/ruby/Gemfile.lock +58 -0
- package/sample/ruby/lib/add.rb +9 -0
- package/sample/ruby/spec/add_spec.rb +11 -0
- package/sample/ts/biome.json +6 -0
- package/sample/ts/package-lock.json +1763 -0
- package/sample/ts/package.json +15 -0
- package/sample/ts/src/add.ts +3 -0
- package/sample/ts/tests/add.test.ts +8 -0
- package/sample/ts/tsconfig.json +12 -0
- package/scripts/aggregate-harness-review.mjs +48 -0
- package/scripts/bootstrap-harness.sh +411 -0
- package/scripts/check-diff-size.mjs +46 -0
- package/scripts/check-e2e-manifest.mjs +35 -0
- package/scripts/check-eval-score-drift.mjs +31 -0
- package/scripts/check-gh-aw-dogfood-scope.mjs +51 -0
- package/scripts/check-issue-spec.mjs +215 -0
- package/scripts/check-l1-readiness.mjs +82 -0
- package/scripts/check-open-pr-limit.mjs +34 -0
- package/scripts/doctor.mjs +177 -0
- package/scripts/emit-gh-aw-dogfood-report.mjs +112 -0
- package/scripts/emit-telemetry-artifact.mjs +99 -0
- package/scripts/fetch-telemetry-artifacts.mjs +176 -0
- package/scripts/harness-drift-report.mjs +99 -0
- package/scripts/lib/bootstrap-copy.mjs +123 -0
- package/scripts/lib/ccsd-contract.mjs +212 -0
- package/scripts/lib/diff-size.mjs +103 -0
- package/scripts/lib/doctor-local.mjs +179 -0
- package/scripts/lib/e2e-manifest.mjs +76 -0
- package/scripts/lib/gh-aw-dogfood.mjs +293 -0
- package/scripts/lib/github-config.mjs +94 -0
- package/scripts/lib/harness-ci-fragments.mjs +98 -0
- package/scripts/lib/harness-review-routing.mjs +244 -0
- package/scripts/lib/harness-review.mjs +388 -0
- package/scripts/lib/issue-form-label-sync.mjs +56 -0
- package/scripts/lib/l1-readiness.mjs +258 -0
- package/scripts/lib/merge-harness-package.mjs +36 -0
- package/scripts/lib/npm-package.mjs +129 -0
- package/scripts/lib/setup-wizard.mjs +224 -0
- package/scripts/lib/stacks.mjs +138 -0
- package/scripts/lib/telemetry-artifact.mjs +253 -0
- package/scripts/lib/template-root.mjs +39 -0
- package/scripts/merge-harness-package.mjs +14 -0
- package/scripts/route-harness-review.mjs +168 -0
- package/scripts/run-e2e-bench.mjs +216 -0
- package/scripts/sdlc-gh-cli.mjs +91 -0
- package/scripts/select-eval-jobs.mjs +41 -0
- package/scripts/setup-github.mjs +242 -0
- package/scripts/setup-github.sh +4 -0
- package/scripts/setup-wizard.mjs +426 -0
- package/scripts/test-bootstrap-guidance-scenarios.mjs +94 -0
- package/scripts/test-diff-size-scenarios.mjs +88 -0
- package/scripts/test-doctor-scenarios.mjs +70 -0
- package/scripts/test-e2e-manifest-scenarios.mjs +65 -0
- package/scripts/test-gh-aw-dogfood-scenarios.mjs +74 -0
- package/scripts/test-harness-review-routing-scenarios.mjs +130 -0
- package/scripts/test-harness-review-scenarios.mjs +92 -0
- package/scripts/test-hooks-scenarios.mjs +44 -0
- package/scripts/test-issue-form-label-sync-scenarios.mjs +48 -0
- package/scripts/test-issue-spec-scenarios.mjs +258 -0
- package/scripts/test-l1-readiness-scenarios.mjs +204 -0
- package/scripts/test-merge-harness-package-scenarios.mjs +53 -0
- package/scripts/test-npm-package-scenarios.mjs +31 -0
- package/scripts/test-sdlc-gh-cli-scenarios.mjs +54 -0
- package/scripts/test-setup-github-scenarios.mjs +103 -0
- package/scripts/test-setup-wizard-scenarios.mjs +114 -0
- package/scripts/test-telemetry-artifact-scenarios.mjs +69 -0
- package/scripts/trim-harness-ci.mjs +18 -0
- package/scripts/validate-gh-aw-compile.mjs +64 -0
- package/scripts/validate-harness.mjs +199 -0
- package/scripts/validate-telemetry.mjs +21 -0
- package/scripts/verify-bootstrap-stacks.sh +192 -0
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "harness-sample-ts",
|
|
3
|
+
"private": true,
|
|
4
|
+
"type": "module",
|
|
5
|
+
"scripts": {
|
|
6
|
+
"lint": "biome check src tests",
|
|
7
|
+
"typecheck": "tsc --noEmit",
|
|
8
|
+
"test": "vitest run"
|
|
9
|
+
},
|
|
10
|
+
"devDependencies": {
|
|
11
|
+
"@biomejs/biome": "^1.9.0",
|
|
12
|
+
"typescript": "^5.7.0",
|
|
13
|
+
"vitest": "^3.0.0"
|
|
14
|
+
}
|
|
15
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Aggregate telemetry artifacts and emit nightly harness review summary.
|
|
4
|
+
*/
|
|
5
|
+
import { appendFileSync, mkdirSync, writeFileSync } from "node:fs";
|
|
6
|
+
import { join } from "node:path";
|
|
7
|
+
import { pathToFileURL } from "node:url";
|
|
8
|
+
import {
|
|
9
|
+
buildHarnessReviewSummary,
|
|
10
|
+
formatHarnessReviewMarkdown,
|
|
11
|
+
REVIEW_OUT_DIR,
|
|
12
|
+
} from "./lib/harness-review.mjs";
|
|
13
|
+
import { DEFAULT_COLLECT_DIR, loadTelemetryJsonFiles } from "./fetch-telemetry-artifacts.mjs";
|
|
14
|
+
|
|
15
|
+
function main() {
|
|
16
|
+
const inputDir = process.env.TELEMETRY_COLLECT_DIR || DEFAULT_COLLECT_DIR;
|
|
17
|
+
const outDir = process.env.HARNESS_REVIEW_OUT_DIR || REVIEW_OUT_DIR;
|
|
18
|
+
const windowHours = Number(process.env.WINDOW_HOURS || 24);
|
|
19
|
+
const repo = process.env.GITHUB_REPOSITORY || "unknown/unknown";
|
|
20
|
+
|
|
21
|
+
const records = loadTelemetryJsonFiles(inputDir);
|
|
22
|
+
const summary = buildHarnessReviewSummary(records, {
|
|
23
|
+
repo,
|
|
24
|
+
windowHours,
|
|
25
|
+
});
|
|
26
|
+
const markdown = formatHarnessReviewMarkdown(summary);
|
|
27
|
+
|
|
28
|
+
mkdirSync(outDir, { recursive: true });
|
|
29
|
+
const jsonPath = join(outDir, "harness-review-summary.json");
|
|
30
|
+
const mdPath = join(outDir, "harness-review-summary.md");
|
|
31
|
+
writeFileSync(jsonPath, `${JSON.stringify(summary, null, 2)}\n`, "utf8");
|
|
32
|
+
writeFileSync(mdPath, markdown, "utf8");
|
|
33
|
+
|
|
34
|
+
console.log(`Wrote ${jsonPath}`);
|
|
35
|
+
console.log(`Wrote ${mdPath}`);
|
|
36
|
+
console.log(
|
|
37
|
+
`::notice::classified_failure_groups=${summary.rollup.failure_groups} telemetry_records=${summary.rollup.telemetry_records}`,
|
|
38
|
+
);
|
|
39
|
+
|
|
40
|
+
const stepSummaryPath = process.env.GITHUB_STEP_SUMMARY;
|
|
41
|
+
if (stepSummaryPath) {
|
|
42
|
+
appendFileSync(stepSummaryPath, `${markdown}\n`, "utf8");
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
const isMain =
|
|
47
|
+
process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href;
|
|
48
|
+
if (isMain) main();
|
|
@@ -0,0 +1,411 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
set -euo pipefail
|
|
3
|
+
|
|
4
|
+
REPO=""
|
|
5
|
+
STACK=""
|
|
6
|
+
MODE=""
|
|
7
|
+
CODEOWNERS_TEAM=""
|
|
8
|
+
YES=0
|
|
9
|
+
TEMPLATE_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
|
10
|
+
|
|
11
|
+
stack_field() {
|
|
12
|
+
node --input-type=module -e "
|
|
13
|
+
import { getStack } from 'file://${TEMPLATE_ROOT}/scripts/lib/stacks.mjs';
|
|
14
|
+
const s = getStack(process.argv[1]);
|
|
15
|
+
console.log(s[process.argv[2]]);
|
|
16
|
+
" "$STACK" "$1"
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
stack_ids() {
|
|
20
|
+
node --input-type=module -e "
|
|
21
|
+
import { stackIds } from 'file://${TEMPLATE_ROOT}/scripts/lib/stacks.mjs';
|
|
22
|
+
console.log(stackIds().join(' '));
|
|
23
|
+
"
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
usage() {
|
|
27
|
+
local ids
|
|
28
|
+
ids="$(stack_ids | tr ' ' '|')"
|
|
29
|
+
cat <<EOF
|
|
30
|
+
Usage: $0 [--repo <path>] [--stack <${ids}>] [--mode new|existing] [--codeowners-team @org/team] [--yes]
|
|
31
|
+
EOF
|
|
32
|
+
exit 1
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
validate_stack() {
|
|
36
|
+
node --input-type=module -e "
|
|
37
|
+
import { getStack } from 'file://${TEMPLATE_ROOT}/scripts/lib/stacks.mjs';
|
|
38
|
+
getStack(process.argv[1]);
|
|
39
|
+
" "$1" >/dev/null 2>&1
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
detect_repo() {
|
|
43
|
+
if [[ -n "$REPO" ]]; then
|
|
44
|
+
return
|
|
45
|
+
fi
|
|
46
|
+
|
|
47
|
+
local git_root=""
|
|
48
|
+
git_root="$(git rev-parse --show-toplevel 2>/dev/null || true)"
|
|
49
|
+
if [[ -n "$git_root" ]] && [[ "$(cd "$git_root" && pwd)" != "$TEMPLATE_ROOT" ]]; then
|
|
50
|
+
REPO="$git_root"
|
|
51
|
+
else
|
|
52
|
+
REPO="$PWD"
|
|
53
|
+
fi
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
detect_stack_json() {
|
|
57
|
+
node --input-type=module -e "
|
|
58
|
+
import { detectStackCandidates } from 'file://${TEMPLATE_ROOT}/scripts/lib/stacks.mjs';
|
|
59
|
+
console.log(JSON.stringify(detectStackCandidates(process.argv[1])));
|
|
60
|
+
" "$REPO"
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
stack_suggested() {
|
|
64
|
+
node --input-type=module -e "
|
|
65
|
+
const input = JSON.parse(process.argv[1]);
|
|
66
|
+
console.log(input.suggested ?? '');
|
|
67
|
+
" "$1"
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
stack_candidates() {
|
|
71
|
+
node --input-type=module -e "
|
|
72
|
+
const input = JSON.parse(process.argv[1]);
|
|
73
|
+
const ids = [...new Set([...input.rootMatches, ...input.nestedMatches].map((m) => m.stackId))];
|
|
74
|
+
console.log(ids.join(' '));
|
|
75
|
+
" "$1"
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
stack_summary() {
|
|
79
|
+
node --input-type=module -e "
|
|
80
|
+
const input = JSON.parse(process.argv[1]);
|
|
81
|
+
for (const match of input.rootMatches) console.log(\`root:\${match.stackId}:\${match.path}\`);
|
|
82
|
+
for (const match of input.nestedMatches) console.log(\`nested:\${match.stackId}:\${match.path}\`);
|
|
83
|
+
" "$1"
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
prompt_stack() {
|
|
87
|
+
local detected_json suggested candidates answer
|
|
88
|
+
detected_json="$(detect_stack_json)"
|
|
89
|
+
suggested="$(stack_suggested "$detected_json")"
|
|
90
|
+
candidates="$(stack_candidates "$detected_json")"
|
|
91
|
+
|
|
92
|
+
if [[ -n "$suggested" ]]; then
|
|
93
|
+
STACK="$suggested"
|
|
94
|
+
echo "Detected stack: $STACK"
|
|
95
|
+
return
|
|
96
|
+
fi
|
|
97
|
+
|
|
98
|
+
if [[ -n "$candidates" ]]; then
|
|
99
|
+
echo "Detected stack candidates:"
|
|
100
|
+
stack_summary "$detected_json"
|
|
101
|
+
if [[ "$YES" -eq 1 ]]; then
|
|
102
|
+
echo "Unable to infer a single stack. Re-run with --stack." >&2
|
|
103
|
+
exit 1
|
|
104
|
+
fi
|
|
105
|
+
read -r -p "Choose stack [${candidates// /, }]: " answer
|
|
106
|
+
else
|
|
107
|
+
if [[ "$YES" -eq 1 ]]; then
|
|
108
|
+
echo "Unable to detect a stack. Re-run with --stack." >&2
|
|
109
|
+
exit 1
|
|
110
|
+
fi
|
|
111
|
+
read -r -p "Stack not detected. Choose stack [$(stack_ids | tr ' ' ', ')]: " answer
|
|
112
|
+
fi
|
|
113
|
+
|
|
114
|
+
if ! validate_stack "$answer"; then
|
|
115
|
+
echo "Unknown stack: $answer" >&2
|
|
116
|
+
exit 1
|
|
117
|
+
fi
|
|
118
|
+
STACK="$answer"
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
detect_mode_json() {
|
|
122
|
+
node --input-type=module -e "
|
|
123
|
+
import { inspectRepoMode } from 'file://${TEMPLATE_ROOT}/scripts/lib/stacks.mjs';
|
|
124
|
+
console.log(JSON.stringify(inspectRepoMode(process.argv[1])));
|
|
125
|
+
" "$REPO"
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
mode_field() {
|
|
129
|
+
node --input-type=module -e "
|
|
130
|
+
const input = JSON.parse(process.argv[1]);
|
|
131
|
+
console.log(input[process.argv[2]] ?? '');
|
|
132
|
+
" "$1" "$2"
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
prompt_mode() {
|
|
136
|
+
local detected_json suggested ambiguous reason answer
|
|
137
|
+
detected_json="$(detect_mode_json)"
|
|
138
|
+
suggested="$(mode_field "$detected_json" suggested)"
|
|
139
|
+
ambiguous="$(mode_field "$detected_json" ambiguous)"
|
|
140
|
+
reason="$(mode_field "$detected_json" reason)"
|
|
141
|
+
|
|
142
|
+
if [[ -n "$suggested" ]] && [[ "$ambiguous" != "true" ]]; then
|
|
143
|
+
MODE="$suggested"
|
|
144
|
+
echo "Detected mode: $MODE ($reason)"
|
|
145
|
+
return
|
|
146
|
+
fi
|
|
147
|
+
|
|
148
|
+
if [[ "$YES" -eq 1 ]]; then
|
|
149
|
+
echo "Unable to infer mode safely ($reason). Re-run with --mode." >&2
|
|
150
|
+
exit 1
|
|
151
|
+
fi
|
|
152
|
+
|
|
153
|
+
echo "Mode requires confirmation: $reason"
|
|
154
|
+
read -r -p "Choose mode [new/existing]: " answer
|
|
155
|
+
case "$answer" in
|
|
156
|
+
new|existing) MODE="$answer" ;;
|
|
157
|
+
*) echo "Unknown mode: $answer" >&2; exit 1 ;;
|
|
158
|
+
esac
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
prompt_codeowners_team() {
|
|
162
|
+
local answer
|
|
163
|
+
if [[ -n "$CODEOWNERS_TEAM" ]]; then
|
|
164
|
+
return
|
|
165
|
+
fi
|
|
166
|
+
|
|
167
|
+
if [[ "$YES" -eq 1 ]]; then
|
|
168
|
+
echo "--codeowners-team is required with --yes." >&2
|
|
169
|
+
exit 1
|
|
170
|
+
fi
|
|
171
|
+
|
|
172
|
+
read -r -p "CODEOWNERS team [@org/team or @username]: " answer
|
|
173
|
+
if [[ ! "$answer" =~ ^@[^/]+/[^/]+$ ]] && [[ ! "$answer" =~ ^@[A-Za-z0-9_.-]+$ ]]; then
|
|
174
|
+
echo "Expected @org/team or @username format." >&2
|
|
175
|
+
exit 1
|
|
176
|
+
fi
|
|
177
|
+
CODEOWNERS_TEAM="$answer"
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
confirm_summary() {
|
|
181
|
+
[[ "$YES" -eq 1 ]] && return
|
|
182
|
+
|
|
183
|
+
local profile product_ci sample_dir
|
|
184
|
+
profile="$(stack_field profile)"
|
|
185
|
+
product_ci="$(stack_field workflow)"
|
|
186
|
+
sample_dir="$(stack_field sampleDir)"
|
|
187
|
+
|
|
188
|
+
cat <<EOF
|
|
189
|
+
Bootstrap summary
|
|
190
|
+
repo: $REPO
|
|
191
|
+
stack: $STACK
|
|
192
|
+
mode: $MODE
|
|
193
|
+
CODEOWNERS team: $CODEOWNERS_TEAM
|
|
194
|
+
profile: $profile
|
|
195
|
+
workflow: $product_ci
|
|
196
|
+
sample copy: $([[ "$MODE" == "new" ]] && echo "sample/$sample_dir -> repo root" || echo "disabled")
|
|
197
|
+
EOF
|
|
198
|
+
read -r -p "Proceed? [y/N]: " answer
|
|
199
|
+
case "$answer" in
|
|
200
|
+
y|Y|yes|YES) ;;
|
|
201
|
+
*) echo "Cancelled."; exit 1 ;;
|
|
202
|
+
esac
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
resolve_github_repo_name() {
|
|
206
|
+
if ! command -v gh >/dev/null 2>&1; then
|
|
207
|
+
return 1
|
|
208
|
+
fi
|
|
209
|
+
gh repo view --json nameWithOwner 2>/dev/null \
|
|
210
|
+
| node --input-type=module -e "
|
|
211
|
+
const chunks = [];
|
|
212
|
+
for await (const chunk of process.stdin) chunks.push(chunk);
|
|
213
|
+
const text = chunks.join('').trim();
|
|
214
|
+
if (!text) process.exit(1);
|
|
215
|
+
const parsed = JSON.parse(text);
|
|
216
|
+
if (!parsed.nameWithOwner) process.exit(1);
|
|
217
|
+
console.log(parsed.nameWithOwner);
|
|
218
|
+
" 2>/dev/null
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
print_next_step() {
|
|
222
|
+
local setup_cmd
|
|
223
|
+
if [[ -d "$REPO/.git" ]] || git -C "$REPO" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
|
|
224
|
+
if (cd "$REPO" && resolve_github_repo_name >/dev/null); then
|
|
225
|
+
setup_cmd="./scripts/setup-wizard.mjs --yes --stack $STACK --codeowners $CODEOWNERS_TEAM"
|
|
226
|
+
else
|
|
227
|
+
setup_cmd="./scripts/setup-wizard.mjs --yes --stack $STACK --codeowners $CODEOWNERS_TEAM --github-repo OWNER/REPO"
|
|
228
|
+
fi
|
|
229
|
+
else
|
|
230
|
+
setup_cmd="./scripts/setup-wizard.mjs --yes --stack $STACK --codeowners $CODEOWNERS_TEAM --github-repo OWNER/REPO"
|
|
231
|
+
fi
|
|
232
|
+
|
|
233
|
+
echo "Next: npx @guilz-dev/sdlc-gh --yes --stack $STACK --codeowners $CODEOWNERS_TEAM"
|
|
234
|
+
echo " Or: $setup_cmd"
|
|
235
|
+
echo " Or: ./scripts/setup-github.sh --yes (after reviewing CODEOWNERS and .harness-stack)"
|
|
236
|
+
echo " Replace OWNER/REPO with your GitHub repository if auto-detection is unavailable."
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
replace_codeowners_placeholder() {
|
|
240
|
+
node --input-type=module -e "
|
|
241
|
+
import { readFileSync, writeFileSync } from 'node:fs';
|
|
242
|
+
const path = process.argv[1];
|
|
243
|
+
const team = process.argv[2];
|
|
244
|
+
const current = readFileSync(path, 'utf8');
|
|
245
|
+
writeFileSync(path, current.replaceAll('@your-org/harness-engineers', team));
|
|
246
|
+
" "$1" "$CODEOWNERS_TEAM"
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
copy_tree() {
|
|
250
|
+
local src="$1" dst="$2"
|
|
251
|
+
mkdir -p "$dst"
|
|
252
|
+
if command -v rsync &>/dev/null; then
|
|
253
|
+
rsync -a \
|
|
254
|
+
--exclude 'node_modules' \
|
|
255
|
+
--exclude '__pycache__' \
|
|
256
|
+
--exclude '.pytest_cache' \
|
|
257
|
+
--exclude '.vite' \
|
|
258
|
+
--exclude 'vendor' \
|
|
259
|
+
--exclude '.bundle' \
|
|
260
|
+
"$src/" "$dst/"
|
|
261
|
+
else
|
|
262
|
+
local item base
|
|
263
|
+
shopt -s dotglob nullglob
|
|
264
|
+
for item in "$src"/*; do
|
|
265
|
+
base="$(basename "$item")"
|
|
266
|
+
case "$base" in
|
|
267
|
+
node_modules|__pycache__|.pytest_cache|.vite|vendor|.bundle) continue ;;
|
|
268
|
+
esac
|
|
269
|
+
cp -R "$item" "$dst/"
|
|
270
|
+
done
|
|
271
|
+
shopt -u dotglob nullglob
|
|
272
|
+
fi
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
while [[ $# -gt 0 ]]; do
|
|
276
|
+
case "$1" in
|
|
277
|
+
--repo) REPO="$2"; shift 2 ;;
|
|
278
|
+
--stack) STACK="$2"; shift 2 ;;
|
|
279
|
+
--mode) MODE="$2"; shift 2 ;;
|
|
280
|
+
--codeowners-team) CODEOWNERS_TEAM="$2"; shift 2 ;;
|
|
281
|
+
--yes) YES=1; shift ;;
|
|
282
|
+
*) usage ;;
|
|
283
|
+
esac
|
|
284
|
+
done
|
|
285
|
+
|
|
286
|
+
detect_repo
|
|
287
|
+
|
|
288
|
+
if [[ -n "$STACK" ]] && ! validate_stack "$STACK"; then
|
|
289
|
+
echo "Unknown stack: $STACK" >&2
|
|
290
|
+
usage
|
|
291
|
+
fi
|
|
292
|
+
|
|
293
|
+
[[ -n "$STACK" ]] || prompt_stack
|
|
294
|
+
|
|
295
|
+
case "$MODE" in
|
|
296
|
+
"" ) prompt_mode ;;
|
|
297
|
+
new|existing) ;;
|
|
298
|
+
* ) echo "Unknown mode: $MODE" >&2; usage ;;
|
|
299
|
+
esac
|
|
300
|
+
|
|
301
|
+
prompt_codeowners_team
|
|
302
|
+
confirm_summary
|
|
303
|
+
|
|
304
|
+
[[ ! -d "$REPO" ]] && mkdir -p "$REPO"
|
|
305
|
+
|
|
306
|
+
PROFILE="$(stack_field profile)"
|
|
307
|
+
PRODUCT_CI="$(stack_field workflow)"
|
|
308
|
+
SAMPLE_DIR="$(stack_field sampleDir)"
|
|
309
|
+
|
|
310
|
+
echo "Bootstrapping harness into $REPO (stack=$STACK, mode=$MODE)"
|
|
311
|
+
|
|
312
|
+
# Core docs
|
|
313
|
+
mkdir -p "$REPO/docs" "$REPO/docs/exceptions"
|
|
314
|
+
for f in operations.md adoption.md auth-boundaries.md failure-taxonomy.md telemetry-schema.md telemetry-artifacts.md nightly-harness-review.md gh-aw-dogfood.md \
|
|
315
|
+
shared-config.md coding-agent-l1.md kpi-baseline.md revert-playbook.md; do
|
|
316
|
+
cp "$TEMPLATE_ROOT/docs/$f" "$REPO/docs/" 2>/dev/null || true
|
|
317
|
+
done
|
|
318
|
+
cp "$TEMPLATE_ROOT/docs/exceptions/README.md" "$REPO/docs/exceptions/" 2>/dev/null || true
|
|
319
|
+
cp "$TEMPLATE_ROOT/docs/exceptions/TEMPLATE.md" "$REPO/docs/exceptions/" 2>/dev/null || true
|
|
320
|
+
cp "$TEMPLATE_ROOT/docs/arch.md" "$REPO/docs/" 2>/dev/null || true
|
|
321
|
+
cp "$TEMPLATE_ROOT/AGENTS.md" "$REPO/"
|
|
322
|
+
cp "$TEMPLATE_ROOT/README.md" "$REPO/HARNESS_README.md" 2>/dev/null || true
|
|
323
|
+
mkdir -p "$REPO/config"
|
|
324
|
+
cp "$TEMPLATE_ROOT/config/stacks.json" "$REPO/config/"
|
|
325
|
+
|
|
326
|
+
# .github harness assets
|
|
327
|
+
mkdir -p "$REPO/.github"
|
|
328
|
+
copy_tree "$TEMPLATE_ROOT/.github/agents" "$REPO/.github/agents"
|
|
329
|
+
copy_tree "$TEMPLATE_ROOT/.github/hooks" "$REPO/.github/hooks"
|
|
330
|
+
copy_tree "$TEMPLATE_ROOT/.github/ISSUE_TEMPLATE" "$REPO/.github/ISSUE_TEMPLATE"
|
|
331
|
+
copy_tree "$TEMPLATE_ROOT/.github/skills" "$REPO/.github/skills"
|
|
332
|
+
cp "$TEMPLATE_ROOT/.github/copilot-instructions.md" "$REPO/.github/"
|
|
333
|
+
cp "$TEMPLATE_ROOT/.github/labels.yml" "$REPO/.github/"
|
|
334
|
+
cp "$TEMPLATE_ROOT/.github/pull_request_template.md" "$REPO/.github/"
|
|
335
|
+
cp "$TEMPLATE_ROOT/.github/CODEOWNERS" "$REPO/.github/"
|
|
336
|
+
replace_codeowners_placeholder "$REPO/.github/CODEOWNERS"
|
|
337
|
+
cp "$TEMPLATE_ROOT/.github/ruleset.example.json" "$REPO/.github/" 2>/dev/null || true
|
|
338
|
+
|
|
339
|
+
mkdir -p "$REPO/.github/instructions/profiles"
|
|
340
|
+
cp "$TEMPLATE_ROOT/.github/instructions/core.instructions.md" "$REPO/.github/instructions/"
|
|
341
|
+
cp "$TEMPLATE_ROOT/.github/instructions/profiles/$PROFILE" "$REPO/.github/instructions/profiles/"
|
|
342
|
+
|
|
343
|
+
# Workflows — core + selected stack product CI + phase 2–4
|
|
344
|
+
mkdir -p "$REPO/.github/workflows"
|
|
345
|
+
for wf in harness-ci.yml copilot-setup-steps.yml l1-readiness-check.yml pr-context-comment.yml eval-ci.yml eval-drift.yml \
|
|
346
|
+
agent-retry-orchestrator.yml harness-sync.yml labels-sync.yml nightly-harness-review.yml gh-aw-dogfood-ci.yml; do
|
|
347
|
+
cp "$TEMPLATE_ROOT/.github/workflows/$wf" "$REPO/.github/workflows/"
|
|
348
|
+
done
|
|
349
|
+
cp "$TEMPLATE_ROOT/.github/workflows/$PRODUCT_CI" "$REPO/.github/workflows/"
|
|
350
|
+
node "$TEMPLATE_ROOT/scripts/trim-harness-ci.mjs" "$STACK" "$REPO/.github/workflows/harness-ci.yml"
|
|
351
|
+
for aw in nightly-harness-review.md weekly-redteam.md nightly-harness-review.lock.yml weekly-redteam.lock.yml; do
|
|
352
|
+
cp "$TEMPLATE_ROOT/.github/workflows/$aw" "$REPO/.github/workflows/" 2>/dev/null || true
|
|
353
|
+
done
|
|
354
|
+
cp "$TEMPLATE_ROOT/.github/ruleset.harness-eval.example.json" "$REPO/.github/" 2>/dev/null || true
|
|
355
|
+
|
|
356
|
+
# Scripts
|
|
357
|
+
mkdir -p "$REPO/scripts/lib"
|
|
358
|
+
for s in validate-harness.mjs check-diff-size.mjs check-issue-spec.mjs select-eval-jobs.mjs \
|
|
359
|
+
check-e2e-manifest.mjs validate-telemetry.mjs emit-telemetry-artifact.mjs fetch-telemetry-artifacts.mjs \
|
|
360
|
+
aggregate-harness-review.mjs route-harness-review.mjs check-l1-readiness.mjs check-gh-aw-dogfood-scope.mjs validate-gh-aw-compile.mjs \
|
|
361
|
+
emit-gh-aw-dogfood-report.mjs check-open-pr-limit.mjs merge-harness-package.mjs test-hooks-scenarios.mjs test-issue-spec-scenarios.mjs \
|
|
362
|
+
test-diff-size-scenarios.mjs test-e2e-manifest-scenarios.mjs test-setup-github-scenarios.mjs test-doctor-scenarios.mjs \
|
|
363
|
+
test-telemetry-artifact-scenarios.mjs test-harness-review-scenarios.mjs test-harness-review-routing-scenarios.mjs test-gh-aw-dogfood-scenarios.mjs \
|
|
364
|
+
test-bootstrap-guidance-scenarios.mjs test-merge-harness-package-scenarios.mjs test-l1-readiness-scenarios.mjs test-setup-wizard-scenarios.mjs \
|
|
365
|
+
harness-drift-report.mjs check-eval-score-drift.mjs run-e2e-bench.mjs doctor.mjs setup-github.mjs setup-wizard.mjs sdlc-gh-cli.mjs test-npm-package-scenarios.mjs; do
|
|
366
|
+
cp "$TEMPLATE_ROOT/scripts/$s" "$REPO/scripts/" 2>/dev/null || true
|
|
367
|
+
done
|
|
368
|
+
for s in bootstrap-harness.sh setup-github.sh verify-bootstrap-stacks.sh; do
|
|
369
|
+
cp "$TEMPLATE_ROOT/scripts/$s" "$REPO/scripts/" 2>/dev/null || true
|
|
370
|
+
done
|
|
371
|
+
for s in stacks.mjs harness-ci-fragments.mjs ccsd-contract.mjs github-config.mjs diff-size.mjs e2e-manifest.mjs \
|
|
372
|
+
doctor-local.mjs bootstrap-copy.mjs l1-readiness.mjs merge-harness-package.mjs telemetry-artifact.mjs harness-review.mjs harness-review-routing.mjs gh-aw-dogfood.mjs setup-wizard.mjs template-root.mjs npm-package.mjs; do
|
|
373
|
+
cp "$TEMPLATE_ROOT/scripts/lib/$s" "$REPO/scripts/lib/" 2>/dev/null || true
|
|
374
|
+
done
|
|
375
|
+
cp "$TEMPLATE_ROOT/scripts/trim-harness-ci.mjs" "$REPO/scripts/" 2>/dev/null || true
|
|
376
|
+
node "$TEMPLATE_ROOT/scripts/merge-harness-package.mjs" "$TEMPLATE_ROOT/package.json" "$REPO/package.json"
|
|
377
|
+
chmod +x "$REPO/scripts/"*.mjs "$REPO/scripts/"*.sh 2>/dev/null || true
|
|
378
|
+
|
|
379
|
+
# Sample for new projects
|
|
380
|
+
if [[ "$MODE" == "new" ]]; then
|
|
381
|
+
copy_tree "$TEMPLATE_ROOT/sample/$SAMPLE_DIR" "$REPO/"
|
|
382
|
+
fi
|
|
383
|
+
|
|
384
|
+
# Evals and prompts
|
|
385
|
+
mkdir -p "$REPO/evals/trajectories" "$REPO/evals/e2e-bench" "$REPO/prompts"
|
|
386
|
+
cp "$TEMPLATE_ROOT/evals/e2e-bench/manifest.json" "$REPO/evals/e2e-bench/" 2>/dev/null || true
|
|
387
|
+
cp "$TEMPLATE_ROOT/evals/e2e-bench/README.md" "$REPO/evals/e2e-bench/" 2>/dev/null || true
|
|
388
|
+
mkdir -p "$REPO/evals/e2e-bench/tasks"
|
|
389
|
+
cp "$TEMPLATE_ROOT/evals/e2e-bench/tasks/"*.yml "$REPO/evals/e2e-bench/tasks/" 2>/dev/null || true
|
|
390
|
+
cp "$TEMPLATE_ROOT/evals/trajectories/test_harness_conventions.py" "$REPO/evals/trajectories/" 2>/dev/null || true
|
|
391
|
+
cp "$TEMPLATE_ROOT/evals/trajectories/rubric.md" "$REPO/evals/trajectories/" 2>/dev/null || true
|
|
392
|
+
cp "$TEMPLATE_ROOT/evals/.score-baseline.json" "$REPO/evals/" 2>/dev/null || true
|
|
393
|
+
cp "$TEMPLATE_ROOT/prompts/"*.prompt.yml "$REPO/prompts/" 2>/dev/null || true
|
|
394
|
+
|
|
395
|
+
# Stack marker for documentation
|
|
396
|
+
echo "$STACK" > "$REPO/.harness-stack"
|
|
397
|
+
|
|
398
|
+
# Infra optional copy
|
|
399
|
+
mkdir -p "$REPO/infra/langfuse" "$REPO/infra/otel" "$REPO/infra/samples"
|
|
400
|
+
cp "$TEMPLATE_ROOT/infra/langfuse/docker-compose.yml" "$REPO/infra/langfuse/" 2>/dev/null || true
|
|
401
|
+
cp "$TEMPLATE_ROOT/infra/otel/collector-config.yml" "$REPO/infra/otel/" 2>/dev/null || true
|
|
402
|
+
cp "$TEMPLATE_ROOT/infra/README.md" "$REPO/infra/" 2>/dev/null || true
|
|
403
|
+
cp "$TEMPLATE_ROOT/infra/samples/telemetry-payload.json" "$REPO/infra/samples/" 2>/dev/null || true
|
|
404
|
+
cp "$TEMPLATE_ROOT/infra/samples/telemetry-artifact.json" "$REPO/infra/samples/" 2>/dev/null || true
|
|
405
|
+
cp "$TEMPLATE_ROOT/infra/samples/gh-aw-dogfood-report.json" "$REPO/infra/samples/" 2>/dev/null || true
|
|
406
|
+
cp "$TEMPLATE_ROOT/infra/samples/harness-review-summary.json" "$REPO/infra/samples/" 2>/dev/null || true
|
|
407
|
+
cp "$TEMPLATE_ROOT/infra/samples/harness-review-routing-plan.json" "$REPO/infra/samples/" 2>/dev/null || true
|
|
408
|
+
|
|
409
|
+
echo "Done. Stack=$STACK mode=$MODE"
|
|
410
|
+
print_next_step
|
|
411
|
+
echo "Then: ./scripts/doctor.mjs --strict"
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Diff size and autonomy gate for PRs.
|
|
4
|
+
*/
|
|
5
|
+
import { execSync } from "node:child_process";
|
|
6
|
+
import { evaluateDiffSize, parseLabelInput } from "./lib/diff-size.mjs";
|
|
7
|
+
|
|
8
|
+
const labels = parseLabelInput(process.env.PR_LABELS || process.argv[2] || "");
|
|
9
|
+
const l1HardFail = process.env.DIFF_SIZE_L1_HARD_FAIL === "1";
|
|
10
|
+
|
|
11
|
+
const base = process.env.BASE_SHA || "origin/main";
|
|
12
|
+
const diffRange = `${base}...HEAD`;
|
|
13
|
+
|
|
14
|
+
function git(cmd) {
|
|
15
|
+
return execSync(cmd, { encoding: "utf8" }).trim();
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
let numstat;
|
|
19
|
+
try {
|
|
20
|
+
numstat = git(`git diff --numstat ${diffRange}`);
|
|
21
|
+
} catch {
|
|
22
|
+
numstat = git("git diff --numstat HEAD~1...HEAD");
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
let diffFiles;
|
|
26
|
+
try {
|
|
27
|
+
diffFiles = git(`git diff --name-only ${diffRange}`).split("\n").filter(Boolean);
|
|
28
|
+
} catch {
|
|
29
|
+
diffFiles = git("git diff --name-only HEAD~1...HEAD").split("\n").filter(Boolean);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const result = evaluateDiffSize({ labels, numstatText: numstat, diffFiles, l1HardFail });
|
|
33
|
+
|
|
34
|
+
console.log(result.summary);
|
|
35
|
+
|
|
36
|
+
if (result.overLimit && result.overLimitMessage) {
|
|
37
|
+
if (result.mode === "hard-fail") {
|
|
38
|
+
console.error(`::error::${result.overLimitMessage}`);
|
|
39
|
+
process.exit(1);
|
|
40
|
+
}
|
|
41
|
+
console.warn(`::warning::${result.overLimitMessage} — split recommended`);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
for (const warning of result.sensitiveWarnings) {
|
|
45
|
+
console.warn(`::warning::${warning}`);
|
|
46
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/** Verify e2e-bench manifest freshness and structural integrity */
|
|
3
|
+
import { readdirSync, readFileSync, existsSync } from "node:fs";
|
|
4
|
+
import { join } from "node:path";
|
|
5
|
+
import { validateManifest } from "./lib/e2e-manifest.mjs";
|
|
6
|
+
|
|
7
|
+
const manifestPath = join(process.cwd(), "evals/e2e-bench/manifest.json");
|
|
8
|
+
const tasksDir = join(process.cwd(), "evals/e2e-bench/tasks");
|
|
9
|
+
|
|
10
|
+
if (!existsSync(manifestPath)) {
|
|
11
|
+
console.error("Missing evals/e2e-bench/manifest.json");
|
|
12
|
+
process.exit(1);
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
const manifest = JSON.parse(readFileSync(manifestPath, "utf8"));
|
|
16
|
+
const taskFileIds = existsSync(tasksDir)
|
|
17
|
+
? readdirSync(tasksDir)
|
|
18
|
+
.filter((name) => name.endsWith(".yml"))
|
|
19
|
+
.map((name) => name.replace(/\.yml$/, ""))
|
|
20
|
+
: [];
|
|
21
|
+
|
|
22
|
+
const { errors, warnings, taskCount, minTasks } = validateManifest(manifest, taskFileIds);
|
|
23
|
+
|
|
24
|
+
console.log(`E2E tasks: ${taskCount} (min ${minTasks})`);
|
|
25
|
+
|
|
26
|
+
for (const warning of warnings) {
|
|
27
|
+
console.warn(`::warning::${warning}`);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
if (errors.length > 0) {
|
|
31
|
+
for (const error of errors) {
|
|
32
|
+
console.error(`::error::${error}`);
|
|
33
|
+
}
|
|
34
|
+
process.exit(1);
|
|
35
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Phase 4: if eval pass rate exceeds production acceptance by DRIFT_PT, flag drift.
|
|
4
|
+
* Reads evals/.score-baseline.json (optional) or uses defaults for template CI.
|
|
5
|
+
*/
|
|
6
|
+
import { readFileSync, existsSync } from "node:fs";
|
|
7
|
+
import { join } from "node:path";
|
|
8
|
+
|
|
9
|
+
const DRIFT_PT = Number(process.env.HARNESS_EVAL_DRIFT_PT || 15);
|
|
10
|
+
const baselinePath = join(process.cwd(), "evals/.score-baseline.json");
|
|
11
|
+
|
|
12
|
+
let evalPass = 85;
|
|
13
|
+
let prodAccept = 70;
|
|
14
|
+
|
|
15
|
+
if (existsSync(baselinePath)) {
|
|
16
|
+
const b = JSON.parse(readFileSync(baselinePath, "utf8"));
|
|
17
|
+
evalPass = Number(b.eval_pass_rate ?? evalPass);
|
|
18
|
+
prodAccept = Number(b.production_acceptance_rate ?? prodAccept);
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
const gap = evalPass - prodAccept;
|
|
22
|
+
console.log(`Eval pass ${evalPass}% vs production acceptance ${prodAccept}% (gap ${gap}pt, threshold ${DRIFT_PT}pt)`);
|
|
23
|
+
|
|
24
|
+
if (gap > DRIFT_PT) {
|
|
25
|
+
console.warn(
|
|
26
|
+
`::warning::Eval/production gap ${gap}pt exceeds ${DRIFT_PT}pt — open bench review issue`,
|
|
27
|
+
);
|
|
28
|
+
process.exit(2);
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
console.log("Eval score drift check passed");
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Enforce narrow file scope for task:gh-aw-dogfood PRs.
|
|
4
|
+
*/
|
|
5
|
+
import { execSync } from "node:child_process";
|
|
6
|
+
import {
|
|
7
|
+
DOGFOOD_TASK_LABEL,
|
|
8
|
+
evaluateDogfoodScope,
|
|
9
|
+
isDogfoodAllowedPath,
|
|
10
|
+
parseDogfoodLabels,
|
|
11
|
+
} from "./lib/gh-aw-dogfood.mjs";
|
|
12
|
+
|
|
13
|
+
function changedFiles() {
|
|
14
|
+
const base = process.env.BASE_SHA || "origin/main";
|
|
15
|
+
try {
|
|
16
|
+
const out = execSync(`git diff --name-only ${base}...HEAD`, { encoding: "utf8" });
|
|
17
|
+
return out.split("\n").filter(Boolean);
|
|
18
|
+
} catch {
|
|
19
|
+
return execSync("git diff --name-only HEAD~1...HEAD", { encoding: "utf8" })
|
|
20
|
+
.split("\n")
|
|
21
|
+
.filter(Boolean);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function main() {
|
|
26
|
+
const labels = parseDogfoodLabels(process.env.PR_LABELS);
|
|
27
|
+
const files = changedFiles();
|
|
28
|
+
const touchesDogfood = files.some((file) => isDogfoodAllowedPath(file));
|
|
29
|
+
const scope = evaluateDogfoodScope(files, labels);
|
|
30
|
+
|
|
31
|
+
if (!scope.enforced) {
|
|
32
|
+
if (touchesDogfood) {
|
|
33
|
+
console.warn(
|
|
34
|
+
`::warning::Dogfood paths changed without ${DOGFOOD_TASK_LABEL}; add label for scoped validation`,
|
|
35
|
+
);
|
|
36
|
+
}
|
|
37
|
+
console.log("Not a gh-aw dogfood task; scope enforcement skipped");
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
if (!scope.ok) {
|
|
42
|
+
console.error(`::error::Out-of-scope paths for ${DOGFOOD_TASK_LABEL}:`);
|
|
43
|
+
for (const issue of scope.issues) console.error(` - ${issue}`);
|
|
44
|
+
console.error("Allowed paths: docs/gh-aw-dogfood.md#allowed-path-scope");
|
|
45
|
+
process.exit(1);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
console.log(`Dogfood scope check passed (${files.length} file(s))`);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
main();
|