@guilz-dev/sdlc-gh 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/CODEOWNERS +5 -0
- package/.github/ISSUE_TEMPLATE/bug_report.yml +68 -0
- package/.github/ISSUE_TEMPLATE/config.yml +1 -0
- package/.github/ISSUE_TEMPLATE/feature_request.yml +39 -0
- package/.github/ISSUE_TEMPLATE/support.yml +56 -0
- package/.github/ISSUE_TEMPLATE/task.yml +89 -0
- package/.github/agents/implementer.agent.md +17 -0
- package/.github/agents/reviewer.agent.md +18 -0
- package/.github/agents/triager.agent.md +13 -0
- package/.github/aw/actions-lock.json +9 -0
- package/.github/copilot-instructions.md +35 -0
- package/.github/hooks/hooks.json +12 -0
- package/.github/instructions/core.instructions.md +11 -0
- package/.github/instructions/profiles/go.instructions.md +10 -0
- package/.github/instructions/profiles/php.instructions.md +11 -0
- package/.github/instructions/profiles/python.instructions.md +11 -0
- package/.github/instructions/profiles/ruby.instructions.md +11 -0
- package/.github/instructions/profiles/typescript.instructions.md +11 -0
- package/.github/labels.yml +55 -0
- package/.github/pull_request_template.md +33 -0
- package/.github/ruleset.example.json +33 -0
- package/.github/ruleset.harness-eval.example.json +29 -0
- package/.github/skills/quality-loop/SKILL.md +23 -0
- package/.github/workflows/agent-retry-orchestrator.yml +161 -0
- package/.github/workflows/copilot-setup-steps.yml +64 -0
- package/.github/workflows/eval-ci.yml +169 -0
- package/.github/workflows/eval-drift.yml +75 -0
- package/.github/workflows/gh-aw-dogfood-ci.yml +73 -0
- package/.github/workflows/harness-ci.yml +244 -0
- package/.github/workflows/harness-sync.yml +28 -0
- package/.github/workflows/l1-readiness-check.yml +45 -0
- package/.github/workflows/labels-sync.yml +24 -0
- package/.github/workflows/nightly-harness-review.lock.yml +1643 -0
- package/.github/workflows/nightly-harness-review.md +87 -0
- package/.github/workflows/nightly-harness-review.yml +63 -0
- package/.github/workflows/npm-publish.yml +49 -0
- package/.github/workflows/pr-context-comment.yml +138 -0
- package/.github/workflows/product-ci-go.yml +33 -0
- package/.github/workflows/product-ci-php.yml +39 -0
- package/.github/workflows/product-ci-python.yml +34 -0
- package/.github/workflows/product-ci-ruby.yml +35 -0
- package/.github/workflows/product-ci-ts.yml +37 -0
- package/.github/workflows/task-issue-label-sync.yml +50 -0
- package/.github/workflows/weekly-redteam.lock.yml +1571 -0
- package/.github/workflows/weekly-redteam.md +76 -0
- package/.github/zizmor.yml +11 -0
- package/AGENTS.md +54 -0
- package/LICENSE +21 -0
- package/README.md +366 -0
- package/config/stacks.json +55 -0
- package/docs/adoption.md +126 -0
- package/docs/arch.md +535 -0
- package/docs/auth-boundaries.md +16 -0
- package/docs/coding-agent-l1.md +152 -0
- package/docs/exceptions/README.md +25 -0
- package/docs/exceptions/TEMPLATE.md +8 -0
- package/docs/failure-taxonomy.md +23 -0
- package/docs/gh-aw-dogfood.md +109 -0
- package/docs/kpi-baseline.md +9 -0
- package/docs/nightly-harness-review.md +94 -0
- package/docs/operations.md +108 -0
- package/docs/publishing.md +79 -0
- package/docs/revert-playbook.md +44 -0
- package/docs/shared-config.md +30 -0
- package/docs/telemetry-artifacts.md +78 -0
- package/docs/telemetry-schema.md +60 -0
- package/evals/.score-baseline.json +6 -0
- package/evals/e2e-bench/README.md +28 -0
- package/evals/e2e-bench/manifest.json +16 -0
- package/evals/e2e-bench/tasks/e2e-001.yml +10 -0
- package/evals/e2e-bench/tasks/e2e-002.yml +11 -0
- package/evals/e2e-bench/tasks/e2e-003.yml +10 -0
- package/evals/e2e-bench/tasks/e2e-004.yml +14 -0
- package/evals/e2e-bench/tasks/e2e-005.yml +11 -0
- package/evals/e2e-bench/tasks/e2e-006.yml +10 -0
- package/evals/e2e-bench/tasks/e2e-007.yml +10 -0
- package/evals/e2e-bench/tasks/e2e-008.yml +10 -0
- package/evals/e2e-bench/tasks/e2e-009.yml +10 -0
- package/evals/trajectories/rubric.md +12 -0
- package/evals/trajectories/test_harness_conventions.py +271 -0
- package/infra/README.md +49 -0
- package/infra/langfuse/docker-compose.yml +25 -0
- package/infra/otel/collector-config.yml +24 -0
- package/infra/samples/gh-aw-dogfood-report.json +44 -0
- package/infra/samples/harness-review-routing-plan.json +19 -0
- package/infra/samples/harness-review-summary.json +61 -0
- package/infra/samples/telemetry-artifact.json +29 -0
- package/infra/samples/telemetry-payload.json +19 -0
- package/package.json +85 -0
- package/prompts/triager-classify.prompt.yml +10 -0
- package/sample/go/add.go +5 -0
- package/sample/go/add_test.go +9 -0
- package/sample/go/go.mod +3 -0
- package/sample/php/composer.json +26 -0
- package/sample/php/composer.lock +1881 -0
- package/sample/php/phpunit.xml +8 -0
- package/sample/php/src/Add.php +13 -0
- package/sample/php/tests/AddTest.php +16 -0
- package/sample/python/requirements-dev.txt +2 -0
- package/sample/python/src/__init__.py +0 -0
- package/sample/python/src/greet.py +3 -0
- package/sample/python/tests/conftest.py +4 -0
- package/sample/python/tests/test_greet.py +5 -0
- package/sample/ruby/.rubocop.yml +10 -0
- package/sample/ruby/Gemfile +6 -0
- package/sample/ruby/Gemfile.lock +58 -0
- package/sample/ruby/lib/add.rb +9 -0
- package/sample/ruby/spec/add_spec.rb +11 -0
- package/sample/ts/biome.json +6 -0
- package/sample/ts/package-lock.json +1763 -0
- package/sample/ts/package.json +15 -0
- package/sample/ts/src/add.ts +3 -0
- package/sample/ts/tests/add.test.ts +8 -0
- package/sample/ts/tsconfig.json +12 -0
- package/scripts/aggregate-harness-review.mjs +48 -0
- package/scripts/bootstrap-harness.sh +411 -0
- package/scripts/check-diff-size.mjs +46 -0
- package/scripts/check-e2e-manifest.mjs +35 -0
- package/scripts/check-eval-score-drift.mjs +31 -0
- package/scripts/check-gh-aw-dogfood-scope.mjs +51 -0
- package/scripts/check-issue-spec.mjs +215 -0
- package/scripts/check-l1-readiness.mjs +82 -0
- package/scripts/check-open-pr-limit.mjs +34 -0
- package/scripts/doctor.mjs +177 -0
- package/scripts/emit-gh-aw-dogfood-report.mjs +112 -0
- package/scripts/emit-telemetry-artifact.mjs +99 -0
- package/scripts/fetch-telemetry-artifacts.mjs +176 -0
- package/scripts/harness-drift-report.mjs +99 -0
- package/scripts/lib/bootstrap-copy.mjs +123 -0
- package/scripts/lib/ccsd-contract.mjs +212 -0
- package/scripts/lib/diff-size.mjs +103 -0
- package/scripts/lib/doctor-local.mjs +179 -0
- package/scripts/lib/e2e-manifest.mjs +76 -0
- package/scripts/lib/gh-aw-dogfood.mjs +293 -0
- package/scripts/lib/github-config.mjs +94 -0
- package/scripts/lib/harness-ci-fragments.mjs +98 -0
- package/scripts/lib/harness-review-routing.mjs +244 -0
- package/scripts/lib/harness-review.mjs +388 -0
- package/scripts/lib/issue-form-label-sync.mjs +56 -0
- package/scripts/lib/l1-readiness.mjs +258 -0
- package/scripts/lib/merge-harness-package.mjs +36 -0
- package/scripts/lib/npm-package.mjs +129 -0
- package/scripts/lib/setup-wizard.mjs +224 -0
- package/scripts/lib/stacks.mjs +138 -0
- package/scripts/lib/telemetry-artifact.mjs +253 -0
- package/scripts/lib/template-root.mjs +39 -0
- package/scripts/merge-harness-package.mjs +14 -0
- package/scripts/route-harness-review.mjs +168 -0
- package/scripts/run-e2e-bench.mjs +216 -0
- package/scripts/sdlc-gh-cli.mjs +91 -0
- package/scripts/select-eval-jobs.mjs +41 -0
- package/scripts/setup-github.mjs +242 -0
- package/scripts/setup-github.sh +4 -0
- package/scripts/setup-wizard.mjs +426 -0
- package/scripts/test-bootstrap-guidance-scenarios.mjs +94 -0
- package/scripts/test-diff-size-scenarios.mjs +88 -0
- package/scripts/test-doctor-scenarios.mjs +70 -0
- package/scripts/test-e2e-manifest-scenarios.mjs +65 -0
- package/scripts/test-gh-aw-dogfood-scenarios.mjs +74 -0
- package/scripts/test-harness-review-routing-scenarios.mjs +130 -0
- package/scripts/test-harness-review-scenarios.mjs +92 -0
- package/scripts/test-hooks-scenarios.mjs +44 -0
- package/scripts/test-issue-form-label-sync-scenarios.mjs +48 -0
- package/scripts/test-issue-spec-scenarios.mjs +258 -0
- package/scripts/test-l1-readiness-scenarios.mjs +204 -0
- package/scripts/test-merge-harness-package-scenarios.mjs +53 -0
- package/scripts/test-npm-package-scenarios.mjs +31 -0
- package/scripts/test-sdlc-gh-cli-scenarios.mjs +54 -0
- package/scripts/test-setup-github-scenarios.mjs +103 -0
- package/scripts/test-setup-wizard-scenarios.mjs +114 -0
- package/scripts/test-telemetry-artifact-scenarios.mjs +69 -0
- package/scripts/trim-harness-ci.mjs +18 -0
- package/scripts/validate-gh-aw-compile.mjs +64 -0
- package/scripts/validate-harness.mjs +199 -0
- package/scripts/validate-telemetry.mjs +21 -0
- package/scripts/verify-bootstrap-stacks.sh +192 -0
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
4
|
+
import { tmpdir } from "node:os";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
import { buildEvalRulesetPayload, buildRulesetPayload, parseLabels } from "./lib/github-config.mjs";
|
|
7
|
+
import { resolveStackId } from "./lib/doctor-local.mjs";
|
|
8
|
+
|
|
9
|
+
const labels = parseLabels(`
|
|
10
|
+
- name: task:docs
|
|
11
|
+
color: "0E8A16"
|
|
12
|
+
description: Documentation changes
|
|
13
|
+
|
|
14
|
+
- name: autonomy:L1
|
|
15
|
+
color: "C2E0C6"
|
|
16
|
+
description: Draft PR, human review required
|
|
17
|
+
`);
|
|
18
|
+
|
|
19
|
+
assert.deepEqual(labels, [
|
|
20
|
+
{
|
|
21
|
+
name: "task:docs",
|
|
22
|
+
color: "0E8A16",
|
|
23
|
+
description: "Documentation changes",
|
|
24
|
+
},
|
|
25
|
+
{
|
|
26
|
+
name: "autonomy:L1",
|
|
27
|
+
color: "C2E0C6",
|
|
28
|
+
description: "Draft PR, human review required",
|
|
29
|
+
},
|
|
30
|
+
]);
|
|
31
|
+
|
|
32
|
+
const tempDir = mkdtempSync(join(tmpdir(), "sdlc-gh-ruleset-"));
|
|
33
|
+
const template = join(tempDir, "ruleset.example.json");
|
|
34
|
+
writeFileSync(
|
|
35
|
+
template,
|
|
36
|
+
JSON.stringify({
|
|
37
|
+
name: "main-protection",
|
|
38
|
+
target: "branch",
|
|
39
|
+
enforcement: "active",
|
|
40
|
+
rules: [
|
|
41
|
+
{
|
|
42
|
+
type: "required_status_checks",
|
|
43
|
+
parameters: {
|
|
44
|
+
strict_required_status_checks_policy: false,
|
|
45
|
+
required_status_checks: [{ context: "harness-static" }],
|
|
46
|
+
},
|
|
47
|
+
},
|
|
48
|
+
],
|
|
49
|
+
_comment: "placeholder",
|
|
50
|
+
}),
|
|
51
|
+
);
|
|
52
|
+
|
|
53
|
+
const payload = buildRulesetPayload(template, "python");
|
|
54
|
+
const contexts = payload.rules
|
|
55
|
+
.find((rule) => rule.type === "required_status_checks")
|
|
56
|
+
.parameters.required_status_checks.map((check) => check.context);
|
|
57
|
+
|
|
58
|
+
assert.equal(payload.name, "main-protection");
|
|
59
|
+
assert.equal(payload.enforcement, "active");
|
|
60
|
+
assert.deepEqual(contexts, ["diff-size", "harness-static", "issue-spec-check", "product-ci-python"]);
|
|
61
|
+
assert.equal("strict_required_status_checks_policy" in payload.rules[0].parameters, true);
|
|
62
|
+
assert.equal(readFileSync(template, "utf8").includes("_comment"), true);
|
|
63
|
+
assert.equal("_comment" in payload, false);
|
|
64
|
+
|
|
65
|
+
const evalTemplate = join(tempDir, "ruleset.harness-eval.example.json");
|
|
66
|
+
writeFileSync(
|
|
67
|
+
evalTemplate,
|
|
68
|
+
JSON.stringify({
|
|
69
|
+
name: "harness-pr-eval-required",
|
|
70
|
+
target: "branch",
|
|
71
|
+
enforcement: "active",
|
|
72
|
+
rules: [
|
|
73
|
+
{
|
|
74
|
+
type: "required_status_checks",
|
|
75
|
+
parameters: {
|
|
76
|
+
strict_required_status_checks_policy: false,
|
|
77
|
+
required_status_checks: [{ context: "harness-static" }],
|
|
78
|
+
},
|
|
79
|
+
},
|
|
80
|
+
],
|
|
81
|
+
_comment: "placeholder",
|
|
82
|
+
}),
|
|
83
|
+
);
|
|
84
|
+
|
|
85
|
+
const evalPayload = buildEvalRulesetPayload(evalTemplate);
|
|
86
|
+
const evalContexts = evalPayload.rules
|
|
87
|
+
.find((rule) => rule.type === "required_status_checks")
|
|
88
|
+
.parameters.required_status_checks.map((check) => check.context);
|
|
89
|
+
|
|
90
|
+
assert.equal(evalPayload.name, "harness-pr-eval-required");
|
|
91
|
+
assert.deepEqual(evalContexts, ["harness-static", "select", "trajectory-conventions"]);
|
|
92
|
+
assert.equal("_comment" in evalPayload, false);
|
|
93
|
+
|
|
94
|
+
const stackRepo = mkdtempSync(join(tmpdir(), "sdlc-gh-stack-resolve-"));
|
|
95
|
+
mkdirSync(join(stackRepo, ".github/workflows"), { recursive: true });
|
|
96
|
+
writeFileSync(join(stackRepo, ".github/workflows/product-ci-go.yml"), "name: product-ci-go\n");
|
|
97
|
+
assert.equal(resolveStackId(stackRepo), "go");
|
|
98
|
+
writeFileSync(join(stackRepo, ".harness-stack"), "python\n");
|
|
99
|
+
assert.equal(resolveStackId(stackRepo), "python");
|
|
100
|
+
|
|
101
|
+
rmSync(tempDir, { recursive: true, force: true });
|
|
102
|
+
rmSync(stackRepo, { recursive: true, force: true });
|
|
103
|
+
console.log("GitHub setup scenario tests passed");
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { mkdtempSync, mkdirSync, writeFileSync, readFileSync } from "node:fs";
|
|
4
|
+
import { tmpdir } from "node:os";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
import { localChecks } from "./lib/doctor-local.mjs";
|
|
7
|
+
import {
|
|
8
|
+
CODEOWNERS_PLACEHOLDER,
|
|
9
|
+
applyCodeownersOwner,
|
|
10
|
+
buildWizardPlan,
|
|
11
|
+
codeownersHasPlaceholder,
|
|
12
|
+
detectHarnessPresent,
|
|
13
|
+
detectRepoProfile,
|
|
14
|
+
isValidCodeownersOwner,
|
|
15
|
+
suggestStack,
|
|
16
|
+
writeHarnessStack,
|
|
17
|
+
} from "./lib/setup-wizard.mjs";
|
|
18
|
+
|
|
19
|
+
assert.equal(isValidCodeownersOwner("@acme/platform"), true);
|
|
20
|
+
assert.equal(isValidCodeownersOwner("@kaz-toc"), true);
|
|
21
|
+
assert.equal(isValidCodeownersOwner("acme/platform"), false);
|
|
22
|
+
|
|
23
|
+
const templateDir = mkdtempSync(join(tmpdir(), "sdlc-gh-wizard-template-"));
|
|
24
|
+
mkdirSync(join(templateDir, ".github/workflows"), { recursive: true });
|
|
25
|
+
mkdirSync(join(templateDir, "sample/ts"), { recursive: true });
|
|
26
|
+
mkdirSync(join(templateDir, "scripts"), { recursive: true });
|
|
27
|
+
writeFileSync(join(templateDir, ".github/workflows/harness-ci.yml"), "name: harness\n");
|
|
28
|
+
writeFileSync(join(templateDir, ".github/workflows/product-ci-ts.yml"), "name: product-ci-ts\n");
|
|
29
|
+
writeFileSync(join(templateDir, ".github/workflows/product-ci-python.yml"), "name: product-ci-python\n");
|
|
30
|
+
writeFileSync(join(templateDir, "scripts/doctor.mjs"), "// doctor\n");
|
|
31
|
+
writeFileSync(join(templateDir, "sample/ts/package.json"), "{}\n");
|
|
32
|
+
writeFileSync(join(templateDir, "package.json"), "{}\n");
|
|
33
|
+
writeFileSync(
|
|
34
|
+
join(templateDir, ".github/CODEOWNERS"),
|
|
35
|
+
`/.github/ ${CODEOWNERS_PLACEHOLDER}\n`,
|
|
36
|
+
);
|
|
37
|
+
|
|
38
|
+
const templateProfile = detectRepoProfile(templateDir);
|
|
39
|
+
assert.equal(templateProfile.kind, "template");
|
|
40
|
+
assert.equal(templateProfile.template, true);
|
|
41
|
+
assert.equal(detectHarnessPresent(templateDir), true);
|
|
42
|
+
assert.equal(suggestStack(templateDir), "ts");
|
|
43
|
+
|
|
44
|
+
writeHarnessStack(templateDir, "ts");
|
|
45
|
+
assert.equal(readFileSync(join(templateDir, ".harness-stack"), "utf8"), "ts\n");
|
|
46
|
+
assert.equal(codeownersHasPlaceholder(templateDir), true);
|
|
47
|
+
|
|
48
|
+
const templateDoctor = localChecks(templateDir, { nodeVersion: "22.0.0", templateMode: true });
|
|
49
|
+
assert.ok(templateDoctor.entries.every((entry) => entry.status === "PASS"));
|
|
50
|
+
|
|
51
|
+
applyCodeownersOwner(templateDir, "@acme/platform");
|
|
52
|
+
assert.equal(codeownersHasPlaceholder(templateDir), false);
|
|
53
|
+
const templateDoctorPersonalized = localChecks(templateDir, { nodeVersion: "22.0.0", templateMode: true });
|
|
54
|
+
assert.ok(templateDoctorPersonalized.entries.some((e) => e.label === "CODEOWNERS" && e.status === "FAIL"));
|
|
55
|
+
|
|
56
|
+
const multiProduct = localChecks(templateDir, { nodeVersion: "22.0.0", templateMode: false });
|
|
57
|
+
assert.ok(multiProduct.entries.some((e) => e.label === "product-ci workflow" && e.status === "FAIL"));
|
|
58
|
+
|
|
59
|
+
const productDir = mkdtempSync(join(tmpdir(), "sdlc-gh-wizard-product-"));
|
|
60
|
+
mkdirSync(join(productDir, ".github/workflows"), { recursive: true });
|
|
61
|
+
writeFileSync(join(productDir, ".harness-stack"), "python\n");
|
|
62
|
+
writeFileSync(join(productDir, ".github/workflows/product-ci-python.yml"), "name: product-ci-python\n");
|
|
63
|
+
writeFileSync(join(productDir, ".github/CODEOWNERS"), "* @acme/platform\n");
|
|
64
|
+
const productDoctor = localChecks(productDir, { nodeVersion: "22.0.0", templateMode: false });
|
|
65
|
+
assert.ok(productDoctor.entries.every((entry) => entry.status === "PASS"));
|
|
66
|
+
|
|
67
|
+
const plan = buildWizardPlan({
|
|
68
|
+
repoRoot: templateDir,
|
|
69
|
+
stackId: "ts",
|
|
70
|
+
owner: "@acme/platform",
|
|
71
|
+
githubRepo: "acme/sdlc-gh",
|
|
72
|
+
template: true,
|
|
73
|
+
withEvalRuleset: false,
|
|
74
|
+
yes: true,
|
|
75
|
+
skipGithub: false,
|
|
76
|
+
dryRun: false,
|
|
77
|
+
writeHarnessStack: true,
|
|
78
|
+
patchCodeowners: true,
|
|
79
|
+
});
|
|
80
|
+
assert.ok(plan.steps.some((step) => step.id === "setup-github"));
|
|
81
|
+
assert.ok(plan.steps.some((step) => step.id === "doctor" && step.detail.includes("--template")));
|
|
82
|
+
|
|
83
|
+
const unchangedPlan = buildWizardPlan({
|
|
84
|
+
repoRoot: templateDir,
|
|
85
|
+
stackId: "ts",
|
|
86
|
+
owner: "@acme/other",
|
|
87
|
+
githubRepo: "acme/sdlc-gh",
|
|
88
|
+
template: false,
|
|
89
|
+
withEvalRuleset: false,
|
|
90
|
+
yes: true,
|
|
91
|
+
skipGithub: true,
|
|
92
|
+
dryRun: false,
|
|
93
|
+
writeHarnessStack: true,
|
|
94
|
+
patchCodeowners: false,
|
|
95
|
+
});
|
|
96
|
+
assert.ok(!unchangedPlan.steps.some((step) => step.id === "codeowners" && step.action === "patch"));
|
|
97
|
+
assert.ok(unchangedPlan.steps.some((step) => step.id === "codeowners" && step.action === "skip"));
|
|
98
|
+
|
|
99
|
+
const templatePlan = buildWizardPlan({
|
|
100
|
+
repoRoot: templateDir,
|
|
101
|
+
stackId: "ts",
|
|
102
|
+
owner: "(unchanged)",
|
|
103
|
+
githubRepo: "acme/sdlc-gh",
|
|
104
|
+
template: true,
|
|
105
|
+
withEvalRuleset: false,
|
|
106
|
+
yes: true,
|
|
107
|
+
skipGithub: false,
|
|
108
|
+
dryRun: false,
|
|
109
|
+
writeHarnessStack: true,
|
|
110
|
+
patchCodeowners: false,
|
|
111
|
+
});
|
|
112
|
+
assert.ok(templatePlan.steps.some((step) => step.id === "codeowners" && step.detail.includes("keep template placeholder")));
|
|
113
|
+
|
|
114
|
+
console.log("Setup wizard scenario tests passed");
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { readFileSync } from "node:fs";
|
|
4
|
+
import {
|
|
5
|
+
artifactFilename,
|
|
6
|
+
buildTelemetryArtifact,
|
|
7
|
+
buildTelemetryPayload,
|
|
8
|
+
mapNameToWallFailureType,
|
|
9
|
+
missingRequiredFields,
|
|
10
|
+
resolveRetryCount,
|
|
11
|
+
resolveTaskClass,
|
|
12
|
+
resolveTaskId,
|
|
13
|
+
wallFailureTypeFromJobResults,
|
|
14
|
+
} from "./lib/telemetry-artifact.mjs";
|
|
15
|
+
import { patchHarnessCi, stacksForHarness } from "./lib/harness-ci-fragments.mjs";
|
|
16
|
+
|
|
17
|
+
const base = buildTelemetryPayload({
|
|
18
|
+
repo: "org/repo",
|
|
19
|
+
pr_number: 42,
|
|
20
|
+
pr_body: "fixes #7",
|
|
21
|
+
labels: "task:docs,autonomy:L2,retry:2",
|
|
22
|
+
wall_failure_type: "diff-size",
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
assert.equal(base.payload.task_id, "7");
|
|
26
|
+
assert.equal(base.payload.task_class, "docs");
|
|
27
|
+
assert.equal(base.payload.autonomy_level, "L2");
|
|
28
|
+
assert.equal(base.payload.retry_count, 2);
|
|
29
|
+
assert.equal(base.payload.wall_failure_type, "diff-size");
|
|
30
|
+
assert.equal(missingRequiredFields(base.payload).length, 0);
|
|
31
|
+
assert.ok(base.placeholders.includes("model"));
|
|
32
|
+
|
|
33
|
+
assert.equal(resolveTaskId("", 99), "pr-99");
|
|
34
|
+
assert.equal(resolveTaskClass(["task:test-fix"]), "test-fix");
|
|
35
|
+
assert.equal(resolveRetryCount(["retry:3"]), 3);
|
|
36
|
+
assert.equal(mapNameToWallFailureType("diff-size"), "diff-size");
|
|
37
|
+
assert.equal(mapNameToWallFailureType("product-ci-ts"), "test");
|
|
38
|
+
assert.equal(
|
|
39
|
+
wallFailureTypeFromJobResults({
|
|
40
|
+
"diff-size": { result: "failure" },
|
|
41
|
+
"harness-static": { result: "success" },
|
|
42
|
+
}),
|
|
43
|
+
"diff-size",
|
|
44
|
+
);
|
|
45
|
+
|
|
46
|
+
const artifact = buildTelemetryArtifact({
|
|
47
|
+
source: "harness-ci",
|
|
48
|
+
TELEMETRY_SOURCE: "harness-ci",
|
|
49
|
+
GITHUB_REPOSITORY: "org/repo",
|
|
50
|
+
GITHUB_RUN_ID: "12345",
|
|
51
|
+
pr_number: 10,
|
|
52
|
+
pr_body: "closes #3",
|
|
53
|
+
});
|
|
54
|
+
assert.equal(artifact.schema_version, "1");
|
|
55
|
+
assert.equal(artifact.source, "harness-ci");
|
|
56
|
+
assert.equal(artifact.payload.pr_number, 10);
|
|
57
|
+
assert.match(artifactFilename({ source: "eval-ci", prNumber: 5, workflowRunId: 9 }), /^eval-ci-pr5-run9\.json$/);
|
|
58
|
+
|
|
59
|
+
const harnessCi = readFileSync(".github/workflows/harness-ci.yml", "utf8");
|
|
60
|
+
const trimmedTs = patchHarnessCi(harnessCi, stacksForHarness("ts"));
|
|
61
|
+
assert.ok(!trimmedTs.includes("product-python:"), "trimmed harness-ci should drop product-python job");
|
|
62
|
+
assert.ok(trimmedTs.includes("product-ts:"), "trimmed harness-ci should keep product-ts job");
|
|
63
|
+
assert.ok(
|
|
64
|
+
!/telemetry:[\s\S]*- product-python/.test(trimmedTs),
|
|
65
|
+
"telemetry needs should not reference trimmed product jobs",
|
|
66
|
+
);
|
|
67
|
+
assert.ok(/telemetry:[\s\S]*- product-ts/.test(trimmedTs), "telemetry needs should reference product-ts");
|
|
68
|
+
|
|
69
|
+
console.log("telemetry-artifact scenarios ok");
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Trim harness-ci.yml to selected stack(s) for bootstrapped repositories.
|
|
4
|
+
* Usage: node scripts/trim-harness-ci.mjs <stack-id> <harness-ci.yml path>
|
|
5
|
+
*/
|
|
6
|
+
import { readFileSync, writeFileSync } from "node:fs";
|
|
7
|
+
import { patchHarnessCi, stacksForHarness } from "./lib/harness-ci-fragments.mjs";
|
|
8
|
+
|
|
9
|
+
const [stackId, targetPath] = process.argv.slice(2);
|
|
10
|
+
if (!stackId || !targetPath) {
|
|
11
|
+
console.error("Usage: node scripts/trim-harness-ci.mjs <stack-id> <harness-ci.yml>");
|
|
12
|
+
process.exit(1);
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
const stacks = stacksForHarness(stackId);
|
|
16
|
+
const content = readFileSync(targetPath, "utf8");
|
|
17
|
+
writeFileSync(targetPath, patchHarnessCi(content, stacks));
|
|
18
|
+
console.log(`Trimmed ${targetPath} to stack=${stackId}`);
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Validate gh-aw source .md workflows compile to committed .lock.yml without drift.
|
|
4
|
+
*/
|
|
5
|
+
import { execSync } from "node:child_process";
|
|
6
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
7
|
+
import { GH_AW_SOURCE_WORKFLOWS } from "./lib/gh-aw-dogfood.mjs";
|
|
8
|
+
|
|
9
|
+
function hasGhAw() {
|
|
10
|
+
try {
|
|
11
|
+
execSync("gh aw version", { stdio: ["pipe", "pipe", "pipe"] });
|
|
12
|
+
return true;
|
|
13
|
+
} catch {
|
|
14
|
+
return false;
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function main() {
|
|
19
|
+
const required = process.env.GH_AW_COMPILE_REQUIRED === "1";
|
|
20
|
+
if (!hasGhAw()) {
|
|
21
|
+
const msg = "gh aw CLI not available; compile validation skipped";
|
|
22
|
+
if (required) {
|
|
23
|
+
console.error(`::error::${msg}`);
|
|
24
|
+
process.exit(1);
|
|
25
|
+
}
|
|
26
|
+
console.warn(`::warning::${msg}`);
|
|
27
|
+
process.exit(0);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
const issues = [];
|
|
31
|
+
|
|
32
|
+
for (const wf of GH_AW_SOURCE_WORKFLOWS) {
|
|
33
|
+
if (!existsSync(wf.md)) {
|
|
34
|
+
issues.push(`${wf.md} missing`);
|
|
35
|
+
continue;
|
|
36
|
+
}
|
|
37
|
+
if (!existsSync(wf.lock)) {
|
|
38
|
+
issues.push(`${wf.lock} missing`);
|
|
39
|
+
continue;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
const before = readFileSync(wf.lock, "utf8");
|
|
43
|
+
try {
|
|
44
|
+
execSync(`gh aw compile ${wf.id}.md`, { stdio: ["pipe", "pipe", "pipe"] });
|
|
45
|
+
} catch (error) {
|
|
46
|
+
issues.push(`gh aw compile failed for ${wf.id}: ${error.stderr?.toString() || error.message}`);
|
|
47
|
+
continue;
|
|
48
|
+
}
|
|
49
|
+
const after = readFileSync(wf.lock, "utf8");
|
|
50
|
+
if (before !== after) {
|
|
51
|
+
issues.push(`${wf.lock} drifted after gh aw compile — commit regenerated lock`);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
if (issues.length) {
|
|
56
|
+
console.error("::error::gh-aw compile validation failed:");
|
|
57
|
+
for (const issue of issues) console.error(` - ${issue}`);
|
|
58
|
+
process.exit(1);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
console.log(`gh-aw compile validation passed (${GH_AW_SOURCE_WORKFLOWS.length} workflow(s))`);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
main();
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Validate harness assets: hooks.json, agent frontmatter, instructions presence.
|
|
4
|
+
*/
|
|
5
|
+
import { readFileSync, readdirSync, statSync, existsSync } from "node:fs";
|
|
6
|
+
import { join, relative } from "node:path";
|
|
7
|
+
import { loadStacks } from "./lib/stacks.mjs";
|
|
8
|
+
import { patchHarnessCi, stacksForHarness } from "./lib/harness-ci-fragments.mjs";
|
|
9
|
+
import {
|
|
10
|
+
BOOTSTRAP_LIB_FILES,
|
|
11
|
+
BOOTSTRAP_SCRIPT_MJS,
|
|
12
|
+
SCRIPT_LIB_IMPORTS,
|
|
13
|
+
} from "./lib/bootstrap-copy.mjs";
|
|
14
|
+
import { CODEOWNERS_PLACEHOLDER, detectRepoProfile } from "./lib/setup-wizard.mjs";
|
|
15
|
+
import {
|
|
16
|
+
NPM_PACKAGE_FILES,
|
|
17
|
+
validateNpmSampleCoverage,
|
|
18
|
+
validatePackageJsonFiles,
|
|
19
|
+
} from "./lib/npm-package.mjs";
|
|
20
|
+
|
|
21
|
+
const ROOT = process.cwd();
|
|
22
|
+
let errors = 0;
|
|
23
|
+
|
|
24
|
+
function fail(msg) {
|
|
25
|
+
console.error(`ERROR: ${msg}`);
|
|
26
|
+
errors++;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function warn(msg) {
|
|
30
|
+
console.warn(`WARN: ${msg}`);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function parseFrontmatter(content) {
|
|
34
|
+
const match = content.match(/^---\r?\n([\s\S]*?)\r?\n---/);
|
|
35
|
+
if (!match) return null;
|
|
36
|
+
const fm = {};
|
|
37
|
+
for (const line of match[1].split("\n")) {
|
|
38
|
+
const m = line.match(/^(\w+):\s*(.+)$/);
|
|
39
|
+
if (m) fm[m[1]] = m[2].replace(/^['"]|['"]$/g, "");
|
|
40
|
+
}
|
|
41
|
+
return fm;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
// hooks.json
|
|
45
|
+
const hooksPath = join(ROOT, ".github/hooks/hooks.json");
|
|
46
|
+
if (existsSync(hooksPath)) {
|
|
47
|
+
try {
|
|
48
|
+
const hooks = JSON.parse(readFileSync(hooksPath, "utf8"));
|
|
49
|
+
if (!Array.isArray(hooks.hooks) && !hooks.hooks) {
|
|
50
|
+
fail("hooks.json: expected hooks array or object with hooks key");
|
|
51
|
+
}
|
|
52
|
+
} catch (e) {
|
|
53
|
+
fail(`hooks.json: invalid JSON — ${e.message}`);
|
|
54
|
+
}
|
|
55
|
+
} else {
|
|
56
|
+
warn("hooks.json not found (optional until Phase 1)");
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
// agents
|
|
60
|
+
const agentsDir = join(ROOT, ".github/agents");
|
|
61
|
+
if (existsSync(agentsDir)) {
|
|
62
|
+
for (const f of readdirSync(agentsDir).filter((x) => x.endsWith(".agent.md"))) {
|
|
63
|
+
const content = readFileSync(join(agentsDir, f), "utf8");
|
|
64
|
+
const fm = parseFrontmatter(content);
|
|
65
|
+
if (!fm) fail(`${f}: missing YAML frontmatter`);
|
|
66
|
+
else {
|
|
67
|
+
if (!fm.name) fail(`${f}: frontmatter missing 'name'`);
|
|
68
|
+
if (!fm.description) fail(`${f}: frontmatter missing 'description'`);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// instructions
|
|
74
|
+
const instrDir = join(ROOT, ".github/instructions");
|
|
75
|
+
if (existsSync(instrDir)) {
|
|
76
|
+
const walk = (dir) => {
|
|
77
|
+
for (const e of readdirSync(dir)) {
|
|
78
|
+
const p = join(dir, e);
|
|
79
|
+
if (statSync(p).isDirectory()) walk(p);
|
|
80
|
+
else if (e.endsWith(".instructions.md")) {
|
|
81
|
+
const fm = parseFrontmatter(readFileSync(p, "utf8"));
|
|
82
|
+
if (!fm?.description) fail(`${relative(ROOT, p)}: missing description in frontmatter`);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
};
|
|
86
|
+
walk(instrDir);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// labels.yml
|
|
90
|
+
if (existsSync(join(ROOT, ".github/labels.yml"))) {
|
|
91
|
+
const labels = readFileSync(join(ROOT, ".github/labels.yml"), "utf8");
|
|
92
|
+
if (!labels.includes("task:")) warn("labels.yml: no task:* labels found");
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// stack catalog consistency
|
|
96
|
+
const catalogPath = join(ROOT, "config/stacks.json");
|
|
97
|
+
const harnessCiPath = join(ROOT, ".github/workflows/harness-ci.yml");
|
|
98
|
+
if (existsSync(catalogPath) && existsSync(harnessCiPath)) {
|
|
99
|
+
const harnessCi = readFileSync(harnessCiPath, "utf8");
|
|
100
|
+
try {
|
|
101
|
+
const expected = patchHarnessCi(harnessCi, stacksForHarness());
|
|
102
|
+
if (harnessCi !== expected) {
|
|
103
|
+
fail("harness-ci.yml: detect/product jobs are out of sync with config/stacks.json");
|
|
104
|
+
}
|
|
105
|
+
} catch (e) {
|
|
106
|
+
fail(`harness-ci.yml: ${e.message}`);
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
for (const stack of loadStacks()) {
|
|
110
|
+
const profilePath = join(ROOT, ".github/instructions/profiles", stack.profile);
|
|
111
|
+
const workflowPath = join(ROOT, ".github/workflows", stack.workflow);
|
|
112
|
+
const sampleMarkerPath = join(ROOT, stack.sampleMarker);
|
|
113
|
+
|
|
114
|
+
if (!existsSync(profilePath)) {
|
|
115
|
+
fail(`missing profile for stack ${stack.id}: ${stack.profile}`);
|
|
116
|
+
}
|
|
117
|
+
if (!existsSync(workflowPath)) {
|
|
118
|
+
fail(`missing workflow for stack ${stack.id}: ${stack.workflow}`);
|
|
119
|
+
}
|
|
120
|
+
if (!existsSync(sampleMarkerPath)) {
|
|
121
|
+
fail(`missing sample marker for stack ${stack.id}: ${stack.sampleMarker}`);
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
const workflow = readFileSync(workflowPath, "utf8");
|
|
125
|
+
const sampleDir = `sample/${stack.sampleDir}`;
|
|
126
|
+
if (!workflow.includes(sampleDir) || !workflow.includes(stack.marker)) {
|
|
127
|
+
fail(`${stack.workflow}: missing sample/root resolution for ${stack.id}`);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
} else {
|
|
131
|
+
warn("config/stacks.json or harness-ci.yml not found — skipping stack catalog checks");
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
// script -> lib import resolution (bootstrap copy drift guard)
|
|
135
|
+
for (const [script, libs] of Object.entries(SCRIPT_LIB_IMPORTS)) {
|
|
136
|
+
const scriptPath = join(ROOT, "scripts", script);
|
|
137
|
+
if (!existsSync(scriptPath)) {
|
|
138
|
+
fail(`missing script entrypoint: scripts/${script}`);
|
|
139
|
+
continue;
|
|
140
|
+
}
|
|
141
|
+
for (const lib of libs) {
|
|
142
|
+
if (!existsSync(join(ROOT, "scripts/lib", lib))) {
|
|
143
|
+
fail(`scripts/${script} requires scripts/lib/${lib} but file is missing`);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
for (const lib of BOOTSTRAP_LIB_FILES) {
|
|
149
|
+
if (!existsSync(join(ROOT, "scripts/lib", lib))) {
|
|
150
|
+
fail(`bootstrap lib manifest missing in template: scripts/lib/${lib}`);
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
for (const script of BOOTSTRAP_SCRIPT_MJS) {
|
|
155
|
+
if (!existsSync(join(ROOT, "scripts", script))) {
|
|
156
|
+
fail(`bootstrap script manifest missing in template: scripts/${script}`);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
const bootstrapSh = readFileSync(join(ROOT, "scripts/bootstrap-harness.sh"), "utf8");
|
|
161
|
+
for (const lib of BOOTSTRAP_LIB_FILES) {
|
|
162
|
+
if (!bootstrapSh.includes(lib)) {
|
|
163
|
+
fail(`bootstrap-harness.sh does not copy scripts/lib/${lib}`);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
const pkgFilesCheck = validatePackageJsonFiles(ROOT);
|
|
168
|
+
if (!pkgFilesCheck.ok) {
|
|
169
|
+
fail(pkgFilesCheck.reason);
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
const npmCoverage = validateNpmSampleCoverage(ROOT);
|
|
173
|
+
for (const entry of npmCoverage.missingOnDisk) {
|
|
174
|
+
fail(`npm package files entry missing on disk: ${entry}`);
|
|
175
|
+
}
|
|
176
|
+
for (const relPath of npmCoverage.notPacked) {
|
|
177
|
+
fail(`sample file not covered by npm files (${NPM_PACKAGE_FILES.length} entries): ${relPath}`);
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
const templateProfile = detectRepoProfile(ROOT);
|
|
181
|
+
if (templateProfile.template) {
|
|
182
|
+
const codeownersPath = join(ROOT, ".github/CODEOWNERS");
|
|
183
|
+
if (!existsSync(codeownersPath)) {
|
|
184
|
+
fail("template repo missing .github/CODEOWNERS");
|
|
185
|
+
} else {
|
|
186
|
+
const codeowners = readFileSync(codeownersPath, "utf8");
|
|
187
|
+
if (!codeowners.includes(CODEOWNERS_PLACEHOLDER)) {
|
|
188
|
+
fail(
|
|
189
|
+
`template repo .github/CODEOWNERS must keep placeholder ${CODEOWNERS_PLACEHOLDER} (do not commit personal or org-specific owners)`,
|
|
190
|
+
);
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
if (errors > 0) {
|
|
196
|
+
console.error(`\n${errors} validation error(s)`);
|
|
197
|
+
process.exit(1);
|
|
198
|
+
}
|
|
199
|
+
console.log("Harness validation passed");
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/** Validate telemetry payload against docs/telemetry-schema.md required fields */
|
|
3
|
+
import { missingRequiredFields, TELEMETRY_REQUIRED_FIELDS } from "./lib/telemetry-artifact.mjs";
|
|
4
|
+
|
|
5
|
+
const raw = process.argv[2] || "{}";
|
|
6
|
+
const parsed = JSON.parse(raw);
|
|
7
|
+
const payload = parsed.payload ?? parsed;
|
|
8
|
+
|
|
9
|
+
const missing = missingRequiredFields(payload);
|
|
10
|
+
if (missing.length) {
|
|
11
|
+
console.error("Missing telemetry fields:", missing.join(", "));
|
|
12
|
+
process.exit(1);
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
if (process.env.HARNESS_STRICT_TELEMETRY === "1" && parsed.placeholders?.length) {
|
|
16
|
+
console.error("Strict telemetry: unresolved placeholders:", parsed.placeholders.join(", "));
|
|
17
|
+
process.exit(1);
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
console.log("Telemetry payload valid");
|
|
21
|
+
console.log(`Required field count: ${TELEMETRY_REQUIRED_FIELDS.length}`);
|