@next-friday/create-app 2.1.9 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/package.json +1 -1
  2. package/templates/base/.agents/agents.json +7 -0
  3. package/templates/base/.agents/hooks/agent-format-after-edit.sh +15 -0
  4. package/templates/base/.agents/hooks/agent-production-gate.sh +10 -0
  5. package/templates/base/.agents/hooks/agent-test-edit-gate.sh +25 -0
  6. package/templates/base/.agents/review.md +21 -0
  7. package/templates/base/.agents/rules/no-suppression.md +1 -1
  8. package/templates/base/.agents/skills/babysit-pr/SKILL.md +8 -0
  9. package/templates/base/.agents/skills/brand-policy/SKILL.md +12 -0
  10. package/templates/base/.agents/skills/compliance-policy/SKILL.md +12 -0
  11. package/templates/base/.agents/skills/security-policy/SKILL.md +12 -0
  12. package/templates/base/.agents/skills/ux-policy/SKILL.md +12 -0
  13. package/templates/base/.agents/skills/write-intent/SKILL.md +30 -0
  14. package/templates/base/.agents/skills/write-spec/SKILL.md +12 -0
  15. package/templates/base/.claude/managed-settings.example.json +26 -0
  16. package/templates/base/.claude/settings.json +31 -0
  17. package/templates/base/.claude/skills/babysit-pr/SKILL.md +8 -0
  18. package/templates/base/.claude/skills/brand-policy/SKILL.md +12 -0
  19. package/templates/base/.claude/skills/compliance-policy/SKILL.md +12 -0
  20. package/templates/base/.claude/skills/security-policy/SKILL.md +12 -0
  21. package/templates/base/.claude/skills/ux-policy/SKILL.md +12 -0
  22. package/templates/base/.claude/skills/write-intent/SKILL.md +30 -0
  23. package/templates/base/.claude/skills/write-spec/SKILL.md +12 -0
  24. package/templates/base/.cursor/BUGBOT.md +1 -0
  25. package/templates/base/.cursor/hooks.json +15 -0
  26. package/templates/base/.cursor/rules/no-suppression.mdc +1 -1
  27. package/templates/base/.cursor/skills/babysit-pr/SKILL.md +8 -0
  28. package/templates/base/.cursor/skills/brand-policy/SKILL.md +12 -0
  29. package/templates/base/.cursor/skills/compliance-policy/SKILL.md +12 -0
  30. package/templates/base/.cursor/skills/security-policy/SKILL.md +12 -0
  31. package/templates/base/.cursor/skills/ux-policy/SKILL.md +12 -0
  32. package/templates/base/.cursor/skills/write-intent/SKILL.md +30 -0
  33. package/templates/base/.cursor/skills/write-spec/SKILL.md +12 -0
  34. package/templates/base/.github/workflows/agent-evals.yml +128 -0
  35. package/templates/base/.github/workflows/claude.yml +39 -0
  36. package/templates/base/.github/zizmor.yml +1 -1
  37. package/templates/base/REVIEW.md +21 -0
  38. package/templates/base/SETUP.md +108 -0
  39. package/templates/base/bands.yaml +10 -0
  40. package/templates/base/evals/README.md +15 -0
  41. package/templates/base/evals/check.sh +17 -0
  42. package/templates/base/intent/README.md +8 -0
  43. package/templates/base/lessons/README.md +3 -0
  44. package/templates/base/scripts/sync-agent-rules.ts +111 -1
  45. package/templates/nestjs/.agents/agents/verifier.md +1 -0
  46. package/templates/nestjs/.claude/agents/verifier.md +7 -0
  47. package/templates/nestjs/.cursor/agents/verifier.md +7 -0
  48. package/templates/nestjs/.github/workflows/ci.yml +44 -0
  49. package/templates/nestjs/AGENTS.md +17 -4
  50. package/templates/nestjs/README.md +1 -16
  51. package/templates/nextjs/.agents/agents/verifier.md +1 -0
  52. package/templates/nextjs/.agents/mcp.json +8 -0
  53. package/templates/nextjs/.claude/agents/verifier.md +7 -0
  54. package/templates/nextjs/.cursor/agents/verifier.md +7 -0
  55. package/templates/nextjs/.cursor/mcp.json +8 -0
  56. package/templates/nextjs/.github/workflows/ci.yml +44 -0
  57. package/templates/nextjs/.mcp.json +8 -0
  58. package/templates/nextjs/AGENTS.md +17 -4
  59. package/templates/nextjs/README.md +1 -16
  60. package/templates/nodejs/.agents/agents/verifier.md +1 -0
  61. package/templates/nodejs/.claude/agents/verifier.md +7 -0
  62. package/templates/nodejs/.cursor/agents/verifier.md +7 -0
  63. package/templates/nodejs/.github/workflows/ci.yml +44 -0
  64. package/templates/nodejs/AGENTS.md +17 -4
  65. package/templates/nodejs/README.md +1 -16
  66. package/templates/payload/.agents/agents/verifier.md +1 -0
  67. package/templates/payload/.agents/mcp.json +8 -0
  68. package/templates/payload/.claude/agents/verifier.md +7 -0
  69. package/templates/payload/.cursor/agents/verifier.md +7 -0
  70. package/templates/payload/.cursor/mcp.json +8 -0
  71. package/templates/payload/.github/workflows/ci.yml +44 -0
  72. package/templates/payload/.mcp.json +8 -0
  73. package/templates/payload/AGENTS.md +17 -4
  74. package/templates/payload/README.md +1 -16
  75. package/templates/turborepo/.agents/agents/verifier.md +1 -0
  76. package/templates/turborepo/.agents/mcp.json +8 -0
  77. package/templates/turborepo/.claude/agents/verifier.md +7 -0
  78. package/templates/turborepo/.cursor/agents/verifier.md +7 -0
  79. package/templates/turborepo/.cursor/mcp.json +8 -0
  80. package/templates/turborepo/.github/workflows/ci.yml +44 -0
  81. package/templates/turborepo/.mcp.json +8 -0
  82. package/templates/turborepo/AGENTS.md +17 -4
  83. package/templates/turborepo/README.md +1 -16
@@ -0,0 +1,128 @@
1
+ name: Agent evals
2
+
3
+ on:
4
+ pull_request:
5
+ types: [opened, synchronize, reopened, ready_for_review]
6
+ schedule:
7
+ - cron: "0 2 * * *"
8
+
9
+ permissions: {}
10
+
11
+ concurrency:
12
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
13
+ cancel-in-progress: true
14
+
15
+ jobs:
16
+ list:
17
+ name: List evals
18
+ runs-on: ubuntu-latest
19
+ timeout-minutes: 5
20
+ permissions:
21
+ contents: read
22
+ outputs:
23
+ evals: ${{ steps.list.outputs.evals }}
24
+ steps:
25
+ - name: Checkout repository
26
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
27
+ with:
28
+ fetch-depth: 0
29
+ persist-credentials: false
30
+ - name: List the evals to run
31
+ id: list
32
+ env:
33
+ EVENT_NAME: ${{ github.event_name }}
34
+ BASE_SHA: ${{ github.event.pull_request.base.sha }}
35
+ run: |
36
+ if [[ "$EVENT_NAME" == "schedule" ]] || git diff --name-only "$BASE_SHA" HEAD | grep -qE '^(CLAUDE\.md|AGENTS\.md|REVIEW\.md|\.mcp\.json|\.agents/|\.claude/|\.cursor/|evals/)'; then
37
+ echo "evals=$(find evals -maxdepth 1 -name '*.json' | sort | jq -R -s -c 'split("\n")[:-1]')" >> "$GITHUB_OUTPUT"
38
+ else
39
+ echo 'evals=[]' >> "$GITHUB_OUTPUT"
40
+ fi
41
+
42
+ eval:
43
+ name: Eval
44
+ needs: list
45
+ if: needs.list.outputs.evals != '[]'
46
+ runs-on: ubuntu-latest
47
+ timeout-minutes: 30
48
+ permissions:
49
+ contents: read
50
+ id-token: write # Needed to authenticate to Anthropic with the workflow's OIDC token.
51
+ strategy:
52
+ fail-fast: false
53
+ matrix:
54
+ eval: ${{ fromJSON(needs.list.outputs.evals) }}
55
+ steps:
56
+ - name: Checkout repository
57
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
58
+ with:
59
+ persist-credentials: false
60
+ - name: Set up Node.js and pnpm
61
+ uses: pnpm/setup@fbda4c85fc2e1e08721cd8763afea8f48d60f024 # v3.0.0
62
+ with:
63
+ cache: true
64
+ require-lockfile: true
65
+ - name: Read the eval prompt
66
+ id: read
67
+ env:
68
+ EVAL: ${{ matrix.eval }}
69
+ run: |
70
+ {
71
+ echo "prompt<<EVAL_PROMPT_END"
72
+ jq -r '.prompt' "$EVAL"
73
+ echo "EVAL_PROMPT_END"
74
+ } >> "$GITHUB_OUTPUT"
75
+ - name: Run the eval task
76
+ id: claude
77
+ uses: anthropics/claude-code-action@61a9c274fa7110a66dfafc1bbbced03833eb25fe # v1.0.269
78
+ with:
79
+ anthropic_federation_rule_id: ${{ vars.ANTHROPIC_FEDERATION_RULE_ID }}
80
+ anthropic_organization_id: ${{ vars.ANTHROPIC_ORGANIZATION_ID }}
81
+ prompt: ${{ steps.read.outputs.prompt }}
82
+ claude_args: --permission-mode acceptEdits --allowedTools "Read,Edit,Bash(pnpm test)"
83
+ - name: Check the result
84
+ env:
85
+ EVAL: ${{ matrix.eval }}
86
+ CONCLUSION: ${{ steps.claude.outputs.conclusion }}
87
+ run: ./evals/check.sh "$EVAL" "$CONCLUSION"
88
+
89
+ evals:
90
+ name: Agent evals
91
+ needs:
92
+ - list
93
+ - eval
94
+ if: always()
95
+ runs-on: ubuntu-latest
96
+ timeout-minutes: 5
97
+ permissions:
98
+ actions: read # Needed to read the result of each eval job.
99
+ steps:
100
+ - name: Check the pass rate
101
+ env:
102
+ EVALS: ${{ needs.list.outputs.evals }}
103
+ LIST_RESULT: ${{ needs.list.result }}
104
+ EVALS_MIN_PASS_RATE: ${{ vars.EVALS_MIN_PASS_RATE }}
105
+ GH_TOKEN: ${{ github.token }}
106
+ REPOSITORY: ${{ github.repository }}
107
+ RUN_ID: ${{ github.run_id }}
108
+ run: |
109
+ if [[ "$LIST_RESULT" != "success" ]]; then
110
+ echo "::error::Listing the evals failed."
111
+ exit 1
112
+ fi
113
+ total=$(jq 'length' <<< "$EVALS")
114
+ if ((total == 0)); then
115
+ echo "No evals to run."
116
+ exit 0
117
+ fi
118
+ if [[ -z "$EVALS_MIN_PASS_RATE" ]]; then
119
+ echo "::error::Set the repository variable EVALS_MIN_PASS_RATE (0 to 100)."
120
+ exit 1
121
+ fi
122
+ passed=$(gh api "repos/$REPOSITORY/actions/runs/$RUN_ID/jobs" --paginate --jq '.jobs[] | select((.name | startswith("Eval (")) and .conclusion == "success") | .id' | wc -l)
123
+ rate=$((passed * 100 / total))
124
+ echo "Pass rate: $rate% ($passed of $total)"
125
+ if ((rate < EVALS_MIN_PASS_RATE)); then
126
+ echo "::error::The pass rate $rate% is below EVALS_MIN_PASS_RATE ($EVALS_MIN_PASS_RATE%)."
127
+ exit 1
128
+ fi
@@ -0,0 +1,39 @@
1
+ name: Claude
2
+
3
+ on:
4
+ pull_request_review_comment:
5
+ types: [created]
6
+ pull_request_review:
7
+ types: [submitted]
8
+
9
+ permissions: {}
10
+
11
+ concurrency:
12
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
13
+ cancel-in-progress: false
14
+
15
+ jobs:
16
+ claude:
17
+ name: Address review comment
18
+ if: |
19
+ (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
20
+ (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude'))
21
+ runs-on: ubuntu-latest
22
+ timeout-minutes: 60
23
+ permissions:
24
+ contents: write # Needed to push the fix.
25
+ pull-requests: write # Needed to reply on the pull request.
26
+ id-token: write # Needed by the action to authenticate.
27
+ actions: read # Needed to read CI results on pull requests.
28
+ steps:
29
+ - name: Checkout repository
30
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
31
+ with:
32
+ fetch-depth: 1
33
+ persist-credentials: false
34
+ - name: Address the comment
35
+ uses: anthropics/claude-code-action@61a9c274fa7110a66dfafc1bbbced03833eb25fe # v1.0.269
36
+ with:
37
+ anthropic_federation_rule_id: ${{ vars.ANTHROPIC_FEDERATION_RULE_ID }}
38
+ anthropic_organization_id: ${{ vars.ANTHROPIC_ORGANIZATION_ID }}
39
+ claude_args: --permission-mode acceptEdits
@@ -1,4 +1,4 @@
1
- # Owner: @your-org/your-team. Reason: Semgrep rejects the `$/` self-repository syntax, so local actions use `./`. Expires: 2027-04-29.
1
+ # Owner: @your-org/your-team. Reason: Semgrep rejects the `$/` self-repository syntax, so local actions use `./`.
2
2
  rules:
3
3
  self-repository:
4
4
  disable: true
@@ -0,0 +1,21 @@
1
+ # Review instructions
2
+
3
+ ## Passes
4
+
5
+ Run three passes and tag each finding with its pass:
6
+
7
+ - Bugs: logic errors, broken edge cases, subtle regressions
8
+ - Security: injection risks, authentication gaps, PII in logs
9
+ - Compliance: the change matches spec.md, plan.md and our design principles
10
+
11
+ ## What Important means here
12
+
13
+ Reserve Important for findings that would break behavior, leak data or breach a policy. Style and naming are nits.
14
+
15
+ ## Cap the nits
16
+
17
+ Report at most five nits per review; summarize the rest as a count.
18
+
19
+ ## Do not report
20
+
21
+ Generated files under src/gen/ and anything CI already enforces.
@@ -0,0 +1,108 @@
1
+ # Setup
2
+
3
+ This project follows Anthropic's [AI-native SDLC Playbook](https://claude.com/resources/articles/the-ai-native-sdlc-playbook) in Claude Code and Cursor. Do the steps below in order. Each step says who does it and how to check that it is done.
4
+
5
+ ## How a change flows
6
+
7
+ 1. **Plan:** an idea, a ticket, or an incident becomes `intent/<change>/intent.md` (`write-intent` skill). The product owner merges or closes it.
8
+ 2. **Design:** `spec.md` is written from the accepted intent with the policy skills loaded (`write-spec` skill). The product owner resolves concerns with the policy owners.
9
+ 3. **Build:** the engineer starts in plan mode, approves the plan, and commits it as `plan.md`. A tech lead approves higher-risk plans.
10
+ 4. **Test:** the session runs **Verifying your work** in `AGENTS.md`; `evals/` tests the agent configuration.
11
+ 5. **Review:** Claude Code Review and Bugbot review by `REVIEW.md`; `@claude` on a review comment makes Claude push a fix. A code owner approves; the agent never does.
12
+ 6. **Deploy:** a failed build gets a triage summary on the pull request; `agent-production-gate.sh` gates production.
13
+ 7. **Maintain:** each incident gets a post-mortem in `lessons/` and an eval; larger work becomes a new `intent.md`.
14
+
15
+ ## 1. First commit and push
16
+
17
+ **Who:** an engineer.
18
+
19
+ 1. Install gitleaks and jq (see [CONTRIBUTING.md](CONTRIBUTING.md)).
20
+ 2. Replace the template's description and owner: `package.json#description`, `repository.url`, and `author`, and any template text that `README.md` → **After creating a project from this template** lists. Change `license` from `UNLICENSED` if the project is open source.
21
+ 3. Create an empty repository on GitHub. Make the first commit on `main` and push it there. The Git hooks allow only this first commit and push on `main`; work on branches after that.
22
+
23
+ ```sh
24
+ git add --all
25
+ git commit -m "chore(repo): start from template"
26
+ git remote add origin https://github.com/OWNER/REPOSITORY.git
27
+ git push --set-upstream origin main
28
+ ```
29
+
30
+ **Check:** `git log --oneline origin/main` prints one commit.
31
+
32
+ ## 2. Owners
33
+
34
+ **Who:** an engineer, with the product owner, a lead, and each policy owner.
35
+
36
+ In `.github/CODEOWNERS`, add these lines in this order. GitHub uses the last matching line for each file ([code owners](https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners)), so the line for every file comes first:
37
+
38
+ ```text
39
+ * @your-org/your-team
40
+ /intent/ @the-product-owner
41
+ /.agents/skills/write-intent/ @the-lead-who-signs-off-the-intent-template
42
+ /.agents/skills/brand-policy/ @the-brand-policy-owner
43
+ /.agents/skills/security-policy/ @the-security-policy-owner
44
+ /.agents/skills/compliance-policy/ @the-compliance-policy-owner
45
+ /.agents/skills/ux-policy/ @the-ux-policy-owner
46
+ ```
47
+
48
+ Also replace the owner placeholder in `.github/zizmor.yml`.
49
+
50
+ **Check:** on GitHub, open `.github/CODEOWNERS`; GitHub shows no errors in the file.
51
+
52
+ ## 3. GitHub repository settings
53
+
54
+ **Who:** a repository admin.
55
+
56
+ - **Actions → General → Workflow permissions:** select **Read repository contents and packages permissions**. Every workflow requests the extra permissions it needs.
57
+ - **Advanced Security:** enable private vulnerability reporting, Dependabot alerts, Dependabot security updates, secret scanning, and push protection. `SECURITY.md` sends reporters to private vulnerability reporting.
58
+ - **Secrets and variables → Actions:**
59
+ - the variables `ANTHROPIC_FEDERATION_RULE_ID` and `ANTHROPIC_ORGANIZATION_ID`, from step 4. They are identifiers, not secrets: `.github/workflows/claude.yml`, `.github/workflows/agent-evals.yml`, and the **Triage failed build** job in `ci.yml` exchange the workflow's OIDC token for short-lived Claude API access, so the repository stores no Claude token.
60
+ - the variable `EVALS_MIN_PASS_RATE`, from 0 to 100, once the team has decided it (step 6).
61
+ - **Rules → Rulesets:** protect `main` with:
62
+ - squash merges only, a linear history, and no force pushes or deletion;
63
+ - the required checks `All Checks`, `Validate pull request title`, and `Agent evals`;
64
+ - a pull request before merging, with at least one approval, review from Code Owners, and stale approvals dismissed when new commits are pushed. Claude Code Review follows the severity, exclusion, and nit-cap rules in `REVIEW.md` only on a default branch protected this way ([customize Code Review](https://code.claude.com/docs/en/code-review-customize)).
65
+
66
+ Rulesets in a private repository need GitHub Pro, Team, or Enterprise; on GitHub Free, make the repository public or upgrade.
67
+
68
+ - Create the `needs triage` label, which the issue forms apply: `gh label create "needs triage"`.
69
+
70
+ **Check:** a test pull request shows the three required checks and asks for a code owner's review.
71
+
72
+ ## 4. Claude
73
+
74
+ **Who:** an Owner of the Claude organization, with an owner of the GitHub organization.
75
+
76
+ - **Code Review:** in [**Admin settings → Claude Code**](https://claude.ai/admin-settings/claude-code#code-review), turn on **Allow code review**, install the Claude GitHub App, and add this repository ([set up Code Review](https://code.claude.com/docs/en/code-review-setup)). It needs a Team or Enterprise plan ([availability](https://code.claude.com/docs/en/code-review#availability)). Keep **Review runs** at **After every push**, the default, so every push to a pull request gets a review. Reviews run on Anthropic's infrastructure, so the repository has no review workflow. The same app also serves `.github/workflows/claude.yml`.
77
+ - **Workload identity federation:** in the [Claude Console](https://platform.claude.com), under **Settings → Workload identity**, register the issuer `https://token.actions.githubusercontent.com`, create a service account in the workspace the workflows should use, and create a federation rule for it that matches this repository's subject prefix, `repo:OWNER/REPOSITORY:` ([setup](https://github.com/anthropics/claude-code-action/blob/main/docs/setup.md), [workload identity federation](https://platform.claude.com/docs/en/manage-claude/workload-identity-federation)). Put the rule's `fdrl_...` ID and the organization ID in the repository variables of step 3. Usage is billed to the Console organization.
78
+ - **Managed settings:** deploy the non-negotiable settings and hooks as [managed settings](https://code.claude.com/docs/en/settings), tailored from the playbook's worked example in `.claude/managed-settings.example.json`. Engineers cannot override them.
79
+ - **Monitoring:** turn on the [OpenTelemetry export](https://code.claude.com/docs/en/monitoring-usage) that the playbook's indicators read.
80
+
81
+ **Check:** the test pull request gets a **Claude Code Review** check run within a few minutes. A review comment that tags `@claude` starts the **Claude** workflow.
82
+
83
+ ## 5. Cursor
84
+
85
+ **Who:** a Cursor team admin.
86
+
87
+ - **Bugbot:** enable the repository at [Bugbot](https://cursor.com/automations/from-cursor/bugbot). Bugbot reviews pull requests by the files `.cursor/BUGBOT.md` links, including `.agents/review.md`.
88
+ - **Sandbox:** set the team's sandbox policy ([Cursor sandbox](https://cursor.com/docs/reference/sandbox)), which repository and user settings cannot weaken.
89
+
90
+ **Check:** the test pull request gets a Bugbot review.
91
+
92
+ ## 6. Team decisions
93
+
94
+ **Who:** the people named in each item.
95
+
96
+ - **Sources of truth (engineering lead):** for each artifact, name one system as the source of truth: this repository, an existing tool such as Jira, or, as the minimum, a link both ways (the record id in the file and the commit SHA in the record).
97
+ - **Higher risk (engineering lead):** decide which changes count as higher risk. A tech lead or architect approves their specs and plans.
98
+ - **Policies (each policy owner, with an engineer):** write the policy in `.agents/skills/<area>-policy/SKILL.md` from the owner's source of truth, then run `pnpm rules:sync`. Test that the skill triggers: ask for the relevant task in different ways and confirm the skill loads each time. The policy owner approves every later change as its code owner.
99
+ - **Production gate (engineering leadership, with change management and compliance):** list the approvals that must survive. A person then changes `.agents/hooks/agent-production-gate.sh` to match this project's production deploy command and approval; agent settings deny the file to agents.
100
+ - **Review (tech lead):** replace `src/gen/` in `.agents/review.md` with this project's generated paths, then run `pnpm rules:sync`, which renders it to `REVIEW.md` for Claude Code Review. Once a month, rate the review findings and tune `.agents/review.md`.
101
+ - **Evals (platform engineer):** collect 20 to 50 real tasks in `evals/` (`evals/README.md`), and set `EVALS_MIN_PASS_RATE` (step 3).
102
+ - **People without Git experience:** connect GitHub to Claude (claude.ai or Cowork), so Claude commits `intent.md` for them.
103
+
104
+ ## 7. Later, as the guardrails mature
105
+
106
+ - Run the `write-spec` skill by hand first, then as an organization-level slash command, then as a job that commits `spec.md` as a pull request when an `intent.md` merges.
107
+ - Make auto mode the default for routine work: a tight `spec.md`, a small blast radius, and code the tests already cover.
108
+ - Replace `bands.yaml` with this project's metric, add a unit-tested detection script that reads it, and add the trigger that runs it.
@@ -0,0 +1,10 @@
1
+ # Placeholder: the example from the AI-native SDLC Playbook. Replace it with this project's metric, and add a
2
+ # version-controlled, unit-tested detection script that reads this file. Detection stays deterministic, with no
3
+ # model involved.
4
+ metric: ci_test_failure_rate
5
+ baseline: rolling_30d
6
+ rules: western_electric
7
+ tiers:
8
+ 1sigma: {action: log}
9
+ 2sigma: {action: diagnose, tools: "Read,Grep,Bash(gh run view *)"}
10
+ 3sigma: {action: propose, routes: [pull_request, runbook:rollback-deploy]}
@@ -0,0 +1,15 @@
1
+ # Agent evals
2
+
3
+ Each eval is one real task from recent work with its accepted outcome, written as `evals/<name>.json`: the prompt, plus the check that defines an acceptable result.
4
+
5
+ ```json
6
+ {
7
+ "prompt": "<the task, as a person would ask it>",
8
+ "check": "<a shell command that exits 0 when the result is acceptable, for example: pnpm test>"
9
+ }
10
+ ```
11
+
12
+ - Collect 20 to 50 real tasks from recent work.
13
+ - `.github/workflows/agent-evals.yml` runs the suite every night, and on every pull request that changes `CLAUDE.md`, `AGENTS.md`, `.mcp.json`, `.agents/`, `.claude/`, `.cursor/`, or `evals/`. `evals/check.sh` checks one eval's result.
14
+ - Set the repository variable `EVALS_MIN_PASS_RATE`, from 0 to 100. When the pass rate falls below it, the `Agent evals` check fails and the pull request cannot merge.
15
+ - Each production incident gets an eval, written by the team that owned the incident, and it stays in the suite.
@@ -0,0 +1,17 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+
4
+ eval_file=$1
5
+ conclusion=$2
6
+
7
+ if [[ $conclusion != success ]]; then
8
+ echo "FAIL $eval_file: Claude Code reported $conclusion"
9
+ exit 1
10
+ fi
11
+
12
+ if bash -c "$(jq -r '.check' "$eval_file")"; then
13
+ echo "PASS $eval_file"
14
+ else
15
+ echo "FAIL $eval_file"
16
+ exit 1
17
+ fi
@@ -0,0 +1,8 @@
1
+ # Intent
2
+
3
+ This folder is the intent home. Every change starts here as an `intent.md`, and its record stays here. Git history keeps the author, the time, and every revision of each file.
4
+
5
+ - Each change has its own folder, `intent/<change>/`, where `<change>` is a short kebab-case name. The folder holds `intent.md` (Plan), then `spec.md` (Design), then `plan.md` (Build).
6
+ - **Plan.** An idea, a ticket, or an incident becomes `intent.md`, written with the `write-intent` skill. The product owner reviews and corrects it before it is committed. Merging its pull request accepts it; closing the pull request rejects it.
7
+ - **Design.** The `write-spec` skill turns the accepted `intent.md` into `spec.md`, committed alongside it. The product owner reviews the spec against the idea and resolves each area of concern with its policy owner. Accepting the spec starts the build.
8
+ - **Build.** The engineer starts in plan mode with `intent.md` and `spec.md`, and commits the approved plan as `plan.md`. When implementation departs from the plan, `plan.md` changes in the same commit.
@@ -0,0 +1,3 @@
1
+ # Lessons
2
+
3
+ Post-mortems, one version-controlled file per incident, named like `lessons/2026-06-checkout-cache.md`. Future investigations read them. When the fix ships, add an eval for the incident, as `evals/README.md` describes.
@@ -1,4 +1,4 @@
1
- import {existsSync, readdirSync, readFileSync, writeFileSync} from "node:fs";
1
+ import {existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync} from "node:fs";
2
2
  import path from "node:path";
3
3
 
4
4
  type RuleConfig = {
@@ -6,6 +6,12 @@ type RuleConfig = {
6
6
  globs?: string[];
7
7
  };
8
8
 
9
+ type SubagentConfig = {
10
+ description: string;
11
+ readonly?: boolean;
12
+ tools?: string[];
13
+ };
14
+
9
15
  const sourceDirectory = ".agents/rules";
10
16
  const configPath = ".agents/rules.json";
11
17
 
@@ -85,9 +91,109 @@ for (const [file, content] of Object.entries(claudeExtras)) {
85
91
  expected.set(path.join(claudeDirectory, file), content);
86
92
  }
87
93
 
94
+ const skillSourceDirectory = ".agents/skills";
95
+ const skillDirectories = [".claude/skills", ".cursor/skills"];
96
+ const subagentSourceDirectory = ".agents/agents";
97
+ const subagentConfigPath = ".agents/agents.json";
98
+ const claudeSubagentDirectory = ".claude/agents";
99
+ const cursorSubagentDirectory = ".cursor/agents";
100
+
101
+ /**
102
+ * List every file under a directory, as paths relative to it.
103
+ * @param directory Directory to read; a missing directory has no files.
104
+ * @returns Relative file paths.
105
+ */
106
+ function listFiles(directory: string): string[] {
107
+ return existsSync(directory)
108
+ ? readdirSync(directory, {
109
+ recursive: true,
110
+ withFileTypes: true,
111
+ })
112
+ .filter(entry => entry.isFile())
113
+ .map(entry => path.relative(directory, path.join(entry.parentPath, entry.name)))
114
+ : [];
115
+ }
116
+
117
+ /**
118
+ * Render a subagent with the frontmatter fields its tool reads.
119
+ * @param name Subagent name.
120
+ * @param fields Frontmatter fields after `name` and `description`.
121
+ * @param subagent Subagent configuration.
122
+ * @param body Shared subagent body.
123
+ * @returns The subagent file content.
124
+ */
125
+ function renderSubagent(
126
+ name: string,
127
+ fields: string[],
128
+ subagent: SubagentConfig,
129
+ body: string,
130
+ ): string {
131
+ return [
132
+ `---`,
133
+ `name: ${name}`,
134
+ `description: ${subagent.description}`,
135
+ ...fields,
136
+ `---`,
137
+ ``,
138
+ body,
139
+ ].join("\n");
140
+ }
141
+
142
+ for (const file of listFiles(skillSourceDirectory)) {
143
+ const content = readFileSync(path.join(skillSourceDirectory, file), "utf8");
144
+
145
+ for (const directory of skillDirectories) {
146
+ expected.set(path.join(directory, file), content);
147
+ }
148
+ }
149
+
150
+ const subagents = existsSync(subagentConfigPath)
151
+ ? (JSON.parse(readFileSync(subagentConfigPath, "utf8")) as Record<string, SubagentConfig>)
152
+ : {};
153
+
154
+ for (const [name, subagent] of Object.entries(subagents)) {
155
+ const body = readFileSync(path.join(subagentSourceDirectory, `${name}.md`), "utf8");
156
+ const claudeFields = subagent.tools === undefined ? [] : [`tools: ${subagent.tools.join(", ")}`];
157
+ const cursorFields = subagent.readonly === undefined ? [] : [`readonly: ${subagent.readonly}`];
158
+
159
+ expected.set(
160
+ path.join(claudeSubagentDirectory, `${name}.md`),
161
+ renderSubagent(name, claudeFields, subagent, body),
162
+ );
163
+
164
+ expected.set(
165
+ path.join(cursorSubagentDirectory, `${name}.md`),
166
+ renderSubagent(name, cursorFields, subagent, body),
167
+ );
168
+ }
169
+
170
+ const sharedFiles: Record<string, string[]> = {
171
+ ".agents/mcp.json": [".mcp.json", ".cursor/mcp.json"],
172
+ ".agents/review.md": ["REVIEW.md"],
173
+ };
174
+
175
+ const sharedSources = Object.entries(sharedFiles).filter(([source]) => existsSync(source));
176
+
177
+ for (const [source, targets] of sharedSources) {
178
+ const content = readFileSync(source, "utf8");
179
+
180
+ for (const target of targets) {
181
+ expected.set(target, content);
182
+ }
183
+ }
184
+
88
185
  const orphans = [
89
186
  ...listNames(claudeDirectory, ".md").map(name => path.join(claudeDirectory, `${name}.md`)),
90
187
  ...listNames(cursorDirectory, ".mdc").map(name => path.join(cursorDirectory, `${name}.mdc`)),
188
+ ...skillDirectories.flatMap(directory =>
189
+ listFiles(directory).map(file => path.join(directory, file)),
190
+ ),
191
+ ...[claudeSubagentDirectory, cursorSubagentDirectory].flatMap(directory =>
192
+ listFiles(directory).map(file => path.join(directory, file)),
193
+ ),
194
+ ...Object.values(sharedFiles)
195
+ .flat()
196
+ .filter(file => existsSync(file)),
91
197
  ].filter(file => !expected.has(file));
92
198
 
93
199
  if (orphans.length > 0) {
@@ -104,6 +210,10 @@ const staleFiles = [...expected].filter(
104
210
 
105
211
  if (!isCheckOnly) {
106
212
  for (const [file, content] of staleFiles) {
213
+ mkdirSync(path.dirname(file), {
214
+ recursive: true,
215
+ });
216
+
107
217
  writeFileSync(file, content);
108
218
  }
109
219
  }
@@ -0,0 +1 @@
1
+ Start the app with `pnpm start:dev`. Exercise the changed behavior and the two nearest neighboring flows. Report what you ran, what you saw, and any behavior that does not match plan.md. Do not fix anything; report only.
@@ -0,0 +1,7 @@
1
+ ---
2
+ name: verifier
3
+ description: Runs the app and checks the change works before the session reports done
4
+ tools: Bash, Read
5
+ ---
6
+
7
+ Start the app with `pnpm start:dev`. Exercise the changed behavior and the two nearest neighboring flows. Report what you ran, what you saw, and any behavior that does not match plan.md. Do not fix anything; report only.
@@ -0,0 +1,7 @@
1
+ ---
2
+ name: verifier
3
+ description: Runs the app and checks the change works before the session reports done
4
+ readonly: true
5
+ ---
6
+
7
+ Start the app with `pnpm start:dev`. Exercise the changed behavior and the two nearest neighboring flows. Report what you ran, what you saw, and any behavior that does not match plan.md. Do not fix anything; report only.
@@ -166,6 +166,50 @@ jobs:
166
166
  environment: dev
167
167
  commit_sha: ${{ github.sha }}
168
168
 
169
+ triage:
170
+ name: Triage failed build
171
+ needs:
172
+ - checks
173
+ - image
174
+ if: failure() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
175
+ runs-on: ubuntu-latest
176
+ timeout-minutes: 15
177
+ permissions:
178
+ actions: read # Needed to read the logs of the failed jobs.
179
+ id-token: write # Needed to authenticate to Anthropic with the workflow's OIDC token.
180
+ pull-requests: write # Needed to post the summary on the pull request.
181
+ steps:
182
+ - name: Collect the build log
183
+ env:
184
+ GH_TOKEN: ${{ github.token }}
185
+ REPOSITORY: ${{ github.repository }}
186
+ RUN_ID: ${{ github.run_id }}
187
+ run: |
188
+ mkdir -p out
189
+ for job in $(gh api "repos/$REPOSITORY/actions/runs/$RUN_ID/jobs" --paginate --jq '.jobs[] | select(.conclusion == "failure") | .id'); do
190
+ gh api "repos/$REPOSITORY/actions/jobs/$job/logs" >> out/build.log
191
+ done
192
+ - name: Triage failed build
193
+ id: triage
194
+ uses: anthropics/claude-code-action@61a9c274fa7110a66dfafc1bbbced03833eb25fe # v1.0.269
195
+ with:
196
+ anthropic_federation_rule_id: ${{ vars.ANTHROPIC_FEDERATION_RULE_ID }}
197
+ anthropic_organization_id: ${{ vars.ANTHROPIC_ORGANIZATION_ID }}
198
+ prompt: >
199
+ Read the build log at out/build.log. Identify the most
200
+ likely cause, say whether the failure looks flaky or real, and write a
201
+ three-line summary for the PR thread.
202
+ claude_args: >-
203
+ --allowedTools "Read"
204
+ --json-schema '{"type":"object","properties":{"summary":{"type":"string"}},"required":["summary"]}'
205
+ - name: Post the summary
206
+ env:
207
+ GH_TOKEN: ${{ github.token }}
208
+ PR_NUMBER: ${{ github.event.pull_request.number }}
209
+ REPOSITORY: ${{ github.repository }}
210
+ SUMMARY: ${{ fromJSON(steps.triage.outputs.structured_output).summary }}
211
+ run: gh pr comment "$PR_NUMBER" --repo "$REPOSITORY" --body "$SUMMARY"
212
+
169
213
  ci:
170
214
  name: All Checks
171
215
  if: always()
@@ -14,19 +14,32 @@ NestJS HTTP API in TypeScript on pnpm and Node.js `>=24.15.0 <25`.
14
14
  | Every check | `pnpm verify` |
15
15
  | Render rules after a change | `pnpm rules:sync` |
16
16
 
17
- ## How it fits together
17
+ ## Verifying your work
18
+
19
+ - Build: `pnpm build` (prints nothing after `$ tsc -p tsconfig.build.json`)
20
+ - Test: `pnpm test:coverage` (all green, then the coverage summary; never skip or delete a failing test)
21
+ - Lint: `pnpm verify:static` (ESLint allows zero warnings; prints `✔ no dependency violations found`)
22
+
23
+ Run all three before reporting any task complete, and paste the output. `pnpm verify` runs all three.
24
+ If a test fails, fix the code, not the test.
25
+
26
+ ## Architecture
18
27
 
19
28
  - `src/main.ts` creates the app from `AppModule` on `parsePort(process.env.PORT)` with Nest's `ConsoleLogger`, which writes JSON lines when `NODE_ENV` is `production`.
20
29
  - `AppModule` imports `HealthModule`, whose Terminus controller serves `/health`. Add a feature as its own module and import it in `AppModule`.
21
30
 
22
- ## Mandatory rules
31
+ ## Conventions
23
32
 
24
- 1. **Follow `.agents/rules/`.** `.agents/rules.json` gives each rule's scope; Claude Code and Cursor load them. After changing either, run `pnpm rules:sync`; `pnpm check` fails on stale copies.
33
+ 1. **Follow `.agents/rules/`.** `.agents/rules.json` gives each rule's scope; Claude Code and Cursor load them. After changing either, or a skill or subagent in `.agents/skills/` or `.agents/agents/`, run `pnpm rules:sync`; `pnpm check` fails on stale copies.
25
34
  2. **Put unit tests next to the code as `src/**/*.test.ts` and end-to-end tests in `test/` as `*.e2e.test.ts`.** `vitest.config.ts` and `vitest.e2e.config.ts` collect only those patterns.
26
35
  3. **Read configuration from environment variables and fail when one is missing**, as `parsePort` does. Document each variable in `.env.example`.
27
36
  4. **Log through Nest's `Logger`.** It goes to the `ConsoleLogger` set in `src/main.ts`.
37
+ 5. **Start from the accepted spec, in plan mode.** Claude Code starts every session in plan mode (`.claude/settings.json`); in Cursor, switch to Plan Mode before the first edit. Give the plan the change's `intent/<change>/intent.md` and `spec.md`, and have it name the files that change, the order of the work, and the tests that prove it. Commit the approved plan as `intent/<change>/plan.md`. When implementation departs from the plan, update `plan.md` in the same commit.
38
+ 6. **Start a bug fix with a failing test.** Reproduce the bug as a test, run it, and confirm it fails for the reason you expect. Commit that test. Then create `.git/bug-fix-in-progress`, so `.agents/hooks/agent-test-edit-gate.sh` blocks test edits, and make the test pass without editing it. Delete `.git/bug-fix-in-progress` when the fix is done.
39
+
40
+ ## Things Claude gets wrong
28
41
 
29
- ## Known pitfalls
42
+ When Claude makes a mistake twice, add the correction here.
30
43
 
31
44
  - **Agent shell gate**: `.agents/hooks/agent-shell-gate.sh` blocks a shell command that writes to a protected path such as `.agents`, `.cursor`, `.claude/rules`, or `.husky`. It is a convenience, not a security boundary: CI and code-owner review are the boundary.
32
45
 
@@ -14,22 +14,7 @@ The server listens on the port set by `PORT` in `.env`, which the CLI created fr
14
14
 
15
15
  ## After creating a project from this template
16
16
 
17
- 1. Install gitleaks and jq (see [CONTRIBUTING.md](CONTRIBUTING.md)). Make the first commit on `main` and push it. The Git hooks allow only this first commit and push on `main`; work on branches after that.
18
-
19
- ```sh
20
- git add --all
21
- git commit -m "chore(repo): start from template"
22
- git push --set-upstream origin main
23
- ```
24
-
25
- 2. Add your team as the owner of every file in `.github/CODEOWNERS`, and replace the owner placeholder in `.github/zizmor.yml`.
26
- 3. Replace the template's description and owner: `package.json#description`, `repository.url`, and `author`. Change `license` from `UNLICENSED` if the project is open source.
27
- 4. In the repository settings:
28
- - **Actions → General → Workflow permissions:** select **Read repository contents and packages permissions**. Every workflow requests the extra permissions it needs.
29
- - **Advanced Security:** enable private vulnerability reporting, Dependabot alerts, Dependabot security updates, secret scanning, and push protection. `SECURITY.md` sends reporters to private vulnerability reporting.
30
- - **Cursor Bugbot:** enable the repository at [Bugbot](https://cursor.com/automations/from-cursor/bugbot). Bugbot reviews pull requests against the rules linked from `.cursor/BUGBOT.md`.
31
- - **Rules → Rulesets:** protect `main` with squash merges only, a linear history, no force pushes or deletion, and the required checks `All Checks` and `Validate pull request title`. Rulesets in a private repository need GitHub Pro, Team, or Enterprise; on GitHub Free, make the repository public or upgrade.
32
- - Create the `needs triage` label, which the issue forms apply: `gh label create "needs triage"`.
17
+ Follow [SETUP.md](SETUP.md) in order. It sets up the repository, the reviews, and the AI-native SDLC.
33
18
 
34
19
  ## Scripts
35
20
 
@@ -0,0 +1 @@
1
+ Start the app with `pnpm dev`. Exercise the changed behavior and the two nearest neighboring flows. Report what you ran, what you saw, and any behavior that does not match plan.md. Do not fix anything; report only.
@@ -0,0 +1,8 @@
1
+ {
2
+ "mcpServers": {
3
+ "playwright": {
4
+ "command": "npx",
5
+ "args": ["@playwright/mcp@0.0.83"]
6
+ }
7
+ }
8
+ }