@next-friday/create-app 4.0.1 → 5.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +10 -10
- package/templates/base/.agents/skills/write-intent/SKILL.md +1 -1
- package/templates/base/.claude/skills/write-intent/SKILL.md +1 -1
- package/templates/base/.cursor/skills/write-intent/SKILL.md +1 -1
- package/templates/base/.husky/pre-push +9 -1
- package/templates/base/SETUP.md +10 -14
- package/templates/base/intent/README.md +1 -1
- package/templates/nestjs/.github/workflows/ci.yml +0 -44
- package/templates/nestjs/package.json +4 -4
- package/templates/nextjs/.github/workflows/ci.yml +0 -44
- package/templates/nextjs/package.json +4 -4
- package/templates/nodejs/.github/workflows/ci.yml +0 -44
- package/templates/nodejs/package.json +4 -4
- package/templates/payload/.github/workflows/ci.yml +0 -44
- package/templates/payload/package.json +4 -4
- package/templates/turborepo/.github/workflows/ci.yml +0 -44
- package/templates/turborepo/apps/app/package.json +2 -2
- package/templates/turborepo/apps/backoffice/package.json +2 -2
- package/templates/turborepo/package.json +3 -3
- package/templates/base/.github/workflows/agent-evals.yml +0 -128
- package/templates/base/.github/workflows/claude.yml +0 -39
- package/templates/base/evals/README.md +0 -15
- package/templates/base/evals/check.sh +0 -17
- package/templates/base/lessons/README.md +0 -3
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@next-friday/create-app",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "5.0.1",
|
|
4
4
|
"description": "Create a Next Friday project from a template.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"create",
|
|
@@ -8,13 +8,13 @@
|
|
|
8
8
|
"template",
|
|
9
9
|
"next-friday"
|
|
10
10
|
],
|
|
11
|
-
"homepage": "https://github.com/next-friday/create-friday-app#readme",
|
|
11
|
+
"homepage": "https://github.com/next-friday/create-next-friday-app#readme",
|
|
12
12
|
"bugs": {
|
|
13
|
-
"url": "https://github.com/next-friday/create-friday-app/issues"
|
|
13
|
+
"url": "https://github.com/next-friday/create-next-friday-app/issues"
|
|
14
14
|
},
|
|
15
15
|
"repository": {
|
|
16
16
|
"type": "git",
|
|
17
|
-
"url": "git+https://github.com/next-friday/create-friday-app.git"
|
|
17
|
+
"url": "git+https://github.com/next-friday/create-next-friday-app.git"
|
|
18
18
|
},
|
|
19
19
|
"license": "MIT",
|
|
20
20
|
"author": "Next Friday",
|
|
@@ -37,22 +37,22 @@
|
|
|
37
37
|
"@changesets/cli": "3.0.3",
|
|
38
38
|
"@commitlint/cli": "21.2.4",
|
|
39
39
|
"@next-friday/commitlint-config-friday": "4.0.0",
|
|
40
|
-
"@next-friday/eslint-config-friday": "5.
|
|
40
|
+
"@next-friday/eslint-config-friday": "5.1.0",
|
|
41
41
|
"@next-friday/prettier-config-friday": "4.0.0",
|
|
42
|
-
"@types/node": "26.6.
|
|
42
|
+
"@types/node": "26.6.6",
|
|
43
43
|
"@vitest/coverage-v8": "5.0.5",
|
|
44
44
|
"editorconfig-checker": "7.0.0",
|
|
45
|
-
"eslint": "10.
|
|
45
|
+
"eslint": "10.14.0",
|
|
46
46
|
"fast-check": "4.10.2",
|
|
47
47
|
"husky": "9.1.7",
|
|
48
48
|
"jiti": "2.7.0",
|
|
49
|
-
"knip": "6.
|
|
49
|
+
"knip": "6.44.1",
|
|
50
50
|
"lint-staged": "17.6.0",
|
|
51
|
-
"pkg-pr-new": "0.0.
|
|
51
|
+
"pkg-pr-new": "0.0.93",
|
|
52
52
|
"prettier": "3.9.10",
|
|
53
53
|
"publint": "0.3.25",
|
|
54
54
|
"sort-package-json": "4.1.0",
|
|
55
|
-
"tsdown": "0.23.
|
|
55
|
+
"tsdown": "0.23.4",
|
|
56
56
|
"typescript": "6.0.3",
|
|
57
57
|
"vitest": "5.0.5"
|
|
58
58
|
},
|
|
@@ -9,7 +9,7 @@ description: Write intent.md, the proto-spec that starts a change. Use whenever
|
|
|
9
9
|
2. Brainstorm until the idea is concrete. Ask the questions an analyst would ask: scope, users, constraints, and what success looks like.
|
|
10
10
|
3. Write the result as `intent/<change>/intent.md`, where `<change>` is a short kebab-case name, using the template below.
|
|
11
11
|
4. Ask the originator to correct anything you misunderstood.
|
|
12
|
-
5. Commit `intent.md` and open a pull request. The product owner accepts it by merging the pull request, or rejects it by
|
|
12
|
+
5. Commit `intent.md` and open a pull request. The product owner accepts it by merging the pull request, or rejects it by setting `Status: rejected.` and merging.
|
|
13
13
|
|
|
14
14
|
## Template
|
|
15
15
|
|
|
@@ -9,7 +9,7 @@ description: Write intent.md, the proto-spec that starts a change. Use whenever
|
|
|
9
9
|
2. Brainstorm until the idea is concrete. Ask the questions an analyst would ask: scope, users, constraints, and what success looks like.
|
|
10
10
|
3. Write the result as `intent/<change>/intent.md`, where `<change>` is a short kebab-case name, using the template below.
|
|
11
11
|
4. Ask the originator to correct anything you misunderstood.
|
|
12
|
-
5. Commit `intent.md` and open a pull request. The product owner accepts it by merging the pull request, or rejects it by
|
|
12
|
+
5. Commit `intent.md` and open a pull request. The product owner accepts it by merging the pull request, or rejects it by setting `Status: rejected.` and merging.
|
|
13
13
|
|
|
14
14
|
## Template
|
|
15
15
|
|
|
@@ -9,7 +9,7 @@ description: Write intent.md, the proto-spec that starts a change. Use whenever
|
|
|
9
9
|
2. Brainstorm until the idea is concrete. Ask the questions an analyst would ask: scope, users, constraints, and what success looks like.
|
|
10
10
|
3. Write the result as `intent/<change>/intent.md`, where `<change>` is a short kebab-case name, using the template below.
|
|
11
11
|
4. Ask the originator to correct anything you misunderstood.
|
|
12
|
-
5. Commit `intent.md` and open a pull request. The product owner accepts it by merging the pull request, or rejects it by
|
|
12
|
+
5. Commit `intent.md` and open a pull request. The product owner accepts it by merging the pull request, or rejects it by setting `Status: rejected.` and merging.
|
|
13
13
|
|
|
14
14
|
## Template
|
|
15
15
|
|
|
@@ -2,11 +2,19 @@
|
|
|
2
2
|
set -e
|
|
3
3
|
|
|
4
4
|
# The first push of a new repository creates main on the remote, so it is the only push allowed there.
|
|
5
|
-
while read -r
|
|
5
|
+
while read -r local_ref _ remote_ref remote_oid; do
|
|
6
6
|
if [ "$remote_ref" = refs/heads/main ] && [ -n "$(printf '%s' "$remote_oid" | tr -d 0)" ]; then
|
|
7
7
|
echo "pre-push: do not push to main. Push your branch and open a pull request." >&2
|
|
8
8
|
exit 1
|
|
9
9
|
fi
|
|
10
|
+
# A branch this clone pushed before that the remote no longer has was deleted when its pull request merged or closed.
|
|
11
|
+
branch="${remote_ref#refs/heads/}"
|
|
12
|
+
if [ "$branch" != "$remote_ref" ] && [ -z "$(printf '%s' "$remote_oid" | tr -d 0)" ] \
|
|
13
|
+
&& { git rev-parse --verify --quiet "refs/remotes/$1/$branch" > /dev/null \
|
|
14
|
+
|| [ "$(git config "branch.${local_ref#refs/heads/}.merge" || true)" = "$remote_ref" ]; }; then
|
|
15
|
+
echo "pre-push: $branch no longer exists on $1, so its pull request merged or closed. Branch from the latest $1/main, push the new branch, and open a new pull request." >&2
|
|
16
|
+
exit 1
|
|
17
|
+
fi
|
|
10
18
|
done
|
|
11
19
|
|
|
12
20
|
pnpm check || {
|
package/templates/base/SETUP.md
CHANGED
|
@@ -4,13 +4,13 @@ This project follows Anthropic's [AI-native SDLC Playbook](https://claude.com/re
|
|
|
4
4
|
|
|
5
5
|
## How a change flows
|
|
6
6
|
|
|
7
|
-
1. **Plan:** an idea, a ticket, or an incident becomes `intent/<change>/intent.md` (`write-intent` skill). The product owner merges or
|
|
7
|
+
1. **Plan:** an idea, a ticket, or an incident becomes `intent/<change>/intent.md` (`write-intent` skill). The product owner merges it with `Status: accepted.` or `Status: rejected.`
|
|
8
8
|
2. **Design:** `spec.md` is written from the accepted intent with the policy skills loaded (`write-spec` skill). The product owner resolves concerns with the policy owners.
|
|
9
9
|
3. **Build:** the engineer starts in plan mode, approves the plan, and commits it as `plan.md`. A tech lead approves higher-risk plans.
|
|
10
|
-
4. **Test:** the session runs **Verifying your work** in `AGENTS.md
|
|
11
|
-
5. **Review:** Claude Code Review and Bugbot review by `REVIEW.md
|
|
12
|
-
6. **Deploy:**
|
|
13
|
-
7. **Maintain:**
|
|
10
|
+
4. **Test:** the session runs **Verifying your work** in `AGENTS.md`.
|
|
11
|
+
5. **Review:** Claude Code Review and Bugbot review by `REVIEW.md`. A code owner approves; the agent never does.
|
|
12
|
+
6. **Deploy:** `agent-production-gate.sh` gates production.
|
|
13
|
+
7. **Maintain:** larger work becomes a new `intent.md`.
|
|
14
14
|
|
|
15
15
|
## 1. First commit and push
|
|
16
16
|
|
|
@@ -54,31 +54,28 @@ Also replace the owner placeholder in `.github/zizmor.yml`.
|
|
|
54
54
|
**Who:** a repository admin.
|
|
55
55
|
|
|
56
56
|
- **Actions → General → Workflow permissions:** select **Read repository contents and packages permissions**. Every workflow requests the extra permissions it needs.
|
|
57
|
+
- **General → Pull Requests:** select **Automatically delete head branches**, so a branch is deleted when its pull request merges.
|
|
57
58
|
- **Advanced Security:** enable private vulnerability reporting, Dependabot alerts, Dependabot security updates, secret scanning, and push protection. `SECURITY.md` sends reporters to private vulnerability reporting.
|
|
58
|
-
- **Secrets and variables → Actions:**
|
|
59
|
-
- the variables `ANTHROPIC_FEDERATION_RULE_ID` and `ANTHROPIC_ORGANIZATION_ID`, from step 4. They are identifiers, not secrets: `.github/workflows/claude.yml`, `.github/workflows/agent-evals.yml`, and the **Triage failed build** job in `ci.yml` exchange the workflow's OIDC token for short-lived Claude API access, so the repository stores no Claude token.
|
|
60
|
-
- the variable `EVALS_MIN_PASS_RATE`, from 0 to 100, once the team has decided it (step 6).
|
|
61
59
|
- **Rules → Rulesets:** protect `main` with:
|
|
62
60
|
- squash merges only, a linear history, and no force pushes or deletion;
|
|
63
|
-
- the required checks `All Checks
|
|
61
|
+
- the required checks `All Checks` and `Validate pull request title`;
|
|
64
62
|
- a pull request before merging, with at least one approval, review from Code Owners, and stale approvals dismissed when new commits are pushed. Claude Code Review follows the severity, exclusion, and nit-cap rules in `REVIEW.md` only on a default branch protected this way ([customize Code Review](https://code.claude.com/docs/en/code-review-customize)).
|
|
65
63
|
|
|
66
64
|
Rulesets in a private repository need GitHub Pro, Team, or Enterprise; on GitHub Free, make the repository public or upgrade.
|
|
67
65
|
|
|
68
66
|
- Create the `needs triage` label, which the issue forms apply: `gh label create "needs triage"`.
|
|
69
67
|
|
|
70
|
-
**Check:** a test pull request shows the
|
|
68
|
+
**Check:** a test pull request shows the two required checks and asks for a code owner's review.
|
|
71
69
|
|
|
72
70
|
## 4. Claude
|
|
73
71
|
|
|
74
72
|
**Who:** an Owner of the Claude organization, with an owner of the GitHub organization.
|
|
75
73
|
|
|
76
|
-
- **Code Review:** in [**Admin settings → Claude Code**](https://claude.ai/admin-settings/claude-code#code-review), turn on **Allow code review**, install the Claude GitHub App, and add this repository ([set up Code Review](https://code.claude.com/docs/en/code-review-setup)). It needs a Team or Enterprise plan ([availability](https://code.claude.com/docs/en/code-review#availability)). Keep **Review runs** at **After every push**, the default, so every push to a pull request gets a review. Reviews run on Anthropic's infrastructure, so the repository has no review workflow.
|
|
77
|
-
- **Workload identity federation:** in the [Claude Console](https://platform.claude.com), under **Settings → Workload identity**, register the issuer `https://token.actions.githubusercontent.com`, create a service account in the workspace the workflows should use, and create a federation rule for it that matches this repository's subject prefix, `repo:OWNER/REPOSITORY:` ([setup](https://github.com/anthropics/claude-code-action/blob/main/docs/setup.md), [workload identity federation](https://platform.claude.com/docs/en/manage-claude/workload-identity-federation)). Put the rule's `fdrl_...` ID and the organization ID in the repository variables of step 3. Usage is billed to the Console organization.
|
|
74
|
+
- **Code Review:** in [**Admin settings → Claude Code**](https://claude.ai/admin-settings/claude-code#code-review), turn on **Allow code review**, install the Claude GitHub App, and add this repository ([set up Code Review](https://code.claude.com/docs/en/code-review-setup)). It needs a Team or Enterprise plan ([availability](https://code.claude.com/docs/en/code-review#availability)). Keep **Review runs** at **After every push**, the default, so every push to a pull request gets a review. Reviews run on Anthropic's infrastructure, so the repository has no review workflow.
|
|
78
75
|
- **Managed settings:** deploy the non-negotiable settings and hooks as [managed settings](https://code.claude.com/docs/en/settings), tailored from the playbook's worked example in `.claude/managed-settings.example.json`. Engineers cannot override them.
|
|
79
76
|
- **Monitoring:** turn on the [OpenTelemetry export](https://code.claude.com/docs/en/monitoring-usage) that the playbook's indicators read.
|
|
80
77
|
|
|
81
|
-
**Check:** the test pull request gets a **Claude Code Review** check run within a few minutes.
|
|
78
|
+
**Check:** the test pull request gets a **Claude Code Review** check run within a few minutes.
|
|
82
79
|
|
|
83
80
|
## 5. Cursor
|
|
84
81
|
|
|
@@ -98,7 +95,6 @@ Also replace the owner placeholder in `.github/zizmor.yml`.
|
|
|
98
95
|
- **Policies (each policy owner, with an engineer):** write the policy in `.agents/skills/<area>-policy/SKILL.md` from the owner's source of truth, then run `pnpm rules:sync`. Test that the skill triggers: ask for the relevant task in different ways and confirm the skill loads each time. The policy owner approves every later change as its code owner.
|
|
99
96
|
- **Production gate (engineering leadership, with change management and compliance):** list the approvals that must survive. A person then changes `.agents/hooks/agent-production-gate.sh` to match this project's production deploy command and approval; agent settings deny the file to agents.
|
|
100
97
|
- **Review (tech lead):** replace `src/gen/` in `.agents/review.md` with this project's generated paths, then run `pnpm rules:sync`, which renders it to `REVIEW.md` for Claude Code Review. Once a month, rate the review findings and tune `.agents/review.md`.
|
|
101
|
-
- **Evals (platform engineer):** collect 20 to 50 real tasks in `evals/` (`evals/README.md`), and set `EVALS_MIN_PASS_RATE` (step 3).
|
|
102
98
|
- **People without Git experience:** connect GitHub to Claude (claude.ai or Cowork), so Claude commits `intent.md` for them.
|
|
103
99
|
|
|
104
100
|
## 7. Later, as the guardrails mature
|
|
@@ -3,6 +3,6 @@
|
|
|
3
3
|
This folder is the intent home. Every change starts here as an `intent.md`, and its record stays here. Git history keeps the author, the time, and every revision of each file.
|
|
4
4
|
|
|
5
5
|
- Each change has its own folder, `intent/<change>/`, where `<change>` is a short kebab-case name. The folder holds `intent.md` (Plan), then `spec.md` (Design), then `plan.md` (Build).
|
|
6
|
-
- **Plan.** An idea, a ticket, or an incident becomes `intent.md`, written with the `write-intent` skill. The product owner reviews and corrects it before it is committed. Merging its pull request accepts it
|
|
6
|
+
- **Plan.** An idea, a ticket, or an incident becomes `intent.md`, written with the `write-intent` skill. The product owner reviews and corrects it before it is committed. Merging its pull request records the decision: `Status: accepted.` accepts it, and `Status: rejected.` rejects it.
|
|
7
7
|
- **Design.** The `write-spec` skill turns the accepted `intent.md` into `spec.md`, committed alongside it. The product owner reviews the spec against the idea and resolves each area of concern with its policy owner. Accepting the spec starts the build.
|
|
8
8
|
- **Build.** The engineer starts in plan mode with `intent.md` and `spec.md`, and commits the approved plan as `plan.md`. When implementation departs from the plan, `plan.md` changes in the same commit.
|
|
@@ -166,50 +166,6 @@ jobs:
|
|
|
166
166
|
environment: dev
|
|
167
167
|
commit_sha: ${{ github.sha }}
|
|
168
168
|
|
|
169
|
-
triage:
|
|
170
|
-
name: Triage failed build
|
|
171
|
-
needs:
|
|
172
|
-
- checks
|
|
173
|
-
- image
|
|
174
|
-
if: failure() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
|
|
175
|
-
runs-on: ubuntu-latest
|
|
176
|
-
timeout-minutes: 15
|
|
177
|
-
permissions:
|
|
178
|
-
actions: read # Needed to read the logs of the failed jobs.
|
|
179
|
-
id-token: write # Needed to authenticate to Anthropic with the workflow's OIDC token.
|
|
180
|
-
pull-requests: write # Needed to post the summary on the pull request.
|
|
181
|
-
steps:
|
|
182
|
-
- name: Collect the build log
|
|
183
|
-
env:
|
|
184
|
-
GH_TOKEN: ${{ github.token }}
|
|
185
|
-
REPOSITORY: ${{ github.repository }}
|
|
186
|
-
RUN_ID: ${{ github.run_id }}
|
|
187
|
-
run: |
|
|
188
|
-
mkdir -p out
|
|
189
|
-
for job in $(gh api "repos/$REPOSITORY/actions/runs/$RUN_ID/jobs" --paginate --jq '.jobs[] | select(.conclusion == "failure") | .id'); do
|
|
190
|
-
gh api "repos/$REPOSITORY/actions/jobs/$job/logs" >> out/build.log
|
|
191
|
-
done
|
|
192
|
-
- name: Triage failed build
|
|
193
|
-
id: triage
|
|
194
|
-
uses: anthropics/claude-code-action@61a9c274fa7110a66dfafc1bbbced03833eb25fe # v1.0.269
|
|
195
|
-
with:
|
|
196
|
-
anthropic_federation_rule_id: ${{ vars.ANTHROPIC_FEDERATION_RULE_ID }}
|
|
197
|
-
anthropic_organization_id: ${{ vars.ANTHROPIC_ORGANIZATION_ID }}
|
|
198
|
-
prompt: >
|
|
199
|
-
Read the build log at out/build.log. Identify the most
|
|
200
|
-
likely cause, say whether the failure looks flaky or real, and write a
|
|
201
|
-
three-line summary for the PR thread.
|
|
202
|
-
claude_args: >-
|
|
203
|
-
--allowedTools "Read"
|
|
204
|
-
--json-schema '{"type":"object","properties":{"summary":{"type":"string"}},"required":["summary"]}'
|
|
205
|
-
- name: Post the summary
|
|
206
|
-
env:
|
|
207
|
-
GH_TOKEN: ${{ github.token }}
|
|
208
|
-
PR_NUMBER: ${{ github.event.pull_request.number }}
|
|
209
|
-
REPOSITORY: ${{ github.repository }}
|
|
210
|
-
SUMMARY: ${{ fromJSON(steps.triage.outputs.structured_output).summary }}
|
|
211
|
-
run: gh pr comment "$PR_NUMBER" --repo "$REPOSITORY" --body "$SUMMARY"
|
|
212
|
-
|
|
213
169
|
ci:
|
|
214
170
|
name: All Checks
|
|
215
171
|
if: always()
|
|
@@ -59,16 +59,16 @@
|
|
|
59
59
|
"@next-friday/commitlint-config-friday": "4.0.0",
|
|
60
60
|
"@next-friday/eslint-config-friday": "5.1.0",
|
|
61
61
|
"@next-friday/prettier-config-friday": "4.0.0",
|
|
62
|
-
"@types/node": "26.6.
|
|
62
|
+
"@types/node": "26.6.6",
|
|
63
63
|
"@types/supertest": "7.2.1",
|
|
64
64
|
"@vitest/coverage-v8": "5.0.5",
|
|
65
|
-
"dependency-cruiser": "18.7.
|
|
65
|
+
"dependency-cruiser": "18.7.1",
|
|
66
66
|
"editorconfig-checker": "7.0.0",
|
|
67
|
-
"eslint": "10.
|
|
67
|
+
"eslint": "10.14.0",
|
|
68
68
|
"fast-check": "4.10.2",
|
|
69
69
|
"husky": "9.1.7",
|
|
70
70
|
"jiti": "2.7.0",
|
|
71
|
-
"knip": "6.
|
|
71
|
+
"knip": "6.44.1",
|
|
72
72
|
"lint-staged": "17.6.0",
|
|
73
73
|
"prettier": "3.9.10",
|
|
74
74
|
"sort-package-json": "4.1.0",
|
|
@@ -172,50 +172,6 @@ jobs:
|
|
|
172
172
|
environment: dev
|
|
173
173
|
commit_sha: ${{ github.sha }}
|
|
174
174
|
|
|
175
|
-
triage:
|
|
176
|
-
name: Triage failed build
|
|
177
|
-
needs:
|
|
178
|
-
- checks
|
|
179
|
-
- image
|
|
180
|
-
if: failure() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
|
|
181
|
-
runs-on: ubuntu-latest
|
|
182
|
-
timeout-minutes: 15
|
|
183
|
-
permissions:
|
|
184
|
-
actions: read # Needed to read the logs of the failed jobs.
|
|
185
|
-
id-token: write # Needed to authenticate to Anthropic with the workflow's OIDC token.
|
|
186
|
-
pull-requests: write # Needed to post the summary on the pull request.
|
|
187
|
-
steps:
|
|
188
|
-
- name: Collect the build log
|
|
189
|
-
env:
|
|
190
|
-
GH_TOKEN: ${{ github.token }}
|
|
191
|
-
REPOSITORY: ${{ github.repository }}
|
|
192
|
-
RUN_ID: ${{ github.run_id }}
|
|
193
|
-
run: |
|
|
194
|
-
mkdir -p out
|
|
195
|
-
for job in $(gh api "repos/$REPOSITORY/actions/runs/$RUN_ID/jobs" --paginate --jq '.jobs[] | select(.conclusion == "failure") | .id'); do
|
|
196
|
-
gh api "repos/$REPOSITORY/actions/jobs/$job/logs" >> out/build.log
|
|
197
|
-
done
|
|
198
|
-
- name: Triage failed build
|
|
199
|
-
id: triage
|
|
200
|
-
uses: anthropics/claude-code-action@61a9c274fa7110a66dfafc1bbbced03833eb25fe # v1.0.269
|
|
201
|
-
with:
|
|
202
|
-
anthropic_federation_rule_id: ${{ vars.ANTHROPIC_FEDERATION_RULE_ID }}
|
|
203
|
-
anthropic_organization_id: ${{ vars.ANTHROPIC_ORGANIZATION_ID }}
|
|
204
|
-
prompt: >
|
|
205
|
-
Read the build log at out/build.log. Identify the most
|
|
206
|
-
likely cause, say whether the failure looks flaky or real, and write a
|
|
207
|
-
three-line summary for the PR thread.
|
|
208
|
-
claude_args: >-
|
|
209
|
-
--allowedTools "Read"
|
|
210
|
-
--json-schema '{"type":"object","properties":{"summary":{"type":"string"}},"required":["summary"]}'
|
|
211
|
-
- name: Post the summary
|
|
212
|
-
env:
|
|
213
|
-
GH_TOKEN: ${{ github.token }}
|
|
214
|
-
PR_NUMBER: ${{ github.event.pull_request.number }}
|
|
215
|
-
REPOSITORY: ${{ github.repository }}
|
|
216
|
-
SUMMARY: ${{ fromJSON(steps.triage.outputs.structured_output).summary }}
|
|
217
|
-
run: gh pr comment "$PR_NUMBER" --repo "$REPOSITORY" --body "$SUMMARY"
|
|
218
|
-
|
|
219
175
|
ci:
|
|
220
176
|
name: All Checks
|
|
221
177
|
if: always()
|
|
@@ -52,17 +52,17 @@
|
|
|
52
52
|
"@next/eslint-plugin-next": "16.4.3",
|
|
53
53
|
"@playwright/test": "1.64.1",
|
|
54
54
|
"@tailwindcss/postcss": "4.3.3",
|
|
55
|
-
"@types/node": "26.6.
|
|
55
|
+
"@types/node": "26.6.6",
|
|
56
56
|
"@types/react": "19.3.0",
|
|
57
57
|
"@types/react-dom": "19.3.0",
|
|
58
58
|
"@vitest/coverage-v8": "5.0.5",
|
|
59
|
-
"dependency-cruiser": "18.7.
|
|
59
|
+
"dependency-cruiser": "18.7.1",
|
|
60
60
|
"editorconfig-checker": "7.0.0",
|
|
61
|
-
"eslint": "10.
|
|
61
|
+
"eslint": "10.14.0",
|
|
62
62
|
"fast-check": "4.10.2",
|
|
63
63
|
"husky": "9.1.7",
|
|
64
64
|
"jiti": "2.7.0",
|
|
65
|
-
"knip": "6.
|
|
65
|
+
"knip": "6.44.1",
|
|
66
66
|
"lint-staged": "17.6.0",
|
|
67
67
|
"prettier": "3.9.10",
|
|
68
68
|
"prettier-plugin-tailwindcss": "0.8.1",
|
|
@@ -164,50 +164,6 @@ jobs:
|
|
|
164
164
|
environment: dev
|
|
165
165
|
commit_sha: ${{ github.sha }}
|
|
166
166
|
|
|
167
|
-
triage:
|
|
168
|
-
name: Triage failed build
|
|
169
|
-
needs:
|
|
170
|
-
- checks
|
|
171
|
-
- image
|
|
172
|
-
if: failure() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
|
|
173
|
-
runs-on: ubuntu-latest
|
|
174
|
-
timeout-minutes: 15
|
|
175
|
-
permissions:
|
|
176
|
-
actions: read # Needed to read the logs of the failed jobs.
|
|
177
|
-
id-token: write # Needed to authenticate to Anthropic with the workflow's OIDC token.
|
|
178
|
-
pull-requests: write # Needed to post the summary on the pull request.
|
|
179
|
-
steps:
|
|
180
|
-
- name: Collect the build log
|
|
181
|
-
env:
|
|
182
|
-
GH_TOKEN: ${{ github.token }}
|
|
183
|
-
REPOSITORY: ${{ github.repository }}
|
|
184
|
-
RUN_ID: ${{ github.run_id }}
|
|
185
|
-
run: |
|
|
186
|
-
mkdir -p out
|
|
187
|
-
for job in $(gh api "repos/$REPOSITORY/actions/runs/$RUN_ID/jobs" --paginate --jq '.jobs[] | select(.conclusion == "failure") | .id'); do
|
|
188
|
-
gh api "repos/$REPOSITORY/actions/jobs/$job/logs" >> out/build.log
|
|
189
|
-
done
|
|
190
|
-
- name: Triage failed build
|
|
191
|
-
id: triage
|
|
192
|
-
uses: anthropics/claude-code-action@61a9c274fa7110a66dfafc1bbbced03833eb25fe # v1.0.269
|
|
193
|
-
with:
|
|
194
|
-
anthropic_federation_rule_id: ${{ vars.ANTHROPIC_FEDERATION_RULE_ID }}
|
|
195
|
-
anthropic_organization_id: ${{ vars.ANTHROPIC_ORGANIZATION_ID }}
|
|
196
|
-
prompt: >
|
|
197
|
-
Read the build log at out/build.log. Identify the most
|
|
198
|
-
likely cause, say whether the failure looks flaky or real, and write a
|
|
199
|
-
three-line summary for the PR thread.
|
|
200
|
-
claude_args: >-
|
|
201
|
-
--allowedTools "Read"
|
|
202
|
-
--json-schema '{"type":"object","properties":{"summary":{"type":"string"}},"required":["summary"]}'
|
|
203
|
-
- name: Post the summary
|
|
204
|
-
env:
|
|
205
|
-
GH_TOKEN: ${{ github.token }}
|
|
206
|
-
PR_NUMBER: ${{ github.event.pull_request.number }}
|
|
207
|
-
REPOSITORY: ${{ github.repository }}
|
|
208
|
-
SUMMARY: ${{ fromJSON(steps.triage.outputs.structured_output).summary }}
|
|
209
|
-
run: gh pr comment "$PR_NUMBER" --repo "$REPOSITORY" --body "$SUMMARY"
|
|
210
|
-
|
|
211
167
|
ci:
|
|
212
168
|
name: All Checks
|
|
213
169
|
if: always()
|
|
@@ -44,15 +44,15 @@
|
|
|
44
44
|
"@next-friday/commitlint-config-friday": "4.0.0",
|
|
45
45
|
"@next-friday/eslint-config-friday": "5.1.0",
|
|
46
46
|
"@next-friday/prettier-config-friday": "4.0.0",
|
|
47
|
-
"@types/node": "26.6.
|
|
47
|
+
"@types/node": "26.6.6",
|
|
48
48
|
"@vitest/coverage-v8": "5.0.5",
|
|
49
|
-
"dependency-cruiser": "18.7.
|
|
49
|
+
"dependency-cruiser": "18.7.1",
|
|
50
50
|
"editorconfig-checker": "7.0.0",
|
|
51
|
-
"eslint": "10.
|
|
51
|
+
"eslint": "10.14.0",
|
|
52
52
|
"fast-check": "4.10.2",
|
|
53
53
|
"husky": "9.1.7",
|
|
54
54
|
"jiti": "2.7.0",
|
|
55
|
-
"knip": "6.
|
|
55
|
+
"knip": "6.44.1",
|
|
56
56
|
"lint-staged": "17.6.0",
|
|
57
57
|
"prettier": "3.9.10",
|
|
58
58
|
"sort-package-json": "4.1.0",
|
|
@@ -189,50 +189,6 @@ jobs:
|
|
|
189
189
|
environment: dev
|
|
190
190
|
commit_sha: ${{ github.sha }}
|
|
191
191
|
|
|
192
|
-
triage:
|
|
193
|
-
name: Triage failed build
|
|
194
|
-
needs:
|
|
195
|
-
- checks
|
|
196
|
-
- image
|
|
197
|
-
if: failure() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
|
|
198
|
-
runs-on: ubuntu-latest
|
|
199
|
-
timeout-minutes: 15
|
|
200
|
-
permissions:
|
|
201
|
-
actions: read # Needed to read the logs of the failed jobs.
|
|
202
|
-
id-token: write # Needed to authenticate to Anthropic with the workflow's OIDC token.
|
|
203
|
-
pull-requests: write # Needed to post the summary on the pull request.
|
|
204
|
-
steps:
|
|
205
|
-
- name: Collect the build log
|
|
206
|
-
env:
|
|
207
|
-
GH_TOKEN: ${{ github.token }}
|
|
208
|
-
REPOSITORY: ${{ github.repository }}
|
|
209
|
-
RUN_ID: ${{ github.run_id }}
|
|
210
|
-
run: |
|
|
211
|
-
mkdir -p out
|
|
212
|
-
for job in $(gh api "repos/$REPOSITORY/actions/runs/$RUN_ID/jobs" --paginate --jq '.jobs[] | select(.conclusion == "failure") | .id'); do
|
|
213
|
-
gh api "repos/$REPOSITORY/actions/jobs/$job/logs" >> out/build.log
|
|
214
|
-
done
|
|
215
|
-
- name: Triage failed build
|
|
216
|
-
id: triage
|
|
217
|
-
uses: anthropics/claude-code-action@61a9c274fa7110a66dfafc1bbbced03833eb25fe # v1.0.269
|
|
218
|
-
with:
|
|
219
|
-
anthropic_federation_rule_id: ${{ vars.ANTHROPIC_FEDERATION_RULE_ID }}
|
|
220
|
-
anthropic_organization_id: ${{ vars.ANTHROPIC_ORGANIZATION_ID }}
|
|
221
|
-
prompt: >
|
|
222
|
-
Read the build log at out/build.log. Identify the most
|
|
223
|
-
likely cause, say whether the failure looks flaky or real, and write a
|
|
224
|
-
three-line summary for the PR thread.
|
|
225
|
-
claude_args: >-
|
|
226
|
-
--allowedTools "Read"
|
|
227
|
-
--json-schema '{"type":"object","properties":{"summary":{"type":"string"}},"required":["summary"]}'
|
|
228
|
-
- name: Post the summary
|
|
229
|
-
env:
|
|
230
|
-
GH_TOKEN: ${{ github.token }}
|
|
231
|
-
PR_NUMBER: ${{ github.event.pull_request.number }}
|
|
232
|
-
REPOSITORY: ${{ github.repository }}
|
|
233
|
-
SUMMARY: ${{ fromJSON(steps.triage.outputs.structured_output).summary }}
|
|
234
|
-
run: gh pr comment "$PR_NUMBER" --repo "$REPOSITORY" --body "$SUMMARY"
|
|
235
|
-
|
|
236
192
|
ci:
|
|
237
193
|
name: All Checks
|
|
238
194
|
if: always()
|
|
@@ -61,17 +61,17 @@
|
|
|
61
61
|
"@next-friday/prettier-config-friday": "4.0.0",
|
|
62
62
|
"@next/eslint-plugin-next": "16.4.3",
|
|
63
63
|
"@playwright/test": "1.64.1",
|
|
64
|
-
"@types/node": "26.6.
|
|
64
|
+
"@types/node": "26.6.6",
|
|
65
65
|
"@types/react": "19.3.0",
|
|
66
66
|
"@types/react-dom": "19.3.0",
|
|
67
67
|
"@vitest/coverage-v8": "5.0.5",
|
|
68
|
-
"dependency-cruiser": "18.7.
|
|
68
|
+
"dependency-cruiser": "18.7.1",
|
|
69
69
|
"editorconfig-checker": "7.0.0",
|
|
70
|
-
"eslint": "10.
|
|
70
|
+
"eslint": "10.14.0",
|
|
71
71
|
"fast-check": "4.10.2",
|
|
72
72
|
"husky": "9.1.7",
|
|
73
73
|
"jiti": "2.7.0",
|
|
74
|
-
"knip": "6.
|
|
74
|
+
"knip": "6.44.1",
|
|
75
75
|
"lint-staged": "17.6.0",
|
|
76
76
|
"prettier": "3.9.10",
|
|
77
77
|
"sort-package-json": "4.1.0",
|
|
@@ -297,50 +297,6 @@ jobs:
|
|
|
297
297
|
environment: dev
|
|
298
298
|
commit_sha: ${{ github.sha }}
|
|
299
299
|
|
|
300
|
-
triage:
|
|
301
|
-
name: Triage failed build
|
|
302
|
-
needs:
|
|
303
|
-
- checks
|
|
304
|
-
- image
|
|
305
|
-
if: failure() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
|
|
306
|
-
runs-on: ubuntu-latest
|
|
307
|
-
timeout-minutes: 15
|
|
308
|
-
permissions:
|
|
309
|
-
actions: read # Needed to read the logs of the failed jobs.
|
|
310
|
-
id-token: write # Needed to authenticate to Anthropic with the workflow's OIDC token.
|
|
311
|
-
pull-requests: write # Needed to post the summary on the pull request.
|
|
312
|
-
steps:
|
|
313
|
-
- name: Collect the build log
|
|
314
|
-
env:
|
|
315
|
-
GH_TOKEN: ${{ github.token }}
|
|
316
|
-
REPOSITORY: ${{ github.repository }}
|
|
317
|
-
RUN_ID: ${{ github.run_id }}
|
|
318
|
-
run: |
|
|
319
|
-
mkdir -p out
|
|
320
|
-
for job in $(gh api "repos/$REPOSITORY/actions/runs/$RUN_ID/jobs" --paginate --jq '.jobs[] | select(.conclusion == "failure") | .id'); do
|
|
321
|
-
gh api "repos/$REPOSITORY/actions/jobs/$job/logs" >> out/build.log
|
|
322
|
-
done
|
|
323
|
-
- name: Triage failed build
|
|
324
|
-
id: triage
|
|
325
|
-
uses: anthropics/claude-code-action@61a9c274fa7110a66dfafc1bbbced03833eb25fe # v1.0.269
|
|
326
|
-
with:
|
|
327
|
-
anthropic_federation_rule_id: ${{ vars.ANTHROPIC_FEDERATION_RULE_ID }}
|
|
328
|
-
anthropic_organization_id: ${{ vars.ANTHROPIC_ORGANIZATION_ID }}
|
|
329
|
-
prompt: >
|
|
330
|
-
Read the build log at out/build.log. Identify the most
|
|
331
|
-
likely cause, say whether the failure looks flaky or real, and write a
|
|
332
|
-
three-line summary for the PR thread.
|
|
333
|
-
claude_args: >-
|
|
334
|
-
--allowedTools "Read"
|
|
335
|
-
--json-schema '{"type":"object","properties":{"summary":{"type":"string"}},"required":["summary"]}'
|
|
336
|
-
- name: Post the summary
|
|
337
|
-
env:
|
|
338
|
-
GH_TOKEN: ${{ github.token }}
|
|
339
|
-
PR_NUMBER: ${{ github.event.pull_request.number }}
|
|
340
|
-
REPOSITORY: ${{ github.repository }}
|
|
341
|
-
SUMMARY: ${{ fromJSON(steps.triage.outputs.structured_output).summary }}
|
|
342
|
-
run: gh pr comment "$PR_NUMBER" --repo "$REPOSITORY" --body "$SUMMARY"
|
|
343
|
-
|
|
344
300
|
ci:
|
|
345
301
|
name: All Checks
|
|
346
302
|
if: always()
|
|
@@ -25,11 +25,11 @@
|
|
|
25
25
|
"devDependencies": {
|
|
26
26
|
"@playwright/test": "1.64.1",
|
|
27
27
|
"@tailwindcss/postcss": "4.3.3",
|
|
28
|
-
"@types/node": "26.6.
|
|
28
|
+
"@types/node": "26.6.6",
|
|
29
29
|
"@types/react": "19.3.0",
|
|
30
30
|
"@types/react-dom": "19.3.0",
|
|
31
31
|
"@vitest/coverage-v8": "5.0.5",
|
|
32
|
-
"dependency-cruiser": "18.7.
|
|
32
|
+
"dependency-cruiser": "18.7.1",
|
|
33
33
|
"fast-check": "4.10.2",
|
|
34
34
|
"tailwindcss": "4.3.3",
|
|
35
35
|
"typescript": "7.0.2",
|
|
@@ -34,11 +34,11 @@
|
|
|
34
34
|
},
|
|
35
35
|
"devDependencies": {
|
|
36
36
|
"@playwright/test": "1.64.1",
|
|
37
|
-
"@types/node": "26.6.
|
|
37
|
+
"@types/node": "26.6.6",
|
|
38
38
|
"@types/react": "19.3.0",
|
|
39
39
|
"@types/react-dom": "19.3.0",
|
|
40
40
|
"@vitest/coverage-v8": "5.0.5",
|
|
41
|
-
"dependency-cruiser": "18.7.
|
|
41
|
+
"dependency-cruiser": "18.7.1",
|
|
42
42
|
"fast-check": "4.10.2",
|
|
43
43
|
"typescript": "7.0.2",
|
|
44
44
|
"vitest": "5.0.5"
|
|
@@ -49,13 +49,13 @@
|
|
|
49
49
|
"@next-friday/eslint-config-friday": "5.1.0",
|
|
50
50
|
"@next-friday/prettier-config-friday": "4.0.0",
|
|
51
51
|
"@next/eslint-plugin-next": "16.4.3",
|
|
52
|
-
"@types/node": "26.6.
|
|
52
|
+
"@types/node": "26.6.6",
|
|
53
53
|
"editorconfig-checker": "7.0.0",
|
|
54
|
-
"eslint": "10.
|
|
54
|
+
"eslint": "10.14.0",
|
|
55
55
|
"fast-check": "4.10.2",
|
|
56
56
|
"husky": "9.1.7",
|
|
57
57
|
"jiti": "2.7.0",
|
|
58
|
-
"knip": "6.
|
|
58
|
+
"knip": "6.44.1",
|
|
59
59
|
"lint-staged": "17.6.0",
|
|
60
60
|
"prettier": "3.9.10",
|
|
61
61
|
"prettier-plugin-tailwindcss": "0.8.1",
|
|
@@ -1,128 +0,0 @@
|
|
|
1
|
-
name: Agent evals
|
|
2
|
-
|
|
3
|
-
on:
|
|
4
|
-
pull_request:
|
|
5
|
-
types: [opened, synchronize, reopened, ready_for_review]
|
|
6
|
-
schedule:
|
|
7
|
-
- cron: "0 2 * * *"
|
|
8
|
-
|
|
9
|
-
permissions: {}
|
|
10
|
-
|
|
11
|
-
concurrency:
|
|
12
|
-
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
|
|
13
|
-
cancel-in-progress: true
|
|
14
|
-
|
|
15
|
-
jobs:
|
|
16
|
-
list:
|
|
17
|
-
name: List evals
|
|
18
|
-
runs-on: ubuntu-latest
|
|
19
|
-
timeout-minutes: 5
|
|
20
|
-
permissions:
|
|
21
|
-
contents: read
|
|
22
|
-
outputs:
|
|
23
|
-
evals: ${{ steps.list.outputs.evals }}
|
|
24
|
-
steps:
|
|
25
|
-
- name: Checkout repository
|
|
26
|
-
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
27
|
-
with:
|
|
28
|
-
fetch-depth: 0
|
|
29
|
-
persist-credentials: false
|
|
30
|
-
- name: List the evals to run
|
|
31
|
-
id: list
|
|
32
|
-
env:
|
|
33
|
-
EVENT_NAME: ${{ github.event_name }}
|
|
34
|
-
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
|
35
|
-
run: |
|
|
36
|
-
if [[ "$EVENT_NAME" == "schedule" ]] || git diff --name-only "$BASE_SHA" HEAD | grep -qE '^(CLAUDE\.md|AGENTS\.md|REVIEW\.md|\.mcp\.json|\.agents/|\.claude/|\.cursor/|evals/)'; then
|
|
37
|
-
echo "evals=$(find evals -maxdepth 1 -name '*.json' | sort | jq -R -s -c 'split("\n")[:-1]')" >> "$GITHUB_OUTPUT"
|
|
38
|
-
else
|
|
39
|
-
echo 'evals=[]' >> "$GITHUB_OUTPUT"
|
|
40
|
-
fi
|
|
41
|
-
|
|
42
|
-
eval:
|
|
43
|
-
name: Eval
|
|
44
|
-
needs: list
|
|
45
|
-
if: needs.list.outputs.evals != '[]'
|
|
46
|
-
runs-on: ubuntu-latest
|
|
47
|
-
timeout-minutes: 30
|
|
48
|
-
permissions:
|
|
49
|
-
contents: read
|
|
50
|
-
id-token: write # Needed to authenticate to Anthropic with the workflow's OIDC token.
|
|
51
|
-
strategy:
|
|
52
|
-
fail-fast: false
|
|
53
|
-
matrix:
|
|
54
|
-
eval: ${{ fromJSON(needs.list.outputs.evals) }}
|
|
55
|
-
steps:
|
|
56
|
-
- name: Checkout repository
|
|
57
|
-
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
58
|
-
with:
|
|
59
|
-
persist-credentials: false
|
|
60
|
-
- name: Set up Node.js and pnpm
|
|
61
|
-
uses: pnpm/setup@fbda4c85fc2e1e08721cd8763afea8f48d60f024 # v3.0.0
|
|
62
|
-
with:
|
|
63
|
-
cache: true
|
|
64
|
-
require-lockfile: true
|
|
65
|
-
- name: Read the eval prompt
|
|
66
|
-
id: read
|
|
67
|
-
env:
|
|
68
|
-
EVAL: ${{ matrix.eval }}
|
|
69
|
-
run: |
|
|
70
|
-
{
|
|
71
|
-
echo "prompt<<EVAL_PROMPT_END"
|
|
72
|
-
jq -r '.prompt' "$EVAL"
|
|
73
|
-
echo "EVAL_PROMPT_END"
|
|
74
|
-
} >> "$GITHUB_OUTPUT"
|
|
75
|
-
- name: Run the eval task
|
|
76
|
-
id: claude
|
|
77
|
-
uses: anthropics/claude-code-action@61a9c274fa7110a66dfafc1bbbced03833eb25fe # v1.0.269
|
|
78
|
-
with:
|
|
79
|
-
anthropic_federation_rule_id: ${{ vars.ANTHROPIC_FEDERATION_RULE_ID }}
|
|
80
|
-
anthropic_organization_id: ${{ vars.ANTHROPIC_ORGANIZATION_ID }}
|
|
81
|
-
prompt: ${{ steps.read.outputs.prompt }}
|
|
82
|
-
claude_args: --permission-mode acceptEdits --allowedTools "Read,Edit,Bash(pnpm test)"
|
|
83
|
-
- name: Check the result
|
|
84
|
-
env:
|
|
85
|
-
EVAL: ${{ matrix.eval }}
|
|
86
|
-
CONCLUSION: ${{ steps.claude.outputs.conclusion }}
|
|
87
|
-
run: ./evals/check.sh "$EVAL" "$CONCLUSION"
|
|
88
|
-
|
|
89
|
-
evals:
|
|
90
|
-
name: Agent evals
|
|
91
|
-
needs:
|
|
92
|
-
- list
|
|
93
|
-
- eval
|
|
94
|
-
if: always()
|
|
95
|
-
runs-on: ubuntu-latest
|
|
96
|
-
timeout-minutes: 5
|
|
97
|
-
permissions:
|
|
98
|
-
actions: read # Needed to read the result of each eval job.
|
|
99
|
-
steps:
|
|
100
|
-
- name: Check the pass rate
|
|
101
|
-
env:
|
|
102
|
-
EVALS: ${{ needs.list.outputs.evals }}
|
|
103
|
-
LIST_RESULT: ${{ needs.list.result }}
|
|
104
|
-
EVALS_MIN_PASS_RATE: ${{ vars.EVALS_MIN_PASS_RATE }}
|
|
105
|
-
GH_TOKEN: ${{ github.token }}
|
|
106
|
-
REPOSITORY: ${{ github.repository }}
|
|
107
|
-
RUN_ID: ${{ github.run_id }}
|
|
108
|
-
run: |
|
|
109
|
-
if [[ "$LIST_RESULT" != "success" ]]; then
|
|
110
|
-
echo "::error::Listing the evals failed."
|
|
111
|
-
exit 1
|
|
112
|
-
fi
|
|
113
|
-
total=$(jq 'length' <<< "$EVALS")
|
|
114
|
-
if ((total == 0)); then
|
|
115
|
-
echo "No evals to run."
|
|
116
|
-
exit 0
|
|
117
|
-
fi
|
|
118
|
-
if [[ -z "$EVALS_MIN_PASS_RATE" ]]; then
|
|
119
|
-
echo "::error::Set the repository variable EVALS_MIN_PASS_RATE (0 to 100)."
|
|
120
|
-
exit 1
|
|
121
|
-
fi
|
|
122
|
-
passed=$(gh api "repos/$REPOSITORY/actions/runs/$RUN_ID/jobs" --paginate --jq '.jobs[] | select((.name | startswith("Eval (")) and .conclusion == "success") | .id' | wc -l)
|
|
123
|
-
rate=$((passed * 100 / total))
|
|
124
|
-
echo "Pass rate: $rate% ($passed of $total)"
|
|
125
|
-
if ((rate < EVALS_MIN_PASS_RATE)); then
|
|
126
|
-
echo "::error::The pass rate $rate% is below EVALS_MIN_PASS_RATE ($EVALS_MIN_PASS_RATE%)."
|
|
127
|
-
exit 1
|
|
128
|
-
fi
|
|
@@ -1,39 +0,0 @@
|
|
|
1
|
-
name: Claude
|
|
2
|
-
|
|
3
|
-
on:
|
|
4
|
-
pull_request_review_comment:
|
|
5
|
-
types: [created]
|
|
6
|
-
pull_request_review:
|
|
7
|
-
types: [submitted]
|
|
8
|
-
|
|
9
|
-
permissions: {}
|
|
10
|
-
|
|
11
|
-
concurrency:
|
|
12
|
-
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
|
|
13
|
-
cancel-in-progress: false
|
|
14
|
-
|
|
15
|
-
jobs:
|
|
16
|
-
claude:
|
|
17
|
-
name: Address review comment
|
|
18
|
-
if: |
|
|
19
|
-
(github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
|
|
20
|
-
(github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude'))
|
|
21
|
-
runs-on: ubuntu-latest
|
|
22
|
-
timeout-minutes: 60
|
|
23
|
-
permissions:
|
|
24
|
-
contents: write # Needed to push the fix.
|
|
25
|
-
pull-requests: write # Needed to reply on the pull request.
|
|
26
|
-
id-token: write # Needed by the action to authenticate.
|
|
27
|
-
actions: read # Needed to read CI results on pull requests.
|
|
28
|
-
steps:
|
|
29
|
-
- name: Checkout repository
|
|
30
|
-
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
31
|
-
with:
|
|
32
|
-
fetch-depth: 1
|
|
33
|
-
persist-credentials: false
|
|
34
|
-
- name: Address the comment
|
|
35
|
-
uses: anthropics/claude-code-action@61a9c274fa7110a66dfafc1bbbced03833eb25fe # v1.0.269
|
|
36
|
-
with:
|
|
37
|
-
anthropic_federation_rule_id: ${{ vars.ANTHROPIC_FEDERATION_RULE_ID }}
|
|
38
|
-
anthropic_organization_id: ${{ vars.ANTHROPIC_ORGANIZATION_ID }}
|
|
39
|
-
claude_args: --permission-mode acceptEdits
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
# Agent evals
|
|
2
|
-
|
|
3
|
-
Each eval is one real task from recent work with its accepted outcome, written as `evals/<name>.json`: the prompt, plus the check that defines an acceptable result.
|
|
4
|
-
|
|
5
|
-
```json
|
|
6
|
-
{
|
|
7
|
-
"prompt": "<the task, as a person would ask it>",
|
|
8
|
-
"check": "<a shell command that exits 0 when the result is acceptable, for example: pnpm test>"
|
|
9
|
-
}
|
|
10
|
-
```
|
|
11
|
-
|
|
12
|
-
- Collect 20 to 50 real tasks from recent work.
|
|
13
|
-
- `.github/workflows/agent-evals.yml` runs the suite every night, and on every pull request that changes `CLAUDE.md`, `AGENTS.md`, `.mcp.json`, `.agents/`, `.claude/`, `.cursor/`, or `evals/`. `evals/check.sh` checks one eval's result.
|
|
14
|
-
- Set the repository variable `EVALS_MIN_PASS_RATE`, from 0 to 100. When the pass rate falls below it, the `Agent evals` check fails and the pull request cannot merge.
|
|
15
|
-
- Each production incident gets an eval, written by the team that owned the incident, and it stays in the suite.
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env bash
|
|
2
|
-
set -euo pipefail
|
|
3
|
-
|
|
4
|
-
eval_file=$1
|
|
5
|
-
conclusion=$2
|
|
6
|
-
|
|
7
|
-
if [[ $conclusion != success ]]; then
|
|
8
|
-
echo "FAIL $eval_file: Claude Code reported $conclusion"
|
|
9
|
-
exit 1
|
|
10
|
-
fi
|
|
11
|
-
|
|
12
|
-
if bash -c "$(jq -r '.check' "$eval_file")"; then
|
|
13
|
-
echo "PASS $eval_file"
|
|
14
|
-
else
|
|
15
|
-
echo "FAIL $eval_file"
|
|
16
|
-
exit 1
|
|
17
|
-
fi
|