@gredescerber/agent-crew 0.0.0-stage → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +24 -0
- package/LICENSE +201 -0
- package/README.md +411 -2
- package/README.ru.md +414 -0
- package/adapters/claude/adapter.js +107 -0
- package/adapters/codex/adapter.js +101 -0
- package/bin/crew.js +60 -0
- package/docs/AGENT-SETUP.md +138 -0
- package/ecc/profile.json +16 -0
- package/evals/README.md +123 -0
- package/evals/fixtures/mini-app/README.md +10 -0
- package/evals/fixtures/mini-app/contracts/health.json +11 -0
- package/evals/fixtures/mini-app/package.json +10 -0
- package/evals/fixtures/mini-app/src/cart.js +14 -0
- package/evals/fixtures/mini-app/src/handlers.js +16 -0
- package/evals/fixtures/mini-app/src/pager.js +7 -0
- package/evals/fixtures/mini-app/test/cart.test.js +10 -0
- package/evals/fixtures/mini-app/test/handlers.test.js +13 -0
- package/evals/fixtures/mini-app/test/pager.test.js +8 -0
- package/evals/tasks/add-input-validation.json +15 -0
- package/evals/tasks/fix-off-by-one.json +14 -0
- package/evals/tasks/handler-per-contract.json +15 -0
- package/evals/tasks/judges/add-input-validation.judge.mjs +28 -0
- package/evals/tasks/judges/fix-off-by-one.judge.mjs +22 -0
- package/evals/tasks/judges/handler-per-contract.judge.mjs +18 -0
- package/package.json +46 -4
- package/roles/analyst.md +11 -0
- package/roles/builder.md +16 -0
- package/roles/implementer.md +17 -0
- package/roles/integrator.md +12 -0
- package/roles/planner.md +19 -0
- package/roles/reviewer.md +16 -0
- package/roles/scout.md +15 -0
- package/skills/crew-brief-analysis/SKILL.md +32 -0
- package/skills/crew-kickoff/SKILL.md +77 -0
- package/skills/crew-kickoff/task-format.md +215 -0
- package/skills/crew-review/SKILL.md +22 -0
- package/skills/crew-workflow/SKILL.md +43 -0
- package/src/commands/adapters.js +85 -0
- package/src/commands/check.js +130 -0
- package/src/commands/commit.js +105 -0
- package/src/commands/compliance.js +109 -0
- package/src/commands/doctor.js +48 -0
- package/src/commands/eval.js +507 -0
- package/src/commands/github.js +169 -0
- package/src/commands/guard.js +101 -0
- package/src/commands/init.js +76 -0
- package/src/commands/kickoff.js +275 -0
- package/src/commands/labels.js +28 -0
- package/src/commands/metrics.js +48 -0
- package/src/commands/pr.js +211 -0
- package/src/commands/shots.js +51 -0
- package/src/commands/status.js +48 -0
- package/src/commands/task.js +569 -0
- package/src/commands/up.js +355 -0
- package/src/commands/worker.js +322 -0
- package/src/core/config.js +77 -0
- package/src/core/exec.js +50 -0
- package/src/core/guard.js +248 -0
- package/src/core/issues.js +154 -0
- package/src/core/lease.js +48 -0
- package/src/core/metrics.js +232 -0
- package/src/core/output.js +56 -0
- package/src/core/prd.js +593 -0
- package/src/core/proc.js +52 -0
- package/src/core/queue.js +92 -0
- package/src/core/registry.js +138 -0
- package/src/core/shots.js +68 -0
- package/src/core/supervisor.js +56 -0
- package/src/core/version.js +44 -0
- package/src/mcp/server.js +71 -0
- package/templates/labels.json +18 -0
- package/templates/mcp.optional.json +23 -0
- package/templates/product/.github/ISSUE_TEMPLATE/epic.yml +22 -0
- package/templates/product/.github/ISSUE_TEMPLATE/feedback.yml +36 -0
- package/templates/product/.github/ISSUE_TEMPLATE/task.yml +62 -0
- package/templates/product/.github/pull_request_template.md +16 -0
- package/templates/product/.github/workflows/crew-guard.yml +45 -0
- package/templates/product/.github/workflows/crew.yml +108 -0
- package/templates/product/AGENTS.md +109 -0
- package/templates/product/contracts/README.md +8 -0
- package/templates/product/docs/architecture.md +41 -0
- package/templates/product/docs/brief-analysis.md +62 -0
- package/templates/product/docs/brief.md +25 -0
- package/templates/product/docs/playbooks/issues.md +91 -0
- package/templates/product/docs/playbooks/preflight.md +28 -0
- package/templates/product/docs/playbooks/timeline.md +52 -0
- package/templates/product/docs/prd.md +58 -0
- package/templates/product/events/event.example.json +10 -0
- package/templates/product/tests/acceptance/README.md +9 -0
- package/templates/stacks/web-fastapi.json +74 -0
- package/templates/stacks/web-node.json +70 -0
package/bin/crew.js
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// crew: control panel for an agent team. See README.md.
|
|
3
|
+
import { readFileSync } from 'node:fs';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
import { commands, findCommand, parseArgs, validate } from '../src/core/registry.js';
|
|
6
|
+
import { CrewError, fail, render } from '../src/core/output.js';
|
|
7
|
+
import { HARNESS_ROOT } from '../src/core/config.js';
|
|
8
|
+
import { recordCommand } from '../src/core/metrics.js';
|
|
9
|
+
|
|
10
|
+
const argv = process.argv.slice(2);
|
|
11
|
+
const json = argv.includes('--json') || process.env.CREW_JSON === '1' || !process.stdout.isTTY;
|
|
12
|
+
const words = argv.filter((a) => a !== '--json');
|
|
13
|
+
|
|
14
|
+
function help() {
|
|
15
|
+
const width = Math.max(...commands.map((c) => c.name.length));
|
|
16
|
+
return [
|
|
17
|
+
'crew: control panel for an agent team (Claude Code, Codex, ...)',
|
|
18
|
+
'',
|
|
19
|
+
'Commands:',
|
|
20
|
+
...commands.map((c) => ` crew ${c.name.padEnd(width)} ${c.summary}`),
|
|
21
|
+
` crew ${'mcp'.padEnd(width)} Start the MCP server (for agents)`,
|
|
22
|
+
'',
|
|
23
|
+
'Flags: --json (machine output; enabled automatically when output is not a terminal)',
|
|
24
|
+
'Environment: CREW_AGENT=claude/opus: who is working (recorded in commits and tasks)',
|
|
25
|
+
].join('\n');
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
async function main() {
|
|
29
|
+
if (!words.length || ['help', '--help', '-h'].includes(words[0])) return console.log(help());
|
|
30
|
+
if (['--version', '-v', 'version'].includes(words[0])) {
|
|
31
|
+
return console.log(JSON.parse(readFileSync(join(HARNESS_ROOT, 'package.json'), 'utf8')).version);
|
|
32
|
+
}
|
|
33
|
+
if (words[0] === 'mcp') {
|
|
34
|
+
const { startServer } = await import('../src/mcp/server.js');
|
|
35
|
+
return startServer();
|
|
36
|
+
}
|
|
37
|
+
const cmd = findCommand(words);
|
|
38
|
+
if (!cmd) {
|
|
39
|
+
console.log(render(fail('unknown_command', `Unknown command: ${words.join(' ')}`, { retry: 'crew help' }), { json }));
|
|
40
|
+
process.exitCode = 2;
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
const opts = parseArgs(cmd, words.slice(cmd.name.split(' ').length));
|
|
44
|
+
const err = validate(cmd, opts);
|
|
45
|
+
const started = Date.now();
|
|
46
|
+
let result;
|
|
47
|
+
if (err) result = fail('bad_input', err, { retry: 'crew help' });
|
|
48
|
+
else {
|
|
49
|
+
try {
|
|
50
|
+
result = await cmd.run(opts, {});
|
|
51
|
+
} catch (e) {
|
|
52
|
+
result = e instanceof CrewError ? e.toResult() : fail('internal', e.message, { stop_when: 'it repeats: report to a human' });
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
recordCommand({ command: cmd.name, opts, result, started }); // .crew/metrics.jsonl; never throws
|
|
56
|
+
console.log(render(result, { json }));
|
|
57
|
+
if (result.status === 'error') process.exitCode = 1;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
main();
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
# Set up agent-crew: instructions for the agent
|
|
2
|
+
|
|
3
|
+
This file is read by an **agent** (Claude Code or Codex), not by a human. The human sends one message (ready-to-paste texts are at the end), and the agent walks through the steps on its own, stopping only where a human is needed.
|
|
4
|
+
|
|
5
|
+
## Rules for the agent
|
|
6
|
+
|
|
7
|
+
- Follow the steps in order. Every `crew` command returns JSON with `status`, `next_actions` and `artifacts`. Read it instead of guessing.
|
|
8
|
+
- `crew up` can be re-run with the same arguments: finished steps are skipped (`state: skipped`) and the command continues from where it failed.
|
|
9
|
+
- **Never type passwords, tokens or codes.** Logging in to GitHub, Claude and Codex is the human's job. Ask them and wait.
|
|
10
|
+
- Do not install anything without the human's consent. Propose the install command and run it after a "yes".
|
|
11
|
+
- `artifacts.human_steps` lists what only a human can do. Relay these steps briefly, as a list, with links.
|
|
12
|
+
- On an error (`status: error`), do what `error.retry` says. If `error.stop_when` applies or the same error repeats twice, stop and show the human the `summary` and `cause_hint`.
|
|
13
|
+
- **Language.** Everything you write to GitHub (issues, PRs, commits, comments) and everything agents write to each other is English. Answer the human in the human's language.
|
|
14
|
+
|
|
15
|
+
## Step 0. The machine and the tool itself
|
|
16
|
+
|
|
17
|
+
1. Check the tools: `node --version` (22+ needed), `git --version`, `gh --version`.
|
|
18
|
+
If something is missing, propose an install command to the human:
|
|
19
|
+
|
|
20
|
+
| | Windows | macOS |
|
|
21
|
+
|---|---|---|
|
|
22
|
+
| Node.js 22 | `winget install OpenJS.NodeJS.LTS` | `brew install node@22` |
|
|
23
|
+
| git | `winget install Git.Git` | `brew install git` |
|
|
24
|
+
| GitHub CLI | `winget install GitHub.cli` | `brew install gh` |
|
|
25
|
+
|
|
26
|
+
On Windows, a new terminal is needed after installing so PATH refreshes. Also run `git config --global core.longpaths true`, otherwise git fails on task worktrees and `node_modules`.
|
|
27
|
+
2. Check `gh auth status`. If not logged in, ask the human to run `gh auth login` (GitHub.com -> HTTPS -> log in through the browser) and wait.
|
|
28
|
+
3. Check `crew --version`. If the command is missing, install the tool straight from the public GitHub repo (no registry, no token):
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
npm install -g @gredescerber/agent-crew
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
Alternative: clone and link.
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
gh repo clone GredesCerber/agent-crew ~/agent-crew
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
In PowerShell write `$HOME` instead of `~`: external programs do not expand `~`. Then, in `~/agent-crew`, check out the latest release (`git tag --sort=-v:refname`, take the first, `git checkout <tag>`) and run `npm link`. There are no dependencies; `npm link` only puts the `crew` command on PATH.
|
|
41
|
+
4. Run `crew --version` again, then `crew doctor`.
|
|
42
|
+
|
|
43
|
+
## Captain: create the project
|
|
44
|
+
|
|
45
|
+
Take the parameters from the human's message. If something is missing, ask once for everything at the same time:
|
|
46
|
+
|
|
47
|
+
- `--repo <owner>/<name>`: the new product repo.
|
|
48
|
+
- `--stack web-node | web-fastapi`: the stack (React + Node.js or React + FastAPI).
|
|
49
|
+
- `--invite alice,bob`: GitHub logins of the members.
|
|
50
|
+
- `--public`: a public repo. On the free GitHub plan a private repo cannot have branch protection, and the `github-setup` step fails with HTTP 403. Unless the human says otherwise, use a public repo (or they need GitHub Pro/Team).
|
|
51
|
+
- `--hosts claude,codex`: agent hosts (default `claude,codex`).
|
|
52
|
+
- Optional event mode: `--start <ISO> --end <ISO>` (with a time zone, e.g. `2026-11-10T10:00:00+03:00`) and `--event-name "<name>"`. Without them, compliance checks are off.
|
|
53
|
+
|
|
54
|
+
Run from the folder **in which** the project should be created:
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
crew up <folder> --repo <owner>/<name> --stack web-node --invite alice,bob --public
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
The command does the following on its own:
|
|
61
|
+
1. creates the project (`crew init`);
|
|
62
|
+
2. writes `events/event.json` (event mode only);
|
|
63
|
+
3. makes the first commit;
|
|
64
|
+
4. creates the GitHub repo and pushes;
|
|
65
|
+
5. invites the members;
|
|
66
|
+
6. sets up protection of `main`, the labels and `CREW_HARNESS`.
|
|
67
|
+
|
|
68
|
+
Then:
|
|
69
|
+
|
|
70
|
+
1. Relay `artifacts.human_steps` to the human (for example, confirming project trust in Codex).
|
|
71
|
+
2. Check CI: `gh run list -R <owner>/<name> -L 3`. If a run failed early, rerun it: `gh run rerun <id> -R <owner>/<name>`. Wait for the result and make sure the status is `success`. If it fails again, show the human the last lines of `gh run view <id> -R <owner>/<name> --log-failed`.
|
|
72
|
+
3. Tell the human what to send to the members: the "Member" message from the end of this file with the repo filled in.
|
|
73
|
+
4. Tell the human how to start agents in the project folder (`next_actions` from `crew up`). The current session must be restarted inside the project folder, because agent configs are read at startup.
|
|
74
|
+
|
|
75
|
+
## Captain: the plan from the brief (after everyone has joined)
|
|
76
|
+
|
|
77
|
+
Only the **captain** plans: `crew up` wrote their login to `crew.config.json -> captain`, and `crew kickoff --apply` refuses for anyone else. In a new session in the project folder (branch `main`), send the "Plan" message from the end of this file together with the brief. The agent then:
|
|
78
|
+
1. puts the brief into `docs/brief.md` and analyzes it (skill `crew-brief-analysis`, including the "Stack" section);
|
|
79
|
+
2. writes `docs/prd.md` (skill `crew-kickoff`) with a `crew-team` block: who owns which area, and the captain as `review`;
|
|
80
|
+
3. shows the plan (`crew kickoff`) and waits for "approved";
|
|
81
|
+
4. runs `crew kickoff --apply`: tasks are assigned to the area owners and the plan is pushed to main;
|
|
82
|
+
5. works the loop: `/loop 10m Work the crew-workflow loop`. Reviews and merges reach the captain first.
|
|
83
|
+
|
|
84
|
+
## Member: join the project
|
|
85
|
+
|
|
86
|
+
Run from the folder **in which** the project should appear:
|
|
87
|
+
|
|
88
|
+
```bash
|
|
89
|
+
crew up --join <owner>/<name>
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
The command does the following on its own:
|
|
93
|
+
1. accepts the invitation to exactly this repo;
|
|
94
|
+
2. clones it;
|
|
95
|
+
3. configures git;
|
|
96
|
+
4. builds the local agent configs (`.mcp.json`, `.codex/config.toml`, skills).
|
|
97
|
+
|
|
98
|
+
Possible stops:
|
|
99
|
+
- `no_access`: the captain has not invited the human yet. The human asks the captain, then repeat the same command.
|
|
100
|
+
- `no_config`: the captain has not set the project up yet. Wait and repeat.
|
|
101
|
+
|
|
102
|
+
Then relay `human_steps` and `next_actions` to the human:
|
|
103
|
+
- **Claude Code**: open the project folder.
|
|
104
|
+
- **Codex**: start it with `CREW_AGENT=codex`:
|
|
105
|
+
- PowerShell: `$env:CREW_AGENT='codex'; codex`;
|
|
106
|
+
- bash: `CREW_AGENT=codex codex`.
|
|
107
|
+
|
|
108
|
+
On the first start, confirm trust in the project.
|
|
109
|
+
|
|
110
|
+
The first message to the agent in a new session (Claude Code, with `/loop`, so the agent checks the queue every 10 minutes on its own):
|
|
111
|
+
|
|
112
|
+
> /loop 10m Work the crew-workflow loop: take reviews and tasks of your area from crew task next and do them. Do not plan or run kickoff; the captain does that.
|
|
113
|
+
|
|
114
|
+
**Codex** has no `/loop`, so the loop runs right in the chat: `crew task next --wait 20` waits for work for up to 20 minutes by itself (verified: Codex waits for a long command and runs many laps in one turn). The first message:
|
|
115
|
+
|
|
116
|
+
> Work the crew-workflow loop without stopping and do not ask me between tasks. One lap: run `crew task next --wait 20` in the terminal and do what its next_actions say (fix: your own PR, merge: merge it, review: review per the crew-review skill, task: claim it, build to the acceptance criteria, crew check, crew commit, crew pr open), then run `crew task next --wait 20` again. If it answers "no free tasks" three times in a row, write me a short summary and stop. Do not plan or run kickoff; the captain does that.
|
|
117
|
+
|
|
118
|
+
If Codex nevertheless ends its turn, tell it "continue the loop".
|
|
119
|
+
|
|
120
|
+
A member **never plans**: no PRD, no `crew kickoff`. If the repo has no tasks yet, they wait for the captain.
|
|
121
|
+
|
|
122
|
+
## Ready-to-paste messages for the human
|
|
123
|
+
|
|
124
|
+
Copy into a Claude Code or Codex chat and fill in your values. You can write to the agent in your own language: it answers you in that language, but everything it writes to GitHub stays English.
|
|
125
|
+
|
|
126
|
+
**Captain:**
|
|
127
|
+
|
|
128
|
+
> Set up our agent-crew. If the `crew` command is missing, install it with `npm install -g @gredescerber/agent-crew`. Then read the `docs/AGENT-SETUP.md` of that package (or fetch it from github.com/GredesCerber/agent-crew) and do step 0 and the "Captain" section. Parameters: repo `<owner>/<name>`, public, stack `web-node`, members `<alice>,<bob>`. Create the project in the current folder. Do not type passwords: tell me what you need from me as a list.
|
|
129
|
+
|
|
130
|
+
**Plan (captain, after everyone has joined):**
|
|
131
|
+
|
|
132
|
+
> You are the captain. Put the brief below into `docs/brief.md`, analyze it with the crew-brief-analysis skill (including the "Stack" section), then use the crew-kickoff skill to write `docs/prd.md` with a crew-team block: take the members from `gh api repos/{owner}/{repo}/collaborators`; roles: `<me>` is backend and review, `<alice>` is frontend, `<bob>` is design and research. Show the plan (`crew kickoff`) and wait for my "approved". After that run `crew kickoff --apply` and work the loop: `/loop 10m Work the crew-workflow loop`. Do not type passwords.
|
|
133
|
+
>
|
|
134
|
+
> <brief>
|
|
135
|
+
|
|
136
|
+
**Member:**
|
|
137
|
+
|
|
138
|
+
> Connect me to the project. If the `crew` command is missing, install it with `npm install -g @gredescerber/agent-crew`. Then read the `docs/AGENT-SETUP.md` of that package (or fetch it from github.com/GredesCerber/agent-crew) and do step 0 and the "Member" section for the repo `<owner>/<name>`. Do not type passwords: tell me what you need from me as a list.
|
package/ecc/profile.json
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
{
|
|
2
|
+
"enabled": true,
|
|
3
|
+
"version": "2.2.3",
|
|
4
|
+
"marketplace": "https://github.com/affaan-m/ECC.git",
|
|
5
|
+
"about": "Which ECC skills to use per role. ECC is an accelerator for Claude Code, not a protocol dependency: everything works without it.",
|
|
6
|
+
"roles": {
|
|
7
|
+
"common": ["search-first", "documentation-lookup", "verification-loop", "strategic-compact", "gateguard", "safety-guard"],
|
|
8
|
+
"planner": ["plan-prd", "product-lens", "council", "contract-first", "api-design", "market-research"],
|
|
9
|
+
"builder": ["orch-add-feature", "orch-fix-defect", "tdd-workflow", "build-fix", "api-connector-builder"],
|
|
10
|
+
"reviewer": ["orch-review", "astra-review", "silent-failure-hunter", "security-review"],
|
|
11
|
+
"qa": ["e2e-testing", "browser-qa", "ai-regression-testing"],
|
|
12
|
+
"demo": ["ui-demo", "frontend-slides", "frontend-design-direction", "investor-materials"],
|
|
13
|
+
"ops": ["opensource-sanitizer", "cost-report", "harness-audit"]
|
|
14
|
+
},
|
|
15
|
+
"avoid": ["multi-*", "email-ops", "chief-of-staff", "messages-ops", "social-publisher", "dev-team", "team-builder"]
|
|
16
|
+
}
|
package/evals/README.md
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
# evals: comparing agents on reference tasks
|
|
2
|
+
|
|
3
|
+
`crew eval` runs agents (Claude Code, Codex, and their models) on the same tasks and computes pass@1, pass@k,
|
|
4
|
+
consistency, time and cost. This is how you check that a change to `templates/product/AGENTS.md`, skills or roles
|
|
5
|
+
did not make the agents worse.
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
crew eval --dry-run # validate tasks and show the plan (no agents are started)
|
|
9
|
+
crew eval --agents claude,codex --runs 3 # a real run: 3 tasks x 2 agents x 3 runs = 18 launches
|
|
10
|
+
crew eval --agents claude:opus,claude:sonnet --task fix-off-by-one --keep
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
| Flag | What it does |
|
|
14
|
+
|---|---|
|
|
15
|
+
| `--agents` | hosts from `adapters/`, comma-separated; a model goes after a colon: `codex:gpt-5`. Default: `hosts` from `crew.config.json` |
|
|
16
|
+
| `--tasks` | a folder with `*.json` or a single file (default `evals/tasks`, looked up from the cwd, then in the tool itself) |
|
|
17
|
+
| `--task` | only these tasks, comma-separated |
|
|
18
|
+
| `--runs` | runs per task and agent (default 3) |
|
|
19
|
+
| `--dry-run` | only validate tasks and show the plan |
|
|
20
|
+
| `--keep` | do not delete the working copies (the path is in `raw[].workdir` of the report) |
|
|
21
|
+
|
|
22
|
+
## How one run goes
|
|
23
|
+
|
|
24
|
+
1. **A fresh copy of the repo.** With `commit`: `git clone` of the repository that contains `repo`, then checkout of that commit;
|
|
25
|
+
without `commit`: a copy of the current folder state (without `.git` and `node_modules`) + `git init` and a commit.
|
|
26
|
+
The initial state is tagged `crew-eval-base`. The copy lives in the OS temp folder and is deleted after the run.
|
|
27
|
+
2. **The agent.** `adapter.headless({ prompt, model, cwd })` -> the command runs without a shell with `cwd` set to the copy.
|
|
28
|
+
The timeout is the task's `timeout_s` (default 900 s).
|
|
29
|
+
3. **Hidden tests.** Files from `judge_files` are copied into `.crew-eval/` of the copy **after** the agent has finished, so the agent never sees them.
|
|
30
|
+
4. **Judges** run in order; the task passes if all of them pass.
|
|
31
|
+
5. **Record:** pass/fail, time, cost (if the agent reports it, e.g. `total_cost_usd` in
|
|
32
|
+
`claude --output-format json`; otherwise `null`), exit code.
|
|
33
|
+
|
|
34
|
+
If an agent's binary cannot start, its remaining runs are skipped with an `agent_failed` error and the report is a warning.
|
|
35
|
+
|
|
36
|
+
## Task format (`evals/tasks/*.json`)
|
|
37
|
+
|
|
38
|
+
```json
|
|
39
|
+
{
|
|
40
|
+
"name": "fix-off-by-one",
|
|
41
|
+
"description": "Fix an off-by-one error in pagination",
|
|
42
|
+
"repo": "../fixtures/mini-app",
|
|
43
|
+
"files": ["src/pager.js"],
|
|
44
|
+
"prompt": "In src/pager.js the function paginate ... Verify with: npm test.",
|
|
45
|
+
"judge": [
|
|
46
|
+
{ "type": "command", "command": "node --test .crew-eval/fix-off-by-one.judge.mjs" },
|
|
47
|
+
{ "type": "command", "command": "node --test" },
|
|
48
|
+
{ "type": "command", "command": "git diff --quiet crew-eval-base -- test/pager.test.js", "expect_exit": 1 },
|
|
49
|
+
{ "type": "grep", "pattern": "TypeError", "files": ["src/cart.js"] }
|
|
50
|
+
],
|
|
51
|
+
"judge_files": ["judges/fix-off-by-one.judge.mjs"],
|
|
52
|
+
"commit": "abc1234",
|
|
53
|
+
"timeout_s": 600
|
|
54
|
+
}
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
| Field | Required | Meaning |
|
|
58
|
+
|---|---|---|
|
|
59
|
+
| `name` | yes | unique name: `a-z0-9._-` |
|
|
60
|
+
| `description` | no | one line for humans |
|
|
61
|
+
| `repo` | yes | folder of the fixture repo, **relative to the task file** |
|
|
62
|
+
| `files` | no | files the task is about (a hint for humans; the default for `grep` without `files`) |
|
|
63
|
+
| `prompt` | yes | what the agent receives, verbatim |
|
|
64
|
+
| `judge` | yes | a non-empty list of deterministic judges |
|
|
65
|
+
| `commit` | no | sha the repo is pinned to (reproducibility across days). Without it the current folder state is used, which suits fixtures inside this repo: their version is pinned by the tool's own commit |
|
|
66
|
+
| `judge_files` | no | *crew extension*: hidden tests, relative to the task file; placed into `.crew-eval/` after the agent has finished |
|
|
67
|
+
| `timeout_s` | no | *crew extension*: agent timeout in seconds |
|
|
68
|
+
|
|
69
|
+
### Judges
|
|
70
|
+
|
|
71
|
+
| `type` | Fields | Passes when |
|
|
72
|
+
|---|---|---|
|
|
73
|
+
| `command` (alias `pytest`) | `command`, `expect_exit` (0), `timeout_s` (300) | the command, run in the copy root (via a shell), returns `expect_exit` |
|
|
74
|
+
| `grep` | `pattern` (JS RegExp), `files` (a path, a glob like `src/**/*.js`, or an array), `flags`, `absent` | the pattern is found in at least one file; with `absent: true`, in none |
|
|
75
|
+
|
|
76
|
+
`llm` judges are not supported: the result must be deterministic. To check "did not touch the contract" or
|
|
77
|
+
"added a test", compare with the initial state: `git diff --exit-code crew-eval-base -- contracts/`.
|
|
78
|
+
|
|
79
|
+
## Compatibility with the agent-eval format
|
|
80
|
+
|
|
81
|
+
The format intentionally follows [agent-eval](https://github.com/joaquinhuigomez/agent-eval), so the tasks can
|
|
82
|
+
also be run with it (after reviewing its source). The differences are JSON instead of YAML (no dependencies) and two extension fields.
|
|
83
|
+
|
|
84
|
+
| agent-eval (YAML) | crew eval (JSON) |
|
|
85
|
+
|---|---|
|
|
86
|
+
| `name`, `description`, `prompt`, `files`, `commit` | the same |
|
|
87
|
+
| `repo: ./my-project` (relative to the launch cwd) | `repo` is **relative to the task file** |
|
|
88
|
+
| `judge: - type: pytest / command` + `command` | the same (`pytest` = `command`) |
|
|
89
|
+
| `judge: - type: grep` + `pattern`, `files` | the same; plus `absent`, `flags` |
|
|
90
|
+
| `judge: - type: llm` | not supported |
|
|
91
|
+
| none | `judge_files`, `timeout_s`, `expect_exit` (crew extensions; agent-eval ignores them) |
|
|
92
|
+
| `agent-eval run --agent claude-code --runs 3` | `crew eval --agents claude --runs 3` |
|
|
93
|
+
| a separate git worktree per run | a temporary copy/clone per run (a worktree of a fixture inside the tool repo is not possible) |
|
|
94
|
+
|
|
95
|
+
YAML -> JSON by hand: `prompt: |` becomes a string with `\n`, lists become arrays. For agent-eval, put the hidden tests from
|
|
96
|
+
`judge_files` into the repo at the pinned commit and replace `.crew-eval/...` in the commands with their path.
|
|
97
|
+
|
|
98
|
+
## Report metrics
|
|
99
|
+
|
|
100
|
+
`artifacts.results` is per agent and task, `artifacts.agents` is per agent averaged over tasks,
|
|
101
|
+
and the full report with every run is `.crew/evals/eval-<time>.json`.
|
|
102
|
+
|
|
103
|
+
| Metric | How it is computed |
|
|
104
|
+
|---|---|
|
|
105
|
+
| `pass_at_1` | share of successful runs, c / n |
|
|
106
|
+
| `pass_at_k` | probability of at least one success in k = n runs (the unbiased estimator of Chen et al.: 1 - C(n-c, k) / C(n, k)) |
|
|
107
|
+
| `consistency` | share of runs that match the majority: max(c, n-c) / n. 1 means the agent behaves the same every time (consistently solves or consistently fails). In agent-eval "consistency" is c/n; here that is `pass_at_1` |
|
|
108
|
+
| `avg_time_ms` | average agent working time (judges excluded) |
|
|
109
|
+
| `avg_cost_usd` / `total_cost_usd` | if the agent reports cost; otherwise `null` |
|
|
110
|
+
|
|
111
|
+
## Fixture and tasks
|
|
112
|
+
|
|
113
|
+
`fixtures/mini-app/` is a small dependency-free Node module with `node:test` tests: a cart, pagination,
|
|
114
|
+
HTTP handlers and the contract `contracts/health.json`.
|
|
115
|
+
|
|
116
|
+
| Task | What it checks |
|
|
117
|
+
|---|---|
|
|
118
|
+
| `fix-off-by-one` | find and fix an off-by-one error, add a regression test |
|
|
119
|
+
| `add-input-validation` | input validation with clear errors, without breaking the old behavior |
|
|
120
|
+
| `handler-per-contract` | implement a route strictly per the contract, without changing the contract or hardcoding the version |
|
|
121
|
+
|
|
122
|
+
A new task: a file `tasks/<name>.json` + (if needed) a hidden test `tasks/judges/<name>.judge.mjs`. The check that a
|
|
123
|
+
task is not already solved and that the judges really catch it lives in `test/eval.test.js` (`npm test`).
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# mini-app
|
|
2
|
+
|
|
3
|
+
A small module for `crew eval` tasks. No dependencies, Node.js 20+.
|
|
4
|
+
|
|
5
|
+
- `src/cart.js` — cart: `createCart`, `addItem`, `total`.
|
|
6
|
+
- `src/pager.js` — pagination `paginate(items, page, size)`, pages start at 1.
|
|
7
|
+
- `src/handlers.js` — HTTP route handlers `handle({ method, path })`.
|
|
8
|
+
- `contracts/` — API contracts. Do not change them, only implement them.
|
|
9
|
+
|
|
10
|
+
Tests: `npm test` (or `node --test`).
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
// Shopping cart. Item: { sku: string, qty: integer > 0, price: number >= 0 }.
|
|
2
|
+
|
|
3
|
+
export function createCart() {
|
|
4
|
+
return { items: [] };
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
export function addItem(cart, item) {
|
|
8
|
+
cart.items.push({ sku: item.sku, qty: item.qty, price: item.price });
|
|
9
|
+
return cart;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export function total(cart) {
|
|
13
|
+
return cart.items.reduce((sum, i) => sum + i.qty * i.price, 0);
|
|
14
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
// HTTP route handlers. Key: "METHOD /path", value: a function with no arguments -> { status, body }.
|
|
2
|
+
// Route contracts live in contracts/.
|
|
3
|
+
|
|
4
|
+
const items = [
|
|
5
|
+
{ id: 1, name: 'tea' },
|
|
6
|
+
{ id: 2, name: 'coffee' },
|
|
7
|
+
];
|
|
8
|
+
|
|
9
|
+
export const routes = {
|
|
10
|
+
'GET /items': () => ({ status: 200, body: items }),
|
|
11
|
+
};
|
|
12
|
+
|
|
13
|
+
export function handle({ method, path }) {
|
|
14
|
+
const h = routes[`${method} ${path}`];
|
|
15
|
+
return h ? h() : { status: 404, body: { error: 'not_found' } };
|
|
16
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
// Pagination. Pages are numbered from 1: page=1 is the first `size` items.
|
|
2
|
+
|
|
3
|
+
export function paginate(items, page = 1, size = 10) {
|
|
4
|
+
const pages = Math.max(1, Math.ceil(items.length / size));
|
|
5
|
+
const start = page * size;
|
|
6
|
+
return { items: items.slice(start, start + size), page, pages };
|
|
7
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { test } from 'node:test';
|
|
2
|
+
import assert from 'node:assert/strict';
|
|
3
|
+
import { createCart, addItem, total } from '../src/cart.js';
|
|
4
|
+
|
|
5
|
+
test('cart total', () => {
|
|
6
|
+
const cart = createCart();
|
|
7
|
+
addItem(cart, { sku: 'tea', qty: 2, price: 150 });
|
|
8
|
+
addItem(cart, { sku: 'cup', qty: 1, price: 300 });
|
|
9
|
+
assert.equal(total(cart), 600);
|
|
10
|
+
});
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { test } from 'node:test';
|
|
2
|
+
import assert from 'node:assert/strict';
|
|
3
|
+
import { handle } from '../src/handlers.js';
|
|
4
|
+
|
|
5
|
+
test('GET /items returns a list', () => {
|
|
6
|
+
const r = handle({ method: 'GET', path: '/items' });
|
|
7
|
+
assert.equal(r.status, 200);
|
|
8
|
+
assert.ok(Array.isArray(r.body));
|
|
9
|
+
});
|
|
10
|
+
|
|
11
|
+
test('unknown route -> 404', () => {
|
|
12
|
+
assert.equal(handle({ method: 'GET', path: '/nope' }).status, 404);
|
|
13
|
+
});
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { test } from 'node:test';
|
|
2
|
+
import assert from 'node:assert/strict';
|
|
3
|
+
import { paginate } from '../src/pager.js';
|
|
4
|
+
|
|
5
|
+
test('number of pages', () => {
|
|
6
|
+
assert.equal(paginate([1, 2, 3, 4, 5], 1, 2).pages, 3);
|
|
7
|
+
assert.equal(paginate([], 1, 10).pages, 1);
|
|
8
|
+
});
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "add-input-validation",
|
|
3
|
+
"description": "Add input validation to addItem",
|
|
4
|
+
"repo": "../fixtures/mini-app",
|
|
5
|
+
"files": ["src/cart.js", "test/cart.test.js"],
|
|
6
|
+
"prompt": "In src/cart.js the function addItem(cart, item) accepts anything. Add validation: sku is a non-empty string, qty is an integer greater than 0, price is a finite number not less than 0. On a violation throw a TypeError and name the field (sku, qty or price) in the message. Valid items must still be added as before. Add tests for the errors in test/cart.test.js. Verify with: npm test.",
|
|
7
|
+
"judge": [
|
|
8
|
+
{ "type": "command", "command": "node --test .crew-eval/add-input-validation.judge.mjs" },
|
|
9
|
+
{ "type": "command", "command": "node --test" },
|
|
10
|
+
{ "type": "grep", "pattern": "TypeError", "files": ["src/cart.js"] },
|
|
11
|
+
{ "type": "grep", "pattern": "throws|rejects", "files": ["test/cart.test.js"] }
|
|
12
|
+
],
|
|
13
|
+
"judge_files": ["judges/add-input-validation.judge.mjs"],
|
|
14
|
+
"timeout_s": 600
|
|
15
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "fix-off-by-one",
|
|
3
|
+
"description": "Fix an off-by-one error in pagination",
|
|
4
|
+
"repo": "../fixtures/mini-app",
|
|
5
|
+
"files": ["src/pager.js"],
|
|
6
|
+
"prompt": "In src/pager.js the function paginate(items, page, size) returns the wrong page: pages are numbered from 1, so page=1 must return the first `size` items, but it currently returns the second chunk. Fix the bug without changing the signature or the response shape { items, page, pages }. Add a test for the first and the last (partial) page in test/pager.test.js. Verify with: npm test.",
|
|
7
|
+
"judge": [
|
|
8
|
+
{ "type": "command", "command": "node --test .crew-eval/fix-off-by-one.judge.mjs" },
|
|
9
|
+
{ "type": "command", "command": "node --test" },
|
|
10
|
+
{ "type": "command", "command": "git diff --quiet crew-eval-base -- test/pager.test.js", "expect_exit": 1 }
|
|
11
|
+
],
|
|
12
|
+
"judge_files": ["judges/fix-off-by-one.judge.mjs"],
|
|
13
|
+
"timeout_s": 600
|
|
14
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "handler-per-contract",
|
|
3
|
+
"description": "Implement the GET /health handler strictly per the contract",
|
|
4
|
+
"repo": "../fixtures/mini-app",
|
|
5
|
+
"files": ["src/handlers.js", "contracts/health.json"],
|
|
6
|
+
"prompt": "Implement the route from the contract contracts/health.json in src/handlers.js (add it to routes). Do not change the contract. Read the version from package.json, do not hardcode it. Add a test in test/handlers.test.js. Verify with: npm test.",
|
|
7
|
+
"judge": [
|
|
8
|
+
{ "type": "command", "command": "node --test .crew-eval/handler-per-contract.judge.mjs" },
|
|
9
|
+
{ "type": "command", "command": "node --test" },
|
|
10
|
+
{ "type": "command", "command": "git diff --exit-code crew-eval-base -- contracts/" },
|
|
11
|
+
{ "type": "grep", "pattern": "['\"]1\\.2\\.0['\"]", "files": ["src/handlers.js"], "absent": true }
|
|
12
|
+
],
|
|
13
|
+
"judge_files": ["judges/handler-per-contract.judge.mjs"],
|
|
14
|
+
"timeout_s": 600
|
|
15
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
// Hidden test of the add-input-validation eval task. Copied into .crew-eval/ only after the agent has finished.
|
|
2
|
+
import { test } from 'node:test';
|
|
3
|
+
import assert from 'node:assert/strict';
|
|
4
|
+
import { createCart, addItem, total } from '../src/cart.js';
|
|
5
|
+
|
|
6
|
+
const good = { sku: 'tea', qty: 2, price: 150 };
|
|
7
|
+
|
|
8
|
+
test('a valid item is added', () => {
|
|
9
|
+
const cart = createCart();
|
|
10
|
+
addItem(cart, good);
|
|
11
|
+
addItem(cart, { sku: 'gift', qty: 1, price: 0 });
|
|
12
|
+
assert.equal(total(cart), 300);
|
|
13
|
+
assert.equal(cart.items.length, 2);
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
for (const [field, values] of Object.entries({
|
|
17
|
+
sku: ['', 42, null, undefined],
|
|
18
|
+
qty: [0, -1, 1.5, '2', NaN],
|
|
19
|
+
price: [-1, Infinity, NaN, '10'],
|
|
20
|
+
})) {
|
|
21
|
+
for (const v of values) {
|
|
22
|
+
test(`${field}=${String(v)} -> TypeError naming the field`, () => {
|
|
23
|
+
const cart = createCart();
|
|
24
|
+
assert.throws(() => addItem(cart, { ...good, [field]: v }), (e) => e instanceof TypeError && e.message.includes(field));
|
|
25
|
+
assert.equal(cart.items.length, 0, 'an invalid item must not end up in the cart');
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
// Hidden test of the fix-off-by-one eval task. Copied into .crew-eval/ only after the agent has finished.
|
|
2
|
+
import { test } from 'node:test';
|
|
3
|
+
import assert from 'node:assert/strict';
|
|
4
|
+
import { paginate } from '../src/pager.js';
|
|
5
|
+
|
|
6
|
+
const list = [1, 2, 3, 4, 5];
|
|
7
|
+
|
|
8
|
+
test('the first page holds the first items', () => {
|
|
9
|
+
assert.deepEqual(paginate(list, 1, 2), { items: [1, 2], page: 1, pages: 3 });
|
|
10
|
+
});
|
|
11
|
+
|
|
12
|
+
test('the last page is partial', () => {
|
|
13
|
+
assert.deepEqual(paginate(list, 3, 2).items, [5]);
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
test('a page out of range is empty', () => {
|
|
17
|
+
assert.deepEqual(paginate(list, 4, 2).items, []);
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
test('defaults: page=1, size=10', () => {
|
|
21
|
+
assert.deepEqual(paginate(list).items, list);
|
|
22
|
+
});
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
// Hidden test of the handler-per-contract eval task. Copied into .crew-eval/ only after the agent has finished.
|
|
2
|
+
import { test } from 'node:test';
|
|
3
|
+
import assert from 'node:assert/strict';
|
|
4
|
+
import { readFileSync } from 'node:fs';
|
|
5
|
+
import { handle } from '../src/handlers.js';
|
|
6
|
+
|
|
7
|
+
const pkg = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8'));
|
|
8
|
+
|
|
9
|
+
test('GET /health matches the contract', () => {
|
|
10
|
+
const r = handle({ method: 'GET', path: '/health' });
|
|
11
|
+
assert.equal(r.status, 200);
|
|
12
|
+
assert.deepEqual(r.body, { status: 'ok', version: pkg.version });
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
test('existing routes still work', () => {
|
|
16
|
+
assert.equal(handle({ method: 'GET', path: '/items' }).status, 200);
|
|
17
|
+
assert.equal(handle({ method: 'POST', path: '/health' }).status, 404);
|
|
18
|
+
});
|
package/package.json
CHANGED
|
@@ -1,6 +1,48 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gredescerber/agent-crew",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.5.0",
|
|
4
|
+
"description": "Vendor-neutral crew of coding agents (Claude Code, Codex, ...) working as one team through GitHub Issues and PRs",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"bin": {
|
|
7
|
+
"crew": "bin/crew.js"
|
|
8
|
+
},
|
|
9
|
+
"engines": {
|
|
10
|
+
"node": ">=22"
|
|
11
|
+
},
|
|
12
|
+
"scripts": {
|
|
13
|
+
"test": "node --test \"test/*.test.js\"",
|
|
14
|
+
"crew": "node bin/crew.js"
|
|
15
|
+
},
|
|
16
|
+
"files": [
|
|
17
|
+
"bin",
|
|
18
|
+
"src",
|
|
19
|
+
"adapters",
|
|
20
|
+
"roles",
|
|
21
|
+
"skills",
|
|
22
|
+
"templates",
|
|
23
|
+
"ecc",
|
|
24
|
+
"AGENTS.md",
|
|
25
|
+
"docs/AGENT-SETUP.md",
|
|
26
|
+
"evals"
|
|
27
|
+
],
|
|
28
|
+
"repository": {
|
|
29
|
+
"type": "git",
|
|
30
|
+
"url": "git+https://github.com/GredesCerber/agent-crew.git"
|
|
31
|
+
},
|
|
32
|
+
"license": "Apache-2.0",
|
|
33
|
+
"keywords": [
|
|
34
|
+
"ai-agents",
|
|
35
|
+
"claude-code",
|
|
36
|
+
"codex",
|
|
37
|
+
"multi-agent",
|
|
38
|
+
"github",
|
|
39
|
+
"mcp"
|
|
40
|
+
],
|
|
41
|
+
"publishConfig": {
|
|
42
|
+
"access": "public"
|
|
43
|
+
},
|
|
44
|
+
"homepage": "https://github.com/GredesCerber/agent-crew#readme",
|
|
45
|
+
"bugs": {
|
|
46
|
+
"url": "https://github.com/GredesCerber/agent-crew/issues"
|
|
47
|
+
}
|
|
48
|
+
}
|
package/roles/analyst.md
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
---
|
|
2
|
+
description: Analyzes the project brief into docs/brief-analysis.md - demo scenario, requirements with IDs and priorities, stakeholder criteria (judging criteria in event mode), ambiguities, risks. Works first, before planning.
|
|
3
|
+
tier: high
|
|
4
|
+
---
|
|
5
|
+
You are the analyst. You do not write code and you do not create tasks.
|
|
6
|
+
|
|
7
|
+
1. Work by the skill `crew-brief-analysis`, 30-minute limit.
|
|
8
|
+
2. The main thing is the **demo scenario**: scope is derived from what stakeholders (judges in event mode) will see in 3 minutes, not from the length of the brief.
|
|
9
|
+
3. Every requirement has an ID `R*` and a source quote. IDs stay stable for the whole project.
|
|
10
|
+
4. Do not wait for answers from the brief's authors — record assumptions.
|
|
11
|
+
5. When done, tell the people briefly: the demo scenario in 3 lines, how many must/should/could, the main ambiguities.
|