@gredescerber/agent-crew 0.0.0-stage → 0.6.0

package/adapters/codex/adapter.js

@@ -0,0 +1,101 @@
+// Codex CLI adapter.
+// Codex reads AGENTS.md from the repo root itself, so no instructions need to be generated.
+// Generates: .codex/config.toml (MCP servers; Codex reads it only if the project is marked trusted),
+// .agents/skills/<name>/SKILL.md (Codex project skills).
+// Codex gets roles as role-<name> skills: no separate subagent format is needed.
+import { statSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+
+const isFile = (path) => { try { return statSync(path).isFile(); } catch { return false; } };
+
+// Do not run npm's .cmd shim: cmd interprets &, %, quotes, etc. even with shell:false.
+// Resolve supported native/npm installations in PATH order, keeping every argument literal.
+function windowsCommand(searchPath) {
+  for (const entry of searchPath.split(';')) {
+    const dir = entry.replace(/^"(.*)"$/, '$1');
+    if (!dir) continue;
+    const prefix = resolve(dir);
+    const binary = join(prefix, 'codex.exe');
+    if (isFile(binary)) return { cmd: binary, args: [] };
+    const script = join(prefix, 'node_modules', '@openai', 'codex', 'bin', 'codex.js');
+    if (isFile(join(prefix, 'codex.cmd')) && isFile(script)) return { cmd: process.execPath, args: [script] };
+  }
+  // Let spawn report a missing native executable; never fall back to a shell script.
+  return { cmd: 'codex.exe', args: [] };
+}
+
+const tomlStr = (s) => JSON.stringify(String(s)); // a JSON string is a valid TOML basic string
+const tomlArr = (a) => `[${a.map(tomlStr).join(', ')}]`;
+
+const serverBlock = (name, srv, platform) => {
+  const winWrap = platform === 'win32' && ['npx', 'crew', 'npm'].includes(srv.command);
+  const command = winWrap ? 'cmd' : srv.command;
+  const args = winWrap ? ['/c', srv.command, ...(srv.args ?? [])] : srv.args ?? [];
+  const lines = [`[mcp_servers.${name}]`, `command = ${tomlStr(command)}`, `args = ${tomlArr(args)}`];
+  if (srv.env_from?.length) lines.push(`env_vars = ${tomlArr(srv.env_from)}`); // secrets only from the environment
+  if (srv.approval) lines.push(`default_tools_approval_mode = ${tomlStr(srv.approval)}`);
+  if (srv.env && Object.keys(srv.env).length) {
+    lines.push(`env = { ${Object.entries(srv.env).map(([k, v]) => `${k} = ${tomlStr(v)}`).join(', ')} }`);
+  }
+  return lines.join('\n');
+};
+
+export default {
+  id: 'codex',
+  binary: 'codex',
+
+  generate({ config, roles, skills, frontmatter, platform }) {
+    const files = [];
+    const blocks = [
+      '# Generated by crew adapters build. Do not edit: change crew.config.json instead.',
+      '# Codex reads this file only if the project is marked as trusted (run codex in the root and confirm).',
+      '',
+      serverBlock('crew', {
+        command: 'crew', args: ['mcp'], env: { CREW_AGENT: 'codex' },
+        // Codex filters the MCP environment. Keep the worker's lease identity and gh auth,
+        // but serialize only variable names, never credentials or a machine-specific id.
+        env_from: ['CREW_AGENT_ID', 'GH_TOKEN', 'GITHUB_TOKEN', 'GH_HOST', 'GH_ENTERPRISE_TOKEN', 'GITHUB_ENTERPRISE_TOKEN'],
+        // #12: codex exec (crew worker) runs with approval policy never; without this every crew_* call is rejected
+        // ("MCP tool call requires approval, but approval policy is never"). We allow only the crew server:
+        // over MCP it exposes only agent commands (agentCommands); dangerous init/labels/github setup are not included.
+        // The key was verified against codex-cli 0.166 in a live run; other MCP servers keep requiring confirmation.
+        approval: 'approve',
+      }, platform),
+    ];
+    for (const [name, srv] of Object.entries(config.mcp ?? {})) if (srv.enabled) blocks.push('', serverBlock(name, srv, platform));
+    files.push({ path: '.codex/config.toml', commit: false, content: blocks.join('\n') + '\n' });
+
+    for (const s of skills) files.push({ path: `.agents/skills/${s.name}/SKILL.md`, commit: false, content: s.content });
+
+    for (const r of roles) {
+      const { meta, body } = frontmatter(r.content);
+      const content = `---\nname: role-${r.name}\ndescription: ${meta.description ?? r.name}\n---\n${body}`;
+      files.push({ path: `.agents/skills/role-${r.name}/SKILL.md`, commit: false, content });
+    }
+    return files;
+  },
+
+  /**
+   * Headless run for crew worker. Flags verified against `codex exec --help` (codex-cli 0.166):
+   *   -C <dir>              agent working root (the task worktree)
+   *   -s workspace-write    file edits and commands inside the worktree, no confirmations
+   *   -c sandbox_workspace_write.network_access=true  - needed for gh/git push in crew pr open
+   *   --add-dir <dir>       extra writable folders: the repo's shared .git (a commit from the worktree writes there)
+   *   --json                JSONL events on stdout (go to the worker log)
+   *   -                     the prompt is read from stdin; on Windows it runs without cmd /c
+   */
+  headless({ prompt, model, cwd, writableDirs = [], platform = process.platform, searchPath = process.env.PATH ?? '' }) {
+    const args = ['exec', '--json', '-s', 'workspace-write', '-c', 'sandbox_workspace_write.network_access=true'];
+    if (cwd) args.push('-C', cwd);
+    for (const d of writableDirs) args.push('--add-dir', d);
+    if (model) args.push('--model', model);
+    args.push('-');
+    const launch = platform === 'win32' ? windowsCommand(searchPath) : { cmd: 'codex', args: [] };
+    return {
+      cmd: launch.cmd,
+      args: [...launch.args, ...args],
+      input: prompt,
+      env: { CREW_AGENT: `codex/${model || 'default'}` },
+    };
+  },
+};

package/bin/crew.js

@@ -0,0 +1,60 @@
+#!/usr/bin/env node
+// crew: control panel for an agent team. See README.md.
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { commands, findCommand, parseArgs, validate } from '../src/core/registry.js';
+import { CrewError, fail, render } from '../src/core/output.js';
+import { HARNESS_ROOT } from '../src/core/config.js';
+import { recordCommand } from '../src/core/metrics.js';
+
+const argv = process.argv.slice(2);
+const json = argv.includes('--json') || process.env.CREW_JSON === '1' || !process.stdout.isTTY;
+const words = argv.filter((a) => a !== '--json');
+
+function help() {
+  const width = Math.max(...commands.map((c) => c.name.length));
+  return [
+    'crew: control panel for an agent team (Claude Code, Codex, ...)',
+    '',
+    'Commands:',
+    ...commands.map((c) => `  crew ${c.name.padEnd(width)}  ${c.summary}`),
+    `  crew ${'mcp'.padEnd(width)}  Start the MCP server (for agents)`,
+    '',
+    'Flags: --json (machine output; enabled automatically when output is not a terminal)',
+    'Environment: CREW_AGENT=claude/opus: who is working (recorded in commits and tasks)',
+  ].join('\n');
+}
+
+async function main() {
+  if (!words.length || ['help', '--help', '-h'].includes(words[0])) return console.log(help());
+  if (['--version', '-v', 'version'].includes(words[0])) {
+    return console.log(JSON.parse(readFileSync(join(HARNESS_ROOT, 'package.json'), 'utf8')).version);
+  }
+  if (words[0] === 'mcp') {
+    const { startServer } = await import('../src/mcp/server.js');
+    return startServer();
+  }
+  const cmd = findCommand(words);
+  if (!cmd) {
+    console.log(render(fail('unknown_command', `Unknown command: ${words.join(' ')}`, { retry: 'crew help' }), { json }));
+    process.exitCode = 2;
+    return;
+  }
+  const opts = parseArgs(cmd, words.slice(cmd.name.split(' ').length));
+  const err = validate(cmd, opts);
+  const started = Date.now();
+  let result;
+  if (err) result = fail('bad_input', err, { retry: 'crew help' });
+  else {
+    try {
+      result = await cmd.run(opts, {});
+    } catch (e) {
+      result = e instanceof CrewError ? e.toResult() : fail('internal', e.message, { stop_when: 'it repeats: report to a human' });
+    }
+  }
+  recordCommand({ command: cmd.name, opts, result, started }); // .crew/metrics.jsonl; never throws
+  console.log(render(result, { json }));
+  if (result.status === 'error') process.exitCode = 1;
+}
+
+main();

package/docs/AGENT-SETUP.md

@@ -0,0 +1,138 @@
+# Set up agent-crew: instructions for the agent
+
+This file is read by an **agent** (Claude Code or Codex), not by a human. The human sends one message (ready-to-paste texts are at the end), and the agent walks through the steps on its own, stopping only where a human is needed.
+
+## Rules for the agent
+
+- Follow the steps in order. Every `crew` command returns JSON with `status`, `next_actions` and `artifacts`. Read it instead of guessing.
+- `crew up` can be re-run with the same arguments: finished steps are skipped (`state: skipped`) and the command continues from where it failed.
+- **Never type passwords, tokens or codes.** Logging in to GitHub, Claude and Codex is the human's job. Ask them and wait.
+- Do not install anything without the human's consent. Propose the install command and run it after a "yes".
+- `artifacts.human_steps` lists what only a human can do. Relay these steps briefly, as a list, with links.
+- On an error (`status: error`), do what `error.retry` says. If `error.stop_when` applies or the same error repeats twice, stop and show the human the `summary` and `cause_hint`.
+- **Language.** Everything you write to GitHub (issues, PRs, commits, comments) and everything agents write to each other is English. Answer the human in the human's language.
+
+## Step 0. The machine and the tool itself
+
+1. Check the tools: `node --version` (22+ needed), `git --version`, `gh --version`.
+   If something is missing, propose an install command to the human:
+
+   | | Windows | macOS |
+   |---|---|---|
+   | Node.js 22 | `winget install OpenJS.NodeJS.LTS` | `brew install node@22` |
+   | git | `winget install Git.Git` | `brew install git` |
+   | GitHub CLI | `winget install GitHub.cli` | `brew install gh` |
+
+   On Windows, a new terminal is needed after installing so PATH refreshes. Also run `git config --global core.longpaths true`, otherwise git fails on task worktrees and `node_modules`.
+2. Check `gh auth status`. If not logged in, ask the human to run `gh auth login` (GitHub.com -> HTTPS -> log in through the browser) and wait.
+3. Check `crew --version`. If the command is missing, install the tool straight from the public GitHub repo (no registry, no token):
+
+   ```bash
+   npm install -g @gredescerber/agent-crew
+   ```
+
+   Alternative: clone and link.
+
+   ```bash
+   gh repo clone GredesCerber/agent-crew ~/agent-crew
+   ```
+
+   In PowerShell write `$HOME` instead of `~`: external programs do not expand `~`. Then, in `~/agent-crew`, check out the latest release (`git tag --sort=-v:refname`, take the first, `git checkout <tag>`) and run `npm link`. There are no dependencies; `npm link` only puts the `crew` command on PATH.
+4. Run `crew --version` again, then `crew doctor`.
+
+## Captain: create the project
+
+Take the parameters from the human's message. If something is missing, ask once for everything at the same time:
+
+- `--repo <owner>/<name>`: the new product repo.
+- `--stack web-node | web-fastapi`: the stack (React + Node.js or React + FastAPI).
+- `--invite alice,bob`: GitHub logins of the members.
+- `--public`: a public repo. On the free GitHub plan a private repo cannot have branch protection, and the `github-setup` step fails with HTTP 403. Unless the human says otherwise, use a public repo (or they need GitHub Pro/Team).
+- `--hosts claude,codex`: agent hosts (default `claude,codex`).
+- Optional event mode: `--start <ISO> --end <ISO>` (with a time zone, e.g. `2026-11-10T10:00:00+03:00`) and `--event-name "<name>"`. Without them, compliance checks are off.
+
+Run from the folder **in which** the project should be created:
+
+```bash
+crew up <folder> --repo <owner>/<name> --stack web-node --invite alice,bob --public
+```
+
+The command does the following on its own:
+1. creates the project (`crew init`);
+2. writes `events/event.json` (event mode only);
+3. makes the first commit;
+4. creates the GitHub repo and pushes;
+5. invites the members;
+6. sets up protection of `main`, the labels and `CREW_HARNESS`.
+
+Then:
+
+1. Relay `artifacts.human_steps` to the human (for example, confirming project trust in Codex).
+2. Check CI: `gh run list -R <owner>/<name> -L 3`. If a run failed early, rerun it: `gh run rerun <id> -R <owner>/<name>`. Wait for the result and make sure the status is `success`. If it fails again, show the human the last lines of `gh run view <id> -R <owner>/<name> --log-failed`.
+3. Tell the human what to send to the members: the "Member" message from the end of this file with the repo filled in.
+4. Tell the human how to start agents in the project folder (`next_actions` from `crew up`). The current session must be restarted inside the project folder, because agent configs are read at startup.
+
+## Captain: the plan from the brief (after everyone has joined)
+
+Only the **captain** plans: `crew up` wrote their login to `crew.config.json -> captain`, and `crew kickoff --apply` refuses for anyone else. In a new session in the project folder (branch `main`), send the "Plan" message from the end of this file together with the brief. The agent then:
+1. puts the brief into `docs/brief.md` and analyzes it (skill `crew-brief-analysis`, including the "Stack" section);
+2. writes `docs/prd.md` (skill `crew-kickoff`) with a `crew-team` block: who owns which area, and the captain as `review`;
+3. shows the plan (`crew kickoff`) and waits for "approved";
+4. runs `crew kickoff --apply`: tasks are assigned to the area owners and the plan is pushed to main;
+5. works the loop: `/loop 10m Work the crew-workflow loop`. Reviews and merges reach the captain first.
+
+## Member: join the project
+
+Run from the folder **in which** the project should appear:
+
+```bash
+crew up --join <owner>/<name>
+```
+
+The command does the following on its own:
+1. accepts the invitation to exactly this repo;
+2. clones it;
+3. configures git;
+4. builds the local agent configs (`.mcp.json`, `.codex/config.toml`, skills).
+
+Possible stops:
+- `no_access`: the captain has not invited the human yet. The human asks the captain, then repeat the same command.
+- `no_config`: the captain has not set the project up yet. Wait and repeat.
+
+Then relay `human_steps` and `next_actions` to the human:
+- **Claude Code**: open the project folder.
+- **Codex**: start it with `CREW_AGENT=codex`:
+  - PowerShell: `$env:CREW_AGENT='codex'; codex`;
+  - bash: `CREW_AGENT=codex codex`.
+
+  On the first start, confirm trust in the project.
+
+The first message to the agent in a new session (Claude Code, with `/loop`, so the agent checks the queue every 10 minutes on its own):
+
+> /loop 10m Work the crew-workflow loop: take reviews and tasks of your area from crew task next and do them. Do not plan or run kickoff; the captain does that.
+
+**Codex** has no `/loop`, so the loop runs right in the chat: `crew task next --wait 20` waits for work for up to 20 minutes by itself (verified: Codex waits for a long command and runs many laps in one turn). The first message:
+
+> Work the crew-workflow loop without stopping and do not ask me between tasks. One lap: run `crew task next --wait 20` in the terminal and do what its next_actions say (fix: your own PR, merge: merge it, review: review per the crew-review skill, task: claim it, build to the acceptance criteria, crew check, crew commit, crew pr open), then run `crew task next --wait 20` again. If it answers "no free tasks" three times in a row, write me a short summary and stop. Do not plan or run kickoff; the captain does that.
+
+If Codex nevertheless ends its turn, tell it "continue the loop".
+
+A member **never plans**: no PRD, no `crew kickoff`. If the repo has no tasks yet, they wait for the captain.
+
+## Ready-to-paste messages for the human
+
+Copy into a Claude Code or Codex chat and fill in your values. You can write to the agent in your own language: it answers you in that language, but everything it writes to GitHub stays English.
+
+**Captain:**
+
+> Set up our agent-crew. If the `crew` command is missing, install it with `npm install -g @gredescerber/agent-crew`. Then read the `docs/AGENT-SETUP.md` of that package (or fetch it from github.com/GredesCerber/agent-crew) and do step 0 and the "Captain" section. Parameters: repo `<owner>/<name>`, public, stack `web-node`, members `<alice>,<bob>`. Create the project in the current folder. Do not type passwords: tell me what you need from me as a list.
+
+**Plan (captain, after everyone has joined):**
+
+> You are the captain. Put the brief below into `docs/brief.md`, analyze it with the crew-brief-analysis skill (including the "Stack" section), then use the crew-kickoff skill to write `docs/prd.md` with a crew-team block: take the members from `gh api repos/{owner}/{repo}/collaborators`; roles: `<me>` is backend and review, `<alice>` is frontend, `<bob>` is design and research. Show the plan (`crew kickoff`) and wait for my "approved". After that run `crew kickoff --apply` and work the loop: `/loop 10m Work the crew-workflow loop`. Do not type passwords.
+>
+> <brief>
+
+**Member:**
+
+> Connect me to the project. If the `crew` command is missing, install it with `npm install -g @gredescerber/agent-crew`. Then read the `docs/AGENT-SETUP.md` of that package (or fetch it from github.com/GredesCerber/agent-crew) and do step 0 and the "Member" section for the repo `<owner>/<name>`. Do not type passwords: tell me what you need from me as a list.

package/ecc/profile.json

@@ -0,0 +1,16 @@
+{
+  "enabled": true,
+  "version": "2.2.3",
+  "marketplace": "https://github.com/affaan-m/ECC.git",
+  "about": "Which ECC skills to use per role. ECC is an accelerator for Claude Code, not a protocol dependency: everything works without it.",
+  "roles": {
+    "common": ["search-first", "documentation-lookup", "verification-loop", "strategic-compact", "gateguard", "safety-guard"],
+    "planner": ["plan-prd", "product-lens", "council", "contract-first", "api-design", "market-research"],
+    "builder": ["orch-add-feature", "orch-fix-defect", "tdd-workflow", "build-fix", "api-connector-builder"],
+    "reviewer": ["orch-review", "astra-review", "silent-failure-hunter", "security-review"],
+    "qa": ["e2e-testing", "browser-qa", "ai-regression-testing"],
+    "demo": ["ui-demo", "frontend-slides", "frontend-design-direction", "investor-materials"],
+    "ops": ["opensource-sanitizer", "cost-report", "harness-audit"]
+  },
+  "avoid": ["multi-*", "email-ops", "chief-of-staff", "messages-ops", "social-publisher", "dev-team", "team-builder"]
+}

package/evals/README.md

@@ -0,0 +1,123 @@
+# evals: comparing agents on reference tasks
+
+`crew eval` runs agents (Claude Code, Codex, and their models) on the same tasks and computes pass@1, pass@k,
+consistency, time and cost. This is how you check that a change to `templates/product/AGENTS.md`, skills or roles
+did not make the agents worse.
+
+```bash
+crew eval --dry-run                                  # validate tasks and show the plan (no agents are started)
+crew eval --agents claude,codex --runs 3             # a real run: 3 tasks x 2 agents x 3 runs = 18 launches
+crew eval --agents claude:opus,claude:sonnet --task fix-off-by-one --keep
+```
+
+| Flag | What it does |
+|---|---|
+| `--agents` | hosts from `adapters/`, comma-separated; a model goes after a colon: `codex:gpt-5`. Default: `hosts` from `crew.config.json` |
+| `--tasks` | a folder with `*.json` or a single file (default `evals/tasks`, looked up from the cwd, then in the tool itself) |
+| `--task` | only these tasks, comma-separated |
+| `--runs` | runs per task and agent (default 3) |
+| `--dry-run` | only validate tasks and show the plan |
+| `--keep` | do not delete the working copies (the path is in `raw[].workdir` of the report) |
+
+## How one run goes
+
+1. **A fresh copy of the repo.** With `commit`: `git clone` of the repository that contains `repo`, then checkout of that commit;
+   without `commit`: a copy of the current folder state (without `.git` and `node_modules`) + `git init` and a commit.
+   The initial state is tagged `crew-eval-base`. The copy lives in the OS temp folder and is deleted after the run.
+2. **The agent.** `adapter.headless({ prompt, model, cwd })` -> the command runs without a shell with `cwd` set to the copy.
+   The timeout is the task's `timeout_s` (default 900 s).
+3. **Hidden tests.** Files from `judge_files` are copied into `.crew-eval/` of the copy **after** the agent has finished, so the agent never sees them.
+4. **Judges** run in order; the task passes if all of them pass.
+5. **Record:** pass/fail, time, cost (if the agent reports it, e.g. `total_cost_usd` in
+   `claude --output-format json`; otherwise `null`), exit code.
+
+If an agent's binary cannot start, its remaining runs are skipped with an `agent_failed` error and the report is a warning.
+
+## Task format (`evals/tasks/*.json`)
+
+```json
+{
+  "name": "fix-off-by-one",
+  "description": "Fix an off-by-one error in pagination",
+  "repo": "../fixtures/mini-app",
+  "files": ["src/pager.js"],
+  "prompt": "In src/pager.js the function paginate ... Verify with: npm test.",
+  "judge": [
+    { "type": "command", "command": "node --test .crew-eval/fix-off-by-one.judge.mjs" },
+    { "type": "command", "command": "node --test" },
+    { "type": "command", "command": "git diff --quiet crew-eval-base -- test/pager.test.js", "expect_exit": 1 },
+    { "type": "grep", "pattern": "TypeError", "files": ["src/cart.js"] }
+  ],
+  "judge_files": ["judges/fix-off-by-one.judge.mjs"],
+  "commit": "abc1234",
+  "timeout_s": 600
+}
+```
+
+| Field | Required | Meaning |
+|---|---|---|
+| `name` | yes | unique name: `a-z0-9._-` |
+| `description` | no | one line for humans |
+| `repo` | yes | folder of the fixture repo, **relative to the task file** |
+| `files` | no | files the task is about (a hint for humans; the default for `grep` without `files`) |
+| `prompt` | yes | what the agent receives, verbatim |
+| `judge` | yes | a non-empty list of deterministic judges |
+| `commit` | no | sha the repo is pinned to (reproducibility across days). Without it the current folder state is used, which suits fixtures inside this repo: their version is pinned by the tool's own commit |
+| `judge_files` | no | *crew extension*: hidden tests, relative to the task file; placed into `.crew-eval/` after the agent has finished |
+| `timeout_s` | no | *crew extension*: agent timeout in seconds |
+
+### Judges
+
+| `type` | Fields | Passes when |
+|---|---|---|
+| `command` (alias `pytest`) | `command`, `expect_exit` (0), `timeout_s` (300) | the command, run in the copy root (via a shell), returns `expect_exit` |
+| `grep` | `pattern` (JS RegExp), `files` (a path, a glob like `src/**/*.js`, or an array), `flags`, `absent` | the pattern is found in at least one file; with `absent: true`, in none |
+
+`llm` judges are not supported: the result must be deterministic. To check "did not touch the contract" or
+"added a test", compare with the initial state: `git diff --exit-code crew-eval-base -- contracts/`.
+
+## Compatibility with the agent-eval format
+
+The format intentionally follows [agent-eval](https://github.com/joaquinhuigomez/agent-eval), so the tasks can
+also be run with it (after reviewing its source). The differences are JSON instead of YAML (no dependencies) and two extension fields.
+
+| agent-eval (YAML) | crew eval (JSON) |
+|---|---|
+| `name`, `description`, `prompt`, `files`, `commit` | the same |
+| `repo: ./my-project` (relative to the launch cwd) | `repo` is **relative to the task file** |
+| `judge: - type: pytest / command` + `command` | the same (`pytest` = `command`) |
+| `judge: - type: grep` + `pattern`, `files` | the same; plus `absent`, `flags` |
+| `judge: - type: llm` | not supported |
+| none | `judge_files`, `timeout_s`, `expect_exit` (crew extensions; agent-eval ignores them) |
+| `agent-eval run --agent claude-code --runs 3` | `crew eval --agents claude --runs 3` |
+| a separate git worktree per run | a temporary copy/clone per run (a worktree of a fixture inside the tool repo is not possible) |
+
+YAML -> JSON by hand: `prompt: |` becomes a string with `\n`, lists become arrays. For agent-eval, put the hidden tests from
+`judge_files` into the repo at the pinned commit and replace `.crew-eval/...` in the commands with their path.
+
+## Report metrics
+
+`artifacts.results` is per agent and task, `artifacts.agents` is per agent averaged over tasks,
+and the full report with every run is `.crew/evals/eval-<time>.json`.
+
+| Metric | How it is computed |
+|---|---|
+| `pass_at_1` | share of successful runs, c / n |
+| `pass_at_k` | probability of at least one success in k = n runs (the unbiased estimator of Chen et al.: 1 - C(n-c, k) / C(n, k)) |
+| `consistency` | share of runs that match the majority: max(c, n-c) / n. 1 means the agent behaves the same every time (consistently solves or consistently fails). In agent-eval "consistency" is c/n; here that is `pass_at_1` |
+| `avg_time_ms` | average agent working time (judges excluded) |
+| `avg_cost_usd` / `total_cost_usd` | if the agent reports cost; otherwise `null` |
+
+## Fixture and tasks
+
+`fixtures/mini-app/` is a small dependency-free Node module with `node:test` tests: a cart, pagination,
+HTTP handlers and the contract `contracts/health.json`.
+
+| Task | What it checks |
+|---|---|
+| `fix-off-by-one` | find and fix an off-by-one error, add a regression test |
+| `add-input-validation` | input validation with clear errors, without breaking the old behavior |
+| `handler-per-contract` | implement a route strictly per the contract, without changing the contract or hardcoding the version |
+
+A new task: a file `tasks/<name>.json` + (if needed) a hidden test `tasks/judges/<name>.judge.mjs`. The check that a
+task is not already solved and that the judges really catch it lives in `test/eval.test.js` (`npm test`).

package/evals/fixtures/mini-app/README.md

@@ -0,0 +1,10 @@
+# mini-app
+
+A small module for `crew eval` tasks. No dependencies, Node.js 20+.
+
+- `src/cart.js` — cart: `createCart`, `addItem`, `total`.
+- `src/pager.js` — pagination `paginate(items, page, size)`, pages start at 1.
+- `src/handlers.js` — HTTP route handlers `handle({ method, path })`.
+- `contracts/` — API contracts. Do not change them, only implement them.
+
+Tests: `npm test` (or `node --test`).

package/evals/fixtures/mini-app/contracts/health.json

@@ -0,0 +1,11 @@
+{
+  "route": "GET /health",
+  "description": "Liveness check of the service for the load balancer",
+  "response": {
+    "status": 200,
+    "body": {
+      "status": "ok",
+      "version": "string: the version field from package.json"
+    }
+  }
+}

package/evals/fixtures/mini-app/package.json

@@ -0,0 +1,10 @@
+{
+  "name": "mini-app",
+  "version": "1.2.0",
+  "private": true,
+  "type": "module",
+  "description": "Reference mini project for crew eval: cart, pagination, HTTP handlers per contract",
+  "scripts": {
+    "test": "node --test"
+  }
+}

package/evals/fixtures/mini-app/src/cart.js

@@ -0,0 +1,14 @@
+// Shopping cart. Item: { sku: string, qty: integer > 0, price: number >= 0 }.
+
+export function createCart() {
+  return { items: [] };
+}
+
+export function addItem(cart, item) {
+  cart.items.push({ sku: item.sku, qty: item.qty, price: item.price });
+  return cart;
+}
+
+export function total(cart) {
+  return cart.items.reduce((sum, i) => sum + i.qty * i.price, 0);
+}

package/evals/fixtures/mini-app/src/handlers.js

@@ -0,0 +1,16 @@
+// HTTP route handlers. Key: "METHOD /path", value: a function with no arguments -> { status, body }.
+// Route contracts live in contracts/.
+
+const items = [
+  { id: 1, name: 'tea' },
+  { id: 2, name: 'coffee' },
+];
+
+export const routes = {
+  'GET /items': () => ({ status: 200, body: items }),
+};
+
+export function handle({ method, path }) {
+  const h = routes[`${method} ${path}`];
+  return h ? h() : { status: 404, body: { error: 'not_found' } };
+}

package/evals/fixtures/mini-app/src/pager.js

@@ -0,0 +1,7 @@
+// Pagination. Pages are numbered from 1: page=1 is the first `size` items.
+
+export function paginate(items, page = 1, size = 10) {
+  const pages = Math.max(1, Math.ceil(items.length / size));
+  const start = page * size;
+  return { items: items.slice(start, start + size), page, pages };
+}

package/evals/fixtures/mini-app/test/cart.test.js

@@ -0,0 +1,10 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { createCart, addItem, total } from '../src/cart.js';
+
+test('cart total', () => {
+  const cart = createCart();
+  addItem(cart, { sku: 'tea', qty: 2, price: 150 });
+  addItem(cart, { sku: 'cup', qty: 1, price: 300 });
+  assert.equal(total(cart), 600);
+});

package/evals/fixtures/mini-app/test/handlers.test.js

@@ -0,0 +1,13 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { handle } from '../src/handlers.js';
+
+test('GET /items returns a list', () => {
+  const r = handle({ method: 'GET', path: '/items' });
+  assert.equal(r.status, 200);
+  assert.ok(Array.isArray(r.body));
+});
+
+test('unknown route -> 404', () => {
+  assert.equal(handle({ method: 'GET', path: '/nope' }).status, 404);
+});

package/evals/fixtures/mini-app/test/pager.test.js

@@ -0,0 +1,8 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { paginate } from '../src/pager.js';
+
+test('number of pages', () => {
+  assert.equal(paginate([1, 2, 3, 4, 5], 1, 2).pages, 3);
+  assert.equal(paginate([], 1, 10).pages, 1);
+});

package/evals/tasks/add-input-validation.json

@@ -0,0 +1,15 @@
+{
+  "name": "add-input-validation",
+  "description": "Add input validation to addItem",
+  "repo": "../fixtures/mini-app",
+  "files": ["src/cart.js", "test/cart.test.js"],
+  "prompt": "In src/cart.js the function addItem(cart, item) accepts anything. Add validation: sku is a non-empty string, qty is an integer greater than 0, price is a finite number not less than 0. On a violation throw a TypeError and name the field (sku, qty or price) in the message. Valid items must still be added as before. Add tests for the errors in test/cart.test.js. Verify with: npm test.",
+  "judge": [
+    { "type": "command", "command": "node --test .crew-eval/add-input-validation.judge.mjs" },
+    { "type": "command", "command": "node --test" },
+    { "type": "grep", "pattern": "TypeError", "files": ["src/cart.js"] },
+    { "type": "grep", "pattern": "throws|rejects", "files": ["test/cart.test.js"] }
+  ],
+  "judge_files": ["judges/add-input-validation.judge.mjs"],
+  "timeout_s": 600
+}

package/evals/tasks/fix-off-by-one.json

@@ -0,0 +1,14 @@
+{
+  "name": "fix-off-by-one",
+  "description": "Fix an off-by-one error in pagination",
+  "repo": "../fixtures/mini-app",
+  "files": ["src/pager.js"],
+  "prompt": "In src/pager.js the function paginate(items, page, size) returns the wrong page: pages are numbered from 1, so page=1 must return the first `size` items, but it currently returns the second chunk. Fix the bug without changing the signature or the response shape { items, page, pages }. Add a test for the first and the last (partial) page in test/pager.test.js. Verify with: npm test.",
+  "judge": [
+    { "type": "command", "command": "node --test .crew-eval/fix-off-by-one.judge.mjs" },
+    { "type": "command", "command": "node --test" },
+    { "type": "command", "command": "git diff --quiet crew-eval-base -- test/pager.test.js", "expect_exit": 1 }
+  ],
+  "judge_files": ["judges/fix-off-by-one.judge.mjs"],
+  "timeout_s": 600
+}

package/evals/tasks/handler-per-contract.json

@@ -0,0 +1,15 @@
+{
+  "name": "handler-per-contract",
+  "description": "Implement the GET /health handler strictly per the contract",
+  "repo": "../fixtures/mini-app",
+  "files": ["src/handlers.js", "contracts/health.json"],
+  "prompt": "Implement the route from the contract contracts/health.json in src/handlers.js (add it to routes). Do not change the contract. Read the version from package.json, do not hardcode it. Add a test in test/handlers.test.js. Verify with: npm test.",
+  "judge": [
+    { "type": "command", "command": "node --test .crew-eval/handler-per-contract.judge.mjs" },
+    { "type": "command", "command": "node --test" },
+    { "type": "command", "command": "git diff --exit-code crew-eval-base -- contracts/" },
+    { "type": "grep", "pattern": "['\"]1\\.2\\.0['\"]", "files": ["src/handlers.js"], "absent": true }
+  ],
+  "judge_files": ["judges/handler-per-contract.judge.mjs"],
+  "timeout_s": 600
+}

package/evals/tasks/judges/add-input-validation.judge.mjs

@@ -0,0 +1,28 @@
+// Hidden test of the add-input-validation eval task. Copied into .crew-eval/ only after the agent has finished.
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { createCart, addItem, total } from '../src/cart.js';
+
+const good = { sku: 'tea', qty: 2, price: 150 };
+
+test('a valid item is added', () => {
+  const cart = createCart();
+  addItem(cart, good);
+  addItem(cart, { sku: 'gift', qty: 1, price: 0 });
+  assert.equal(total(cart), 300);
+  assert.equal(cart.items.length, 2);
+});
+
+for (const [field, values] of Object.entries({
+  sku: ['', 42, null, undefined],
+  qty: [0, -1, 1.5, '2', NaN],
+  price: [-1, Infinity, NaN, '10'],
+})) {
+  for (const v of values) {
+    test(`${field}=${String(v)} -> TypeError naming the field`, () => {
+      const cart = createCart();
+      assert.throws(() => addItem(cart, { ...good, [field]: v }), (e) => e instanceof TypeError && e.message.includes(field));
+      assert.equal(cart.items.length, 0, 'an invalid item must not end up in the cart');
+    });
+  }
+}