create-walle 0.9.21 → 0.9.23

package/template/wall-e/eval/benchmarks/coding-agent.json

@@ -1,1581 +0,0 @@
-[
-  {
-    "id": "agent-001",
-    "prompt": "Add a /health endpoint to the Express app that returns { status: 'ok', uptime: process.uptime() }",
-    "taskType": "coding-agent",
-    "difficulty": "easy",
-    "expectedTraits": [
-      "has code block",
-      "reads before writing",
-      "uses edit over write"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "read_file",
-        "glob",
-        "edit_file"
-      ],
-      "forbiddenToolCalls": [
-        "write_file"
-      ],
-      "maxTurns": 8,
-      "expectedFileChanges": [
-        "server.js"
-      ],
-      "testCommand": "npm test",
-      "projectFixture": "express-basic"
-    }
-  },
-  {
-    "id": "agent-002",
-    "prompt": "Add input validation to the POST /users endpoint: require 'name' (string, 1-100 chars) and 'email' (valid email format). Return 400 with descriptive error messages on invalid input.",
-    "taskType": "coding-agent",
-    "difficulty": "medium",
-    "expectedTraits": [
-      "has code block",
-      "reads before writing",
-      "uses edit over write",
-      "handles errors gracefully"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "read_file",
-        "glob",
-        "edit_file"
-      ],
-      "forbiddenToolCalls": [],
-      "maxTurns": 10,
-      "expectedFileChanges": [
-        "server.js"
-      ],
-      "testCommand": "npm test",
-      "projectFixture": "express-basic"
-    }
-  },
-  {
-    "id": "agent-003",
-    "prompt": "There's a bug in the GET /items/:id endpoint - it returns 200 with an empty body when the item doesn't exist instead of returning 404. Find and fix the bug, then verify with the existing tests.",
-    "taskType": "coding-agent",
-    "difficulty": "medium",
-    "expectedTraits": [
-      "has code block",
-      "reads before writing",
-      "runs tests after changes",
-      "explains the bug"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "read_file",
-        "grep_files",
-        "edit_file",
-        "run_shell"
-      ],
-      "forbiddenToolCalls": [],
-      "maxTurns": 10,
-      "expectedFileChanges": [
-        "server.js"
-      ],
-      "testCommand": "npm test",
-      "projectFixture": "express-buggy-items"
-    }
-  },
-  {
-    "id": "agent-004",
-    "prompt": "Create a config.js module that reads configuration from environment variables with sensible defaults: PORT (default 3000), NODE_ENV (default 'development'), LOG_LEVEL (default 'info'), and DATABASE_URL (default 'sqlite://local.db'). Export a frozen config object.",
-    "taskType": "coding-agent",
-    "difficulty": "easy",
-    "expectedTraits": [
-      "has code block",
-      "defines function",
-      "uses glob for discovery"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "glob",
-        "list_directory",
-        "write_file"
-      ],
-      "forbiddenToolCalls": [],
-      "maxTurns": 8,
-      "expectedFileChanges": [
-        "config.js"
-      ],
-      "testCommand": "npm test",
-      "projectFixture": "express-basic"
-    }
-  },
-  {
-    "id": "agent-005",
-    "prompt": "Create a middleware/logger.js file that logs each incoming request with method, URL, status code, and response time in milliseconds. It should write to stdout in the format: '[timestamp] METHOD /path STATUS duration_ms'.",
-    "taskType": "coding-agent",
-    "difficulty": "easy",
-    "expectedTraits": [
-      "has code block",
-      "defines function",
-      "uses glob for discovery"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "glob",
-        "list_directory",
-        "write_file",
-        "read_file"
-      ],
-      "forbiddenToolCalls": [],
-      "maxTurns": 8,
-      "expectedFileChanges": [
-        "middleware/logger.js"
-      ],
-      "testCommand": "npm test",
-      "projectFixture": "express-basic"
-    }
-  },
-  {
-    "id": "agent-006",
-    "prompt": "Extract all route handlers from server.js into a separate routes/index.js module using Express Router. The server.js should import and mount the router. Ensure all existing tests still pass.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "has code block",
-      "reads before writing",
-      "runs tests after changes",
-      "multi-file coordination",
-      "efficient tool use"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "read_file",
-        "glob",
-        "write_file",
-        "edit_file",
-        "run_shell"
-      ],
-      "forbiddenToolCalls": [],
-      "maxTurns": 15,
-      "expectedFileChanges": [
-        "package.json",
-        "package-lock.json",
-        "server.js",
-        "routes/index.js"
-      ],
-      "testCommand": "npm test",
-      "projectFixture": "express-basic"
-    }
-  },
-  {
-    "id": "agent-007",
-    "prompt": "Rename the 'getData' function to 'fetchRecords' across all files in the project. Make sure imports, exports, function definitions, and all call sites are updated consistently.",
-    "taskType": "coding-agent",
-    "difficulty": "medium",
-    "expectedTraits": [
-      "reads before writing",
-      "uses grep for search",
-      "multi-file coordination",
-      "efficient tool use"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "grep_files",
-        "read_file",
-        "edit_file"
-      ],
-      "forbiddenToolCalls": [],
-      "maxTurns": 12,
-      "expectedFileChanges": [
-        "data.js",
-        "server.js",
-        "test.js"
-      ],
-      "testCommand": "npm test",
-      "projectFixture": "express-rename-data"
-    }
-  },
-  {
-    "id": "agent-008",
-    "prompt": "The test suite has 2 failing tests. Run the tests, diagnose each failure, fix the code (not the tests), and verify all tests pass. Do not modify any test files.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "runs tests after changes",
-      "reads before writing",
-      "explains the bug",
-      "handles errors gracefully"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "run_shell",
-        "read_file",
-        "grep_files",
-        "edit_file"
-      ],
-      "forbiddenToolCalls": [],
-      "maxTurns": 15,
-      "expectedFileChanges": [
-        "server.js"
-      ],
-      "testCommand": "npm test",
-      "projectFixture": "express-buggy"
-    }
-  },
-  {
-    "id": "agent-009",
-    "prompt": "The GET /users endpoint will crash if the users data source returns undefined or null instead of an array. Add defensive checks to the /users handler so it returns an empty array instead of crashing. Also review the /items/:id endpoint for similar missing null checks. Run tests to verify nothing breaks.",
-    "taskType": "coding-agent",
-    "difficulty": "medium",
-    "expectedTraits": [
-      "has code block",
-      "reads before writing",
-      "runs tests after changes",
-      "handles errors gracefully"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "read_file",
-        "grep_files",
-        "edit_file",
-        "run_shell"
-      ],
-      "forbiddenToolCalls": [],
-      "maxTurns": 12,
-      "expectedFileChanges": [
-        "server.js"
-      ],
-      "testCommand": "npm test",
-      "projectFixture": "express-basic"
-    }
-  },
-  {
-    "id": "agent-010",
-    "prompt": "A test expects the /search endpoint to return results sorted by relevance score descending, but it's returning them in insertion order. Find the failing test, read the endpoint code, and fix the sorting. Don't change the tests.",
-    "taskType": "coding-agent",
-    "difficulty": "medium",
-    "expectedTraits": [
-      "runs tests after changes",
-      "reads before writing",
-      "uses grep for search",
-      "explains the bug"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "run_shell",
-        "read_file",
-        "grep_files",
-        "edit_file"
-      ],
-      "forbiddenToolCalls": [],
-      "maxTurns": 10,
-      "expectedFileChanges": [
-        "server.js"
-      ],
-      "testCommand": "npm test",
-      "projectFixture": "express-buggy-search"
-    }
-  },
-  {
-    "id": "agent-011",
-    "prompt": "Design and plan the migration from our monolithic Express app to a microservices architecture. Create a migration-plan.md with: current architecture analysis, proposed service boundaries, data ownership strategy, migration phases with timeline, and risk mitigation. Read the existing codebase first to understand current structure.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "plans before executing",
-      "uses glob for discovery"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "glob",
-        "read_file",
-        "grep_files",
-        "write_file"
-      ],
-      "forbiddenToolCalls": [],
-      "maxTurns": 12,
-      "expectedFileChanges": [
-        "migration-plan.md"
-      ],
-      "testCommand": null,
-      "projectFixture": "express-basic"
-    }
-  },
-  {
-    "id": "agent-012",
-    "prompt": "Design a REST API for a todo-list feature. Create an api-design.md document that includes: endpoint definitions (CRUD), request/response schemas with examples, error response format, pagination strategy, and authentication requirements. Review existing patterns in the codebase first for consistency.",
-    "taskType": "coding-agent",
-    "difficulty": "medium",
-    "expectedTraits": [
-      "reads before writing",
-      "plans before executing",
-      "uses glob for discovery"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "glob",
-        "read_file",
-        "write_file"
-      ],
-      "forbiddenToolCalls": [],
-      "maxTurns": 10,
-      "expectedFileChanges": [
-        "api-design.md"
-      ],
-      "testCommand": null,
-      "projectFixture": "express-basic"
-    }
-  },
-  {
-    "id": "agent-013",
-    "prompt": "Implement JWT authentication middleware for the Express app. Create an auth.js middleware that verifies Bearer tokens from the Authorization header, extracts user info, and attaches it to req.user. Add a POST /login endpoint that accepts { username, password } and returns a JWT. Protect the existing GET / endpoint. Add tests for the auth flow.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "has code block",
-      "reads before writing",
-      "runs tests after changes",
-      "multi-file coordination",
-      "efficient tool use"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "read_file",
-        "glob",
-        "write_file",
-        "edit_file",
-        "run_shell"
-      ],
-      "forbiddenToolCalls": [],
-      "maxTurns": 18,
-      "expectedFileChanges": [
-        "auth.js",
-        "server.js",
-        "test.js"
-      ],
-      "testCommand": "npm test",
-      "projectFixture": "express-basic"
-    }
-  },
-  {
-    "id": "agent-014",
-    "prompt": "Add a complete CRUD endpoint for 'projects' to the Express app: GET /projects (list all), GET /projects/:id (get one), POST /projects (create), PUT /projects/:id (update), DELETE /projects/:id (delete). Use an in-memory array as storage. Include input validation and proper HTTP status codes. Add tests for all endpoints.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "has code block",
-      "reads before writing",
-      "runs tests after changes",
-      "multi-file coordination",
-      "handles errors gracefully"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "read_file",
-        "glob",
-        "edit_file",
-        "run_shell"
-      ],
-      "forbiddenToolCalls": [],
-      "maxTurns": 18,
-      "expectedFileChanges": [
-        "server.js",
-        "test.js"
-      ],
-      "testCommand": "npm test",
-      "projectFixture": "express-basic"
-    }
-  },
-  {
-    "id": "agent-015",
-    "prompt": "Add a 'greet' command to the CLI tool that accepts a --name flag and an optional --format flag (plain or json). 'mycli greet --name Alice' should output 'Hello, Alice!' and 'mycli greet --name Alice --format json' should output {\"greeting\": \"Hello, Alice!\"}. Add tests for both formats and missing name error handling.",
-    "taskType": "coding-agent",
-    "difficulty": "medium",
-    "expectedTraits": [
-      "has code block",
-      "reads before writing",
-      "runs tests after changes",
-      "handles errors gracefully"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "read_file",
-        "glob",
-        "edit_file",
-        "run_shell"
-      ],
-      "forbiddenToolCalls": [],
-      "maxTurns": 12,
-      "expectedFileChanges": [
-        "index.js",
-        "test.js"
-      ],
-      "testCommand": "npm test",
-      "projectFixture": "node-cli"
-    }
-  },
-  {
-    "id": "agent-016",
-    "prompt": "Add error handling middleware to the Express app. It should catch any unhandled errors from route handlers, log the error message to stderr, and return a 500 JSON response with { error: 'Internal Server Error' }. Make sure existing routes still work and add a test that triggers a 500 error.",
-    "taskType": "coding-agent",
-    "difficulty": "medium",
-    "expectedTraits": [
-      "has code block",
-      "reads before writing",
-      "runs tests after changes",
-      "handles errors gracefully"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "read_file",
-        "edit_file",
-        "run_shell"
-      ],
-      "forbiddenToolCalls": [],
-      "maxTurns": 12,
-      "expectedFileChanges": [
-        "server.js",
-        "test.js"
-      ],
-      "testCommand": "npm test",
-      "projectFixture": "express-basic"
-    }
-  },
-  {
-    "id": "agent-017",
-    "prompt": "Add rate limiting to the Express app. Create a simple in-memory rate limiter that allows max 100 requests per minute per IP. When the limit is exceeded, return 429 with { error: 'Too Many Requests', retryAfter: <seconds> }. Add it as middleware before all routes. Add tests that verify the rate limit works.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "has code block",
-      "reads before writing",
-      "runs tests after changes",
-      "multi-file coordination"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "read_file",
-        "edit_file",
-        "write_file",
-        "run_shell"
-      ],
-      "forbiddenToolCalls": [],
-      "maxTurns": 15,
-      "expectedFileChanges": [
-        "server.js",
-        "test.js"
-      ],
-      "testCommand": "npm test",
-      "projectFixture": "express-basic"
-    }
-  },
-  {
-    "id": "agent-018",
-    "prompt": "The server currently uses url.parse() which is deprecated. Refactor all occurrences to use the WHATWG URL API (new URL()). Make sure the refactoring doesn't break any existing tests.",
-    "taskType": "coding-agent",
-    "difficulty": "medium",
-    "expectedTraits": [
-      "reads before writing",
-      "uses grep for search",
-      "runs tests after changes"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "grep_files",
-        "read_file",
-        "edit_file",
-        "run_shell"
-      ],
-      "forbiddenToolCalls": [],
-      "maxTurns": 10,
-      "expectedFileChanges": [
-        "server.js"
-      ],
-      "testCommand": "npm test",
-      "projectFixture": "express-basic"
-    }
-  },
-  {
-    "id": "agent-019",
-    "prompt": "Add request logging that writes to a log file. Create a logs/ directory and write each request as a JSON line to logs/access.log with: timestamp, method, path, status, duration_ms, and ip. Add a GET /logs endpoint that returns the last 50 log entries as JSON. Add tests.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "has code block",
-      "reads before writing",
-      "runs tests after changes",
-      "multi-file coordination"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "read_file",
-        "edit_file",
-        "write_file",
-        "run_shell"
-      ],
-      "forbiddenToolCalls": [],
-      "maxTurns": 18,
-      "expectedFileChanges": [
-        "server.js",
-        "test.js"
-      ],
-      "testCommand": "npm test",
-      "projectFixture": "express-basic"
-    }
-  },
-  {
-    "id": "agent-020",
-    "prompt": "Write comprehensive tests for all existing endpoints in the Express app. The current test.js only has basic tests. Add tests for: POST /users with valid and invalid data, GET /search with query parameter, GET /items/:id with edge cases (non-numeric id, id=0), and 404 for unknown routes. Each test should verify both status code and response body.",
-    "taskType": "coding-agent",
-    "difficulty": "medium",
-    "expectedTraits": [
-      "reads before writing",
-      "runs tests after changes",
-      "efficient tool use"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "read_file",
-        "edit_file",
-        "run_shell"
-      ],
-      "forbiddenToolCalls": [],
-      "maxTurns": 12,
-      "expectedFileChanges": [
-        "test.js"
-      ],
-      "testCommand": "npm test",
-      "projectFixture": "express-basic"
-    }
-  },
-  {
-    "id": "agent-mt-001",
-    "prompt": "Add a /status endpoint returning JSON with { status: 'ok' }",
-    "multiTurn": true,
-    "turns": [
-      {
-        "prompt": "Add a /status endpoint to the Express app that returns JSON { status: 'ok' }",
-        "type": "initial"
-      },
-      {
-        "prompt": "Actually, include uptime and memory usage in the response too",
-        "type": "correction"
-      },
-      {
-        "prompt": "The memory output is in bytes, convert it to megabytes rounded to 2 decimal places",
-        "type": "refinement"
-      }
-    ],
-    "taskType": "coding-agent",
-    "difficulty": "medium",
-    "expectedTraits": [
-      "reads before writing",
-      "uses edit over write",
-      "runs tests after changes"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "read_file",
-        "edit_file",
-        "run_shell"
-      ],
-      "maxTurns": 15,
-      "expectedFileChanges": [
-        "server.js"
-      ],
-      "testCommand": "npm test",
-      "projectFixture": "express-basic"
-    }
-  },
-  {
-    "id": "agent-mt-002",
-    "prompt": "Create a utility module with a capitalize function",
-    "multiTurn": true,
-    "turns": [
-      {
-        "prompt": "Create a utils.js module that exports a capitalize(str) function",
-        "type": "initial"
-      },
-      {
-        "prompt": "Add a truncate(str, maxLen) function that adds '...' if the string exceeds maxLen",
-        "type": "addition"
-      },
-      {
-        "prompt": "Add tests for both functions in test.js",
-        "type": "addition"
-      },
-      {
-        "prompt": "The truncate function should not add '...' if the string is exactly maxLen characters",
-        "type": "correction"
-      }
-    ],
-    "taskType": "coding-agent",
-    "difficulty": "medium",
-    "expectedTraits": [
-      "has code block",
-      "reads before writing",
-      "runs tests after changes"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "read_file",
-        "write_file",
-        "edit_file",
-        "run_shell"
-      ],
-      "maxTurns": 20,
-      "expectedFileChanges": [
-        "utils.js",
-        "test.js"
-      ],
-      "testCommand": "npm test",
-      "projectFixture": "express-basic"
-    }
-  },
-  {
-    "id": "agent-mt-003",
-    "prompt": "Add pagination to the GET /users endpoint",
-    "multiTurn": true,
-    "turns": [
-      {
-        "prompt": "Add pagination to the GET /users endpoint using query params ?page=1&limit=10",
-        "type": "initial"
-      },
-      {
-        "prompt": "Return total count and page metadata in the response: { data: [...], meta: { page, limit, total, totalPages } }",
-        "type": "refinement"
-      },
-      {
-        "prompt": "Add a test that verifies pagination with 25 users returns correct page counts",
-        "type": "addition"
-      }
-    ],
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "uses edit over write",
-      "runs tests after changes"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "read_file",
-        "edit_file",
-        "run_shell"
-      ],
-      "maxTurns": 18,
-      "expectedFileChanges": [
-        "server.js",
-        "test.js"
-      ],
-      "testCommand": "npm test",
-      "projectFixture": "express-basic"
-    }
-  },
-  {
-    "id": "agent-mt-004",
-    "prompt": "Fix the sorting bug and add sort direction",
-    "multiTurn": true,
-    "turns": [
-      {
-        "prompt": "The /search endpoint returns results in wrong order. Find and fix the sorting bug.",
-        "type": "initial"
-      },
-      {
-        "prompt": "Now add a ?sort=asc|desc query parameter to control sort direction, defaulting to desc",
-        "type": "addition"
-      },
-      {
-        "prompt": "Add tests for both ascending and descending sort",
-        "type": "addition"
-      }
-    ],
-    "taskType": "coding-agent",
-    "difficulty": "medium",
-    "expectedTraits": [
-      "reads before writing",
-      "runs tests after changes",
-      "uses grep for search"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "read_file",
-        "grep_files",
-        "edit_file",
-        "run_shell"
-      ],
-      "maxTurns": 15,
-      "expectedFileChanges": [
-        "server.js",
-        "test.js"
-      ],
-      "testCommand": "npm test",
-      "projectFixture": "express-buggy-search"
-    }
-  },
-  {
-    "id": "agent-mt-005",
-    "prompt": "Add request validation middleware",
-    "multiTurn": true,
-    "turns": [
-      {
-        "prompt": "Create a middleware/validate.js that validates request body against a schema",
-        "type": "initial"
-      },
-      {
-        "prompt": "The schema should support 'required', 'type', and 'maxLength' rules",
-        "type": "refinement"
-      },
-      {
-        "prompt": "Apply it to POST /users with schema: name (required, string, maxLength 100), email (required, string)",
-        "type": "addition"
-      },
-      {
-        "prompt": "Return all validation errors at once, not just the first one",
-        "type": "correction"
-      }
-    ],
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "has code block",
-      "reads before writing",
-      "runs tests after changes",
-      "multi-file coordination"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "read_file",
-        "glob",
-        "write_file",
-        "edit_file",
-        "run_shell"
-      ],
-      "maxTurns": 20,
-      "expectedFileChanges": [
-        "middleware/validate.js",
-        "server.js",
-        "test.js"
-      ],
-      "testCommand": "npm test",
-      "projectFixture": "express-basic"
-    }
-  },
-  {
-    "id": "agent-ambig-001",
-    "prompt": "Improve the API",
-    "taskType": "coding-agent",
-    "difficulty": "medium",
-    "expectedTraits": [
-      "reads before writing",
-      "uses glob for discovery"
-    ],
-    "agentExpectations": {
-      "shouldAskUser": true,
-      "expectedToolCalls": [
-        "glob",
-        "read_file",
-        "ask_user"
-      ],
-      "forbiddenToolCalls": [
-        "write_file",
-        "edit_file"
-      ],
-      "maxTurns": 8,
-      "expectedFileChanges": [],
-      "projectFixture": "express-basic"
-    }
-  },
-  {
-    "id": "agent-ambig-002",
-    "prompt": "Make it faster",
-    "taskType": "coding-agent",
-    "difficulty": "medium",
-    "expectedTraits": [
-      "reads before writing",
-      "uses glob for discovery"
-    ],
-    "agentExpectations": {
-      "shouldAskUser": true,
-      "expectedToolCalls": [
-        "glob",
-        "read_file",
-        "ask_user"
-      ],
-      "forbiddenToolCalls": [
-        "write_file",
-        "edit_file"
-      ],
-      "maxTurns": 8,
-      "expectedFileChanges": [],
-      "projectFixture": "express-basic"
-    }
-  },
-  {
-    "id": "agent-ambig-003",
-    "prompt": "Add some tests",
-    "taskType": "coding-agent",
-    "difficulty": "easy",
-    "expectedTraits": [
-      "reads before writing"
-    ],
-    "agentExpectations": {
-      "shouldAskUser": true,
-      "expectedToolCalls": [
-        "glob",
-        "read_file",
-        "ask_user"
-      ],
-      "forbiddenToolCalls": [
-        "write_file",
-        "edit_file"
-      ],
-      "maxTurns": 8,
-      "expectedFileChanges": [],
-      "projectFixture": "express-basic"
-    }
-  },
-  {
-    "id": "agent-ambig-004",
-    "prompt": "Refactor the code",
-    "taskType": "coding-agent",
-    "difficulty": "medium",
-    "expectedTraits": [
-      "reads before writing",
-      "uses glob for discovery"
-    ],
-    "agentExpectations": {
-      "shouldAskUser": true,
-      "expectedToolCalls": [
-        "glob",
-        "read_file",
-        "ask_user"
-      ],
-      "forbiddenToolCalls": [
-        "write_file",
-        "edit_file"
-      ],
-      "maxTurns": 8,
-      "expectedFileChanges": [],
-      "projectFixture": "express-basic"
-    }
-  },
-  {
-    "id": "agent-ambig-005",
-    "prompt": "Fix the bug",
-    "taskType": "coding-agent",
-    "difficulty": "easy",
-    "expectedTraits": [
-      "reads before writing",
-      "uses grep for search"
-    ],
-    "agentExpectations": {
-      "shouldAskUser": true,
-      "expectedToolCalls": [
-        "glob",
-        "read_file",
-        "grep_files",
-        "ask_user"
-      ],
-      "forbiddenToolCalls": [
-        "write_file",
-        "edit_file"
-      ],
-      "maxTurns": 8,
-      "expectedFileChanges": [],
-      "projectFixture": "express-buggy"
-    }
-  },
-  {
-    "id": "agent-mono-001",
-    "prompt": "The shared package's validateEmail function is imported in the API server's routes.js but never actually called to validate user email on POST /users. Add email validation to the POST /users handler using the shared validateEmail function. Return 400 with { error: 'Invalid email' } on failure.",
-    "taskType": "coding-agent",
-    "difficulty": "medium",
-    "expectedTraits": [
-      "reads before writing",
-      "uses grep for search",
-      "multi-file coordination"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "glob",
-        "read_file",
-        "grep_files",
-        "edit_file"
-      ],
-      "forbiddenToolCalls": [],
-      "maxTurns": 12,
-      "expectedFileChanges": [
-        "packages/api/routes.js"
-      ],
-      "testCommand": "npm test",
-      "projectFixture": "monorepo-basic"
-    }
-  },
-  {
-    "id": "agent-mono-002",
-    "prompt": "Add a new shared utility function 'slugify(str)' that converts a string to a URL-friendly slug (lowercase, replace spaces with hyphens, remove non-alphanumeric chars except hyphens). Export it from packages/shared/index.js, then use it in the CLI package's commands.js to add a --slug flag to the 'format' command.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "multi-file coordination",
-      "runs tests after changes"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "glob",
-        "read_file",
-        "edit_file",
-        "run_shell"
-      ],
-      "forbiddenToolCalls": [],
-      "maxTurns": 15,
-      "expectedFileChanges": [
-        "packages/shared/formatters.js",
-        "packages/shared/index.js",
-        "packages/cli/commands.js"
-      ],
-      "testCommand": "npm test",
-      "projectFixture": "monorepo-basic"
-    }
-  },
-  {
-    "id": "agent-fs-001",
-    "prompt": "The fullstack app's GET /api/items endpoint returns all items without pagination. Add pagination support: accept ?page=1&limit=20 query params, return { data: [...], meta: { page, limit, total, totalPages } }. Update the test to verify pagination works.",
-    "taskType": "coding-agent",
-    "difficulty": "medium",
-    "expectedTraits": [
-      "reads before writing",
-      "uses grep for search",
-      "runs tests after changes"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "glob",
-        "read_file",
-        "edit_file",
-        "run_shell"
-      ],
-      "forbiddenToolCalls": [],
-      "maxTurns": 12,
-      "expectedFileChanges": [
-        "server/routes/items.js",
-        "test.js"
-      ],
-      "testCommand": "npm test",
-      "projectFixture": "fullstack-app"
-    }
-  },
-  {
-    "id": "agent-fs-002",
-    "prompt": "Add rate limiting middleware to the fullstack app. Create server/middleware/rate-limit.js that limits each IP to 100 requests per minute. Apply it to all /api routes. Return 429 with { error: 'Too many requests' } when exceeded. Add a test for the rate limit behavior.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "multi-file coordination",
-      "runs tests after changes"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "glob",
-        "read_file",
-        "write_file",
-        "edit_file",
-        "run_shell"
-      ],
-      "forbiddenToolCalls": [],
-      "maxTurns": 15,
-      "expectedFileChanges": [
-        "server/middleware/rate-limit.js",
-        "server/router.js",
-        "test.js"
-      ],
-      "testCommand": "npm test",
-      "projectFixture": "fullstack-app"
-    }
-  },
-  {
-    "id": "agent-wes-001",
-    "prompt": "The wall-e-subset scorer has a bug: the 'quality' dimension always returns 0 or 1 (see scoreFromTraits line 69: 'ratio >= 0.8 ? 1.0 : ratio'). This is actually correct for ratio but the quality weight (0.2) means it under-contributes. The real bug is in brain.js: the stats() method computes avgComposite but doesn't round it, leading to floating point noise. Fix stats() to round avgComposite to 3 decimal places, and add a test case verifying the fix.",
-    "taskType": "coding-agent",
-    "difficulty": "medium",
-    "expectedTraits": [
-      "reads before writing",
-      "uses grep for search",
-      "runs tests after changes",
-      "explains the bug"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "read_file",
-        "grep_files",
-        "edit_file",
-        "run_shell"
-      ],
-      "forbiddenToolCalls": [],
-      "maxTurns": 10,
-      "expectedFileChanges": [
-        "brain.js",
-        "test.js"
-      ],
-      "testCommand": "npm test",
-      "projectFixture": "wall-e-subset"
-    }
-  },
-  {
-    "id": "agent-py-001",
-    "prompt": "Add a /health endpoint to the Flask app that returns JSON with status 'ok' and uptime in seconds (time since the app started).",
-    "taskType": "coding-agent",
-    "difficulty": "easy",
-    "expectedTraits": [
-      "has code block",
-      "reads before writing",
-      "uses edit over write"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "read_file",
-        "glob",
-        "edit_file"
-      ],
-      "forbiddenToolCalls": [],
-      "maxTurns": 8,
-      "expectedFileChanges": [
-        "app.py"
-      ],
-      "testCommand": "python -m pytest",
-      "projectFixture": "python-flask"
-    }
-  },
-  {
-    "id": "agent-py-002",
-    "prompt": "Add input validation to the Flask app: create a POST /items endpoint that accepts JSON with 'name' (required string, 1-100 chars) and 'price' (required number, > 0). Return 400 with descriptive errors on invalid input, 201 with the created item on success.",
-    "taskType": "coding-agent",
-    "difficulty": "medium",
-    "expectedTraits": [
-      "has code block",
-      "reads before writing",
-      "handles errors gracefully"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "read_file",
-        "glob",
-        "edit_file"
-      ],
-      "forbiddenToolCalls": [],
-      "maxTurns": 10,
-      "expectedFileChanges": [
-        "app.py"
-      ],
-      "testCommand": "python -m pytest",
-      "projectFixture": "python-flask"
-    }
-  },
-  {
-    "id": "agent-py-003",
-    "prompt": "There's a bug in the Flask app: the GET / endpoint returns a datetime field that uses naive datetime (no timezone info), but the API contract requires UTC ISO 8601 format. Fix the serialization to always return timezone-aware UTC timestamps.",
-    "taskType": "coding-agent",
-    "difficulty": "medium",
-    "expectedTraits": [
-      "has code block",
-      "reads before writing",
-      "explains the bug"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "read_file",
-        "grep_files",
-        "edit_file"
-      ],
-      "forbiddenToolCalls": [],
-      "maxTurns": 10,
-      "expectedFileChanges": [
-        "app.py"
-      ],
-      "testCommand": "python -m pytest",
-      "projectFixture": "python-flask"
-    }
-  },
-  {
-    "id": "agent-py-004",
-    "prompt": "Add a SQLite persistence layer to the Flask app. Create a db.py module that initializes a SQLite database with an 'items' table (id INTEGER PRIMARY KEY, name TEXT, price REAL, created_at TEXT). Add functions to insert and retrieve items. Wire it into the Flask app so POST /items stores to the database and GET /items reads from it.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "has code block",
-      "reads before writing",
-      "multi-file coordination",
-      "runs tests after changes"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "read_file",
-        "glob",
-        "write_file",
-        "edit_file",
-        "run_shell"
-      ],
-      "forbiddenToolCalls": [],
-      "maxTurns": 18,
-      "expectedFileChanges": [
-        "db.py",
-        "app.py"
-      ],
-      "testCommand": "python -m pytest",
-      "projectFixture": "python-flask"
-    }
-  },
-  {
-    "id": "agent-py-005",
-    "prompt": "Refactor the Flask app: split the monolithic app.py into blueprints. Create a blueprints/ directory with items.py and health.py. Move route handlers into their respective blueprints. Create a shared config.py for app configuration. Ensure all existing tests still pass.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "has code block",
-      "reads before writing",
-      "multi-file coordination",
-      "runs tests after changes"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "read_file",
-        "glob",
-        "write_file",
-        "edit_file",
-        "run_shell"
-      ],
-      "forbiddenToolCalls": [],
-      "maxTurns": 20,
-      "expectedFileChanges": [
-        "app.py",
-        "config.py",
-        "blueprints/items.py",
-        "blueprints/health.py"
-      ],
-      "testCommand": "python -m pytest",
-      "projectFixture": "python-flask"
-    }
-  },
-  {
-    "id": "agent-lsp-001",
-    "prompt": "Rename the function 'processData' to 'transformPayload' across the entire project. Update all callers, imports, exports, and tests. Use LSP references to find all usages before making changes.",
-    "taskType": "coding-agent",
-    "difficulty": "medium",
-    "expectedTraits": [
-      "reads before writing",
-      "uses grep for search",
-      "multi-file coordination",
-      "efficient tool use"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "read_file",
-        "grep_files",
-        "edit_file",
-        "lsp_references"
-      ],
-      "forbiddenToolCalls": [],
-      "maxTurns": 12,
-      "expectedFileChanges": [
-        "server.js",
-        "test.js"
-      ],
-      "testCommand": "npm test",
-      "projectFixture": "express-basic"
-    }
-  },
-  {
-    "id": "agent-lsp-002",
-    "prompt": "This project uses JSDoc type annotations but has type errors. Run diagnostics to find all type-related issues, then fix them. The function signatures should match their documented types.",
-    "taskType": "coding-agent",
-    "difficulty": "medium",
-    "expectedTraits": [
-      "reads before writing",
-      "uses LSP diagnostics",
-      "runs tests after changes"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "read_file",
-        "glob",
-        "edit_file",
-        "lsp_diagnostics"
-      ],
-      "forbiddenToolCalls": [],
-      "maxTurns": 15,
-      "expectedFileChanges": [
-        "src/utils.js",
-        "src/handlers.js"
-      ],
-      "testCommand": "npm test",
-      "projectFixture": "node-typed"
-    }
-  },
-  {
-    "id": "agent-lsp-003",
-    "prompt": "Extract all inline SQL queries from the route handlers into a separate db.js module. Use LSP to find all references to the database connection, create a clean data access layer, and update all imports. Ensure tests still pass.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "multi-file coordination",
-      "runs tests after changes"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "read_file",
-        "grep_files",
-        "write_file",
-        "edit_file",
-        "lsp_definition",
-        "lsp_references",
-        "run_shell"
-      ],
-      "forbiddenToolCalls": [],
-      "maxTurns": 18,
-      "expectedFileChanges": [
-        "server/db.js",
-        "server/routes/items.js",
-        "server/routes/users.js"
-      ],
-      "testCommand": "npm test",
-      "projectFixture": "fullstack-app"
-    }
-  },
-  {
-    "id": "agent-sec-001",
-    "prompt": "The user login endpoint in this app has a SQL injection vulnerability. The query uses string concatenation instead of parameterized queries. Find and fix the SQL injection vulnerability, and add a test that proves the injection is no longer possible.",
-    "taskType": "coding-agent",
-    "difficulty": "medium",
-    "expectedTraits": [
-      "has code block",
-      "reads before writing",
-      "uses grep for search",
-      "runs tests after changes"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "read_file",
-        "grep_files",
-        "edit_file",
-        "run_shell"
-      ],
-      "forbiddenToolCalls": [],
-      "maxTurns": 12,
-      "expectedFileChanges": [
-        "server.js",
-        "test.js"
-      ],
-      "testCommand": "npm test",
-      "projectFixture": "express-buggy"
-    }
-  },
-  {
-    "id": "agent-sec-002",
-    "prompt": "This Express app renders user input directly in HTML templates without sanitization, creating XSS vulnerabilities. Find all places where user input is rendered unsanitized, fix them by adding proper escaping/sanitization, and add tests to verify the fixes.",
-    "taskType": "coding-agent",
-    "difficulty": "medium",
-    "expectedTraits": [
-      "has code block",
-      "reads before writing",
-      "uses grep for search",
-      "runs tests after changes"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "read_file",
-        "grep_files",
-        "edit_file",
-        "run_shell"
-      ],
-      "forbiddenToolCalls": [],
-      "maxTurns": 15,
-      "expectedFileChanges": [
-        "server.js",
-        "views/profile.ejs",
-        "test.js"
-      ],
-      "testCommand": "npm test",
-      "projectFixture": "express-xss"
-    }
-  },
-  {
-    "id": "agent-sec-003",
-    "prompt": "Add rate limiting and CSRF protection to all POST routes in the Express app. Create a rate limiter middleware (100 requests per minute per IP) and a CSRF token middleware. Apply both to all POST endpoints. Add tests verifying rate limiting returns 429 and CSRF validation rejects requests without valid tokens.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "has code block",
-      "reads before writing",
-      "multi-file coordination",
-      "runs tests after changes"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "read_file",
-        "glob",
-        "write_file",
-        "edit_file",
-        "run_shell"
-      ],
-      "forbiddenToolCalls": [],
-      "maxTurns": 18,
-      "expectedFileChanges": [
-        "server.js",
-        "middleware/rate-limit.js",
-        "middleware/csrf.js",
-        "test.js"
-      ],
-      "testCommand": "npm test",
-      "projectFixture": "express-basic"
-    }
-  },
-  {
-    "id": "agent-sec-004",
-    "prompt": "Audit the project's dependencies for known security vulnerabilities. Run npm audit, identify vulnerable packages, upgrade them to safe versions, and verify no breaking changes were introduced. Document findings in a SECURITY.md file.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "runs tests after changes"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "run_shell",
-        "read_file",
-        "edit_file",
-        "write_file"
-      ],
-      "forbiddenToolCalls": [],
-      "maxTurns": 15,
-      "expectedFileChanges": [
-        "package.json",
-        "SECURITY.md"
-      ],
-      "testCommand": "npm test",
-      "projectFixture": "fullstack-app"
-    }
-  },
-  {
-    "id": "agent-mf-001",
-    "prompt": "Add pagination to the GET /api/items endpoint: accept ?page and ?limit query params. Also update the frontend public/js/items.js to display page controls (Previous/Next buttons) and call the paginated API. Return pagination metadata in the API response.",
-    "taskType": "coding-agent",
-    "difficulty": "medium",
-    "expectedTraits": [
-      "reads before writing",
-      "multi-file coordination",
-      "runs tests after changes"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "glob",
-        "read_file",
-        "edit_file",
-        "run_shell"
-      ],
-      "forbiddenToolCalls": [],
-      "maxTurns": 15,
-      "expectedFileChanges": [
-        "server/routes/items.js",
-        "public/js/items.js",
-        "test.js"
-      ],
-      "testCommand": "npm test",
-      "projectFixture": "fullstack-app"
-    }
-  },
-  {
-    "id": "agent-mf-002",
-    "prompt": "Add authentication middleware to the fullstack app. Create a JWT-based auth middleware that protects all /api routes except POST /api/auth/login. Add a login page at public/login.html. Update the frontend to include the JWT token in all API requests. Add tests for protected and unprotected routes.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "multi-file coordination",
-      "runs tests after changes"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "glob",
-        "read_file",
-        "write_file",
-        "edit_file",
-        "run_shell"
-      ],
-      "forbiddenToolCalls": [],
-      "maxTurns": 20,
-      "expectedFileChanges": [
-        "server/middleware/auth.js",
-        "server/routes/auth.js",
-        "server/router.js",
-        "public/login.html",
-        "test.js"
-      ],
-      "testCommand": "npm test",
-      "projectFixture": "fullstack-app"
-    }
-  },
-  {
-    "id": "agent-mf-003",
-    "prompt": "Convert the monorepo from CommonJS (require/module.exports) to ES Modules (import/export). Update all package.json files to add \"type\": \"module\", convert all require() calls to import statements, convert all module.exports to export statements. Ensure all tests pass after conversion.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "uses grep for search",
-      "multi-file coordination",
-      "runs tests after changes"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "glob",
-        "grep_files",
-        "read_file",
-        "edit_file",
-        "run_shell"
-      ],
-      "forbiddenToolCalls": [],
-      "maxTurns": 20,
-      "expectedFileChanges": [
-        "package.json",
-        "packages/shared/index.js",
-        "packages/shared/formatters.js",
-        "packages/shared/validators.js",
-        "packages/api/server.js",
-        "packages/api/routes.js",
-        "packages/cli/index.js",
-        "packages/cli/commands.js"
-      ],
-      "testCommand": "npm test",
-      "projectFixture": "monorepo-basic"
-    }
-  },
-  {
-    "id": "agent-mf-004",
-    "prompt": "Add a CI/CD pipeline and containerization to the fullstack app. Create a .github/workflows/ci.yml that runs tests on push, a Dockerfile for the app, and a docker-compose.yml that includes the app and a volume for the SQLite database. The CI should lint, test, and build the Docker image.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "multi-file coordination"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "glob",
-        "read_file",
-        "write_file"
-      ],
-      "forbiddenToolCalls": [],
-      "maxTurns": 15,
-      "expectedFileChanges": [
-        ".github/workflows/ci.yml",
-        "Dockerfile",
-        "docker-compose.yml"
-      ],
-      "testCommand": null,
-      "projectFixture": "fullstack-app"
-    }
-  },
-  {
-    "id": "agent-ambig-006",
-    "prompt": "Make the API faster",
-    "taskType": "coding-agent",
-    "difficulty": "medium",
-    "expectedTraits": [
-      "reads before writing",
-      "uses glob for discovery"
-    ],
-    "agentExpectations": {
-      "shouldAskUser": true,
-      "expectedToolCalls": [
-        "glob",
-        "read_file",
-        "ask_user"
-      ],
-      "forbiddenToolCalls": [
-        "write_file",
-        "edit_file"
-      ],
-      "maxTurns": 8,
-      "expectedFileChanges": [],
-      "projectFixture": "fullstack-app"
-    }
-  },
-  {
-    "id": "agent-ambig-007",
-    "prompt": "Add caching",
-    "taskType": "coding-agent",
-    "difficulty": "medium",
-    "expectedTraits": [
-      "reads before writing",
-      "uses glob for discovery"
-    ],
-    "agentExpectations": {
-      "shouldAskUser": true,
-      "expectedToolCalls": [
-        "glob",
-        "read_file",
-        "ask_user"
-      ],
-      "forbiddenToolCalls": [
-        "write_file",
-        "edit_file"
-      ],
-      "maxTurns": 8,
-      "expectedFileChanges": [],
-      "projectFixture": "express-basic"
-    }
-  },
-  {
-    "id": "agent-ambig-008",
-    "prompt": "Refactor the database layer",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "uses glob for discovery"
-    ],
-    "agentExpectations": {
-      "shouldAskUser": true,
-      "expectedToolCalls": [
-        "glob",
-        "read_file",
-        "grep_files",
-        "ask_user"
-      ],
-      "forbiddenToolCalls": [
-        "write_file",
-        "edit_file"
-      ],
-      "maxTurns": 8,
-      "expectedFileChanges": [],
-      "projectFixture": "fullstack-app"
-    }
-  },
-  {
-    "id": "agent-ambig-009",
-    "prompt": "The tests are flaky, fix them",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "runs tests after changes",
-      "uses grep for search"
-    ],
-    "agentExpectations": {
-      "shouldAskUser": true,
-      "expectedToolCalls": [
-        "run_shell",
-        "read_file",
-        "grep_files",
-        "ask_user"
-      ],
-      "forbiddenToolCalls": [
-        "write_file",
-        "edit_file"
-      ],
-      "maxTurns": 10,
-      "expectedFileChanges": [],
-      "projectFixture": "fullstack-app"
-    }
-  }
-]