create-walle 0.9.13 → 0.9.15

package/template/wall-e/eval/benchmarks/coding-agent.json

@@ -1577,601 +1577,5 @@
       "expectedFileChanges": [],
       "projectFixture": "fullstack-app"
     }
-  },
-  {
-    "id": "agent-session-e8413ef1",
-    "prompt": "I'll start by exploring the project structure to understand the Express app setup.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "uses edit over write"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "list_directory",
-        "read_file",
-        "edit_file",
-        "run_shell"
-      ],
-      "maxTurns": 22,
-      "expectedFileChanges": [
-        "server.js",
-        "test.js"
-      ]
-    },
-    "sourceSessionId": "8a31c591-bbaf-403f-b82e-3ec093618584",
-    "classifiedType": "coding:generation",
-    "expectedDiff": null,
-    "complexityIndicator": 2
-  },
-  {
-    "id": "agent-session-02a0e2b9",
-    "prompt": "I'll help you rename the 'getData' function to 'fetchRecords' across the project. Let me start by exploring the project structure and finding all occurrences.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "uses edit over write"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "list_directory",
-        "read_file",
-        "grep_files",
-        "glob",
-        "ask_user",
-        "edit_file",
-        "run_shell"
-      ],
-      "maxTurns": 50,
-      "expectedFileChanges": [
-        "server.js",
-        "test.js"
-      ]
-    },
-    "sourceSessionId": "07b948cd-2324-442b-849b-ff88de26783d",
-    "classifiedType": "coding:refactoring",
-    "expectedDiff": null,
-    "complexityIndicator": 2
-  },
-  {
-    "id": "agent-session-a78f1dc7",
-    "prompt": "I'll help you find and fix the SQL injection vulnerability. Let me start by exploring the project structure.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "uses edit over write"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "list_directory",
-        "read_file",
-        "grep_files",
-        "edit_file",
-        "run_shell"
-      ],
-      "maxTurns": 30,
-      "expectedFileChanges": [
-        "server.js",
-        "test.js"
-      ]
-    },
-    "sourceSessionId": "dcce8928-0376-4058-b5fd-321f50c39783",
-    "classifiedType": "coding:debugging",
-    "expectedDiff": null,
-    "complexityIndicator": 2
-  },
-  {
-    "id": "agent-session-1348957e",
-    "prompt": "I'll help you add tests for both ascending and descending sort. Let me start by exploring the project structure to understand what we're working with.",
-    "taskType": "coding-agent",
-    "difficulty": "medium",
-    "expectedTraits": [
-      "reads before writing",
-      "uses edit over write"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "list_directory",
-        "read_file",
-        "run_shell",
-        "ask_user",
-        "edit_file"
-      ],
-      "maxTurns": 20,
-      "expectedFileChanges": [
-        "server.js",
-        "test.js"
-      ]
-    },
-    "sourceSessionId": "31c2980b-95ce-4685-bdd5-46c92fe9aa36",
-    "classifiedType": "coding:testing",
-    "expectedDiff": null,
-    "complexityIndicator": 2
-  },
-  {
-    "id": "agent-session-f7edd420",
-    "prompt": "I'll start by exploring the project structure to understand the CLI tool setup.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "uses edit over write"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "list_directory",
-        "read_file",
-        "edit_file",
-        "run_shell"
-      ],
-      "maxTurns": 22,
-      "expectedFileChanges": [
-        "index.js",
-        "test.js"
-      ]
-    },
-    "sourceSessionId": "5719d4dd-fcdb-47b6-9874-60f4b9fbfb71",
-    "classifiedType": "coding:generation",
-    "expectedDiff": null,
-    "complexityIndicator": 2
-  },
-  {
-    "id": "agent-session-b36c5122",
-    "prompt": "I'll start by exploring the project structure to understand what we're working with.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "uses edit over write",
-      "plans before executing"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "list_directory",
-        "read_file",
-        "run_shell",
-        "update_todos",
-        "edit_file"
-      ],
-      "maxTurns": 36,
-      "expectedFileChanges": [
-        "server/db.js",
-        "server/index.js"
-      ]
-    },
-    "sourceSessionId": "f78fa371-7e29-4a52-8f4f-b992fdaccc0d",
-    "classifiedType": "coding:generation",
-    "expectedDiff": null,
-    "complexityIndicator": 2
-  },
-  {
-    "id": "agent-session-29b64889",
-    "prompt": "I'll start by exploring the project structure and understanding what tests exist.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "uses edit over write"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "list_directory",
-        "read_file",
-        "run_shell",
-        "grep_files",
-        "edit_file"
-      ],
-      "maxTurns": 50,
-      "expectedFileChanges": [
-        "server/test.js",
-        "server/db.js"
-      ]
-    },
-    "sourceSessionId": "25da8640-eaba-42a6-9eb2-a5831c2162fd",
-    "classifiedType": "coding:testing",
-    "expectedDiff": null,
-    "complexityIndicator": 2
-  },
-  {
-    "id": "agent-session-2494dd1d",
-    "prompt": "I'll help you extract inline SQL queries from route handlers into a separate db.js module. Let me start by exploring the project structure.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "uses edit over write",
-      "plans before executing"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "list_directory",
-        "read_file",
-        "run_shell",
-        "update_todos",
-        "lsp_references",
-        "grep_files",
-        "edit_file"
-      ],
-      "maxTurns": 50,
-      "expectedFileChanges": [
-        "server/db.js",
-        "server/routes/auth.js",
-        "server/routes/users.js",
-        "server/routes/items.js",
-        "server/routes/health.js",
-        "server/middleware/auth.js"
-      ]
-    },
-    "sourceSessionId": "08d10d2e-3e63-4354-bf05-8786388b2420",
-    "classifiedType": "coding:refactoring",
-    "expectedDiff": null,
-    "complexityIndicator": 6
-  },
-  {
-    "id": "agent-session-5fcb2df4",
-    "prompt": "I'll start by exploring the project structure to understand the fullstack app, then create the CI/CD pipeline and containerization files.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "list_directory",
-        "read_file",
-        "glob",
-        "write_file",
-        "edit_file",
-        "run_shell"
-      ],
-      "maxTurns": 50,
-      "expectedFileChanges": [
-        ".github/workflows/ci.yml",
-        "Dockerfile",
-        "docker-compose.yml",
-        ".dockerignore",
-        "config/default.js",
-        "DOCKER.md",
-        "CI_CD_SETUP.md",
-        "QUICKSTART.md",
-        "IMPLEMENTATION_SUMMARY.md",
-        "DEPLOYMENT_CHECKLIST.md",
-        "CHANGES.md",
-        "README_CI_CD.md"
-      ]
-    },
-    "sourceSessionId": "9055411d-427f-498b-b501-b7f695b22214",
-    "classifiedType": "coding:generation",
-    "expectedDiff": null,
-    "complexityIndicator": 12
-  },
-  {
-    "id": "agent-session-1c9213cc",
-    "prompt": "I'll start by exploring the project structure to understand the current implementation.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "uses edit over write"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "list_directory",
-        "read_file",
-        "edit_file",
-        "run_shell"
-      ],
-      "maxTurns": 50,
-      "expectedFileChanges": [
-        "server/routes/items.js",
-        "public/js/items.js",
-        "public/js/app.js",
-        "server/test.js"
-      ]
-    },
-    "sourceSessionId": "8a0a1609-9031-4bcc-86ff-e690fef3b290",
-    "classifiedType": "coding:generation",
-    "expectedDiff": null,
-    "complexityIndicator": 4
-  },
-  {
-    "id": "agent-session-63ada37e",
-    "prompt": "I'll help you rename the function 'processData' to 'transformPayload' across the entire project. Let me start by exploring the project structure and finding all usages.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "uses edit over write"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "list_directory",
-        "read_file",
-        "grep_files",
-        "run_shell",
-        "ask_user",
-        "edit_file",
-        "lsp_references"
-      ],
-      "maxTurns": 50,
-      "expectedFileChanges": [
-        "server.js",
-        "test.js"
-      ]
-    },
-    "sourceSessionId": "a7b388ad-58e9-48ea-9278-b817ba300726",
-    "classifiedType": "coding:refactoring",
-    "expectedDiff": null,
-    "complexityIndicator": 2
-  },
-  {
-    "id": "agent-session-3a3b9214",
-    "prompt": "I'll start by exploring the project structure to understand the Express app and then implement rate limiting and CSRF protection.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "uses edit over write"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "list_directory",
-        "read_file",
-        "edit_file",
-        "run_shell"
-      ],
-      "maxTurns": 36,
-      "expectedFileChanges": [
-        "package.json",
-        "server.js",
-        "test.js"
-      ]
-    },
-    "sourceSessionId": "4a7b82b6-88ce-4bb3-a41a-d5ba58f2c83b",
-    "classifiedType": "coding:generation",
-    "expectedDiff": null,
-    "complexityIndicator": 3
-  },
-  {
-    "id": "agent-session-c28e561a",
-    "prompt": "I'll help you add a test for pagination with 25 users. Let me start by exploring the project structure to understand the codebase.",
-    "taskType": "coding-agent",
-    "difficulty": "medium",
-    "expectedTraits": [
-      "reads before writing",
-      "uses edit over write"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "list_directory",
-        "read_file",
-        "edit_file",
-        "run_shell"
-      ],
-      "maxTurns": 14,
-      "expectedFileChanges": [
-        "server.js",
-        "test.js"
-      ]
-    },
-    "sourceSessionId": "91ccf531-0e27-45be-880b-9e37fd8b8349",
-    "classifiedType": "coding:testing",
-    "expectedDiff": null,
-    "complexityIndicator": 2
-  },
-  {
-    "id": "agent-session-ecc4ff3f",
-    "prompt": "I'll start by exploring the project structure to understand the codebase and locate the relevant files.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "uses edit over write"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "list_directory",
-        "read_file",
-        "run_shell",
-        "edit_file"
-      ],
-      "maxTurns": 48,
-      "expectedFileChanges": [
-        "brain.js",
-        "test.js"
-      ]
-    },
-    "sourceSessionId": "aeae6b54-c335-4fe4-a903-3e6b440fbe66",
-    "classifiedType": "coding:generation",
-    "expectedDiff": null,
-    "complexityIndicator": 2
-  },
-  {
-    "id": "agent-session-a29571f9",
-    "prompt": "I'll help you add tests for both ascending and descending sort. Let me start by exploring the project structure.",
-    "taskType": "coding-agent",
-    "difficulty": "medium",
-    "expectedTraits": [
-      "reads before writing",
-      "uses edit over write"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "list_directory",
-        "read_file",
-        "edit_file",
-        "run_shell"
-      ],
-      "maxTurns": 14,
-      "expectedFileChanges": [
-        "server.js",
-        "test.js"
-      ]
-    },
-    "sourceSessionId": "bdc47279-f629-465b-bc8e-eaa22cb65266",
-    "classifiedType": "coding:testing",
-    "expectedDiff": null,
-    "complexityIndicator": 2
-  },
-  {
-    "id": "agent-session-8abb12e4",
-    "prompt": "I'll start by exploring the project structure to understand the codebase.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "uses edit over write"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "list_directory",
-        "read_file",
-        "edit_file",
-        "run_shell"
-      ],
-      "maxTurns": 50,
-      "expectedFileChanges": [
-        "server/routes/items.js",
-        "public/js/items.js",
-        "public/js/app.js",
-        "public/css/style.css",
-        "server/test.js",
-        "server/index.js"
-      ]
-    },
-    "sourceSessionId": "d25075ab-ec95-4e1d-813d-79c5dbf5bd68",
-    "classifiedType": "coding:generation",
-    "expectedDiff": null,
-    "complexityIndicator": 6
-  },
-  {
-    "id": "agent-session-cfd12d6c",
-    "prompt": "I'll help you convert this monorepo from CommonJS to ES Modules. Let me start by exploring the project structure.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "plans before executing"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "list_directory",
-        "read_file",
-        "run_shell",
-        "update_todos",
-        "edit_file",
-        "write_file"
-      ],
-      "maxTurns": 50,
-      "expectedFileChanges": [
-        "package.json",
-        "packages/shared/package.json",
-        "packages/api/package.json",
-        "packages/cli/package.json",
-        "packages/shared/formatters.js",
-        "packages/shared/validators.js",
-        "packages/shared/index.js",
-        "packages/shared/test.js",
-        "packages/api/data.js",
-        "packages/api/middleware.js",
-        "packages/api/routes.js",
-        "packages/api/server.js",
-        "packages/api/test.js",
-        "packages/cli/output.js",
-        "packages/cli/commands.js",
-        "packages/cli/index.js",
-        "packages/cli/test.js",
-        "test.js"
-      ]
-    },
-    "sourceSessionId": "ab4c19bb-d588-43dd-994b-fdf05431e525",
-    "classifiedType": "coding:generation",
-    "expectedDiff": null,
-    "complexityIndicator": 18
-  },
-  {
-    "id": "agent-session-f21db69e",
-    "prompt": "I'll start by exploring the project structure to understand the Flask app setup.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "uses edit over write"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "list_directory",
-        "read_file",
-        "edit_file",
-        "run_shell"
-      ],
-      "maxTurns": 30,
-      "expectedFileChanges": [
-        "app.py",
-        "test_app.py"
-      ]
-    },
-    "sourceSessionId": "d44e8d0e-ef75-4f49-8340-2c4628858fe9",
-    "classifiedType": "coding:generation",
-    "expectedDiff": null,
-    "complexityIndicator": 2
-  },
-  {
-    "id": "agent-session-e8eb3472",
-    "prompt": "I'll help you add a test for pagination with 25 users. Let me start by exploring the project structure.",
-    "taskType": "coding-agent",
-    "difficulty": "medium",
-    "expectedTraits": [
-      "reads before writing",
-      "uses edit over write"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "list_directory",
-        "read_file",
-        "edit_file",
-        "run_shell"
-      ],
-      "maxTurns": 14,
-      "expectedFileChanges": [
-        "server.js",
-        "test.js"
-      ]
-    },
-    "sourceSessionId": "991abf0f-4bb1-48fa-8aa8-ddd97d45d6f8",
-    "classifiedType": "coding:testing",
-    "expectedDiff": null,
-    "complexityIndicator": 2
-  },
-  {
-    "id": "agent-session-bd5f99de",
-    "prompt": "I'll start by exploring the project structure and understanding the codebase, then run diagnostics to find type-related issues.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "uses edit over write"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "list_directory",
-        "read_file",
-        "lsp_diagnostics",
-        "run_shell",
-        "edit_file"
-      ],
-      "maxTurns": 32,
-      "expectedFileChanges": [
-        "src/handlers.js",
-        "src/utils.js"
-      ]
-    },
-    "sourceSessionId": "abeceb68-cda7-4791-a201-b0d8e00bbb90",
-    "classifiedType": "coding:generation",
-    "expectedDiff": null,
-    "complexityIndicator": 2
   }
 ]

package/template/wall-e/eval/cc-replay.js

@@ -289,6 +289,7 @@ async function replayAndScore(session, sandbox, runAgentLoop, opts = {}) {
       provider,
       model,
       mode: 'build',
+      persistTranscript: false,
     });
     const hardTimeout = new Promise((_, reject) =>
       setTimeout(() => reject(new Error('cc-replay hard timeout exceeded')), timeoutMs + 60_000)