create-walle 0.9.13 → 0.9.14

package/template/wall-e/eval/benchmark-generator.js

@@ -18,11 +18,31 @@ function classifyCodingType(session) {
   return 'coding:generation';
 }
 
+function isReplayableBenchmarkPrompt(prompt) {
+  const text = String(prompt || '').trim();
+  if (text.length < 25) return false;
+
+  // Session-mined prompts must be the user's task, not the assistant's first
+  // progress narration. Assistant prose turns the benchmark into "continue the
+  // previous assistant's work", which is not replayable from a fresh sandbox.
+  if (/^(i('|’)ll|i will|i can|i('|’)m going to|let me|sure[, ]|happy to help|i('|’)ll help|i('|’)ll start)\b/i.test(text)) {
+    return false;
+  }
+
+  if (/^\s*(go ahead|continue|proceed|do it|yes|yep|ok|okay|thanks|thank you)\b/i.test(text)) {
+    return false;
+  }
+
+  return /\b(fix|implement|add|change|update|refactor|test|debug|make|write|delete|remove|harden|wire|bug|failing|error|regression|feature|endpoint|api|ui|server|component|code review|review.*code)\b/i.test(text);
+}
+
 /**
  * Convert a coding agent session object to a benchmark entry.
  * Returns null if the session would be a duplicate (id already in existingIds).
  */
 function sessionToBenchmark(session, existingIds = new Set()) {
+  if (!isReplayableBenchmarkPrompt(session.prompt)) return null;
+
   const id = `agent-session-${crypto.createHash('sha256').update(session.prompt || '').digest('hex').slice(0, 8)}`;
   if (existingIds.has(id)) return null; // dedup
 
@@ -90,4 +110,4 @@ async function generateBenchmarks(brain, { minSignificance = 0.5, limit = 50 } =
   return benchmarks;
 }
 
-module.exports = { classifyCodingType, sessionToBenchmark, generateBenchmarks };
+module.exports = { classifyCodingType, isReplayableBenchmarkPrompt, sessionToBenchmark, generateBenchmarks };

package/template/wall-e/eval/benchmarks/coding-agent.json

@@ -1577,601 +1577,5 @@
       "expectedFileChanges": [],
       "projectFixture": "fullstack-app"
     }
-  },
-  {
-    "id": "agent-session-e8413ef1",
-    "prompt": "I'll start by exploring the project structure to understand the Express app setup.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "uses edit over write"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "list_directory",
-        "read_file",
-        "edit_file",
-        "run_shell"
-      ],
-      "maxTurns": 22,
-      "expectedFileChanges": [
-        "server.js",
-        "test.js"
-      ]
-    },
-    "sourceSessionId": "8a31c591-bbaf-403f-b82e-3ec093618584",
-    "classifiedType": "coding:generation",
-    "expectedDiff": null,
-    "complexityIndicator": 2
-  },
-  {
-    "id": "agent-session-02a0e2b9",
-    "prompt": "I'll help you rename the 'getData' function to 'fetchRecords' across the project. Let me start by exploring the project structure and finding all occurrences.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "uses edit over write"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "list_directory",
-        "read_file",
-        "grep_files",
-        "glob",
-        "ask_user",
-        "edit_file",
-        "run_shell"
-      ],
-      "maxTurns": 50,
-      "expectedFileChanges": [
-        "server.js",
-        "test.js"
-      ]
-    },
-    "sourceSessionId": "07b948cd-2324-442b-849b-ff88de26783d",
-    "classifiedType": "coding:refactoring",
-    "expectedDiff": null,
-    "complexityIndicator": 2
-  },
-  {
-    "id": "agent-session-a78f1dc7",
-    "prompt": "I'll help you find and fix the SQL injection vulnerability. Let me start by exploring the project structure.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "uses edit over write"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "list_directory",
-        "read_file",
-        "grep_files",
-        "edit_file",
-        "run_shell"
-      ],
-      "maxTurns": 30,
-      "expectedFileChanges": [
-        "server.js",
-        "test.js"
-      ]
-    },
-    "sourceSessionId": "dcce8928-0376-4058-b5fd-321f50c39783",
-    "classifiedType": "coding:debugging",
-    "expectedDiff": null,
-    "complexityIndicator": 2
-  },
-  {
-    "id": "agent-session-1348957e",
-    "prompt": "I'll help you add tests for both ascending and descending sort. Let me start by exploring the project structure to understand what we're working with.",
-    "taskType": "coding-agent",
-    "difficulty": "medium",
-    "expectedTraits": [
-      "reads before writing",
-      "uses edit over write"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "list_directory",
-        "read_file",
-        "run_shell",
-        "ask_user",
-        "edit_file"
-      ],
-      "maxTurns": 20,
-      "expectedFileChanges": [
-        "server.js",
-        "test.js"
-      ]
-    },
-    "sourceSessionId": "31c2980b-95ce-4685-bdd5-46c92fe9aa36",
-    "classifiedType": "coding:testing",
-    "expectedDiff": null,
-    "complexityIndicator": 2
-  },
-  {
-    "id": "agent-session-f7edd420",
-    "prompt": "I'll start by exploring the project structure to understand the CLI tool setup.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "uses edit over write"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "list_directory",
-        "read_file",
-        "edit_file",
-        "run_shell"
-      ],
-      "maxTurns": 22,
-      "expectedFileChanges": [
-        "index.js",
-        "test.js"
-      ]
-    },
-    "sourceSessionId": "5719d4dd-fcdb-47b6-9874-60f4b9fbfb71",
-    "classifiedType": "coding:generation",
-    "expectedDiff": null,
-    "complexityIndicator": 2
-  },
-  {
-    "id": "agent-session-b36c5122",
-    "prompt": "I'll start by exploring the project structure to understand what we're working with.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "uses edit over write",
-      "plans before executing"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "list_directory",
-        "read_file",
-        "run_shell",
-        "update_todos",
-        "edit_file"
-      ],
-      "maxTurns": 36,
-      "expectedFileChanges": [
-        "server/db.js",
-        "server/index.js"
-      ]
-    },
-    "sourceSessionId": "f78fa371-7e29-4a52-8f4f-b992fdaccc0d",
-    "classifiedType": "coding:generation",
-    "expectedDiff": null,
-    "complexityIndicator": 2
-  },
-  {
-    "id": "agent-session-29b64889",
-    "prompt": "I'll start by exploring the project structure and understanding what tests exist.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "uses edit over write"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "list_directory",
-        "read_file",
-        "run_shell",
-        "grep_files",
-        "edit_file"
-      ],
-      "maxTurns": 50,
-      "expectedFileChanges": [
-        "server/test.js",
-        "server/db.js"
-      ]
-    },
-    "sourceSessionId": "25da8640-eaba-42a6-9eb2-a5831c2162fd",
-    "classifiedType": "coding:testing",
-    "expectedDiff": null,
-    "complexityIndicator": 2
-  },
-  {
-    "id": "agent-session-2494dd1d",
-    "prompt": "I'll help you extract inline SQL queries from route handlers into a separate db.js module. Let me start by exploring the project structure.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "uses edit over write",
-      "plans before executing"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "list_directory",
-        "read_file",
-        "run_shell",
-        "update_todos",
-        "lsp_references",
-        "grep_files",
-        "edit_file"
-      ],
-      "maxTurns": 50,
-      "expectedFileChanges": [
-        "server/db.js",
-        "server/routes/auth.js",
-        "server/routes/users.js",
-        "server/routes/items.js",
-        "server/routes/health.js",
-        "server/middleware/auth.js"
-      ]
-    },
-    "sourceSessionId": "08d10d2e-3e63-4354-bf05-8786388b2420",
-    "classifiedType": "coding:refactoring",
-    "expectedDiff": null,
-    "complexityIndicator": 6
-  },
-  {
-    "id": "agent-session-5fcb2df4",
-    "prompt": "I'll start by exploring the project structure to understand the fullstack app, then create the CI/CD pipeline and containerization files.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "list_directory",
-        "read_file",
-        "glob",
-        "write_file",
-        "edit_file",
-        "run_shell"
-      ],
-      "maxTurns": 50,
-      "expectedFileChanges": [
-        ".github/workflows/ci.yml",
-        "Dockerfile",
-        "docker-compose.yml",
-        ".dockerignore",
-        "config/default.js",
-        "DOCKER.md",
-        "CI_CD_SETUP.md",
-        "QUICKSTART.md",
-        "IMPLEMENTATION_SUMMARY.md",
-        "DEPLOYMENT_CHECKLIST.md",
-        "CHANGES.md",
-        "README_CI_CD.md"
-      ]
-    },
-    "sourceSessionId": "9055411d-427f-498b-b501-b7f695b22214",
-    "classifiedType": "coding:generation",
-    "expectedDiff": null,
-    "complexityIndicator": 12
-  },
-  {
-    "id": "agent-session-1c9213cc",
-    "prompt": "I'll start by exploring the project structure to understand the current implementation.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "uses edit over write"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "list_directory",
-        "read_file",
-        "edit_file",
-        "run_shell"
-      ],
-      "maxTurns": 50,
-      "expectedFileChanges": [
-        "server/routes/items.js",
-        "public/js/items.js",
-        "public/js/app.js",
-        "server/test.js"
-      ]
-    },
-    "sourceSessionId": "8a0a1609-9031-4bcc-86ff-e690fef3b290",
-    "classifiedType": "coding:generation",
-    "expectedDiff": null,
-    "complexityIndicator": 4
-  },
-  {
-    "id": "agent-session-63ada37e",
-    "prompt": "I'll help you rename the function 'processData' to 'transformPayload' across the entire project. Let me start by exploring the project structure and finding all usages.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "uses edit over write"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "list_directory",
-        "read_file",
-        "grep_files",
-        "run_shell",
-        "ask_user",
-        "edit_file",
-        "lsp_references"
-      ],
-      "maxTurns": 50,
-      "expectedFileChanges": [
-        "server.js",
-        "test.js"
-      ]
-    },
-    "sourceSessionId": "a7b388ad-58e9-48ea-9278-b817ba300726",
-    "classifiedType": "coding:refactoring",
-    "expectedDiff": null,
-    "complexityIndicator": 2
-  },
-  {
-    "id": "agent-session-3a3b9214",
-    "prompt": "I'll start by exploring the project structure to understand the Express app and then implement rate limiting and CSRF protection.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "uses edit over write"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "list_directory",
-        "read_file",
-        "edit_file",
-        "run_shell"
-      ],
-      "maxTurns": 36,
-      "expectedFileChanges": [
-        "package.json",
-        "server.js",
-        "test.js"
-      ]
-    },
-    "sourceSessionId": "4a7b82b6-88ce-4bb3-a41a-d5ba58f2c83b",
-    "classifiedType": "coding:generation",
-    "expectedDiff": null,
-    "complexityIndicator": 3
-  },
-  {
-    "id": "agent-session-c28e561a",
-    "prompt": "I'll help you add a test for pagination with 25 users. Let me start by exploring the project structure to understand the codebase.",
-    "taskType": "coding-agent",
-    "difficulty": "medium",
-    "expectedTraits": [
-      "reads before writing",
-      "uses edit over write"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "list_directory",
-        "read_file",
-        "edit_file",
-        "run_shell"
-      ],
-      "maxTurns": 14,
-      "expectedFileChanges": [
-        "server.js",
-        "test.js"
-      ]
-    },
-    "sourceSessionId": "91ccf531-0e27-45be-880b-9e37fd8b8349",
-    "classifiedType": "coding:testing",
-    "expectedDiff": null,
-    "complexityIndicator": 2
-  },
-  {
-    "id": "agent-session-ecc4ff3f",
-    "prompt": "I'll start by exploring the project structure to understand the codebase and locate the relevant files.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "uses edit over write"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "list_directory",
-        "read_file",
-        "run_shell",
-        "edit_file"
-      ],
-      "maxTurns": 48,
-      "expectedFileChanges": [
-        "brain.js",
-        "test.js"
-      ]
-    },
-    "sourceSessionId": "aeae6b54-c335-4fe4-a903-3e6b440fbe66",
-    "classifiedType": "coding:generation",
-    "expectedDiff": null,
-    "complexityIndicator": 2
-  },
-  {
-    "id": "agent-session-a29571f9",
-    "prompt": "I'll help you add tests for both ascending and descending sort. Let me start by exploring the project structure.",
-    "taskType": "coding-agent",
-    "difficulty": "medium",
-    "expectedTraits": [
-      "reads before writing",
-      "uses edit over write"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "list_directory",
-        "read_file",
-        "edit_file",
-        "run_shell"
-      ],
-      "maxTurns": 14,
-      "expectedFileChanges": [
-        "server.js",
-        "test.js"
-      ]
-    },
-    "sourceSessionId": "bdc47279-f629-465b-bc8e-eaa22cb65266",
-    "classifiedType": "coding:testing",
-    "expectedDiff": null,
-    "complexityIndicator": 2
-  },
-  {
-    "id": "agent-session-8abb12e4",
-    "prompt": "I'll start by exploring the project structure to understand the codebase.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "uses edit over write"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "list_directory",
-        "read_file",
-        "edit_file",
-        "run_shell"
-      ],
-      "maxTurns": 50,
-      "expectedFileChanges": [
-        "server/routes/items.js",
-        "public/js/items.js",
-        "public/js/app.js",
-        "public/css/style.css",
-        "server/test.js",
-        "server/index.js"
-      ]
-    },
-    "sourceSessionId": "d25075ab-ec95-4e1d-813d-79c5dbf5bd68",
-    "classifiedType": "coding:generation",
-    "expectedDiff": null,
-    "complexityIndicator": 6
-  },
-  {
-    "id": "agent-session-cfd12d6c",
-    "prompt": "I'll help you convert this monorepo from CommonJS to ES Modules. Let me start by exploring the project structure.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "plans before executing"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "list_directory",
-        "read_file",
-        "run_shell",
-        "update_todos",
-        "edit_file",
-        "write_file"
-      ],
-      "maxTurns": 50,
-      "expectedFileChanges": [
-        "package.json",
-        "packages/shared/package.json",
-        "packages/api/package.json",
-        "packages/cli/package.json",
-        "packages/shared/formatters.js",
-        "packages/shared/validators.js",
-        "packages/shared/index.js",
-        "packages/shared/test.js",
-        "packages/api/data.js",
-        "packages/api/middleware.js",
-        "packages/api/routes.js",
-        "packages/api/server.js",
-        "packages/api/test.js",
-        "packages/cli/output.js",
-        "packages/cli/commands.js",
-        "packages/cli/index.js",
-        "packages/cli/test.js",
-        "test.js"
-      ]
-    },
-    "sourceSessionId": "ab4c19bb-d588-43dd-994b-fdf05431e525",
-    "classifiedType": "coding:generation",
-    "expectedDiff": null,
-    "complexityIndicator": 18
-  },
-  {
-    "id": "agent-session-f21db69e",
-    "prompt": "I'll start by exploring the project structure to understand the Flask app setup.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "uses edit over write"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "list_directory",
-        "read_file",
-        "edit_file",
-        "run_shell"
-      ],
-      "maxTurns": 30,
-      "expectedFileChanges": [
-        "app.py",
-        "test_app.py"
-      ]
-    },
-    "sourceSessionId": "d44e8d0e-ef75-4f49-8340-2c4628858fe9",
-    "classifiedType": "coding:generation",
-    "expectedDiff": null,
-    "complexityIndicator": 2
-  },
-  {
-    "id": "agent-session-e8eb3472",
-    "prompt": "I'll help you add a test for pagination with 25 users. Let me start by exploring the project structure.",
-    "taskType": "coding-agent",
-    "difficulty": "medium",
-    "expectedTraits": [
-      "reads before writing",
-      "uses edit over write"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "list_directory",
-        "read_file",
-        "edit_file",
-        "run_shell"
-      ],
-      "maxTurns": 14,
-      "expectedFileChanges": [
-        "server.js",
-        "test.js"
-      ]
-    },
-    "sourceSessionId": "991abf0f-4bb1-48fa-8aa8-ddd97d45d6f8",
-    "classifiedType": "coding:testing",
-    "expectedDiff": null,
-    "complexityIndicator": 2
-  },
-  {
-    "id": "agent-session-bd5f99de",
-    "prompt": "I'll start by exploring the project structure and understanding the codebase, then run diagnostics to find type-related issues.",
-    "taskType": "coding-agent",
-    "difficulty": "hard",
-    "expectedTraits": [
-      "reads before writing",
-      "uses edit over write"
-    ],
-    "agentExpectations": {
-      "expectedToolCalls": [
-        "list_directory",
-        "read_file",
-        "lsp_diagnostics",
-        "run_shell",
-        "edit_file"
-      ],
-      "maxTurns": 32,
-      "expectedFileChanges": [
-        "src/handlers.js",
-        "src/utils.js"
-      ]
-    },
-    "sourceSessionId": "abeceb68-cda7-4791-a201-b0d8e00bbb90",
-    "classifiedType": "coding:generation",
-    "expectedDiff": null,
-    "complexityIndicator": 2
   }
 ]