create-walle 0.9.13 → 0.9.15

package/template/wall-e/eval/benchmark-generator.js

@@ -18,11 +18,31 @@ function classifyCodingType(session) {
   return 'coding:generation';
 }
 
+function isReplayableBenchmarkPrompt(prompt) {
+  const text = String(prompt || '').trim();
+  if (text.length < 25) return false;
+
+  // Session-mined prompts must be the user's task, not the assistant's first
+  // progress narration. Assistant prose turns the benchmark into "continue the
+  // previous assistant's work", which is not replayable from a fresh sandbox.
+  if (/^(i('|’)ll|i will|i can|i('|’)m going to|let me|sure[, ]|happy to help|i('|’)ll help|i('|’)ll start)\b/i.test(text)) {
+    return false;
+  }
+
+  if (/^\s*(go ahead|continue|proceed|do it|yes|yep|ok|okay|thanks|thank you)\b/i.test(text)) {
+    return false;
+  }
+
+  return /\b(fix|implement|add|change|update|refactor|test|debug|make|write|delete|remove|harden|wire|bug|failing|error|regression|feature|endpoint|api|ui|server|component|code review|review.*code)\b/i.test(text);
+}
+
 /**
  * Convert a coding agent session object to a benchmark entry.
  * Returns null if the session would be a duplicate (id already in existingIds).
  */
 function sessionToBenchmark(session, existingIds = new Set()) {
+  if (!isReplayableBenchmarkPrompt(session.prompt)) return null;
+
   const id = `agent-session-${crypto.createHash('sha256').update(session.prompt || '').digest('hex').slice(0, 8)}`;
   if (existingIds.has(id)) return null; // dedup
 
@@ -90,4 +110,4 @@ async function generateBenchmarks(brain, { minSignificance = 0.5, limit = 50 } =
   return benchmarks;
 }
 
-module.exports = { classifyCodingType, sessionToBenchmark, generateBenchmarks };
+module.exports = { classifyCodingType, isReplayableBenchmarkPrompt, sessionToBenchmark, generateBenchmarks };

package/template/wall-e/eval/benchmarks/chat-eval.json

@@ -89,13 +89,13 @@
   },
   {
     "id": "ce-B2",
-    "prompt": "Who is Sarah Chen and what's her role?",
+    "prompt": "Who is Alex Example and what's their role?",
     "taskType": "chat",
     "difficulty": "medium",
     "category": "knowledge",
     "expectedIntent": "knowledge",
     "expectedTools": ["search_memories"],
-    "forbiddenTools": ["calendar_events", "run_shell"],
+    "forbiddenTools": ["calendar_events", "run_shell", "web_fetch", "slack_search"],
     "maxToolCalls": 5,
     "latencyBudgetMs": 15000,
     "mockToolResults": {
@@ -105,18 +105,18 @@
         "total_slack": 400,
         "search_method": "fts5_bm25",
         "memories": [
-          {"source": "slack", "channel": "general", "content": "Sarah Chen mentioned she's leading the platform team restructuring. She reports directly to the VP of Engineering.", "timestamp": "2026-04-07T11:00:00Z"},
-          {"source": "slack", "channel": "engineering", "content": "1:1 with Sarah Chen — discussed Q2 roadmap priorities and hiring plan for 3 new engineers.", "timestamp": "2026-04-05T09:30:00Z"}
+          {"source": "slack", "channel": "general", "content": "Alex Example mentioned they're leading the platform team restructuring. They report directly to the VP of Engineering.", "timestamp": "2026-04-07T11:00:00Z"},
+          {"source": "slack", "channel": "engineering", "content": "1:1 with Alex Example — discussed Q2 roadmap priorities and hiring plan for 3 new engineers.", "timestamp": "2026-04-05T09:30:00Z"}
         ]
       },
       "lookup_person": {
-        "name": "Sarah Chen",
+        "name": "Alex Example",
         "relationship": "manager",
         "trust_level": 0.9,
         "notes": "VP Engineering, leads platform team"
       }
     },
-    "expectedInReply": ["sarah"],
+    "expectedInReply": ["alex"],
     "forbiddenInReply": [],
     "minReplyLength": 30,
     "maxReplyLength": 1500,
@@ -125,6 +125,66 @@
   },
   {
     "id": "ce-B3",
+    "prompt": "What did we decide last time about the MCP auto-config approach?",
+    "taskType": "chat",
+    "difficulty": "medium",
+    "category": "knowledge",
+    "expectedIntent": "knowledge",
+    "expectedTools": ["search_memories"],
+    "forbiddenTools": ["calendar_events", "run_shell", "web_fetch", "slack_search"],
+    "maxToolCalls": 5,
+    "latencyBudgetMs": 15000,
+    "mockToolResults": {
+      "search_memories": {
+        "count": 2,
+        "total_memories": 500,
+        "total_slack": 200,
+        "search_method": "fts5_bm25",
+        "memories": [
+          {"source": "ctm", "content": "Decision: MCP auto-config should be handled in the Wall-E install/startup path, not with a local-only Codex skill.", "timestamp": "2026-05-01T18:30:00Z"},
+          {"source": "ctm", "content": "Follow-up: add portable agent instructions for Claude and Codex so memory routing works for all npx users.", "timestamp": "2026-05-01T18:35:00Z"}
+        ]
+      }
+    },
+    "expectedInReply": ["auto-config", "portable"],
+    "forbiddenInReply": [],
+    "minReplyLength": 30,
+    "maxReplyLength": 1500,
+    "expectedTraits": ["references context"],
+    "tags": ["knowledge", "recall", "private-context"]
+  },
+  {
+    "id": "ce-B3-session-recall",
+    "prompt": "What was the parser regression session about and what should we do next?",
+    "taskType": "chat",
+    "difficulty": "medium",
+    "category": "knowledge",
+    "expectedIntent": "knowledge",
+    "expectedTools": ["search_memories"],
+    "forbiddenTools": ["calendar_events", "run_shell", "web_fetch", "slack_search"],
+    "maxToolCalls": 5,
+    "latencyBudgetMs": 15000,
+    "mockToolResults": {
+      "search_memories": {
+        "count": 2,
+        "total_memories": 500,
+        "total_slack": 200,
+        "search_method": "fts5_bm25",
+        "memories": [
+          {"source": "ctm", "content": "Session summary: Parser regression was caused by stale cached CTM session rows after restart.", "timestamp": "2026-05-01T20:00:00Z"},
+          {"source": "walle-diary", "content": "Next steps: verify restart recovery with CTM render tests and keep the session title from cached metadata.", "timestamp": "2026-05-01T20:05:00Z"}
+        ]
+      }
+    },
+    "expectedInReply": ["parser", "restart"],
+    "forbiddenInReply": [],
+    "minReplyLength": 30,
+    "maxReplyLength": 1500,
+    "expectedTraits": ["references context"],
+    "tags": ["knowledge", "session-recall", "private-context"]
+  },
+  {
+    "id": "ce-B4",
     "prompt": "What topics come up most in my slack conversations?",
     "taskType": "chat",
     "difficulty": "medium",