create-walle 0.9.11 → 0.9.13

package/template/wall-e/eval/eval-orchestrator.js

@@ -8,6 +8,7 @@ const { pLimit, getAvailableProviders } = require('./head-to-head');
 const { runAgentBenchmark, runMultiTurnBenchmark } = require('./agent-runner');
 const { createClient } = require('../llm/client');
 const { createAnthropicFromEnv } = require('../llm/anthropic');
+const { decorateBenchmarkResult, DEFAULT_SCORER_VERSION } = require('./manifest');
 
 // ============================================================
 // Benchmark suite loader
@@ -22,14 +23,89 @@ const SUITE_FILES = {
   'chat-eval':    'chat-eval.json',
   'reasoning':    'reasoning.json',
   'memory-retrieval': 'memory-retrieval.json',
-  'humaneval-plus': null,    // handled by humaneval-adapter.js (no JSON file)
-  'livecodebench':  null,    // handled by livecodebench-adapter.js (no JSON file)
+};
+
+const ADAPTER_SUITE_METADATA = {
+  'humaneval-plus': {
+    name: 'humaneval-plus',
+    count: null,
+    taskTypes: ['coding'],
+    difficulties: ['easy', 'medium', 'hard'],
+    adapter: true,
+    description: 'EvalPlus HumanEval+ Python function-generation tasks',
+  },
+  livecodebench: {
+    name: 'livecodebench',
+    count: null,
+    taskTypes: ['coding'],
+    difficulties: ['easy', 'medium', 'hard'],
+    adapter: true,
+    description: 'LiveCodeBench code-generation tasks with date filtering',
+  },
+  'swebench-lite': {
+    name: 'swebench-lite',
+    count: 30,
+    taskTypes: ['coding-agent'],
+    difficulties: ['medium', 'hard'],
+    adapter: true,
+    description: 'Curated SWE-bench Lite issue-fixing tasks',
+  },
+};
+
+function isAdapterSuite(suiteName) {
+  return !!ADAPTER_SUITE_METADATA[suiteName];
+}
+
+function listAdapterSuites() {
+  return Object.values(ADAPTER_SUITE_METADATA).map((s) => ({ ...s }));
+}
+
+const DEFAULT_ADAPTER_RUNNERS = {
+  'humaneval-plus': async ({ brain, providerInfo, model, runId, maxTasks, signal }) => {
+    const { runHumanEvalSuite } = require('./humaneval-adapter');
+    return runHumanEvalSuite({
+      brain,
+      providerType: providerInfo.provider,
+      config: { apiKey: providerInfo.apiKey, baseUrl: providerInfo.baseUrl },
+      model,
+      runId,
+      maxTasks,
+      signal,
+    });
+  },
+  livecodebench: async ({ brain, providerInfo, model, runId, maxTasks, afterDate, signal }) => {
+    const { runLiveCodeBenchSuite } = require('./livecodebench-adapter');
+    return runLiveCodeBenchSuite({
+      brain,
+      providerType: providerInfo.provider,
+      config: { apiKey: providerInfo.apiKey, baseUrl: providerInfo.baseUrl },
+      model,
+      runId,
+      maxTasks,
+      afterDate,
+      signal,
+    });
+  },
+  'swebench-lite': async ({ brain, providerInfo, model, runId, maxTasks, signal, runAgentLoop, timeoutMs }) => {
+    const { runSWEBenchSuite } = require('./swebench-adapter');
+    return runSWEBenchSuite({
+      brain,
+      provider: providerInfo.client,
+      providerType: providerInfo.provider,
+      model,
+      runId,
+      maxTasks,
+      signal,
+      runAgentLoop,
+      timeoutMs,
+    });
+  },
 };
 
 function loadSuite(suiteName) {
+  if (isAdapterSuite(suiteName)) return [];
   if (!(suiteName in SUITE_FILES)) throw new Error(`Unknown suite: ${suiteName}`);
   const file = SUITE_FILES[suiteName];
-  if (!file) return []; // adapter-based suite (humaneval-plus, livecodebench) — loaded externally
   const filePath = path.join(BENCHMARKS_DIR, file);
   if (!fs.existsSync(filePath)) throw new Error(`Suite file not found: ${filePath}`);
   return JSON.parse(fs.readFileSync(filePath, 'utf8'));
@@ -83,6 +159,7 @@ class EvalOrchestrator extends EventEmitter {
     this.totalSpent = 0;
     this.running = false;
     this.aborted = false;
+    this.adapterRunners = { ...DEFAULT_ADAPTER_RUNNERS, ...(options.adapterRunners || {}) };
   }
 
   /**
@@ -94,12 +171,16 @@ class EvalOrchestrator extends EventEmitter {
    * @param {Function} params.runAgentLoop - Agent loop function for coding benchmarks
    * @returns {Promise<object>} Run summary
    */
-  async run({ suite, models, benchmarkIds, runAgentLoop }) {
+  async run({ suite, models, benchmarkIds, runAgentLoop, maxTasks, afterDate }) {
     if (this.running) throw new Error('Orchestrator is already running');
     this.running = true;
     this.aborted = false;
 
     try {
+      if (isAdapterSuite(suite)) {
+        return await this._runAdapterSuite({ suite, models, benchmarkIds, runAgentLoop, maxTasks, afterDate });
+      }
+
       // 1. Load benchmarks
       const benchmarks = suite === 'all' ? loadAllSuites() : loadSuite(suite);
       const filtered = benchmarkIds && benchmarkIds.length > 0
@@ -330,7 +411,7 @@ class EvalOrchestrator extends EventEmitter {
         } else {
           client = createClient(providerType, { apiKey, baseUrl });
         }
-        map[model] = { client, provider: providerType, registryId };
+        map[model] = { client, provider: providerType, registryId, apiKey, baseUrl };
       } catch (err) {
         this.emit('error', { benchmarkId: null, model, error: `Failed to create client: ${err.message}` });
       }
@@ -338,6 +419,102 @@ class EvalOrchestrator extends EventEmitter {
     return map;
   }
 
+  async _runAdapterSuite({ suite, models, benchmarkIds, runAgentLoop, maxTasks, afterDate }) {
+    const runner = this.adapterRunners[suite];
+    if (!runner) throw new Error(`No adapter runner configured for suite: ${suite}`);
+
+    if (!models || models.length === 0) {
+      const summary = { runId: this.runId, status: 'error', error: 'No models specified', models: {}, totalBenchmarks: 0, totalSpent: 0 };
+      this.emit('error', { benchmarkId: null, model: null, error: 'No models specified. Provide at least one model.' });
+      this.emit('run-complete', { runId: this.runId, summary });
+      return summary;
+    }
+
+    const providerMap = this._resolveProviders(models);
+    const adapterMaxTasks = maxTasks || (benchmarkIds && benchmarkIds.length ? benchmarkIds.length : undefined);
+    const limit = pLimit(this.concurrency);
+    const modelResults = {};
+
+    const orchestrator = this;
+    const adapterSignal = { get aborted() { return orchestrator.aborted; } };
+    const tasks = models.map((model) => limit(async () => {
+      const providerInfo = providerMap[model];
+      if (!providerInfo) {
+        this.emit('error', { benchmarkId: suite, model, error: `No provider found for model: ${model}` });
+        return null;
+      }
+
+      this.emit('benchmark-start', {
+        benchmarkId: suite,
+        model,
+        startedAt: new Date().toISOString(),
+      });
+
+      let result;
+      try {
+        result = await runner({
+          brain: this.brain,
+          providerInfo,
+          model,
+          runId: this.runId,
+          maxTasks: adapterMaxTasks,
+          afterDate,
+          signal: adapterSignal,
+          runAgentLoop,
+          timeoutMs: this.timeoutMs,
+        });
+      } catch (err) {
+        result = {
+          suite,
+          model,
+          totalTasks: 0,
+          avgScore: 0,
+          totalCost: 0,
+          error: err.message,
+          results: [],
+        };
+        this.emit('error', { benchmarkId: suite, model, error: err.message });
+      }
+
+      const totalCost = result.totalCost || 0;
+      if (!isLocalModel(providerInfo.provider)) {
+        this.spent[model] = (this.spent[model] || 0) + totalCost;
+        this.totalSpent += totalCost;
+      }
+      modelResults[model] = result;
+      this.emit('model-complete', {
+        model,
+        avgScore: Math.round((result.avgScore || 0) * 1000) / 1000,
+        totalCost: Math.round(totalCost * 1_000_000) / 1_000_000,
+        benchmarksRun: result.totalTasks || result.results?.length || 0,
+      });
+      return result;
+    }));
+
+    await Promise.all(tasks);
+
+    const summary = {
+      runId: this.runId,
+      status: this.aborted ? 'aborted' : 'complete',
+      totalBenchmarks: Object.values(modelResults).reduce((s, r) => s + (r.totalTasks || r.results?.length || 0), 0),
+      totalSpent: Math.round(this.totalSpent * 1_000_000) / 1_000_000,
+      models: {},
+      suite,
+      adapter: true,
+    };
+    for (const [model, result] of Object.entries(modelResults)) {
+      summary.models[model] = {
+        avgScore: Math.round((result.avgScore || 0) * 1000) / 1000,
+        totalCost: Math.round((result.totalCost || 0) * 1_000_000) / 1_000_000,
+        benchmarksRun: result.totalTasks || result.results?.length || 0,
+        errors: (result.results || []).filter(r => r.error).length + (result.error ? 1 : 0),
+      };
+    }
+
+    this.emit('run-complete', { runId: this.runId, summary });
+    return summary;
+  }
+
   /**
    * Get set of completed benchmark keys for this runId (for resume).
    */
@@ -363,7 +540,10 @@ class EvalOrchestrator extends EventEmitter {
     if (!this.brain || typeof this.brain.insertBenchmarkResult !== 'function') return;
 
     try {
-      this.brain.insertBenchmarkResult({
+      const scoringMethod = item.benchmark.agentExpectations?.testCommand
+        ? 'agent-rubric+tests'
+        : 'agent-rubric';
+      this.brain.insertBenchmarkResult(decorateBenchmarkResult({
         runId: this.runId,
         suite: item.benchmark._suite || 'coding-agent',
         promptId: item.benchmark.id,
@@ -384,7 +564,31 @@ class EvalOrchestrator extends EventEmitter {
         dimensionsJson: result.dimensionsJson || null,
         inputTokens: result.inputTokens ?? null,
         outputTokens: result.outputTokens ?? null,
-      });
+        scorerVersion: DEFAULT_SCORER_VERSION,
+        scoringMethod,
+        trusted: !result.error && result.testsPassed === true,
+        runConfig: {
+          timeoutMs: this.timeoutMs,
+          concurrency: this.concurrency,
+          budgetDollars: this.budgetDollars,
+          scoringMethod,
+        },
+      }, {
+        suite: item.benchmark._suite || 'coding-agent',
+        benchmark: item.benchmark,
+        runId: this.runId,
+        provider: item.provider.provider,
+        model: item.model,
+        scoringMethod,
+        scorerVersion: DEFAULT_SCORER_VERSION,
+        trusted: !result.error && result.testsPassed === true,
+        runConfig: {
+          timeoutMs: this.timeoutMs,
+          concurrency: this.concurrency,
+          budgetDollars: this.budgetDollars,
+          scoringMethod,
+        },
+      }));
     } catch { /* non-fatal */ }
   }
 
@@ -427,4 +631,10 @@ class EvalOrchestrator extends EventEmitter {
   }
 }
 
-module.exports = { EvalOrchestrator };
+module.exports = {
+  EvalOrchestrator,
+  ADAPTER_SUITE_METADATA,
+  isAdapterSuite,
+  listAdapterSuites,
+  loadSuite,
+};

package/template/wall-e/eval/harvester.js

@@ -5,6 +5,17 @@ const path = require('path');
 const { createHash } = require('crypto');
 const { execFileSync } = require('child_process');
 
+let claudeDesktopSessions = null;
+function getClaudeDesktopSessions() {
+  if (claudeDesktopSessions) return claudeDesktopSessions;
+  try {
+    claudeDesktopSessions = require('../../claude-task-manager/lib/claude-desktop-sessions');
+  } catch {
+    return null;
+  }
+  return claudeDesktopSessions;
+}
+
 // --- Task type classification ---
 
 function classifyTaskType(content) {
@@ -86,6 +97,46 @@ async function harvestClaudeCodeSessions(since) {
   return samples;
 }
 
+// --- Claude Desktop Session Harvesting ---
+
+async function harvestClaudeDesktopSessions(since) {
+  const reader = getClaudeDesktopSessions();
+  if (!reader) return [];
+
+  const sessions = reader.listSessions();
+  const samples = [];
+
+  for (const session of sessions) {
+    if (since && session.updatedAt && session.updatedAt <= since) continue;
+    const messages = Array.isArray(session.messages) ? session.messages : [];
+    for (let i = 0; i < messages.length - 1; i++) {
+      const userMsg = messages[i];
+      const assistantMsg = messages[i + 1];
+      if (userMsg.role !== 'user' || assistantMsg.role !== 'assistant') continue;
+      const userContent = userMsg.text || '';
+      const assistantContent = assistantMsg.text || '';
+      if (!userContent || userContent.length < 20) continue;
+      if (!assistantContent || assistantContent.length < 20) continue;
+
+      samples.push({
+        id: contentHash('claude-desktop', `${session.uuid}:${i}:${userContent}`),
+        source: 'claude-desktop',
+        session_id: session.uuid,
+        timestamp: userMsg.timestamp || session.updatedAt || session.createdAt || new Date().toISOString(),
+        task_type: classifyTaskType(userContent),
+        prompt: userContent,
+        response: assistantContent,
+        tool_calls: [],
+        outcome: 'unknown',
+        outcome_signal: { git_committed: false, git_diff: null, task_status: null, user_corrected: false },
+        model: session.model || 'unknown',
+        quality_label: 0.5,
+      });
+    }
+  }
+  return samples;
+}
+
 // --- Codex Session Harvesting ---
 
 async function harvestCodexSessions(since) {
@@ -144,12 +195,13 @@ async function harvestCodexSessions(since) {
 
 // --- CTM Session Harvesting ---
 
-async function harvestCtmSessions(since) {
-  const dataDir = process.env.WALL_E_DATA_DIR || path.join(process.env.HOME, '.walle', 'data');
+async function harvestCtmSessions(since, dataDirOverride = null) {
+  const dataDir = dataDirOverride || process.env.WALL_E_DATA_DIR || path.join(process.env.HOME, '.walle', 'data');
   const ctmDbPath = path.join(dataDir, 'task-manager.db');
   if (!fs.existsSync(ctmDbPath)) return [];
 
-  const Database = require('better-sqlite3');
+  let Database;
+  try { Database = require('better-sqlite3'); } catch { return []; }
   let ctmDb;
   try {
     ctmDb = new Database(ctmDbPath, { readonly: true, fileMustExist: true });
@@ -534,10 +586,11 @@ async function runHarvest({ incremental = true, brain, dataDir } = {}) {
 
   // Harvest from each source
   const claudeSamples = await harvestClaudeCodeSessions(getSince('claude-code'));
+  const claudeDesktopSamples = await harvestClaudeDesktopSessions(getSince('claude-desktop'));
   const codexSamples = await harvestCodexSessions(getSince('codex'));
   const chatSamples = await harvestWalleChat(brain, getSince('walle-chat'));
   const taskSamples = await harvestWalleTasks(brain, getSince('walle-task'));
-  const ctmSamples = await harvestCtmSessions(getSince('ctm-sessions'));
+  const ctmSamples = await harvestCtmSessions(getSince('ctm-sessions'), dataDir);
 
   // Harvest coding agent sessions and store in brain
   const codingAgentSessions = await harvestCodingAgentSessions(getSince('coding-agent'));
@@ -549,7 +602,7 @@ async function runHarvest({ incremental = true, brain, dataDir } = {}) {
     if (typeof brain.insertCodingSession === 'function') brain.insertCodingSession(session);
   }
 
-  allSamples.push(...claudeSamples, ...codexSamples, ...chatSamples, ...taskSamples, ...ctmSamples);
+  allSamples.push(...claudeSamples, ...claudeDesktopSamples, ...codexSamples, ...chatSamples, ...taskSamples, ...ctmSamples);
 
   // Deduplicate by content hash
   const seen = new Set();
@@ -584,6 +637,9 @@ async function runHarvest({ incremental = true, brain, dataDir } = {}) {
     if (claudeSamples.length > 0) {
       brain.updateHarvestState('claude-code', { lastProcessedAt: now, totalHarvested: claudeSamples.length });
     }
+    if (claudeDesktopSamples.length > 0) {
+      brain.updateHarvestState('claude-desktop', { lastProcessedAt: now, totalHarvested: claudeDesktopSamples.length });
+    }
     if (codexSamples.length > 0) {
       brain.updateHarvestState('codex', { lastProcessedAt: now, totalHarvested: codexSamples.length });
     }
@@ -610,6 +666,7 @@ module.exports = {
   classifyTaskType,
   contentHash,
   harvestClaudeCodeSessions,
+  harvestClaudeDesktopSessions,
   harvestCodexSessions,
   harvestCtmSessions,
   harvestWalleChat,

package/template/wall-e/eval/head-to-head.js

@@ -3,7 +3,7 @@
 const { randomUUID } = require('node:crypto');
 const { createClient } = require('../llm/client');
 const { heuristicScore } = require('./evaluator');
-const { scoreTraits } = require('./benchmarks');
+const { scoreTraitsDetailed, TRAIT_MATCHERS, UNSCORABLE_TRAITS } = require('./benchmarks');
 
 // ============================================================
 // Concurrency limiter (inline, no external dependency)
@@ -122,7 +122,28 @@ function getAvailableProviders(brain) {
 
 function traitScore(response, expectedTraits) {
   if (!expectedTraits || expectedTraits.length === 0) return null;
-  return scoreTraits(response, expectedTraits);
+  const text = String(response || '').toLowerCase();
+  let matched = 0;
+  let scored = 0;
+
+  const knownTraits = [];
+  for (const trait of expectedTraits) {
+    if (UNSCORABLE_TRAITS.has(trait)) continue;
+    if (TRAIT_MATCHERS[trait]) {
+      knownTraits.push(trait);
+      continue;
+    }
+    scored++;
+    if (text.includes(String(trait).toLowerCase())) matched++;
+  }
+
+  if (knownTraits.length > 0) {
+    const detail = scoreTraitsDetailed(response, knownTraits);
+    matched += detail.matched.length;
+    scored += detail.scoredCount;
+  }
+
+  return scored > 0 ? matched / scored : 0;
 }
 
 // ============================================================

package/template/wall-e/eval/humaneval-adapter.js

@@ -6,6 +6,7 @@ const crypto = require('crypto');
 const { execFileSync } = require('child_process');
 const { createClient } = require('../llm/client');
 const { resolveModelName } = require('./agent-runner');
+const { decorateBenchmarkResult, DEFAULT_SCORER_VERSION } = require('./manifest');
 
 const SUITE_NAME = 'humaneval-plus';
 const CACHE_DIR = path.join(os.homedir(), '.walle', 'eval-cache');
@@ -192,7 +193,8 @@ async function runHumanEvalTask(task, options = {}) {
     costDollars,
     response: response.slice(0, 2000),
     code: code.slice(0, 2000),
-    error: error || testError || null,
+    error: error || null,
+    testError: testError || null,
     usage,
   };
 }
@@ -222,13 +224,13 @@ function scoreHumanEvalQuality(code) {
  * @returns {Promise<object>} Suite results
  */
 async function runHumanEvalSuite(options = {}) {
-  const { brain, providerType, config, model, maxTasks, signal } = options;
+  const { brain, providerType, config, model, maxTasks, signal, runId: providedRunId } = options;
 
   const allTasks = await loadHumanEvalDataset();
   const tasks = maxTasks ? allTasks.slice(0, maxTasks) : allTasks;
 
   const client = createClient(providerType || 'anthropic', config || {});
-  const runId = crypto.randomUUID();
+  const runId = providedRunId || crypto.randomUUID();
   const results = [];
   let totalPassed = 0;
 
@@ -247,7 +249,8 @@ async function runHumanEvalSuite(options = {}) {
     // Store in brain
     if (brain && typeof brain.insertBenchmarkResult === 'function') {
       try {
-        brain.insertBenchmarkResult({
+        const scoringMethod = 'executable-tests';
+        brain.insertBenchmarkResult(decorateBenchmarkResult({
           runId,
           suite: SUITE_NAME,
           promptId: task.task_id,
@@ -266,7 +269,29 @@ async function runHumanEvalSuite(options = {}) {
           testsAfter: result.passed ? 1 : 0,
           totalTests: 1,
           dimensionsJson: JSON.stringify(result.score.dimensions),
-        });
+          modelMetadataJson: JSON.stringify({ testError: result.testError || null }),
+          datasetVersion: 'humaneval-plus:evalplus-master',
+          scorerVersion: DEFAULT_SCORER_VERSION,
+          scoringMethod,
+          trusted: !result.error,
+          runConfig: { maxTasks, scoringMethod },
+        }, {
+          suite: SUITE_NAME,
+          benchmark: {
+            id: task.task_id,
+            prompt: task.prompt,
+            taskType: 'coding',
+            difficulty: taskDifficulty(task.task_id),
+            datasetVersion: 'humaneval-plus:evalplus-master',
+          },
+          runId,
+          provider: providerType || 'unknown',
+          model: resolveModelName(model),
+          scorerVersion: DEFAULT_SCORER_VERSION,
+          scoringMethod,
+          trusted: !result.error,
+          runConfig: { maxTasks, scoringMethod },
+        }));
       } catch {}
     }
   }

package/template/wall-e/eval/livecodebench-adapter.js

@@ -7,6 +7,7 @@ const crypto = require('crypto');
 const { execFileSync } = require('child_process');
 const { createClient } = require('../llm/client');
 const { resolveModelName } = require('./agent-runner');
+const { decorateBenchmarkResult, DEFAULT_SCORER_VERSION } = require('./manifest');
 
 const SUITE_NAME = 'livecodebench';
 const CACHE_DIR = path.join(os.homedir(), '.walle', 'eval-cache');
@@ -187,7 +188,7 @@ async function runLiveCodeBenchTask(task, options = {}) {
  * Run the full LiveCodeBench suite.
  */
 async function runLiveCodeBenchSuite(options = {}) {
-  const { brain, providerType, config, model, maxTasks, signal, afterDate } = options;
+  const { brain, providerType, config, model, maxTasks, signal, afterDate, runId: providedRunId } = options;
 
   let allTasks = await loadLiveCodeBenchDataset();
 
@@ -201,7 +202,7 @@ async function runLiveCodeBenchSuite(options = {}) {
 
   const tasks = maxTasks ? allTasks.slice(0, maxTasks) : allTasks;
   const client = createClient(providerType || 'anthropic', config || {});
-  const runId = crypto.randomUUID();
+  const runId = providedRunId || crypto.randomUUID();
   const results = [];
   let totalPassed = 0;
 
@@ -219,7 +220,9 @@ async function runLiveCodeBenchSuite(options = {}) {
     // Store in brain
     if (brain && typeof brain.insertBenchmarkResult === 'function') {
       try {
-        brain.insertBenchmarkResult({
+        const scoringMethod = 'executable-tests';
+        const prompt = task.question_content || task.prompt || task.description || '';
+        brain.insertBenchmarkResult(decorateBenchmarkResult({
           runId,
           suite: SUITE_NAME,
           promptId: result.taskId,
@@ -227,7 +230,7 @@ async function runLiveCodeBenchSuite(options = {}) {
           difficulty: taskDifficulty(task),
           provider: providerType || 'unknown',
           model: resolveModelName(model),
-          prompt: (task.question_content || '').slice(0, 2000),
+          prompt: prompt.slice(0, 2000),
           response: result.response || '',
           traitScore: null,
           compositeScore: result.score.composite,
@@ -238,7 +241,28 @@ async function runLiveCodeBenchSuite(options = {}) {
           testsAfter: result.passedCases || 0,
           totalTests: result.totalCases || 0,
           dimensionsJson: JSON.stringify(result.score.dimensions),
-        });
+          datasetVersion: 'livecodebench:release_v6',
+          scorerVersion: DEFAULT_SCORER_VERSION,
+          scoringMethod,
+          trusted: !result.error,
+          runConfig: { maxTasks, afterDate, scoringMethod },
+        }, {
+          suite: SUITE_NAME,
+          benchmark: {
+            id: result.taskId,
+            prompt,
+            taskType: 'coding',
+            difficulty: taskDifficulty(task),
+            datasetVersion: 'livecodebench:release_v6',
+          },
+          runId,
+          provider: providerType || 'unknown',
+          model: resolveModelName(model),
+          scorerVersion: DEFAULT_SCORER_VERSION,
+          scoringMethod,
+          trusted: !result.error,
+          runConfig: { maxTasks, afterDate, scoringMethod },
+        }));
       } catch {}
     }
   }