create-walle 0.9.21 → 0.9.23

package/template/wall-e/eval/context-compactor.js

@@ -1,251 +0,0 @@
-'use strict';
-
-/**
- * Smart context compaction for the coding agent.
- * Replaces blunt character truncation with semantic compaction.
- * Prioritizes: errors > test results > file reads > tool outputs > successes.
- */
-
-const PRIORITY = {
-  ERROR: 10,
-  TEST_RESULT: 9,
-  DIAGNOSTIC: 8,
-  FILE_READ: 5,
-  TOOL_OUTPUT: 4,
-  PLAN: 6,
-  SUCCESS_MSG: 2,
-  ASSISTANT_TEXT: 3,
-  DEFAULT: 3,
-};
-
-/**
- * Classify a message by its content to determine compaction priority.
- * @param {object} msg - { role, content, toolCall?, toolResult? }
- * @returns {number} Priority score (higher = keep longer)
- */
-function classifyMessage(msg) {
-  const content = typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content || '');
-  const toolName = msg.toolCall?.name || msg.tool_use?.name || '';
-
-  // Errors are highest priority — always keep
-  if (/error|Error|ERROR|FAIL|failed|exception|TypeError|SyntaxError|ReferenceError/i.test(content)) {
-    return PRIORITY.ERROR;
-  }
-
-  // Test results
-  if (/test|PASS|FAIL|✓|✗|assertion|expect/i.test(content) && /run_shell/.test(toolName)) {
-    return PRIORITY.TEST_RESULT;
-  }
-
-  // LSP diagnostics
-  if (/diagnostic|lsp_diagnostics|warning.*line/i.test(content) || toolName === 'lsp_diagnostics') {
-    return PRIORITY.DIAGNOSTIC;
-  }
-
-  // Planning
-  if (toolName === 'update_todos' || /plan|step\s+\d|todo/i.test(content)) {
-    return PRIORITY.PLAN;
-  }
-
-  // File reads
-  if (/read_file|glob|grep_files|list_directory/.test(toolName)) {
-    return PRIORITY.FILE_READ;
-  }
-
-  // General tool output
-  if (toolName) {
-    return PRIORITY.TOOL_OUTPUT;
-  }
-
-  // Success messages (low priority — can be summarized)
-  if (/success|done|complete|finished|created|updated/i.test(content) && content.length < 200) {
-    return PRIORITY.SUCCESS_MSG;
-  }
-
-  // Assistant text
-  if (msg.role === 'assistant') {
-    return PRIORITY.ASSISTANT_TEXT;
-  }
-
-  return PRIORITY.DEFAULT;
-}
-
-/**
- * Compact a message array to fit within a character budget.
- * Preserves high-priority messages, summarizes low-priority ones.
- *
- * @param {Array} messages - Array of { role, content, ... } messages
- * @param {number} budgetChars - Maximum total characters
- * @param {object} [options]
- * @param {number} [options.keepFirst] - Always keep first N messages (default: 2)
- * @param {number} [options.keepLast] - Always keep last N messages (default: 3)
- * @returns {Array} Compacted messages
- */
-function compactMessages(messages, budgetChars, options = {}) {
-  const { keepFirst = 2, keepLast = 3 } = options;
-
-  if (!messages || messages.length === 0) return [];
-
-  // Compute total size
-  const totalSize = messages.reduce((sum, m) => sum + messageSize(m), 0);
-  if (totalSize <= budgetChars) return messages; // no compaction needed
-
-  // Score and tag each message
-  const scored = messages.map((msg, idx) => ({
-    msg,
-    idx,
-    priority: classifyMessage(msg),
-    size: messageSize(msg),
-    protected: idx < keepFirst || idx >= messages.length - keepLast,
-  }));
-
-  // Protected messages are always kept
-  const protectedMsgs = scored.filter(s => s.protected);
-  const evictable = scored.filter(s => !s.protected);
-
-  // Sort evictable by priority ascending (lowest priority evicted first)
-  evictable.sort((a, b) => a.priority - b.priority);
-
-  let usedBudget = protectedMsgs.reduce((sum, s) => sum + s.size, 0);
-  const kept = new Set(protectedMsgs.map(s => s.idx));
-
-  // Add evictable messages in priority order until budget is exceeded
-  for (const item of evictable.reverse()) {
-    if (usedBudget + item.size <= budgetChars) {
-      kept.add(item.idx);
-      usedBudget += item.size;
-    } else {
-      // Try to include a truncated version for high-priority messages
-      if (item.priority >= PRIORITY.PLAN) {
-        const truncated = truncateMessage(item.msg, Math.max(200, budgetChars - usedBudget));
-        if (truncated) {
-          item.msg = truncated;
-          kept.add(item.idx);
-          usedBudget += messageSize(truncated);
-        }
-      }
-    }
-  }
-
-  // Rebuild in original order, inserting compaction markers where messages were dropped
-  const result = [];
-  let droppedCount = 0;
-  const scoredByIdx = new Map(scored.map(s => [s.idx, s]));
-
-  for (let i = 0; i < messages.length; i++) {
-    if (kept.has(i)) {
-      if (droppedCount > 0) {
-        result.push({
-          role: 'system',
-          content: `[${droppedCount} message${droppedCount > 1 ? 's' : ''} compacted]`,
-        });
-        droppedCount = 0;
-      }
-      const item = scoredByIdx.get(i);
-      result.push(item ? item.msg : messages[i]);
-    } else {
-      droppedCount++;
-    }
-  }
-
-  if (droppedCount > 0) {
-    result.push({
-      role: 'system',
-      content: `[${droppedCount} message${droppedCount > 1 ? 's' : ''} compacted]`,
-    });
-  }
-
-  return result;
-}
-
-/**
- * Compact a cumulative context string with semantic awareness.
- * @param {string} context - Raw cumulative context
- * @param {number} budgetChars - Max characters
- * @returns {string} Compacted context
- */
-function compactContext(context, budgetChars) {
-  if (!context || context.length <= budgetChars) return context;
-
-  // Split into sections by step markers
-  const sections = context.split(/(?=\n\[Step \d+)/);
-  if (sections.length <= 1) {
-    return context.slice(-budgetChars);
-  }
-
-  // Score sections
-  const scored = sections.map((section, idx) => ({
-    section,
-    idx,
-    priority: /error|fail|Error/i.test(section) ? PRIORITY.ERROR :
-              /test|PASS|FAIL/i.test(section) ? PRIORITY.TEST_RESULT :
-              PRIORITY.SUCCESS_MSG,
-    size: section.length,
-  }));
-
-  // Always keep last 2 sections and any error sections
-  let usedBudget = 0;
-  const kept = [];
-
-  // Keep last 2 sections always
-  const lastTwo = scored.slice(-2);
-  for (const s of lastTwo) {
-    kept.push(s);
-    usedBudget += s.size;
-  }
-
-  // Add error sections
-  for (const s of scored.slice(0, -2)) {
-    if (s.priority >= PRIORITY.ERROR && usedBudget + s.size <= budgetChars) {
-      kept.push(s);
-      usedBudget += s.size;
-    }
-  }
-
-  // Fill remaining budget with other sections (newest first)
-  for (const s of scored.slice(0, -2).reverse()) {
-    if (kept.includes(s)) continue;
-    if (usedBudget + s.size <= budgetChars) {
-      kept.push(s);
-      usedBudget += s.size;
-    }
-  }
-
-  // Sort by original index
-  kept.sort((a, b) => a.idx - b.idx);
-
-  const compactedCount = scored.length - kept.length;
-  let result = kept.map(s => s.section).join('');
-
-  if (compactedCount > 0) {
-    result = `[${compactedCount} earlier step${compactedCount > 1 ? 's' : ''} compacted]\n` + result;
-  }
-
-  // Final safety: if still over budget, fall back to tail truncation
-  if (result.length > budgetChars) {
-    result = result.slice(-budgetChars);
-  }
-
-  return result;
-}
-
-function messageSize(msg) {
-  const content = typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content || '');
-  return content.length;
-}
-
-function truncateMessage(msg, maxChars) {
-  const content = typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content || '');
-  if (content.length <= maxChars) return msg;
-  return {
-    ...msg,
-    content: content.slice(0, maxChars - 20) + '\n[...truncated]',
-  };
-}
-
-module.exports = {
-  PRIORITY,
-  classifyMessage,
-  compactMessages,
-  compactContext,
-};

package/template/wall-e/eval/debug-agent003.js

@@ -1,68 +0,0 @@
-#!/usr/bin/env node
-'use strict';
-const path = require('path');
-process.chdir(path.join(__dirname, '..'));
-
-// Load .env
-try {
-  const envPath = path.resolve(__dirname, '..', '..', '.env');
-  const lines = require('fs').readFileSync(envPath, 'utf8').split('\n');
-  for (const line of lines) {
-    const m = line.match(/^([A-Z_]+)=(.*)$/);
-    if (m && !process.env[m[1]]) process.env[m[1]] = m[2];
-  }
-} catch {}
-
-const { setupSandbox, cleanupSandbox } = require('./agent-runner');
-const { runAgentLoop } = require('../coding-orchestrator');
-const benchmarks = require('./benchmarks/coding-agent.json');
-
-const benchId = process.argv[2] || 'agent-003';
-const bench = benchmarks.find(b => b.id === benchId);
-if (!bench) { console.error('No benchmark:', benchId); process.exit(1); }
-
-(async () => {
-  const fixture = bench.agentExpectations?.projectFixture || 'express-basic';
-  const dir = setupSandbox(fixture);
-  console.log(`Benchmark: ${bench.id} | Fixture: ${fixture} | Sandbox: ${dir}`);
-
-  try {
-    const result = await runAgentLoop(bench.prompt, {
-      cwd: dir,
-      timeoutMs: 120000,
-      mode: 'build',
-      persistTranscript: false,
-      onProgress: (e) => {
-        if (e.message) {
-          const detail = e.detail ? ` ${JSON.stringify(e.detail).slice(0, 120)}` : '';
-          console.log(`  [${e.phase}/${e.step}] ${e.message}${detail}`);
-        }
-      }
-    });
-
-    console.log('\n=== RESULT ===');
-    console.log('Success:', result.success);
-    console.log('Turns:', (result.log || []).length);
-
-    for (const entry of (result.log || [])) {
-      const tools = (entry.toolCalls || []).map(t => t.name);
-      console.log(`  Turn ${entry.turn}: stop=${entry.stopReason} tools=[${tools.join(', ')}]`);
-      if (entry.content) console.log(`    "${(entry.content || '').slice(0, 150)}"`);
-    }
-
-    console.log('\nOutput:', (result.output || '').slice(0, 400));
-
-    const { execFileSync } = require('child_process');
-    const diff = execFileSync('git', ['diff', '--name-only', 'HEAD'], { cwd: dir, encoding: 'utf8' });
-    const untracked = execFileSync('git', ['ls-files', '--others', '--exclude-standard'], { cwd: dir, encoding: 'utf8' });
-    console.log('Modified:', diff.trim() || 'none');
-    console.log('Untracked:', untracked.trim() || 'none');
-
-    try {
-      execFileSync('node', ['test.js'], { cwd: dir, timeout: 10000, stdio: 'pipe' });
-      console.log('Tests: PASS');
-    } catch { console.log('Tests: FAIL'); }
-  } finally {
-    cleanupSandbox(dir);
-  }
-})().catch(e => { console.error(e.message); process.exit(1); });

package/template/wall-e/eval/diagnostics.js

@@ -1,216 +0,0 @@
-'use strict';
-
-/**
- * Per-session failure analysis — auto-classifies root causes
- * when any scoring dimension falls below threshold.
- *
- * Root causes: "doom_loop" | "no_edits" | "timeout" | "test_failure" | "path_error"
- */
-
-const crypto = require('crypto');
-
-const FAILING_THRESHOLD = 0.3;
-
-const ROOT_CAUSE_PATTERNS = [
-  { name: 'timeout',      test: (_seq, meta) => meta?.error?.includes('timeout') || meta?.error?.includes('TIMEOUT') },
-  { name: 'doom_loop',    test: (seq) => hasDoomLoop(seq) },
-  { name: 'path_error',   test: (seq) => seq.some(t => t.result?.includes?.('ENOENT') || t.result?.includes?.('no such file')) },
-  { name: 'test_failure', test: (_seq, meta) => meta?.testsPassed === false },
-  { name: 'no_edits',     test: (seq) => seq.length > 0 && !seq.some(t => /write_file|edit_file|Edit|Write/.test(t.name)) },
-];
-
-/**
- * Detect doom loops — same tool call pattern repeated 3+ times consecutively.
- */
-function hasDoomLoop(toolSequence) {
-  if (toolSequence.length < 6) return false;
-  for (let windowSize = 2; windowSize <= 4; windowSize++) {
-    for (let i = 0; i <= toolSequence.length - windowSize * 3; i++) {
-      const pattern = toolSequence.slice(i, i + windowSize).map(t => t.name).join(',');
-      let repeats = 1;
-      for (let j = i + windowSize; j <= toolSequence.length - windowSize; j += windowSize) {
-        const next = toolSequence.slice(j, j + windowSize).map(t => t.name).join(',');
-        if (next === pattern) repeats++;
-        else break;
-      }
-      if (repeats >= 3) return true;
-    }
-  }
-  return false;
-}
-
-/**
- * Classify root cause from tool call sequence and metadata.
- */
-function classifyRootCause(toolSequence, meta) {
-  for (const pattern of ROOT_CAUSE_PATTERNS) {
-    if (pattern.test(toolSequence, meta)) return pattern.name;
-  }
-  return 'unknown';
-}
-
-/**
- * Analyze a benchmark result and store diagnostics for any failing dimensions.
- * @param {Object} brain - Brain module
- * @param {Object} result - Benchmark result with score.dimensions and toolCallDetails
- * @returns {Array} diagnostics entries created
- */
-function analyzeBenchmarkResult(brain, result) {
-  const entries = [];
-  const dimensions = result.score?.dimensions || {};
-  const toolSequence = result.toolCallDetails || result.actualToolCalls?.map(name => ({ name })) || [];
-  const meta = {
-    error: result.error,
-    testsPassed: result.testsPassed,
-  };
-
-  for (const [dim, score] of Object.entries(dimensions)) {
-    if (typeof score !== 'number' || score >= FAILING_THRESHOLD) continue;
-
-    const rootCause = classifyRootCause(toolSequence, meta);
-    const entry = {
-      id: crypto.randomUUID(),
-      sessionId: result.sessionId || null,
-      benchmarkId: result.benchmarkId,
-      failingDimension: dim,
-      score,
-      toolSequence: JSON.stringify(toolSequence.map(t => t.name || t)),
-      rootCause,
-    };
-
-    entries.push(entry);
-
-    if (brain) {
-      try {
-        const db = brain.getDb();
-        db.prepare(`
-          INSERT OR IGNORE INTO eval_diagnostics (id, session_id, benchmark_id, failing_dimension, score, tool_sequence, root_cause)
-          VALUES (?, ?, ?, ?, ?, ?, ?)
-        `).run(entry.id, entry.sessionId, entry.benchmarkId, entry.failingDimension, entry.score, entry.toolSequence, entry.rootCause);
-      } catch { /* non-fatal */ }
-    }
-  }
-
-  return entries;
-}
-
-/**
- * Query recent diagnostics.
- * @param {Object} brain - Brain module
- * @param {Object} [opts]
- * @param {number} [opts.days=7] - Look-back window
- * @param {string} [opts.rootCause] - Filter by root cause
- * @param {string} [opts.benchmarkId] - Filter by benchmark
- * @returns {Array} diagnostic entries
- */
-function queryDiagnostics(brain, opts = {}) {
-  const { days = 7, rootCause, benchmarkId } = opts;
-  const db = brain.getDb();
-
-  let sql = `SELECT * FROM eval_diagnostics WHERE created_at >= datetime('now', ?)`;
-  const params = [`-${days} days`];
-
-  if (rootCause) {
-    sql += ' AND root_cause = ?';
-    params.push(rootCause);
-  }
-  if (benchmarkId) {
-    sql += ' AND benchmark_id = ?';
-    params.push(benchmarkId);
-  }
-
-  sql += ' ORDER BY created_at DESC LIMIT 200';
-
-  try {
-    return db.prepare(sql).all(...params);
-  } catch {
-    return [];
-  }
-}
-
-/**
- * Get a summary of root cause distribution.
- */
-function getDiagnosticsSummary(brain, days = 7) {
-  const db = brain.getDb();
-  try {
-    return db.prepare(`
-      SELECT root_cause, COUNT(*) as count, AVG(score) as avg_score,
-             GROUP_CONCAT(DISTINCT benchmark_id) as benchmarks
-      FROM eval_diagnostics
-      WHERE created_at >= datetime('now', ?)
-      GROUP BY root_cause
-      ORDER BY count DESC
-    `).all(`-${days} days`);
-  } catch {
-    return [];
-  }
-}
-
-// Ensure eval_diagnostics table exists
-function ensureTable(brain) {
-  try {
-    brain.getDb().exec(`
-      CREATE TABLE IF NOT EXISTS eval_diagnostics (
-        id TEXT PRIMARY KEY,
-        session_id TEXT,
-        benchmark_id TEXT,
-        failing_dimension TEXT,
-        score REAL,
-        tool_sequence TEXT,
-        root_cause TEXT,
-        created_at TEXT DEFAULT (datetime('now'))
-      )
-    `);
-  } catch { /* table may already exist */ }
-}
-
-// CLI: node eval/diagnostics.js [--days 7] [--root-cause doom_loop]
-if (require.main === module) {
-  const args = process.argv.slice(2);
-  const daysIdx = args.indexOf('--days');
-  const days = daysIdx >= 0 ? parseInt(args[daysIdx + 1], 10) : 7;
-  const rcIdx = args.indexOf('--root-cause');
-  const rootCause = rcIdx >= 0 ? args[rcIdx + 1] : undefined;
-
-  let brain;
-  try {
-    brain = require('../brain');
-    brain.initDb();
-    ensureTable(brain);
-  } catch (err) {
-    console.error('Brain not available:', err.message);
-    process.exit(1);
-  }
-
-  console.log(`=== Eval Diagnostics (last ${days} days) ===\n`);
-
-  const summary = getDiagnosticsSummary(brain, days);
-  if (summary.length === 0) {
-    console.log('No diagnostics found.');
-  } else {
-    console.log('Root cause distribution:');
-    for (const row of summary) {
-      console.log(`  ${row.root_cause}: ${row.count} failures (avg score ${row.avg_score.toFixed(3)})`);
-      console.log(`    Benchmarks: ${row.benchmarks}`);
-    }
-  }
-
-  const entries = queryDiagnostics(brain, { days, rootCause });
-  if (entries.length > 0) {
-    console.log(`\nRecent entries (${entries.length}):`);
-    for (const e of entries.slice(0, 20)) {
-      console.log(`  [${e.created_at}] ${e.benchmark_id} — ${e.failing_dimension}: ${e.score.toFixed(3)} (${e.root_cause})`);
-    }
-  }
-}
-
-module.exports = {
-  FAILING_THRESHOLD,
-  classifyRootCause,
-  hasDoomLoop,
-  analyzeBenchmarkResult,
-  queryDiagnostics,
-  getDiagnosticsSummary,
-  ensureTable,
-};