create-walle 0.9.21 → 0.9.23

package/template/wall-e/eval/chat-eval.js

@@ -1,525 +0,0 @@
-'use strict';
-
-const fs = require('fs');
-const path = require('path');
-
-const SUITE_PATH = path.join(__dirname, 'benchmarks', 'chat-eval.json');
-const PASS_THRESHOLD = 0.70;
-
-// Cost budgets per intent
-const COST_BUDGETS = {
-  conversational: 0.01,
-  direct_action: 0.05,
-  knowledge: 0.10,
-  'multi-tool': 0.15,
-  'multi-turn': 0.10,
-  calendar: 0.05,
-  tasks: 0.05,
-  slack: 0.10,
-  system: 0.05,
-  weather: 0.05,
-  email: 0.15,
-  'edge-case': 0.10,
-  hallucination: 0.10,
-  'error-resilience': 0.10,
-  permission: 0.10,
-  coding: 0.20,
-  tone: 0.05,
-  'cost-check': 0.005,
-  'ghost-action': 0.005,
-  'wrong-tool': 0.10,
-  adversarial: 0.05,
-  consistency: 0.10,
-  orchestration: 0.15,
-  'context-switch': 0.10,
-};
-
-// ============================================================
-// Load test suite
-// ============================================================
-
-function loadSuite(filterTag) {
-  const raw = JSON.parse(fs.readFileSync(SUITE_PATH, 'utf8'))
-    .filter(tc => tc.id); // skip _comment entries
-  if (filterTag) return raw.filter(tc => tc.tags && tc.tags.includes(filterTag));
-  return raw;
-}
-
-// ============================================================
-// Tool interceptor builder
-// ============================================================
-
-// Side-effect-free tools that are safe to fall through to real execution in eval.
-const EVAL_SAFE_PASSTHROUGH = new Set([
-  'think', 'search_memories', 'lookup_person', 'remember_fact',
-  'list_mcp_tools', 'list_tasks', 'calendar_events', 'calendar_list', 'system_info',
-  'clipboard_read', 'mail_messages', 'mail_search', 'mail_read',
-]);
-
-function buildToolInterceptor(mockToolResults) {
-  if (!mockToolResults || Object.keys(mockToolResults).length === 0) return undefined;
-  return (name, _input) => {
-    if (name in mockToolResults) return mockToolResults[name];
-    // Safe read-only tools can fall through to real execution
-    if (EVAL_SAFE_PASSTHROUGH.has(name)) return undefined;
-    // Block all other tools to prevent side effects (notifications, reminders, file writes, etc.)
-    return { error: `Tool "${name}" is not mocked and not safe for eval — blocked` };
-  };
-}
-
-// ============================================================
-// Scoring
-// ============================================================
-
-function scoreIntent(testCase, actualIntent) {
-  if (!testCase.expectedIntent) return { score: 1.0, detail: 'no expectation' };
-  const match = actualIntent === testCase.expectedIntent;
-  return { score: match ? 1.0 : 0.0, detail: match ? 'match' : `expected ${testCase.expectedIntent}, got ${actualIntent}` };
-}
-
-function scoreTools(testCase, toolCalls) {
-  const actualTools = toolCalls.map(tc => tc.tool);
-  const expectedTools = testCase.expectedTools || [];
-  const forbiddenTools = testCase.forbiddenTools || [];
-  const maxToolCalls = testCase.maxToolCalls || 10;
-
-  // Recall: what fraction of expected tools were used?
-  let recall = 1.0;
-  const missing = [];
-  if (expectedTools.length > 0) {
-    let found = 0;
-    for (const et of expectedTools) {
-      if (actualTools.includes(et)) { found++; } else { missing.push(et); }
-    }
-    recall = found / expectedTools.length;
-  }
-
-  // Forbidden: were any forbidden tools used?
-  const forbiddenUsed = forbiddenTools.filter(ft => actualTools.includes(ft));
-  const forbiddenClean = forbiddenUsed.length === 0 ? 1.0 : 0.0;
-
-  // Efficiency: did it stay within maxToolCalls?
-  const efficiency = actualTools.length <= maxToolCalls
-    ? 1.0
-    : Math.max(0, 1.0 - (actualTools.length - maxToolCalls) / maxToolCalls);
-
-  const score = 0.6 * recall + 0.2 * forbiddenClean + 0.2 * efficiency;
-  const details = [];
-  if (missing.length) details.push(`missing: ${missing.join(', ')}`);
-  if (forbiddenUsed.length) details.push(`forbidden used: ${forbiddenUsed.join(', ')}`);
-  if (actualTools.length > maxToolCalls) details.push(`${actualTools.length} calls > max ${maxToolCalls}`);
-
-  return { score, detail: details.join('; ') || `${actualTools.length} tools, all expected found` };
-}
-
-function scoreContent(testCase, reply) {
-  if (!reply) return { score: 0, detail: 'empty reply' };
-  const lowerReply = reply.toLowerCase();
-
-  // expectedInReply hit rate
-  const expectedHits = testCase.expectedInReply || [];
-  let hitRate = 1.0;
-  const missingContent = [];
-  if (expectedHits.length > 0) {
-    let found = 0;
-    for (const term of expectedHits) {
-      if (lowerReply.includes(term.toLowerCase())) { found++; } else { missingContent.push(term); }
-    }
-    hitRate = found / expectedHits.length;
-  }
-
-  // forbiddenInReply check
-  const forbiddenHits = (testCase.forbiddenInReply || []).filter(term => lowerReply.includes(term.toLowerCase()));
-  const forbiddenClean = forbiddenHits.length === 0 ? 1.0 : 0.0;
-
-  // Length bounds
-  const minLen = testCase.minReplyLength || 0;
-  const maxLen = testCase.maxReplyLength || 10000;
-  const inBounds = reply.length >= minLen && reply.length <= maxLen;
-  const lengthScore = inBounds ? 1.0 : 0.5;
-
-  // Trait check (reuse simple regex matchers)
-  let traitScore = 1.0;
-  const traitDetails = [];
-  if (testCase.expectedTraits && testCase.expectedTraits.length > 0) {
-    const SIMPLE_TRAITS = {
-      greeting: (r) => /\b(hi|hello|hey|welcome|greetings|good\s+(morning|afternoon|evening))\b/i.test(r),
-      'friendly tone': (r) => /\b(glad|happy|great|wonderful|welcome|you're welcome|of course|sure|congrat|exciting|fantastic)\b|!\s/i.test(r),
-      concise: (r) => r.length < 3000,
-      'references context': (r) => r.length > 30, // proxy: if it references mock data, it'll be longer
-      'has code block': (r) => /```[\s\S]*?```/.test(r),
-    };
-    const matched = testCase.expectedTraits.filter(t => {
-      const fn = SIMPLE_TRAITS[t];
-      return fn ? fn(reply) : true; // unknown traits pass
-    });
-    traitScore = matched.length / testCase.expectedTraits.length;
-    const failedTraits = testCase.expectedTraits.filter(t => {
-      const fn = SIMPLE_TRAITS[t];
-      return fn && !fn(reply);
-    });
-    if (failedTraits.length) traitDetails.push(`failed traits: ${failedTraits.join(', ')}`);
-  }
-
-  const score = 0.50 * hitRate + 0.20 * forbiddenClean + 0.15 * lengthScore + 0.15 * traitScore;
-  const details = [];
-  if (missingContent.length) details.push(`missing in reply: ${missingContent.join(', ')}`);
-  if (forbiddenHits.length) details.push(`forbidden found: ${forbiddenHits.join(', ')}`);
-  if (!inBounds) details.push(`length ${reply.length} outside [${minLen}, ${maxLen}]`);
-  details.push(...traitDetails);
-
-  return { score, detail: details.join('; ') || 'ok' };
-}
-
-function scoreLatency(testCase, latencyMs) {
-  const budget = testCase.latencyBudgetMs || 15000;
-  if (latencyMs <= budget) return { score: 1.0, detail: `${(latencyMs / 1000).toFixed(1)}s <= ${(budget / 1000).toFixed(1)}s budget` };
-  const score = Math.max(0, 1.0 - (latencyMs - budget) / budget);
-  return { score, detail: `${(latencyMs / 1000).toFixed(1)}s > ${(budget / 1000).toFixed(1)}s budget (${Math.round(score * 100)}%)` };
-}
-
-function scoreCost(testCase, cost) {
-  const category = testCase.category || 'knowledge';
-  const budget = COST_BUDGETS[category] || COST_BUDGETS.knowledge;
-  if (cost <= budget) return { score: 1.0, detail: `$${cost.toFixed(4)} <= $${budget.toFixed(3)} budget` };
-  const score = Math.max(0, 1.0 - (cost - budget) / budget);
-  return { score, detail: `$${cost.toFixed(4)} > $${budget.toFixed(3)} budget (${Math.round(score * 100)}%)` };
-}
-
-function scoreResult(testCase, chatResult, actualIntent) {
-  const intent = scoreIntent(testCase, actualIntent);
-  const tools = scoreTools(testCase, chatResult.toolCalls || []);
-  const content = scoreContent(testCase, chatResult.reply || '');
-  const latency = scoreLatency(testCase, chatResult.latencyMs || 0);
-  const cost = scoreCost(testCase, chatResult.cost || 0);
-
-  const composite = 0.15 * intent.score + 0.25 * tools.score + 0.25 * content.score + 0.15 * latency.score + 0.20 * cost.score;
-  const passed = composite >= PASS_THRESHOLD;
-
-  return {
-    composite: Math.round(composite * 1000) / 1000,
-    passed,
-    dimensions: { intent, tools, content, latency, cost },
-  };
-}
-
-// ============================================================
-// Runner
-// ============================================================
-
-async function runChatEval(options = {}) {
-  const { filter, model, verbose } = options;
-  const suite = loadSuite(filter);
-  const { chat } = require('../chat');
-  const { classifyTopics, classifyIntent } = require('../context/topic-matcher');
-
-  console.log(`\n=== Wall-E Chat Eval ===`);
-  console.log(`Cases: ${suite.length}${filter ? ` (filter: ${filter})` : ''}`);
-  console.log(`Model: ${model || 'default'}\n`);
-
-  const results = [];
-
-  // Track multi-turn conversation session IDs
-  const conversationSessions = new Map();
-
-  for (const tc of suite) {
-    const label = `[${tc.id}] "${tc.prompt.slice(0, 50)}${tc.prompt.length > 50 ? '...' : ''}"`;
-    process.stdout.write(`  ${label} ... `);
-
-    // 1. Validate intent classification
-    let actualIntent = 'conversational';
-    try {
-      const topics = classifyTopics(tc.prompt);
-      actualIntent = classifyIntent(topics);
-    } catch {}
-
-    // 2. Build tool interceptor
-    const toolInterceptor = buildToolInterceptor(tc.mockToolResults);
-
-    // 3. Determine session ID — multi-turn tests share a session
-    let sessionId;
-    if (tc.conversationId) {
-      if (!conversationSessions.has(tc.conversationId)) {
-        conversationSessions.set(tc.conversationId, `eval-${tc.conversationId}-${Date.now()}`);
-      }
-      sessionId = conversationSessions.get(tc.conversationId);
-    } else {
-      sessionId = `eval-${tc.id}-${Date.now()}`;
-    }
-
-    // 4. Run chat
-    let chatResult;
-    try {
-      chatResult = await chat(tc.prompt, {
-        channel: 'eval',
-        session_id: sessionId,
-        toolInterceptor,
-        ...(model ? { model } : {}),
-      });
-    } catch (err) {
-      chatResult = {
-        reply: `[ERROR] ${err.message}`,
-        model: 'error', provider: 'error',
-        latencyMs: 0, tokens: { input: 0, output: 0 },
-        cost: 0, toolCalls: [],
-      };
-    }
-
-    // 5. Score
-    const score = scoreResult(tc, chatResult, actualIntent);
-
-    const status = score.passed ? 'PASS' : 'FAIL';
-    const emoji = score.passed ? '+' : 'X';
-    console.log(`[${emoji}] ${status}  ${score.composite.toFixed(3)}  ${(chatResult.latencyMs / 1000).toFixed(1)}s  $${(chatResult.cost || 0).toFixed(4)}  ${chatResult.model || '?'}`);
-
-    if (verbose || !score.passed) {
-      const dims = score.dimensions;
-      if (dims.intent.score < 1) console.log(`    intent: ${dims.intent.detail}`);
-      if (dims.tools.score < 1) console.log(`    tools: ${dims.tools.detail}`);
-      if (dims.content.score < 1) console.log(`    content: ${dims.content.detail}`);
-      if (dims.latency.score < 1) console.log(`    latency: ${dims.latency.detail}`);
-      if (dims.cost.score < 1) console.log(`    cost: ${dims.cost.detail}`);
-      if (!score.passed && verbose) {
-        console.log(`    reply: ${(chatResult.reply || '').slice(0, 200)}...`);
-        console.log(`    tools used: ${(chatResult.toolCalls || []).map(t => t.tool).join(', ') || 'none'}`);
-      }
-    }
-
-    results.push({
-      id: tc.id,
-      prompt: tc.prompt,
-      category: tc.category,
-      expectedIntent: tc.expectedIntent,
-      actualIntent,
-      score,
-      chatResult: {
-        reply: (chatResult.reply || '').slice(0, 500),
-        model: chatResult.model,
-        provider: chatResult.provider,
-        latencyMs: chatResult.latencyMs,
-        tokens: chatResult.tokens,
-        cost: chatResult.cost,
-        toolCalls: chatResult.toolCalls || [],
-      },
-    });
-  }
-
-  return formatReport(results, model);
-}
-
-// ============================================================
-// Report
-// ============================================================
-
-function formatReport(results, model) {
-  const total = results.length;
-  const passed = results.filter(r => r.score.passed).length;
-  const failed = total - passed;
-
-  // Category breakdown
-  const categories = {};
-  for (const r of results) {
-    const cat = r.category || 'unknown';
-    if (!categories[cat]) categories[cat] = { cases: 0, passed: 0, totalScore: 0, totalLatency: 0, totalCost: 0 };
-    categories[cat].cases++;
-    if (r.score.passed) categories[cat].passed++;
-    categories[cat].totalScore += r.score.composite;
-    categories[cat].totalLatency += r.chatResult.latencyMs;
-    categories[cat].totalCost += r.chatResult.cost;
-  }
-
-  // Dimension averages
-  const dimAvgs = { intent: 0, tools: 0, content: 0, latency: 0, cost: 0 };
-  for (const r of results) {
-    for (const dim of Object.keys(dimAvgs)) {
-      dimAvgs[dim] += r.score.dimensions[dim].score;
-    }
-  }
-  for (const dim of Object.keys(dimAvgs)) dimAvgs[dim] = Math.round((dimAvgs[dim] / total) * 100) / 100;
-
-  const totalCost = results.reduce((s, r) => s + r.chatResult.cost, 0);
-  const totalTime = results.reduce((s, r) => s + r.chatResult.latencyMs, 0);
-
-  // Print summary
-  console.log(`\n${'='.repeat(60)}`);
-  console.log(`  Wall-E Chat Eval Summary`);
-  console.log(`  Model: ${model || 'default'} | ${total} cases | ${passed} passed | ${failed} failed`);
-  console.log(`${'='.repeat(60)}`);
-
-  console.log(`\n  Category          Pass   Avg    Latency    Cost`);
-  console.log(`  ${'─'.repeat(54)}`);
-  for (const [cat, data] of Object.entries(categories).sort((a, b) => a[0].localeCompare(b[0]))) {
-    const avg = (data.totalScore / data.cases).toFixed(2);
-    const latency = (data.totalLatency / data.cases / 1000).toFixed(1);
-    const cost = (data.totalCost / data.cases).toFixed(4);
-    const passStr = `${data.passed}/${data.cases}`;
-    console.log(`  ${cat.padEnd(18)} ${passStr.padEnd(6)} ${avg.padStart(5)}  ${(latency + 's').padStart(8)}  $${cost}`);
-  }
-
-  console.log(`\n  Dimension Averages:`);
-  for (const [dim, avg] of Object.entries(dimAvgs)) {
-    console.log(`    ${dim.padEnd(10)} ${avg.toFixed(2)}`);
-  }
-
-  if (failed > 0) {
-    console.log(`\n  Failed Cases:`);
-    for (const r of results.filter(r => !r.score.passed)) {
-      const dims = r.score.dimensions;
-      const reasons = [];
-      if (dims.intent.score < 1) reasons.push(`intent: ${dims.intent.detail}`);
-      if (dims.tools.score < 0.5) reasons.push(`tools: ${dims.tools.detail}`);
-      if (dims.content.score < 0.5) reasons.push(`content: ${dims.content.detail}`);
-      if (dims.latency.score < 0.5) reasons.push(`latency: ${dims.latency.detail}`);
-      if (dims.cost.score < 0.5) reasons.push(`cost: ${dims.cost.detail}`);
-      console.log(`    ${r.id} (${r.score.composite.toFixed(3)}) — ${reasons.join('; ') || 'composite below threshold'}`);
-    }
-  }
-
-  console.log(`\n  Total cost: $${totalCost.toFixed(4)} | Total time: ${(totalTime / 1000).toFixed(1)}s`);
-  console.log(`${'='.repeat(60)}\n`);
-
-  return {
-    summary: { total, passed, failed, avgComposite: Math.round((results.reduce((s, r) => s + r.score.composite, 0) / total) * 1000) / 1000, totalCost, totalTimeMs: totalTime },
-    byCategory: categories,
-    byDimension: dimAvgs,
-    results,
-    failures: results.filter(r => !r.score.passed),
-  };
-}
-
-// ============================================================
-// Result persistence — timestamped history + latest copy
-// ============================================================
-
-const RESULTS_DIR = path.join(__dirname, 'results');
-
-function saveReport(report, model) {
-  if (!fs.existsSync(RESULTS_DIR)) fs.mkdirSync(RESULTS_DIR, { recursive: true });
-
-  const ts = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
-  const tag = model || 'default';
-  const filename = `chat-eval-${ts}-${tag}.json`;
-  const filepath = path.join(RESULTS_DIR, filename);
-
-  // Save full report
-  fs.writeFileSync(filepath, JSON.stringify(report, null, 2));
-
-  // Also save as latest (overwrite)
-  const latestPath = path.join(__dirname, 'chat-eval-report.json');
-  fs.writeFileSync(latestPath, JSON.stringify(report, null, 2));
-
-  // Append summary line to history index
-  const historyPath = path.join(RESULTS_DIR, 'history.jsonl');
-  const entry = {
-    timestamp: new Date().toISOString(),
-    file: filename,
-    model: tag,
-    total: report.summary.total,
-    passed: report.summary.passed,
-    failed: report.summary.failed,
-    avgComposite: report.summary.avgComposite,
-    totalCost: Math.round(report.summary.totalCost * 10000) / 10000,
-    totalTimeMs: report.summary.totalTimeMs,
-    dimensions: report.byDimension,
-    perCase: report.results.map(r => ({
-      id: r.id,
-      passed: r.score.passed,
-      composite: r.score.composite,
-      latencyMs: r.chatResult.latencyMs,
-      cost: r.chatResult.cost,
-      model: r.chatResult.model,
-    })),
-  };
-  fs.appendFileSync(historyPath, JSON.stringify(entry) + '\n');
-
-  console.log(`Report saved: ${filepath}`);
-  console.log(`History appended: ${historyPath}`);
-  return filepath;
-}
-
-/**
- * Print regression comparison against previous run.
- */
-function compareWithPrevious(report) {
-  const historyPath = path.join(RESULTS_DIR, 'history.jsonl');
-  if (!fs.existsSync(historyPath)) return;
-
-  const lines = fs.readFileSync(historyPath, 'utf8').trim().split('\n').filter(Boolean);
-  if (lines.length < 2) return; // need at least 2 runs to compare
-
-  const prev = JSON.parse(lines[lines.length - 2]); // second-to-last is previous
-  const curr = report.summary;
-
-  console.log(`\n  Regression Check (vs ${prev.timestamp.slice(0, 16)}):`);
-  const delta = (curr.avgComposite - prev.avgComposite).toFixed(3);
-  const passDelta = curr.passed - prev.passed;
-  const costDelta = (curr.totalCost - prev.totalCost).toFixed(4);
-  const sign = (v) => v > 0 ? `+${v}` : `${v}`;
-
-  console.log(`    Composite: ${prev.avgComposite.toFixed(3)} → ${curr.avgComposite.toFixed(3)} (${sign(delta)})`);
-  console.log(`    Pass rate: ${prev.passed}/${prev.total} → ${curr.passed}/${curr.total} (${sign(passDelta)})`);
-  console.log(`    Cost:      $${prev.totalCost.toFixed(4)} → $${curr.totalCost.toFixed(4)} (${sign(costDelta)})`);
-
-  if (curr.avgComposite < prev.avgComposite - 0.05) {
-    console.log(`    !! REGRESSION: composite score dropped >5%`);
-  }
-  if (curr.total === prev.total && curr.passed < prev.passed) {
-    console.log(`    !! REGRESSION: fewer tests passing`);
-  }
-
-  // Per-case regression: find tests that were passing before but fail now
-  if (prev.perCase && report.results) {
-    const prevMap = new Map(prev.perCase.map(p => [p.id, p]));
-    const regressions = [];
-    const improvements = [];
-    for (const r of report.results) {
-      const p = prevMap.get(r.id);
-      if (!p) continue;
-      if (p.passed && !r.score.passed) regressions.push(r.id);
-      if (!p.passed && r.score.passed) improvements.push(r.id);
-    }
-    if (regressions.length) console.log(`    !! Newly failing: ${regressions.join(', ')}`);
-    if (improvements.length) console.log(`    ++ Newly passing: ${improvements.join(', ')}`);
-  }
-}
-
-// ============================================================
-// CLI
-// ============================================================
-
-if (require.main === module) {
-  const args = process.argv.slice(2);
-  const filter = args.find(a => a.startsWith('--filter='))?.split('=')[1];
-  const model = args.find(a => a.startsWith('--model='))?.split('=')[1];
-  const verbose = args.includes('--verbose') || args.includes('-v');
-  const history = args.includes('--history');
-
-  if (history) {
-    // Print history summary
-    const historyPath = path.join(RESULTS_DIR, 'history.jsonl');
-    if (!fs.existsSync(historyPath)) { console.log('No history yet.'); process.exit(0); }
-    const lines = fs.readFileSync(historyPath, 'utf8').trim().split('\n').filter(Boolean);
-    console.log(`\n  Chat Eval History (${lines.length} runs)\n`);
-    console.log(`  ${'Date'.padEnd(20)} ${'Model'.padEnd(22)} ${'Pass'.padEnd(8)} ${'Score'.padEnd(8)} ${'Cost'.padEnd(10)} Time`);
-    console.log(`  ${'─'.repeat(76)}`);
-    for (const line of lines) {
-      const e = JSON.parse(line);
-      console.log(`  ${e.timestamp.slice(0, 19).padEnd(20)} ${e.model.padEnd(22)} ${(e.passed + '/' + e.total).padEnd(8)} ${e.avgComposite.toFixed(3).padEnd(8)} $${e.totalCost.toFixed(4).padEnd(9)} ${(e.totalTimeMs / 1000).toFixed(0)}s`);
-    }
-    console.log();
-    process.exit(0);
-  }
-
-  runChatEval({ filter, model, verbose })
-    .then(report => {
-      saveReport(report, model);
-      compareWithPrevious(report);
-      process.exit(report.summary.failed > 0 ? 1 : 0);
-    })
-    .catch(err => {
-      console.error('Chat eval failed:', err);
-      process.exit(2);
-    });
-}
-
-module.exports = { runChatEval, scoreResult, loadSuite, scoreIntent, scoreTools, scoreContent, scoreLatency, scoreCost };

package/template/wall-e/eval/check-keys.js

@@ -1,15 +0,0 @@
-#!/usr/bin/env node
-'use strict';
-const path = require('path');
-process.chdir(path.join(__dirname, '..'));
-const brain = require('../brain');
-brain.initDb();
-const db = brain.getDb();
-
-const providers = db.prepare('SELECT type, id, base_url, substr(api_key_encrypted, 1, 15) as prefix, length(api_key_encrypted) as len FROM model_providers WHERE enabled = 1').all();
-providers.forEach(p => console.log(`${p.type} | ${p.id} | prefix=${p.prefix || 'none'} | len=${p.len || 0} | url=${p.base_url || 'default'}`));
-
-// Check Portkey-related env or metadata
-const meta = db.prepare("SELECT key, value FROM brain_metadata WHERE key LIKE '%provider%' OR key LIKE '%portkey%' OR key LIKE '%model%' OR key LIKE '%anthropic%'").all();
-console.log('\nBrain metadata:');
-meta.forEach(r => console.log(`  ${r.key} = ${r.value}`));

package/template/wall-e/eval/check-providers.js

@@ -1,42 +0,0 @@
-#!/usr/bin/env node
-'use strict';
-const path = require('path');
-process.chdir(path.join(__dirname, '..'));
-
-const brain = require('../brain');
-brain.initDb();
-const db = brain.getDb();
-
-// Default provider
-const meta = db.prepare("SELECT key, value FROM brain_metadata WHERE key IN ('walle_provider', 'walle_model')").all();
-console.log('=== Brain defaults ===');
-meta.forEach(r => console.log(`  ${r.key}: ${r.value}`));
-
-// Providers
-console.log('\n=== Active Providers ===');
-const providers = db.prepare('SELECT id, type, base_url, enabled, api_key_encrypted IS NOT NULL as has_key FROM model_providers WHERE enabled = 1').all();
-providers.forEach(p => console.log(`  ${p.type} | id=${p.id} | has_key=${p.has_key} | url=${p.base_url || 'default'}`));
-
-// Count models per provider
-console.log('\n=== Model Counts ===');
-const counts = db.prepare(`
-  SELECT mp.type, COUNT(*) as cnt
-  FROM model_registry mr JOIN model_providers mp ON mr.provider_id = mp.id
-  WHERE mp.enabled = 1
-  GROUP BY mp.type ORDER BY cnt DESC
-`).all();
-counts.forEach(c => console.log(`  ${c.type}: ${c.cnt} models`));
-
-// Best coding models per provider
-console.log('\n=== Coding-capable Models (sample) ===');
-for (const prov of providers) {
-  const models = db.prepare(`
-    SELECT model_id, display_name, capabilities, speed_tier, cost_per_1m_input
-    FROM model_registry WHERE provider_id = ? ORDER BY model_id LIMIT 8
-  `).all(prov.id);
-  console.log(`\n  --- ${prov.type} (id=${prov.id}) ---`);
-  models.forEach(m => {
-    const caps = m.capabilities ? JSON.parse(m.capabilities) : [];
-    console.log(`    ${m.model_id} | speed=${m.speed_tier || '?'} | cost=$${m.cost_per_1m_input || '?'}/1M | caps=${caps.join(',') || 'none'}`);
-  });
-}