create-walle 0.9.21 → 0.9.23

package/template/wall-e/eval/exporter.js

@@ -1,212 +0,0 @@
-'use strict';
-
-const fs = require('fs');
-const path = require('path');
-
-const ALL_SOURCES = ['knowledge', 'chat', 'patterns'];
-
-/**
- * Export training data from brain DB to JSONL format.
- * @param {Object} options
- * @param {Object} options.brain - Brain module (for dependency injection)
- * @param {string} [options.outputPath] - Where to write JSONL file
- * @param {string[]} [options.sources] - Which sources: ['knowledge', 'chat', 'patterns']
- * @returns {{ samples: Array, stats: { total, bySource } }}
- */
-async function exportTrainingData({ brain, outputPath, sources } = {}) {
-  const activeSources = sources || ALL_SOURCES;
-  const allSamples = [];
-  const bySource = {};
-
-  for (const src of ALL_SOURCES) {
-    bySource[src] = 0;
-  }
-
-  // 1. Knowledge triples -> Q&A pairs
-  if (activeSources.includes('knowledge')) {
-    const entries = brain.findKnowledge ? brain.findKnowledge() : [];
-    for (const entry of entries) {
-      if (entry.confidence < 0.5) continue;
-      if (entry.status !== 'active') continue;
-      const instruction = `What ${entry.predicate} ${entry.subject}?`;
-      const output = entry.object;
-      allSamples.push({ instruction, output, source: 'knowledge' });
-    }
-  }
-
-  // 2. Chat messages -> conversation pairs
-  if (activeSources.includes('chat')) {
-    const messages = brain.listChatMessages ? brain.listChatMessages() : [];
-    // Group by session
-    const sessions = new Map();
-    for (const msg of messages) {
-      const sid = msg.session_id || '__default__';
-      if (!sessions.has(sid)) sessions.set(sid, []);
-      sessions.get(sid).push(msg);
-    }
-    // Create pairs from consecutive user/assistant turns
-    for (const [, msgs] of sessions) {
-      for (let i = 0; i < msgs.length - 1; i++) {
-        const userMsg = msgs[i];
-        const assistantMsg = msgs[i + 1];
-        if (userMsg.role !== 'user' || assistantMsg.role !== 'assistant') continue;
-        // Filter short messages
-        if (!userMsg.content || userMsg.content.length < 20) continue;
-        if (!assistantMsg.content || assistantMsg.content.length < 20) continue;
-        // Filter tool-use-only messages
-        if (isToolUseOnly(assistantMsg.content)) continue;
-        allSamples.push({
-          instruction: userMsg.content,
-          output: assistantMsg.content,
-          source: 'chat',
-        });
-      }
-    }
-  }
-
-  // 3. Patterns -> instruction pairs
-  if (activeSources.includes('patterns')) {
-    const patterns = (brain.listPatterns ? brain.listPatterns() : []);
-    for (const p of patterns) {
-      const instruction = `Describe the pattern: ${p.pattern_type}`;
-      const output = p.description;
-      allSamples.push({ instruction, output, source: 'patterns' });
-    }
-  }
-
-  // 4. Quality filtering
-  const filtered = [];
-  const seen = new Set();
-  for (const sample of allSamples) {
-    // Skip empty instruction/output
-    if (!sample.instruction || !sample.instruction.trim()) continue;
-    if (!sample.output || sample.output.length < 10) continue;
-    // Dedup by instruction
-    if (seen.has(sample.instruction)) continue;
-    seen.add(sample.instruction);
-    filtered.push(sample);
-  }
-
-  // 5. Compute stats
-  for (const sample of filtered) {
-    bySource[sample.source] = (bySource[sample.source] || 0) + 1;
-  }
-  const stats = { total: filtered.length, bySource };
-
-  // 6. Write JSONL if outputPath provided
-  if (outputPath) {
-    const dir = path.dirname(outputPath);
-    if (!fs.existsSync(dir)) {
-      fs.mkdirSync(dir, { recursive: true });
-    }
-    const lines = filtered.map(s => JSON.stringify(s));
-    fs.writeFileSync(outputPath, lines.join('\n') + (lines.length ? '\n' : ''), 'utf8');
-  }
-
-  return { samples: filtered, stats };
-}
-
-/**
- * Detect if a message is tool-use only (JSON array of tool_use blocks).
- */
-function isToolUseOnly(content) {
-  if (!content) return false;
-  const trimmed = content.trim();
-  if (!trimmed.startsWith('[')) return false;
-  try {
-    const parsed = JSON.parse(trimmed);
-    if (Array.isArray(parsed) && parsed.every(item => item.type === 'tool_use')) {
-      return true;
-    }
-  } catch {
-    // Not JSON, not tool-use only
-  }
-  return false;
-}
-
-/**
- * Export harvested training data from JSONL files.
- * @param {Object} options
- * @param {string} options.harvestDir - Directory containing harvested/*.jsonl files
- * @param {string} [options.outputPath] - Where to write output JSONL
- * @param {number} [options.minQuality=0.6] - Minimum quality_label to include
- * @param {string} [options.format='alpaca'] - Output format: 'alpaca' or 'chat-ml'
- * @param {number} [options.maxPerTaskType=0] - Max samples per task type (0=unlimited)
- * @returns {{ samples: Array, stats: { total, bySource, byTaskType, filtered } }}
- */
-async function exportHarvestedData({ harvestDir, outputPath, minQuality = 0.6, format = 'alpaca', maxPerTaskType = 0 } = {}) {
-  const harvestedDir = path.join(harvestDir, 'training', 'harvested');
-  if (!fs.existsSync(harvestedDir)) return { samples: [], stats: { total: 0, bySource: {}, byTaskType: {}, filtered: 0 } };
-
-  // Read all JSONL files
-  const files = fs.readdirSync(harvestedDir).filter(f => f.endsWith('.jsonl')).sort();
-  let allSamples = [];
-  for (const file of files) {
-    const lines = fs.readFileSync(path.join(harvestedDir, file), 'utf8').split('\n').filter(Boolean);
-    for (const line of lines) {
-      try { allSamples.push(JSON.parse(line)); } catch {}
-    }
-  }
-
-  // Filter by quality
-  const filtered = allSamples.length;
-  allSamples = allSamples.filter(s => (s.quality_label || 0) >= minQuality);
-
-  // Deduplicate by id
-  const seen = new Set();
-  allSamples = allSamples.filter(s => {
-    if (seen.has(s.id)) return false;
-    seen.add(s.id);
-    return true;
-  });
-
-  // Balance by task_type if maxPerTaskType > 0
-  if (maxPerTaskType > 0) {
-    const byType = {};
-    const balanced = [];
-    for (const s of allSamples) {
-      const t = s.task_type || 'unknown';
-      byType[t] = (byType[t] || 0) + 1;
-      if (byType[t] <= maxPerTaskType) balanced.push(s);
-    }
-    allSamples = balanced;
-  }
-
-  // Convert to output format
-  const outputSamples = allSamples.map(s => {
-    if (format === 'chat-ml') {
-      return {
-        messages: [
-          { role: 'system', content: 'You are a helpful AI assistant.' },
-          { role: 'user', content: s.prompt },
-          { role: 'assistant', content: s.response },
-        ],
-      };
-    }
-    // Default: alpaca format
-    return {
-      instruction: s.prompt,
-      input: '',
-      output: s.response,
-    };
-  });
-
-  // Build stats
-  const stats = { total: outputSamples.length, bySource: {}, byTaskType: {}, filtered: filtered - allSamples.length };
-  for (const s of allSamples) {
-    stats.bySource[s.source] = (stats.bySource[s.source] || 0) + 1;
-    stats.byTaskType[s.task_type] = (stats.byTaskType[s.task_type] || 0) + 1;
-  }
-
-  // Write to file if outputPath specified
-  if (outputPath) {
-    const dir = path.dirname(outputPath);
-    if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
-    const jsonl = outputSamples.map(s => JSON.stringify(s)).join('\n') + '\n';
-    fs.writeFileSync(outputPath, jsonl);
-  }
-
-  return { samples: outputSamples, stats };
-}
-
-module.exports = { exportTrainingData, exportHarvestedData };

package/template/wall-e/eval/fixtures/express-basic/package.json

@@ -1,9 +0,0 @@
-{
-  "name": "express-basic-fixture",
-  "version": "1.0.0",
-  "scripts": {
-    "test": "node test.js",
-    "start": "node server.js"
-  },
-  "dependencies": {}
-}

package/template/wall-e/eval/fixtures/express-basic/server.js

@@ -1,115 +0,0 @@
-const http = require('http');
-const url = require('url');
-
-// In-memory data stores
-const users = [
-  { id: 1, name: 'Alice', email: 'alice@example.com' },
-  { id: 2, name: 'Bob', email: 'bob@example.com' },
-];
-
-const items = [
-  { id: 1, name: 'Widget', price: 9.99 },
-  { id: 2, name: 'Gadget', price: 19.99 },
-  { id: 3, name: 'Doohickey', price: 4.99 },
-];
-
-const searchData = [
-  { id: 1, title: 'Getting Started', relevance: 0.9 },
-  { id: 2, title: 'Advanced Guide', relevance: 0.7 },
-  { id: 3, title: 'FAQ', relevance: 0.5 },
-  { id: 4, title: 'API Reference', relevance: 0.8 },
-];
-
-// Helper: parse JSON body
-function parseBody(req) {
-  return new Promise((resolve, reject) => {
-    let body = '';
-    req.on('data', chunk => (body += chunk));
-    req.on('end', () => {
-      try { resolve(body ? JSON.parse(body) : {}); }
-      catch { reject(new Error('Invalid JSON')); }
-    });
-  });
-}
-
-const routes = {};
-
-routes['GET /'] = (req, res) => {
-  res.writeHead(200, { 'Content-Type': 'application/json' });
-  res.end(JSON.stringify({ message: 'hello' }));
-};
-
-routes['GET /users'] = (req, res) => {
-  const data = users || [];
-  res.writeHead(200, { 'Content-Type': 'application/json' });
-  res.end(JSON.stringify(data.map(u => ({ id: u.id, name: u.name }))));
-};
-
-routes['POST /users'] = async (req, res) => {
-  try {
-    const body = await parseBody(req);
-    const newUser = { id: users.length + 1, ...body };
-    users.push(newUser);
-    res.writeHead(201, { 'Content-Type': 'application/json' });
-    res.end(JSON.stringify(newUser));
-  } catch {
-    res.writeHead(400, { 'Content-Type': 'application/json' });
-    res.end(JSON.stringify({ error: 'Invalid request body' }));
-  }
-};
-
-routes['GET /search'] = (req, res) => {
-  const parsed = url.parse(req.url, true);
-  const q = (parsed.query.q || '').toLowerCase();
-  let results = searchData;
-  if (q) {
-    results = searchData.filter(item =>
-      item.title.toLowerCase().includes(q)
-    );
-  }
-  results.sort((a, b) => b.relevance - a.relevance);
-  res.writeHead(200, { 'Content-Type': 'application/json' });
-  res.end(JSON.stringify(results));
-};
-
-const server = http.createServer((req, res) => {
-  const parsed = url.parse(req.url, true);
-  const pathname = parsed.pathname;
-
-  // Dynamic route: GET /items/:id
-  if (req.method === 'GET' && pathname.startsWith('/items/')) {
-    const id = parseInt(pathname.split('/')[2]);
-    const item = items.find(i => i.id === id);
-    if (!item) {
-      res.writeHead(404, { 'Content-Type': 'application/json' });
-      res.end(JSON.stringify({ error: 'item not found' }));
-      return;
-    }
-    res.writeHead(200, { 'Content-Type': 'application/json' });
-    res.end(JSON.stringify(item));
-    return;
-  }
-
-  // Static routes
-  const key = `${req.method} ${pathname}`;
-  const handler = routes[key];
-  if (handler) {
-    Promise.resolve(handler(req, res)).catch(err => {
-      res.writeHead(500, { 'Content-Type': 'application/json' });
-      res.end(JSON.stringify({ error: err.message }));
-    });
-  } else {
-    res.writeHead(404, { 'Content-Type': 'application/json' });
-    res.end(JSON.stringify({ error: 'not found' }));
-  }
-});
-
-const PORT = process.env.PORT || 3000;
-
-if (require.main === module) {
-  server.listen(PORT, () => {
-    console.log(`Server running on port ${PORT}`);
-  });
-}
-
-module.exports = server;

package/template/wall-e/eval/fixtures/express-basic/test.js

@@ -1,83 +0,0 @@
-const http = require('http');
-const app = require('./server');
-
-let server;
-let passed = 0;
-let failed = 0;
-
-function request(method, path, body) {
-  return new Promise((resolve, reject) => {
-    const addr = server.address();
-    const options = { hostname: '127.0.0.1', port: addr.port, path, method };
-    if (body) options.headers = { 'Content-Type': 'application/json' };
-
-    const req = http.request(options, (res) => {
-      let data = '';
-      res.on('data', (chunk) => (data += chunk));
-      res.on('end', () => {
-        try { resolve({ status: res.statusCode, body: JSON.parse(data) }); }
-        catch { resolve({ status: res.statusCode, body: data }); }
-      });
-    });
-    req.on('error', reject);
-    if (body) req.write(JSON.stringify(body));
-    req.end();
-  });
-}
-
-function assert(condition, msg) {
-  if (condition) { passed++; console.log(`  PASS: ${msg}`); }
-  else { failed++; console.log(`  FAIL: ${msg}`); }
-}
-
-async function run() {
-  server = app.listen(0);
-
-  try {
-    // Basic endpoint
-    console.log('GET /');
-    const root = await request('GET', '/');
-    assert(root.status === 200, 'root status is 200');
-    assert(root.body.message === 'hello', 'root body has message: hello');
-
-    // Users
-    console.log('GET /users');
-    const usersRes = await request('GET', '/users');
-    assert(usersRes.status === 200, 'users status is 200');
-    assert(Array.isArray(usersRes.body), 'users body is array');
-    assert(usersRes.body.length >= 2, 'users has at least 2 entries');
-
-    // Items - existing item
-    console.log('GET /items/1');
-    const item1 = await request('GET', '/items/1');
-    assert(item1.status === 200, 'item 1 status is 200');
-    assert(item1.body.name === 'Widget', 'item 1 name is Widget');
-
-    // Items - non-existent item (should be 404)
-    console.log('GET /items/999');
-    const item999 = await request('GET', '/items/999');
-    assert(item999.status === 404, 'non-existent item returns 404');
-
-    // Search - should return results sorted by relevance descending
-    console.log('GET /search');
-    const searchRes = await request('GET', '/search');
-    assert(searchRes.status === 200, 'search status is 200');
-    assert(Array.isArray(searchRes.body), 'search body is array');
-    // Verify sorted by relevance (highest first)
-    const relevances = searchRes.body.map(r => r.relevance);
-    const sorted = [...relevances].sort((a, b) => b - a);
-    assert(JSON.stringify(relevances) === JSON.stringify(sorted), 'search results sorted by relevance desc');
-
-    // 404 for unknown route
-    console.log('GET /unknown');
-    const unknown = await request('GET', '/unknown');
-    assert(unknown.status === 404, 'unknown route returns 404');
-  } finally {
-    server.close();
-  }
-
-  console.log(`\nResults: ${passed} passed, ${failed} failed`);
-  process.exit(failed > 0 ? 1 : 0);
-}
-
-run().catch((err) => { console.error(err); process.exit(1); });

package/template/wall-e/eval/fixtures/express-buggy/package.json

@@ -1,9 +0,0 @@
-{
-  "name": "express-basic-fixture",
-  "version": "1.0.0",
-  "scripts": {
-    "test": "node test.js",
-    "start": "node server.js"
-  },
-  "dependencies": {}
-}

package/template/wall-e/eval/fixtures/express-buggy/server.js

@@ -1,113 +0,0 @@
-const http = require('http');
-const url = require('url');
-
-// In-memory data stores
-const users = [
-  { id: 1, name: 'Alice', email: 'alice@example.com' },
-  { id: 2, name: 'Bob', email: 'bob@example.com' },
-];
-
-const items = [
-  { id: 1, name: 'Widget', price: 9.99 },
-  { id: 2, name: 'Gadget', price: 19.99 },
-  { id: 3, name: 'Doohickey', price: 4.99 },
-];
-
-// agent-008 prompt explicitly asks the agent to fix BOTH this latent
-// /search-sort bug AND the items/:id 404 bug. Single-bug benchmarks
-// (agent-003 / agent-010 / agent-mt-004) use the express-buggy-items
-// or express-buggy-search variant fixtures so they aren't gated by an
-// unrelated failing test.
-const searchData = [
-  { id: 1, title: 'Getting Started', relevance: 0.9 },
-  { id: 2, title: 'Advanced Guide', relevance: 0.7 },
-  { id: 3, title: 'FAQ', relevance: 0.5 },
-  { id: 4, title: 'API Reference', relevance: 0.8 },
-];
-
-function parseBody(req) {
-  return new Promise((resolve, reject) => {
-    let body = '';
-    req.on('data', chunk => (body += chunk));
-    req.on('end', () => {
-      try { resolve(body ? JSON.parse(body) : {}); }
-      catch { reject(new Error('Invalid JSON')); }
-    });
-  });
-}
-
-const routes = {};
-
-routes['GET /'] = (req, res) => {
-  res.writeHead(200, { 'Content-Type': 'application/json' });
-  res.end(JSON.stringify({ message: 'hello' }));
-};
-
-routes['GET /users'] = (req, res) => {
-  const data = users;
-  res.writeHead(200, { 'Content-Type': 'application/json' });
-  res.end(JSON.stringify(data.map(u => ({ id: u.id, name: u.name }))));
-};
-
-routes['POST /users'] = async (req, res) => {
-  try {
-    const body = await parseBody(req);
-    const newUser = { id: users.length + 1, ...body };
-    users.push(newUser);
-    res.writeHead(201, { 'Content-Type': 'application/json' });
-    res.end(JSON.stringify(newUser));
-  } catch {
-    res.writeHead(400, { 'Content-Type': 'application/json' });
-    res.end(JSON.stringify({ error: 'Invalid request body' }));
-  }
-};
-
-routes['GET /search'] = (req, res) => {
-  const parsed = url.parse(req.url, true);
-  const q = (parsed.query.q || '').toLowerCase();
-  let results = searchData;
-  if (q) {
-    results = searchData.filter(item =>
-      item.title.toLowerCase().includes(q)
-    );
-  }
-  res.writeHead(200, { 'Content-Type': 'application/json' });
-  res.end(JSON.stringify(results));
-};
-
-const server = http.createServer((req, res) => {
-  const parsed = url.parse(req.url, true);
-  const pathname = parsed.pathname;
-
-  // Dynamic route: GET /items/:id
-  if (req.method === 'GET' && pathname.startsWith('/items/')) {
-    const id = parseInt(pathname.split('/')[2]);
-    const item = items.find(i => i.id === id);
-    res.writeHead(200, { 'Content-Type': 'application/json' });
-    res.end(JSON.stringify(item));
-    return;
-  }
-
-  // Static routes
-  const key = `${req.method} ${pathname}`;
-  const handler = routes[key];
-  if (handler) {
-    Promise.resolve(handler(req, res)).catch(err => {
-      res.writeHead(500, { 'Content-Type': 'application/json' });
-      res.end(JSON.stringify({ error: err.message }));
-    });
-  } else {
-    res.writeHead(404, { 'Content-Type': 'application/json' });
-    res.end(JSON.stringify({ error: 'not found' }));
-  }
-});
-
-const PORT = process.env.PORT || 3000;
-
-if (require.main === module) {
-  server.listen(PORT, () => {
-    console.log(`Server running on port ${PORT}`);
-  });
-}
-
-module.exports = server;

package/template/wall-e/eval/fixtures/express-buggy/test.js

@@ -1,83 +0,0 @@
-const http = require('http');
-const app = require('./server');
-
-let server;
-let passed = 0;
-let failed = 0;
-
-function request(method, path, body) {
-  return new Promise((resolve, reject) => {
-    const addr = server.address();
-    const options = { hostname: '127.0.0.1', port: addr.port, path, method };
-    if (body) options.headers = { 'Content-Type': 'application/json' };
-
-    const req = http.request(options, (res) => {
-      let data = '';
-      res.on('data', (chunk) => (data += chunk));
-      res.on('end', () => {
-        try { resolve({ status: res.statusCode, body: JSON.parse(data) }); }
-        catch { resolve({ status: res.statusCode, body: data }); }
-      });
-    });
-    req.on('error', reject);
-    if (body) req.write(JSON.stringify(body));
-    req.end();
-  });
-}
-
-function assert(condition, msg) {
-  if (condition) { passed++; console.log(`  PASS: ${msg}`); }
-  else { failed++; console.log(`  FAIL: ${msg}`); }
-}
-
-async function run() {
-  server = app.listen(0);
-
-  try {
-    // Basic endpoint
-    console.log('GET /');
-    const root = await request('GET', '/');
-    assert(root.status === 200, 'root status is 200');
-    assert(root.body.message === 'hello', 'root body has message: hello');
-
-    // Users
-    console.log('GET /users');
-    const usersRes = await request('GET', '/users');
-    assert(usersRes.status === 200, 'users status is 200');
-    assert(Array.isArray(usersRes.body), 'users body is array');
-    assert(usersRes.body.length >= 2, 'users has at least 2 entries');
-
-    // Items - existing item
-    console.log('GET /items/1');
-    const item1 = await request('GET', '/items/1');
-    assert(item1.status === 200, 'item 1 status is 200');
-    assert(item1.body.name === 'Widget', 'item 1 name is Widget');
-
-    // Items - non-existent item (should be 404)
-    console.log('GET /items/999');
-    const item999 = await request('GET', '/items/999');
-    assert(item999.status === 404, 'non-existent item returns 404');
-
-    // Search - should return results sorted by relevance descending
-    console.log('GET /search');
-    const searchRes = await request('GET', '/search');
-    assert(searchRes.status === 200, 'search status is 200');
-    assert(Array.isArray(searchRes.body), 'search body is array');
-    // Verify sorted by relevance (highest first)
-    const relevances = searchRes.body.map(r => r.relevance);
-    const sorted = [...relevances].sort((a, b) => b - a);
-    assert(JSON.stringify(relevances) === JSON.stringify(sorted), 'search results sorted by relevance desc');
-
-    // 404 for unknown route
-    console.log('GET /unknown');
-    const unknown = await request('GET', '/unknown');
-    assert(unknown.status === 404, 'unknown route returns 404');
-  } finally {
-    server.close();
-  }
-
-  console.log(`\nResults: ${passed} passed, ${failed} failed`);
-  process.exit(failed > 0 ? 1 : 0);
-}
-
-run().catch((err) => { console.error(err); process.exit(1); });

package/template/wall-e/eval/fixtures/express-buggy-items/package.json

@@ -1,9 +0,0 @@
-{
-  "name": "express-basic-fixture",
-  "version": "1.0.0",
-  "scripts": {
-    "test": "node test.js",
-    "start": "node server.js"
-  },
-  "dependencies": {}
-}