create-walle 0.9.21 → 0.9.23

package/template/wall-e/eval/fixtures/node-typed/package.json

@@ -1,8 +0,0 @@
-{
-  "name": "node-typed-fixture",
-  "version": "1.0.0",
-  "private": true,
-  "scripts": {
-    "test": "node test.js"
-  }
-}

package/template/wall-e/eval/fixtures/node-typed/src/handlers.js

@@ -1,31 +0,0 @@
-'use strict';
-const { capitalize, sum, buildUrl } = require('./utils');
-
-/**
- * @param {{ name: string, age: number }} user
- * @returns {{ greeting: string, category: string }}
- */
-function greetUser(user) {
-  const greeting = `Hello, ${capitalize(user.name)}!`;
-  // BUG: age comparison uses string instead of number
-  const category = user.age > '18' ? 'adult' : 'minor';
-  return { greeting, category };
-}
-
-/**
- * @param {number[]} scores
- * @returns {{ total: number, average: number, grade: string }}
- */
-function computeGrade(scores) {
-  const total = sum(scores);
-  const average = total / scores.length;
-  // BUG: grade thresholds should use numbers, not strings
-  let grade;
-  if (average >= 90) grade = 'A';
-  else if (average >= '80') grade = 'B';
-  else if (average >= '70') grade = 'C';
-  else grade = 'F';
-  return { total, average, grade };
-}
-
-module.exports = { greetUser, computeGrade };

package/template/wall-e/eval/fixtures/node-typed/src/utils.js

@@ -1,33 +0,0 @@
-'use strict';
-
-/**
- * @param {string} str - The string to capitalize
- * @returns {string} The capitalized string
- */
-function capitalize(str) {
-  // BUG: doesn't check if str is actually a string
-  return str.charAt(0).toUpperCase() + str.slice(1);
-}
-
-/**
- * @param {number[]} numbers - Array of numbers
- * @returns {number} The sum of all numbers
- */
-function sum(numbers) {
-  // BUG: returns string concatenation if array has non-numbers
-  return numbers.reduce((a, b) => a + b, 0);
-}
-
-/**
- * @param {object} config
- * @param {string} config.host - Server hostname
- * @param {number} config.port - Server port
- * @returns {{ url: string, isSecure: boolean }}
- */
-function buildUrl(config) {
-  // BUG: port is used as string in comparison, isSecure check is wrong
-  const isSecure = config.port == '443';
-  return { url: `http://${config.host}:${config.port}`, isSecure };
-}
-
-module.exports = { capitalize, sum, buildUrl };

package/template/wall-e/eval/fixtures/node-typed/test.js

@@ -1,36 +0,0 @@
-const { greetUser, computeGrade } = require('./src/handlers');
-const { capitalize, sum, buildUrl } = require('./src/utils');
-
-let passed = 0;
-let failed = 0;
-
-function assert(condition, msg) {
-  if (condition) { passed++; console.log(`  PASS: ${msg}`); }
-  else { failed++; console.log(`  FAIL: ${msg}`); }
-}
-
-// Utils tests
-assert(capitalize('hello') === 'Hello', 'capitalize works');
-assert(sum([1, 2, 3]) === 6, 'sum works with numbers');
-const urlResult = buildUrl({ host: 'localhost', port: 443 });
-assert(urlResult.isSecure === true, 'buildUrl detects secure port');
-assert(urlResult.url === 'https://localhost:443' || urlResult.url === 'http://localhost:443', 'buildUrl includes host and port');
-
-// Handler tests
-const greeting = greetUser({ name: 'alice', age: 25 });
-assert(greeting.greeting === 'Hello, Alice!', 'greetUser capitalizes name');
-assert(greeting.category === 'adult', 'greetUser identifies adult');
-
-const minor = greetUser({ name: 'bob', age: 15 });
-assert(minor.category === 'minor', 'greetUser identifies minor');
-
-const grade = computeGrade([90, 85, 95]);
-assert(grade.total === 270, 'computeGrade calculates total');
-assert(grade.average === 90, 'computeGrade calculates average');
-assert(grade.grade === 'A', 'computeGrade assigns correct grade');
-
-const gradeB = computeGrade([80, 85, 75]);
-assert(gradeB.grade === 'B', 'computeGrade assigns B grade');
-
-console.log(`\nResults: ${passed} passed, ${failed} failed`);
-process.exit(failed > 0 ? 1 : 0);

package/template/wall-e/eval/fixtures/python-flask/app.py

@@ -1,14 +0,0 @@
-from flask import Flask, jsonify
-
-app = Flask(__name__)
-
-
-@app.route("/")
-def index():
-    return jsonify({"message": "hello"})
-
-
-if __name__ == "__main__":
-    import os
-    port = int(os.environ.get("PORT", 5000))
-    app.run(host="0.0.0.0", port=port)

package/template/wall-e/eval/fixtures/python-flask/requirements.txt

@@ -1,2 +0,0 @@
-flask>=2.0
-pytest

package/template/wall-e/eval/fixtures/python-flask/test_app.py

@@ -1,25 +0,0 @@
-import pytest
-from app import app
-
-
-@pytest.fixture
-def client():
-    app.config["TESTING"] = True
-    with app.test_client() as client:
-        yield client
-
-
-def test_index_status(client):
-    response = client.get("/")
-    assert response.status_code == 200
-
-
-def test_index_json(client):
-    response = client.get("/")
-    data = response.get_json()
-    assert data == {"message": "hello"}
-
-
-def test_index_content_type(client):
-    response = client.get("/")
-    assert response.content_type == "application/json"

package/template/wall-e/eval/fixtures/wall-e-subset/brain.js

@@ -1,105 +0,0 @@
-'use strict';
-
-/**
- * Simplified in-memory "brain" for benchmark results.
- * No SQLite — just objects in memory.
- */
-class Brain {
-  constructor() {
-    this.results = [];
-    this.metadata = new Map();
-  }
-
-  /**
-   * Insert a benchmark result.
-   * @param {object} result - { benchmarkId, model, scores, timestamp }
-   * @returns {object} the stored result with an auto-generated id
-   */
-  insert(result) {
-    const entry = {
-      id: this.results.length + 1,
-      benchmarkId: result.benchmarkId,
-      model: result.model || 'unknown',
-      scores: result.scores || {},
-      timestamp: result.timestamp || Date.now(),
-      raw: result.raw || null,
-    };
-    this.results.push(entry);
-    return entry;
-  }
-
-  /**
-   * Query results with optional filters.
-   * @param {object} [filters] - { model, benchmarkId, minScore, limit }
-   * @returns {object[]} matching results
-   */
-  query(filters = {}) {
-    let out = [...this.results];
-
-    if (filters.model) {
-      out = out.filter(r => r.model === filters.model);
-    }
-    if (filters.benchmarkId) {
-      out = out.filter(r => r.benchmarkId === filters.benchmarkId);
-    }
-    if (typeof filters.minScore === 'number') {
-      out = out.filter(r => {
-        const composite = r.scores.composite ?? 0;
-        return composite >= filters.minScore;
-      });
-    }
-
-    // Sort newest first
-    out.sort((a, b) => b.timestamp - a.timestamp);
-
-    if (filters.limit) {
-      out = out.slice(0, filters.limit);
-    }
-
-    return out;
-  }
-
-  /**
-   * Get aggregate stats for a model.
-   * @param {string} model
-   * @returns {object} { count, avgComposite, best, worst }
-   */
-  stats(model) {
-    const entries = this.results.filter(r => r.model === model);
-    if (entries.length === 0) return { count: 0, avgComposite: 0, best: 0, worst: 0 };
-
-    const composites = entries.map(r => r.scores.composite ?? 0);
-    const sum = composites.reduce((a, b) => a + b, 0);
-
-    return {
-      count: entries.length,
-      avgComposite: sum / entries.length,
-      best: Math.max(...composites),
-      worst: Math.min(...composites),
-    };
-  }
-
-  /**
-   * Set metadata key/value.
-   */
-  setMeta(key, value) {
-    this.metadata.set(key, value);
-  }
-
-  /**
-   * Get metadata value.
-   */
-  getMeta(key) {
-    return this.metadata.get(key);
-  }
-
-  /**
-   * Clear all results.
-   */
-  clear() {
-    this.results = [];
-    this.metadata.clear();
-  }
-}
-
-module.exports = { Brain };

package/template/wall-e/eval/fixtures/wall-e-subset/eval/aggregator.js

@@ -1,101 +0,0 @@
-'use strict';
-
-/**
- * Rolling window score aggregation.
- */
-class Aggregator {
-  /**
-   * @param {number} windowSize - max entries in rolling window
-   */
-  constructor(windowSize = 50) {
-    this.windowSize = windowSize;
-    this.entries = [];
-  }
-
-  /**
-   * Add a scored result to the window.
-   * @param {object} entry - { model, scores: { composite, correctness, efficiency, quality } }
-   */
-  add(entry) {
-    this.entries.push({
-      model: entry.model,
-      scores: entry.scores,
-      timestamp: entry.timestamp || Date.now(),
-    });
-
-    // Trim to window size
-    if (this.entries.length > this.windowSize) {
-      this.entries = this.entries.slice(-this.windowSize);
-    }
-  }
-
-  /**
-   * Get average scores across the window, optionally filtered by model.
-   * @param {string} [model]
-   * @returns {object} { count, avgComposite, avgCorrectness, avgEfficiency, avgQuality }
-   */
-  average(model) {
-    let subset = this.entries;
-    if (model) {
-      subset = subset.filter(e => e.model === model);
-    }
-    if (subset.length === 0) {
-      return { count: 0, avgComposite: 0, avgCorrectness: 0, avgEfficiency: 0, avgQuality: 0 };
-    }
-
-    const sum = { composite: 0, correctness: 0, efficiency: 0, quality: 0 };
-    for (const e of subset) {
-      sum.composite   += e.scores.composite   ?? 0;
-      sum.correctness += e.scores.correctness ?? 0;
-      sum.efficiency  += e.scores.efficiency  ?? 0;
-      sum.quality     += e.scores.quality     ?? 0;
-    }
-
-    const n = subset.length;
-    return {
-      count: n,
-      avgComposite:   Math.round((sum.composite / n) * 1000) / 1000,
-      avgCorrectness: Math.round((sum.correctness / n) * 1000) / 1000,
-      avgEfficiency:  Math.round((sum.efficiency / n) * 1000) / 1000,
-      avgQuality:     Math.round((sum.quality / n) * 1000) / 1000,
-    };
-  }
-
-  /**
-   * Get the trend (delta between first half and second half averages).
-   * @param {string} [model]
-   * @returns {number} positive = improving, negative = declining
-   */
-  trend(model) {
-    let subset = this.entries;
-    if (model) {
-      subset = subset.filter(e => e.model === model);
-    }
-    if (subset.length < 4) return 0;
-
-    const mid = Math.floor(subset.length / 2);
-    const firstHalf = subset.slice(0, mid);
-    const secondHalf = subset.slice(mid);
-
-    const avgFirst = firstHalf.reduce((s, e) => s + (e.scores.composite ?? 0), 0) / firstHalf.length;
-    const avgSecond = secondHalf.reduce((s, e) => s + (e.scores.composite ?? 0), 0) / secondHalf.length;
-
-    return Math.round((avgSecond - avgFirst) * 1000) / 1000;
-  }
-
-  /**
-   * Return the number of entries in the window.
-   */
-  get size() {
-    return this.entries.length;
-  }
-
-  /**
-   * Clear all entries.
-   */
-  clear() {
-    this.entries = [];
-  }
-}
-
-module.exports = { Aggregator };

package/template/wall-e/eval/fixtures/wall-e-subset/eval/benchmarks/chat.json

@@ -1,20 +0,0 @@
-[
-  {
-    "id": "chat-001",
-    "prompt": "Explain the difference between let, const, and var in JavaScript",
-    "difficulty": "easy",
-    "expectedTraits": ["has-conditional"]
-  },
-  {
-    "id": "chat-002",
-    "prompt": "Show me how to use async/await with error handling in a real-world example",
-    "difficulty": "medium",
-    "expectedTraits": ["has-function", "has-async", "has-error-handling"]
-  },
-  {
-    "id": "chat-003",
-    "prompt": "Describe the observer pattern and provide a simple implementation in JavaScript",
-    "difficulty": "medium",
-    "expectedTraits": ["has-class", "has-function", "has-loop"]
-  }
-]

package/template/wall-e/eval/fixtures/wall-e-subset/eval/benchmarks/coding.json

@@ -1,32 +0,0 @@
-[
-  {
-    "id": "coding-001",
-    "prompt": "Write a function that reverses a string without using built-in reverse methods",
-    "difficulty": "easy",
-    "expectedTraits": ["has-function", "has-loop"]
-  },
-  {
-    "id": "coding-002",
-    "prompt": "Create an async function that fetches data from a URL and handles errors gracefully",
-    "difficulty": "medium",
-    "expectedTraits": ["has-function", "has-async", "has-error-handling"]
-  },
-  {
-    "id": "coding-003",
-    "prompt": "Write a Node.js module that exports a class with methods for stack operations (push, pop, peek)",
-    "difficulty": "easy",
-    "expectedTraits": ["has-class", "has-export", "has-function"]
-  },
-  {
-    "id": "coding-004",
-    "prompt": "Create a test suite for a calculator module using assertions",
-    "difficulty": "medium",
-    "expectedTraits": ["has-test", "has-function", "has-import"]
-  },
-  {
-    "id": "coding-005",
-    "prompt": "Write a well-documented utility module with JSDoc comments that processes an array of objects, filtering and mapping with error handling",
-    "difficulty": "hard",
-    "expectedTraits": ["has-function", "has-jsdoc", "has-error-handling", "has-loop", "has-export"]
-  }
-]

package/template/wall-e/eval/fixtures/wall-e-subset/eval/benchmarks.js

@@ -1,64 +0,0 @@
-'use strict';
-
-const fs = require('fs');
-const path = require('path');
-
-/**
- * 10 regex-based trait matchers for detecting code qualities in responses.
- */
-const TRAIT_MATCHERS = [
-  { trait: 'has-function',      regex: /function\s+\w+|const\s+\w+\s*=\s*(?:async\s*)?\(/ },
-  { trait: 'has-class',         regex: /class\s+\w+/ },
-  { trait: 'has-error-handling', regex: /try\s*\{|\.catch\(|if\s*\(\s*err/ },
-  { trait: 'has-async',         regex: /async\s|await\s|\.then\(/ },
-  { trait: 'has-test',          regex: /test\(|describe\(|it\(|assert/ },
-  { trait: 'has-import',        regex: /require\(|import\s+/ },
-  { trait: 'has-export',        regex: /module\.exports|export\s/ },
-  { trait: 'has-loop',          regex: /for\s*\(|\.forEach\(|\.map\(|while\s*\(/ },
-  { trait: 'has-conditional',   regex: /if\s*\(|switch\s*\(|\?\s*:/ },
-  { trait: 'has-jsdoc',         regex: /\/\*\*[\s\S]*?\*\// },
-];
-
-/**
- * Detect traits in a text string.
- * @param {string} text
- * @returns {string[]} matched trait names
- */
-function detectTraits(text) {
-  if (!text) return [];
-  return TRAIT_MATCHERS
-    .filter(m => m.regex.test(text))
-    .map(m => m.trait);
-}
-
-/**
- * Load benchmarks from a JSON file.
- * @param {string} filePath - path to JSON file
- * @returns {object[]} array of benchmark objects
- */
-function loadBenchmarks(filePath) {
-  const raw = fs.readFileSync(filePath, 'utf8');
-  const data = JSON.parse(raw);
-  return Array.isArray(data) ? data : data.benchmarks || [];
-}
-
-/**
- * Load all benchmarks from the benchmarks directory.
- * @param {string} [dir] - benchmarks directory
- * @returns {object[]} all benchmarks with category tag
- */
-function loadAll(dir) {
-  const benchDir = dir || path.join(__dirname, 'benchmarks');
-  const files = fs.readdirSync(benchDir).filter(f => f.endsWith('.json'));
-  const all = [];
-  for (const f of files) {
-    const category = path.basename(f, '.json');
-    const benchmarks = loadBenchmarks(path.join(benchDir, f));
-    for (const b of benchmarks) {
-      all.push({ ...b, category });
-    }
-  }
-  return all;
-}
-
-module.exports = { detectTraits, loadBenchmarks, loadAll, TRAIT_MATCHERS };

package/template/wall-e/eval/fixtures/wall-e-subset/eval/fixtures/simple-project/package.json

@@ -1,6 +0,0 @@
-{
-  "name": "simple-project",
-  "version": "1.0.0",
-  "description": "A tiny project fixture for eval sandbox testing",
-  "main": "server.js"
-}

package/template/wall-e/eval/fixtures/wall-e-subset/eval/fixtures/simple-project/server.js

@@ -1,31 +0,0 @@
-'use strict';
-
-const http = require('http');
-const { add } = require('./utils');
-
-const server = http.createServer((req, res) => {
-  if (req.url === '/health') {
-    res.writeHead(200, { 'Content-Type': 'application/json' });
-    res.end(JSON.stringify({ status: 'ok' }));
-    return;
-  }
-
-  if (req.url === '/add' && req.method === 'GET') {
-    const url = new URL(req.url, `http://${req.headers.host}`);
-    const a = Number(url.searchParams.get('a') || 0);
-    const b = Number(url.searchParams.get('b') || 0);
-    res.writeHead(200, { 'Content-Type': 'application/json' });
-    res.end(JSON.stringify({ result: add(a, b) }));
-    return;
-  }
-
-  res.writeHead(404);
-  res.end('Not Found');
-});
-
-if (require.main === module) {
-  const port = process.env.PORT || 3000;
-  server.listen(port, () => console.log(`Listening on ${port}`));
-}
-
-module.exports = { server };

package/template/wall-e/eval/fixtures/wall-e-subset/eval/fixtures/simple-project/test.js

@@ -1,18 +0,0 @@
-'use strict';
-
-const { describe, it } = require('node:test');
-const assert = require('node:assert/strict');
-const { add, clamp, simpleHash } = require('./utils');
-
-describe('utils', () => {
-  it('should add two numbers', () => {
-    assert.equal(add(2, 3), 5);
-    assert.equal(add(-1, 1), 0);
-  });
-
-  it('should clamp values', () => {
-    assert.equal(clamp(5, 0, 10), 5);
-    assert.equal(clamp(-1, 0, 10), 0);
-    assert.equal(clamp(15, 0, 10), 10);
-  });
-});

package/template/wall-e/eval/fixtures/wall-e-subset/eval/fixtures/simple-project/utils.js

@@ -1,34 +0,0 @@
-'use strict';
-
-/**
- * Add two numbers.
- * @param {number} a
- * @param {number} b
- * @returns {number}
- */
-function add(a, b) {
-  return a + b;
-}
-
-/**
- * Clamp a number to a range.
- * @param {number} val
- * @param {number} min
- * @param {number} max
- * @returns {number}
- */
-function clamp(val, min, max) {
-  return Math.max(min, Math.min(max, val));
-}
-
-/**
- * Generate a simple hash from a string.
- * @param {string} str
- * @returns {string}
- */
-function simpleHash(str) {
-  const crypto = require('crypto');
-  return crypto.createHash('sha256').update(str).digest('hex').slice(0, 8);
-}
-
-module.exports = { add, clamp, simpleHash };

package/template/wall-e/eval/fixtures/wall-e-subset/eval/runner.js

@@ -1,104 +0,0 @@
-'use strict';
-
-const fs = require('fs');
-const path = require('path');
-const os = require('os');
-const crypto = require('crypto');
-const { scoreFromTraits } = require('./scorer');
-const { detectTraits } = require('./benchmarks');
-
-/**
- * Simplified benchmark runner.
- * Sets up a sandbox dir, runs a mock agent, scores the result.
- */
-
-/**
- * Create a temporary sandbox directory.
- * @returns {string} path to sandbox
- */
-function createSandbox() {
-  const id = crypto.randomBytes(4).toString('hex');
-  const dir = path.join(os.tmpdir(), `walle-eval-${id}`);
-  fs.mkdirSync(dir, { recursive: true });
-  return dir;
-}
-
-/**
- * Clean up sandbox directory.
- * @param {string} dir
- */
-function cleanSandbox(dir) {
-  if (dir && dir.startsWith(os.tmpdir())) {
-    fs.rmSync(dir, { recursive: true, force: true });
-  }
-}
-
-/**
- * Run a single benchmark with a mock agent function.
- * @param {object} benchmark - { id, prompt, expectedTraits, difficulty }
- * @param {function} agentFn - async (prompt, sandboxDir) => string (response text)
- * @returns {object} { benchmarkId, response, detectedTraits, scores, sandboxDir }
- */
-async function runBenchmark(benchmark, agentFn) {
-  const sandbox = createSandbox();
-
-  try {
-    // Copy fixture files into sandbox if specified
-    if (benchmark.fixtureDir && fs.existsSync(benchmark.fixtureDir)) {
-      copyDir(benchmark.fixtureDir, sandbox);
-    }
-
-    // Run the mock agent
-    const response = await agentFn(benchmark.prompt, sandbox);
-
-    // Detect traits in the response
-    const detectedTraits = detectTraits(response);
-
-    // Score against expected traits
-    const scores = scoreFromTraits(benchmark.expectedTraits || [], detectedTraits);
-
-    return {
-      benchmarkId: benchmark.id,
-      response,
-      detectedTraits,
-      scores,
-      sandboxDir: sandbox,
-    };
-  } finally {
-    cleanSandbox(sandbox);
-  }
-}
-
-/**
- * Run all benchmarks with a mock agent.
- * @param {object[]} benchmarks
- * @param {function} agentFn
- * @returns {object[]} results
- */
-async function runAll(benchmarks, agentFn) {
-  const results = [];
-  for (const b of benchmarks) {
-    const result = await runBenchmark(b, agentFn);
-    results.push(result);
-  }
-  return results;
-}
-
-/**
- * Recursively copy a directory.
- */
-function copyDir(src, dest) {
-  const entries = fs.readdirSync(src, { withFileTypes: true });
-  for (const entry of entries) {
-    const srcPath = path.join(src, entry.name);
-    const destPath = path.join(dest, entry.name);
-    if (entry.isDirectory()) {
-      fs.mkdirSync(destPath, { recursive: true });
-      copyDir(srcPath, destPath);
-    } else {
-      fs.copyFileSync(srcPath, destPath);
-    }
-  }
-}
-
-module.exports = { runBenchmark, runAll, createSandbox, cleanSandbox };