create-walle 0.9.21 → 0.9.23

package/template/wall-e/eval/swebench-adapter.js

@@ -1,345 +0,0 @@
-'use strict';
-
-/**
- * SWE-bench Lite adapter for Wall-E evaluation pipeline.
- *
- * Downloads the SWE-bench Lite dataset from HuggingFace (JSON),
- * maps tasks to Wall-E prompt format, and delegates execution
- * to Docker containers (or returns placeholder results when Docker
- * is unavailable).
- *
- * Results are stored with suite = 'swebench-lite'.
- */
-
-const fs = require('fs');
-const path = require('path');
-const os = require('os');
-const https = require('https');
-const crypto = require('crypto');
-const { resolveModelName } = require('./agent-runner');
-const { decorateBenchmarkResult, DEFAULT_SCORER_VERSION } = require('./manifest');
-
-const CACHE_DIR = path.join(os.homedir(), '.walle', 'swebench-cache');
-const DATASET_URL =
-  'https://raw.githubusercontent.com/princeton-nlp/SWE-bench/main/swebench/collect/tasks/swe-bench-lite.json';
-const CACHE_FILE = path.join(CACHE_DIR, 'swe-bench-lite.json');
-
-/**
- * Download and cache the SWE-bench Lite dataset.
- * Returns the parsed JSON array of tasks.
- */
-async function downloadDataset() {
-  fs.mkdirSync(CACHE_DIR, { recursive: true });
-
-  // Return cached version if it exists and is less than 7 days old
-  if (fs.existsSync(CACHE_FILE)) {
-    const stat = fs.statSync(CACHE_FILE);
-    const ageMs = Date.now() - stat.mtimeMs;
-    if (ageMs < 7 * 24 * 60 * 60 * 1000) {
-      return JSON.parse(fs.readFileSync(CACHE_FILE, 'utf8'));
-    }
-  }
-
-  // Download from HuggingFace / GitHub mirror
-  const data = await new Promise((resolve, reject) => {
-    const request = (url, redirectCount = 0) => {
-      if (redirectCount > 5) return reject(new Error('Too many redirects'));
-
-      https.get(url, { headers: { 'User-Agent': 'wall-e-eval/1.0' } }, (res) => {
-        // Follow redirects
-        if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
-          return request(res.headers.location, redirectCount + 1);
-        }
-        if (res.statusCode !== 200) {
-          return reject(new Error(`HTTP ${res.statusCode} fetching SWE-bench dataset`));
-        }
-
-        const chunks = [];
-        res.on('data', (chunk) => chunks.push(chunk));
-        res.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
-        res.on('error', reject);
-      }).on('error', reject);
-    };
-    request(DATASET_URL);
-  });
-
-  fs.writeFileSync(CACHE_FILE, data, 'utf8');
-  return JSON.parse(data);
-}
-
-/**
- * Convert a SWE-bench task to Wall-E prompt format.
- *
- * @param {Object} task - SWE-bench task with instance_id, repo,
- *   base_commit, problem_statement, hints_text, test_patch, patch
- * @returns {Object} Wall-E-compatible benchmark entry
- */
-function mapTaskToPrompt(task) {
-  let prompt = `Fix the following issue in ${task.repo}:\n\n${task.problem_statement}`;
-  if (task.hints_text) {
-    prompt += `\n\nHints:\n${task.hints_text}`;
-  }
-
-  return {
-    id: task.instance_id,
-    prompt,
-    suite: 'swebench-lite',
-    repo: task.repo,
-    baseCommit: task.base_commit,
-    goldenPatch: task.patch,
-    testPatch: task.test_patch,
-  };
-}
-
-/**
- * Run a single SWE-bench task.
- *
- * This is the adapter layer: actual execution depends on Docker
- * availability. When Docker is present, delegates to swebench-docker.js
- * to spin up a container, apply the agent's patch, and run the test suite.
- *
- * @param {Object} task - Mapped task from mapTaskToPrompt()
- * @param {Object} options
- * @param {Function} options.runAgentLoop - The agent orchestrator function
- * @param {Object}   options.brain - Wall-E brain instance (for DB / keys)
- * @param {Object}   options.provider - LLM provider client
- * @param {string}   options.model - Model ID to use
- * @param {number}   options.timeoutMs - Timeout in ms (default 900000 = 15min)
- * @returns {Object} Result with { taskId, success, score, error, ... }
- */
-async function runSWEBenchTask(task, options = {}) {
-  const { runAgentLoop, brain, provider, model, timeoutMs = 900_000 } = options;
-
-  const mapped = typeof task.prompt === 'string' ? task : mapTaskToPrompt(task);
-  const startTime = Date.now();
-
-  // Check Docker availability
-  let dockerAvailable = false;
-  try {
-    const docker = require('./swebench-docker');
-    dockerAvailable = await docker.isDockerAvailable();
-  } catch {
-    dockerAvailable = false;
-  }
-
-  if (!dockerAvailable) {
-    return {
-      taskId: mapped.id,
-      suite: 'swebench-lite',
-      success: false,
-      error: 'Docker not available — SWE-bench tasks require Docker for sandboxed execution',
-      score: { composite: 0, dimensions: {} },
-      elapsedMs: Date.now() - startTime,
-    };
-  }
-
-  // Docker-based execution
-  const docker = require('./swebench-docker');
-  let containerId = null;
-
-  try {
-    // 1. Build container with the repo at the base commit
-    containerId = await docker.buildContainer(mapped.repo, mapped.baseCommit);
-
-    // 2. Apply test patch so the new tests exist
-    if (mapped.testPatch) {
-      await docker.applyPatch(containerId, mapped.testPatch);
-    }
-
-    // 3. Run the agent to produce a fix
-    // The agent receives the issue description and must produce a patch
-    let agentResult = null;
-    if (runAgentLoop) {
-      try {
-        let timeoutHandle;
-        agentResult = await Promise.race([
-          runAgentLoop(mapped.prompt, {
-            brain,
-            provider,
-            model,
-            maxTurns: 30,
-            persistTranscript: false,
-          }),
-          new Promise((_, reject) => {
-            timeoutHandle = setTimeout(() => reject(new Error('Agent timeout')), timeoutMs);
-            if (typeof timeoutHandle.unref === 'function') timeoutHandle.unref();
-          }),
-        ]);
-        if (timeoutHandle) clearTimeout(timeoutHandle);
-      } catch (err) {
-        return {
-          taskId: mapped.id,
-          suite: 'swebench-lite',
-          success: false,
-          error: `Agent execution failed: ${err.message}`,
-          score: { composite: 0, dimensions: {} },
-          elapsedMs: Date.now() - startTime,
-        };
-      }
-    }
-
-    // 4. Apply agent's produced patch
-    if (agentResult?.patch) {
-      await docker.applyPatch(containerId, agentResult.patch);
-    }
-
-    // 5. Run tests to determine pass/fail
-    const testResult = await docker.runTests(containerId, 'python -m pytest --tb=short -q');
-    const passed = testResult.exitCode === 0;
-
-    // 6. Score with Wall-E's 11-dimension scorer if available
-    let score = { composite: passed ? 1.0 : 0.0, dimensions: {} };
-    try {
-      const { scoreAgentResult } = require('./agent-scorer');
-      if (scoreAgentResult && agentResult) {
-        score = scoreAgentResult({
-          success: passed,
-          actualToolCalls: agentResult.toolCalls || [],
-          turnCount: agentResult.turns || 0,
-          fileChanges: agentResult.fileChanges || [],
-        }, {
-          expectedToolCalls: ['read_file', 'edit_file', 'glob'],
-          maxTurns: 30,
-        });
-        // Override correctness with binary pass/fail from tests
-        if (score.dimensions) {
-          score.dimensions.correctness = passed ? 1.0 : 0.0;
-        }
-      }
-    } catch {
-      // Scorer not available; binary score is fine
-    }
-
-    return {
-      taskId: mapped.id,
-      suite: 'swebench-lite',
-      success: passed,
-      score,
-      testOutput: testResult.stdout?.slice(0, 2000),
-      agentTurns: agentResult?.turns || 0,
-      elapsedMs: Date.now() - startTime,
-    };
-  } finally {
-    // Cleanup container
-    if (containerId) {
-      try { await docker.cleanup(containerId); } catch {}
-    }
-  }
-}
-
-/**
- * Load the curated 30-task subset (swebench-lite-30.json).
- * These are selected for diversity across repos, difficulty, and category.
- */
-async function loadCuratedSubset() {
-  const filePath = path.join(__dirname, 'benchmarks', 'swebench-lite-30.json');
-  return JSON.parse(fs.readFileSync(filePath, 'utf8'));
-}
-
-async function runSWEBenchSuite(options = {}) {
-  const {
-    brain,
-    runAgentLoop,
-    provider,
-    providerType,
-    model,
-    maxTasks,
-    signal,
-    timeoutMs,
-    runId: providedRunId,
-  } = options;
-
-  const allTasks = await loadCuratedSubset();
-  const tasks = maxTasks ? allTasks.slice(0, maxTasks) : allTasks;
-  const runId = providedRunId || crypto.randomUUID();
-  const results = [];
-  let totalPassed = 0;
-
-  for (const rawTask of tasks) {
-    if (signal?.aborted) break;
-
-    const mapped = mapTaskToPrompt(rawTask);
-    const result = await runSWEBenchTask(mapped, {
-      brain,
-      runAgentLoop,
-      provider,
-      model,
-      timeoutMs,
-    });
-    results.push(result);
-    if (result.success) totalPassed++;
-
-    if (brain && typeof brain.insertBenchmarkResult === 'function') {
-      try {
-        const scoringMethod = 'swebench-docker-tests';
-        brain.insertBenchmarkResult(decorateBenchmarkResult({
-          runId,
-          suite: 'swebench-lite',
-          promptId: mapped.id,
-          taskType: 'coding-agent',
-          difficulty: rawTask.difficulty || 'medium',
-          provider: providerType || 'unknown',
-          model: resolveModelName(model),
-          prompt: mapped.prompt,
-          response: result.testOutput || '',
-          traitScore: null,
-          compositeScore: result.score?.composite || 0,
-          latencyMs: result.elapsedMs || null,
-          error: result.error || null,
-          testsBefore: null,
-          testsAfter: result.success ? 1 : 0,
-          totalTests: result.error ? 0 : 1,
-          dimensionsJson: JSON.stringify(result.score?.dimensions || {}),
-          modelMetadataJson: JSON.stringify({
-            repo: mapped.repo,
-            baseCommit: mapped.baseCommit,
-            agentTurns: result.agentTurns || 0,
-            testOutput: result.testOutput || null,
-          }),
-          datasetVersion: 'swebench-lite:curated-30',
-          scorerVersion: DEFAULT_SCORER_VERSION,
-          scoringMethod,
-          trusted: !result.error,
-          runConfig: { maxTasks, timeoutMs, scoringMethod },
-        }, {
-          suite: 'swebench-lite',
-          benchmark: {
-            id: mapped.id,
-            prompt: mapped.prompt,
-            taskType: 'coding-agent',
-            difficulty: rawTask.difficulty || 'medium',
-            datasetVersion: 'swebench-lite:curated-30',
-          },
-          runId,
-          provider: providerType || 'unknown',
-          model: resolveModelName(model),
-          scorerVersion: DEFAULT_SCORER_VERSION,
-          scoringMethod,
-          trusted: !result.error,
-          runConfig: { maxTasks, timeoutMs, scoringMethod },
-        }));
-      } catch {}
-    }
-  }
-
-  return {
-    runId,
-    suite: 'swebench-lite',
-    model: resolveModelName(model),
-    totalTasks: tasks.length,
-    passed: totalPassed,
-    passAt1: tasks.length > 0 ? totalPassed / tasks.length : 0,
-    avgScore: results.reduce((s, r) => s + (r.score?.composite || 0), 0) / Math.max(results.length, 1),
-    totalCost: results.reduce((s, r) => s + (r.costDollars || 0), 0),
-    results,
-  };
-}
-
-module.exports = {
-  downloadDataset,
-  mapTaskToPrompt,
-  runSWEBenchTask,
-  runSWEBenchSuite,
-  loadCuratedSubset,
-  CACHE_DIR,
-};

package/template/wall-e/eval/swebench-docker.js

@@ -1,192 +0,0 @@
-'use strict';
-
-/**
- * Docker container lifecycle management for SWE-bench tasks.
- *
- * Each task gets an isolated container with the target repo checked
- * out at the specified base commit. The agent's patch is applied
- * inside the container, then tests are run to determine pass/fail.
- */
-
-const { execFile } = require('child_process');
-const { promisify } = require('util');
-const crypto = require('crypto');
-const fs = require('fs');
-const os = require('os');
-const path = require('path');
-const execFileAsync = promisify(execFile);
-
-// Strict validation patterns for inputs interpolated into shell commands
-const REPO_PATTERN = /^[\w.-]+\/[\w.-]+$/;
-const COMMIT_PATTERN = /^[0-9a-f]{7,40}$/i;
-
-const DOCKER_TIMEOUT = 5000;
-const DEFAULT_RUN_TIMEOUT = 900_000; // 15 minutes
-
-/**
- * Check whether Docker is installed and the daemon is running.
- */
-async function isDockerAvailable() {
-  try {
-    await execFileAsync('docker', ['info'], { timeout: DOCKER_TIMEOUT });
-    return true;
-  } catch {
-    return false;
-  }
-}
-
-/**
- * Build a Docker container for the given repo at the given commit.
- *
- * Strategy: pull a Python base image, clone the repo, checkout the
- * commit, install dependencies, and return the container ID.
- *
- * @param {string} repo - GitHub repo in "owner/name" format (e.g. "django/django")
- * @param {string} baseCommit - Git commit SHA to check out
- * @returns {string} Container ID
- */
-async function buildContainer(repo, baseCommit) {
-  // Validate inputs to prevent shell injection
-  if (!REPO_PATTERN.test(repo)) {
-    throw new Error(`Invalid repo format: ${repo} (expected owner/name)`);
-  }
-  if (!COMMIT_PATTERN.test(baseCommit)) {
-    throw new Error(`Invalid commit SHA: ${baseCommit}`);
-  }
-
-  const containerName = `swebench-${crypto.randomUUID().slice(0, 8)}`;
-
-  // Create a container from python:3.11-slim, clone repo at commit
-  const setupScript = [
-    'apt-get update -qq && apt-get install -y -qq git > /dev/null 2>&1',
-    `git clone --quiet https://github.com/${repo}.git /workspace`,
-    `cd /workspace && git checkout ${baseCommit}`,
-    'cd /workspace && if [ -f requirements.txt ]; then pip install -q -r requirements.txt; fi',
-    'cd /workspace && if [ -f setup.py ]; then pip install -q -e .; fi',
-    'cd /workspace && if [ -f pyproject.toml ]; then pip install -q -e . 2>/dev/null || true; fi',
-  ].join(' && ');
-
-  // Start a long-running container
-  const { stdout: containerId } = await execFileAsync('docker', [
-    'run', '-d',
-    '--name', containerName,
-    '--workdir', '/workspace',
-    'python:3.11-slim',
-    'bash', '-c', `${setupScript} && tail -f /dev/null`,
-  ], { timeout: 300_000 }); // 5 min for clone + install
-
-  const id = containerId.trim();
-
-  // Wait for setup to complete by polling for the workspace dir
-  let ready = false;
-  for (let i = 0; i < 60; i++) {
-    try {
-      await execFileAsync('docker', [
-        'exec', id, 'test', '-d', '/workspace/.git',
-      ], { timeout: 5000 });
-      ready = true;
-      break;
-    } catch {
-      await new Promise(r => setTimeout(r, 5000));
-    }
-  }
-
-  if (!ready) {
-    await cleanup(id).catch(() => {});
-    throw new Error(`Container setup timed out for ${repo}@${baseCommit}`);
-  }
-
-  return id;
-}
-
-/**
- * Execute a command inside the container and return stdout/stderr.
- *
- * @param {string} containerId
- * @param {string} command - Shell command to run
- * @param {Object} opts
- * @param {number} opts.timeoutMs - Timeout in ms (default 15min)
- * @returns {{ stdout: string, stderr: string, exitCode: number }}
- */
-async function runInContainer(containerId, command, { timeoutMs = DEFAULT_RUN_TIMEOUT } = {}) {
-  try {
-    const { stdout, stderr } = await execFileAsync('docker', [
-      'exec', containerId, 'bash', '-c', command,
-    ], { timeout: timeoutMs, maxBuffer: 10 * 1024 * 1024 });
-    return { stdout, stderr, exitCode: 0 };
-  } catch (err) {
-    return {
-      stdout: err.stdout || '',
-      stderr: err.stderr || err.message,
-      exitCode: err.code || 1,
-    };
-  }
-}
-
-/**
- * Apply a git patch inside the container.
- *
- * @param {string} containerId
- * @param {string} patch - Unified diff / git patch content
- */
-async function applyPatch(containerId, patch) {
-  // Write patch to a local temp file, then docker cp it in (avoids shell injection)
-  const tmpFile = path.join(os.tmpdir(), `swebench-patch-${crypto.randomUUID().slice(0, 8)}.patch`);
-  try {
-    fs.writeFileSync(tmpFile, patch);
-    await execFileAsync('docker', ['cp', tmpFile, `${containerId}:/tmp/agent.patch`], { timeout: 15_000 });
-  } finally {
-    try { fs.unlinkSync(tmpFile); } catch {}
-  }
-
-  const result = await runInContainer(containerId,
-    'cd /workspace && git apply /tmp/agent.patch',
-    { timeoutMs: 30_000 },
-  );
-
-  if (result.exitCode !== 0) {
-    // Try with --3way as fallback
-    const fallback = await runInContainer(containerId,
-      'cd /workspace && git apply --3way /tmp/agent.patch',
-      { timeoutMs: 30_000 },
-    );
-    if (fallback.exitCode !== 0) {
-      throw new Error(`Failed to apply patch: ${fallback.stderr.slice(0, 500)}`);
-    }
-  }
-}
-
-/**
- * Run the test suite inside the container.
- *
- * @param {string} containerId
- * @param {string} testCommand - e.g. "python -m pytest --tb=short -q"
- * @returns {{ stdout: string, stderr: string, exitCode: number }}
- */
-async function runTests(containerId, testCommand) {
-  return runInContainer(containerId, `cd /workspace && ${testCommand}`, {
-    timeoutMs: DEFAULT_RUN_TIMEOUT,
-  });
-}
-
-/**
- * Stop and remove the container.
- *
- * @param {string} containerId
- */
-async function cleanup(containerId) {
-  try {
-    await execFileAsync('docker', ['rm', '-f', containerId], { timeout: 15_000 });
-  } catch {
-    // Best-effort cleanup
-  }
-}
-
-module.exports = {
-  isDockerAvailable,
-  buildContainer,
-  runInContainer,
-  applyPatch,
-  runTests,
-  cleanup,
-};