create-walle 0.9.21 → 0.9.23

package/template/wall-e/eval/mail-integration.js

@@ -1,443 +0,0 @@
-'use strict';
-
-/**
- * mail-integration.js — Integration tests for mail tools.
- *
- * Unlike chat-eval.js (which mocks tool results), these tests call the REAL
- * searchMail(), getMailMessages(), and readMailMessage() functions against
- * the live macOS Mail app. They catch the class of bugs that mocks cannot:
- * - AppleScript/JXA timeouts on large mailboxes
- * - INBOX case sensitivity
- * - Sent folder name mismatches across accounts (Gmail vs iCloud vs Exchange)
- * - TCC/FDA permission failures
- * - Delimiter collisions in email content
- * - Envelope Index (Swift fast path) compilation & access
- *
- * Usage:
- *   node eval/mail-integration.js              # run all tests
- *   node eval/mail-integration.js --filter timeout   # run matching tests
- *   node eval/mail-integration.js --dry-run     # show tests without running
- */
-
-const path = require('path');
-const { execFileAsync } = (() => {
-  const { execFile } = require('child_process');
-  const { promisify } = require('util');
-  return { execFileAsync: promisify(execFile) };
-})();
-
-// Load real tool functions
-const {
-  getMailMessages, readMailMessage, searchMail,
-} = require('../tools/local-tools');
-
-const TIMEOUT_MS = 120000; // per-test timeout (Mail.app scripting bridge takes ~24-47s on large mailboxes, up to 120s without FDA)
-
-// ============================================================
-// Test helpers
-// ============================================================
-
-async function runWithTimeout(fn, ms) {
-  return Promise.race([
-    fn(),
-    new Promise((_, reject) => setTimeout(() => reject(new Error(`Timed out after ${ms}ms`)), ms)),
-  ]);
-}
-
-function assert(condition, msg) {
-  if (!condition) throw new Error(`Assertion failed: ${msg}`);
-}
-
-// ============================================================
-// Test cases — each returns { passed, detail } or throws
-// ============================================================
-
-const TESTS = [
-  // --- Basic functionality ---
-  {
-    id: 'mail-int-01',
-    name: 'getMailMessages returns structured JSON with count and messages array',
-    tags: ['basic', 'mail_messages'],
-    async run() {
-      const result = await runWithTimeout(() => getMailMessages({ hours_back: 72, limit: 5 }), TIMEOUT_MS);
-      assert(typeof result === 'object', 'result should be an object');
-      assert('count' in result || 'error' in result, 'result should have count or error');
-      if (result.error) {
-        // Acceptable errors: Mail not running, automation denied, scripting bridge timeout
-        assert(
-          /automation|not running|timed out|timeout|Full Disk|Command failed/i.test(result.error),
-          `Unexpected error: ${result.error}`
-        );
-        return { passed: true, detail: `Expected error: ${result.error.slice(0, 100)}` };
-      }
-      assert(typeof result.count === 'number', 'count should be a number');
-      assert(Array.isArray(result.messages), 'messages should be an array');
-      assert(result.count === result.messages.length, `count (${result.count}) should match messages.length (${result.messages.length})`);
-      if (result.messages.length > 0) {
-        const msg = result.messages[0];
-        assert('subject' in msg, 'message should have subject');
-        assert('from' in msg, 'message should have from');
-        assert('date' in msg, 'message should have date');
-        assert('read' in msg, 'message should have read status');
-        assert('messageId' in msg, 'message should have messageId');
-      }
-      const extra = result.slow_accounts ? ` (slow: ${result.slow_accounts.join(', ')})` : '';
-      return { passed: true, detail: `Got ${result.count} messages${extra}` };
-    },
-  },
-
-  {
-    id: 'mail-int-02',
-    name: 'getMailMessages completes within 90 seconds (performance regression)',
-    tags: ['performance', 'mail_messages', 'timeout'],
-    async run() {
-      const start = Date.now();
-      const result = await runWithTimeout(() => getMailMessages({ hours_back: 24, limit: 10 }), 90000);
-      const elapsed = Date.now() - start;
-      if (result.error && /automation|not running|timed out|timeout|Full Disk/i.test(result.error)) {
-        return { passed: true, detail: `Skipped (expected): ${result.error.slice(0, 80)}` };
-      }
-      assert(elapsed < 90000, `Took ${elapsed}ms — should be under 90s.`);
-      const extra = result.slow_accounts ? ` (slow: ${result.slow_accounts.join(', ')})` : '';
-      return { passed: true, detail: `${elapsed}ms for ${result.count} messages${extra}` };
-    },
-  },
-
-  {
-    id: 'mail-int-03',
-    name: 'getMailMessages with INBOX (uppercase) returns results',
-    tags: ['basic', 'mail_messages', 'inbox-case'],
-    async run() {
-      const result = await runWithTimeout(() => getMailMessages({ hours_back: 24, mailbox: 'INBOX', limit: 3 }), TIMEOUT_MS);
-      if (result.error && /automation|not running|timed out|timeout|Full Disk/i.test(result.error)) {
-        return { passed: true, detail: `Skipped: ${result.error.slice(0, 80)}` };
-      }
-      // INBOX (uppercase) should work on all accounts (may have 0 results if all accounts were slow)
-      assert(!result.error, `INBOX (uppercase) failed: ${result.error}`);
-      const extra = result.slow_accounts ? ` (slow: ${result.slow_accounts.join(', ')})` : '';
-      return { passed: true, detail: `INBOX returned ${result.count} messages${extra}` };
-    },
-  },
-
-  {
-    id: 'mail-int-04',
-    name: 'getMailMessages with lowercase inbox should also work (case sensitivity bug)',
-    tags: ['regression', 'mail_messages', 'inbox-case'],
-    async run() {
-      // This was the original bug: lowercase "inbox" silently returned 0 results on some accounts
-      const upper = await runWithTimeout(() => getMailMessages({ hours_back: 24, mailbox: 'INBOX', limit: 5 }), TIMEOUT_MS);
-      if (upper.error && /automation|not running|timed out|timeout|Full Disk/i.test(upper.error)) {
-        return { passed: true, detail: `Skipped: ${upper.error.slice(0, 80)}` };
-      }
-      // The tool now defaults to 'INBOX' — verify the default is uppercase
-      const defaultResult = await runWithTimeout(() => getMailMessages({ hours_back: 24, limit: 5 }), TIMEOUT_MS);
-      if (defaultResult.error) {
-        return { passed: true, detail: `Skipped: ${defaultResult.error.slice(0, 80)}` };
-      }
-      // Default should return same or similar count as explicit INBOX
-      assert(
-        Math.abs(upper.count - defaultResult.count) <= 2,
-        `Default mailbox (${defaultResult.count}) vs INBOX (${upper.count}) differ significantly — possible case sensitivity issue`
-      );
-      return { passed: true, detail: `Default=${defaultResult.count}, INBOX=${upper.count}` };
-    },
-  },
-
-  // --- searchMail ---
-  {
-    id: 'mail-int-05',
-    name: 'searchMail returns structured JSON with count and messages array',
-    tags: ['basic', 'mail_search'],
-    async run() {
-      // Search for something common — "re:" exists in most inboxes
-      const result = await runWithTimeout(() => searchMail({ query: 're:', days_back: 7, limit: 3 }), TIMEOUT_MS);
-      assert(typeof result === 'object', 'result should be an object');
-      assert('count' in result || 'error' in result, 'result should have count or error');
-      if (result.error) {
-        assert(
-          /automation|not running|timed out|timeout|Full Disk|Command failed/i.test(result.error),
-          `Unexpected error: ${result.error}`
-        );
-        return { passed: true, detail: `Expected error: ${result.error.slice(0, 100)}` };
-      }
-      assert(typeof result.count === 'number', 'count should be a number');
-      assert(Array.isArray(result.messages), 'messages should be an array');
-      if (result.messages.length > 0) {
-        const msg = result.messages[0];
-        assert('subject' in msg, 'message should have subject');
-        assert('from' in msg || 'sender' in msg, 'message should have from/sender');
-        assert('date' in msg, 'message should have date');
-      }
-      return { passed: true, detail: `Got ${result.count} results` };
-    },
-  },
-
-  {
-    id: 'mail-int-06',
-    name: 'searchMail completes within 45 seconds with days_back=30 (performance regression)',
-    tags: ['performance', 'mail_search', 'timeout'],
-    async run() {
-      const start = Date.now();
-      const result = await runWithTimeout(() => searchMail({ query: 'test', days_back: 30, limit: 5 }), 45000);
-      const elapsed = Date.now() - start;
-      if (result.error && /automation|not running|timed out|timeout|Full Disk/i.test(result.error)) {
-        return { passed: true, detail: `Skipped: ${result.error.slice(0, 60)}` };
-      }
-      assert(elapsed < 45000, `Took ${elapsed}ms — should be under 45s. The JXA whose clause is O(n) on large mailboxes.`);
-      return { passed: true, detail: `${elapsed}ms for ${result.count} results` };
-    },
-  },
-
-  {
-    id: 'mail-int-07',
-    name: 'searchMail with empty query and no sender returns error',
-    tags: ['validation', 'mail_search'],
-    async run() {
-      const result = await searchMail({});
-      assert(result.error, 'Empty search should return an error');
-      assert(result.count === 0, 'Empty search should return count 0');
-      return { passed: true, detail: `Error: ${result.error}` };
-    },
-  },
-
-  {
-    id: 'mail-int-08',
-    name: 'searchMail with include_content returns email body',
-    tags: ['basic', 'mail_search', 'include-content'],
-    async run() {
-      const result = await runWithTimeout(() => searchMail({ query: 're:', days_back: 7, limit: 1, include_content: true }), TIMEOUT_MS);
-      if (result.error && /automation|not running|timed out|timeout|Full Disk/i.test(result.error)) {
-        return { passed: true, detail: `Skipped: ${result.error.slice(0, 60)}` };
-      }
-      if (result.count === 0) {
-        return { passed: true, detail: 'No messages found to verify content' };
-      }
-      const msg = result.messages[0];
-      assert('content' in msg, 'Message should have content when include_content=true');
-      assert(typeof msg.content === 'string', 'Content should be a string');
-      return { passed: true, detail: `Got content (${msg.content.length} chars) for "${msg.subject?.slice(0, 40)}"` };
-    },
-  },
-
-  {
-    id: 'mail-int-09',
-    name: 'searchMail searches sent folder (cross-provider folder name bug)',
-    tags: ['regression', 'mail_search', 'sent-folder'],
-    async run() {
-      // This was the bug: "sent messages" only works for iCloud, not Gmail/Exchange
-      const result = await runWithTimeout(() => searchMail({ query: 're:', days_back: 30, limit: 5 }), TIMEOUT_MS);
-      if (result.error && /automation|not running|timed out|timeout|Full Disk/i.test(result.error)) {
-        return { passed: true, detail: `Skipped: ${result.error.slice(0, 60)}` };
-      }
-      // Check if any results come from a sent folder
-      const sentMsgs = result.messages.filter(m =>
-        /sent/i.test(m.mailbox || '') || /sent/i.test(m.folder || '')
-      );
-      // We can't guarantee sent results, but verify the structure is correct
-      return { passed: true, detail: `${result.count} results, ${sentMsgs.length} from sent folders` };
-    },
-  },
-
-  {
-    id: 'mail-int-10',
-    name: 'searchMail does not produce delimiter collisions in results',
-    tags: ['regression', 'mail_search', 'delimiter'],
-    async run() {
-      // This was the bug: emails containing " || " in content corrupted parsing
-      // The fix was to switch from AppleScript+delimiter to JXA+JSON
-      const result = await runWithTimeout(() => searchMail({ query: 're:', days_back: 7, limit: 10, include_content: true }), TIMEOUT_MS);
-      if (result.error && /automation|not running|timed out|timeout|Full Disk/i.test(result.error)) {
-        return { passed: true, detail: `Skipped: ${result.error.slice(0, 60)}` };
-      }
-      // Verify count matches actual array length (delimiter collision would inflate count)
-      assert(
-        result.count === result.messages.length,
-        `count (${result.count}) !== messages.length (${result.messages.length}) — possible delimiter collision`
-      );
-      // Verify each message has well-formed fields
-      for (const msg of result.messages) {
-        assert(typeof msg.subject === 'string', `Malformed subject: ${JSON.stringify(msg.subject)}`);
-        assert(typeof msg.date === 'string', `Malformed date: ${JSON.stringify(msg.date)}`);
-      }
-      return { passed: true, detail: `${result.count} results, all well-formed` };
-    },
-  },
-
-  // --- Swift fast path ---
-  {
-    id: 'mail-int-11',
-    name: 'Swift Envelope Index binary compiles or is cached',
-    tags: ['fast-path', 'swift'],
-    async run() {
-      const binPath = path.join(__dirname, '..', 'skills', '_bundled', 'email-sync', '.mail-search');
-      const srcPath = path.join(__dirname, '..', 'skills', '_bundled', 'email-sync', 'mail-search.swift');
-      const fs = require('fs');
-      assert(fs.existsSync(srcPath), `Swift source not found: ${srcPath}`);
-      // Try to compile if binary doesn't exist
-      try {
-        await execFileAsync('swiftc', ['-O', srcPath, '-o', binPath], { timeout: 30000 });
-        assert(fs.existsSync(binPath), 'Binary should exist after compilation');
-        return { passed: true, detail: 'Compiled successfully' };
-      } catch (err) {
-        if (/not found|ENOENT/.test(err.message)) {
-          return { passed: true, detail: 'Swift compiler not available — fast path skipped at runtime' };
-        }
-        return { passed: false, detail: `Compilation failed: ${err.message.slice(0, 100)}` };
-      }
-    },
-  },
-
-  {
-    id: 'mail-int-12',
-    name: 'Swift fast path returns results or gracefully falls back',
-    tags: ['fast-path', 'swift', 'tcc'],
-    async run() {
-      // The fast path requires Full Disk Access (FDA). If not granted, it should
-      // return null (not crash), and searchMail falls back to JXA.
-      const result = await runWithTimeout(() => searchMail({ query: 're:', days_back: 7, limit: 3 }), TIMEOUT_MS);
-      if (result.error && /automation|not running|timed out|timeout|Full Disk/i.test(result.error)) {
-        return { passed: true, detail: `Skipped: ${result.error.slice(0, 60)}` };
-      }
-      // The key assertion: searchMail should ALWAYS return valid JSON, regardless
-      // of whether the fast path worked or fell back
-      assert(typeof result === 'object', 'Should return object');
-      assert('count' in result, 'Should have count');
-      assert(Array.isArray(result.messages), 'Should have messages array');
-      const method = result.search_method || 'unknown';
-      return { passed: true, detail: `${result.count} results via ${method}` };
-    },
-  },
-
-  // --- Sender search ---
-  {
-    id: 'mail-int-13',
-    name: 'searchMail by sender only (no subject query)',
-    tags: ['basic', 'mail_search', 'sender'],
-    async run() {
-      // Search by sender alone — should work without a subject query
-      const result = await runWithTimeout(() => searchMail({ sender: '@', days_back: 7, limit: 3 }), TIMEOUT_MS);
-      if (result.error) {
-        if (/automation|not running|timed out|timeout|Full Disk|Command failed/i.test(result.error)) {
-          return { passed: true, detail: `Skipped: ${result.error.slice(0, 80)}` };
-        }
-        assert(false, `Sender-only search failed: ${result.error}`);
-      }
-      return { passed: true, detail: `${result.count} results for sender '@'` };
-    },
-  },
-
-  // --- Error handling ---
-  {
-    id: 'mail-int-14',
-    name: 'searchMail with SQL injection-like query does not crash',
-    tags: ['security', 'mail_search'],
-    async run() {
-      const result = await runWithTimeout(
-        () => searchMail({ query: '\'; DROP TABLE messages; --', days_back: 7, limit: 1 }),
-        TIMEOUT_MS
-      );
-      // Should return 0 results or an error, never crash
-      assert(typeof result === 'object', 'Should return object even with malicious input');
-      return { passed: true, detail: result.error ? `Error (expected): ${result.error.slice(0, 60)}` : `${result.count} results (safely handled)` };
-    },
-  },
-
-  {
-    id: 'mail-int-15',
-    name: 'searchMail with very large days_back still completes',
-    tags: ['performance', 'mail_search'],
-    async run() {
-      const start = Date.now();
-      const result = await runWithTimeout(() => searchMail({ query: 'xyznonexistent99', days_back: 365, limit: 1 }), TIMEOUT_MS);
-      const elapsed = Date.now() - start;
-      if (result.error && /automation|not running|timed out|timeout|Full Disk/i.test(result.error)) {
-        return { passed: true, detail: `Skipped: ${result.error.slice(0, 60)}` };
-      }
-      assert(elapsed < TIMEOUT_MS, `Took ${elapsed}ms with days_back=365 — potential full-scan`);
-      return { passed: true, detail: `${elapsed}ms with days_back=365, ${result.count} results` };
-    },
-  },
-];
-
-// ============================================================
-// Runner
-// ============================================================
-
-async function run(filterTag) {
-  const tests = filterTag
-    ? TESTS.filter(t => t.tags.includes(filterTag) || t.id.includes(filterTag) || t.name.toLowerCase().includes(filterTag.toLowerCase()))
-    : TESTS;
-
-  console.log(`\n=== Mail Integration Tests ===`);
-  console.log(`Running ${tests.length}/${TESTS.length} tests\n`);
-
-  let passed = 0;
-  let failed = 0;
-  let skipped = 0;
-  const failures = [];
-
-  for (const test of tests) {
-    process.stdout.write(`  [${test.id}] ${test.name.slice(0, 60).padEnd(60)} `);
-    try {
-      const result = await test.run();
-      if (result.passed) {
-        const isSkip = /^Skipped:/.test(result.detail);
-        if (isSkip) {
-          skipped++;
-          console.log(`[SKIP] ${result.detail}`);
-        } else {
-          passed++;
-          console.log(`[PASS] ${result.detail}`);
-        }
-      } else {
-        failed++;
-        console.log(`[FAIL] ${result.detail}`);
-        failures.push({ id: test.id, name: test.name, detail: result.detail });
-      }
-    } catch (err) {
-      failed++;
-      const detail = err.message.slice(0, 120);
-      console.log(`[FAIL] ${detail}`);
-      failures.push({ id: test.id, name: test.name, detail });
-    }
-  }
-
-  console.log(`\n${'='.repeat(60)}`);
-  console.log(`  ${passed} passed, ${failed} failed, ${skipped} skipped (${tests.length} total)`);
-  if (failures.length) {
-    console.log(`\n  Failures:`);
-    for (const f of failures) {
-      console.log(`    ${f.id}: ${f.detail}`);
-    }
-  }
-  console.log(`${'='.repeat(60)}\n`);
-
-  return { passed, failed, skipped, total: tests.length, failures };
-}
-
-// ============================================================
-// CLI
-// ============================================================
-
-if (require.main === module) {
-  const args = process.argv.slice(2);
-  const filter = args.find(a => a.startsWith('--filter='))?.split('=')[1]
-    || args.find(a => a.startsWith('--filter'))?.split(' ')[1]
-    || (args[0] && !args[0].startsWith('-') ? args[0] : null);
-  const dryRun = args.includes('--dry-run');
-
-  if (dryRun) {
-    console.log(`\nMail Integration Tests (${TESTS.length} total):\n`);
-    for (const t of TESTS) {
-      console.log(`  ${t.id}  [${t.tags.join(', ')}]`);
-      console.log(`    ${t.name}\n`);
-    }
-    process.exit(0);
-  }
-
-  run(filter)
-    .then(r => process.exit(r.failed > 0 ? 1 : 0))
-    .catch(err => { console.error('Fatal:', err); process.exit(2); });
-}
-
-module.exports = { run, TESTS };

package/template/wall-e/eval/manifest.js

@@ -1,186 +0,0 @@
-'use strict';
-
-const crypto = require('crypto');
-const path = require('path');
-const { execFileSync } = require('child_process');
-
-const DEFAULT_DATASET_VERSION = 'local-v1';
-const DEFAULT_SCORER_VERSION = 'wall-e-eval-v2';
-const DEFAULT_EVALUATOR_VERSION = 'wall-e-evaluator-v2';
-
-let cachedRepoSha;
-
-function stableStringify(value) {
-  if (value === null || typeof value !== 'object') return JSON.stringify(value);
-  if (Array.isArray(value)) return '[' + value.map(stableStringify).join(',') + ']';
-  return '{' + Object.keys(value).sort().map((key) => (
-    JSON.stringify(key) + ':' + stableStringify(value[key])
-  )).join(',') + '}';
-}
-
-function sha256(value) {
-  return crypto.createHash('sha256').update(String(value ?? '')).digest('hex');
-}
-
-function hashObject(value) {
-  return sha256(stableStringify(value));
-}
-
-function getRepoSha(cwd = path.resolve(__dirname, '..')) {
-  if (cachedRepoSha !== undefined) return cachedRepoSha;
-  try {
-    cachedRepoSha = execFileSync('git', ['rev-parse', 'HEAD'], {
-      cwd,
-      encoding: 'utf8',
-      stdio: ['ignore', 'pipe', 'ignore'],
-    }).trim() || null;
-  } catch {
-    cachedRepoSha = null;
-  }
-  return cachedRepoSha;
-}
-
-function safeJson(value) {
-  if (value == null) return null;
-  if (typeof value === 'string') return value;
-  try { return JSON.stringify(value); } catch { return null; }
-}
-
-function pickRunConfig(config = {}) {
-  const allowed = [
-    'temperature', 'seed', 'maxTokens', 'timeoutMs', 'concurrency',
-    'budgetDollars', 'suite', 'taskType', 'scoringMethod',
-  ];
-  const out = {};
-  for (const key of allowed) {
-    if (config[key] !== undefined) out[key] = config[key];
-  }
-  return out;
-}
-
-function samplePayloadForHash(benchmark = {}) {
-  return {
-    id: benchmark.id || benchmark.promptId || benchmark.benchmark_id || null,
-    prompt: benchmark.prompt || '',
-    taskType: benchmark.taskType || null,
-    difficulty: benchmark.difficulty || null,
-    expectedTraits: benchmark.expectedTraits || null,
-    expectedInReply: benchmark.expectedInReply || null,
-    agentExpectations: benchmark.agentExpectations || null,
-  };
-}
-
-function buildEvalManifest({
-  suite,
-  benchmark = {},
-  runId,
-  provider,
-  model,
-  runConfig = {},
-  scorerVersion = DEFAULT_SCORER_VERSION,
-  evaluatorVersion = DEFAULT_EVALUATOR_VERSION,
-  scoringMethod,
-  artifactPath,
-  trusted,
-} = {}) {
-  const sampleId = benchmark.sampleId || benchmark.id || benchmark.promptId || benchmark.benchmark_id || null;
-  const datasetVersion = benchmark.datasetVersion || `${suite || 'unknown'}:${DEFAULT_DATASET_VERSION}`;
-  const datasetHash = benchmark.datasetHash || hashObject({
-    suite: suite || 'unknown',
-    datasetVersion,
-    sample: samplePayloadForHash(benchmark),
-  });
-  const promptHash = sha256(benchmark.prompt || '');
-  const sanitizedConfig = pickRunConfig({ ...runConfig, suite, scoringMethod });
-  const repoSha = getRepoSha();
-
-  const manifest = {
-    runId: runId || null,
-    suite: suite || null,
-    datasetVersion,
-    datasetHash,
-    sampleId,
-    promptHash,
-    provider: provider || null,
-    model: model || null,
-    modelSnapshot: benchmark.modelSnapshot || model || null,
-    scorerVersion,
-    evaluatorVersion,
-    scoringMethod: scoringMethod || null,
-    repoSha,
-    runConfig: sanitizedConfig,
-    artifactPath: artifactPath || null,
-    trusted: trusted === undefined ? null : !!trusted,
-  };
-
-  return {
-    sampleId,
-    datasetVersion,
-    datasetHash,
-    promptHash,
-    repoSha,
-    scorerVersion,
-    evaluatorVersion,
-    scoringMethod: scoringMethod || null,
-    runConfigJson: safeJson(sanitizedConfig),
-    evalManifestJson: safeJson(manifest),
-    artifactPath: artifactPath || null,
-    modelSnapshot: benchmark.modelSnapshot || model || null,
-    temperature: sanitizedConfig.temperature ?? null,
-    seed: sanitizedConfig.seed ?? null,
-  };
-}
-
-function decorateBenchmarkResult(entry = {}, context = {}) {
-  const suite = entry.suite || context.suite;
-  const benchmark = {
-    ...(context.benchmark || {}),
-    id: entry.promptId || entry.benchmark_id || context.benchmark?.id,
-    prompt: entry.prompt || context.benchmark?.prompt,
-    taskType: entry.taskType || context.benchmark?.taskType,
-    difficulty: entry.difficulty || context.benchmark?.difficulty,
-  };
-  const manifest = buildEvalManifest({
-    suite,
-    benchmark,
-    runId: entry.runId || context.runId,
-    provider: entry.provider || context.provider,
-    model: entry.model || context.model,
-    runConfig: context.runConfig || {},
-    scorerVersion: entry.scorerVersion || context.scorerVersion,
-    evaluatorVersion: entry.evaluatorVersion || context.evaluatorVersion,
-    scoringMethod: entry.scoringMethod || context.scoringMethod,
-    artifactPath: entry.artifactPath || context.artifactPath,
-    trusted: entry.trusted ?? context.trusted,
-  });
-
-  return {
-    ...entry,
-    sampleId: entry.sampleId || manifest.sampleId,
-    datasetVersion: entry.datasetVersion || manifest.datasetVersion,
-    datasetHash: entry.datasetHash || manifest.datasetHash,
-    promptHash: entry.promptHash || manifest.promptHash,
-    repoSha: entry.repoSha || manifest.repoSha,
-    scorerVersion: entry.scorerVersion || manifest.scorerVersion,
-    evaluatorVersion: entry.evaluatorVersion || manifest.evaluatorVersion,
-    scoringMethod: entry.scoringMethod || manifest.scoringMethod,
-    runConfigJson: entry.runConfigJson || manifest.runConfigJson,
-    evalManifestJson: entry.evalManifestJson || manifest.evalManifestJson,
-    artifactPath: entry.artifactPath || manifest.artifactPath,
-    modelSnapshot: entry.modelSnapshot || manifest.modelSnapshot,
-    temperature: entry.temperature ?? manifest.temperature,
-    seed: entry.seed ?? manifest.seed,
-  };
-}
-
-module.exports = {
-  DEFAULT_DATASET_VERSION,
-  DEFAULT_SCORER_VERSION,
-  DEFAULT_EVALUATOR_VERSION,
-  stableStringify,
-  sha256,
-  hashObject,
-  getRepoSha,
-  buildEvalManifest,
-  decorateBenchmarkResult,
-};

package/template/wall-e/eval/meta-harness/adapters/coding-agent.js

@@ -1,57 +0,0 @@
-'use strict';
-
-const path = require('node:path');
-
-const { runAgentBenchmark } = require('../../agent-runner');
-
-function createCodingAgentAdapter({
-  runAgentLoop,
-  provider = null,
-  model = null,
-  timeoutMs = null,
-  benchmarks = null,
-} = {}) {
-  const suite = benchmarks || loadCodingAgentBenchmarks();
-  const byId = new Map(suite.map((benchmark) => [benchmark.id, benchmark]));
-  return {
-    id: 'coding-agent',
-    listTasks(taskIds = []) {
-      return taskIds.map((taskId) => {
-        const benchmark = byId.get(taskId);
-        if (!benchmark) throw new Error(`Unknown coding-agent benchmark: ${taskId}`);
-        return benchmark;
-      });
-    },
-    async runTask({ taskId, harness, candidateId, artifactDir, domainSpec } = {}) {
-      const benchmark = byId.get(taskId);
-      if (!benchmark) throw new Error(`Unknown coding-agent benchmark: ${taskId}`);
-      if (!runAgentLoop) throw new Error('runAgentLoop is required for coding-agent adapter');
-      const adaptedBenchmark = {
-        ...benchmark,
-        originalPrompt: benchmark.prompt,
-        prompt: harness?.transformPrompt
-          ? harness.transformPrompt(benchmark.prompt, { benchmark, taskId, candidateId, domainSpec })
-          : benchmark.prompt,
-      };
-      const baseOptions = { runAgentLoop, provider, model, timeoutMs, artifactDir };
-      const runOptions = harness?.buildRunOptions
-        ? harness.buildRunOptions({ benchmark: adaptedBenchmark, taskId, candidateId, domainSpec, options: baseOptions })
-        : baseOptions;
-      return runAgentBenchmark(adaptedBenchmark, {
-        ...baseOptions,
-        ...runOptions,
-        artifactDir,
-        runAgentLoop: runOptions.runAgentLoop || runAgentLoop,
-      });
-    },
-  };
-}
-
-function loadCodingAgentBenchmarks() {
-  return require(path.resolve(__dirname, '../../benchmarks/coding-agent.json'));
-}
-
-module.exports = {
-  createCodingAgentAdapter,
-  loadCodingAgentBenchmarks,
-};