create-walle 0.9.21 → 0.9.23

package/template/wall-e/eval/meta-harness/experience-store.js

@@ -1,174 +0,0 @@
-'use strict';
-
-const fs = require('node:fs');
-const path = require('node:path');
-const crypto = require('node:crypto');
-
-const SAFE_SEGMENT_RE = /^[A-Za-z0-9][A-Za-z0-9_.:-]*$/;
-
-class ExperienceStore {
-  constructor({ domainSpec, rootDir, runName, now = () => new Date().toISOString() } = {}) {
-    if (!domainSpec || typeof domainSpec !== 'object') throw new Error('domainSpec is required');
-    this.domainSpec = domainSpec;
-    this.now = now;
-    this.rootDir = path.resolve(rootDir || domainSpec.paths?.runRoot || path.join(process.cwd(), 'wall-e', 'eval', 'meta-runs'));
-    this.runName = safeSegment(runName || `${domainSpec.id}-${timestampForName(now())}`, 'runName');
-    this.runDir = path.join(this.rootDir, this.runName);
-    this.paths = {
-      rootDir: this.rootDir,
-      runDir: this.runDir,
-      candidatesDir: path.join(this.runDir, 'candidates'),
-      iterationsDir: path.join(this.runDir, 'iterations'),
-      reportsDir: path.join(this.runDir, 'reports'),
-      telemetryFile: path.join(this.runDir, 'telemetry.jsonl'),
-      manifestFile: path.join(this.runDir, 'run_manifest.json'),
-      experienceFile: path.join(this.runDir, 'experience.jsonl'),
-      frontierFile: path.join(this.runDir, 'frontier.json'),
-    };
-  }
-
-  ensureRun(extraManifest = {}) {
-    for (const dir of [
-      this.paths.runDir,
-      this.paths.candidatesDir,
-      this.paths.iterationsDir,
-      this.paths.reportsDir,
-    ]) {
-      fs.mkdirSync(dir, { recursive: true });
-    }
-    if (!fs.existsSync(this.paths.manifestFile)) {
-      this.writeRunManifest(extraManifest);
-    }
-    return this.paths;
-  }
-
-  writeRunManifest(extra = {}) {
-    const manifest = {
-      runName: this.runName,
-      domainId: this.domainSpec.id,
-      taskType: this.domainSpec.taskType,
-      createdAt: this.now(),
-      domainSpecHash: this.domainSpec.specHash || null,
-      domainSpecPath: this.domainSpec.specPath || null,
-      projectRoot: this.domainSpec.paths?.projectRoot || this.domainSpec.projectRoot || null,
-      ...extra,
-    };
-    this.writeJsonAbsolute(this.paths.manifestFile, manifest);
-    return manifest;
-  }
-
-  iterationDir(iteration) {
-    return path.join(this.paths.iterationsDir, `iter-${padIteration(iteration)}`);
-  }
-
-  candidateDir(iteration, candidateId) {
-    const dir = path.join(this.iterationDir(iteration), 'candidates', safeSegment(candidateId, 'candidateId'));
-    fs.mkdirSync(dir, { recursive: true });
-    return dir;
-  }
-
-  evalTaskDir(iteration, candidateId, taskId) {
-    const dir = path.join(this.candidateDir(iteration, candidateId), 'tasks', safeSegment(taskId, 'taskId'));
-    fs.mkdirSync(dir, { recursive: true });
-    return dir;
-  }
-
-  writeCandidateFile(candidateId, fileName, contents) {
-    const dir = path.join(this.paths.candidatesDir, safeSegment(candidateId, 'candidateId'));
-    fs.mkdirSync(dir, { recursive: true });
-    const target = path.join(dir, safeSegment(fileName, 'fileName'));
-    atomicWrite(target, String(contents || ''));
-    return target;
-  }
-
-  writeJson(relativePath, data) {
-    const target = this.resolveRunPath(relativePath);
-    this.writeJsonAbsolute(target, data);
-    return target;
-  }
-
-  writeJsonAbsolute(target, data) {
-    atomicWrite(target, JSON.stringify(data, null, 2) + '\n');
-  }
-
-  readJson(relativePath, fallback = undefined) {
-    const target = this.resolveRunPath(relativePath);
-    if (!fs.existsSync(target)) return fallback;
-    return JSON.parse(fs.readFileSync(target, 'utf8'));
-  }
-
-  writeText(relativePath, text) {
-    const target = this.resolveRunPath(relativePath);
-    atomicWrite(target, String(text || ''));
-    return target;
-  }
-
-  appendJsonl(relativePath, row) {
-    const target = this.resolveRunPath(relativePath);
-    fs.mkdirSync(path.dirname(target), { recursive: true });
-    fs.appendFileSync(target, JSON.stringify(row) + '\n', 'utf8');
-    return target;
-  }
-
-  appendExperience(row) {
-    this.ensureRun();
-    return this.appendJsonl('experience.jsonl', {
-      timestamp: this.now(),
-      domainId: this.domainSpec.id,
-      ...row,
-    });
-  }
-
-  readJsonl(relativePath) {
-    const target = this.resolveRunPath(relativePath);
-    if (!fs.existsSync(target)) return [];
-    return fs.readFileSync(target, 'utf8')
-      .split('\n')
-      .filter((line) => line.trim())
-      .map((line) => JSON.parse(line));
-  }
-
-  resolveRunPath(relativePath) {
-    if (!relativePath || typeof relativePath !== 'string') throw new Error('relative path is required');
-    if (path.isAbsolute(relativePath)) throw new Error(`absolute paths are not allowed in run store: ${relativePath}`);
-    const target = path.resolve(this.runDir, relativePath);
-    if (!target.startsWith(this.runDir + path.sep) && target !== this.runDir) {
-      throw new Error(`path escapes run directory: ${relativePath}`);
-    }
-    return target;
-  }
-}
-
-function atomicWrite(target, contents) {
-  fs.mkdirSync(path.dirname(target), { recursive: true });
-  const suffix = crypto.randomBytes(6).toString('hex');
-  const tmp = `${target}.${process.pid}.${suffix}.tmp`;
-  fs.writeFileSync(tmp, contents, 'utf8');
-  fs.renameSync(tmp, target);
-}
-
-function padIteration(iteration) {
-  const n = Number(iteration);
-  if (!Number.isInteger(n) || n < 0) throw new Error(`invalid iteration: ${iteration}`);
-  return String(n).padStart(3, '0');
-}
-
-function safeSegment(value, label = 'segment') {
-  const text = String(value || '');
-  if (!SAFE_SEGMENT_RE.test(text) || text.includes('..') || /[\\/]/.test(text)) {
-    throw new Error(`invalid ${label}: ${text || '<empty>'}`);
-  }
-  return text;
-}
-
-function timestampForName(iso) {
-  return String(iso || new Date().toISOString()).replace(/[^0-9T]/g, '').slice(0, 15) || 'run';
-}
-
-module.exports = {
-  ExperienceStore,
-  SAFE_SEGMENT_RE,
-  atomicWrite,
-  padIteration,
-  safeSegment,
-};

package/template/wall-e/eval/meta-harness/frontier.js

@@ -1,96 +0,0 @@
-'use strict';
-
-const path = require('node:path');
-
-function getMetricValue(result, metricPath = 'score.composite') {
-  const parts = String(metricPath || '').split('.').filter(Boolean);
-  let value = result;
-  for (const part of parts) {
-    if (value == null) return null;
-    value = value[part];
-  }
-  const n = Number(value);
-  return Number.isFinite(n) ? n : null;
-}
-
-function summarizeCandidateResults({ candidateId, iteration, split, results = [], metricPath = 'score.composite' } = {}) {
-  const scored = results.map((result) => getMetricValue(result, metricPath)).filter((value) => value !== null);
-  const aggregateScore = scored.length > 0
-    ? scored.reduce((sum, value) => sum + value, 0) / scored.length
-    : 0;
-  return {
-    candidateId,
-    iteration,
-    split,
-    metricPath,
-    aggregateScore,
-    taskCount: results.length,
-    scoredTaskCount: scored.length,
-    successCount: results.filter((result) => result.success === true).length,
-    results: results.map((result) => ({
-      benchmarkId: result.benchmarkId || result.taskId || null,
-      success: result.success === true,
-      score: getMetricValue(result, metricPath),
-      artifactPath: result.artifactPath || null,
-      error: result.error || null,
-    })),
-  };
-}
-
-function updateFrontier(frontier = {}, candidateSummary = {}, domainSpec = {}) {
-  const higherIsBetter = domainSpec.metrics?.higherIsBetter !== false;
-  const current = normalizeFrontier(frontier);
-  const candidateScore = Number(candidateSummary.aggregateScore || 0);
-  const bestScore = current.bestCandidate ? Number(current.bestCandidate.aggregateScore || 0) : null;
-  const improves = bestScore === null || (higherIsBetter ? candidateScore > bestScore : candidateScore < bestScore);
-  const next = {
-    ...current,
-    metricPath: candidateSummary.metricPath || current.metricPath || domainSpec.metrics?.primary || 'score.composite',
-    updatedAt: new Date().toISOString(),
-    history: [...current.history, candidateSummary],
-  };
-  if (improves) next.bestCandidate = candidateSummary;
-  for (const result of candidateSummary.results || []) {
-    if (!result.benchmarkId || result.score === null) continue;
-    const existing = next.perTask[result.benchmarkId];
-    const resultImproves = !existing || (higherIsBetter ? result.score > existing.score : result.score < existing.score);
-    if (resultImproves) {
-      next.perTask[result.benchmarkId] = {
-        candidateId: candidateSummary.candidateId,
-        iteration: candidateSummary.iteration,
-        score: result.score,
-        success: result.success,
-        artifactPath: result.artifactPath,
-      };
-    }
-  }
-  return next;
-}
-
-function normalizeFrontier(frontier = {}) {
-  return {
-    metricPath: frontier.metricPath || 'score.composite',
-    bestCandidate: frontier.bestCandidate || null,
-    perTask: { ...(frontier.perTask || {}) },
-    history: Array.isArray(frontier.history) ? frontier.history.slice() : [],
-    updatedAt: frontier.updatedAt || null,
-  };
-}
-
-function readFrontier(store) {
-  return normalizeFrontier(store.readJson('frontier.json', {}));
-}
-
-function writeFrontier(store, frontier) {
-  store.writeJson('frontier.json', normalizeFrontier(frontier));
-  return path.join(store.runDir, 'frontier.json');
-}
-
-module.exports = {
-  getMetricValue,
-  summarizeCandidateResults,
-  updateFrontier,
-  normalizeFrontier,
-  readFrontier,
-  writeFrontier,
-};

package/template/wall-e/eval/meta-harness/harness-interface.js

@@ -1,90 +0,0 @@
-'use strict';
-
-const { validateCandidateModule } = require('./validation');
-
-const BASE_HARNESS_VERSION = 'walle-meta-harness-v1';
-
-function createBaseHarness(overrides = {}) {
-  const base = {
-    id: 'walle-coding-agent-base',
-    version: BASE_HARNESS_VERSION,
-    surfaces: {
-      systemPromptPrelude: '',
-      contextBuilder: null,
-      toolPolicy: null,
-      environmentBootstrap: null,
-      runtimeOptions: {},
-    },
-    transformPrompt(prompt, context = {}) {
-      const prelude = String(this.surfaces.systemPromptPrelude || '').trim();
-      const env = typeof this.surfaces.environmentBootstrap === 'function'
-        ? String(this.surfaces.environmentBootstrap(context) || '').trim()
-        : String(this.surfaces.environmentBootstrap || '').trim();
-      return [prelude, env, String(prompt || '')].filter(Boolean).join('\n\n');
-    },
-    buildRunOptions(context = {}) {
-      return {
-        ...(context.options || {}),
-        ...(this.surfaces.runtimeOptions || {}),
-      };
-    },
-  };
-  return mergeHarness(base, overrides);
-}
-
-function applyCandidate(baseHarness, candidateModule, context = {}) {
-  const candidate = validateCandidateModule(candidateModule, context);
-  const inputHarness = mergeHarness(createBaseHarness(), baseHarness || {});
-  const applied = candidate.apply(inputHarness, context);
-  if (!applied || typeof applied !== 'object') {
-    throw new Error(`Candidate ${candidate.manifest.id} apply() must return a harness object`);
-  }
-  const harness = mergeHarness(inputHarness, applied);
-  validateHarness(harness);
-  return {
-    ...harness,
-    candidateManifest: candidate.manifest,
-    candidateId: candidate.manifest.id,
-  };
-}
-
-function normalizeCandidateModule(candidateModule) {
-  if (!candidateModule || typeof candidateModule !== 'object') return candidateModule;
-  return candidateModule.default || candidateModule.candidate || candidateModule;
-}
-
-function validateHarness(harness) {
-  if (!harness || typeof harness !== 'object') throw new Error('harness must be an object');
-  if (typeof harness.id !== 'string' || !harness.id) throw new Error('harness.id is required');
-  if (!harness.surfaces || typeof harness.surfaces !== 'object') throw new Error('harness.surfaces is required');
-  if (typeof harness.transformPrompt !== 'function') throw new Error('harness.transformPrompt must be a function');
-  if (typeof harness.buildRunOptions !== 'function') throw new Error('harness.buildRunOptions must be a function');
-  return true;
-}
-
-function mergeHarness(base, overlay) {
-  const merged = {
-    ...base,
-    ...overlay,
-    surfaces: {
-      ...(base.surfaces || {}),
-      ...(overlay.surfaces || {}),
-      runtimeOptions: {
-        ...(base.surfaces?.runtimeOptions || {}),
-        ...(overlay.surfaces?.runtimeOptions || {}),
-      },
-    },
-  };
-  if (!overlay.transformPrompt && base.transformPrompt) merged.transformPrompt = base.transformPrompt;
-  if (!overlay.buildRunOptions && base.buildRunOptions) merged.buildRunOptions = base.buildRunOptions;
-  return merged;
-}
-
-module.exports = {
-  BASE_HARNESS_VERSION,
-  createBaseHarness,
-  applyCandidate,
-  normalizeCandidateModule,
-  validateHarness,
-  mergeHarness,
-};

package/template/wall-e/eval/meta-harness/leakage-guard.js

@@ -1,80 +0,0 @@
-'use strict';
-
-const fs = require('node:fs');
-const path = require('node:path');
-
-function getHeldoutTaskIds(domainSpec = {}) {
-  return new Set(domainSpec.datasets?.heldout?.taskIds || []);
-}
-
-function scanCandidateSourceForLeakage(source, domainSpec = {}) {
-  const text = String(source || '');
-  const violations = [];
-  for (const taskId of getHeldoutTaskIds(domainSpec)) {
-    if (taskId && text.includes(taskId)) violations.push({ type: 'heldout_task_id', value: taskId });
-  }
-  if (/held[-_]?out/i.test(text) && domainSpec.leakagePolicy?.proposerCanReadHeldout === false) {
-    violations.push({ type: 'heldout_reference', value: 'heldout' });
-  }
-  return violations;
-}
-
-function assertNoHeldoutLeakage({ source, domainSpec, candidateId = 'candidate' } = {}) {
-  const violations = scanCandidateSourceForLeakage(source, domainSpec);
-  if (violations.length > 0) {
-    const details = violations.map((violation) => `${violation.type}:${violation.value}`).join(', ');
-    throw new Error(`${candidateId} failed held-out leakage scan: ${details}`);
-  }
-  return true;
-}
-
-function assertProposerPathAllowed(filePath, { store, domainSpec } = {}) {
-  const absolute = path.resolve(filePath);
-  const runDir = path.resolve(store.runDir);
-  if (!absolute.startsWith(runDir + path.sep) && absolute !== runDir) {
-    throw new Error(`proposer path escapes run directory: ${filePath}`);
-  }
-  const rel = path.relative(runDir, absolute).split(path.sep).join('/');
-  if (domainSpec?.leakagePolicy?.proposerCanReadHeldout === false && /(^|\/)heldout(\/|$)/i.test(rel)) {
-    throw new Error(`proposer path is held-out protected: ${rel}`);
-  }
-  for (const taskId of getHeldoutTaskIds(domainSpec)) {
-    if (rel.includes(taskId)) throw new Error(`proposer path includes held-out task id ${taskId}: ${rel}`);
-  }
-  return true;
-}
-
-function listProposerVisibleArtifacts({ store, domainSpec } = {}) {
-  const out = [];
-  walk(store.runDir, (filePath) => {
-    try {
-      assertProposerPathAllowed(filePath, { store, domainSpec });
-      out.push(filePath);
-    } catch {
-      // Hidden from proposer context.
-    }
-  });
-  return out.sort();
-}
-
-function walk(dir, onFile) {
-  let entries = [];
-  try {
-    entries = fs.readdirSync(dir, { withFileTypes: true });
-  } catch {
-    return;
-  }
-  for (const entry of entries) {
-    const full = path.join(dir, entry.name);
-    if (entry.isDirectory()) walk(full, onFile);
-    else if (entry.isFile()) onFile(full);
-  }
-}
-
-module.exports = {
-  getHeldoutTaskIds,
-  scanCandidateSourceForLeakage,
-  assertNoHeldoutLeakage,
-  assertProposerPathAllowed,
-  listProposerVisibleArtifacts,
-};

package/template/wall-e/eval/meta-harness/optimizer.js

@@ -1,207 +0,0 @@
-'use strict';
-
-const path = require('node:path');
-
-const { ExperienceStore, padIteration, safeSegment } = require('./experience-store');
-const { CandidateStore } = require('./candidate-store');
-const { createBaseHarness, applyCandidate } = require('./harness-interface');
-const { recordMetaHarnessTelemetry } = require('./telemetry');
-const {
-  readFrontier,
-  summarizeCandidateResults,
-  updateFrontier,
-  writeFrontier,
-} = require('./frontier');
-const { writeIterationReport, writeRunSummary } = require('./reporting');
-
-async function runMetaHarness({
-  domainSpec,
-  store = null,
-  adapter,
-  proposer = null,
-  candidateStore = null,
-  maxIterations = null,
-  evaluateBaseline = true,
-} = {}) {
-  if (!domainSpec) throw new Error('domainSpec is required');
-  if (!adapter || typeof adapter.runTask !== 'function') throw new Error('adapter.runTask is required');
-  const runStore = store || new ExperienceStore({ domainSpec });
-  runStore.ensureRun({ optimizer: 'meta-harness' });
-  const candidates = candidateStore || new CandidateStore({ store: runStore, domainSpec });
-  let frontier = readFrontier(runStore);
-  const summaries = [];
-  recordMetaHarnessTelemetry(runStore, {
-    type: 'optimizer_start',
-    domainId: domainSpec.id,
-    maxIterations: maxIterations || domainSpec.budget?.maxIterations || 1,
-  });
-
-  if (evaluateBaseline && frontier.history.length === 0) {
-    const baselineSummary = await evaluateCandidate({
-      store: runStore,
-      adapter,
-      domainSpec,
-      iteration: 0,
-      candidateId: 'baseline',
-      harness: createBaseHarness(),
-      split: 'search',
-    });
-    frontier = updateFrontier(frontier, baselineSummary, domainSpec);
-    writeFrontier(runStore, frontier);
-    summaries.push(baselineSummary);
-  }
-
-  const iterations = Math.min(
-    Number(maxIterations || domainSpec.budget?.maxIterations || 1),
-    Number(domainSpec.budget?.maxIterations || maxIterations || 1),
-  );
-  for (let iteration = 1; iteration <= iterations; iteration++) {
-    const iterationSummaries = [];
-    const proposals = await resolveProposals(proposer, { iteration, store: runStore, domainSpec, frontier });
-    const limited = proposals.slice(0, Number(domainSpec.budget?.maxCandidatesPerIteration || proposals.length || 1));
-    for (const proposal of limited) {
-      const registered = candidates.registerCandidate({ ...proposal, iteration });
-      let harness;
-      try {
-        harness = applyCandidate(createBaseHarness(), registered.module, { domainSpec, iteration, store: runStore });
-      } catch (err) {
-        runStore.appendExperience({ event: 'candidate_rejected', candidateId: registered.id, iteration, error: err.message });
-        continue;
-      }
-
-      const smokeSummary = await evaluateCandidate({
-        store: runStore,
-        adapter,
-        domainSpec,
-        iteration,
-        candidateId: registered.id,
-        harness,
-        split: 'smoke',
-      });
-      summaries.push(smokeSummary);
-      iterationSummaries.push(smokeSummary);
-      const minimum = Number(domainSpec.metrics?.minimumViableScore ?? 0);
-      if (smokeSummary.aggregateScore < minimum) {
-        runStore.appendExperience({
-          event: 'candidate_smoke_rejected',
-          candidateId: registered.id,
-          iteration,
-          aggregateScore: smokeSummary.aggregateScore,
-          minimum,
-        });
-        continue;
-      }
-
-      const searchSummary = await evaluateCandidate({
-        store: runStore,
-        adapter,
-        domainSpec,
-        iteration,
-        candidateId: registered.id,
-        harness,
-        split: 'search',
-      });
-      frontier = updateFrontier(frontier, searchSummary, domainSpec);
-      writeFrontier(runStore, frontier);
-      summaries.push(searchSummary);
-      iterationSummaries.push(searchSummary);
-    }
-    writeIterationReport({ store: runStore, iteration, frontier, summaries: iterationSummaries });
-  }
-
-  writeRunSummary({ store: runStore, frontier, summaries });
-  recordMetaHarnessTelemetry(runStore, {
-    type: 'optimizer_complete',
-    domainId: domainSpec.id,
-    bestCandidate: frontier.bestCandidate?.candidateId || null,
-    bestScore: frontier.bestCandidate?.aggregateScore ?? null,
-    summaryCount: summaries.length,
-  });
-  return { store: runStore, frontier, summaries };
-}
-
-async function evaluateCandidate({ store, adapter, domainSpec, iteration, candidateId, harness, split = 'search' } = {}) {
-  const dataset = domainSpec.datasets?.[split];
-  if (!dataset) throw new Error(`Unknown dataset split: ${split}`);
-  const maxTasks = Number(domainSpec.budget?.maxTasksPerCandidate || dataset.taskIds.length);
-  const taskIds = dataset.taskIds.slice(0, maxTasks);
-  const results = [];
-  for (const taskId of taskIds) {
-    const artifactDir = store.evalTaskDir(iteration, candidateId, taskId);
-    const result = await adapter.runTask({
-      taskId,
-      candidateId,
-      harness,
-      artifactDir,
-      split,
-      domainSpec,
-    });
-    const normalized = {
-      taskId,
-      benchmarkId: result.benchmarkId || taskId,
-      ...result,
-      artifactPath: result.artifactPath || artifactDir,
-    };
-    results.push(normalized);
-    store.appendExperience({
-      event: 'task_evaluated',
-      candidateId,
-      iteration,
-      split,
-      taskId,
-      success: normalized.success === true,
-      score: normalized.score?.composite ?? null,
-      artifactPath: normalized.artifactPath,
-      error: normalized.error || null,
-    });
-    recordMetaHarnessTelemetry(store, {
-      type: 'task_evaluated',
-      candidateId,
-      iteration,
-      split,
-      taskId,
-      success: normalized.success === true,
-      score: normalized.score?.composite ?? null,
-    });
-  }
-  const summary = summarizeCandidateResults({
-    candidateId,
-    iteration,
-    split,
-    results,
-    metricPath: domainSpec.metrics?.primary || 'score.composite',
-  });
-  const summaryPath = path.join(
-    'iterations',
-    `iter-${padIteration(iteration)}`,
-    'candidates',
-    safeSegment(candidateId, 'candidateId'),
-    `${safeSegment(split, 'split')}_summary.json`,
-  );
-  store.writeJson(summaryPath, summary);
-  store.appendExperience({
-    event: 'candidate_evaluated',
-    candidateId,
-    iteration,
-    split,
-    aggregateScore: summary.aggregateScore,
-    taskCount: summary.taskCount,
-    successCount: summary.successCount,
-  });
-  return summary;
-}
-
-async function resolveProposals(proposer, context) {
-  if (!proposer) return [];
-  const proposals = typeof proposer === 'function'
-    ? await proposer(context)
-    : await proposer.propose(context);
-  if (!Array.isArray(proposals)) throw new Error('proposer must return an array of candidate proposals');
-  return proposals;
-}
-
-module.exports = {
-  runMetaHarness,
-  evaluateCandidate,
-  resolveProposals,
-};