create-walle 0.9.21 → 0.9.23

package/template/wall-e/eval/meta-harness/bootstrap-snapshot.js

@@ -1,149 +0,0 @@
-'use strict';
-
-const fs = require('node:fs');
-const path = require('node:path');
-const { execFileSync } = require('node:child_process');
-
-const DEFAULT_TIMEOUT_MS = 1500;
-const DEFAULT_MAX_OUTPUT_BYTES = 4096;
-
-function gatherEnvironmentSnapshot({
-  cwd = process.cwd(),
-  commands = defaultCommands(),
-  timeoutMs = DEFAULT_TIMEOUT_MS,
-  maxOutputBytes = DEFAULT_MAX_OUTPUT_BYTES,
-} = {}) {
-  const root = path.resolve(cwd);
-  const packageFiles = [
-    'package.json',
-    'pnpm-lock.yaml',
-    'package-lock.json',
-    'yarn.lock',
-    'requirements.txt',
-    'pyproject.toml',
-    'Cargo.toml',
-    'go.mod',
-  ].filter((name) => fs.existsSync(path.join(root, name)));
-  const entries = safeListDir(root).slice(0, 80);
-  const commandResults = commands.map((cmd) => runSnapshotCommand(cmd, {
-    cwd: root,
-    timeoutMs,
-    maxOutputBytes,
-  }));
-  return {
-    cwd: root,
-    packageFiles,
-    entries,
-    commands: commandResults,
-  };
-}
-
-function formatEnvironmentSnapshot(snapshot = {}) {
-  const lines = [
-    '# Environment Snapshot',
-    `cwd: ${snapshot.cwd || ''}`,
-    `package files: ${(snapshot.packageFiles || []).join(', ') || 'none detected'}`,
-    `top-level entries: ${(snapshot.entries || []).join(', ') || 'none'}`,
-    '',
-    '## Command Probes',
-  ];
-  for (const result of snapshot.commands || []) {
-    const status = result.ok ? 'ok' : 'unavailable';
-    const output = String(result.output || result.error || '').trim();
-    lines.push(`- ${result.label || result.command}: ${status}${output ? ` - ${singleLine(output)}` : ''}`);
-  }
-  return lines.join('\n');
-}
-
-function runSnapshotCommand(commandSpec, { cwd, timeoutMs, maxOutputBytes } = {}) {
-  const normalized = normalizeCommand(commandSpec);
-  try {
-    const output = execFileSync(normalized.command, normalized.args, {
-      cwd,
-      timeout: timeoutMs,
-      encoding: 'utf8',
-      stdio: ['ignore', 'pipe', 'pipe'],
-      maxBuffer: Math.max(maxOutputBytes * 2, 1024),
-    });
-    return {
-      label: normalized.label,
-      command: [normalized.command, ...normalized.args].join(' '),
-      ok: true,
-      output: limitBytes(redactSecrets(output), maxOutputBytes),
-    };
-  } catch (err) {
-    const stderr = err.stderr ? String(err.stderr) : '';
-    return {
-      label: normalized.label,
-      command: [normalized.command, ...normalized.args].join(' '),
-      ok: false,
-      error: limitBytes(redactSecrets(stderr || err.message || 'command failed'), maxOutputBytes),
-    };
-  }
-}
-
-function defaultCommands() {
-  return [
-    { label: 'node', command: 'node', args: ['--version'] },
-    { label: 'npm', command: 'npm', args: ['--version'] },
-    { label: 'python3', command: 'python3', args: ['--version'] },
-    { label: 'git', command: 'git', args: ['--version'] },
-    { label: 'go', command: 'go', args: ['version'] },
-    { label: 'rustc', command: 'rustc', args: ['--version'] },
-  ];
-}
-
-function normalizeCommand(commandSpec) {
-  if (typeof commandSpec === 'string') {
-    return { label: commandSpec, command: commandSpec, args: [] };
-  }
-  if (!commandSpec || typeof commandSpec !== 'object') throw new Error('command spec must be a string or object');
-  if (!commandSpec.command || typeof commandSpec.command !== 'string') throw new Error('command spec command is required');
-  return {
-    label: commandSpec.label || commandSpec.command,
-    command: commandSpec.command,
-    args: Array.isArray(commandSpec.args) ? commandSpec.args.map(String) : [],
-  };
-}
-
-function safeListDir(cwd) {
-  try {
-    return fs.readdirSync(cwd, { withFileTypes: true })
-      .filter((entry) => entry.name !== '.git' && entry.name !== 'node_modules')
-      .map((entry) => entry.isDirectory() ? `${entry.name}/` : entry.name)
-      .sort();
-  } catch {
-    return [];
-  }
-}
-
-function redactSecrets(text) {
-  return String(text || '')
-    .replace(/\b(sk-[A-Za-z0-9_-]{10,})\b/g, '[redacted-api-key]')
-    .replace(/\b(AKIA[0-9A-Z]{12,})\b/g, '[redacted-aws-key]')
-    .replace(/\b(xox[bpas]-[A-Za-z0-9-]{10,})\b/g, '[redacted-slack-token]')
-    .replace(/\b(ghp_[A-Za-z0-9_]{10,})\b/g, '[redacted-github-token]')
-    .replace(/\b([A-Z0-9_]*(?:KEY|TOKEN|SECRET|PASSWORD)[A-Z0-9_]*=)([^\s]+)/gi, '$1[redacted]');
-}
-
-function limitBytes(text, maxBytes) {
-  const value = String(text || '');
-  const max = Math.max(1, Number(maxBytes || DEFAULT_MAX_OUTPUT_BYTES));
-  if (Buffer.byteLength(value, 'utf8') <= max) return value;
-  return Buffer.from(value, 'utf8').subarray(0, max).toString('utf8') + '\n[snapshot output truncated]';
-}
-
-function singleLine(text) {
-  return String(text || '').replace(/\s+/g, ' ').trim();
-}
-
-module.exports = {
-  DEFAULT_TIMEOUT_MS,
-  DEFAULT_MAX_OUTPUT_BYTES,
-  gatherEnvironmentSnapshot,
-  formatEnvironmentSnapshot,
-  runSnapshotCommand,
-  defaultCommands,
-  redactSecrets,
-  limitBytes,
-};

package/template/wall-e/eval/meta-harness/candidate-store.js

@@ -1,117 +0,0 @@
-'use strict';
-
-const fs = require('node:fs');
-const path = require('node:path');
-const crypto = require('node:crypto');
-
-const { loadCandidateModule, validateCandidateModule } = require('./validation');
-const { safeSegment } = require('./experience-store');
-const { assertNoHeldoutLeakage } = require('./leakage-guard');
-const { recordMetaHarnessTelemetry } = require('./telemetry');
-
-class CandidateStore {
-  constructor({ store, domainSpec } = {}) {
-    if (!store) throw new Error('store is required');
-    this.store = store;
-    this.domainSpec = domainSpec || store.domainSpec;
-  }
-
-  registerCandidate({ candidatePath, source, manifest, iteration = 0 } = {}) {
-    this.store.ensureRun();
-    let candidateModule = null;
-    let candidateSource = source;
-    if (candidatePath) {
-      const absolutePath = path.resolve(candidatePath);
-      candidateSource = fs.readFileSync(absolutePath, 'utf8');
-      candidateModule = loadCandidateModule(absolutePath);
-    }
-    if (!candidateModule && source) {
-      const tmpPath = path.join(this.store.paths.runDir, `.candidate-${process.pid}-${Date.now()}.js`);
-      fs.writeFileSync(tmpPath, source, 'utf8');
-      try {
-        candidateModule = loadCandidateModule(tmpPath);
-      } finally {
-        fs.rmSync(tmpPath, { force: true });
-      }
-    }
-    if (!candidateModule && manifest) {
-      candidateModule = { manifest, apply: (base) => base };
-    }
-    const candidate = validateCandidateModule(candidateModule, { domainSpec: this.domainSpec });
-    const id = safeSegment(candidate.manifest.id, 'candidateId');
-    if (candidateSource && this.domainSpec?.leakagePolicy?.candidateSourceScan !== false) {
-      try {
-        assertNoHeldoutLeakage({ source: candidateSource, domainSpec: this.domainSpec, candidateId: id });
-      } catch (err) {
-        recordMetaHarnessTelemetry(this.store, {
-          type: 'leakage_blocked',
-          candidateId: id,
-          phase: 'candidate_register',
-          error: err.message,
-        });
-        throw err;
-      }
-    }
-    const sourceHash = sha256(candidateSource || '');
-    const candidateDir = path.join(this.store.paths.candidatesDir, id);
-    fs.mkdirSync(candidateDir, { recursive: true });
-    const storedSourcePath = path.join(candidateDir, 'candidate.js');
-    if (candidateSource) fs.writeFileSync(storedSourcePath, candidateSource, 'utf8');
-    const storedManifest = {
-      ...candidate.manifest,
-      sourceHash,
-      sourcePath: candidateSource ? storedSourcePath : null,
-      registeredAt: this.store.now(),
-      iteration,
-    };
-    this.store.writeJson(path.relative(this.store.runDir, path.join(candidateDir, 'manifest.json')), storedManifest);
-    this.store.appendExperience({
-      event: 'candidate_registered',
-      candidateId: id,
-      iteration,
-      surfaces: candidate.manifest.surfaces,
-      sourceHash,
-    });
-    return {
-      id,
-      manifest: storedManifest,
-      candidateDir,
-      sourcePath: storedSourcePath,
-      module: candidate,
-    };
-  }
-
-  loadCandidate(candidateId) {
-    const id = safeSegment(candidateId, 'candidateId');
-    const candidateDir = path.join(this.store.paths.candidatesDir, id);
-    const manifestPath = path.join(candidateDir, 'manifest.json');
-    const sourcePath = path.join(candidateDir, 'candidate.js');
-    if (!fs.existsSync(manifestPath)) throw new Error(`candidate manifest not found: ${id}`);
-    if (!fs.existsSync(sourcePath)) throw new Error(`candidate source not found: ${id}`);
-    const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
-    const module = validateCandidateModule(loadCandidateModule(sourcePath), { domainSpec: this.domainSpec });
-    return { id, manifest, sourcePath, module };
-  }
-
-  listCandidates() {
-    if (!fs.existsSync(this.store.paths.candidatesDir)) return [];
-    return fs.readdirSync(this.store.paths.candidatesDir, { withFileTypes: true })
-      .filter((entry) => entry.isDirectory())
-      .map((entry) => {
-        const manifestPath = path.join(this.store.paths.candidatesDir, entry.name, 'manifest.json');
-        if (!fs.existsSync(manifestPath)) return null;
-        return JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
-      })
-      .filter(Boolean)
-      .sort((a, b) => String(a.id).localeCompare(String(b.id)));
-  }
-}
-
-function sha256(text) {
-  return crypto.createHash('sha256').update(String(text || '')).digest('hex');
-}
-
-module.exports = {
-  CandidateStore,
-  sha256,
-};

package/template/wall-e/eval/meta-harness/cli.js

@@ -1,86 +0,0 @@
-#!/usr/bin/env node
-'use strict';
-
-const path = require('node:path');
-
-const { loadDomainSpec } = require('./domain-spec');
-const { ExperienceStore } = require('./experience-store');
-const { createCodingAgentAdapter } = require('./adapters/coding-agent');
-const { runMetaHarness } = require('./optimizer');
-
-async function main(argv = process.argv.slice(2)) {
-  const args = parseArgs(argv);
-  const projectRoot = path.resolve(__dirname, '..', '..', '..');
-  const domainPath = args.domain || path.join(__dirname, 'domains', 'coding-agent.domain.json');
-  const domainSpec = loadDomainSpec(domainPath, { projectRoot });
-  const store = new ExperienceStore({
-    domainSpec,
-    rootDir: args.runRoot || undefined,
-    runName: args.runName || undefined,
-  });
-  store.ensureRun({ cli: true, dryRun: !!args.dryRun });
-
-  if (args.dryRun) {
-    console.log(JSON.stringify({
-      ok: true,
-      domainId: domainSpec.id,
-      runDir: store.runDir,
-      searchTasks: domainSpec.datasets.search.taskIds.length,
-      maxIterations: Number(args.iterations || domainSpec.budget.maxIterations),
-    }, null, 2));
-    return { domainSpec, store };
-  }
-
-  let runAgentLoop;
-  try {
-    runAgentLoop = require('../../coding-orchestrator').runAgentLoop;
-  } catch (err) {
-    throw new Error(`Failed to load Wall-E coding orchestrator: ${err.message}`);
-  }
-
-  const adapter = createCodingAgentAdapter({ runAgentLoop });
-  const result = await runMetaHarness({
-    domainSpec,
-    store,
-    adapter,
-    maxIterations: args.iterations ? Number(args.iterations) : undefined,
-  });
-  console.log(JSON.stringify({
-    ok: true,
-    runDir: result.store.runDir,
-    bestCandidate: result.frontier.bestCandidate?.candidateId || null,
-    bestScore: result.frontier.bestCandidate?.aggregateScore ?? null,
-  }, null, 2));
-  return result;
-}
-
-function parseArgs(argv) {
-  const out = {};
-  for (let i = 0; i < argv.length; i++) {
-    const arg = argv[i];
-    if (arg === '--dry-run') out.dryRun = true;
-    else if (arg === '--domain') out.domain = argv[++i];
-    else if (arg === '--run-name') out.runName = argv[++i];
-    else if (arg === '--run-root') out.runRoot = argv[++i];
-    else if (arg === '--iterations') out.iterations = Number(argv[++i]);
-    else if (arg === '--help' || arg === '-h') {
-      console.log('Usage: node eval/meta-harness/cli.js [--domain path] [--run-root dir] [--run-name name] [--iterations n] [--dry-run]');
-      process.exit(0);
-    } else {
-      throw new Error(`Unknown argument: ${arg}`);
-    }
-  }
-  return out;
-}
-
-if (require.main === module) {
-  main().catch((err) => {
-    console.error(err.stack || err.message);
-    process.exit(1);
-  });
-}
-
-module.exports = {
-  main,
-  parseArgs,
-};

package/template/wall-e/eval/meta-harness/domain-spec.js

@@ -1,154 +0,0 @@
-'use strict';
-
-const fs = require('node:fs');
-const path = require('node:path');
-const crypto = require('node:crypto');
-
-const DEFAULT_PROJECT_ROOT = path.resolve(__dirname, '..', '..', '..');
-const SAFE_ID_RE = /^[A-Za-z0-9][A-Za-z0-9_.:-]*$/;
-
-function loadDomainSpec(filePath, opts = {}) {
-  if (!filePath) throw new Error('domain spec path is required');
-  const absolutePath = path.resolve(filePath);
-  let parsed;
-  try {
-    parsed = JSON.parse(fs.readFileSync(absolutePath, 'utf8'));
-  } catch (err) {
-    throw new Error(`Failed to read domain spec ${absolutePath}: ${err.message}`);
-  }
-  return normalizeDomainSpec(parsed, {
-    ...opts,
-    specPath: absolutePath,
-  });
-}
-
-function normalizeDomainSpec(spec, opts = {}) {
-  const projectRoot = path.resolve(opts.projectRoot || DEFAULT_PROJECT_ROOT);
-  validateDomainSpec(spec);
-  const normalized = {
-    ...spec,
-    projectRoot,
-    specPath: opts.specPath ? path.resolve(opts.specPath) : spec.specPath || null,
-    specHash: hashObject(withoutVolatileSpecFields(spec)),
-  };
-  normalized.paths = resolveDomainPaths(normalized, { projectRoot });
-  return normalized;
-}
-
-function validateDomainSpec(spec) {
-  if (!spec || typeof spec !== 'object' || Array.isArray(spec)) {
-    throw new Error('domain spec must be an object');
-  }
-  requireString(spec, 'id');
-  if (!SAFE_ID_RE.test(spec.id)) throw new Error(`domain spec id is not safe: ${spec.id}`);
-  requireString(spec, 'taskType');
-  requireObject(spec, 'fixed');
-  requireString(spec.fixed, 'baseRuntime', 'fixed.baseRuntime');
-  requireString(spec.fixed, 'baseModel', 'fixed.baseModel');
-  requireArray(spec.fixed, 'toolSurface', 'fixed.toolSurface');
-  requireArray(spec, 'allowedCandidateSurfaces');
-  requireObject(spec, 'datasets');
-  requireDataset(spec.datasets, 'smoke');
-  requireDataset(spec.datasets, 'search');
-  if (spec.datasets.heldout) requireDataset(spec.datasets, 'heldout');
-  requireObject(spec, 'metrics');
-  requireString(spec.metrics, 'primary', 'metrics.primary');
-  requireObject(spec, 'budget');
-  requirePositiveInteger(spec.budget, 'maxIterations', 'budget.maxIterations');
-  requirePositiveInteger(spec.budget, 'maxCandidatesPerIteration', 'budget.maxCandidatesPerIteration');
-  requireObject(spec, 'leakagePolicy');
-  if (typeof spec.leakagePolicy.proposerCanReadHeldout !== 'boolean') {
-    throw new Error('leakagePolicy.proposerCanReadHeldout must be boolean');
-  }
-  for (const surface of spec.allowedCandidateSurfaces) {
-    if (typeof surface !== 'string' || !SAFE_ID_RE.test(surface)) {
-      throw new Error(`allowedCandidateSurfaces contains unsafe surface: ${surface}`);
-    }
-  }
-  return true;
-}
-
-function resolveDomainPaths(spec, opts = {}) {
-  const projectRoot = path.resolve(opts.projectRoot || spec.projectRoot || DEFAULT_PROJECT_ROOT);
-  const runRoot = spec.runRoot || path.join('wall-e', 'eval', 'meta-runs');
-  return {
-    projectRoot,
-    runRoot: path.resolve(projectRoot, runRoot),
-    specPath: spec.specPath ? path.resolve(spec.specPath) : null,
-  };
-}
-
-function requireDataset(datasets, key) {
-  const dataset = datasets[key];
-  if (!dataset || typeof dataset !== 'object' || Array.isArray(dataset)) {
-    throw new Error(`datasets.${key} must be an object`);
-  }
-  requireString(dataset, 'suite', `datasets.${key}.suite`);
-  requireArray(dataset, 'taskIds', `datasets.${key}.taskIds`);
-  if (dataset.taskIds.length === 0) throw new Error(`datasets.${key}.taskIds must not be empty`);
-  for (const id of dataset.taskIds) {
-    if (typeof id !== 'string' || !SAFE_ID_RE.test(id)) {
-      throw new Error(`datasets.${key}.taskIds contains unsafe id: ${id}`);
-    }
-  }
-}
-
-function requireObject(obj, key, label = key) {
-  const value = obj[key];
-  if (!value || typeof value !== 'object' || Array.isArray(value)) {
-    throw new Error(`${label} must be an object`);
-  }
-  return value;
-}
-
-function requireString(obj, key, label = key) {
-  const value = obj[key];
-  if (typeof value !== 'string' || value.trim() === '') {
-    throw new Error(`${label} must be a non-empty string`);
-  }
-  return value;
-}
-
-function requireArray(obj, key, label = key) {
-  const value = obj[key];
-  if (!Array.isArray(value)) throw new Error(`${label} must be an array`);
-  return value;
-}
-
-function requirePositiveInteger(obj, key, label = key) {
-  const value = Number(obj[key]);
-  if (!Number.isInteger(value) || value < 1) throw new Error(`${label} must be a positive integer`);
-  return value;
-}
-
-function withoutVolatileSpecFields(spec) {
-  const copy = { ...spec };
-  delete copy.projectRoot;
-  delete copy.specPath;
-  delete copy.specHash;
-  delete copy.paths;
-  return copy;
-}
-
-function stableStringify(value) {
-  if (value === null || typeof value !== 'object') return JSON.stringify(value);
-  if (Array.isArray(value)) return '[' + value.map(stableStringify).join(',') + ']';
-  return '{' + Object.keys(value).sort().map((key) => (
-    JSON.stringify(key) + ':' + stableStringify(value[key])
-  )).join(',') + '}';
-}
-
-function hashObject(value) {
-  return crypto.createHash('sha256').update(stableStringify(value)).digest('hex');
-}
-
-module.exports = {
-  DEFAULT_PROJECT_ROOT,
-  SAFE_ID_RE,
-  loadDomainSpec,
-  normalizeDomainSpec,
-  validateDomainSpec,
-  resolveDomainPaths,
-  stableStringify,
-  hashObject,
-};

package/template/wall-e/eval/meta-harness/domains/coding-agent.domain.json

@@ -1,84 +0,0 @@
-{
-  "id": "coding-agent",
-  "version": "2026-05-05",
-  "taskType": "coding-agent",
-  "description": "Meta-Harness optimization domain for Wall-E coding-agent evals. The fixed baseline is the current Wall-E coding runtime, model/provider config, tool surface, and scorer; candidates may only adjust explicitly listed surfaces.",
-  "runRoot": "wall-e/eval/meta-runs",
-  "fixed": {
-    "baseRuntime": "wall-e-coding-agent",
-    "baseModel": "configured-provider",
-    "runner": "wall-e/eval/agent-runner.js",
-    "scorer": "agent-rubric+tests",
-    "toolSurface": [
-      "read_file",
-      "list_directory",
-      "glob",
-      "grep_files",
-      "edit_file",
-      "write_file",
-      "run_shell"
-    ]
-  },
-  "allowedCandidateSurfaces": [
-    "systemPromptPrelude",
-    "contextBuilder",
-    "toolPolicy",
-    "environmentBootstrap",
-    "runtimeOptions"
-  ],
-  "datasets": {
-    "smoke": {
-      "suite": "coding-agent",
-      "taskIds": [
-        "agent-001",
-        "agent-004"
-      ]
-    },
-    "search": {
-      "suite": "coding-agent",
-      "taskIds": [
-        "agent-001",
-        "agent-002",
-        "agent-003",
-        "agent-004",
-        "agent-005",
-        "agent-006"
-      ]
-    },
-    "heldout": {
-      "suite": "coding-agent",
-      "taskIds": [
-        "agent-007",
-        "agent-008",
-        "agent-009"
-      ]
-    }
-  },
-  "metrics": {
-    "primary": "score.composite",
-    "higherIsBetter": true,
-    "secondary": [
-      "testsPassed",
-      "latencyMs",
-      "costDollars"
-    ],
-    "minimumViableScore": 0.01
-  },
-  "budget": {
-    "maxIterations": 3,
-    "maxCandidatesPerIteration": 3,
-    "maxTasksPerCandidate": 6,
-    "timeoutMs": 600000
-  },
-  "leakagePolicy": {
-    "proposerCanReadHeldout": false,
-    "heldoutResultAccess": "blocked",
-    "candidateSourceScan": true
-  },
-  "artifactRetention": {
-    "keepTranscript": true,
-    "keepDiff": true,
-    "keepWorkspaceManifest": true,
-    "keepRawResult": true
-  }
-}

package/template/wall-e/eval/meta-harness/examples/env-bootstrap-candidate.js

@@ -1,29 +0,0 @@
-'use strict';
-
-const {
-  gatherEnvironmentSnapshot,
-  formatEnvironmentSnapshot,
-} = require('../bootstrap-snapshot');
-
-module.exports = {
-  manifest: {
-    id: 'env-bootstrap-v1',
-    hypothesis: 'Injecting a bounded environment snapshot reduces first-turn discovery shell calls and improves early task execution.',
-    surfaces: ['environmentBootstrap'],
-  },
-  apply(baseHarness) {
-    return {
-      ...baseHarness,
-      surfaces: {
-        ...baseHarness.surfaces,
-        environmentBootstrap(context = {}) {
-          const snapshot = context.environmentSnapshot || gatherEnvironmentSnapshot({
-            cwd: context.cwd || process.cwd(),
-            commands: context.bootstrapCommands,
-          });
-          return formatEnvironmentSnapshot(snapshot);
-        },
-      },
-    };
-  },
-};