atris 3.35.0 → 3.36.0

package/lib/bench/context.js

@@ -0,0 +1,147 @@
+'use strict';
+
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { spawnSync } = require('node:child_process');
+
+const PROFILE_ENV_KEYS = [
+  'ATRIS_PROFILE',
+  'ATRIS_PROFILE_EMAIL',
+  'ATRIS_AUTH_PROFILE',
+  'ATRIS_ACTIVE_PROFILE',
+  'ATRIS_ACCOUNT',
+  'ATRIS_ACCOUNT_ID',
+  'ATRIS_BUSINESS_ID',
+  'ATRIS_BUSINESS_SLUG',
+  'ATRIS_OWNER_ID',
+  'ATRIS_OWNER_SLUG',
+  'AX_PROFILE',
+  'AX_ACCOUNT',
+];
+
+function defaultRepoRoot() {
+  return path.resolve(__dirname, '..', '..');
+}
+
+function hermeticEnv(home, extra = {}) {
+  const env = { ...process.env };
+  for (const key of PROFILE_ENV_KEYS) env[key] = '';
+  for (const key of Object.keys(env)) {
+    if (key.startsWith('NODE_TEST_')) delete env[key];
+  }
+  env.HOME = home;
+  env.USERPROFILE = home;
+  env.XDG_CONFIG_HOME = path.join(home, '.config');
+  env.XDG_CACHE_HOME = path.join(home, '.cache');
+  env.XDG_DATA_HOME = path.join(home, '.local', 'share');
+  env.ATRIS_SKIP_UPDATE_CHECK = '1';
+  env.NO_UPDATE_NOTIFIER = '1';
+  env.NODE_NO_WARNINGS = '1';
+  env.ATRIS_AGENT_PROOF_ONLY = '0';
+  env.ATRIS_TASKS_DB = path.join(home, '.atris', 'tasks.db');
+  env.ATRIS_AGENT_ID = env.ATRIS_AGENT_ID || 'bench';
+  return { ...env, ...extra };
+}
+
+function spawnErrorFromResult(result, command) {
+  const source = result && result.error ? result.error : new Error(`failed to spawn ${command}`);
+  const err = new Error(source.message || `failed to spawn ${command}`);
+  err.name = source.name || err.name;
+  err.code = source.code || 'SPAWN_ERROR';
+  err.errno = source.errno;
+  err.syscall = source.syscall || `spawn ${command}`;
+  err.path = source.path || command;
+  err.spawnError = true;
+  err.result = result;
+  return err;
+}
+
+function createBenchContext(options = {}) {
+  const repoRoot = path.resolve(options.repoRoot || defaultRepoRoot());
+  const cliPath = options.cliPath || path.join(repoRoot, 'bin', 'atris.js');
+  const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), `atris-bench-${options.taskId || 'task'}-`));
+  const home = path.join(tempRoot, 'home');
+  const workspace = path.join(tempRoot, 'workspace');
+  fs.mkdirSync(home, { recursive: true });
+  fs.mkdirSync(workspace, { recursive: true });
+  const env = hermeticEnv(home, options.env || {});
+  const defaultTimeout = Number(options.timeoutMs || 30000);
+
+  const ctx = {
+    repoRoot,
+    cliPath,
+    env,
+    workspace,
+    run(command, args = [], runOptions = {}) {
+      const result = spawnSync(command, args, {
+        cwd: runOptions.cwd || workspace,
+        input: runOptions.input,
+        encoding: 'utf8',
+        timeout: Number(runOptions.timeoutMs || runOptions.timeout || defaultTimeout),
+        maxBuffer: Number(runOptions.maxBuffer || 16 * 1024 * 1024),
+        env: {
+          ...env,
+          ...(runOptions.env || {}),
+        },
+      });
+      if (result.error) throw spawnErrorFromResult(result, command);
+      return result;
+    },
+    runCli(args = [], runOptions = {}) {
+      const result = spawnSync(process.execPath, [cliPath, ...args], {
+        cwd: workspace,
+        input: runOptions.input,
+        encoding: 'utf8',
+        timeout: Number(runOptions.timeoutMs || runOptions.timeout || defaultTimeout),
+        maxBuffer: Number(runOptions.maxBuffer || 16 * 1024 * 1024),
+        env: {
+          ...env,
+          ...(runOptions.env || {}),
+        },
+      });
+      if (result.error) throw spawnErrorFromResult(result, process.execPath);
+      return result;
+    },
+    requireCmd(command, args = ['--version']) {
+      const result = spawnSync(command, args, {
+        cwd: workspace,
+        encoding: 'utf8',
+        timeout: 5000,
+        env,
+      });
+      if (result.error || result.status !== 0) {
+        const err = result.error
+          ? spawnErrorFromResult(result, command)
+          : new Error(`${command} exited ${result.status}`);
+        err.code = err.code || 'COMMAND_UNAVAILABLE';
+        err.command = command;
+        throw err;
+      }
+      return command;
+    },
+  };
+
+  return {
+    ctx,
+    teardown() {
+      fs.rmSync(tempRoot, { recursive: true, force: true, maxRetries: 3, retryDelay: 50 });
+    },
+  };
+}
+
+async function withBenchContext(options, fn) {
+  const { ctx, teardown } = createBenchContext(options);
+  try {
+    return await fn(ctx);
+  } finally {
+    teardown();
+  }
+}
+
+module.exports = {
+  PROFILE_ENV_KEYS,
+  createBenchContext,
+  hermeticEnv,
+  withBenchContext,
+};

package/lib/bench/engines.js

@@ -0,0 +1,141 @@
+'use strict';
+
+const fs = require('node:fs');
+const path = require('node:path');
+const { spawnSync } = require('node:child_process');
+
+const ENGINE_NAMES = Object.freeze([
+  'codex',
+  'cursor',
+  'claude',
+  'atris-fast',
+  'null',
+  'solution',
+]);
+
+const ENGINE_BINS = Object.freeze({
+  codex: 'codex',
+  cursor: 'cursor-agent',
+  claude: 'claude',
+  'atris-fast': 'atris',
+});
+
+function normalizeEngineName(name) {
+  const normalized = String(name || '').trim();
+  if (!normalized) return '';
+  if (!ENGINE_NAMES.includes(normalized)) {
+    throw new Error(`unknown bench engine: ${normalized}`);
+  }
+  return normalized;
+}
+
+function commandAvailable(command) {
+  const result = spawnSync('/bin/sh', ['-c', `command -v ${command}`], {
+    encoding: 'utf8',
+    env: process.env,
+  });
+  return result.status === 0 && Boolean(String(result.stdout || '').trim());
+}
+
+function spawnEngine(command, args, workspaceDir, timeoutMs, options = {}) {
+  const result = spawnSync(command, args, {
+    cwd: workspaceDir,
+    input: options.input,
+    encoding: 'utf8',
+    timeout: Number(timeoutMs || 300000),
+    maxBuffer: Number(options.maxBuffer || 16 * 1024 * 1024),
+    env: {
+      ...process.env,
+      ...(options.env || {}),
+    },
+  });
+  return {
+    status: result.status === null ? 1 : result.status,
+    stdout: result.stdout || '',
+    stderr: result.error ? String(result.error.message || result.error) : (result.stderr || ''),
+  };
+}
+
+function realEngineAdapter(name, bin, runArgs) {
+  return {
+    name,
+    available(workspaceDir) {
+      if (!commandAvailable(bin)) {
+        return { available: false, reason: `${bin} not found` };
+      }
+      if (name === 'atris-fast' && !fs.existsSync(path.join(workspaceDir, 'atris'))) {
+        return { available: false, reason: 'workspace root is not an Atris workspace' };
+      }
+      return { available: true, reason: 'available' };
+    },
+    run(promptText, workspaceDir, timeoutMs) {
+      return spawnEngine(bin, runArgs(promptText, workspaceDir), workspaceDir, timeoutMs);
+    },
+  };
+}
+
+function nullEngineAdapter() {
+  return {
+    name: 'null',
+    available() {
+      return { available: true, reason: 'available' };
+    },
+    run() {
+      return { status: 0, stdout: '', stderr: '' };
+    },
+  };
+}
+
+function solutionEngineAdapter(solutionPath) {
+  return {
+    name: 'solution',
+    available() {
+      if (!solutionPath || !fs.existsSync(solutionPath)) {
+        return { available: false, reason: 'solution.sh not found' };
+      }
+      return { available: true, reason: 'available' };
+    },
+    run(_promptText, workspaceDir, timeoutMs) {
+      return spawnEngine('/bin/sh', [solutionPath], workspaceDir, timeoutMs);
+    },
+  };
+}
+
+function getEngineAdapter(name, options = {}) {
+  const normalized = normalizeEngineName(name);
+  if (normalized === 'null') return nullEngineAdapter();
+  if (normalized === 'solution') return solutionEngineAdapter(options.solutionPath);
+  if (normalized === 'codex') {
+    return realEngineAdapter('codex', ENGINE_BINS.codex, (promptText, workspaceDir) => [
+      'exec',
+      '--dangerously-bypass-approvals-and-sandbox',
+      '-C',
+      workspaceDir,
+      promptText,
+    ]);
+  }
+  if (normalized === 'cursor') {
+    return realEngineAdapter('cursor', ENGINE_BINS.cursor, (promptText) => ['--trust', '-p', promptText]);
+  }
+  if (normalized === 'claude') {
+    return realEngineAdapter('claude', ENGINE_BINS.claude, (promptText) => [
+      '-p',
+      promptText,
+      '--dangerously-skip-permissions',
+    ]);
+  }
+  if (normalized === 'atris-fast') {
+    return realEngineAdapter('atris-fast', ENGINE_BINS['atris-fast'], (promptText) => [
+      'chat',
+      '--print',
+      promptText,
+    ]);
+  }
+  throw new Error(`unknown bench engine: ${name}`);
+}
+
+module.exports = {
+  ENGINE_NAMES,
+  getEngineAdapter,
+  normalizeEngineName,
+};

package/lib/bench/report.js

@@ -0,0 +1,140 @@
+'use strict';
+
+const path = require('node:path');
+const { readResultRecords, taskMetadata } = require('./runner');
+
+const REPORT_SCHEMA = 'atris.bench.report.v1';
+const DEFAULT_PACK = 'agents-v1';
+const CATEGORY_ORDER = ['navigate', 'edit', 'contract', 'build', 'recover'];
+
+function repoRootFromHere() {
+  return path.resolve(__dirname, '..', '..');
+}
+
+function isFullRun(record, totalTaskCount) {
+  return Array.isArray(record.tasks) && totalTaskCount > 0 && record.tasks.length === totalTaskCount;
+}
+
+// Latest full run wins outright; partial runs (from --task filtering) are only
+// merged together when the engine never produced a full run of the pack.
+function effectiveTasksForEngine(records, totalTaskCount) {
+  const sorted = [...records].sort((a, b) => new Date(a.started) - new Date(b.started));
+  const fullRuns = sorted.filter((record) => isFullRun(record, totalTaskCount));
+  if (fullRuns.length) {
+    return fullRuns[fullRuns.length - 1].tasks;
+  }
+  const merged = new Map();
+  for (const record of sorted) {
+    for (const task of record.tasks || []) {
+      merged.set(task.id, task);
+    }
+  }
+  return Array.from(merged.values());
+}
+
+function meanDurationMs(tasks) {
+  const durations = tasks
+    .filter((task) => !task.skipped)
+    .map((task) => Number(task.duration_ms) || 0);
+  if (!durations.length) return null;
+  const sum = durations.reduce((total, value) => total + value, 0);
+  return Math.round(sum / durations.length);
+}
+
+function buildBenchReport(options = {}) {
+  const repoRoot = path.resolve(options.repoRoot || repoRootFromHere());
+  const stateRoot = options.stateRoot || process.cwd();
+  const pack = options.pack || DEFAULT_PACK;
+
+  const records = readResultRecords({ stateRoot }).filter((record) => record.pack === pack);
+  if (!records.length) {
+    return { schema: REPORT_SCHEMA, pack, engines: [] };
+  }
+
+  const taskCategories = new Map();
+  let totalTaskCount = 0;
+  try {
+    const tasks = taskMetadata({ repoRoot, pack });
+    totalTaskCount = tasks.length;
+    for (const task of tasks) taskCategories.set(task.id, task.category || null);
+  } catch (err) {
+    // degrade instead of crashing the report: fall back to the largest observed run
+    totalTaskCount = records.reduce((max, record) => Math.max(max, (record.tasks || []).length), 0);
+  }
+
+  const byEngine = new Map();
+  for (const record of records) {
+    const engineName = record.engine || 'unknown';
+    if (!byEngine.has(engineName)) byEngine.set(engineName, []);
+    byEngine.get(engineName).push(record);
+  }
+
+  const categoryTotals = new Map();
+  for (const category of taskCategories.values()) {
+    if (!category) continue;
+    categoryTotals.set(category, (categoryTotals.get(category) || 0) + 1);
+  }
+
+  const engines = Array.from(byEngine.keys())
+    .sort((a, b) => a.localeCompare(b))
+    .map((engineName) => {
+      const tasks = effectiveTasksForEngine(byEngine.get(engineName), totalTaskCount);
+      const passed = tasks.filter((task) => task.passed && !task.skipped);
+      const failed = tasks.filter((task) => !task.passed && !task.skipped);
+      const skipped = tasks.filter((task) => task.skipped);
+
+      const categories = CATEGORY_ORDER.map((name) => {
+        const categoryTasks = tasks.filter((task) => taskCategories.get(task.id) === name);
+        const categoryPassed = categoryTasks.filter((task) => task.passed && !task.skipped);
+        return {
+          name,
+          passed: categoryPassed.length,
+          total: categoryTotals.get(name) || 0,
+        };
+      });
+
+      return {
+        engine: engineName,
+        passed: passed.length,
+        total: totalTaskCount,
+        categories,
+        meanDurationMs: meanDurationMs(tasks),
+        failed: failed.map((task) => task.id),
+        skipped: skipped.length,
+      };
+    });
+
+  return { schema: REPORT_SCHEMA, pack, engines };
+}
+
+function formatDuration(ms) {
+  if (ms === null || ms === undefined) return 'n/a';
+  if (ms < 1000) return `${ms}ms`;
+  return `${(ms / 1000).toFixed(1)}s`;
+}
+
+function renderBenchReportText(report) {
+  if (!report.engines.length) {
+    return `no runs recorded for pack ${report.pack}`;
+  }
+  const lines = [`pack: ${report.pack}`, ''];
+  for (const engine of report.engines) {
+    lines.push(`engine: ${engine.engine}`);
+    lines.push(`  score      ${engine.passed}/${engine.total} passed`);
+    for (const category of engine.categories) {
+      lines.push(`  ${category.name.padEnd(10)} ${category.passed}/${category.total}`);
+    }
+    lines.push(`  mean dur   ${formatDuration(engine.meanDurationMs)}`);
+    lines.push(`  skipped    ${engine.skipped}`);
+    lines.push(`  failed     ${engine.failed.length ? engine.failed.join(', ') : 'none'}`);
+    lines.push('');
+  }
+  return lines.join('\n').trimEnd();
+}
+
+module.exports = {
+  DEFAULT_PACK,
+  REPORT_SCHEMA,
+  buildBenchReport,
+  renderBenchReportText,
+};