atris 2.6.3 → 3.0.0

package/commands/experiments.js

@@ -1,6 +1,22 @@
 const fs = require('fs');
 const path = require('path');
 const { spawnSync } = require('child_process');
+const { runTaskOnce } = require('./autopilot');
+const {
+  appendResultsRow,
+  buildRunId,
+  buildWikiArtifact,
+  compareEndstateArtifacts,
+  collectChangedFiles,
+  getArtifactScore,
+  getGitHead,
+  inferTestResults,
+  readLatestArtifact,
+  readTextIfExists,
+  scoreEndstateArtifact,
+  summarizeReview,
+  writeArtifact,
+} = require('../lib/endstate');
 
 const SLUG_RE = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
 const ROOT_FILES = ['README.md', 'validate.py', 'benchmark_validate.py', 'benchmark_runtime.py'];
@@ -183,6 +199,327 @@ function experimentsBenchmark(kind = 'all') {
   }
 }
 
+function parseRunOptions(args) {
+  const options = {
+    dryRun: false,
+    verbose: false,
+    backendPath: path.resolve(process.cwd(), '../atrisos-backend'),
+    interventions: [],
+  };
+
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i];
+
+    if (arg === '--dry-run' || arg === '-n') {
+      options.dryRun = true;
+      continue;
+    }
+
+    if (arg === '--verbose') {
+      options.verbose = true;
+      continue;
+    }
+
+    if (arg === '--backend' && args[i + 1]) {
+      options.backendPath = path.resolve(process.cwd(), args[++i]);
+      continue;
+    }
+
+    if (arg.startsWith('--backend=')) {
+      options.backendPath = path.resolve(process.cwd(), arg.split('=')[1]);
+      continue;
+    }
+
+    if (arg === '--intervention' && args[i + 1]) {
+      options.interventions.push(args[++i]);
+      continue;
+    }
+
+    if (arg.startsWith('--intervention=')) {
+      options.interventions.push(arg.split('=')[1]);
+      continue;
+    }
+  }
+
+  return options;
+}
+
+function isEndstatePack(name) {
+  return name === 'endstate-baseline' || name === 'endstate-stack';
+}
+
+function readTaskBrief(packDir) {
+  const programPath = path.join(packDir, 'program.md');
+  const content = fs.readFileSync(programPath, 'utf8');
+  return content.replace(/^# Program\s*/i, '').trim();
+}
+
+function readRunnerConfig(packDir, name) {
+  const runnerPath = path.join(packDir, 'runner.json');
+  const fallback = name.endsWith('stack')
+    ? {
+        name: 'stack-coordinated',
+        strategy: 'stack',
+        summary: 'Coordinated stack benchmark run.',
+        context_note: 'Runner profile: stack-coordinated\nProtocol: coordinated stack run',
+      }
+    : {
+        name: 'baseline-single',
+        strategy: 'single',
+        summary: 'Single-model direct benchmark run.',
+        context_note: 'Runner profile: baseline-single\nProtocol: single-model direct run',
+      };
+
+  if (!fs.existsSync(runnerPath)) return fallback;
+
+  try {
+    const parsed = JSON.parse(fs.readFileSync(runnerPath, 'utf8'));
+    return {
+      ...fallback,
+      ...parsed,
+      name: parsed.name || fallback.name,
+      strategy: parsed.strategy || fallback.strategy,
+      summary: parsed.summary || fallback.summary,
+      context_note: parsed.context_note || fallback.context_note,
+    };
+  } catch {
+    return fallback;
+  }
+}
+
+function buildPromptContext(name, runnerConfig, taskBrief, phaseResults) {
+  const sections = [
+    `pack: ${name}`,
+    `runner: ${runnerConfig.name}`,
+    `runner strategy: ${runnerConfig.strategy}`,
+    `runner summary:\n${runnerConfig.summary}`,
+    `task brief:\n${taskBrief}`,
+  ];
+
+  if (phaseResults) {
+    for (const phase of ['plan', 'do', 'review']) {
+      if (!phaseResults[phase]) continue;
+      sections.push(`${phase} prompt:\n${phaseResults[phase].prompt}`);
+      sections.push(`${phase} output:\n${phaseResults[phase].output || ''}`);
+    }
+  }
+
+  return sections.join('\n\n');
+}
+
+function buildBenchmarkArtifact(name, packDir, options) {
+  const track = name.endsWith('stack') ? 'stack' : 'baseline';
+  const cwd = process.cwd();
+  const backendPath = options.backendPath;
+  const taskBrief = readTaskBrief(packDir);
+  const runnerConfig = readRunnerConfig(packDir, name);
+  const runId = buildRunId(track);
+  const beforeCli = getGitHead(cwd);
+  const beforeBackend = getGitHead(backendPath);
+  const wikiBefore = readTextIfExists(path.join(cwd, 'atris', 'wiki', 'STATUS.md'));
+
+  let execution = null;
+  let reviewStatus = options.dryRun ? 'draft' : 'fail';
+  let reviewSummary = options.dryRun ? 'dry run — benchmark task not executed' : 'benchmark run failed before review';
+  let tests = [{
+    command: options.dryRun ? 'benchmark dry-run' : '(no explicit test command captured)',
+    status: 'not_run',
+  }];
+  let notes = options.dryRun ? 'dry-run artifact only' : '';
+
+  if (!options.dryRun) {
+    try {
+      execution = runTaskOnce(
+        { task: taskBrief, kind: 'benchmark' },
+        {
+          verbose: options.verbose,
+          benchmarkStrategy: runnerConfig.strategy,
+          runnerName: runnerConfig.name,
+          extraReadFiles: [
+            'atris/features/endstate/contract.md',
+            'atris/features/endstate/artifact-schema.json',
+            path.relative(cwd, path.join(packDir, 'program.md')),
+            '../atrisos-backend/atris/MAP.md',
+            '../atrisos-backend/atris/TODO.md',
+          ],
+          contextNote: [
+            `Project Endstate track: ${track}`,
+            `Pack: atris/experiments/${name}/program.md`,
+            runnerConfig.context_note,
+            'Stay within the exact Level 1 contract. The outer runner records the receipt.',
+          ].join('\n'),
+        }
+      );
+
+      reviewStatus = execution.success ? 'pass' : 'fail';
+      reviewSummary = summarizeReview(execution.reviewOutput);
+      tests = inferTestResults(execution.reviewOutput);
+    } catch (error) {
+      reviewStatus = 'fail';
+      reviewSummary = summarizeReview(error.message || String(error));
+      notes = 'runner threw before completion';
+    }
+  }
+
+  const afterCli = getGitHead(cwd);
+  const afterBackend = getGitHead(backendPath);
+  const changedFiles = options.dryRun
+    ? []
+    : [
+        ...collectChangedFiles(cwd, beforeCli, afterCli),
+        ...collectChangedFiles(backendPath, beforeBackend, afterBackend, '../atrisos-backend/'),
+      ];
+  const wikiAfter = readTextIfExists(path.join(cwd, 'atris', 'wiki', 'STATUS.md'));
+
+  const artifact = {
+    run_id: runId,
+    track,
+    repo_commits: {
+      atris_cli: beforeCli || 'missing',
+      atrisos_backend: beforeBackend || 'missing',
+    },
+    repo_commits_after: {
+      atris_cli: afterCli || 'missing',
+      atrisos_backend: afterBackend || 'missing',
+    },
+    task_brief: taskBrief,
+    prompt_context: buildPromptContext(name, runnerConfig, taskBrief, execution?.phaseResults),
+    changed_files: changedFiles,
+    tests,
+    review: {
+      status: reviewStatus,
+      summary: reviewSummary,
+    },
+    wiki: buildWikiArtifact(wikiBefore, wikiAfter),
+    elapsed_seconds: execution?.elapsedSeconds || 0,
+    interventions: {
+      count: options.interventions.length,
+      events: options.interventions,
+    },
+    notes: [notes, `runner=${runnerConfig.name}`].filter(Boolean).join(' | '),
+  };
+
+  const score = scoreEndstateArtifact(artifact);
+  artifact.score = score.total;
+  artifact.score_breakdown = score.breakdown;
+
+  return { artifact, score };
+}
+
+function experimentsRun(name, ...args) {
+  const { experimentsDir } = ensureExperimentsFramework();
+
+  if (!name) {
+    console.error('Usage: atris experiments run <slug> [--dry-run] [--verbose]');
+    process.exit(1);
+  }
+
+  const packDir = path.join(experimentsDir, name);
+  if (!fs.existsSync(packDir)) {
+    console.error(`✗ Experiment "${name}" not found at atris/experiments/${name}/`);
+    process.exit(1);
+  }
+
+  const options = parseRunOptions(args);
+
+  if (!isEndstatePack(name)) {
+    const loopPath = path.join(packDir, 'loop.py');
+    if (!fs.existsSync(loopPath)) {
+      console.error(`✗ Experiment "${name}" has no loop.py to run.`);
+      process.exit(1);
+    }
+    if (options.dryRun) {
+      console.log(`✓ Dry run: would execute atris/experiments/${name}/loop.py`);
+      return;
+    }
+    runPython(loopPath, [], packDir);
+    return;
+  }
+
+  const schemaPath = path.join(process.cwd(), 'atris', 'features', 'endstate', 'artifact-schema.json');
+  const contractPath = path.join(process.cwd(), 'atris', 'features', 'endstate', 'contract.md');
+  if (!fs.existsSync(schemaPath) || !fs.existsSync(contractPath)) {
+    console.error('✗ Endstate contract missing. Expected atris/features/endstate/{contract.md,artifact-schema.json}.');
+    process.exit(1);
+  }
+
+  const { artifact, score } = buildBenchmarkArtifact(name, packDir, options);
+  const artifactPath = writeArtifact(packDir, artifact);
+  appendResultsRow(path.join(packDir, 'results.tsv'), artifactPath, artifact, score);
+
+  console.log(`✓ Endstate ${artifact.track} run recorded`);
+  console.log(`  artifact: ${path.relative(process.cwd(), artifactPath)}`);
+  console.log(`  score: ${score.total}/100`);
+  console.log(`  review: ${artifact.review.status}`);
+  console.log(`  interventions: ${artifact.interventions.count}`);
+
+  if (artifact.review.status === 'fail' && !options.dryRun) {
+    process.exit(1);
+  }
+}
+
+function formatCompareLine(label, entry) {
+  const artifact = entry.artifact;
+  const score = getArtifactScore(artifact);
+  const review = artifact.review?.status || 'unknown';
+  const interventions = artifact.interventions?.count || 0;
+  return `  ${label}: ${score}/100 | review: ${review} | interventions: ${interventions} | artifact: ${path.relative(process.cwd(), entry.filePath)}`;
+}
+
+function experimentsCompare(target = 'endstate') {
+  const { experimentsDir } = ensureExperimentsFramework(process.cwd(), { silent: true });
+
+  if (target !== 'endstate') {
+    console.error('Usage: atris experiments compare endstate');
+    process.exit(1);
+  }
+
+  const baselineDir = path.join(experimentsDir, 'endstate-baseline');
+  const stackDir = path.join(experimentsDir, 'endstate-stack');
+
+  try {
+    const baselineEntry = readLatestArtifact(baselineDir);
+    const stackEntry = readLatestArtifact(stackDir);
+    const comparison = compareEndstateArtifacts(baselineEntry, stackEntry);
+
+    console.log('✓ Endstate comparison ready');
+    console.log(formatCompareLine('baseline', baselineEntry));
+    console.log(formatCompareLine('stack', stackEntry));
+    console.log('');
+    if (comparison.winner === 'stack') {
+      console.log('Decision: stack wins.');
+    } else {
+      console.log('Decision: no winner yet.');
+    }
+    console.log(`Reason: ${comparison.reason}`);
+  } catch (error) {
+    console.error(`✗ ${error.message || error}`);
+    process.exit(1);
+  }
+}
+
+function experimentsReplay(target = 'endstate') {
+  if (target !== 'endstate') {
+    console.error('Usage: atris experiments replay endstate');
+    process.exit(1);
+  }
+
+  console.log('Replay: validate baseline pack');
+  experimentsValidate('endstate-baseline');
+  console.log('');
+  console.log('Replay: validate stack pack');
+  experimentsValidate('endstate-stack');
+  console.log('');
+  console.log('Replay: baseline dry run');
+  experimentsRun('endstate-baseline', '--dry-run');
+  console.log('');
+  console.log('Replay: stack dry run');
+  experimentsRun('endstate-stack', '--dry-run');
+  console.log('');
+  console.log('Replay: compare latest receipts');
+  experimentsCompare('endstate');
+}
+
 function experimentsCommand(subcommand, ...args) {
   switch (subcommand) {
     case 'init':
@@ -192,6 +529,12 @@ function experimentsCommand(subcommand, ...args) {
       return experimentsValidate(args[0]);
     case 'benchmark':
       return experimentsBenchmark(args[0] || 'all');
+    case 'compare':
+      return experimentsCompare(args[0] || 'endstate');
+    case 'replay':
+      return experimentsReplay(args[0] || 'endstate');
+    case 'run':
+      return experimentsRun(args[0], ...args.slice(1));
     default:
       console.log('');
       console.log('Usage: atris experiments <subcommand> [name]');
@@ -199,12 +542,18 @@ function experimentsCommand(subcommand, ...args) {
       console.log('Subcommands:');
       console.log('  init [slug]          Prepare atris/experiments/ or scaffold a new pack');
       console.log('  validate [path|slug] Run structural validation on packs or a single pack');
+      console.log('  run <slug>           Execute a pack or record an Endstate benchmark receipt');
+      console.log('  compare endstate     Compare the latest baseline and stack receipts');
+      console.log('  replay endstate      Validate, dry-run, and compare the public benchmark flow');
       console.log('  benchmark [mode]     Run validate/runtime/all benchmark harness');
       console.log('');
       console.log('Examples:');
       console.log('  atris experiments init');
       console.log('  atris experiments init self-heal');
       console.log('  atris experiments validate');
+      console.log('  atris experiments run endstate-baseline --dry-run');
+      console.log('  atris experiments compare endstate');
+      console.log('  atris experiments replay endstate');
       console.log('  atris experiments benchmark runtime');
       console.log('');
   }
@@ -213,4 +562,6 @@ function experimentsCommand(subcommand, ...args) {
 module.exports = {
   experimentsCommand,
   ensureExperimentsFramework,
+  buildBenchmarkArtifact,
+  parseRunOptions,
 };

package/commands/feedback.js

@@ -0,0 +1,150 @@
+/**
+ * Feedback command for Atris CLI
+ *
+ * Usage:
+ *   atris feedback "message here"  - Submit feedback
+ *   atris feedback                 - List your feedback
+ *   atris feedback list            - List your feedback
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { loadCredentials } = require('../utils/auth');
+const { apiRequestJson } = require('../utils/api');
+
+function getAuth() {
+  const creds = loadCredentials();
+  if (!creds || !creds.token) {
+    console.error('Not logged in. Run: atris login');
+    process.exit(1);
+  }
+  return { token: creds.token, email: creds.email || 'unknown' };
+}
+
+function getBusinessId() {
+  // 1. Check .atris/business.json in current directory
+  const bizFile = path.join(process.cwd(), '.atris', 'business.json');
+  if (fs.existsSync(bizFile)) {
+    try {
+      const biz = JSON.parse(fs.readFileSync(bizFile, 'utf8'));
+      if (biz.business_id) return biz.business_id;
+    } catch {}
+  }
+
+  // 2. Check ~/.atris/businesses.json (first connected business)
+  const home = require('os').homedir();
+  const globalBizFile = path.join(home, '.atris', 'businesses.json');
+  if (fs.existsSync(globalBizFile)) {
+    try {
+      const businesses = JSON.parse(fs.readFileSync(globalBizFile, 'utf8'));
+      const slugs = Object.keys(businesses);
+      if (slugs.length > 0 && businesses[slugs[0]].business_id) {
+        return businesses[slugs[0]].business_id;
+      }
+    } catch {}
+  }
+
+  return null;
+}
+
+async function submitFeedback(message) {
+  if (!message) {
+    console.error('Usage: atris feedback "your message here"');
+    process.exit(1);
+  }
+
+  const { token } = getAuth();
+  const businessId = getBusinessId();
+
+  const body = {
+    message,
+    source: 'cli',
+  };
+  if (businessId) {
+    body.business_id = businessId;
+  }
+
+  const result = await apiRequestJson('/feedback', {
+    method: 'POST',
+    token,
+    body,
+  });
+
+  if (!result.ok) {
+    console.error(`Error: ${result.error || 'Failed to submit feedback'}`);
+    process.exit(1);
+  }
+
+  console.log('Feedback submitted.');
+  if (result.data?.feedback_id) {
+    console.log(`  ID: ${result.data.feedback_id}`);
+  }
+}
+
+async function listFeedback() {
+  const { token } = getAuth();
+  const businessId = getBusinessId();
+
+  let url = '/feedback?limit=20';
+  if (businessId) {
+    url += `&business_id=${businessId}`;
+  }
+
+  const result = await apiRequestJson(url, {
+    method: 'GET',
+    token,
+  });
+
+  if (!result.ok) {
+    console.error(`Error: ${result.error || 'Failed to fetch feedback'}`);
+    process.exit(1);
+  }
+
+  const items = result.data?.feedback || [];
+
+  if (items.length === 0) {
+    console.log('No feedback found.');
+    return;
+  }
+
+  console.log(`${items.length} feedback item${items.length !== 1 ? 's' : ''}:\n`);
+
+  for (const item of items) {
+    const date = item.created_at
+      ? new Date(item.created_at).toLocaleDateString('en-US', { month: 'short', day: 'numeric', year: 'numeric' })
+      : '';
+    const status = item.status || 'open';
+    const msg = item.message || '';
+    const preview = msg.length > 80 ? msg.substring(0, 80) + '...' : msg;
+
+    console.log(`  [${status}] ${preview}`);
+    if (date || item.id) {
+      const parts = [];
+      if (date) parts.push(date);
+      if (item.id) parts.push(item.id.substring(0, 8));
+      console.log(`         ${parts.join('  ')}`);
+    }
+    console.log('');
+  }
+}
+
+async function feedbackCommand() {
+  const subcommand = process.argv[3];
+
+  if (!subcommand || subcommand === 'list') {
+    await listFeedback();
+  } else if (subcommand === '--help' || subcommand === '-h') {
+    console.log('');
+    console.log('Usage:');
+    console.log('  atris feedback "message"   Submit feedback');
+    console.log('  atris feedback             List your feedback');
+    console.log('  atris feedback list        List your feedback');
+    console.log('');
+  } else {
+    // Everything else is a feedback message
+    const message = process.argv.slice(3).join(' ');
+    await submitFeedback(message);
+  }
+}
+
+module.exports = { feedbackCommand };