@jackwener/opencli 1.6.0 → 1.6.2

package/autoresearch/engine.ts

@@ -0,0 +1,359 @@
+/**
+ * AutoResearch Engine — Karpathy's 8-phase autonomous iteration loop.
+ *
+ * Phase 0: Precondition checks (git clean, no locks)
+ * Phase 1: Review (read scope files + log + git history)
+ * Phase 2: Ideate (select next change based on history)
+ * Phase 3: Modify (one atomic change — delegated to caller)
+ * Phase 4: Commit (git add + commit with experiment prefix)
+ * Phase 5: Verify (run verify command, extract metric)
+ * Phase 5.5: Guard (optional regression check)
+ * Phase 6: Decide (keep/discard/crash + rollback)
+ * Phase 7: Log (append TSV)
+ * Phase 8: Repeat
+ */
+
+import { execSync } from 'node:child_process';
+import { readFileSync, existsSync } from 'node:fs';
+import { join } from 'node:path';
+import { type AutoResearchConfig, type IterationResult, type IterationStatus, extractMetric } from './config.js';
+import { Logger } from './logger.js';
+
+export interface EngineCallbacks {
+  /** Called at Phase 2-3: review context, ideate, and make ONE change.
+   *  Return a one-sentence description of what was changed, or null to skip. */
+  modify(context: ModifyContext): Promise<string | null>;
+
+  /** Called when engine needs to report status */
+  onStatus?(msg: string): void;
+}
+
+export interface ModifyContext {
+  iteration: number;
+  bestMetric: number;
+  currentMetric: number;
+  recentLog: IterationResult[];
+  gitLog: string;
+  scopeFiles: string[];
+  consecutiveDiscards: number;
+  stuckHint: string | null;
+}
+
+const ROOT = join(import.meta.dirname ?? process.cwd(), '..');
+
+function exec(cmd: string, opts?: { timeout?: number; cwd?: string }): string {
+  try {
+    return execSync(cmd, {
+      cwd: opts?.cwd ?? ROOT,
+      timeout: opts?.timeout ?? 120_000,
+      encoding: 'utf-8',
+      stdio: ['pipe', 'pipe', 'pipe'],
+      env: process.env,
+    }).trim();
+  } catch (err: any) {
+    return err.stdout?.trim() ?? err.message ?? '';
+  }
+}
+
+function execStrict(cmd: string, opts?: { timeout?: number }): string {
+  return execSync(cmd, {
+    cwd: ROOT,
+    timeout: opts?.timeout ?? 120_000,
+    encoding: 'utf-8',
+    stdio: ['pipe', 'pipe', 'pipe'],
+    env: process.env,
+  }).trim();
+}
+
+export class Engine {
+  private config: AutoResearchConfig;
+  private logger: Logger;
+  private callbacks: EngineCallbacks;
+  private bestMetric: number = 0;
+  private currentMetric: number = 0;
+  private iteration: number = 0;
+
+  constructor(config: AutoResearchConfig, logPath: string, callbacks: EngineCallbacks) {
+    this.config = config;
+    this.logger = new Logger(logPath);
+    this.callbacks = callbacks;
+  }
+
+  private log(msg: string): void {
+    this.callbacks.onStatus?.(msg);
+  }
+
+  /** Phase 0: Precondition checks */
+  private checkPreconditions(): void {
+    // Git repo exists
+    try { execStrict('git rev-parse --git-dir'); }
+    catch { throw new Error('Not a git repository'); }
+
+    // Clean working tree
+    const status = exec('git status --porcelain');
+    if (status) throw new Error(`Working tree not clean:\n${status}`);
+
+    // No stale locks
+    if (existsSync(join(ROOT, '.git', 'index.lock'))) {
+      throw new Error('Stale .git/index.lock found — remove it first');
+    }
+
+    // Not detached HEAD
+    try { execStrict('git symbolic-ref HEAD'); }
+    catch { throw new Error('Detached HEAD — checkout a branch first'); }
+  }
+
+  /** Phase 5: Run verify command and extract metric */
+  private runVerify(): number | null {
+    this.log('  verify...');
+    const output = exec(this.config.verify, { timeout: 300_000 });
+    return extractMetric(output);
+  }
+
+  /** Phase 5.5: Run guard command */
+  private runGuard(): boolean {
+    if (!this.config.guard) return true;
+    this.log('  guard...');
+    try {
+      execStrict(this.config.guard, { timeout: 300_000 });
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  /** Phase 4: Commit changes */
+  private commit(description: string): string | null {
+    // Stage all changes in scope (but not untracked outside scope)
+    exec('git add -A');
+    const diff = exec('git diff --cached --quiet; echo $?');
+    if (diff === '0') return null; // no changes
+
+    try {
+      execStrict(`git commit -m "experiment(operate): ${description.replace(/"/g, '\\"')}"`);
+      return exec('git rev-parse --short HEAD');
+    } catch {
+      // Hook failure
+      exec('git reset HEAD');
+      return 'hook-blocked';
+    }
+  }
+
+  /** Phase 6: Rollback */
+  private safeRevert(): void {
+    try {
+      execStrict('git revert HEAD --no-edit');
+    } catch {
+      exec('git revert --abort');
+      exec('git reset --hard HEAD~1');
+    }
+  }
+
+  /** Get stuck hint when >5 consecutive discards */
+  private getStuckHint(discards: number): string | null {
+    if (discards < 5) return null;
+    const hints = [
+      'Re-read ALL scope files from scratch. Try a completely different approach.',
+      'Review entire results log — what worked before? Try combining successful changes.',
+      'Try the OPPOSITE of what has been failing.',
+      'Try a radical architectural change instead of incremental tweaks.',
+      'Simplify — remove complexity rather than adding it.',
+    ];
+    return hints[Math.min(discards - 5, hints.length - 1)];
+  }
+
+  /** Run the main loop */
+  async run(): Promise<IterationResult[]> {
+    const results: IterationResult[] = [];
+
+    // Phase 0: Preconditions
+    this.log('Phase 0: Precondition checks...');
+    this.checkPreconditions();
+
+    // Initialize logger
+    this.logger.init(this.config);
+
+    // Baseline measurement
+    this.log('Measuring baseline...');
+    const baseline = this.runVerify();
+    if (baseline == null) throw new Error('Verify command returned no metric for baseline');
+    this.bestMetric = baseline;
+    this.currentMetric = baseline;
+
+    const baselineCommit = exec('git rev-parse --short HEAD');
+    const baselineResult: IterationResult = {
+      iteration: 0,
+      commit: baselineCommit,
+      metric: baseline,
+      delta: 0,
+      guard: this.config.guard ? (this.runGuard() ? 'pass' : 'fail') : '-',
+      status: 'baseline',
+      description: `initial state — ${this.config.metric} ${baseline}`,
+    };
+    this.logger.append(baselineResult);
+    results.push(baselineResult);
+    this.log(`Baseline: ${this.config.metric} = ${baseline}`);
+
+    // Main loop
+    const maxIter = this.config.iterations ?? Infinity;
+    for (this.iteration = 1; this.iteration <= maxIter; this.iteration++) {
+      this.log(`\n━━━ Iteration ${this.iteration}${maxIter < Infinity ? `/${maxIter}` : ''} ━━━`);
+
+      // Phase 1: Review
+      const gitLog = exec('git log --oneline -20');
+      const recentLog = this.logger.readLast(20);
+      const scopeFiles = this.config.scope;
+      const consecutiveDiscards = this.logger.consecutiveDiscards();
+
+      // Phase 2-3: Ideate + Modify (delegated to callback)
+      const context: ModifyContext = {
+        iteration: this.iteration,
+        bestMetric: this.bestMetric,
+        currentMetric: this.currentMetric,
+        recentLog,
+        gitLog,
+        scopeFiles,
+        consecutiveDiscards,
+        stuckHint: this.getStuckHint(consecutiveDiscards),
+      };
+
+      let description: string | null;
+      try {
+        description = await this.callbacks.modify(context);
+      } catch (err: any) {
+        this.log(`  modify error: ${err.message}`);
+        const result: IterationResult = {
+          iteration: this.iteration,
+          commit: '-',
+          metric: this.currentMetric,
+          delta: 0,
+          guard: '-',
+          status: 'crash',
+          description: `modify crashed: ${err.message?.slice(0, 80)}`,
+        };
+        this.logger.append(result);
+        results.push(result);
+        continue;
+      }
+
+      if (!description) {
+        const result: IterationResult = {
+          iteration: this.iteration,
+          commit: '-',
+          metric: this.currentMetric,
+          delta: 0,
+          guard: '-',
+          status: 'no-op',
+          description: 'no changes made',
+        };
+        this.logger.append(result);
+        results.push(result);
+        continue;
+      }
+
+      // Phase 4: Commit
+      this.log(`  commit: ${description}`);
+      const commitHash = this.commit(description);
+      if (!commitHash) {
+        const result: IterationResult = {
+          iteration: this.iteration,
+          commit: '-',
+          metric: this.currentMetric,
+          delta: 0,
+          guard: '-',
+          status: 'no-op',
+          description: `no diff after: ${description}`,
+        };
+        this.logger.append(result);
+        results.push(result);
+        continue;
+      }
+      if (commitHash === 'hook-blocked') {
+        const result: IterationResult = {
+          iteration: this.iteration,
+          commit: '-',
+          metric: this.currentMetric,
+          delta: 0,
+          guard: '-',
+          status: 'hook-blocked',
+          description: `hook rejected: ${description}`,
+        };
+        this.logger.append(result);
+        results.push(result);
+        continue;
+      }
+
+      // Phase 5: Verify
+      const metric = this.runVerify();
+      if (metric == null) {
+        this.log('  verify crashed — reverting');
+        this.safeRevert();
+        const result: IterationResult = {
+          iteration: this.iteration,
+          commit: '-',
+          metric: this.currentMetric,
+          delta: 0,
+          guard: '-',
+          status: 'crash',
+          description: `verify crashed: ${description}`,
+        };
+        this.logger.append(result);
+        results.push(result);
+        continue;
+      }
+
+      const improved = this.config.direction === 'higher'
+        ? metric > this.bestMetric
+        : metric < this.bestMetric;
+      const delta = +(metric - this.bestMetric).toFixed(4);
+      const absDelta = Math.abs(delta);
+      const minDelta = this.config.minDelta ?? 0;
+
+      // Phase 5.5: Guard
+      let guardResult: 'pass' | 'fail' | '-' = '-';
+      if (this.config.guard && improved && absDelta >= minDelta) {
+        guardResult = this.runGuard() ? 'pass' : 'fail';
+      }
+
+      // Phase 6: Decide
+      let status: IterationStatus;
+      if (improved && absDelta >= minDelta && (guardResult !== 'fail')) {
+        status = 'keep';
+        this.bestMetric = metric;
+        this.currentMetric = metric;
+        this.log(`  ✓ KEEP — ${this.config.metric}: ${metric} (${delta >= 0 ? '+' : ''}${delta})`);
+      } else if (improved && guardResult === 'fail') {
+        this.log('  guard failed — reverting');
+        this.safeRevert();
+        status = 'discard';
+        this.log(`  ✗ DISCARD (guard) — ${description}`);
+      } else {
+        this.safeRevert();
+        status = 'discard';
+        const reason = absDelta < minDelta ? 'below min delta' : 'no improvement';
+        this.log(`  ✗ DISCARD (${reason}) — ${this.config.metric}: ${metric} (${delta >= 0 ? '+' : ''}${delta})`);
+      }
+
+      const result: IterationResult = {
+        iteration: this.iteration,
+        commit: status === 'keep' ? commitHash : '-',
+        metric,
+        delta,
+        guard: guardResult,
+        status,
+        description,
+      };
+      this.logger.append(result);
+      results.push(result);
+    }
+
+    // Summary
+    const keeps = results.filter(r => r.status === 'keep' || r.status === 'keep (reworked)');
+    const discards = results.filter(r => r.status === 'discard');
+    this.log(`\n${'━'.repeat(50)}`);
+    this.log(`Done: ${this.iteration - 1} iterations, ${keeps.length} kept, ${discards.length} discarded`);
+    this.log(`Final ${this.config.metric}: ${this.bestMetric} (started at ${results[0]?.metric})`);
+
+    return results;
+  }
+}

package/autoresearch/eval-all.ts

@@ -0,0 +1,127 @@
+#!/usr/bin/env npx tsx
+/**
+ * Combined Test Suite Runner — runs browse + V2EX + Zhihu tasks.
+ * Reports combined score for AutoResearch iteration.
+ *
+ * Usage:
+ *   npx tsx autoresearch/eval-all.ts              # Run all
+ *   npx tsx autoresearch/eval-all.ts --suite v2ex  # Run one suite
+ */
+
+import { execSync } from 'node:child_process';
+import { readFileSync, writeFileSync, mkdirSync, readdirSync } from 'node:fs';
+import { join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = join(__dirname, '..');
+const RESULTS_DIR = join(__dirname, 'results');
+
+interface SuiteResult {
+  name: string;
+  passed: number;
+  total: number;
+  failures: string[];
+  duration: number;
+}
+
+function runSuite(name: string, script: string): SuiteResult {
+  const start = Date.now();
+  try {
+    const output = execSync(`npx tsx ${script}`, {
+      cwd: ROOT,
+      timeout: 600_000,
+      encoding: 'utf-8',
+      env: process.env,
+      stdio: ['pipe', 'pipe', 'pipe'],
+    });
+
+    // Parse SCORE=X/Y from output
+    const scoreMatch = output.match(/SCORE=(\d+)\/(\d+)/);
+    const passed = scoreMatch ? parseInt(scoreMatch[1], 10) : 0;
+    const total = scoreMatch ? parseInt(scoreMatch[2], 10) : 0;
+
+    // Parse failures
+    const failures: string[] = [];
+    const failLines = output.match(/✗.*$/gm) || [];
+    for (const line of failLines) {
+      const m = line.match(/✗\s+(?:\[.*?\]\s+)?(\S+)/);
+      if (m) failures.push(m[1].replace(/:$/, ''));
+    }
+
+    return { name, passed, total, failures, duration: Date.now() - start };
+  } catch (err: any) {
+    const output = err.stdout ?? '';
+    const scoreMatch = output.match(/SCORE=(\d+)\/(\d+)/);
+    const passed = scoreMatch ? parseInt(scoreMatch[1], 10) : 0;
+    const total = scoreMatch ? parseInt(scoreMatch[2], 10) : 0;
+    const failures: string[] = [];
+    const failLines = output.match(/✗.*$/gm) || [];
+    for (const line of failLines) {
+      const m = line.match(/✗\s+(?:\[.*?\]\s+)?(\S+)/);
+      if (m) failures.push(m[1].replace(/:$/, ''));
+    }
+    return { name, passed, total, failures, duration: Date.now() - start };
+  }
+}
+
+function main() {
+  const args = process.argv.slice(2);
+  const singleSuite = args.includes('--suite') ? args[args.indexOf('--suite') + 1] : null;
+
+  const suites = [
+    { name: 'browse', script: 'autoresearch/eval-browse.ts' },
+    { name: 'v2ex', script: 'autoresearch/eval-v2ex.ts' },
+    { name: 'zhihu', script: 'autoresearch/eval-zhihu.ts' },
+  ].filter(s => !singleSuite || s.name === singleSuite);
+
+  console.log(`\n🔬 Combined AutoResearch — ${suites.length} suites\n`);
+
+  const results: SuiteResult[] = [];
+  for (const suite of suites) {
+    console.log(`  Running ${suite.name}...`);
+    const result = runSuite(suite.name, suite.script);
+    results.push(result);
+    const icon = result.passed === result.total ? '✓' : '✗';
+    console.log(`    ${icon} ${result.name}: ${result.passed}/${result.total} (${Math.round(result.duration / 1000)}s)`);
+    if (result.failures.length > 0) {
+      for (const f of result.failures.slice(0, 5)) {
+        console.log(`      ✗ ${f}`);
+      }
+    }
+  }
+
+  // Summary
+  const totalPassed = results.reduce((s, r) => s + r.passed, 0);
+  const totalTasks = results.reduce((s, r) => s + r.total, 0);
+  const totalDuration = results.reduce((s, r) => s + r.duration, 0);
+  const allFailures = results.flatMap(r => r.failures.map(f => `${r.name}:${f}`));
+
+  console.log(`\n${'━'.repeat(50)}`);
+  console.log(`  Combined: ${totalPassed}/${totalTasks}`);
+  for (const r of results) {
+    console.log(`    ${r.name}: ${r.passed}/${r.total}`);
+  }
+  console.log(`  Time: ${Math.round(totalDuration / 60000)}min`);
+  if (allFailures.length > 0) {
+    console.log(`\n  All failures:`);
+    for (const f of allFailures) console.log(`    ✗ ${f}`);
+  }
+
+  // Save result
+  mkdirSync(RESULTS_DIR, { recursive: true });
+  const existing = readdirSync(RESULTS_DIR).filter(f => f.startsWith('all-')).length;
+  const roundNum = String(existing + 1).padStart(3, '0');
+  const resultPath = join(RESULTS_DIR, `all-${roundNum}.json`);
+  writeFileSync(resultPath, JSON.stringify({
+    timestamp: new Date().toISOString(),
+    score: `${totalPassed}/${totalTasks}`,
+    suites: Object.fromEntries(results.map(r => [r.name, `${r.passed}/${r.total}`])),
+    failures: allFailures,
+    duration: `${Math.round(totalDuration / 60000)}min`,
+  }, null, 2), 'utf-8');
+  console.log(`\n  Results saved to: ${resultPath}`);
+  console.log(`\nSCORE=${totalPassed}/${totalTasks}`);
+}
+
+main();

package/autoresearch/eval-browse.ts

@@ -76,7 +76,7 @@ function runCommand(cmd: string): string {
       stdio: ['pipe', 'pipe', 'pipe'],
     }).trim();
   } catch (err: any) {
-    return err.stdout?.trim() ?? '';
+    return err.stdout?.trim() || err.stderr?.trim() || '';
   }
 }