create-walle 0.9.21 → 0.9.23

package/template/wall-e/eval/weekly-eval-loop.js

@@ -1,241 +0,0 @@
-'use strict';
-
-/**
- * Weekly evaluation loop — automated Sunday cycle.
- * Schedule: Sun 2am benchmarks, 3am regressions, 4am mining, 5am promotions.
- */
-
-const crypto = require('crypto');
-
-/**
- * Run the full weekly evaluation cycle.
- * @param {Object} brain - Brain module
- * @param {Object} [opts] - Options
- * @param {boolean} [opts.dryRun=false] - Print plan without executing
- * @param {Function} [opts.runAgentLoop] - Agent loop function (required if not dryRun)
- * @returns {Promise<Object>} Summary of actions taken
- */
-async function run(brain, opts = {}) {
-  const { dryRun = false, runAgentLoop } = opts;
-  const summary = {
-    runId: crypto.randomUUID(),
-    startedAt: new Date().toISOString(),
-    steps: [],
-    benchmarkResults: null,
-    regressions: [],
-    minedBenchmarks: 0,
-    promotions: { candidates: 0, alerts: 0 },
-    demotions: { candidates: 0, alerts: 0 },
-  };
-
-  // Step 1: Run benchmarks (custom + SWE-bench 30-task subset)
-  const step1 = { name: 'benchmarks', status: 'pending', detail: '' };
-  summary.steps.push(step1);
-
-  if (dryRun) {
-    step1.status = 'skipped (dry-run)';
-    step1.detail = 'Would run 30 custom benchmarks on default model';
-  } else {
-    try {
-      const { runAgentBenchmarkSuite, detectRegressions, resolveModelName } = require('./agent-runner');
-      const result = await runAgentBenchmarkSuite({
-        brain,
-        runAgentLoop,
-        model: resolveModelName(),
-        timeoutMs: 600000,
-      });
-      step1.status = 'completed';
-      step1.detail = `${result.completed}/${result.totalBenchmarks} benchmarks, avg score: ${result.avgScore.toFixed(3)}`;
-      summary.benchmarkResults = result;
-
-      // Step 2: Detect regressions
-      const step2 = { name: 'regressions', status: 'pending', detail: '' };
-      summary.steps.push(step2);
-      const regressions = detectRegressions(brain, result.results);
-      summary.regressions = regressions;
-
-      if (regressions.length > 0) {
-        // Store regression alerts
-        const db = brain.getDb();
-        for (const reg of regressions) {
-          try {
-            db.prepare(`
-              INSERT OR IGNORE INTO eval_regression_alerts (id, benchmark_id, model, provider, baseline_avg, current_score, drop_pct)
-              VALUES (?, ?, ?, ?, ?, ?, ?)
-            `).run(
-              crypto.randomUUID(), reg.benchmarkId,
-              'default', 'default',
-              reg.baselineAvg, reg.currentScore, reg.dropPct
-            );
-          } catch { /* non-fatal */ }
-        }
-        step2.status = 'completed';
-        step2.detail = `${regressions.length} regressions detected and stored`;
-      } else {
-        step2.status = 'completed';
-        step2.detail = 'No regressions detected';
-      }
-    } catch (err) {
-      step1.status = 'error';
-      step1.detail = err.message;
-    }
-  }
-
-  // Step 3: Mine sessions for new benchmarks
-  const step3 = { name: 'session-mining', status: 'pending', detail: '' };
-  summary.steps.push(step3);
-
-  if (dryRun) {
-    step3.status = 'skipped (dry-run)';
-    step3.detail = 'Would mine high-significance sessions for benchmark generation';
-  } else {
-    try {
-      const { mineAndGenerate } = require('./session-miner');
-      const mineResult = await mineAndGenerate(brain, { maxPerWeek: 5 });
-      summary.minedBenchmarks = mineResult.generated.length;
-      step3.status = 'completed';
-      step3.detail = `${mineResult.generated.length} new benchmarks from ${mineResult.candidates.length} candidates`;
-    } catch (err) {
-      step3.status = 'error';
-      step3.detail = err.message;
-    }
-  }
-
-  // Step 4: Check promotions and demotions
-  const step4 = { name: 'model-evaluation', status: 'pending', detail: '' };
-  summary.steps.push(step4);
-
-  if (dryRun) {
-    step4.status = 'skipped (dry-run)';
-    step4.detail = 'Would check model promotions and demotions';
-  } else {
-    try {
-      const { checkPromotions, checkDemotions } = require('./promoter');
-      const promoResult = checkPromotions(brain);
-      summary.promotions = { candidates: promoResult.candidates.length, alerts: promoResult.alertsCreated };
-
-      let demoResult = { demotions: [], alertsCreated: 0 };
-      if (typeof checkDemotions === 'function') {
-        demoResult = checkDemotions(brain);
-      }
-      summary.demotions = { candidates: demoResult.demotions?.length || 0, alerts: demoResult.alertsCreated || 0 };
-
-      step4.status = 'completed';
-      step4.detail = `Promotions: ${promoResult.alertsCreated} alerts, Demotions: ${demoResult.alertsCreated || 0} alerts`;
-    } catch (err) {
-      step4.status = 'error';
-      step4.detail = err.message;
-    }
-  }
-
-  // Step 5: Aggregate scores
-  const step5 = { name: 'score-aggregation', status: 'pending', detail: '' };
-  summary.steps.push(step5);
-
-  if (dryRun) {
-    step5.status = 'skipped (dry-run)';
-    step5.detail = 'Would update all model task scores';
-  } else {
-    try {
-      const { updateAllModelTaskScores } = require('./aggregator');
-      const aggResult = updateAllModelTaskScores(brain);
-      step5.status = 'completed';
-      step5.detail = `Updated ${aggResult.updated} model/task score pairs`;
-    } catch (err) {
-      step5.status = 'error';
-      step5.detail = err.message;
-    }
-  }
-
-  // Step 6: Create Monday briefing item with eval summary
-  const step6 = { name: 'monday-briefing', status: 'pending', detail: '' };
-  summary.steps.push(step6);
-
-  if (dryRun) {
-    step6.status = 'skipped (dry-run)';
-    step6.detail = 'Would create Monday briefing item with eval summary';
-  } else {
-    try {
-      const parts = [];
-      if (summary.benchmarkResults) {
-        parts.push(`Benchmarks: ${summary.benchmarkResults.totalBenchmarks || 0} run`);
-      }
-      if (summary.regressions.length > 0) {
-        parts.push(`Regressions: ${summary.regressions.length} detected`);
-      }
-      if (summary.promotions.alerts > 0) {
-        parts.push(`Promotions: ${summary.promotions.alerts}`);
-      }
-      if (summary.demotions.alerts > 0) {
-        parts.push(`Demotions: ${summary.demotions.alerts}`);
-      }
-      if (summary.minedBenchmarks > 0) {
-        parts.push(`New benchmarks mined: ${summary.minedBenchmarks}`);
-      }
-
-      const briefingText = parts.length > 0
-        ? `Weekly eval summary: ${parts.join(', ')}.`
-        : 'Weekly eval completed with no notable changes.';
-
-      if (brain && typeof brain.insertTask === 'function') {
-        brain.insertTask({
-          title: 'Weekly Eval Results',
-          description: briefingText,
-          type: 'once',
-          priority: summary.regressions.length > 0 ? 'high' : 'normal',
-        });
-      }
-      step6.status = 'completed';
-      step6.detail = briefingText;
-    } catch (err) {
-      step6.status = 'error';
-      step6.detail = err.message;
-    }
-  }
-
-  summary.completedAt = new Date().toISOString();
-  return summary;
-}
-
-// CLI: node eval/weekly-eval-loop.js --dry-run
-if (require.main === module) {
-  const args = process.argv.slice(2);
-  const dryRun = args.includes('--dry-run');
-
-  (async () => {
-    let brain = null;
-    try {
-      brain = require('../brain');
-      brain.initDb();
-    } catch (err) {
-      console.error('Brain not available:', err.message);
-      process.exit(1);
-    }
-
-    console.log('=== Weekly Evaluation Loop ===');
-    console.log(`Mode: ${dryRun ? 'DRY RUN' : 'LIVE'}`);
-    console.log('');
-
-    const summary = await run(brain, { dryRun });
-
-    console.log('Steps:');
-    for (const step of summary.steps) {
-      console.log(`  ${step.name}: ${step.status}`);
-      if (step.detail) console.log(`    ${step.detail}`);
-    }
-
-    if (summary.regressions.length > 0) {
-      console.log(`\nRegressions: ${summary.regressions.length}`);
-      for (const r of summary.regressions) {
-        console.log(`  ${r.benchmarkId}: dropped ${r.dropPct}%`);
-      }
-    }
-
-    console.log(`\nCompleted at: ${summary.completedAt}`);
-  })().catch(err => {
-    console.error('Fatal:', err);
-    process.exit(1);
-  });
-}
-
-module.exports = { run };