opengstack 0.14.0 → 0.14.2

package/scripts/eval-select.ts

@@ -0,0 +1,86 @@
+#!/usr/bin/env bun
+/**
+ * Show which E2E and LLM-judge tests would run based on the current git diff.
+ *
+ * Usage:
+ * bun run eval:select # human-readable output
+ * bun run eval:select --json # machine-readable JSON
+ * bun run eval:select --base main # override base branch
+ */
+
+import * as path from 'path';
+import {
+ selectTests,
+ detectBaseBranch,
+ getChangedFiles,
+ E2E_TOUCHFILES,
+ LLM_JUDGE_TOUCHFILES,
+ GLOBAL_TOUCHFILES,
+} from '../test/helpers/touchfiles';
+
+const ROOT = path.resolve(import.meta.dir, '..');
+const args = process.argv.slice(2);
+const jsonMode = args.includes('--json');
+const baseIdx = args.indexOf('--base');
+const baseOverride = baseIdx >= 0 ? args[baseIdx + 1] : undefined;
+
+// Detect base branch
+const baseBranch = baseOverride || detectBaseBranch(ROOT) || 'main';
+const changedFiles = getChangedFiles(baseBranch, ROOT);
+
+if (changedFiles.length === 0) {
+ if (jsonMode) {
+ console.log(JSON.stringify({ base: baseBranch, changed_files: 0, e2e: 'all', llm_judge: 'all', reason: 'no diff — would run all tests' }));
+ } else {
+ console.log(`Base: ${baseBranch}`);
+ console.log('No changed files detected — all tests would run.');
+ }
+ process.exit(0);
+}
+
+const e2eSelection = selectTests(changedFiles, E2E_TOUCHFILES, GLOBAL_TOUCHFILES);
+const llmSelection = selectTests(changedFiles, LLM_JUDGE_TOUCHFILES, GLOBAL_TOUCHFILES);
+
+if (jsonMode) {
+ console.log(JSON.stringify({
+ base: baseBranch,
+ changed_files: changedFiles,
+ e2e: {
+ selected: e2eSelection.selected,
+ skipped: e2eSelection.skipped,
+ reason: e2eSelection.reason,
+ count: `${e2eSelection.selected.length}/${Object.keys(E2E_TOUCHFILES).length}`,
+ },
+ llm_judge: {
+ selected: llmSelection.selected,
+ skipped: llmSelection.skipped,
+ reason: llmSelection.reason,
+ count: `${llmSelection.selected.length}/${Object.keys(LLM_JUDGE_TOUCHFILES).length}`,
+ },
+ }, null, 2));
+} else {
+ console.log(`Base: ${baseBranch}`);
+ console.log(`Changed files: ${changedFiles.length}`);
+ console.log();
+
+ console.log(`E2E (${e2eSelection.reason}): ${e2eSelection.selected.length}/${Object.keys(E2E_TOUCHFILES).length} tests`);
+ if (e2eSelection.selected.length > 0 && e2eSelection.selected.length < Object.keys(E2E_TOUCHFILES).length) {
+ console.log(` Selected: ${e2eSelection.selected.join(', ')}`);
+ console.log(` Skipped: ${e2eSelection.skipped.join(', ')}`);
+ } else if (e2eSelection.selected.length === 0) {
+ console.log(' No E2E tests affected.');
+ } else {
+ console.log(' All E2E tests selected.');
+ }
+ console.log();
+
+ console.log(`LLM-judge (${llmSelection.reason}): ${llmSelection.selected.length}/${Object.keys(LLM_JUDGE_TOUCHFILES).length} tests`);
+ if (llmSelection.selected.length > 0 && llmSelection.selected.length < Object.keys(LLM_JUDGE_TOUCHFILES).length) {
+ console.log(` Selected: ${llmSelection.selected.join(', ')}`);
+ console.log(` Skipped: ${llmSelection.skipped.join(', ')}`);
+ } else if (llmSelection.selected.length === 0) {
+ console.log(' No LLM-judge tests affected.');
+ } else {
+ console.log(' All LLM-judge tests selected.');
+ }
+}

package/scripts/eval-summary.ts

@@ -0,0 +1,188 @@
+#!/usr/bin/env bun
+/**
+ * Aggregate summary of all eval runs from ~/.opengstack-dev/evals/
+ *
+ * Usage: bun run eval:summary
+ */
+
+import * as fs from 'fs';
+import * as path from 'path';
+import * as os from 'os';
+import type { EvalResult } from '../test/helpers/eval-store';
+import { getProjectEvalDir } from '../test/helpers/eval-store';
+
+const EVAL_DIR = getProjectEvalDir();
+
+let files: string[];
+try {
+ files = fs.readdirSync(EVAL_DIR).filter(f => f.endsWith('.json'));
+} catch {
+ console.log('No eval runs yet. Run: EVALS=1 bun run test:evals');
+ process.exit(0);
+}
+
+if (files.length === 0) {
+ console.log('No eval runs yet. Run: EVALS=1 bun run test:evals');
+ process.exit(0);
+}
+
+// Load all results
+const results: EvalResult[] = [];
+for (const file of files) {
+ try {
+ results.push(JSON.parse(fs.readFileSync(path.join(EVAL_DIR, file), 'utf-8')));
+ } catch { continue; }
+}
+
+// Aggregate stats
+const e2eRuns = results.filter(r => r.tier === 'e2e');
+const judgeRuns = results.filter(r => r.tier === 'llm-judge');
+const totalCost = results.reduce((s, r) => s + (r.total_cost_usd || 0), 0);
+const avgE2ECost = e2eRuns.length > 0 ? e2eRuns.reduce((s, r) => s + r.total_cost_usd, 0) / e2eRuns.length : 0;
+const avgJudgeCost = judgeRuns.length > 0 ? judgeRuns.reduce((s, r) => s + r.total_cost_usd, 0) / judgeRuns.length : 0;
+
+// Duration + turns from E2E runs
+const avgE2EDuration = e2eRuns.length > 0
+ ? e2eRuns.reduce((s, r) => s + (r.total_duration_ms || 0), 0) / e2eRuns.length
+ : 0;
+const e2eTurns: number[] = [];
+for (const r of e2eRuns) {
+ const runTurns = r.tests.reduce((s, t) => s + (t.turns_used || 0), 0);
+ if (runTurns > 0) e2eTurns.push(runTurns);
+}
+const avgE2ETurns = e2eTurns.length > 0
+ ? e2eTurns.reduce((a, b) => a + b, 0) / e2eTurns.length
+ : 0;
+
+// Per-test efficiency stats (avg turns + duration across runs)
+const testEfficiency = new Map<string, { turns: number[]; durations: number[]; costs: number[] }>();
+for (const r of e2eRuns) {
+ for (const t of r.tests) {
+ if (!testEfficiency.has(t.name)) {
+ testEfficiency.set(t.name, { turns: [], durations: [], costs: [] });
+ }
+ const stats = testEfficiency.get(t.name)!;
+ if (t.turns_used !== undefined) stats.turns.push(t.turns_used);
+ if (t.duration_ms > 0) stats.durations.push(t.duration_ms);
+ if (t.cost_usd > 0) stats.costs.push(t.cost_usd);
+ }
+}
+
+// Detection rates from outcome evals
+const detectionRates: number[] = [];
+for (const r of e2eRuns) {
+ for (const t of r.tests) {
+ if (t.detection_rate !== undefined) {
+ detectionRates.push(t.detection_rate);
+ }
+ }
+}
+const avgDetection = detectionRates.length > 0
+ ? detectionRates.reduce((a, b) => a + b, 0) / detectionRates.length
+ : null;
+
+// Flaky tests (passed in some runs, failed in others)
+const testResults = new Map<string, boolean[]>();
+for (const r of results) {
+ for (const t of r.tests) {
+ const key = `${r.tier}:${t.name}`;
+ if (!testResults.has(key)) testResults.set(key, []);
+ testResults.get(key)!.push(t.passed);
+ }
+}
+const flakyTests: string[] = [];
+for (const [name, outcomes] of testResults) {
+ if (outcomes.length >= 2) {
+ const hasPass = outcomes.some(o => o);
+ const hasFail = outcomes.some(o => !o);
+ if (hasPass && hasFail) flakyTests.push(name);
+ }
+}
+
+// Branch stats
+const branchStats = new Map<string, { runs: number; avgDetection: number; detections: number[] }>();
+for (const r of e2eRuns) {
+ if (!branchStats.has(r.branch)) {
+ branchStats.set(r.branch, { runs: 0, avgDetection: 0, detections: [] });
+ }
+ const stats = branchStats.get(r.branch)!;
+ stats.runs++;
+ for (const t of r.tests) {
+ if (t.detection_rate !== undefined) {
+ stats.detections.push(t.detection_rate);
+ }
+ }
+}
+for (const stats of branchStats.values()) {
+ stats.avgDetection = stats.detections.length > 0
+ ? stats.detections.reduce((a, b) => a + b, 0) / stats.detections.length
+ : 0;
+}
+
+// Print summary
+console.log('');
+console.log('Eval Summary');
+console.log('═'.repeat(70));
+console.log(` Total runs: ${results.length} (${e2eRuns.length} e2e, ${judgeRuns.length} llm-judge)`);
+console.log(` Total spend: $${totalCost.toFixed(2)}`);
+console.log(` Avg cost/e2e: $${avgE2ECost.toFixed(2)}`);
+console.log(` Avg cost/judge: $${avgJudgeCost.toFixed(2)}`);
+if (avgE2EDuration > 0) {
+ console.log(` Avg duration/e2e: ${Math.round(avgE2EDuration / 1000)}s`);
+}
+if (avgE2ETurns > 0) {
+ console.log(` Avg turns/e2e: ${Math.round(avgE2ETurns)}`);
+}
+if (avgDetection !== null) {
+ console.log(` Avg detection: ${avgDetection.toFixed(1)} bugs`);
+}
+console.log('─'.repeat(70));
+
+// Per-test efficiency averages (only if we have enough data)
+if (testEfficiency.size > 0 && e2eRuns.length >= 2) {
+ console.log(' Per-test efficiency (averages across runs):');
+ const sorted = [...testEfficiency.entries()]
+ .filter(([, s]) => s.turns.length >= 2)
+ .sort((a, b) => {
+ const avgA = a[1].costs.reduce((s, c) => s + c, 0) / a[1].costs.length;
+ const avgB = b[1].costs.reduce((s, c) => s + c, 0) / b[1].costs.length;
+ return avgB - avgA;
+ });
+ for (const [name, stats] of sorted) {
+ const avgT = Math.round(stats.turns.reduce((a, b) => a + b, 0) / stats.turns.length);
+ const avgD = Math.round(stats.durations.reduce((a, b) => a + b, 0) / stats.durations.length / 1000);
+ const avgC = (stats.costs.reduce((a, b) => a + b, 0) / stats.costs.length).toFixed(2);
+ const label = name.length > 30 ? name.slice(0, 27) + '...' : name.padEnd(30);
+ console.log(` ${label} $${avgC} ${avgT}t ${avgD}s (${stats.turns.length} runs)`);
+ }
+ console.log('─'.repeat(70));
+}
+
+if (flakyTests.length > 0) {
+ console.log(` Flaky tests (${flakyTests.length}):`);
+ for (const name of flakyTests) {
+ console.log(` - ${name}`);
+ }
+ console.log('─'.repeat(70));
+}
+
+if (branchStats.size > 0) {
+ console.log(' Branches:');
+ const sorted = [...branchStats.entries()].sort((a, b) => b[1].avgDetection - a[1].avgDetection);
+ for (const [branch, stats] of sorted) {
+ const det = stats.detections.length > 0 ? ` avg det: ${stats.avgDetection.toFixed(1)}` : '';
+ console.log(` ${branch.padEnd(30)} ${stats.runs} runs${det}`);
+ }
+ console.log('─'.repeat(70));
+}
+
+// Date range
+const timestamps = results.map(r => r.timestamp).filter(Boolean).sort();
+if (timestamps.length > 0) {
+ const first = timestamps[0].replace('T', ' ').slice(0, 16);
+ const last = timestamps[timestamps.length - 1].replace('T', ' ').slice(0, 16);
+ console.log(` Date range: ${first} → ${last}`);
+}
+
+console.log(` Dir: ${EVAL_DIR}`);
+console.log('');

package/scripts/eval-watch.ts

@@ -0,0 +1,172 @@
+/**
+ * Live E2E test watcher dashboard.
+ *
+ * Reads heartbeat (e2e-live.json) for current test status and
+ * partial eval results (_partial-e2e.json) for completed tests.
+ * Renders a terminal dashboard every 1s.
+ *
+ * Usage: bun run eval:watch [--tail]
+ */
+
+import * as fs from 'fs';
+import * as path from 'path';
+import * as os from 'os';
+
+const OpenGStack_DEV_DIR = path.join(os.homedir(), '.opengstack-dev');
+const HEARTBEAT_PATH = path.join(OpenGStack_DEV_DIR, 'e2e-live.json');
+const PARTIAL_PATH = path.join(OpenGStack_DEV_DIR, 'evals', '_partial-e2e.json');
+const STALE_THRESHOLD_SEC = 600; // 10 minutes
+
+export interface HeartbeatData {
+ runId: string;
+ pid?: number;
+ startedAt: string;
+ currentTest: string;
+ status: string;
+ turn: number;
+ toolCount: number;
+ lastTool: string;
+ lastToolAt: string;
+ elapsedSec: number;
+}
+
+export interface PartialData {
+ tests: Array<{
+ name: string;
+ passed: boolean;
+ cost_usd: number;
+ duration_ms: number;
+ turns_used?: number;
+ exit_reason?: string;
+ }>;
+ total_cost_usd: number;
+ _partial?: boolean;
+}
+
+/** Read and parse a JSON file, returning null on any error. */
+function readJSON<T>(filePath: string): T | null {
+ try {
+ return JSON.parse(fs.readFileSync(filePath, 'utf-8'));
+ } catch {
+ return null;
+ }
+}
+
+/** Check if a process is alive (signal 0 = existence check, doesn't kill). */
+function isProcessAlive(pid: number): boolean {
+ try {
+ process.kill(pid, 0);
+ return true;
+ } catch {
+ return false;
+ }
+}
+
+/** Format seconds as Xm Ys */
+function formatDuration(sec: number): string {
+ if (sec < 60) return `${sec}s`;
+ const m = Math.floor(sec / 60);
+ const s = sec % 60;
+ return `${m}m ${s}s`;
+}
+
+/** Render dashboard from heartbeat + partial data. Pure function for testability. */
+export function renderDashboard(heartbeat: HeartbeatData | null, partial: PartialData | null): string {
+ const lines: string[] = [];
+
+ if (!heartbeat && !partial) {
+ lines.push('E2E Watch — No active run detected');
+ lines.push('');
+ lines.push(`Heartbeat: ${HEARTBEAT_PATH} (not found)`);
+ lines.push(`Partial: ${PARTIAL_PATH} (not found)`);
+ lines.push('');
+ lines.push('Start a run with: EVALS=1 bun test test/skill-e2e-*.test.ts');
+ return lines.join('\n');
+ }
+
+ const runId = heartbeat?.runId || 'unknown';
+ const elapsed = heartbeat?.elapsedSec || 0;
+ lines.push(`E2E Watch \u2014 Run ${runId} \u2014 ${formatDuration(elapsed)}`);
+ lines.push('\u2550'.repeat(55));
+
+ // Completed tests from partial
+ if (partial?.tests) {
+ for (const t of partial.tests) {
+ const icon = t.passed ? '\u2713' : '\u2717';
+ const cost = `$${t.cost_usd.toFixed(2)}`;
+ const dur = `${Math.round(t.duration_ms / 1000)}s`;
+ const turns = t.turns_used !== undefined ? `${t.turns_used} turns` : '';
+ const name = t.name.length > 30 ? t.name.slice(0, 27) + '...' : t.name.padEnd(30);
+ lines.push(` ${icon} ${name} ${cost.padStart(6)} ${dur.padStart(5)} ${turns}`);
+ }
+ }
+
+ // Current test from heartbeat
+ if (heartbeat && heartbeat.status === 'running') {
+ const name = heartbeat.currentTest.length > 30
+ ? heartbeat.currentTest.slice(0, 27) + '...'
+ : heartbeat.currentTest.padEnd(30);
+ lines.push(` \u29D6 ${name} ${formatDuration(heartbeat.elapsedSec).padStart(6)} turn ${heartbeat.turn} last: ${heartbeat.lastTool}`);
+
+ // Stale detection
+ const lastToolTime = new Date(heartbeat.lastToolAt).getTime();
+ const staleSec = Math.round((Date.now() - lastToolTime) / 1000);
+ if (staleSec > STALE_THRESHOLD_SEC) {
+ lines.push(` \u26A0 STALE: last tool call was ${formatDuration(staleSec)} ago \u2014 run may have crashed`);
+ }
+ }
+
+ lines.push('\u2500'.repeat(55));
+
+ // Summary
+ const completedCount = partial?.tests?.length || 0;
+ const totalCost = partial?.total_cost_usd || 0;
+ const running = heartbeat?.status === 'running' ? 1 : 0;
+ lines.push(` Completed: ${completedCount} Running: ${running} Cost: $${totalCost.toFixed(2)} Elapsed: ${formatDuration(elapsed)}`);
+
+ if (heartbeat?.runId) {
+ const logPath = path.join(OpenGStack_DEV_DIR, 'e2e-runs', heartbeat.runId, 'progress.log');
+ lines.push(` Logs: ${logPath}`);
+ }
+
+ return lines.join('\n');
+}
+
+// --- Main ---
+
+if (import.meta.main) {
+ const showTail = process.argv.includes('--tail');
+
+ const render = () => {
+ let heartbeat = readJSON<HeartbeatData>(HEARTBEAT_PATH);
+ const partial = readJSON<PartialData>(PARTIAL_PATH);
+
+ // Auto-clear heartbeat if the process is dead
+ if (heartbeat?.pid && !isProcessAlive(heartbeat.pid)) {
+ try { fs.unlinkSync(HEARTBEAT_PATH); } catch { /* already gone */ }
+ process.stdout.write('\x1B[2J\x1B[H');
+ process.stdout.write(`Cleared stale heartbeat — PID ${heartbeat.pid} is no longer running.\n\n`);
+ heartbeat = null;
+ }
+
+ // Clear screen
+ process.stdout.write('\x1B[2J\x1B[H');
+ process.stdout.write(renderDashboard(heartbeat, partial) + '\n');
+
+ // --tail: show last 10 lines of progress.log
+ if (showTail && heartbeat?.runId) {
+ const logPath = path.join(OpenGStack_DEV_DIR, 'e2e-runs', heartbeat.runId, 'progress.log');
+ try {
+ const content = fs.readFileSync(logPath, 'utf-8');
+ const tail = content.split('\n').filter(l => l.trim()).slice(-10);
+ process.stdout.write('\nRecent progress:\n');
+ for (const line of tail) {
+ process.stdout.write(line + '\n');
+ }
+ } catch { /* log file may not exist yet */ }
+ }
+ };
+
+ render();
+ setInterval(render, 1000);
+}