@recombine-ai/bosun 0.10.1 → 0.11.2

package/src/backend/tests/data.ts

@@ -1,5 +1,5 @@
 import { tryReadConfig, writeConfig } from './config'
-import { access, readdir, readFile } from 'node:fs/promises'
+import { access, readdir, readFile, stat, writeFile } from 'node:fs/promises'
 import path from 'node:path'
 import {
 	getSuiteFilePath,
@@ -12,8 +12,14 @@ import {
 } from './suite'
 import { getRootPath } from '../lib/get-root-path'
 import z from 'zod'
-import type { ResultEntry, WrittenRunResult } from './academy'
+import type {
+	ResultEntry,
+	WrittenRunReport,
+	WrittenRunReportConfig,
+	WrittenRunResult
+} from './academy'
 import { makeContext, makeContextDiff } from './context-utils'
+import { listAgentReportFiles, readReportFile, reportTimestamp } from './report-files'
 
 export type CsvRow = Record<string, string>
 
@@ -51,16 +57,52 @@ export interface TestResult {
 	test_id: string
 	success_rate_percent?: number
 	success_rate?: string
-	results: RunResult[]
+	results: LatestRunResultSummary[]
 	report_timestamp: string // ISO
 	is_old: boolean
 	previous?: PreviousTestResultSummary
 }
+
+export type LatestRunResultSummary = Pick<RunResult, 'success' | 'result' | 'evaluation' | 'errors'>
 export interface TestTable {
 	rows: TestRow[]
 	headers: string[]
 }
 
+export interface TestRunHistorySummary {
+	timestamp: string
+	runScope: 'all' | 'pinned'
+	testCount: number
+	successCount: number
+	totalRuns: number
+	successRatePercent: number
+	successRate: string
+	passed: boolean
+	repeats: number
+	concurrency: number
+}
+
+export interface TestRunHistoryResultEntry {
+	success: boolean
+	result: string
+	evaluation: string
+	errors: string
+}
+
+export interface TestRunHistoryRow {
+	test: TestRow
+	success_rate?: string
+	success_rate_percent?: number
+	results: TestRunHistoryResultEntry[]
+}
+
+export interface TestRunHistoryDetail {
+	timestamp: string
+	runScope: 'all' | 'pinned'
+	runConfig: WrittenRunReportConfig
+	rows: TestRunHistoryRow[]
+}
+
 const defaultTestRow: Omit<TestRow, 'id'> = {
 	pinned: 'FALSE',
 	scenario: 'New scenario',
@@ -71,7 +113,7 @@ const defaultTestRow: Omit<TestRow, 'id'> = {
 	conversation: 'user: ...',
 	expected_output: '',
 	timeout: '0',
-	similarity_threshold: '60',
+	similarity_threshold: '50',
 	context: {}
 }
 
@@ -90,7 +132,8 @@ export async function loadTests(agent: string): Promise<TestTable> {
 			seenIds.add(id)
 			// previously tests were storing context as a string...
 			const context = typeof r.context === 'string' ? JSON.parse(r.context) : r.context
-			return { ...r, context, id }
+			const mocks = typeof r.mocks === 'string' ? JSON.parse(r.mocks) : r.mocks
+			return typeof mocks === 'undefined' ? { ...r, context, id } : { ...r, context, mocks, id }
 		})
 		if (changed) {
 			await writeSuite(agent, { ...suite, tests: rows })
@@ -106,9 +149,11 @@ export async function loadTests(agent: string): Promise<TestTable> {
 			'assertion_value',
 			'conversation',
 			'expected_output',
+			'capture_mode',
 			'timeout',
 			'similarity_threshold',
-			'context'
+			'context',
+			'mocks'
 		]
 		return { rows, headers }
 	})
@@ -133,12 +178,16 @@ export async function patchTest(
 }
 
 function preparePatch(current: TestRow, patch: Partial<TestRow>) {
-	return {
+	const updated: TestRow = {
 		...defaultTestRow,
 		...current,
 		...patch,
 		context: makeContext(patch.context || current.context)
 	}
+	if (typeof patch.mocks !== 'undefined' || typeof current.mocks !== 'undefined') {
+		updated.mocks = makeContext(patch.mocks ?? current.mocks)
+	}
+	return updated
 }
 
 // Batch update multiple rows in a single read/write to avoid race conditions
@@ -183,6 +232,9 @@ export async function addTest(
 				? makeContextDiff(seed?.context, suite.meta?.defaultContext)
 				: makeContext(seed?.context)
 		}
+		if (typeof seed?.mocks !== 'undefined') {
+			newRow.mocks = makeContext(seed.mocks)
+		}
 		await writeSuite(agent, { ...(suite as any), tests: [newRow, ...suite.tests] })
 		await ensureTestExpandedByDefault(agent, newRow.id)
 		return newRow
@@ -207,7 +259,7 @@ async function ensureTestExpandedByDefault(agent: string, id: string): Promise<v
 async function ensureConfigExists(agent: string): Promise<void> {
 	const existing = await tryReadConfig(agent)
 	if (existing) return
-	await writeConfig(agent, { repeats: 1, concurrency: 1, expandedTestIds: [] })
+	await writeConfig(agent, { repeats: 1, concurrency: 75, expandedTestIds: [] })
 }
 
 async function loadSuiteOrEmpty(agent: string): Promise<TestSuite> {
@@ -273,7 +325,7 @@ export async function loadConfig(agent: string): Promise<RunConfig> {
 		: []
 	return {
 		repeats: typeof (cfg as any).repeats === 'number' ? (cfg as any).repeats : 1,
-		concurrency: typeof (cfg as any).concurrency === 'number' ? (cfg as any).concurrency : 1,
+		concurrency: typeof (cfg as any).concurrency === 'number' ? (cfg as any).concurrency : 75,
 		defaultContext,
 		expandedTestIds
 	}
@@ -306,72 +358,394 @@ export async function saveConfig(agent: string, cfg: RunConfig): Promise<void> {
 
 // We no longer migrate legacy reports automatically. Please update old reports manually.
 
-export async function loadLatestResultsByTest(agent: string): Promise<Record<string, TestResult>> {
-	const projectRoot = path.dirname(await getRootPath())
-	const outDir = path.join(projectRoot, 'data', 'test_runs')
-	let entries: string[] = []
+type ReportFileState = {
+	name: string
+	size: number
+	mtimeMs: number
+}
+
+type LatestResultsCacheEntry = {
+	files: ReportFileState[]
+	results: Record<string, TestResult>
+}
+
+type PersistedLatestResultsCache = LatestResultsCacheEntry & {
+	version: 1
+}
+
+const latestResultsCache = new Map<string, LatestResultsCacheEntry>()
+const latestResultsLoads = new Map<string, Promise<Record<string, TestResult>>>()
+
+function latestResultsCachePath(outDir: string, agent: string): string {
+	return path.join(outDir, `.bosun-latest-results-${encodeURIComponent(agent)}.json`)
+}
+
+async function reportFileStates(outDir: string, files: string[]): Promise<ReportFileState[]> {
+	return Promise.all(
+		files.map(async (name) => {
+			const info = await stat(path.join(outDir, name))
+			return { name, size: info.size, mtimeMs: info.mtimeMs }
+		})
+	)
+}
+
+function sameReportFile(a: ReportFileState, b: ReportFileState): boolean {
+	return a.name === b.name && a.size === b.size && a.mtimeMs === b.mtimeMs
+}
+
+function incrementalReportStart(
+	previous: ReportFileState[],
+	current: ReportFileState[]
+): number | null {
+	if (current.length < previous.length) return -1
+
+	const sharedLength = Math.min(previous.length, current.length)
+	for (let index = 0; index < sharedLength; index += 1) {
+		if (sameReportFile(previous[index], current[index])) continue
+		return index >= Math.max(0, previous.length - 1) ? index : -1
+	}
+
+	return current.length === previous.length ? null : previous.length
+}
+
+function isPersistedLatestResultsCache(value: unknown): value is PersistedLatestResultsCache {
+	if (!value || typeof value !== 'object') return false
+	const candidate = value as Partial<PersistedLatestResultsCache>
+	return (
+		candidate.version === 1 &&
+		Array.isArray(candidate.files) &&
+		candidate.files.every(
+			(file) =>
+				file &&
+				typeof file.name === 'string' &&
+				typeof file.size === 'number' &&
+				Number.isFinite(file.size) &&
+				typeof file.mtimeMs === 'number' &&
+				Number.isFinite(file.mtimeMs)
+		) &&
+		!!candidate.results &&
+		typeof candidate.results === 'object' &&
+		!Array.isArray(candidate.results)
+	)
+}
+
+async function readPersistedLatestResultsCache(
+	outDir: string,
+	agent: string
+): Promise<LatestResultsCacheEntry | null> {
 	try {
-		entries = await readdir(outDir)
+		const raw = await readFile(latestResultsCachePath(outDir, agent), 'utf8')
+		const parsed: unknown = JSON.parse(raw)
+		if (!isPersistedLatestResultsCache(parsed)) return null
+		return { files: parsed.files, results: parsed.results }
 	} catch {
-		return {}
+		return null
 	}
-	const prefix = `${agent}.`
-	const suffix = `.report.json`
-	const files = entries.filter((n) => n.startsWith(prefix) && n.endsWith(suffix)).sort()
-	if (!files.length) return {}
+}
 
-	const latestFile = files[files.length - 1]
-	const latestTs = latestFile
-		.slice(prefix.length, latestFile.length - suffix.length)
-		.replace(/-/g, ':')
+async function persistLatestResultsCache(
+	outDir: string,
+	agent: string,
+	entry: LatestResultsCacheEntry
+): Promise<void> {
+	try {
+		await writeFile(
+			latestResultsCachePath(outDir, agent),
+			JSON.stringify({ version: 1, ...entry } satisfies PersistedLatestResultsCache),
+			'utf8'
+		)
+	} catch {
+		// The in-memory cache still avoids repeat work for the current Bosun process.
+	}
+}
 
-	const results: Record<string, TestResult> = {}
-	for (let i = files.length - 1; i >= 0; i--) {
-		const name = files[i]
-		const full = path.join(outDir, name)
-		const raw = await readFile(full, 'utf8')
-		const parsed = JSON.parse(raw) as any
-		const rows = Array.isArray(parsed) ? parsed : Array.isArray(parsed?.rows) ? parsed.rows : []
-		const ts = name.slice(prefix.length, name.length - suffix.length).replace(/-/g, ':')
-		for (const r of rows) {
-			const test_id = (r as any).test_id
-			if (!test_id) {
+function summarizeLatestRun(run: RunResult): LatestRunResultSummary {
+	return {
+		success: !!run?.success,
+		result: String(run?.result ?? ''),
+		evaluation: String(run?.evaluation ?? ''),
+		errors: String(run?.errors ?? '')
+	}
+}
+
+function reportRows(parsed: any): any[] {
+	return Array.isArray(parsed) ? parsed : Array.isArray(parsed?.rows) ? parsed.rows : []
+}
+
+function resultSummary(result: TestResult): PreviousTestResultSummary {
+	return {
+		success_count: result.results.reduce((count, run) => count + (run.success ? 1 : 0), 0),
+		total_runs: result.results.length,
+		report_timestamp: result.report_timestamp
+	}
+}
+
+function resultFromReportRow(
+	row: any,
+	timestamp: string,
+	latestTimestamp: string,
+	previous?: PreviousTestResultSummary
+): TestResult {
+	return {
+		test_id: String(row.test_id),
+		success_rate_percent: row.success_rate_percent,
+		success_rate: row.success_rate,
+		results: (Array.isArray(row.results) ? (row.results as RunResult[]) : []).map(
+			summarizeLatestRun
+		),
+		report_timestamp: timestamp,
+		is_old: timestamp < latestTimestamp,
+		...(previous ? { previous } : {})
+	}
+}
+
+function cloneLatestResults(results: Record<string, TestResult>): Record<string, TestResult> {
+	return Object.fromEntries(
+		Object.entries(results).map(([testId, result]) => [
+			testId,
+			{
+				...result,
+				results: result.results.map((run) => ({ ...run })),
+				...(result.previous ? { previous: { ...result.previous } } : {})
+			}
+		])
+	)
+}
+
+async function mergeIncrementalReports(
+	outDir: string,
+	files: ReportFileState[],
+	startIndex: number,
+	baseResults: Record<string, TestResult>,
+	agent: string
+): Promise<Record<string, TestResult>> {
+	const results = cloneLatestResults(baseResults)
+	const latestTimestamp = reportTimestamp(files[files.length - 1].name, agent)
+
+	for (let index = startIndex; index < files.length; index += 1) {
+		const file = files[index]
+		const timestamp = reportTimestamp(file.name, agent)
+		const parsed = await readReportFile<any>(path.join(outDir, file.name))
+
+		for (const row of reportRows(parsed)) {
+			const testId = String(row.test_id ?? '').trim()
+			if (!testId) {
 				throw new Error(
-					`Missing test_id in report ${name}. Please add 'test_id' and provenance columns manually.`
+					`Missing test_id in report ${file.name}. Please add 'test_id' and provenance columns manually.`
 				)
 			}
-			const par = String((r as any).pinned_at_run ?? 'TRUE').toUpperCase()
-			if (par === 'FALSE') continue
-
-			const runResults: RunResult[] = Array.isArray((r as any).results)
-				? ((r as any).results as RunResult[])
-				: []
-			const summary: PreviousTestResultSummary = {
-				success_count: runResults.reduce((acc, rr) => acc + (rr?.success ? 1 : 0), 0),
-				total_runs: runResults.length,
-				report_timestamp: ts
+			if (String(row.pinned_at_run ?? 'TRUE').toUpperCase() === 'FALSE') continue
+
+			const current = results[testId]
+			if (!current || timestamp > current.report_timestamp) {
+				results[testId] = resultFromReportRow(
+					row,
+					timestamp,
+					latestTimestamp,
+					current ? resultSummary(current) : undefined
+				)
+			} else if (timestamp === current.report_timestamp) {
+				results[testId] = resultFromReportRow(row, timestamp, latestTimestamp, current.previous)
 			}
+		}
+	}
 
-			const current = results[test_id]
-			if (!current) {
-				results[test_id] = {
-					test_id,
-					success_rate_percent: (r as any).success_rate_percent,
-					success_rate: (r as any).success_rate,
-					results: runResults,
-					report_timestamp: ts,
-					is_old: ts < latestTs
-				}
-				continue
+	for (const result of Object.values(results)) {
+		result.is_old = result.report_timestamp < latestTimestamp
+	}
+	return results
+}
+
+async function rebuildLatestResults(
+	agent: string,
+	outDir: string,
+	files: ReportFileState[]
+): Promise<Record<string, TestResult>> {
+	const latestTimestamp = reportTimestamp(files[files.length - 1].name, agent)
+	const wantedTestIds = new Set(
+		(await readSuite(agent)).tests.map((test) => String(test.id ?? '').trim()).filter(Boolean)
+	)
+	if (!wantedTestIds.size) return {}
+
+	const unresolvedLatest = new Set(wantedTestIds)
+	const unresolvedPrevious = new Set<string>()
+	const results: Record<string, TestResult> = {}
+
+	for (let index = files.length - 1; index >= 0; index -= 1) {
+		const file = files[index]
+		const timestamp = reportTimestamp(file.name, agent)
+		const parsed = await readReportFile<any>(path.join(outDir, file.name))
+
+		for (const row of reportRows(parsed)) {
+			const testId = String(row.test_id ?? '').trim()
+			if (!testId) {
+				throw new Error(
+					`Missing test_id in report ${file.name}. Please add 'test_id' and provenance columns manually.`
+				)
 			}
-			if (!current.previous && ts < current.report_timestamp) {
-				current.previous = summary
+			if (
+				!wantedTestIds.has(testId) ||
+				String(row.pinned_at_run ?? 'TRUE').toUpperCase() === 'FALSE'
+			)
+				continue
+
+			const current = results[testId]
+			if (!current) {
+				results[testId] = resultFromReportRow(row, timestamp, latestTimestamp)
+				unresolvedLatest.delete(testId)
+				if (timestamp === latestTimestamp) unresolvedPrevious.add(testId)
+			} else if (unresolvedPrevious.has(testId) && timestamp < current.report_timestamp) {
+				current.previous = resultSummary(resultFromReportRow(row, timestamp, latestTimestamp))
+				unresolvedPrevious.delete(testId)
 			}
 		}
+
+		if (!unresolvedLatest.size && !unresolvedPrevious.size) break
+	}
+
+	return results
+}
+
+async function loadLatestResultsByTestUncached(agent: string): Promise<Record<string, TestResult>> {
+	const projectRoot = path.dirname(await getRootPath())
+	const outDir = path.join(projectRoot, 'data', 'test_runs')
+	let entries: string[] = []
+	try {
+		entries = await readdir(outDir)
+	} catch {
+		return {}
+	}
+	const files = listAgentReportFiles(entries, agent)
+	if (!files.length) return {}
+	const states = await reportFileStates(outDir, files)
+	const cached =
+		latestResultsCache.get(agent) ?? (await readPersistedLatestResultsCache(outDir, agent))
+	const incrementalStart = cached ? incrementalReportStart(cached.files, states) : -1
+	if (cached && incrementalStart === null) {
+		latestResultsCache.set(agent, cached)
+		return cached.results
 	}
+
+	let results: Record<string, TestResult>
+	if (cached && incrementalStart !== null && incrementalStart >= 0) {
+		results = await mergeIncrementalReports(outDir, states, incrementalStart, cached.results, agent)
+	} else {
+		results = await rebuildLatestResults(agent, outDir, states)
+	}
+	const entry = { files: states, results }
+	latestResultsCache.set(agent, entry)
+	await persistLatestResultsCache(outDir, agent, entry)
 	return results
 }
 
+export function loadLatestResultsByTest(agent: string): Promise<Record<string, TestResult>> {
+	const pending = latestResultsLoads.get(agent)
+	if (pending) return pending
+
+	const load = loadLatestResultsByTestUncached(agent).finally(() => {
+		if (latestResultsLoads.get(agent) === load) latestResultsLoads.delete(agent)
+	})
+	latestResultsLoads.set(agent, load)
+	return load
+}
+
+function summarizeReport(report: WrittenRunReport, timestamp: string): TestRunHistorySummary {
+	requireTestSnapshots(report)
+	const rows = report.rows
+	const testCount = rows.length
+	const successCount = rows.reduce((acc, row) => {
+		return acc + row.results.reduce((inner, result) => inner + (result.success ? 1 : 0), 0)
+	}, 0)
+	const totalRuns = rows.reduce((acc, row) => {
+		return acc + row.results.length
+	}, 0)
+	const successRatePercent = totalRuns ? Math.round((successCount / totalRuns) * 100) : 0
+
+	return {
+		timestamp,
+		runScope: report.run_scope,
+		testCount,
+		successCount,
+		totalRuns,
+		successRatePercent,
+		successRate: `${successCount}/${totalRuns}`,
+		passed: totalRuns > 0 && successCount === totalRuns,
+		repeats: report.run_config.repeats,
+		concurrency: report.run_config.concurrency
+	}
+}
+
+async function getRunReportDir() {
+	const projectRoot = path.dirname(await getRootPath())
+	return path.join(projectRoot, 'data', 'test_runs')
+}
+
+async function listRunReportFiles(agent: string): Promise<string[]> {
+	const outDir = await getRunReportDir()
+	let entries: string[] = []
+	try {
+		entries = await readdir(outDir)
+	} catch {
+		return []
+	}
+	return listAgentReportFiles(entries, agent)
+}
+
+function requireTestSnapshots(report: WrittenRunReport): void {
+	for (const row of report.rows) {
+		if (!row.test) {
+			throw new Error(`Report ${report.run_timestamp} is missing preserved test snapshots`)
+		}
+	}
+}
+
+export async function listRunHistory(agent: string): Promise<TestRunHistorySummary[]> {
+	const outDir = await getRunReportDir()
+	const files = await listRunReportFiles(agent)
+	const summaries: TestRunHistorySummary[] = []
+
+	for (const name of files.slice().reverse()) {
+		try {
+			const parsed = await readReportFile<WrittenRunReport>(path.join(outDir, name))
+			summaries.push(summarizeReport(parsed, parsed.run_timestamp))
+		} catch (e) {
+			console.error(`Failed to read test report ${name}`, e)
+		}
+	}
+
+	return summaries
+}
+
+function normalizeHistoryRow(row: WrittenRunReport['rows'][number]): TestRunHistoryRow {
+	const rawResults = row.results
+	return {
+		test: row.test,
+		success_rate: row.success_rate,
+		success_rate_percent: row.success_rate_percent,
+		results: rawResults.map((result) => ({
+			success: result.success,
+			result: result.result,
+			evaluation: result.evaluation,
+			errors: result.errors
+		}))
+	}
+}
+
+export async function getRunHistory(
+	agent: string,
+	timestamp: string
+): Promise<TestRunHistoryDetail> {
+	const report = await getRun(agent, timestamp)
+	requireTestSnapshots(report)
+
+	return {
+		timestamp: report.run_timestamp,
+		runScope: report.run_scope,
+		runConfig: report.run_config,
+		rows: report.rows.map(normalizeHistoryRow)
+	}
+}
+
 // Fallback UUID for Node environments without global crypto
 function randomUUID() {
 	const rnd = () =>
@@ -384,22 +758,26 @@ function randomUUID() {
 async function getRun(agent: string, ts?: string) {
 	const projectRoot = path.dirname(await getRootPath())
 	const outDir = path.join(projectRoot, 'data', 'test_runs')
-	const file = `${agent}.${(ts || '').replace(/:/g, '-')}.report.json`
-	const reportPath = ts ? path.join(outDir, file) : null
+	const logicalFile = `${agent}.${(ts || '').replace(/:/g, '-')}.report.json`
 
 	async function readLatestPath(): Promise<string> {
 		const entries = await readdir(outDir)
-		const prefix = `${agent}.`
-		const suffix = `.report.json`
-		const candidates = entries.filter((n) => n.startsWith(prefix) && n.endsWith(suffix)).sort()
+		const candidates = listAgentReportFiles(entries, agent)
 		if (!candidates.length) throw new Error('No reports found')
 		return path.join(outDir, candidates[candidates.length - 1])
 	}
 
-	const chosen = reportPath || (await readLatestPath())
-	const raw = await readFile(chosen, 'utf8')
-	const parsed = JSON.parse(raw) as WrittenRunResult
-	return parsed
+	async function readTimestampPath(): Promise<string> {
+		const entries = await readdir(outDir)
+		const candidate = listAgentReportFiles(entries, agent).find((name) =>
+			name.startsWith(logicalFile)
+		)
+		if (!candidate) throw new Error(`Report not found: ${ts}`)
+		return path.join(outDir, candidate)
+	}
+
+	const chosen = ts ? await readTimestampPath() : await readLatestPath()
+	return readReportFile<WrittenRunResult>(chosen)
 }
 
 type ResultKey = keyof Omit<ResultEntry, 'success'>
@@ -442,5 +820,11 @@ export async function getResultEntry(
 	}
 
 	// TODO get separate RPCs for logs and traces (list)
-	return Array.isArray(entryContent) ? entryContent.join('\n') : entryContent
+	if (Array.isArray(entryContent)) {
+		return entryContent
+			.map((entry) => (typeof entry === 'string' ? entry : JSON.stringify(entry)))
+			.join('\n')
+	}
+	if (typeof entryContent === 'string') return entryContent
+	return entryContent == null ? '' : JSON.stringify(entryContent)
 }