@recombine-ai/bosun 0.16.0 → 0.16.2-cloud-1

package/src/backend/tests/data.ts

@@ -1,731 +0,0 @@
-import { createHash, randomUUID } from 'node:crypto'
-import { readdir, readFile, stat, writeFile } from 'node:fs/promises'
-import path from 'node:path'
-import {
-	readSuite,
-	type TestSuite,
-	withSuiteMutationLock,
-	writeSuite,
-	type TestRow as SuiteRow
-} from './suite'
-import { getRootPath } from '../lib/get-root-path'
-import type {
-	ResultEntry,
-	WrittenRunReport,
-	WrittenRunReportConfig,
-	WrittenRunResult
-} from './academy'
-import { makeContext } from './context-utils'
-import { listAgentReportFiles, readReportFile, reportTimestamp } from './report-files'
-
-export type CsvRow = Record<string, string>
-
-export type TestRow = SuiteRow
-
-export interface RunResult {
-	success: boolean
-	result: string
-	evaluation: string
-	rendered_prompt: string
-	received_prompt: string
-	stringified_conversation: string
-	received_context: string
-	traces: any[]
-	errors: string
-	logs: string[]
-}
-
-export interface PreviousTestResultSummary {
-	success_count: number
-	total_runs: number
-	report_timestamp: string // ISO
-}
-
-export interface TestResult {
-	test_id: string
-	success_rate_percent?: number
-	success_rate?: string
-	results: LatestRunResultSummary[]
-	report_timestamp: string // ISO
-	is_old: boolean
-	previous?: PreviousTestResultSummary
-}
-
-export type LatestRunResultSummary = Pick<RunResult, 'success' | 'result' | 'evaluation' | 'errors'>
-export interface TestTable {
-	rows: TestRow[]
-	headers: string[]
-	settingsKey: string
-}
-
-export interface TestRunHistorySummary {
-	timestamp: string
-	runScope: 'all' | 'pinned'
-	testCount: number
-	successCount: number
-	totalRuns: number
-	successRatePercent: number
-	successRate: string
-	passed: boolean
-	repeats: number
-	concurrency: number
-}
-
-export interface TestRunHistoryResultEntry {
-	success: boolean
-	result: string
-	evaluation: string
-	errors: string
-}
-
-export interface TestRunHistoryRow {
-	test: TestRow
-	success_rate?: string
-	success_rate_percent?: number
-	results: TestRunHistoryResultEntry[]
-}
-
-export interface TestRunHistoryDetail {
-	timestamp: string
-	runScope: 'all' | 'pinned'
-	runConfig: WrittenRunReportConfig
-	rows: TestRunHistoryRow[]
-}
-
-const defaultTestRow: Omit<TestRow, 'id'> = {
-	order: 0,
-	pinned: 'FALSE',
-	scenario: 'New scenario',
-	author: '',
-	category: '',
-	step: 'main',
-	evaluation_mode: 'assertion',
-	context_mode: 'merge',
-	assertion_type: 'llm-rubric',
-	assertion_value: '',
-	conversation: 'user: ...',
-	expected_output: '',
-	timeout: '0',
-	similarity_threshold: '50',
-	context: {}
-}
-
-// Utility: ensure each row has a unique id; persist if any are missing/duplicated.
-export async function loadTests(agent: string): Promise<TestTable> {
-	return withSuiteMutationLock(agent, async () => {
-		const suite = await readSuite(agent)
-		let changed = false
-		const seenIds = new Set<string>()
-		const rows: TestRow[] = suite.tests.map((r) => {
-			let id = String(r.id || '').trim()
-			if (!id || seenIds.has(id)) {
-				changed = true
-				id = randomUUID()
-			}
-			seenIds.add(id)
-			// previously tests were storing context as a string...
-			const context = typeof r.context === 'string' ? JSON.parse(r.context) : r.context
-			const mocks = typeof r.mocks === 'string' ? JSON.parse(r.mocks) : r.mocks
-			return typeof mocks === 'undefined' ? { ...r, context, id } : { ...r, context, mocks, id }
-		})
-		if (changed) {
-			await writeSuite(agent, { ...suite, tests: rows })
-		}
-		const headers = [
-			'id',
-			'pinned',
-			'scenario',
-			'step',
-			'author',
-			'category',
-			'assertion_type',
-			'assertion_value',
-			'conversation',
-			'expected_output',
-			'evaluation_mode',
-			'context_mode',
-			'timeout',
-			'similarity_threshold',
-			'context',
-			'mocks'
-		]
-		const projectPath = path.dirname(await getRootPath())
-		const workspaceId = createHash('sha256').update(projectPath).digest('hex').slice(0, 16)
-		return { rows, headers, settingsKey: `bosun:test-settings:${workspaceId}:${agent}` }
-	})
-}
-
-// Utility: update a single row by id and write the CSV
-export async function patchTest(
-	agent: string,
-	id: string,
-	patch: Partial<TestRow>
-): Promise<TestRow | null> {
-	return withSuiteMutationLock(agent, async () => {
-		const suite = await readSuite(agent)
-		const idx = suite.tests.findIndex((r) => r.id === id)
-		if (idx < 0) return null
-		const current = suite.tests[idx]
-		const updated = preparePatch(current, patch)
-		const nextRows = suite.tests.map((r, i) => (i === idx ? updated : r))
-		await writeSuite(agent, { ...suite, tests: nextRows })
-		return updated
-	})
-}
-
-function preparePatch(current: TestRow, patch: Partial<TestRow>) {
-	const updated: TestRow = {
-		...defaultTestRow,
-		...current,
-		...patch,
-		context: makeContext(patch.context || current.context)
-	}
-	if (typeof patch.mocks !== 'undefined' || typeof current.mocks !== 'undefined') {
-		updated.mocks = makeContext(patch.mocks ?? current.mocks)
-	}
-	return updated
-}
-
-// Batch update multiple rows in a single read/write to avoid race conditions
-export async function patchTestsBatch(
-	agent: string,
-	updates: Array<{ id: string; patch: Partial<TestRow> }>
-): Promise<TestRow[]> {
-	if (!Array.isArray(updates) || updates.length === 0) return []
-	return withSuiteMutationLock(agent, async () => {
-		const suite = await readSuite(agent)
-		const byId = new Map(suite.tests.map((r) => [r.id, r]))
-		const updatedRows: TestRow[] = []
-		for (const { id, patch } of updates) {
-			const current = byId.get(id)
-			if (!current) continue
-			const updated = preparePatch(current, patch)
-			byId.set(id, updated)
-			updatedRows.push(updated)
-		}
-		const nextRows: TestRow[] = suite.tests.map((r) => byId.get(r.id) ?? r)
-		await writeSuite(agent, { ...suite, tests: nextRows })
-		return updatedRows
-	})
-}
-
-// Add a new row at the top
-export async function addTest(agent: string, seed?: Partial<TestRow>): Promise<TestRow> {
-	return withSuiteMutationLock(agent, async () => {
-		const suite = await loadSuiteOrEmpty(agent)
-		const id = randomUUID()
-		const firstOrder = suite.tests.reduce(
-			(minimum, row) => Math.min(minimum, row.order),
-			Number.POSITIVE_INFINITY
-		)
-
-		const newRow: TestRow = {
-			...defaultTestRow,
-			id,
-			order: Number.isFinite(firstOrder) ? firstOrder - 1024 : 0,
-			...seed,
-			context: makeContext(seed?.context)
-		}
-		if (typeof seed?.mocks !== 'undefined') {
-			newRow.mocks = makeContext(seed.mocks)
-		}
-		await writeSuite(agent, { tests: [newRow, ...suite.tests] })
-		return newRow
-	})
-}
-
-async function loadSuiteOrEmpty(agent: string): Promise<TestSuite> {
-	return readSuite(agent)
-}
-
-export async function deleteTest(agent: string, id: string): Promise<boolean> {
-	return withSuiteMutationLock(agent, async () => {
-		const suite = await readSuite(agent)
-		const filtered = suite.tests.filter((r) => r.id !== id)
-		if (filtered.length === suite.tests.length) return false
-		await writeSuite(agent, { ...suite, tests: filtered })
-		return true
-	})
-}
-
-export async function reorderTests(agent: string, order: string[]): Promise<void> {
-	await withSuiteMutationLock(agent, async () => {
-		const suite = await readSuite(agent)
-		const map = new Map(suite.tests.map((r) => [r.id, r]))
-		const next: TestRow[] = []
-		for (const id of order) {
-			const r = map.get(id)
-			if (r) {
-				next.push({ ...r, order: next.length * 1024 })
-				map.delete(id)
-			}
-		}
-		// Append any rows that were not included in order (safety)
-		for (const r of map.values()) next.push({ ...r, order: next.length * 1024 })
-		await writeSuite(agent, { ...suite, tests: next })
-	})
-}
-
-// We no longer migrate legacy reports automatically. Please update old reports manually.
-
-type ReportFileState = {
-	name: string
-	size: number
-	mtimeMs: number
-}
-
-type LatestResultsCacheEntry = {
-	files: ReportFileState[]
-	results: Record<string, TestResult>
-}
-
-type PersistedLatestResultsCache = LatestResultsCacheEntry & {
-	version: 1
-}
-
-const latestResultsCache = new Map<string, LatestResultsCacheEntry>()
-const latestResultsLoads = new Map<string, Promise<Record<string, TestResult>>>()
-
-function latestResultsCachePath(outDir: string, agent: string): string {
-	return path.join(outDir, `.bosun-latest-results-${encodeURIComponent(agent)}.json`)
-}
-
-async function reportFileStates(outDir: string, files: string[]): Promise<ReportFileState[]> {
-	return Promise.all(
-		files.map(async (name) => {
-			const info = await stat(path.join(outDir, name))
-			return { name, size: info.size, mtimeMs: info.mtimeMs }
-		})
-	)
-}
-
-function sameReportFile(a: ReportFileState, b: ReportFileState): boolean {
-	return a.name === b.name && a.size === b.size && a.mtimeMs === b.mtimeMs
-}
-
-function incrementalReportStart(
-	previous: ReportFileState[],
-	current: ReportFileState[]
-): number | null {
-	if (current.length < previous.length) return -1
-
-	const sharedLength = Math.min(previous.length, current.length)
-	for (let index = 0; index < sharedLength; index += 1) {
-		if (sameReportFile(previous[index], current[index])) continue
-		return index >= Math.max(0, previous.length - 1) ? index : -1
-	}
-
-	return current.length === previous.length ? null : previous.length
-}
-
-function isPersistedLatestResultsCache(value: unknown): value is PersistedLatestResultsCache {
-	if (!value || typeof value !== 'object') return false
-	const candidate = value as Partial<PersistedLatestResultsCache>
-	return (
-		candidate.version === 1 &&
-		Array.isArray(candidate.files) &&
-		candidate.files.every(
-			(file) =>
-				file &&
-				typeof file.name === 'string' &&
-				typeof file.size === 'number' &&
-				Number.isFinite(file.size) &&
-				typeof file.mtimeMs === 'number' &&
-				Number.isFinite(file.mtimeMs)
-		) &&
-		!!candidate.results &&
-		typeof candidate.results === 'object' &&
-		!Array.isArray(candidate.results)
-	)
-}
-
-async function readPersistedLatestResultsCache(
-	outDir: string,
-	agent: string
-): Promise<LatestResultsCacheEntry | null> {
-	try {
-		const raw = await readFile(latestResultsCachePath(outDir, agent), 'utf8')
-		const parsed: unknown = JSON.parse(raw)
-		if (!isPersistedLatestResultsCache(parsed)) return null
-		return { files: parsed.files, results: parsed.results }
-	} catch {
-		return null
-	}
-}
-
-async function persistLatestResultsCache(
-	outDir: string,
-	agent: string,
-	entry: LatestResultsCacheEntry
-): Promise<void> {
-	try {
-		await writeFile(
-			latestResultsCachePath(outDir, agent),
-			JSON.stringify({ version: 1, ...entry } satisfies PersistedLatestResultsCache),
-			'utf8'
-		)
-	} catch {
-		// The in-memory cache still avoids repeat work for the current Bosun process.
-	}
-}
-
-function summarizeLatestRun(run: RunResult): LatestRunResultSummary {
-	return {
-		success: !!run?.success,
-		result: String(run?.result ?? ''),
-		evaluation: String(run?.evaluation ?? ''),
-		errors: String(run?.errors ?? '')
-	}
-}
-
-function reportRows(parsed: any): any[] {
-	return Array.isArray(parsed) ? parsed : Array.isArray(parsed?.rows) ? parsed.rows : []
-}
-
-function resultSummary(result: TestResult): PreviousTestResultSummary {
-	return {
-		success_count: result.results.reduce((count, run) => count + (run.success ? 1 : 0), 0),
-		total_runs: result.results.length,
-		report_timestamp: result.report_timestamp
-	}
-}
-
-function resultFromReportRow(
-	row: any,
-	timestamp: string,
-	latestTimestamp: string,
-	previous?: PreviousTestResultSummary
-): TestResult {
-	return {
-		test_id: String(row.test_id),
-		success_rate_percent: row.success_rate_percent,
-		success_rate: row.success_rate,
-		results: (Array.isArray(row.results) ? (row.results as RunResult[]) : []).map(
-			summarizeLatestRun
-		),
-		report_timestamp: timestamp,
-		is_old: timestamp < latestTimestamp,
-		...(previous ? { previous } : {})
-	}
-}
-
-function cloneLatestResults(results: Record<string, TestResult>): Record<string, TestResult> {
-	return Object.fromEntries(
-		Object.entries(results).map(([testId, result]) => [
-			testId,
-			{
-				...result,
-				results: result.results.map((run) => ({ ...run })),
-				...(result.previous ? { previous: { ...result.previous } } : {})
-			}
-		])
-	)
-}
-
-async function mergeIncrementalReports(
-	outDir: string,
-	files: ReportFileState[],
-	startIndex: number,
-	baseResults: Record<string, TestResult>,
-	agent: string
-): Promise<Record<string, TestResult>> {
-	const results = cloneLatestResults(baseResults)
-	const latestTimestamp = reportTimestamp(files[files.length - 1].name, agent)
-
-	for (let index = startIndex; index < files.length; index += 1) {
-		const file = files[index]
-		const timestamp = reportTimestamp(file.name, agent)
-		const parsed = await readReportFile<any>(path.join(outDir, file.name))
-
-		for (const row of reportRows(parsed)) {
-			const testId = String(row.test_id ?? '').trim()
-			if (!testId) {
-				throw new Error(
-					`Missing test_id in report ${file.name}. Please add 'test_id' and provenance columns manually.`
-				)
-			}
-			if (String(row.pinned_at_run ?? 'TRUE').toUpperCase() === 'FALSE') continue
-
-			const current = results[testId]
-			if (!current || timestamp > current.report_timestamp) {
-				results[testId] = resultFromReportRow(
-					row,
-					timestamp,
-					latestTimestamp,
-					current ? resultSummary(current) : undefined
-				)
-			} else if (timestamp === current.report_timestamp) {
-				results[testId] = resultFromReportRow(row, timestamp, latestTimestamp, current.previous)
-			}
-		}
-	}
-
-	for (const result of Object.values(results)) {
-		result.is_old = result.report_timestamp < latestTimestamp
-	}
-	return results
-}
-
-async function rebuildLatestResults(
-	agent: string,
-	outDir: string,
-	files: ReportFileState[]
-): Promise<Record<string, TestResult>> {
-	const latestTimestamp = reportTimestamp(files[files.length - 1].name, agent)
-	const wantedTestIds = new Set(
-		(await readSuite(agent)).tests.map((test) => String(test.id ?? '').trim()).filter(Boolean)
-	)
-	if (!wantedTestIds.size) return {}
-
-	const unresolvedLatest = new Set(wantedTestIds)
-	const unresolvedPrevious = new Set<string>()
-	const results: Record<string, TestResult> = {}
-
-	for (let index = files.length - 1; index >= 0; index -= 1) {
-		const file = files[index]
-		const timestamp = reportTimestamp(file.name, agent)
-		const parsed = await readReportFile<any>(path.join(outDir, file.name))
-
-		for (const row of reportRows(parsed)) {
-			const testId = String(row.test_id ?? '').trim()
-			if (!testId) {
-				throw new Error(
-					`Missing test_id in report ${file.name}. Please add 'test_id' and provenance columns manually.`
-				)
-			}
-			if (
-				!wantedTestIds.has(testId) ||
-				String(row.pinned_at_run ?? 'TRUE').toUpperCase() === 'FALSE'
-			)
-				continue
-
-			const current = results[testId]
-			if (!current) {
-				results[testId] = resultFromReportRow(row, timestamp, latestTimestamp)
-				unresolvedLatest.delete(testId)
-				if (timestamp === latestTimestamp) unresolvedPrevious.add(testId)
-			} else if (unresolvedPrevious.has(testId) && timestamp < current.report_timestamp) {
-				current.previous = resultSummary(resultFromReportRow(row, timestamp, latestTimestamp))
-				unresolvedPrevious.delete(testId)
-			}
-		}
-
-		if (!unresolvedLatest.size && !unresolvedPrevious.size) break
-	}
-
-	return results
-}
-
-async function loadLatestResultsByTestUncached(agent: string): Promise<Record<string, TestResult>> {
-	const projectRoot = path.dirname(await getRootPath())
-	const outDir = path.join(projectRoot, 'data', 'test_runs')
-	let entries: string[] = []
-	try {
-		entries = await readdir(outDir)
-	} catch {
-		return {}
-	}
-	const files = listAgentReportFiles(entries, agent)
-	if (!files.length) return {}
-	const states = await reportFileStates(outDir, files)
-	const cached =
-		latestResultsCache.get(agent) ?? (await readPersistedLatestResultsCache(outDir, agent))
-	const incrementalStart = cached ? incrementalReportStart(cached.files, states) : -1
-	if (cached && incrementalStart === null) {
-		latestResultsCache.set(agent, cached)
-		return cached.results
-	}
-
-	let results: Record<string, TestResult>
-	if (cached && incrementalStart !== null && incrementalStart >= 0) {
-		results = await mergeIncrementalReports(outDir, states, incrementalStart, cached.results, agent)
-	} else {
-		results = await rebuildLatestResults(agent, outDir, states)
-	}
-	const entry = { files: states, results }
-	latestResultsCache.set(agent, entry)
-	await persistLatestResultsCache(outDir, agent, entry)
-	return results
-}
-
-export function loadLatestResultsByTest(agent: string): Promise<Record<string, TestResult>> {
-	const pending = latestResultsLoads.get(agent)
-	if (pending) return pending
-
-	const load = loadLatestResultsByTestUncached(agent).finally(() => {
-		if (latestResultsLoads.get(agent) === load) latestResultsLoads.delete(agent)
-	})
-	latestResultsLoads.set(agent, load)
-	return load
-}
-
-function summarizeReport(report: WrittenRunReport, timestamp: string): TestRunHistorySummary {
-	requireTestSnapshots(report)
-	const rows = report.rows
-	const testCount = rows.length
-	const successCount = rows.reduce((acc, row) => {
-		return acc + row.results.reduce((inner, result) => inner + (result.success ? 1 : 0), 0)
-	}, 0)
-	const totalRuns = rows.reduce((acc, row) => {
-		return acc + row.results.length
-	}, 0)
-	const successRatePercent = totalRuns ? Math.round((successCount / totalRuns) * 100) : 0
-
-	return {
-		timestamp,
-		runScope: report.run_scope,
-		testCount,
-		successCount,
-		totalRuns,
-		successRatePercent,
-		successRate: `${successCount}/${totalRuns}`,
-		passed: totalRuns > 0 && successCount === totalRuns,
-		repeats: report.run_config.repeats,
-		concurrency: report.run_config.concurrency
-	}
-}
-
-async function getRunReportDir() {
-	const projectRoot = path.dirname(await getRootPath())
-	return path.join(projectRoot, 'data', 'test_runs')
-}
-
-async function listRunReportFiles(agent: string): Promise<string[]> {
-	const outDir = await getRunReportDir()
-	let entries: string[] = []
-	try {
-		entries = await readdir(outDir)
-	} catch {
-		return []
-	}
-	return listAgentReportFiles(entries, agent)
-}
-
-function requireTestSnapshots(report: WrittenRunReport): void {
-	for (const row of report.rows) {
-		if (!row.test) {
-			throw new Error(`Report ${report.run_timestamp} is missing preserved test snapshots`)
-		}
-	}
-}
-
-export async function listRunHistory(agent: string): Promise<TestRunHistorySummary[]> {
-	const outDir = await getRunReportDir()
-	const files = await listRunReportFiles(agent)
-	const summaries: TestRunHistorySummary[] = []
-
-	for (const name of files.slice().reverse()) {
-		try {
-			const parsed = await readReportFile<WrittenRunReport>(path.join(outDir, name))
-			summaries.push(summarizeReport(parsed, parsed.run_timestamp))
-		} catch (e) {
-			console.error(`Failed to read test report ${name}`, e)
-		}
-	}
-
-	return summaries
-}
-
-function normalizeHistoryRow(row: WrittenRunReport['rows'][number]): TestRunHistoryRow {
-	const rawResults = row.results
-	return {
-		test: row.test,
-		success_rate: row.success_rate,
-		success_rate_percent: row.success_rate_percent,
-		results: rawResults.map((result) => ({
-			success: result.success,
-			result: result.result,
-			evaluation: result.evaluation,
-			errors: result.errors
-		}))
-	}
-}
-
-export async function getRunHistory(
-	agent: string,
-	timestamp: string
-): Promise<TestRunHistoryDetail> {
-	const report = await getRun(agent, timestamp)
-	requireTestSnapshots(report)
-
-	return {
-		timestamp: report.run_timestamp,
-		runScope: report.run_scope,
-		runConfig: report.run_config,
-		rows: report.rows.map(normalizeHistoryRow)
-	}
-}
-
-async function getRun(agent: string, ts?: string) {
-	const projectRoot = path.dirname(await getRootPath())
-	const outDir = path.join(projectRoot, 'data', 'test_runs')
-	const logicalFile = `${agent}.${(ts || '').replace(/:/g, '-')}.report.json`
-
-	async function readLatestPath(): Promise<string> {
-		const entries = await readdir(outDir)
-		const candidates = listAgentReportFiles(entries, agent)
-		if (!candidates.length) throw new Error('No reports found')
-		return path.join(outDir, candidates[candidates.length - 1])
-	}
-
-	async function readTimestampPath(): Promise<string> {
-		const entries = await readdir(outDir)
-		const candidate = listAgentReportFiles(entries, agent).find((name) =>
-			name.startsWith(logicalFile)
-		)
-		if (!candidate) throw new Error(`Report not found: ${ts}`)
-		return path.join(outDir, candidate)
-	}
-
-	const chosen = ts ? await readTimestampPath() : await readLatestPath()
-	return readReportFile<WrittenRunResult>(chosen)
-}
-
-type ResultKey = keyof Omit<ResultEntry, 'success'>
-
-export async function getRunResultEntry(
-	agent: string,
-	testId: string,
-	index = 0,
-	ts?: string
-): Promise<ResultEntry> {
-	const result = await getRun(agent, ts)
-
-	const row = result.rows.find((r) => r.test_id === testId)
-	if (!row) throw new Error('Row not found in report')
-
-	const run = row.results[index]
-	if (!run) throw new Error(`Run not found for test ${testId}, index: ${index}`)
-
-	return run
-}
-
-export async function getResultEntry(
-	agent: string,
-	testId: string,
-	resultKey: ResultKey,
-	index = 0,
-	ts?: string
-): Promise<string> {
-	const result = await getRun(agent, ts)
-
-	const row = result.rows.find((r) => r.test_id === testId)
-	if (!row) throw new Error('Row not found in report')
-
-	const run = row.results[index]
-	if (!run) throw new Error(`Run not found for test ${testId}, index: ${index}`)
-
-	let entryContent = run[resultKey]
-	if (resultKey === 'traces') {
-		entryContent = JSON.stringify(entryContent)
-	}
-
-	// TODO get separate RPCs for logs and traces (list)
-	if (Array.isArray(entryContent)) {
-		return entryContent
-			.map((entry) => (typeof entry === 'string' ? entry : JSON.stringify(entry)))
-			.join('\n')
-	}
-	if (typeof entryContent === 'string') return entryContent
-	return entryContent == null ? '' : JSON.stringify(entryContent)
-}