@recombine-ai/bosun 0.11.1 → 0.11.2

package/src/backend/tests/academy.ts

@@ -1,9 +1,8 @@
 // cspell:words conv
-import { type ProviderOptions } from 'promptfoo'
-import * as promptfoo from 'promptfoo'
 import { type Message, type StepTrace, type StepTracer } from '@recombine-ai/engine'
 import { loadAgents } from './emulate-agents'
-import { type Assertion as PFAssertion } from 'promptfoo'
+import { installTestPromptCacheBust } from './openai-prompt-cache'
+import { promptfoo } from './promptfoo-runtime'
 // Backend: per-project JSON test suites (no CSV)
 import { tryReadConfig } from './config'
 import { mkdir, rename, unlink, writeFile } from 'node:fs/promises'
@@ -11,11 +10,18 @@ import path from 'node:path'
 import { readSuite, type TestRow as SuiteTestRow } from './suite'
 // Use types derived from promptfoo exports instead of incorrect default type imports
 
-import type { ApiProvider, CallApiContextParams, ProviderResponse } from 'promptfoo'
+import type {
+	ApiProvider,
+	Assertion as PFAssertion,
+	CallApiContextParams,
+	ProviderOptions,
+	ProviderResponse
+} from 'promptfoo'
 import deepmerge from 'deepmerge'
 import fs from 'node:fs'
 import { getRootPath } from '../lib/get-root-path'
 import { makeContext } from './context-utils'
+import { queueReportCompression, REPORT_JSON_SUFFIX } from './report-files'
 
 // ----------------- Type aliases & helpers -----------------
 type Context = Record<string, unknown>
@@ -93,6 +99,30 @@ type CaptureMode = 'immediate' | 'final'
 
 const activeRunsBySuiteKey = new Map<string, AbortController>()
 
+export interface PromptTestsProgress {
+	plannedTestIds: string[]
+	runningTestIds: string[]
+	completedTestIds: string[]
+}
+
+type MutablePromptTestsProgress = PromptTestsProgress & {
+	expectedRuns: number
+	startedTestIds: Set<string>
+	completedRunsByTestId: Map<string, number>
+}
+
+const activeProgressBySuiteKey = new Map<string, MutablePromptTestsProgress>()
+
+export function getPromptTestsProgress(suiteKey: string): PromptTestsProgress | null {
+	const progress = activeProgressBySuiteKey.get(suiteKey)
+	if (!progress) return null
+	return {
+		plannedTestIds: [...progress.plannedTestIds],
+		runningTestIds: [...progress.runningTestIds],
+		completedTestIds: [...progress.completedTestIds]
+	}
+}
+
 export type SuiteSelection = {
 	// `suiteKey` is the filesystem/report key (e.g. `voiceSupport.en-gb`).
 	// `agentId` is the canonical factory key (workflow export key, e.g. `voiceSupport`).
@@ -122,6 +152,7 @@ export async function runPromptTestsManaged(
 		return await runPromptTests(suite, selection, { signal: controller.signal })
 	} finally {
 		activeRunsBySuiteKey.delete(suite.suiteKey)
+		activeProgressBySuiteKey.delete(suite.suiteKey)
 	}
 }
 
@@ -234,6 +265,7 @@ export class RecombineAgentProvider implements ApiProvider {
 	protected providerId: string
 	public config: Record<string, unknown>
 	private abortSignal?: AbortSignal
+	private onTestStarted?: (testId: string) => void
 
 	constructor(options: ProviderOptions) {
 		this.providerId = options.id || 'recombine-agent'
@@ -244,6 +276,10 @@ export class RecombineAgentProvider implements ApiProvider {
 		this.abortSignal = signal
 	}
 
+	setOnTestStarted(callback: (testId: string) => void) {
+		this.onTestStarted = callback
+	}
+
 	id() {
 		return this.providerId
 	}
@@ -252,6 +288,7 @@ export class RecombineAgentProvider implements ApiProvider {
 		const projectRoot = path.dirname(await getRootPath())
 		const vars = (context?.vars || {}) as unknown as {
 			agentId: string
+			testId: string
 			region?: string
 			conversation: Conversation
 			conversationParseError?: string
@@ -259,6 +296,8 @@ export class RecombineAgentProvider implements ApiProvider {
 			mocks?: Context
 			captureMode?: CaptureMode | string
 		}
+		const testId = String(vars.testId ?? '')
+		if (testId) this.onTestStarted?.(testId)
 
 		if (typeof vars.conversationParseError === 'string' && vars.conversationParseError.trim()) {
 			return { output: '', error: vars.conversationParseError }
@@ -293,7 +332,6 @@ export class RecombineAgentProvider implements ApiProvider {
 		const stepTracer: StepTracer = {
 			addStepTrace: (stepTrace: StepTrace) => {
 				try {
-					console.log(`trace: ${stepTrace.name}`)
 					capturedStepTraces.push(stepTrace)
 				} catch {
 					// best-effort capture; ignore serialization issues
@@ -438,6 +476,7 @@ export class RecombineAgentProvider implements ApiProvider {
 				}
 			}
 		} catch (err) {
+			if (this.abortSignal?.aborted) throw err
 			const msg = err instanceof Error ? `${err.name}: ${err.message}` : String(err)
 			capturedLogs.push(`[ERROR] ${msg}`)
 			try {
@@ -574,7 +613,7 @@ export async function runPromptTests(
 	const cfg = (await tryReadConfig(suiteKey)) || {}
 
 	const repeats = Number(cfg?.repeats ?? 1)
-	const concurrency = Number(cfg?.concurrency ?? 1)
+	const concurrency = Math.min(75, Math.max(1, Math.floor(Number(cfg?.concurrency ?? 75))))
 	const baseContext = loadedSuite?.meta?.defaultContext || {}
 
 	// Add an index field to each row for mapping
@@ -586,12 +625,27 @@ export async function runPromptTests(
 	}
 
 	const tests = toTestInputs(indexedRows, baseContext, selection)
+	const plannedTestIds = tests.map((test) => String(test.id ?? '')).filter(Boolean)
+	const progress: MutablePromptTestsProgress = {
+		plannedTestIds,
+		runningTestIds: [],
+		completedTestIds: [],
+		expectedRuns: repeats,
+		startedTestIds: new Set(),
+		completedRunsByTestId: new Map()
+	}
+	activeProgressBySuiteKey.set(suiteKey, progress)
 
 	// Provider function with stable label
 	const providerInstance = new RecombineAgentProvider({
 		id: 'recombine-agent'
 	})
 	providerInstance.setAbortSignal(signal)
+	providerInstance.setOnTestStarted((testId) => {
+		if (progress.completedTestIds.includes(testId)) return
+		progress.startedTestIds.add(testId)
+		progress.runningTestIds = [...progress.startedTestIds]
+	})
 	const providerFn = Object.assign(
 		(_promptStr: string, ctx?: CallApiContextParams) => {
 			if (signal?.aborted) {
@@ -605,39 +659,40 @@ export async function runPromptTests(
 	const hookKey = `__bosun_promptfoo_hook_${process.pid}_${Date.now()}_${Math.random().toString(16).slice(2)}`
 	const extensionCode = `globalThis[${JSON.stringify(hookKey)}]?.(output, context)`
 
-	const testSuites = tests.map((test) => ({
+	const testSuite = {
 		prompts: ['agent-run'], // placeholder; provider ignores
-		scenario: test.scenario,
-		tests: [
-			{
-				metadata: { testId: test.id },
-				assert: test.assertions,
-				model: 'not used!',
-				vars: {
-					agentId: suite.agentId,
-					...(suite.region ? { region: suite.region } : {}),
-					context: test.context,
-					...(typeof test.mocks !== 'undefined' ? { mocks: test.mocks } : {}),
-					captureMode: test.captureMode,
-					conversation: test.conversation,
-					...(test.conversationParseError
-						? { conversationParseError: test.conversationParseError }
-						: {})
-				}
+		tests: tests.map((test) => ({
+			description: test.scenario,
+			metadata: { testId: test.id },
+			assert: test.assertions,
+			model: 'not used!',
+			vars: {
+				agentId: suite.agentId,
+				testId: String(test.id ?? ''),
+				...(suite.region ? { region: suite.region } : {}),
+				context: test.context,
+				...(typeof test.mocks !== 'undefined' ? { mocks: test.mocks } : {}),
+				captureMode: test.captureMode,
+				conversation: test.conversation,
+				...(test.conversationParseError
+					? { conversationParseError: test.conversationParseError }
+					: {})
 			}
-		],
+		})),
 		providers: [providerFn],
 		extensions: [extensionCode]
-	}))
+	}
 
-	const projectRoot = path.dirname(await getRootPath())
+	const rootPath = await getRootPath()
+	installTestPromptCacheBust(rootPath)
+	const projectRoot = path.dirname(rootPath)
 	const outDir = path.join(projectRoot, 'data', 'test_runs')
 	await mkdir(outDir, { recursive: true })
 
 	// Use full ISO timestamp and sanitize ':' for filesystem safety
 	const iso = new Date().toISOString()
 	const isoFile = iso.replace(/:/g, '-')
-	const outPath = path.join(outDir, `${suiteKey}.${isoFile}.report.json`)
+	const outPath = path.join(outDir, `${suiteKey}.${isoFile}${REPORT_JSON_SUFFIX}`)
 
 	async function writeReport(report: WrittenRunReport) {
 		const dir = path.dirname(outPath)
@@ -746,6 +801,15 @@ export async function runPromptTests(
 
 			const entry = toResultEntry(r)
 			reportRow.results.push(entry)
+			const completedRuns = (progress.completedRunsByTestId.get(testId) ?? 0) + 1
+			progress.completedRunsByTestId.set(testId, completedRuns)
+			if (completedRuns >= progress.expectedRuns) {
+				progress.startedTestIds.delete(testId)
+				progress.runningTestIds = [...progress.startedTestIds]
+				if (!progress.completedTestIds.includes(testId)) {
+					progress.completedTestIds = [...progress.completedTestIds, testId]
+				}
+			}
 
 			const counts = countsByTestId.get(testId) ?? { pass: 0, total: 0 }
 			counts.total += 1
@@ -764,77 +828,68 @@ export async function runPromptTests(
 		}
 	}
 
-	try {
-		for (const suite of testSuites) {
-			if (signal?.aborted) break
-			const testId = String((suite.tests?.[0] as any)?.metadata?.testId ?? '')
-			const originalRow = testId ? typedRows.find((r) => String(r.id) === testId) : undefined
-			try {
-				console.log(`Running: ${suite.scenario}`)
-				const ev = await promptfoo.evaluate(suite, {
-					repeat: repeats,
-					cache: false,
-					maxConcurrency: concurrency,
-					abortSignal: signal
-				})
+	const selectedOriginalRows = plannedTestIds
+		.map((testId) => originalRowById.get(testId))
+		.filter((row): row is SuiteTestRow => !!row)
+	const persistEvaluation = (evaluation: PFEval) => {
+		for (const rowToWrite of buildTableRows([evaluation], selectedOriginalRows, suiteKey)) {
+			const testId = String(rowToWrite.id ?? '')
+			const originalRow = originalRowById.get(testId)
+			if (!testId || !originalRow) continue
+			const pinned: 'TRUE' | 'FALSE' =
+				(rowToWrite.pinned || '').toUpperCase() === 'TRUE' ? 'TRUE' : 'FALSE'
+			byTestId.set(testId, {
+				...rowToWrite,
+				test: originalRow,
+				agent: suiteKey,
+				test_id: testId,
+				run_timestamp: iso,
+				run_config: JSON.stringify(runConfig),
+				pinned_at_run: scope === 'all' || !!onlyTestId ? 'TRUE' : pinned,
+				run_scope: scope
+			})
+		}
+		body.rows = Array.from(byTestId.values())
+	}
 
-				if (originalRow && testId) {
-					const rowToWrite = buildTableRows([ev], [originalRow], suiteKey)[0]
-					const pinned: 'TRUE' | 'FALSE' =
-						(rowToWrite.pinned || '').toUpperCase() === 'TRUE' ? 'TRUE' : 'FALSE'
-					byTestId.set(testId, {
-						...rowToWrite,
-						test: originalRow,
-						agent: suiteKey,
-						test_id: testId,
-						run_timestamp: iso,
-						run_config: JSON.stringify(runConfig),
-						pinned_at_run: scope === 'all' || !!onlyTestId ? 'TRUE' : pinned,
-						run_scope: scope
-					})
-					body.rows = Array.from(byTestId.values())
-					await enqueueWrite()
-				}
-			} catch (err) {
-				const cancelled =
-					!!signal?.aborted ||
-					(err instanceof Error && /operation cancelled/i.test(err.message || ''))
-				if (cancelled) {
-					break
-				}
-				const msg = err instanceof Error ? `${err.name}: ${err.message}` : String(err)
-				const ev = {
-					results: [
-						{
+	const previousIsTest = process.env.IS_TEST
+	process.env.IS_TEST = 'true'
+	try {
+		if (tests.length > 0 && !signal?.aborted) {
+			const evaluation = await promptfoo.evaluate(testSuite, {
+				repeat: repeats,
+				cache: false,
+				maxConcurrency: concurrency,
+				// Bosun reports run progress in the tests UI. Promptfoo's terminal progress bars
+				// continuously rewrite stdout (up to 20 bars at higher concurrency).
+				showProgressBar: false,
+				abortSignal: signal
+			})
+			persistEvaluation(evaluation)
+			await enqueueWrite()
+		}
+	} catch (err) {
+		const cancelled =
+			!!signal?.aborted || (err instanceof Error && /operation cancelled/i.test(err.message || ''))
+		if (!cancelled) {
+			const msg = err instanceof Error ? `${err.name}: ${err.message}` : String(err)
+			const evaluation = {
+				results: plannedTestIds.map(
+					(testId) =>
+						({
 							success: false,
 							response: { output: '' },
 							error: msg,
-							metadata: { testId: (suite.tests?.[0] as any)?.metadata?.testId }
-						} as any
-					]
-				} as any
-
-				// Persist failure row immediately for real-time UI updates.
-				if (originalRow && testId) {
-					const rowToWrite = buildTableRows([ev], [originalRow], suiteKey)[0]
-					const pinned: 'TRUE' | 'FALSE' =
-						(rowToWrite.pinned || '').toUpperCase() === 'TRUE' ? 'TRUE' : 'FALSE'
-					byTestId.set(testId, {
-						...rowToWrite,
-						test: originalRow,
-						agent: suiteKey,
-						test_id: testId,
-						run_timestamp: iso,
-						run_config: JSON.stringify(runConfig),
-						pinned_at_run: scope === 'all' || !!onlyTestId ? 'TRUE' : pinned,
-						run_scope: scope
-					})
-					body.rows = Array.from(byTestId.values())
-					await enqueueWrite()
-				}
-			}
+							metadata: { testId }
+						}) as any
+				)
+			} as PFEval
+			persistEvaluation(evaluation)
+			await enqueueWrite()
 		}
 	} finally {
+		if (previousIsTest == null) delete process.env.IS_TEST
+		else process.env.IS_TEST = previousIsTest
 		if (writeTimer) clearTimeout(writeTimer)
 		writeTimer = null
 		delete (globalThis as any)[hookKey]
@@ -844,6 +899,9 @@ export async function runPromptTests(
 	// Ensure the final report is written even if no tests ran (or an update was skipped).
 	body.rows = Array.from(byTestId.values())
 	await enqueueWrite()
+	void queueReportCompression(outPath).catch((error) => {
+		console.error(`Failed to compress completed test report ${path.basename(outPath)}`, error)
+	})
 
 	return body
 }

package/src/backend/tests/analyze.ts

@@ -107,9 +107,6 @@ export async function analyzeTraceQuotes(input: {
 	}
 
 	const conversation = decodeHtmlEntities(String(input.stringifiedConversation ?? ''))
-	if (!conversation) {
-		throw new Error('Missing conversation')
-	}
 
 	const resolvedModel = resolveModel(input.model)
 	if (!resolvedModel) {

package/src/backend/tests/data.spec.ts

@@ -0,0 +1,139 @@
+import { mkdtemp, mkdir, readdir, readFile, rm, stat, utimes, writeFile } from 'node:fs/promises'
+import os from 'node:os'
+import path from 'node:path'
+import { afterEach, describe, expect, it, vi } from 'vitest'
+
+const root = vi.hoisted(() => ({ path: '' }))
+
+vi.mock('../lib/get-root-path', () => ({
+	getRootPath: async () => root.path
+}))
+
+import { getRunResultEntry, loadLatestResultsByTest } from './data'
+import { compressReportFile } from './report-files'
+
+const temporaryDirectories: string[] = []
+
+afterEach(async () => {
+	await Promise.all(temporaryDirectories.splice(0).map((dir) => rm(dir, { recursive: true })))
+})
+
+function testRow(id: string) {
+	return {
+		id,
+		pinned: 'TRUE',
+		scenario: id,
+		assertion_type: 'equals',
+		assertion_value: '',
+		conversation: 'user: hello',
+		expected_output: 'hello',
+		timeout: '0',
+		similarity_threshold: '50',
+		context: {}
+	}
+}
+
+function reportRow(testId: string, success: boolean) {
+	return {
+		test_id: testId,
+		pinned_at_run: 'TRUE',
+		success_rate_percent: success ? 100 : 0,
+		success_rate: success ? '1/1' : '0/1',
+		results: [
+			{
+				success,
+				result: `${testId}-${success ? 'pass' : 'fail'}`,
+				evaluation: '',
+				errors: ''
+			}
+		]
+	}
+}
+
+async function writeReport(dir: string, agent: string, timestamp: string, rows: unknown[]) {
+	const file = path.join(dir, `${agent}.${timestamp}.report.json`)
+	await writeFile(file, JSON.stringify({ rows }), 'utf8')
+	return file
+}
+
+describe('loadLatestResultsByTest', () => {
+	it('persists summaries and incrementally reads appended reports', async () => {
+		const projectDir = await mkdtemp(path.join(os.tmpdir(), 'bosun-latest-results-'))
+		temporaryDirectories.push(projectDir)
+		const bosunDir = path.join(projectDir, 'src', 'lib', 'bosun')
+		const reportDir = path.join(bosunDir, 'data', 'test_runs')
+		await mkdir(reportDir, { recursive: true })
+		root.path = path.join(bosunDir, 'index.ts')
+
+		const agent = `agent-${Date.now()}-${Math.random().toString(16).slice(2)}`
+		await writeFile(
+			path.join(bosunDir, `${agent}.tests.jsonl`),
+			[
+				JSON.stringify({ version: 1, meta: {} }),
+				JSON.stringify(testRow('a')),
+				JSON.stringify(testRow('b'))
+			].join('\n') + '\n',
+			'utf8'
+		)
+
+		const firstReport = await writeReport(reportDir, agent, '2026-07-16T10-00-00.000Z', [
+			reportRow('a', false),
+			reportRow('b', false)
+		])
+		await utimes(firstReport, 1_700_000_000, 1_700_000_000)
+		await writeReport(reportDir, agent, '2026-07-16T11-00-00.000Z', [reportRow('a', true)])
+
+		const initial = await loadLatestResultsByTest(agent)
+		expect(initial.a.results[0].success).toBe(true)
+		expect(initial.a.previous).toMatchObject({ success_count: 0, total_runs: 1 })
+		expect(initial.b.is_old).toBe(true)
+
+		expect((await readdir(reportDir)).some((name) => name.includes('latest-results'))).toBe(true)
+
+		// Keep the cached file signature but make the old report unreadable as JSON. An incremental
+		// update must only read the newly appended report and therefore still succeed.
+		const firstState = await stat(firstReport)
+		const original = await readFile(firstReport)
+		await writeFile(firstReport, Buffer.alloc(original.length, 0x20))
+		await utimes(firstReport, firstState.atimeMs / 1000, firstState.mtimeMs / 1000)
+		expect((await stat(firstReport)).mtimeMs).toBe(firstState.mtimeMs)
+		await writeReport(reportDir, agent, '2026-07-16T12-00-00.000Z', [reportRow('b', true)])
+
+		const updated = await loadLatestResultsByTest(agent)
+		expect(updated.a.is_old).toBe(true)
+		expect(updated.b.results[0]).toMatchObject({ success: true, result: 'b-pass' })
+		expect(updated.b.previous).toMatchObject({ success_count: 0, total_runs: 1 })
+	})
+
+	it('loads compressed reports and keeps the cache valid when the latest report is compressed', async () => {
+		const projectDir = await mkdtemp(path.join(os.tmpdir(), 'bosun-gzip-results-'))
+		temporaryDirectories.push(projectDir)
+		const bosunDir = path.join(projectDir, 'src', 'lib', 'bosun')
+		const reportDir = path.join(bosunDir, 'data', 'test_runs')
+		await mkdir(reportDir, { recursive: true })
+		root.path = path.join(bosunDir, 'index.ts')
+
+		const agent = `agent-${Date.now()}-${Math.random().toString(16).slice(2)}`
+		await writeFile(
+			path.join(bosunDir, `${agent}.tests.jsonl`),
+			[JSON.stringify({ version: 1, meta: {} }), JSON.stringify(testRow('a'))].join('\n') + '\n',
+			'utf8'
+		)
+
+		await compressReportFile(
+			await writeReport(reportDir, agent, '2026-07-16T10-00-00.000Z', [reportRow('a', false)])
+		)
+		const latestReport = await writeReport(reportDir, agent, '2026-07-16T11-00-00.000Z', [
+			reportRow('a', true)
+		])
+
+		const initial = await loadLatestResultsByTest(agent)
+		expect(initial.a.results[0].success).toBe(true)
+		expect(initial.a.previous).toMatchObject({ success_count: 0, total_runs: 1 })
+
+		await compressReportFile(latestReport)
+		const compressed = await loadLatestResultsByTest(agent)
+		expect(compressed.a.results[0]).toMatchObject({ success: true, result: 'a-pass' })
+		expect((await getRunResultEntry(agent, 'a')).success).toBe(true)
+	})
+})