@recombine-ai/bosun 0.10.1 → 0.11.2

package/src/backend/roleplay/variants.ts

@@ -28,28 +28,30 @@ export async function generateReplyVariants(props: {
 	agentId: string
 	region?: string | null
 	states: AgentState[]
-	prevStateId: string | null
-	rawUserMessage: string
 	count: number
 	context: Record<string, any>
 }): Promise<GeneratedVariant[]> {
-	const { agentId, region, states, prevStateId, rawUserMessage, count, context } = props
+	const { agentId, region, states, count, context } = props
 	if (!Number.isFinite(count) || count <= 0) return []
+	const targetState = states.at(-1)
+	if (!targetState) throw new Error('No saved roleplay state found for variants')
 
 	const rootPath = await getRootPath()
 	const { factories } = await getAgentFactories(rootPath, getImportNonce())
 	const factory = factories[agentId]
 	if (!factory) throw new Error(`Cannot find agent ${agentId}`)
 
-	const statesById = new Map<string, AgentState>()
-	for (const state of states) statesById.set(state.id, clone(state))
+	const statesById = new Map(states.map((state) => [state.id, clone(state)]))
 
+	const prevStateId = targetState.prev
 	const prevState = prevStateId ? (statesById.get(prevStateId) ?? null) : null
 	if (prevStateId && !prevState) {
 		throw new Error('Could not find the previous roleplay state for this reply')
 	}
 
-	const message = parseRawUserMessage(rawUserMessage)
+	const message = latestInputMessage(targetState)
+	if (!message) throw new Error('Could not reconstruct the input message for variants')
+
 	return Promise.all(
 		Array.from({ length: count }, (_, index) =>
 			generateSingleReplyVariant({
@@ -59,7 +61,6 @@ export async function generateReplyVariants(props: {
 				context,
 				prevState,
 				statesById,
-				rawUserMessage,
 				message
 			})
 		)
@@ -84,20 +85,6 @@ export function replaceSavedState(
 	return next
 }
 
-function parseRawUserMessage(raw: string): Message {
-	const trimmed = String(raw ?? '').trim()
-	if (!trimmed) throw new Error('Could not reconstruct the user message for variants')
-
-	const imageMatch = /^\[image\]\s+(.+)$/u.exec(trimmed)
-	if (imageMatch) {
-		const imageUrl = (imageMatch[1] ?? '').trim()
-		if (!imageUrl) throw new Error('Image variant message is missing its URL')
-		return { sender: 'user', text: '', imageUrl }
-	}
-
-	return { sender: 'user', text: trimmed }
-}
-
 function collectMessagesUpToState(
 	statesById: Map<string, AgentState>,
 	stateId: string | null
@@ -124,30 +111,32 @@ function collectMessagesUpToState(
 }
 
 function collectMessagesFromHistory(history: AgentState['history']): Message[] {
-	const messages: Message[] = []
-	for (const entry of history) {
-		if (entry.type !== 'message') continue
-		messages.push(entry.payload as Message)
-	}
-	return messages
+	return history.filter((entry) => entry.type === 'message').map((entry) => entry.payload)
+}
+
+function latestInputMessage(state: AgentState): Message | null {
+	const entry = [...state.history]
+		.reverse()
+		.find(
+			(entry) =>
+				entry.type === 'message' &&
+				(entry.payload.sender === 'user' || entry.payload.sender === 'system')
+		)
+	return entry ? clone(entry.payload as Message) : null
 }
 
 function extractVariantText(state: AgentState): string {
-	for (let index = state.history.length - 1; index >= 0; index--) {
-		const entry = state.history[index]
-		if (entry.type !== 'conversational-trace') continue
-		const trace = entry.payload as ConversationalTrace
-		if (trace.eventName === 'finished-main-streaming-step' || trace.eventName === 'sent-message') {
-			return String(trace.content ?? '')
-		}
-	}
+	const traceEntry = [...state.history].reverse().find((entry) => {
+		if (entry.type !== 'conversational-trace') return false
+		const trace = entry.payload
+		return trace.eventName === 'finished-main-streaming-step' || trace.eventName === 'sent-message'
+	})
+	if (traceEntry) return String((traceEntry.payload as ConversationalTrace).content ?? '')
 
-	for (let index = state.history.length - 1; index >= 0; index--) {
-		const entry = state.history[index]
-		if (entry.type !== 'message') continue
-		const payload = entry.payload as Message
-		if (payload.sender === 'agent') return String(payload.text ?? '')
-	}
+	const messageEntry = [...state.history]
+		.reverse()
+		.find((entry) => entry.type === 'message' && entry.payload.sender === 'agent')
+	if (messageEntry) return String((messageEntry.payload as Message).text ?? '')
 
 	return ''
 }
@@ -171,10 +160,9 @@ async function generateSingleReplyVariant(props: {
 	context: Record<string, any>
 	prevState: AgentState | null
 	statesById: Map<string, AgentState>
-	rawUserMessage: string
 	message: Message
 }): Promise<GeneratedVariant> {
-	const { index, factory, region, context, prevState, statesById, rawUserMessage, message } = props
+	const { index, factory, region, context, prevState, statesById, message } = props
 
 	const state = createVariantState(prevState?.id ?? null, index)
 	const scheduleSeed = clone(prevState?.schedule ?? {})
@@ -212,9 +200,6 @@ async function generateSingleReplyVariant(props: {
 	}
 	const conversationalTracer: ConversationalTracer = {
 		addConversationalTrace: (trace: ConversationalTrace) => {
-			;(trace as any).bosunStateId = state.id
-			;(trace as any).bosunPrevStateId = state.prev
-			;(trace as any).bosunUserMessage = rawUserMessage
 			state.history.push({
 				type: 'conversational-trace',
 				payload: trace
@@ -228,9 +213,6 @@ async function generateSingleReplyVariant(props: {
 		const current = (ctx.get() as Record<string, unknown> | null | undefined) || {}
 		ctx.set({ ...current, recombineRegion: region })
 	}
-	if (context && typeof context === 'object') {
-		ctx.swap(region ? { ...context, recombineRegion: region } : context)
-	}
 
 	const getMessages = () => [
 		...collectMessagesUpToState(statesById, state.prev),
@@ -263,6 +245,8 @@ async function generateSingleReplyVariant(props: {
 		}
 	})
 
+	ctx.swap(region ? { ...context, recombineRegion: region } : context)
+
 	scheduler.setState(scheduleSeed)
 
 	if (!state.prev && hasStart(agent)) {

package/src/backend/tests/academy.ts

@@ -1,21 +1,27 @@
 // cspell:words conv
-import { type ProviderOptions } from 'promptfoo'
-import * as promptfoo from 'promptfoo'
 import { type Message, type StepTrace, type StepTracer } from '@recombine-ai/engine'
 import { loadAgents } from './emulate-agents'
-import { type Assertion as PFAssertion } from 'promptfoo'
+import { installTestPromptCacheBust } from './openai-prompt-cache'
+import { promptfoo } from './promptfoo-runtime'
 // Backend: per-project JSON test suites (no CSV)
 import { tryReadConfig } from './config'
 import { mkdir, rename, unlink, writeFile } from 'node:fs/promises'
 import path from 'node:path'
-import { readSuite } from './suite'
+import { readSuite, type TestRow as SuiteTestRow } from './suite'
 // Use types derived from promptfoo exports instead of incorrect default type imports
 
-import type { ApiProvider, CallApiContextParams, ProviderResponse } from 'promptfoo'
+import type {
+	ApiProvider,
+	Assertion as PFAssertion,
+	CallApiContextParams,
+	ProviderOptions,
+	ProviderResponse
+} from 'promptfoo'
 import deepmerge from 'deepmerge'
 import fs from 'node:fs'
 import { getRootPath } from '../lib/get-root-path'
 import { makeContext } from './context-utils'
+import { queueReportCompression, REPORT_JSON_SUFFIX } from './report-files'
 
 // ----------------- Type aliases & helpers -----------------
 type Context = Record<string, unknown>
@@ -30,9 +36,13 @@ interface CsvRow {
 	expected_output: string
 	similarity_threshold: number | string
 	pinned: 'TRUE' | 'FALSE'
-	context: string
+	context: string | Record<string, unknown>
+	mocks?: string | Record<string, unknown>
 	timeout: string
+	capture_mode?: string
 	scenario?: string
+	step?: string
+	author?: string
 }
 
 export interface ResultEntry {
@@ -65,6 +75,7 @@ export interface WrittenRunReportConfig {
 }
 
 export interface WrittenRunReportRow extends TableRow {
+	test: SuiteTestRow
 	agent: string
 	test_id: string
 	run_timestamp: string
@@ -84,9 +95,34 @@ export interface WrittenRunReport {
 type PromptTestsScope = 'all' | 'pinned'
 
 type PromptTestsSelection = { scope: PromptTestsScope } | { testId: string }
+type CaptureMode = 'immediate' | 'final'
 
 const activeRunsBySuiteKey = new Map<string, AbortController>()
 
+export interface PromptTestsProgress {
+	plannedTestIds: string[]
+	runningTestIds: string[]
+	completedTestIds: string[]
+}
+
+type MutablePromptTestsProgress = PromptTestsProgress & {
+	expectedRuns: number
+	startedTestIds: Set<string>
+	completedRunsByTestId: Map<string, number>
+}
+
+const activeProgressBySuiteKey = new Map<string, MutablePromptTestsProgress>()
+
+export function getPromptTestsProgress(suiteKey: string): PromptTestsProgress | null {
+	const progress = activeProgressBySuiteKey.get(suiteKey)
+	if (!progress) return null
+	return {
+		plannedTestIds: [...progress.plannedTestIds],
+		runningTestIds: [...progress.runningTestIds],
+		completedTestIds: [...progress.completedTestIds]
+	}
+}
+
 export type SuiteSelection = {
 	// `suiteKey` is the filesystem/report key (e.g. `voiceSupport.en-gb`).
 	// `agentId` is the canonical factory key (workflow export key, e.g. `voiceSupport`).
@@ -116,6 +152,7 @@ export async function runPromptTestsManaged(
 		return await runPromptTests(suite, selection, { signal: controller.signal })
 	} finally {
 		activeRunsBySuiteKey.delete(suite.suiteKey)
+		activeProgressBySuiteKey.delete(suite.suiteKey)
 	}
 }
 
@@ -209,7 +246,7 @@ function toTestInputs(
 						value: row.assertion_value
 					},
 					{
-						type: 'similar' as 'similar',
+						type: 'similar' as const,
 						value: String(row.expected_output),
 						threshold: Number(row.similarity_threshold) / 100
 					}
@@ -217,7 +254,9 @@ function toTestInputs(
 				context: (() => {
 					const testCtx = makeContext(row.context)
 					return defaultContext ? mergeContexts(defaultContext, testCtx) : testCtx
-				})()
+				})(),
+				captureMode: row.capture_mode === 'final' ? 'final' : 'immediate',
+				mocks: typeof row.mocks === 'undefined' ? undefined : makeContext(row.mocks)
 			}
 		})
 }
@@ -226,6 +265,7 @@ export class RecombineAgentProvider implements ApiProvider {
 	protected providerId: string
 	public config: Record<string, unknown>
 	private abortSignal?: AbortSignal
+	private onTestStarted?: (testId: string) => void
 
 	constructor(options: ProviderOptions) {
 		this.providerId = options.id || 'recombine-agent'
@@ -236,6 +276,10 @@ export class RecombineAgentProvider implements ApiProvider {
 		this.abortSignal = signal
 	}
 
+	setOnTestStarted(callback: (testId: string) => void) {
+		this.onTestStarted = callback
+	}
+
 	id() {
 		return this.providerId
 	}
@@ -244,17 +288,25 @@ export class RecombineAgentProvider implements ApiProvider {
 		const projectRoot = path.dirname(await getRootPath())
 		const vars = (context?.vars || {}) as unknown as {
 			agentId: string
+			testId: string
 			region?: string
 			conversation: Conversation
 			conversationParseError?: string
 			context: Context
+			mocks?: Context
+			captureMode?: CaptureMode | string
 		}
+		const testId = String(vars.testId ?? '')
+		if (testId) this.onTestStarted?.(testId)
 
 		if (typeof vars.conversationParseError === 'string' && vars.conversationParseError.trim()) {
 			return { output: '', error: vars.conversationParseError }
 		}
 
+		const captureMode: CaptureMode = vars.captureMode === 'final' ? 'final' : 'immediate'
 		let captured: string = ''
+		let captureEnabled = false
+		const finalCaptureErrors: string[] = []
 		const capturedLogs: string[] = []
 		const capturedStepTraces: StepTrace[] = []
 
@@ -265,7 +317,7 @@ export class RecombineAgentProvider implements ApiProvider {
 		// Clone the conversation so parallel runs don't share the same array reference.
 		// Shallow-clone each message; messages are simple { sender, text } objects in this flow.
 		const conv: Conversation = Array.isArray(vars.conversation)
-			? (vars.conversation as Conversation).map((m) => ({ ...m }))
+			? vars.conversation.map((m) => ({ ...m }))
 			: []
 
 		// Prepare a streaming log writer so logs are written as they occur
@@ -280,7 +332,6 @@ export class RecombineAgentProvider implements ApiProvider {
 		const stepTracer: StepTracer = {
 			addStepTrace: (stepTrace: StepTrace) => {
 				try {
-					console.log(`trace: ${stepTrace.name}`)
 					capturedStepTraces.push(stepTrace)
 				} catch {
 					// best-effort capture; ignore serialization issues
@@ -307,6 +358,9 @@ export class RecombineAgentProvider implements ApiProvider {
 					line = line.slice(0, 5000) + '... [TRUNCATED]'
 				}
 				capturedLogs.push(line)
+				if (captureMode === 'final' && captureEnabled && level === 'error') {
+					finalCaptureErrors.push(line)
+				}
 				try {
 					logStream.write(line + '\n')
 				} catch {
@@ -331,14 +385,24 @@ export class RecombineAgentProvider implements ApiProvider {
 				getMessages: () => conv,
 				sendMessage: async (message: Message | string) => {
 					const m = typeof message === 'string' ? { sender: 'agent', text: message } : message
-					if (m.sender === 'agent') {
-						captured = typeof m.text === 'string' ? m.text : JSON.stringify(m)
-						logger.log('Agent message captured:', captured)
+					if (captureEnabled && m.sender === 'agent') {
+						const text = typeof m.text === 'string' ? m.text : JSON.stringify(m)
+						if (captureMode === 'final' || !captured) {
+							captured = text
+							logger.log('Agent message captured:', captured)
+						} else {
+							logger.log('Agent message ignored after immediate capture:', text)
+						}
 					}
 				},
+				onBeforeTestResponse: () => {
+					captured = ''
+					captureEnabled = true
+				},
 				stepTracer,
 				logger,
 				context: vars.context || {},
+				mocks: vars.mocks,
 				abortSignal: this.abortSignal
 			})
 
@@ -378,6 +442,19 @@ export class RecombineAgentProvider implements ApiProvider {
 				? capturedStepTraces.map(sanitizeTrace)
 				: []
 
+			if (captureMode === 'final' && finalCaptureErrors.length > 0) {
+				const detail = finalCaptureErrors.at(-1) ?? 'Unknown final capture error'
+				return {
+					output: captured,
+					error: `Final capture failed because a subsequent step emitted an error: ${detail}`,
+					metadata: {
+						logs: capturedLogs,
+						stepTraces: sanitizedStepTraces,
+						rawStepTraces: capturedStepTraces
+					}
+				}
+			}
+
 			if (captured && captured.trim()) {
 				return {
 					output: captured,
@@ -399,6 +476,7 @@ export class RecombineAgentProvider implements ApiProvider {
 				}
 			}
 		} catch (err) {
+			if (this.abortSignal?.aborted) throw err
 			const msg = err instanceof Error ? `${err.name}: ${err.message}` : String(err)
 			capturedLogs.push(`[ERROR] ${msg}`)
 			try {
@@ -531,15 +609,15 @@ export async function runPromptTests(
 	const onlyTestId = 'testId' in selection ? selection.testId : undefined
 	const suiteKey = suite.suiteKey
 	const loadedSuite = await readSuite(suiteKey)
-	const rows = loadedSuite.tests as unknown as CsvRow[]
+	const rows = loadedSuite.tests
 	const cfg = (await tryReadConfig(suiteKey)) || {}
 
 	const repeats = Number(cfg?.repeats ?? 1)
-	const concurrency = Number(cfg?.concurrency ?? 1)
+	const concurrency = Math.min(75, Math.max(1, Math.floor(Number(cfg?.concurrency ?? 75))))
 	const baseContext = loadedSuite?.meta?.defaultContext || {}
 
 	// Add an index field to each row for mapping
-	const typedRows = rows as unknown as CsvRow[]
+	const typedRows = rows
 	const indexedRows = typedRows.map((row, index) => ({ ...row, index }))
 
 	if (onlyTestId && !indexedRows.some((row) => String(row.id ?? '') === String(onlyTestId))) {
@@ -547,12 +625,27 @@ export async function runPromptTests(
 	}
 
 	const tests = toTestInputs(indexedRows, baseContext, selection)
+	const plannedTestIds = tests.map((test) => String(test.id ?? '')).filter(Boolean)
+	const progress: MutablePromptTestsProgress = {
+		plannedTestIds,
+		runningTestIds: [],
+		completedTestIds: [],
+		expectedRuns: repeats,
+		startedTestIds: new Set(),
+		completedRunsByTestId: new Map()
+	}
+	activeProgressBySuiteKey.set(suiteKey, progress)
 
 	// Provider function with stable label
 	const providerInstance = new RecombineAgentProvider({
 		id: 'recombine-agent'
 	})
 	providerInstance.setAbortSignal(signal)
+	providerInstance.setOnTestStarted((testId) => {
+		if (progress.completedTestIds.includes(testId)) return
+		progress.startedTestIds.add(testId)
+		progress.runningTestIds = [...progress.startedTestIds]
+	})
 	const providerFn = Object.assign(
 		(_promptStr: string, ctx?: CallApiContextParams) => {
 			if (signal?.aborted) {
@@ -566,37 +659,40 @@ export async function runPromptTests(
 	const hookKey = `__bosun_promptfoo_hook_${process.pid}_${Date.now()}_${Math.random().toString(16).slice(2)}`
 	const extensionCode = `globalThis[${JSON.stringify(hookKey)}]?.(output, context)`
 
-	const testSuites = tests.map((test) => ({
+	const testSuite = {
 		prompts: ['agent-run'], // placeholder; provider ignores
-		scenario: test.scenario,
-		tests: [
-			{
-				metadata: { testId: test.id },
-				assert: test.assertions,
-				model: 'not used!',
-				vars: {
-					agentId: suite.agentId,
-					...(suite.region ? { region: suite.region } : {}),
-					context: test.context,
-					conversation: test.conversation,
-					...(test.conversationParseError
-						? { conversationParseError: test.conversationParseError }
-						: {})
-				}
+		tests: tests.map((test) => ({
+			description: test.scenario,
+			metadata: { testId: test.id },
+			assert: test.assertions,
+			model: 'not used!',
+			vars: {
+				agentId: suite.agentId,
+				testId: String(test.id ?? ''),
+				...(suite.region ? { region: suite.region } : {}),
+				context: test.context,
+				...(typeof test.mocks !== 'undefined' ? { mocks: test.mocks } : {}),
+				captureMode: test.captureMode,
+				conversation: test.conversation,
+				...(test.conversationParseError
+					? { conversationParseError: test.conversationParseError }
+					: {})
 			}
-		],
+		})),
 		providers: [providerFn],
 		extensions: [extensionCode]
-	}))
+	}
 
-	const projectRoot = path.dirname(await getRootPath())
+	const rootPath = await getRootPath()
+	installTestPromptCacheBust(rootPath)
+	const projectRoot = path.dirname(rootPath)
 	const outDir = path.join(projectRoot, 'data', 'test_runs')
 	await mkdir(outDir, { recursive: true })
 
 	// Use full ISO timestamp and sanitize ':' for filesystem safety
 	const iso = new Date().toISOString()
 	const isoFile = iso.replace(/:/g, '-')
-	const outPath = path.join(outDir, `${suiteKey}.${isoFile}.report.json`)
+	const outPath = path.join(outDir, `${suiteKey}.${isoFile}${REPORT_JSON_SUFFIX}`)
 
 	async function writeReport(report: WrittenRunReport) {
 		const dir = path.dirname(outPath)
@@ -639,7 +735,7 @@ export async function runPromptTests(
 
 	const byTestId = new Map<string, WrittenRunReportRow>()
 
-	const originalRowById = new Map<string, CsvRow>()
+	const originalRowById = new Map<string, SuiteTestRow>()
 	for (const r of typedRows) {
 		const id = String((r as any)?.id ?? '').trim()
 		if (id) originalRowById.set(id, r)
@@ -685,6 +781,7 @@ export async function runPromptTests(
 					String((originalRow as any).pinned || '').toUpperCase() === 'TRUE' ? 'TRUE' : 'FALSE'
 				const nextRow: WrittenRunReportRow = {
 					...(originalRow as any),
+					test: originalRow,
 					prompts: `suite://${suiteKey}?testId=${testId}`,
 					success_rate_percent: 0,
 					success_rate: '0/0',
@@ -704,6 +801,15 @@ export async function runPromptTests(
 
 			const entry = toResultEntry(r)
 			reportRow.results.push(entry)
+			const completedRuns = (progress.completedRunsByTestId.get(testId) ?? 0) + 1
+			progress.completedRunsByTestId.set(testId, completedRuns)
+			if (completedRuns >= progress.expectedRuns) {
+				progress.startedTestIds.delete(testId)
+				progress.runningTestIds = [...progress.startedTestIds]
+				if (!progress.completedTestIds.includes(testId)) {
+					progress.completedTestIds = [...progress.completedTestIds, testId]
+				}
+			}
 
 			const counts = countsByTestId.get(testId) ?? { pass: 0, total: 0 }
 			counts.total += 1
@@ -722,75 +828,68 @@ export async function runPromptTests(
 		}
 	}
 
-	try {
-		for (const suite of testSuites) {
-			if (signal?.aborted) break
-			const testId = String((suite.tests?.[0] as any)?.metadata?.testId ?? '')
-			const originalRow = testId ? typedRows.find((r) => String(r.id) === testId) : undefined
-			try {
-				console.log(`Running: ${suite.scenario}`)
-				const ev = await promptfoo.evaluate(suite, {
-					repeat: repeats,
-					cache: false,
-					maxConcurrency: concurrency,
-					abortSignal: signal
-				})
+	const selectedOriginalRows = plannedTestIds
+		.map((testId) => originalRowById.get(testId))
+		.filter((row): row is SuiteTestRow => !!row)
+	const persistEvaluation = (evaluation: PFEval) => {
+		for (const rowToWrite of buildTableRows([evaluation], selectedOriginalRows, suiteKey)) {
+			const testId = String(rowToWrite.id ?? '')
+			const originalRow = originalRowById.get(testId)
+			if (!testId || !originalRow) continue
+			const pinned: 'TRUE' | 'FALSE' =
+				(rowToWrite.pinned || '').toUpperCase() === 'TRUE' ? 'TRUE' : 'FALSE'
+			byTestId.set(testId, {
+				...rowToWrite,
+				test: originalRow,
+				agent: suiteKey,
+				test_id: testId,
+				run_timestamp: iso,
+				run_config: JSON.stringify(runConfig),
+				pinned_at_run: scope === 'all' || !!onlyTestId ? 'TRUE' : pinned,
+				run_scope: scope
+			})
+		}
+		body.rows = Array.from(byTestId.values())
+	}
 
-				if (originalRow && testId) {
-					const rowToWrite = buildTableRows([ev], [originalRow], suiteKey)[0]
-					const pinned: 'TRUE' | 'FALSE' =
-						(rowToWrite.pinned || '').toUpperCase() === 'TRUE' ? 'TRUE' : 'FALSE'
-					byTestId.set(testId, {
-						...rowToWrite,
-						agent: suiteKey,
-						test_id: testId,
-						run_timestamp: iso,
-						run_config: JSON.stringify(runConfig),
-						pinned_at_run: scope === 'all' || !!onlyTestId ? 'TRUE' : pinned,
-						run_scope: scope
-					})
-					body.rows = Array.from(byTestId.values())
-					await enqueueWrite()
-				}
-			} catch (err) {
-				const cancelled =
-					!!signal?.aborted ||
-					(err instanceof Error && /operation cancelled/i.test(err.message || ''))
-				if (cancelled) {
-					break
-				}
-				const msg = err instanceof Error ? `${err.name}: ${err.message}` : String(err)
-				const ev = {
-					results: [
-						{
+	const previousIsTest = process.env.IS_TEST
+	process.env.IS_TEST = 'true'
+	try {
+		if (tests.length > 0 && !signal?.aborted) {
+			const evaluation = await promptfoo.evaluate(testSuite, {
+				repeat: repeats,
+				cache: false,
+				maxConcurrency: concurrency,
+				// Bosun reports run progress in the tests UI. Promptfoo's terminal progress bars
+				// continuously rewrite stdout (up to 20 bars at higher concurrency).
+				showProgressBar: false,
+				abortSignal: signal
+			})
+			persistEvaluation(evaluation)
+			await enqueueWrite()
+		}
+	} catch (err) {
+		const cancelled =
+			!!signal?.aborted || (err instanceof Error && /operation cancelled/i.test(err.message || ''))
+		if (!cancelled) {
+			const msg = err instanceof Error ? `${err.name}: ${err.message}` : String(err)
+			const evaluation = {
+				results: plannedTestIds.map(
+					(testId) =>
+						({
 							success: false,
 							response: { output: '' },
 							error: msg,
-							metadata: { testId: (suite.tests?.[0] as any)?.metadata?.testId }
-						} as any
-					]
-				} as any
-
-				// Persist failure row immediately for real-time UI updates.
-				if (originalRow && testId) {
-					const rowToWrite = buildTableRows([ev], [originalRow], suiteKey)[0]
-					const pinned: 'TRUE' | 'FALSE' =
-						(rowToWrite.pinned || '').toUpperCase() === 'TRUE' ? 'TRUE' : 'FALSE'
-					byTestId.set(testId, {
-						...rowToWrite,
-						agent: suiteKey,
-						test_id: testId,
-						run_timestamp: iso,
-						run_config: JSON.stringify(runConfig),
-						pinned_at_run: scope === 'all' || !!onlyTestId ? 'TRUE' : pinned,
-						run_scope: scope
-					})
-					body.rows = Array.from(byTestId.values())
-					await enqueueWrite()
-				}
-			}
+							metadata: { testId }
+						}) as any
+				)
+			} as PFEval
+			persistEvaluation(evaluation)
+			await enqueueWrite()
 		}
 	} finally {
+		if (previousIsTest == null) delete process.env.IS_TEST
+		else process.env.IS_TEST = previousIsTest
 		if (writeTimer) clearTimeout(writeTimer)
 		writeTimer = null
 		delete (globalThis as any)[hookKey]
@@ -800,6 +899,9 @@ export async function runPromptTests(
 	// Ensure the final report is written even if no tests ran (or an update was skipped).
 	body.rows = Array.from(byTestId.values())
 	await enqueueWrite()
+	void queueReportCompression(outPath).catch((error) => {
+		console.error(`Failed to compress completed test report ${path.basename(outPath)}`, error)
+	})
 
 	return body
 }