@recombine-ai/bosun 0.18.0 → 0.20.0

package/src/backend/tests/academy.ts

@@ -1,960 +0,0 @@
-// cspell:words conv
-import { type Message, type StepTrace, type StepTracer } from '@recombine-ai/engine'
-import { loadAgents } from './emulate-agents'
-import { installTestPromptCacheBust } from './openai-prompt-cache'
-import { promptfoo } from './promptfoo-runtime'
-// Backend: per-agent YAML test cases.
-import { mkdir, rename, unlink, writeFile } from 'node:fs/promises'
-import path from 'node:path'
-import { readSuite, type TestRow as SuiteTestRow } from './suite'
-import { isMainStepName } from '../../lib/steps'
-// Use types derived from promptfoo exports instead of incorrect default type imports
-
-import type {
-	ApiProvider,
-	Assertion as PFAssertion,
-	CallApiContextParams,
-	ProviderOptions,
-	ProviderResponse
-} from 'promptfoo'
-import { getRootPath } from '../lib/get-root-path'
-import { ActiveTestRuns } from './active-test-runs'
-import { createBestEffortLogWriter, type BestEffortLogWriter } from './best-effort-log-writer'
-import { makeContext } from './context-utils'
-import { gradeJsonExpectedFields } from './json-field-match'
-import { queueReportCompression, REPORT_JSON_SUFFIX } from './report-files'
-
-// ----------------- Type aliases & helpers -----------------
-type Context = Record<string, unknown>
-
-type Conversation = Message[]
-
-interface CsvRow {
-	id?: string
-	assertion_type: string
-	assertion_value: string
-	conversation: string
-	expected_output: string
-	evaluation_mode?: string
-	case_sensitive?: boolean
-	context_mode?: string
-	similarity_threshold: number | string
-	pinned: 'TRUE' | 'FALSE'
-	context: string | Record<string, unknown>
-	mocks?: string | Record<string, unknown>
-	timeout: string
-	scenario?: string
-	step?: string
-	author?: string
-}
-
-export interface ResultEntry {
-	success: boolean
-	result: string
-	evaluation: string
-	rendered_prompt: string
-	received_prompt: string
-	stringified_conversation: string
-	received_context: string
-	// Raw traces array for this run (sanitized where necessary)
-	traces: StepTrace[]
-	errors: string
-	logs: string[]
-}
-
-export interface TableRow extends CsvRow {
-	// Link to details view for this row
-	prompts: string
-	success_rate_percent: number
-	success_rate: string
-	// Per-run results (new format)
-	results: ResultEntry[]
-}
-
-export interface WrittenRunReportConfig {
-	repeats: number
-	concurrency: number
-}
-
-export type PromptTestsRunConfig = Pick<WrittenRunReportConfig, 'repeats' | 'concurrency'>
-export type TestHistoryOverrides = Record<string, { conversation: string }>
-
-export function applyTestHistoryOverrides<T extends { id?: unknown; conversation: string }>(
-	rows: T[],
-	overrides: TestHistoryOverrides
-): T[] {
-	return rows.map((row) => {
-		const override = overrides[typeof row.id === 'string' ? row.id : '']
-		return override ? { ...row, conversation: override.conversation } : row
-	})
-}
-
-export interface WrittenRunReportRow extends TableRow {
-	test: SuiteTestRow
-	agent: string
-	test_id: string
-	run_timestamp: string
-	run_config: string
-	pinned_at_run: 'TRUE' | 'FALSE'
-	run_scope: 'all' | 'pinned'
-}
-
-export interface WrittenRunReport {
-	agent: string
-	run_timestamp: string
-	run_scope: 'all' | 'pinned'
-	run_config: WrittenRunReportConfig
-	rows: WrittenRunReportRow[]
-}
-
-type PromptTestsScope = 'all' | 'pinned'
-
-type PromptTestsSelection = { scope: PromptTestsScope; step?: string } | { testId: string }
-
-export interface PromptTestsProgress {
-	plannedTestIds: string[]
-	runningTestIds: string[]
-	completedTestIds: string[]
-}
-
-type MutablePromptTestsProgress = PromptTestsProgress & {
-	expectedRuns: number
-	startedTestIds: Set<string>
-	completedRunsByTestId: Map<string, number>
-}
-
-const activeRuns = new ActiveTestRuns<MutablePromptTestsProgress>()
-
-export function getPromptTestsProgress(suiteKey: string): PromptTestsProgress | null {
-	const progress = activeRuns.getProgress(suiteKey)
-	if (!progress) return null
-	return {
-		plannedTestIds: [...progress.plannedTestIds],
-		runningTestIds: [...progress.runningTestIds],
-		completedTestIds: [...progress.completedTestIds]
-	}
-}
-
-export type SuiteSelection = {
-	// `suiteKey` is the filesystem/report key (e.g. `voiceSupport.en-gb`).
-	// `agentId` is the canonical factory key (workflow export key, e.g. `voiceSupport`).
-	// Keeping these separate avoids any parsing/guessing in the test runner.
-	suiteKey: string
-	agentId: string
-	region?: string
-}
-
-export function cancelPromptTests(suiteKey: string): boolean {
-	return activeRuns.cancel(suiteKey)
-}
-
-export async function runPromptTestsManaged(
-	suite: SuiteSelection,
-	selection: PromptTestsSelection,
-	requestedConfig: PromptTestsRunConfig,
-	promptOverrides: Record<string, string> = {},
-	testHistoryOverrides: TestHistoryOverrides = {}
-): Promise<WrittenRunReport> {
-	const run = activeRuns.start(suite.suiteKey)
-	if (!run) throw new Error('Tests already running for this agent')
-	try {
-		return await runPromptTests(suite, selection, requestedConfig, {
-			signal: run.controller.signal,
-			promptOverrides,
-			testHistoryOverrides,
-			setProgress: (progress) => activeRuns.setProgress(suite.suiteKey, run, progress)
-		})
-	} finally {
-		activeRuns.finish(suite.suiteKey, run)
-	}
-}
-
-// Parse a multi-line sheet conversation string into engine messages
-function parseConversationFromText(conversation: string | undefined | null): Conversation {
-	if (!conversation) return []
-
-	// Greedily accumulate lines for a role until the next role marker.
-	// Recognized markers (case-insensitive) at the beginning of a line:
-	//   agent:, assistant:, user:, system:
-	const lines = conversation.split(/\r?\n/)
-	const messages: Conversation = []
-
-	let currentRole: 'agent' | 'user' | 'system' | null = null
-	let buffer: string[] = []
-
-	const roleMatch = (line: string) => line.match(/^\s*(agent|assistant|user|system)\s*:\s*(.*)$/i)
-	const flush = () => {
-		if (currentRole !== null) {
-			messages.push({ sender: currentRole, text: buffer.join('\n') })
-		}
-	}
-
-	for (let i = 0; i < lines.length; i++) {
-		const line = lines[i] ?? ''
-		const m = roleMatch(line)
-		if (m) {
-			// Starting a new role block; flush previous
-			flush()
-			const roleRaw = (m[1] || '').toLowerCase()
-			currentRole = roleRaw === 'assistant' ? 'agent' : (roleRaw as 'agent' | 'user' | 'system')
-			buffer = [m[2] ?? '']
-		} else if (currentRole !== null) {
-			// Continuation of the current role block (allow blank lines)
-			buffer.push(line)
-		} else {
-			// No current role block yet.
-			// Previously we silently ignored preamble lines; that's almost always a test authoring mistake
-			// (e.g. a malformed "User:"/"Agent:" prefix like "?Agent:").
-			if (line.trim()) {
-				const snippet = line.length > 200 ? `${line.slice(0, 200)}…` : line
-				throw new Error(
-					`Unparsed conversation content before the first role block at line ${i + 1}: ${snippet}\n` +
-						`Start each block with "User:", "Agent:" (or "Assistant:"), or "System:".`
-				)
-			}
-		}
-	}
-
-	// Flush the last buffered message
-	flush()
-
-	return messages.filter(
-		(m) => m.sender === 'user' || m.sender === 'agent' || m.sender === 'system'
-	)
-}
-
-// Map pinned sheet rows to common test inputs
-function toTestInputs(
-	rows: (CsvRow & { index: number })[],
-	selection: PromptTestsSelection = { scope: 'pinned' }
-) {
-	return rows
-		.filter((r) => {
-			if ('testId' in selection) return String(r.id ?? '') === String(selection.testId)
-			if (selection.step) {
-				const step = String(r.step || 'main')
-				// A row against the primary step belongs to whichever variant is selected; see isMainStepName.
-				const matches =
-					step === selection.step || (isMainStepName(step) && isMainStepName(selection.step))
-				if (!matches) return false
-			}
-			return selection.scope === 'all' ? true : r.pinned === 'TRUE'
-		})
-		.map((row) => {
-			const step = String(row.step || 'main')
-			let conversation: Conversation = []
-			let conversationParseError: string | undefined
-			if (isMainStepName(step)) {
-				try {
-					conversation = parseConversationFromText(row.conversation)
-				} catch (err) {
-					conversation = []
-					conversationParseError = err instanceof Error ? err.message : String(err)
-				}
-			}
-
-			const assertions: PFAssertion[] =
-				row.evaluation_mode === 'json-deep-equal'
-					? [
-							{
-								type: 'javascript',
-								value: (output: string) =>
-									gradeJsonExpectedFields(output, row.expected_output, row.case_sensitive === true)
-							}
-						]
-					: [
-							{
-								type: row.assertion_type as PFAssertion['type'],
-								value: row.assertion_value
-							}
-						]
-
-			return {
-				id: row.id,
-				scenario: row.scenario,
-				step,
-				conversation,
-				stringifiedConversation: String(row.conversation || ''),
-				conversationParseError,
-				assertions,
-				context: makeContext(row.context),
-				mocks: typeof row.mocks === 'undefined' ? undefined : makeContext(row.mocks)
-			}
-		})
-}
-
-export class RecombineAgentProvider implements ApiProvider {
-	protected providerId: string
-	public config: Record<string, unknown>
-	private abortSignal?: AbortSignal
-	private onTestStarted?: (testId: string) => void
-
-	constructor(options: ProviderOptions) {
-		this.providerId = options.id || 'recombine-agent'
-		this.config = options.config || {}
-	}
-
-	setAbortSignal(signal: AbortSignal | undefined) {
-		this.abortSignal = signal
-	}
-
-	setOnTestStarted(callback: (testId: string) => void) {
-		this.onTestStarted = callback
-	}
-
-	id() {
-		return this.providerId
-	}
-
-	async callApi(_prompt: string, context?: CallApiContextParams): Promise<ProviderResponse> {
-		const projectRoot = path.dirname(await getRootPath())
-		const vars = (context?.vars || {}) as unknown as {
-			agentId: string
-			testId: string
-			region?: string
-			conversation: Conversation
-			stringifiedConversation?: string
-			conversationParseError?: string
-			context: Context
-			mocks?: Context
-			step?: string
-			promptOverrides?: Record<string, string>
-		}
-		const testId = String(vars.testId ?? '')
-		if (testId) this.onTestStarted?.(testId)
-
-		if (typeof vars.conversationParseError === 'string' && vars.conversationParseError.trim()) {
-			return { output: '', error: vars.conversationParseError }
-		}
-
-		const targetStep = String(vars.step || 'main')
-		let captured: string = ''
-		let captureEnabled = false
-		let targetStepMatched = false
-		const capturedLogs: string[] = []
-		const capturedStepTraces: StepTrace[] = []
-
-		// Streaming log writer; initialized after basic validation
-		let logWriter: BestEffortLogWriter | null = null
-		let logsFilePath = ''
-
-		// Clone the conversation so parallel runs don't share the same array reference.
-		// Shallow-clone each message; messages are simple { sender, text } objects in this flow.
-		const conv: Conversation = Array.isArray(vars.conversation)
-			? vars.conversation.map((m) => ({ ...m }))
-			: []
-
-		// Prepare a streaming log writer so logs are written as they occur
-		{
-			const runId = `${Date.now()}_${Math.random().toString(36).slice(2, 8)}`
-			const logsDir = path.join(projectRoot, 'data', 'logs')
-			await mkdir(logsDir, { recursive: true })
-			logsFilePath = path.join(logsDir, `capturedLogs_${runId}.log`)
-			logWriter = createBestEffortLogWriter(logsFilePath)
-		}
-
-		const stepTracer: StepTracer = {
-			addStepTrace: (stepTrace: StepTrace) => {
-				try {
-					capturedStepTraces.push(stepTrace)
-					if (
-						captureEnabled &&
-						!isMainStepName(targetStep) &&
-						!targetStepMatched &&
-						stepTrace.name === targetStep
-					) {
-						targetStepMatched = true
-						captured = toStringSafe(stepTrace.response ?? '')
-						logger.log(`Step ${targetStep} captured:`, captured)
-					}
-				} catch {
-					// best-effort capture; ignore serialization issues
-				}
-			},
-			async flush() {}
-		}
-
-		const push = (level: 'log' | 'debug' | 'warn' | 'error', args: any[]) => {
-			try {
-				const raw = args
-					.map((a) => {
-						try {
-							return typeof a === 'string' ? a : JSON.stringify(a)
-						} catch {
-							return String(a)
-						}
-					})
-					.join(' ')
-				const prefix = `[${level.toUpperCase()}] `
-				let line = `${prefix}${raw}`
-				// Enforce max 5000 chars per entry for logs
-				if (line.length > 5000) {
-					line = line.slice(0, 5000) + '... [TRUNCATED]'
-				}
-				capturedLogs.push(line)
-				logWriter?.write(line + '\n')
-			} catch {
-				// best-effort capture
-			}
-		}
-		const logger = {
-			log: (...args: any[]) => push('log', args),
-			debug: (...args: any[]) => push('debug', args),
-			warn: (...args: any[]) => push('warn', args),
-			error: (...args: any[]) => push('error', args)
-		}
-
-		try {
-			// Run the agent via loadAgents using provided conversation
-			await loadAgents({
-				agentId: vars.agentId,
-				region: vars.region,
-				getMessages: () => conv,
-				sendMessage: async (message: Message | string) => {
-					const m = typeof message === 'string' ? { sender: 'agent', text: message } : message
-					if (captureEnabled && isMainStepName(targetStep) && m.sender === 'agent') {
-						const text = typeof m.text === 'string' ? m.text : JSON.stringify(m)
-						if (!captured) {
-							captured = text
-							logger.log('Agent message captured:', captured)
-						} else {
-							logger.log('Agent message ignored after immediate capture:', text)
-						}
-					}
-				},
-				onBeforeTestResponse: () => {
-					captured = ''
-					targetStepMatched = false
-					captureEnabled = true
-				},
-				stepTracer,
-				logger,
-				context: vars.context || {},
-				mocks: vars.mocks,
-				promptOverrides: vars.promptOverrides,
-				abortSignal: this.abortSignal,
-				...(isMainStepName(targetStep)
-					? {}
-					: {
-							testStep: {
-								name: targetStep,
-								context: vars.context || {},
-								stringifiedConversation: vars.stringifiedConversation || ''
-							}
-						})
-			})
-
-			const logsDir = path.join(projectRoot, 'data', 'logs')
-			await mkdir(logsDir, { recursive: true })
-			const filePath = path.join(logsDir, `capturedStepTraces_${Date.now()}.json`)
-			await writeFile(filePath, JSON.stringify(capturedStepTraces, null, 2))
-
-			// Prepare sanitized traces for sheet output
-			const sanitizeTrace = (t: StepTrace): any => {
-				try {
-					const clone: any = { ...t }
-					const isMain = isMainStepName(clone?.name)
-					// Truncate large prompt fields on non-main steps
-					const truncate = (s: any, max = 5000) => {
-						if (typeof s !== 'string') return s
-						return s.length > max ? s.slice(0, max) + '... [TRUNCATED]' : s
-					}
-					if (isMain) {
-						// Hide main step rendered/received prompts entirely in traces
-						if ('renderedPrompt' in clone) delete clone.renderedPrompt
-						if ('receivedPrompt' in clone) delete clone.receivedPrompt
-					} else {
-						if (typeof clone.renderedPrompt !== 'undefined') {
-							clone.renderedPrompt = truncate(clone.renderedPrompt)
-						}
-						if (typeof clone.receivedPrompt !== 'undefined') {
-							clone.receivedPrompt = truncate(clone.receivedPrompt)
-						}
-					}
-					return clone
-				} catch {
-					return t
-				}
-			}
-			const sanitizedStepTraces = Array.isArray(capturedStepTraces)
-				? capturedStepTraces.map(sanitizeTrace)
-				: []
-
-			if (!isMainStepName(targetStep) && targetStepMatched) {
-				return {
-					output: captured,
-					metadata: {
-						targetStep,
-						logs: capturedLogs,
-						stepTraces: sanitizedStepTraces,
-						rawStepTraces: capturedStepTraces
-					}
-				}
-			}
-
-			if (captured && captured.trim()) {
-				return {
-					output: captured,
-					metadata: {
-						targetStep,
-						logs: capturedLogs,
-						stepTraces: sanitizedStepTraces,
-						rawStepTraces: capturedStepTraces
-					}
-				}
-			}
-
-			return {
-				output: '',
-				error: isMainStepName(targetStep)
-					? 'No agent reply captured'
-					: `Step "${targetStep}" did not execute after the test boundary`,
-				metadata: {
-					targetStep,
-					logs: capturedLogs,
-					stepTraces: sanitizedStepTraces,
-					rawStepTraces: capturedStepTraces
-				}
-			}
-		} catch (err) {
-			if (this.abortSignal?.aborted) throw err
-			const msg = err instanceof Error ? `${err.name}: ${err.message}` : String(err)
-			capturedLogs.push(`[ERROR] ${msg}`)
-			logWriter?.write(`[ERROR] ${msg}\n`)
-			return {
-				output: '',
-				error: msg,
-				metadata: { targetStep, logs: capturedLogs, stepTraces: capturedStepTraces }
-			}
-		} finally {
-			// Ensure the stream flushes all data
-			await logWriter?.close()
-		}
-	}
-}
-
-// ----------------- Helpers -----------------
-// Derive the result type returned inside Eval from promptfoo.evaluate
-type PFEval = Awaited<ReturnType<typeof promptfoo.evaluate>>
-type PFEvalResult = PFEval['results'][number]
-
-const toStringSafe = (v: unknown): string => {
-	if (typeof v === 'string') return v
-	try {
-		return JSON.stringify(v)
-	} catch {
-		return String(v)
-	}
-}
-
-function toResultEntry(r: any): ResultEntry {
-	const rawTraces = Array.isArray(r.metadata?.rawStepTraces)
-		? r.metadata?.rawStepTraces
-		: Array.isArray(r.metadata?.stepTraces)
-			? r.metadata?.stepTraces
-			: []
-	const targetStep = String(r.metadata?.targetStep || 'main')
-	const sourceTrace = isMainStepName(targetStep)
-		? // A row targeting the primary step matches whichever variant actually ran.
-			[...rawTraces].reverse().find((trace: any) => isMainStepName(trace?.name))
-		: rawTraces.find((trace: any) => trace?.name === targetStep)
-	const rendered_prompt =
-		sourceTrace && typeof sourceTrace.renderedPrompt !== 'undefined'
-			? toStringSafe(sourceTrace.renderedPrompt)
-			: 'NO RENDERED PROMPT'
-	const received_prompt =
-		sourceTrace && typeof sourceTrace.receivedPrompt !== 'undefined'
-			? toStringSafe(sourceTrace.receivedPrompt)
-			: 'NO RECEIVED PROMPT'
-	const stringified_conversation =
-		sourceTrace && typeof sourceTrace.stringifiedConversation !== 'undefined'
-			? toStringSafe(sourceTrace.stringifiedConversation)
-			: 'NO STRINGIFIED CONVERSATION'
-	const received_context =
-		sourceTrace && typeof sourceTrace.receivedContext !== 'undefined'
-			? toStringSafe(sourceTrace.receivedContext)
-			: 'NO RECEIVED CONTEXT'
-
-	const logs = Array.isArray(r.metadata?.logs) ? (r.metadata?.logs as string[]) : []
-	const traces = Array.isArray(r.metadata?.stepTraces) ? r.metadata?.stepTraces : []
-	return {
-		success: !!r.success,
-		result: String(r.response?.output ?? ''),
-		evaluation: String(r.gradingResult?.reason ?? ''),
-		rendered_prompt,
-		received_prompt,
-		stringified_conversation,
-		received_context,
-		traces,
-		errors: String(r.error ?? ''),
-		logs
-	}
-}
-
-function buildTableRows(
-	evals: Array<Awaited<ReturnType<typeof promptfoo.evaluate>>>,
-	csvRows: CsvRow[],
-	agentId: string
-): TableRow[] {
-	const bucket = new Map<string, PFEvalResult[]>()
-
-	for (const ev of evals) {
-		for (const res of ev.results) {
-			const testId = String(res.metadata?.testId ?? '')
-			;(bucket.get(testId) ?? bucket.set(testId, []).get(testId)!).push(res)
-		}
-	}
-
-	return csvRows.map((row) => {
-		const rowId = row.id
-		const evalResults = bucket.get(String(rowId)) ?? []
-		const passCount = evalResults.filter((r) => r.success).length
-		const totalRuns = evalResults.length
-		const pct = totalRuns ? Math.round((passCount / totalRuns) * 100) : 0
-		const rateStr = `${passCount}/${totalRuns}`
-
-		const results: ResultEntry[] = evalResults.map((r) => toResultEntry(r))
-
-		return {
-			...row,
-			prompts: `suite://${agentId}?testId=${rowId}`,
-			success_rate_percent: pct,
-			success_rate: rateStr,
-			results
-		}
-	})
-}
-
-export async function runPromptTests(
-	suite: SuiteSelection,
-	selection: PromptTestsSelection,
-	requestedConfig: PromptTestsRunConfig,
-	opts?: {
-		signal?: AbortSignal
-		promptOverrides?: Record<string, string>
-		testHistoryOverrides?: TestHistoryOverrides
-		setProgress?: (progress: MutablePromptTestsProgress) => boolean
-	}
-): Promise<WrittenRunReport> {
-	const signal = opts?.signal
-	const scope = 'scope' in selection ? selection.scope : 'pinned'
-	const onlyTestId = 'testId' in selection ? selection.testId : undefined
-	const suiteKey = suite.suiteKey
-	const loadedSuite = await readSuite(suiteKey)
-	const rows = applyTestHistoryOverrides(loadedSuite.tests, opts?.testHistoryOverrides ?? {})
-
-	const repeats = Math.min(500, Math.max(1, Math.floor(Number(requestedConfig.repeats) || 1)))
-	const concurrency = Math.min(
-		75,
-		Math.max(1, Math.floor(Number(requestedConfig.concurrency) || 75))
-	)
-
-	// Add an index field to each row for mapping
-	const typedRows = rows
-	const indexedRows = typedRows.map((row, index) => ({ ...row, index }))
-
-	if (onlyTestId && !indexedRows.some((row) => String(row.id ?? '') === String(onlyTestId))) {
-		throw new Error(`Test not found: ${onlyTestId}`)
-	}
-
-	const tests = toTestInputs(indexedRows, selection)
-	const plannedTestIds = tests.map((test) => String(test.id ?? '')).filter(Boolean)
-	const progress: MutablePromptTestsProgress = {
-		plannedTestIds,
-		runningTestIds: [],
-		completedTestIds: [],
-		expectedRuns: repeats,
-		startedTestIds: new Set(),
-		completedRunsByTestId: new Map()
-	}
-	if (opts?.setProgress && !opts.setProgress(progress)) throw new Error('Operation cancelled')
-
-	// Provider function with stable label
-	const providerInstance = new RecombineAgentProvider({
-		id: 'recombine-agent'
-	})
-	providerInstance.setAbortSignal(signal)
-	providerInstance.setOnTestStarted((testId) => {
-		if (progress.completedTestIds.includes(testId)) return
-		progress.startedTestIds.add(testId)
-		progress.runningTestIds = [...progress.startedTestIds]
-	})
-	const providerFn = Object.assign(
-		(_promptStr: string, ctx?: CallApiContextParams) => {
-			if (signal?.aborted) {
-				throw new Error('Operation cancelled')
-			}
-			return providerInstance.callApi(_promptStr, ctx)
-		},
-		{ label: 'recombine-agent' }
-	)
-
-	const hookKey = `__bosun_promptfoo_hook_${process.pid}_${Date.now()}_${Math.random().toString(16).slice(2)}`
-	const extensionCode = `globalThis[${JSON.stringify(hookKey)}]?.(output, context)`
-
-	const testSuite = {
-		prompts: ['agent-run'], // placeholder; provider ignores
-		tests: tests.map((test) => ({
-			description: test.scenario,
-			metadata: { testId: test.id },
-			assert: test.assertions,
-			model: 'not used!',
-			vars: {
-				agentId: suite.agentId,
-				testId: String(test.id ?? ''),
-				...(suite.region ? { region: suite.region } : {}),
-				context: test.context,
-				step: test.step,
-				promptOverrides: opts?.promptOverrides ?? {},
-				...(typeof test.mocks !== 'undefined' ? { mocks: test.mocks } : {}),
-				conversation: test.conversation,
-				stringifiedConversation: test.stringifiedConversation,
-				...(test.conversationParseError
-					? { conversationParseError: test.conversationParseError }
-					: {})
-			}
-		})),
-		providers: [providerFn],
-		extensions: [extensionCode]
-	}
-
-	const rootPath = await getRootPath()
-	installTestPromptCacheBust(rootPath)
-	const projectRoot = path.dirname(rootPath)
-	const outDir = path.join(projectRoot, 'data', 'test_runs')
-	await mkdir(outDir, { recursive: true })
-
-	// Use full ISO timestamp and sanitize ':' for filesystem safety
-	const iso = new Date().toISOString()
-	const isoFile = iso.replace(/:/g, '-')
-	const outPath = path.join(outDir, `${suiteKey}.${isoFile}${REPORT_JSON_SUFFIX}`)
-
-	async function writeReport(report: WrittenRunReport) {
-		const dir = path.dirname(outPath)
-		const base = path.basename(outPath)
-		const tmpPath = path.join(
-			dir,
-			`.${base}.${process.pid}.${Date.now()}.${Math.random().toString(16).slice(2)}.tmp`
-		)
-		const json = JSON.stringify(report, null, 2)
-		await writeFile(tmpPath, json, 'utf8')
-		try {
-			await rename(tmpPath, outPath)
-		} catch (e: any) {
-			// Windows doesn't allow atomic rename over an existing file.
-			try {
-				await unlink(outPath)
-			} catch {
-				// ignore
-			}
-			await rename(tmpPath, outPath)
-		}
-	}
-
-	const runConfig: WrittenRunReportConfig = {
-		repeats,
-		concurrency
-	}
-
-	const body: WrittenRunReport = {
-		agent: suiteKey,
-		run_timestamp: iso,
-		run_scope: scope,
-		run_config: runConfig,
-		rows: []
-	}
-
-	// Write an empty report immediately so clients can start polling this run.
-	await writeReport(body)
-
-	const byTestId = new Map<string, WrittenRunReportRow>()
-
-	const originalRowById = new Map<string, SuiteTestRow>()
-	for (const r of typedRows) {
-		const id = String((r as any)?.id ?? '').trim()
-		if (id) originalRowById.set(id, r)
-	}
-
-	const countsByTestId = new Map<string, { pass: number; total: number }>()
-
-	let writeTimer: ReturnType<typeof setTimeout> | null = null
-	let writePending = false
-	let writeChain: Promise<void> = Promise.resolve()
-	const enqueueWrite = () => {
-		writeChain = writeChain.catch(() => {}).then(() => writeReport(body))
-		return writeChain
-	}
-	const requestWrite = () => {
-		writePending = true
-		if (writeTimer) return
-		writeTimer = setTimeout(() => {
-			writeTimer = null
-			if (!writePending) return
-			writePending = false
-			void enqueueWrite()
-		}, 200)
-	}
-
-	;(globalThis as any)[hookKey] = (hookName: string, ctx: any) => {
-		try {
-			if (hookName !== 'afterEach') return
-			if (signal?.aborted) {
-				throw new Error('Operation cancelled')
-			}
-			const testId = String(ctx?.test?.metadata?.testId ?? '')
-			if (!testId) return
-			const r = ctx?.result
-			if (!r) return
-
-			const originalRow = originalRowById.get(testId)
-			if (!originalRow) return
-
-			let reportRow = byTestId.get(testId)
-			if (!reportRow) {
-				const pinned: 'TRUE' | 'FALSE' =
-					String((originalRow as any).pinned || '').toUpperCase() === 'TRUE' ? 'TRUE' : 'FALSE'
-				const nextRow: WrittenRunReportRow = {
-					...(originalRow as any),
-					test: originalRow,
-					prompts: `suite://${suiteKey}?testId=${testId}`,
-					success_rate_percent: 0,
-					success_rate: '0/0',
-					results: [],
-					agent: suiteKey,
-					test_id: testId,
-					run_timestamp: iso,
-					run_config: JSON.stringify(runConfig),
-					pinned_at_run: scope === 'all' || !!onlyTestId ? 'TRUE' : pinned,
-					run_scope: scope
-				}
-				reportRow = nextRow
-				byTestId.set(testId, nextRow)
-				body.rows = Array.from(byTestId.values())
-			}
-			if (!reportRow) return
-
-			const entry = toResultEntry(r)
-			reportRow.results.push(entry)
-			const completedRuns = (progress.completedRunsByTestId.get(testId) ?? 0) + 1
-			progress.completedRunsByTestId.set(testId, completedRuns)
-			if (completedRuns >= progress.expectedRuns) {
-				progress.startedTestIds.delete(testId)
-				progress.runningTestIds = [...progress.startedTestIds]
-				if (!progress.completedTestIds.includes(testId)) {
-					progress.completedTestIds = [...progress.completedTestIds, testId]
-				}
-			}
-
-			const counts = countsByTestId.get(testId) ?? { pass: 0, total: 0 }
-			counts.total += 1
-			if (entry.success) counts.pass += 1
-			countsByTestId.set(testId, counts)
-
-			reportRow.success_rate = `${counts.pass}/${counts.total}`
-			reportRow.success_rate_percent = counts.total
-				? Math.round((counts.pass / counts.total) * 100)
-				: 0
-
-			requestWrite()
-		} catch (err) {
-			if (signal?.aborted) throw err
-			// ignore
-		}
-	}
-
-	const selectedOriginalRows = plannedTestIds
-		.map((testId) => originalRowById.get(testId))
-		.filter((row): row is SuiteTestRow => !!row)
-	const persistEvaluation = (evaluation: PFEval) => {
-		for (const rowToWrite of buildTableRows([evaluation], selectedOriginalRows, suiteKey)) {
-			const testId = String(rowToWrite.id ?? '')
-			const originalRow = originalRowById.get(testId)
-			if (!testId || !originalRow) continue
-			const pinned: 'TRUE' | 'FALSE' =
-				(rowToWrite.pinned || '').toUpperCase() === 'TRUE' ? 'TRUE' : 'FALSE'
-			byTestId.set(testId, {
-				...rowToWrite,
-				test: originalRow,
-				agent: suiteKey,
-				test_id: testId,
-				run_timestamp: iso,
-				run_config: JSON.stringify(runConfig),
-				pinned_at_run: scope === 'all' || !!onlyTestId ? 'TRUE' : pinned,
-				run_scope: scope
-			})
-		}
-		body.rows = Array.from(byTestId.values())
-	}
-
-	const previousIsTest = process.env.IS_TEST
-	process.env.IS_TEST = 'true'
-	try {
-		if (tests.length > 0 && !signal?.aborted) {
-			const evaluation = await promptfoo.evaluate(testSuite, {
-				repeat: repeats,
-				cache: false,
-				maxConcurrency: concurrency,
-				// Bosun reports run progress in the tests UI. Promptfoo's terminal progress bars
-				// continuously rewrite stdout (up to 20 bars at higher concurrency).
-				showProgressBar: false,
-				abortSignal: signal
-			})
-			persistEvaluation(evaluation)
-			await enqueueWrite()
-		}
-	} catch (err) {
-		const cancelled =
-			!!signal?.aborted || (err instanceof Error && /operation cancelled/i.test(err.message || ''))
-		if (!cancelled) {
-			const msg = err instanceof Error ? `${err.name}: ${err.message}` : String(err)
-			const evaluation = {
-				results: plannedTestIds.map(
-					(testId) =>
-						({
-							success: false,
-							response: { output: '' },
-							error: msg,
-							metadata: { testId }
-						}) as any
-				)
-			} as PFEval
-			persistEvaluation(evaluation)
-			await enqueueWrite()
-		}
-	} finally {
-		if (previousIsTest == null) delete process.env.IS_TEST
-		else process.env.IS_TEST = previousIsTest
-		if (writeTimer) clearTimeout(writeTimer)
-		writeTimer = null
-		delete (globalThis as any)[hookKey]
-		await writeChain
-	}
-
-	if (signal?.aborted) {
-		try {
-			await unlink(outPath)
-		} catch {
-			// The report may not have been installed yet if cancellation happened during setup.
-		}
-		return body
-	}
-
-	// Ensure the final report is written even if no tests ran (or an update was skipped).
-	body.rows = Array.from(byTestId.values())
-	await enqueueWrite()
-	if (signal?.aborted) {
-		try {
-			await unlink(outPath)
-		} catch {
-			// Best-effort cancellation cleanup.
-		}
-		return body
-	}
-	void queueReportCompression(outPath).catch((error) => {
-		console.error(`Failed to compress completed test report ${path.basename(outPath)}`, error)
-	})
-
-	return body
-}
-
-export type WrittenRunResult = Awaited<ReturnType<typeof runPromptTests>>