@recombine-ai/bosun 0.3.7 → 0.3.9

package/src/backend/tests/academy.ts

@@ -6,7 +6,7 @@ import { loadAgents } from './emulate-agents'
 import { type Assertion as PFAssertion } from 'promptfoo'
 // Backend: per-project JSON test suites (no CSV)
 import { tryReadConfig } from './config'
-import { mkdir, writeFile } from 'node:fs/promises'
+import { mkdir, rename, unlink, writeFile } from 'node:fs/promises'
 import path from 'node:path'
 import { readSuite } from './suite'
 // Use types derived from promptfoo exports instead of incorrect default type imports
@@ -58,6 +58,58 @@ export interface TableRow extends CsvRow {
 	results: ResultEntry[]
 }
 
+export interface WrittenRunReportConfig {
+	repeats: number
+	concurrency: number
+	defaultContext: Record<string, any>
+}
+
+export interface WrittenRunReportRow extends TableRow {
+	agent: string
+	test_id: string
+	run_timestamp: string
+	run_config: string
+	pinned_at_run: 'TRUE' | 'FALSE'
+	run_scope: 'all' | 'pinned'
+}
+
+export interface WrittenRunReport {
+	agent: string
+	run_timestamp: string
+	run_scope: 'all' | 'pinned'
+	run_config: WrittenRunReportConfig
+	rows: WrittenRunReportRow[]
+}
+
+type PromptTestsScope = 'all' | 'pinned'
+
+type PromptTestsSelection = { scope: PromptTestsScope } | { testId: string }
+
+const activeRunsByAgent = new Map<string, AbortController>()
+
+export function cancelPromptTests(agent: string): boolean {
+	const controller = activeRunsByAgent.get(agent)
+	if (!controller) return false
+	controller.abort()
+	return true
+}
+
+export async function runPromptTestsManaged(
+	agent: string,
+	selection: PromptTestsSelection = { scope: 'pinned' }
+): Promise<WrittenRunReport> {
+	if (activeRunsByAgent.has(agent)) {
+		throw new Error('Tests already running for this agent')
+	}
+	const controller = new AbortController()
+	activeRunsByAgent.set(agent, controller)
+	try {
+		return await runPromptTests(agent, selection, { signal: controller.signal })
+	} finally {
+		activeRunsByAgent.delete(agent)
+	}
+}
+
 // Deep merge helper using 'deepmerge' with array replacement semantics
 const mergeContexts = (baseCtx: Record<string, any>, testCtx: Record<string, any>) =>
 	deepmerge(baseCtx || {}, testCtx || {}, { arrayMerge: (_d, s, _o) => s })
@@ -111,10 +163,13 @@ function parseConversationFromSheet(conversation: string | undefined | null): Co
 function toTestInputs(
 	rows: (CsvRow & { index: number })[],
 	defaultContext?: Record<string, any>,
-	scope: 'all' | 'pinned' = 'pinned'
+	selection: PromptTestsSelection = { scope: 'pinned' }
 ) {
 	return rows
-		.filter((r) => (scope === 'all' ? true : r.pinned === 'TRUE'))
+		.filter((r) => {
+			if ('testId' in selection) return String(r.id ?? '') === String(selection.testId)
+			return selection.scope === 'all' ? true : r.pinned === 'TRUE'
+		})
 		.map((row) => ({
 			id: row.id,
 			scenario: row.scenario,
@@ -157,12 +212,17 @@ function toTestInputs(
 export class RecombineAgentProvider implements ApiProvider {
 	protected providerId: string
 	public config: Record<string, unknown>
+	private abortSignal?: AbortSignal
 
 	constructor(options: ProviderOptions) {
 		this.providerId = options.id || 'recombine-agent'
 		this.config = options.config || {}
 	}
 
+	setAbortSignal(signal: AbortSignal | undefined) {
+		this.abortSignal = signal
+	}
+
 	id() {
 		return this.providerId
 	}
@@ -269,7 +329,8 @@ export class RecombineAgentProvider implements ApiProvider {
 				stepTracer,
 				logger,
 				context: vars.context || {},
-				timeoutSeconds: Number(vars.timeoutSeconds)
+				timeoutSeconds: Number(vars.timeoutSeconds),
+				abortSignal: this.abortSignal
 			})
 
 			const logsDir = path.join(projectRoot, 'data', 'logs')
@@ -361,6 +422,62 @@ export class RecombineAgentProvider implements ApiProvider {
 type PFEval = Awaited<ReturnType<typeof promptfoo.evaluate>>
 type PFEvalResult = PFEval['results'][number]
 
+const toStringSafe = (v: unknown): string => {
+	if (typeof v === 'string') return v
+	try {
+		return JSON.stringify(v)
+	} catch {
+		return String(v)
+	}
+}
+
+function toResultEntry(r: any): ResultEntry {
+	const rawTraces = Array.isArray(r.metadata?.rawStepTraces)
+		? r.metadata?.rawStepTraces
+		: Array.isArray(r.metadata?.stepTraces)
+			? r.metadata?.stepTraces
+			: []
+	// Pick the last trace whose name === 'main'
+	const lastMain = (() => {
+		for (let i = rawTraces.length - 1; i >= 0; i--) {
+			const t = rawTraces[i]
+			if (t && t.name === 'main') return t
+		}
+		return undefined
+	})()
+	const rendered_prompt =
+		lastMain && typeof lastMain.renderedPrompt !== 'undefined'
+			? toStringSafe(lastMain.renderedPrompt)
+			: 'NO RENDERED PROMPT'
+	const received_prompt =
+		lastMain && typeof lastMain.receivedPrompt !== 'undefined'
+			? toStringSafe(lastMain.receivedPrompt)
+			: 'NO RECEIVED PROMPT'
+	const stringified_conversation =
+		lastMain && typeof lastMain.stringifiedConversation !== 'undefined'
+			? toStringSafe(lastMain.stringifiedConversation)
+			: 'NO STRINGIFIED CONVERSATION'
+	const received_context =
+		lastMain && typeof lastMain.receivedContext !== 'undefined'
+			? toStringSafe(lastMain.receivedContext)
+			: 'NO RECEIVED CONTEXT'
+
+	const logs = Array.isArray(r.metadata?.logs) ? (r.metadata?.logs as string[]) : []
+	const traces = Array.isArray(r.metadata?.stepTraces) ? r.metadata?.stepTraces : []
+	return {
+		success: !!r.success,
+		result: String(r.response?.output ?? ''),
+		evaluation: String(r.gradingResult?.reason ?? ''),
+		rendered_prompt,
+		received_prompt,
+		stringified_conversation,
+		received_context,
+		traces,
+		errors: String(r.error ?? ''),
+		logs
+	}
+}
+
 function buildTableRows(
 	evals: Array<Awaited<ReturnType<typeof promptfoo.evaluate>>>,
 	csvRows: CsvRow[],
@@ -375,15 +492,6 @@ function buildTableRows(
 		}
 	}
 
-	const toStringSafe = (v: unknown): string => {
-		if (typeof v === 'string') return v
-		try {
-			return JSON.stringify(v)
-		} catch {
-			return String(v)
-		}
-	}
-
 	return csvRows.map((row) => {
 		const rowId = row.id
 		const evalResults = bucket.get(String(rowId)) ?? []
@@ -392,52 +500,7 @@ function buildTableRows(
 		const pct = totalRuns ? Math.round((passCount / totalRuns) * 100) : 0
 		const rateStr = `${passCount}/${totalRuns}`
 
-		const results: ResultEntry[] = evalResults.map((r) => {
-			const rawTraces = Array.isArray(r.metadata?.rawStepTraces)
-				? r.metadata?.rawStepTraces
-				: Array.isArray(r.metadata?.stepTraces)
-					? r.metadata?.stepTraces
-					: []
-			// Pick the last trace whose name === 'main'
-			const lastMain = (() => {
-				for (let i = rawTraces.length - 1; i >= 0; i--) {
-					const t = rawTraces[i]
-					if (t && t.name === 'main') return t
-				}
-				return undefined
-			})()
-			const rendered_prompt =
-				lastMain && typeof lastMain.renderedPrompt !== 'undefined'
-					? toStringSafe(lastMain.renderedPrompt)
-					: 'NO RENDERED PROMPT'
-			const received_prompt =
-				lastMain && typeof lastMain.receivedPrompt !== 'undefined'
-					? toStringSafe(lastMain.receivedPrompt)
-					: 'NO RECEIVED PROMPT'
-			const stringified_conversation =
-				lastMain && typeof lastMain.stringifiedConversation !== 'undefined'
-					? toStringSafe(lastMain.stringifiedConversation)
-					: 'NO STRINGIFIED CONVERSATION'
-			const received_context =
-				lastMain && typeof lastMain.receivedContext !== 'undefined'
-					? toStringSafe(lastMain.receivedContext)
-					: 'NO RECEIVED CONTEXT'
-
-			const logs = Array.isArray(r.metadata?.logs) ? (r.metadata?.logs as string[]) : []
-			const traces = Array.isArray(r.metadata?.stepTraces) ? r.metadata?.stepTraces : []
-			return {
-				success: !!r.success,
-				result: String(r.response?.output ?? ''),
-				evaluation: String(r.gradingResult?.reason ?? ''),
-				rendered_prompt,
-				received_prompt,
-				stringified_conversation,
-				received_context,
-				traces,
-				errors: String(r.error ?? ''),
-				logs
-			}
-		})
+		const results: ResultEntry[] = evalResults.map((r) => toResultEntry(r))
 
 		return {
 			...row,
@@ -449,7 +512,14 @@ function buildTableRows(
 	})
 }
 
-export async function runPromptTests(agent: string, scope: 'all' | 'pinned' = 'pinned') {
+export async function runPromptTests(
+	agent: string,
+	selection: PromptTestsSelection = { scope: 'pinned' },
+	opts?: { signal?: AbortSignal }
+): Promise<WrittenRunReport> {
+	const signal = opts?.signal
+	const scope = 'scope' in selection ? selection.scope : 'pinned'
+	const onlyTestId = 'testId' in selection ? selection.testId : undefined
 	const suite = await readSuite(agent)
 	const rows = suite.tests as unknown as CsvRow[]
 	const cfg = (await tryReadConfig(agent)) || {}
@@ -462,17 +532,30 @@ export async function runPromptTests(agent: string, scope: 'all' | 'pinned' = 'p
 	const typedRows = rows as unknown as CsvRow[]
 	const indexedRows = typedRows.map((row, index) => ({ ...row, index }))
 
-	const tests = toTestInputs(indexedRows, baseContext, scope || 'pinned')
+	if (onlyTestId && !indexedRows.some((row) => String(row.id ?? '') === String(onlyTestId))) {
+		throw new Error(`Test not found: ${onlyTestId}`)
+	}
+
+	const tests = toTestInputs(indexedRows, baseContext, selection)
 
-	// Provider function with stable label
-	const providerInstance = new RecombineAgentProvider({
-		id: 'recombine-agent'
-	})
-	const providerFn = Object.assign(
-		(_promptStr: string, ctx?: CallApiContextParams) => providerInstance.callApi(_promptStr, ctx),
+		// Provider function with stable label
+		const providerInstance = new RecombineAgentProvider({
+			id: 'recombine-agent'
+		})
+		providerInstance.setAbortSignal(signal)
+		const providerFn = Object.assign(
+			(_promptStr: string, ctx?: CallApiContextParams) => {
+				if (signal?.aborted) {
+					throw new Error('Operation cancelled')
+			}
+			return providerInstance.callApi(_promptStr, ctx)
+		},
 		{ label: 'recombine-agent' }
 	)
 
+	const hookKey = `__bosun_promptfoo_hook_${process.pid}_${Date.now()}_${Math.random().toString(16).slice(2)}`
+	const extensionCode = `globalThis[${JSON.stringify(hookKey)}]?.(output, context)`
+
 	const testSuites = tests.map((test) => ({
 		prompts: ['agent-run'], // placeholder; provider ignores
 		scenario: test.scenario,
@@ -487,68 +570,223 @@ export async function runPromptTests(agent: string, scope: 'all' | 'pinned' = 'p
 					conversation: test.conversation,
 					timeoutSeconds: (test as any).timeoutSeconds
 				}
-			}
-		],
-		providers: [providerFn]
+				}
+			],
+		providers: [providerFn],
+		extensions: [extensionCode]
 	}))
 
-	const results: Array<Awaited<ReturnType<typeof promptfoo.evaluate>>> = []
-	for (const suite of testSuites) {
-		try {
-			console.log(`Running: ${suite.scenario}`)
-			const ev = await promptfoo.evaluate(suite, {
-				repeat: repeats,
-				cache: false,
-				maxConcurrency: concurrency
-			})
-			results.push(ev)
-		} catch (err) {
-			const msg = err instanceof Error ? `${err.name}: ${err.message}` : String(err)
-			results.push({
-				results: [
-					{
-						success: false,
-						response: { output: '' },
-						error: msg,
-						metadata: { testId: (suite.tests?.[0] as any)?.metadata?.testId }
-					} as any
-				]
-			} as any)
-		}
-	}
-
-	// Build rows with per-run results, including last-step trace derived fields.
-	const rowsToWrite = buildTableRows(results, typedRows, agent)
-
 	const projectRoot = path.dirname(await getRootPath())
 	const outDir = path.join(projectRoot, 'data', 'test_runs')
 	await mkdir(outDir, { recursive: true })
+
 	// Use full ISO timestamp and sanitize ':' for filesystem safety
 	const iso = new Date().toISOString()
 	const isoFile = iso.replace(/:/g, '-')
 	const outPath = path.join(outDir, `${agent}.${isoFile}.report.json`)
-	// Attach provenance columns to each row before writing
-	const withProvenance = rowsToWrite.map((row, idx) => {
-		const test_id = row.id ?? String(idx)
-		const pinned = (row.pinned || '').toUpperCase() === 'TRUE' ? 'TRUE' : 'FALSE'
-		return {
-			...row,
-			agent,
-			test_id,
-			run_timestamp: iso,
-			run_config: JSON.stringify({ repeats, concurrency, defaultContext: baseContext }),
-			pinned_at_run: scope === 'all' ? 'TRUE' : pinned,
-			run_scope: scope
+
+	async function writeReport(report: WrittenRunReport) {
+		const dir = path.dirname(outPath)
+		const base = path.basename(outPath)
+		const tmpPath = path.join(
+			dir,
+			`.${base}.${process.pid}.${Date.now()}.${Math.random().toString(16).slice(2)}.tmp`
+		)
+		const json = JSON.stringify(report, null, 2)
+		await writeFile(tmpPath, json, 'utf8')
+		try {
+			await rename(tmpPath, outPath)
+		} catch (e: any) {
+			// Windows doesn't allow atomic rename over an existing file.
+			try {
+				await unlink(outPath)
+			} catch {
+				// ignore
+			}
+			await rename(tmpPath, outPath)
 		}
-	})
-	const body = {
+	}
+
+	const runConfig: WrittenRunReportConfig = {
+		repeats,
+		concurrency,
+		defaultContext: baseContext
+	}
+
+	const body: WrittenRunReport = {
 		agent,
 		run_timestamp: iso,
 		run_scope: scope,
-		run_config: { repeats, concurrency, defaultContext: baseContext },
-		rows: withProvenance
+		run_config: runConfig,
+		rows: []
+	}
+
+	// Write an empty report immediately so clients can start polling this run.
+	await writeReport(body)
+
+	const byTestId = new Map<string, WrittenRunReportRow>()
+
+	const originalRowById = new Map<string, CsvRow>()
+	for (const r of typedRows) {
+		const id = String((r as any)?.id ?? '').trim()
+		if (id) originalRowById.set(id, r)
+	}
+
+	const countsByTestId = new Map<string, { pass: number; total: number }>()
+
+	let writeTimer: ReturnType<typeof setTimeout> | null = null
+	let writePending = false
+	let writeChain: Promise<void> = Promise.resolve()
+	const enqueueWrite = () => {
+		writeChain = writeChain
+			.catch(() => {})
+			.then(() => writeReport(body))
+		return writeChain
 	}
-	await writeFile(outPath, JSON.stringify(body, null, 2), 'utf8')
+	const requestWrite = () => {
+		writePending = true
+		if (writeTimer) return
+		writeTimer = setTimeout(() => {
+			writeTimer = null
+			if (!writePending) return
+			writePending = false
+			void enqueueWrite()
+		}, 200)
+	}
+
+	;(globalThis as any)[hookKey] = (hookName: string, ctx: any) => {
+		try {
+			if (hookName !== 'afterEach') return
+			if (signal?.aborted) {
+				throw new Error('Operation cancelled')
+			}
+			const testId = String(ctx?.test?.metadata?.testId ?? '')
+			if (!testId) return
+			const r = ctx?.result
+			if (!r) return
+
+			const originalRow = originalRowById.get(testId)
+			if (!originalRow) return
+
+			let reportRow = byTestId.get(testId)
+			if (!reportRow) {
+				const pinned: 'TRUE' | 'FALSE' =
+					String((originalRow as any).pinned || '').toUpperCase() === 'TRUE' ? 'TRUE' : 'FALSE'
+					const nextRow: WrittenRunReportRow = {
+						...(originalRow as any),
+						prompts: `suite://${agent}?testId=${testId}`,
+						success_rate_percent: 0,
+						success_rate: '0/0',
+						results: [],
+						agent,
+						test_id: testId,
+						run_timestamp: iso,
+						run_config: JSON.stringify(runConfig),
+						pinned_at_run: scope === 'all' || !!onlyTestId ? 'TRUE' : pinned,
+						run_scope: scope
+					}
+				reportRow = nextRow
+				byTestId.set(testId, nextRow)
+				body.rows = Array.from(byTestId.values())
+			}
+			if (!reportRow) return
+
+			const entry = toResultEntry(r)
+			reportRow.results.push(entry)
+
+			const counts = countsByTestId.get(testId) ?? { pass: 0, total: 0 }
+			counts.total += 1
+			if (entry.success) counts.pass += 1
+			countsByTestId.set(testId, counts)
+
+			reportRow.success_rate = `${counts.pass}/${counts.total}`
+			reportRow.success_rate_percent = counts.total ? Math.round((counts.pass / counts.total) * 100) : 0
+
+			requestWrite()
+		} catch (err) {
+			if (signal?.aborted) throw err
+			// ignore
+		}
+	}
+
+	try {
+		for (const suite of testSuites) {
+			if (signal?.aborted) break
+			const testId = String((suite.tests?.[0] as any)?.metadata?.testId ?? '')
+			const originalRow = testId ? typedRows.find((r) => String(r.id) === testId) : undefined
+			try {
+				console.log(`Running: ${suite.scenario}`)
+				const ev = await promptfoo.evaluate(suite, {
+					repeat: repeats,
+					cache: false,
+					maxConcurrency: concurrency,
+					abortSignal: signal
+				})
+
+				if (originalRow && testId) {
+					const rowToWrite = buildTableRows([ev], [originalRow], agent)[0]
+					const pinned: 'TRUE' | 'FALSE' =
+						(rowToWrite.pinned || '').toUpperCase() === 'TRUE' ? 'TRUE' : 'FALSE'
+						byTestId.set(testId, {
+							...rowToWrite,
+							agent,
+							test_id: testId,
+							run_timestamp: iso,
+							run_config: JSON.stringify(runConfig),
+							pinned_at_run: scope === 'all' || !!onlyTestId ? 'TRUE' : pinned,
+							run_scope: scope
+						})
+					body.rows = Array.from(byTestId.values())
+					await enqueueWrite()
+				}
+			} catch (err) {
+				const cancelled =
+					!!signal?.aborted ||
+					(err instanceof Error && /operation cancelled/i.test(err.message || ''))
+				if (cancelled) {
+					break
+				}
+				const msg = err instanceof Error ? `${err.name}: ${err.message}` : String(err)
+				const ev = {
+					results: [
+						{
+							success: false,
+							response: { output: '' },
+							error: msg,
+							metadata: { testId: (suite.tests?.[0] as any)?.metadata?.testId }
+						} as any
+					]
+				} as any
+
+				// Persist failure row immediately for real-time UI updates.
+				if (originalRow && testId) {
+					const rowToWrite = buildTableRows([ev], [originalRow], agent)[0]
+					const pinned: 'TRUE' | 'FALSE' =
+						(rowToWrite.pinned || '').toUpperCase() === 'TRUE' ? 'TRUE' : 'FALSE'
+						byTestId.set(testId, {
+							...rowToWrite,
+							agent,
+							test_id: testId,
+							run_timestamp: iso,
+							run_config: JSON.stringify(runConfig),
+							pinned_at_run: scope === 'all' || !!onlyTestId ? 'TRUE' : pinned,
+							run_scope: scope
+						})
+					body.rows = Array.from(byTestId.values())
+					await enqueueWrite()
+				}
+			}
+		}
+	} finally {
+		if (writeTimer) clearTimeout(writeTimer)
+		writeTimer = null
+		delete (globalThis as any)[hookKey]
+		await writeChain
+	}
+
+	// Ensure the final report is written even if no tests ran (or an update was skipped).
+	body.rows = Array.from(byTestId.values())
+	await enqueueWrite()
 
 	return body
 }