@recombine-ai/bosun 0.3.12 → 0.4.1

package/src/backend/tests/academy.ts

@@ -85,28 +85,37 @@ type PromptTestsScope = 'all' | 'pinned'
 
 type PromptTestsSelection = { scope: PromptTestsScope } | { testId: string }
 
-const activeRunsByAgent = new Map<string, AbortController>()
+const activeRunsBySuiteKey = new Map<string, AbortController>()
 
-export function cancelPromptTests(agent: string): boolean {
-	const controller = activeRunsByAgent.get(agent)
+export type SuiteSelection = {
+	// `suiteKey` is the filesystem/report key (e.g. `voiceSupport.en-gb`).
+	// `agentId` is the canonical factory key (workflow export key, e.g. `voiceSupport`).
+	// Keeping these separate avoids any parsing/guessing in the test runner.
+	suiteKey: string
+	agentId: string
+	region?: string
+}
+
+export function cancelPromptTests(suiteKey: string): boolean {
+	const controller = activeRunsBySuiteKey.get(suiteKey)
 	if (!controller) return false
 	controller.abort()
 	return true
 }
 
 export async function runPromptTestsManaged(
-	agent: string,
+	suite: SuiteSelection,
 	selection: PromptTestsSelection = { scope: 'pinned' }
 ): Promise<WrittenRunReport> {
-	if (activeRunsByAgent.has(agent)) {
+	if (activeRunsBySuiteKey.has(suite.suiteKey)) {
 		throw new Error('Tests already running for this agent')
 	}
 	const controller = new AbortController()
-	activeRunsByAgent.set(agent, controller)
+	activeRunsBySuiteKey.set(suite.suiteKey, controller)
 	try {
-		return await runPromptTests(agent, selection, { signal: controller.signal })
+		return await runPromptTests(suite, selection, { signal: controller.signal })
 	} finally {
-		activeRunsByAgent.delete(agent)
+		activeRunsBySuiteKey.delete(suite.suiteKey)
 	}
 }
 
@@ -134,7 +143,8 @@ function parseConversationFromSheet(conversation: string | undefined | null): Co
 		}
 	}
 
-	for (const line of lines) {
+	for (let i = 0; i < lines.length; i++) {
+		const line = lines[i] ?? ''
 		const m = roleMatch(line)
 		if (m) {
 			// Starting a new role block; flush previous
@@ -146,8 +156,16 @@ function parseConversationFromSheet(conversation: string | undefined | null): Co
 			// Continuation of the current role block (allow blank lines)
 			buffer.push(line)
 		} else {
-			// Ignore lines until we encounter the first role marker
-			continue
+			// No current role block yet.
+			// Previously we silently ignored preamble lines; that's almost always a test authoring mistake
+			// (e.g. a malformed "User:"/"Agent:" prefix like "?Agent:").
+			if (line.trim()) {
+				const snippet = line.length > 200 ? `${line.slice(0, 200)}…` : line
+				throw new Error(
+					`Unparsed conversation content before the first role block at line ${i + 1}: ${snippet}\n` +
+						`Start each block with "User:", "Agent:" (or "Assistant:"), or "System:".`
+				)
+			}
 		}
 	}
 
@@ -170,43 +188,55 @@ function toTestInputs(
 			if ('testId' in selection) return String(r.id ?? '') === String(selection.testId)
 			return selection.scope === 'all' ? true : r.pinned === 'TRUE'
 		})
-		.map((row) => ({
-			id: row.id,
-			scenario: row.scenario,
-			conversation: parseConversationFromSheet(row.conversation),
-			assertions: [
-				{
-					type: row.assertion_type as PFAssertion['type'],
-					value: row.assertion_value
-				},
-				{
-					type: 'similar' as 'similar',
-					value: String(row.expected_output),
-					threshold: Number(row.similarity_threshold) / 100
-				}
-			],
-			context: (() => {
-				const testCtx = makeContext(row.context)
-				return defaultContext ? mergeContexts(defaultContext, testCtx) : testCtx
-			})(),
-			timeoutSeconds: (() => {
-				const rawValue = row.timeout
-				if (rawValue === undefined || rawValue === null || String(rawValue).trim() === '') {
-					throw new Error('Timeout value is required')
-				}
-				const raw = Number(rawValue)
-				if (isNaN(raw)) {
-					throw new Error(`Invalid timeout value: ${rawValue}`)
-				}
-				if (raw > 1000) {
-					throw new Error(`Timeout value exceeds maximum of 1000: ${raw}`)
-				}
-				if (raw < 0) {
-					throw new Error(`Timeout value must be non-negative: ${raw}`)
-				}
-				return raw
-			})()
-		}))
+		.map((row) => {
+			let conversation: Conversation = []
+			let conversationParseError: string | undefined
+			try {
+				conversation = parseConversationFromSheet(row.conversation)
+			} catch (err) {
+				conversation = []
+				conversationParseError = err instanceof Error ? err.message : String(err)
+			}
+
+			return {
+				id: row.id,
+				scenario: row.scenario,
+				conversation,
+				conversationParseError,
+				assertions: [
+					{
+						type: row.assertion_type as PFAssertion['type'],
+						value: row.assertion_value
+					},
+					{
+						type: 'similar' as 'similar',
+						value: String(row.expected_output),
+						threshold: Number(row.similarity_threshold) / 100
+					}
+				],
+				context: (() => {
+					const testCtx = makeContext(row.context)
+					return defaultContext ? mergeContexts(defaultContext, testCtx) : testCtx
+				})(),
+				timeoutSeconds: (() => {
+					const rawValue = row.timeout
+					if (rawValue === undefined || rawValue === null || String(rawValue).trim() === '') {
+						throw new Error('Timeout value is required')
+					}
+					const raw = Number(rawValue)
+					if (isNaN(raw)) {
+						throw new Error(`Invalid timeout value: ${rawValue}`)
+					}
+					if (raw > 1000) {
+						throw new Error(`Timeout value exceeds maximum of 1000: ${raw}`)
+					}
+					if (raw < 0) {
+						throw new Error(`Timeout value must be non-negative: ${raw}`)
+					}
+					return raw
+				})()
+			}
+		})
 }
 
 export class RecombineAgentProvider implements ApiProvider {
@@ -230,12 +260,18 @@ export class RecombineAgentProvider implements ApiProvider {
 	async callApi(_prompt: string, context?: CallApiContextParams): Promise<ProviderResponse> {
 		const projectRoot = path.dirname(await getRootPath())
 		const vars = (context?.vars || {}) as unknown as {
-			agent: string
+			agentId: string
+			region?: string
 			conversation: Conversation
+			conversationParseError?: string
 			context: Context
 			timeoutSeconds?: number
 		}
 
+		if (typeof vars.conversationParseError === 'string' && vars.conversationParseError.trim()) {
+			return { output: '', error: vars.conversationParseError }
+		}
+
 		let captured: string = ''
 		const capturedLogs: string[] = []
 		const capturedStepTraces: StepTrace[] = []
@@ -249,15 +285,6 @@ export class RecombineAgentProvider implements ApiProvider {
 		const conv: Conversation = Array.isArray(vars.conversation)
 			? (vars.conversation as Conversation).map((m) => ({ ...m }))
 			: []
-		const lastUserIndex = (() => {
-			for (let i = conv.length - 1; i >= 0; i--) {
-				if (conv[i]?.sender === 'user') return i
-			}
-			return -1
-		})()
-		if (lastUserIndex < 0) {
-			return { output: '', error: 'No user message found in conversation' }
-		}
 
 		// Prepare a streaming log writer so logs are written as they occur
 		{
@@ -317,7 +344,8 @@ export class RecombineAgentProvider implements ApiProvider {
 		try {
 			// Run the agent via loadAgents using provided conversation
 			await loadAgents({
-				agentId: vars.agent,
+				agentId: vars.agentId,
+				region: vars.region,
 				getMessages: () => conv,
 				sendMessage: async (message: Message | string) => {
 					const m = typeof message === 'string' ? { sender: 'agent', text: message } : message
@@ -513,20 +541,21 @@ function buildTableRows(
 }
 
 export async function runPromptTests(
-	agent: string,
+	suite: SuiteSelection,
 	selection: PromptTestsSelection = { scope: 'pinned' },
 	opts?: { signal?: AbortSignal }
 ): Promise<WrittenRunReport> {
 	const signal = opts?.signal
 	const scope = 'scope' in selection ? selection.scope : 'pinned'
 	const onlyTestId = 'testId' in selection ? selection.testId : undefined
-	const suite = await readSuite(agent)
-	const rows = suite.tests as unknown as CsvRow[]
-	const cfg = (await tryReadConfig(agent)) || {}
+	const suiteKey = suite.suiteKey
+	const loadedSuite = await readSuite(suiteKey)
+	const rows = loadedSuite.tests as unknown as CsvRow[]
+	const cfg = (await tryReadConfig(suiteKey)) || {}
 
 	const repeats = Number(cfg?.repeats ?? 1)
 	const concurrency = Number(cfg?.concurrency ?? 1)
-	const baseContext = suite?.meta?.defaultContext || {}
+	const baseContext = loadedSuite?.meta?.defaultContext || {}
 
 	// Add an index field to each row for mapping
 	const typedRows = rows as unknown as CsvRow[]
@@ -565,9 +594,13 @@ export async function runPromptTests(
 				assert: test.assertions,
 				model: 'not used!',
 				vars: {
-					agent,
+					agentId: suite.agentId,
+					...(suite.region ? { region: suite.region } : {}),
 					context: test.context,
 					conversation: test.conversation,
+					...(test.conversationParseError
+						? { conversationParseError: test.conversationParseError }
+						: {}),
 					timeoutSeconds: (test as any).timeoutSeconds
 				}
 			}
@@ -583,7 +616,7 @@ export async function runPromptTests(
 	// Use full ISO timestamp and sanitize ':' for filesystem safety
 	const iso = new Date().toISOString()
 	const isoFile = iso.replace(/:/g, '-')
-	const outPath = path.join(outDir, `${agent}.${isoFile}.report.json`)
+	const outPath = path.join(outDir, `${suiteKey}.${isoFile}.report.json`)
 
 	async function writeReport(report: WrittenRunReport) {
 		const dir = path.dirname(outPath)
@@ -614,7 +647,7 @@ export async function runPromptTests(
 	}
 
 	const body: WrittenRunReport = {
-		agent,
+		agent: suiteKey,
 		run_timestamp: iso,
 		run_scope: scope,
 		run_config: runConfig,
@@ -672,11 +705,11 @@ export async function runPromptTests(
 					String((originalRow as any).pinned || '').toUpperCase() === 'TRUE' ? 'TRUE' : 'FALSE'
 				const nextRow: WrittenRunReportRow = {
 					...(originalRow as any),
-					prompts: `suite://${agent}?testId=${testId}`,
+					prompts: `suite://${suiteKey}?testId=${testId}`,
 					success_rate_percent: 0,
 					success_rate: '0/0',
 					results: [],
-					agent,
+					agent: suiteKey,
 					test_id: testId,
 					run_timestamp: iso,
 					run_config: JSON.stringify(runConfig),
@@ -724,12 +757,12 @@ export async function runPromptTests(
 				})
 
 				if (originalRow && testId) {
-					const rowToWrite = buildTableRows([ev], [originalRow], agent)[0]
+					const rowToWrite = buildTableRows([ev], [originalRow], suiteKey)[0]
 					const pinned: 'TRUE' | 'FALSE' =
 						(rowToWrite.pinned || '').toUpperCase() === 'TRUE' ? 'TRUE' : 'FALSE'
 					byTestId.set(testId, {
 						...rowToWrite,
-						agent,
+						agent: suiteKey,
 						test_id: testId,
 						run_timestamp: iso,
 						run_config: JSON.stringify(runConfig),
@@ -760,12 +793,12 @@ export async function runPromptTests(
 
 				// Persist failure row immediately for real-time UI updates.
 				if (originalRow && testId) {
-					const rowToWrite = buildTableRows([ev], [originalRow], agent)[0]
+					const rowToWrite = buildTableRows([ev], [originalRow], suiteKey)[0]
 					const pinned: 'TRUE' | 'FALSE' =
 						(rowToWrite.pinned || '').toUpperCase() === 'TRUE' ? 'TRUE' : 'FALSE'
 					byTestId.set(testId, {
 						...rowToWrite,
-						agent,
+						agent: suiteKey,
 						test_id: testId,
 						run_timestamp: iso,
 						run_config: JSON.stringify(runConfig),

package/src/backend/tests/emulate-agents.ts

@@ -7,10 +7,12 @@ import {
 	type TestVoiceAgent
 } from '@recombine-ai/engine'
 import { getAgentFactories } from '../lib/get-agent-factories'
+import { getImportNonce } from '../lib/import-nonce'
 import { getRootPath } from '../lib/get-root-path'
 
 export async function loadAgents(props: {
 	agentId: string
+	region?: string | null
 	getMessages: () => Message[]
 	sendMessage: (message: Message | string) => Promise<void>
 	stepTracer: StepTracer
@@ -21,6 +23,7 @@ export async function loadAgents(props: {
 }): Promise<void> {
 	const {
 		agentId,
+		region,
 		getMessages,
 		sendMessage,
 		stepTracer: stepTracer,
@@ -29,7 +32,8 @@ export async function loadAgents(props: {
 		timeoutSeconds,
 		abortSignal
 	} = props
-	const { factories } = await getAgentFactories(await getRootPath())
+	const importNonce = getImportNonce()
+	const { factories } = await getAgentFactories(await getRootPath(), importNonce)
 	const factory = factories[agentId]
 	if (!factory) throw new Error('Factory not found')
 
@@ -68,6 +72,13 @@ export async function loadAgents(props: {
 		addStep: () => {}
 	}
 
+	if (region) {
+		// The UI-selected region is stored in Bosun's runtime ctx and consumed by the project's workflows.
+		// Setting it before calling the factory ensures region-scoped config/strings are used for this run.
+		const current = (ctx.get() as Record<string, unknown> | null | undefined) || {}
+		ctx.set({ ...current, recombineRegion: region })
+	}
+
 	const agent = await factory({
 		stepTracer: stepTracer,
 		stepRegistry,
@@ -123,6 +134,10 @@ export async function loadAgents(props: {
 	}
 
 	try {
+		if (hasStart(agent)) {
+			// Some project workflows (e.g. voice) persist required per-run state (like region) during `start()`.
+			await withAbort(Promise.resolve().then(() => agent.start()))
+		}
 		if (isVoiceAgent(agent)) {
 			await withAbort(agent.streamResponse())
 		} else {
@@ -150,3 +165,7 @@ export async function loadAgents(props: {
 function isVoiceAgent(agent: TestAgent): agent is TestVoiceAgent {
 	return 'streamResponse' in agent
 }
+
+function hasStart(agent: TestAgent): agent is TestAgent & { start: () => Promise<void> | void } {
+	return typeof (agent as { start?: unknown }).start === 'function'
+}