@recombine-ai/bosun 0.10.0 → 0.11.1

package/src/backend/roleplay/router.ts

@@ -104,7 +104,7 @@ export function createRolePlayRouter(t: TRPCRootObject<TrpcContext, any, any>) {
 						if (entry.type !== 'message') continue
 						const payload = entry.payload
 						if (!payload || typeof payload !== 'object') continue
-						if (payload.sender !== 'user') continue
+						if (payload.sender !== 'user' && payload.sender !== 'system') continue
 
 						const rawText = payload.text
 						if (typeof rawText !== 'string') continue
@@ -130,8 +130,6 @@ export function createRolePlayRouter(t: TRPCRootObject<TrpcContext, any, any>) {
 				z.object({
 					agentId: AgentIdSchema,
 					region: z.string().trim().min(1).max(128).nullable().optional(),
-					prevStateId: z.string().trim().min(1).max(2048).nullable(),
-					rawUserMessage: z.string().trim().min(1).max(100_000),
 					context: z.record(z.string(), z.unknown()).nullable().optional(),
 					promptUpdate: z
 						.object({
@@ -155,16 +153,13 @@ export function createRolePlayRouter(t: TRPCRootObject<TrpcContext, any, any>) {
 				const rawStates = await fsStatePersistence.load(stateKey)
 				const states = Array.isArray(rawStates) ? rawStates : []
 				const context =
-					input.context && typeof input.context === 'object'
-						? input.context
-						: ((await fsContextPersistence.load('default')) as Record<string, any>) || {}
+					input.context ??
+					(((await fsContextPersistence.load('default')) as Record<string, any> | null) || {})
 
 				const variants = await generateReplyVariants({
 					agentId: input.agentId,
 					region: input.region ?? null,
 					states,
-					prevStateId: input.prevStateId ?? null,
-					rawUserMessage: input.rawUserMessage,
 					count: input.count,
 					context
 				})
@@ -183,7 +178,6 @@ export function createRolePlayRouter(t: TRPCRootObject<TrpcContext, any, any>) {
 				z.object({
 					agentId: AgentIdSchema,
 					region: z.string().trim().min(1).max(128).nullable().optional(),
-					targetStateId: z.string().trim().min(1).max(2048),
 					state: z.object({
 						history: z.unknown(),
 						schedule: z.unknown()
@@ -194,12 +188,10 @@ export function createRolePlayRouter(t: TRPCRootObject<TrpcContext, any, any>) {
 				const stateKey = resolveAgentStateKey(input.agentId, input.region ?? null)
 				const rawStates = await fsStatePersistence.load(stateKey)
 				const states = Array.isArray(rawStates) ? rawStates : []
-				if (!states.length) throw new Error('No saved roleplay states found for this agent')
-				if (states[states.length - 1]?.id !== input.targetStateId) {
-					throw new Error('Variants can only be applied to the latest saved reply')
-				}
+				const latestState = states.at(-1)
+				if (!latestState) throw new Error('No saved roleplay states found for this agent')
 
-				const nextStates = replaceSavedState(states, input.targetStateId, {
+				const nextStates = replaceSavedState(states, latestState.id, {
 					history: input.state.history as AgentState['history'],
 					schedule: input.state.schedule as AgentState['schedule']
 				})

package/src/backend/roleplay/trace-serialization.spec.ts

@@ -0,0 +1,35 @@
+import { describe, expect, it } from 'vitest'
+import { normalizeTracePayload } from './trace-serialization'
+
+describe('normalizeTracePayload', () => {
+	it('keeps error details when trace payloads are JSON serialized', () => {
+		const cause = new Error('schema expected array')
+		const error = new Error('Response parsing error') as Error & {
+			cause?: unknown
+			operation?: string
+		}
+		error.cause = cause
+		error.operation = 'fetchCustomerDetails'
+
+		const normalized = normalizeTracePayload({
+			name: 'request-customer-details',
+			error
+		})
+
+		expect(normalized.error).toMatchObject({
+			name: 'Error',
+			message: 'Response parsing error',
+			operation: 'fetchCustomerDetails',
+			cause: {
+				name: 'Error',
+				message: 'schema expected array'
+			}
+		})
+		expect(JSON.parse(JSON.stringify(normalized)).error).toMatchObject({
+			message: 'Response parsing error',
+			cause: {
+				message: 'schema expected array'
+			}
+		})
+	})
+})

package/src/backend/roleplay/trace-serialization.ts

@@ -0,0 +1,57 @@
+export function normalizeTracePayload<T>(value: T): T {
+	return normalizeValue(value, new WeakMap<object, unknown>()) as T
+}
+
+function normalizeValue(value: unknown, seen: WeakMap<object, unknown>): unknown {
+	if (value == null) return value
+	if (typeof value === 'bigint') return value.toString()
+	if (typeof value !== 'object') return value
+	if (value instanceof Error) return normalizeError(value, seen)
+	if (value instanceof Date) return value.toISOString()
+
+	const existing = seen.get(value)
+	if (existing) return '[Circular]'
+
+	if (Array.isArray(value)) {
+		const next: unknown[] = []
+		seen.set(value, next)
+		for (const item of value) {
+			const normalized = normalizeValue(item, seen)
+			next.push(normalized === undefined ? null : normalized)
+		}
+		return next
+	}
+
+	const next: Record<string, unknown> = {}
+	seen.set(value, next)
+	for (const [key, item] of Object.entries(value)) {
+		next[key] = normalizeValue(item, seen)
+	}
+	return next
+}
+
+function normalizeError(error: Error, seen: WeakMap<object, unknown>): Record<string, unknown> {
+	const existing = seen.get(error)
+	if (existing) return { name: error.name || 'Error', message: '[Circular]' }
+
+	const next: Record<string, unknown> = {
+		name: error.name || 'Error',
+		message: error.message || String(error)
+	}
+	seen.set(error, next)
+
+	if (typeof error.stack === 'string' && error.stack) {
+		next.stack = error.stack
+	}
+
+	if ('cause' in error) {
+		next.cause = normalizeValue((error as Error & { cause?: unknown }).cause, seen)
+	}
+
+	for (const key of Object.getOwnPropertyNames(error)) {
+		if (key in next) continue
+		next[key] = normalizeValue((error as unknown as Record<string, unknown>)[key], seen)
+	}
+
+	return next
+}

package/src/backend/roleplay/variants.ts

@@ -28,28 +28,30 @@ export async function generateReplyVariants(props: {
 	agentId: string
 	region?: string | null
 	states: AgentState[]
-	prevStateId: string | null
-	rawUserMessage: string
 	count: number
 	context: Record<string, any>
 }): Promise<GeneratedVariant[]> {
-	const { agentId, region, states, prevStateId, rawUserMessage, count, context } = props
+	const { agentId, region, states, count, context } = props
 	if (!Number.isFinite(count) || count <= 0) return []
+	const targetState = states.at(-1)
+	if (!targetState) throw new Error('No saved roleplay state found for variants')
 
 	const rootPath = await getRootPath()
 	const { factories } = await getAgentFactories(rootPath, getImportNonce())
 	const factory = factories[agentId]
 	if (!factory) throw new Error(`Cannot find agent ${agentId}`)
 
-	const statesById = new Map<string, AgentState>()
-	for (const state of states) statesById.set(state.id, clone(state))
+	const statesById = new Map(states.map((state) => [state.id, clone(state)]))
 
+	const prevStateId = targetState.prev
 	const prevState = prevStateId ? (statesById.get(prevStateId) ?? null) : null
 	if (prevStateId && !prevState) {
 		throw new Error('Could not find the previous roleplay state for this reply')
 	}
 
-	const message = parseRawUserMessage(rawUserMessage)
+	const message = latestInputMessage(targetState)
+	if (!message) throw new Error('Could not reconstruct the input message for variants')
+
 	return Promise.all(
 		Array.from({ length: count }, (_, index) =>
 			generateSingleReplyVariant({
@@ -59,7 +61,6 @@ export async function generateReplyVariants(props: {
 				context,
 				prevState,
 				statesById,
-				rawUserMessage,
 				message
 			})
 		)
@@ -84,20 +85,6 @@ export function replaceSavedState(
 	return next
 }
 
-function parseRawUserMessage(raw: string): Message {
-	const trimmed = String(raw ?? '').trim()
-	if (!trimmed) throw new Error('Could not reconstruct the user message for variants')
-
-	const imageMatch = /^\[image\]\s+(.+)$/u.exec(trimmed)
-	if (imageMatch) {
-		const imageUrl = (imageMatch[1] ?? '').trim()
-		if (!imageUrl) throw new Error('Image variant message is missing its URL')
-		return { sender: 'user', text: '', imageUrl }
-	}
-
-	return { sender: 'user', text: trimmed }
-}
-
 function collectMessagesUpToState(
 	statesById: Map<string, AgentState>,
 	stateId: string | null
@@ -124,30 +111,32 @@ function collectMessagesUpToState(
 }
 
 function collectMessagesFromHistory(history: AgentState['history']): Message[] {
-	const messages: Message[] = []
-	for (const entry of history) {
-		if (entry.type !== 'message') continue
-		messages.push(entry.payload as Message)
-	}
-	return messages
+	return history.filter((entry) => entry.type === 'message').map((entry) => entry.payload)
+}
+
+function latestInputMessage(state: AgentState): Message | null {
+	const entry = [...state.history]
+		.reverse()
+		.find(
+			(entry) =>
+				entry.type === 'message' &&
+				(entry.payload.sender === 'user' || entry.payload.sender === 'system')
+		)
+	return entry ? clone(entry.payload as Message) : null
 }
 
 function extractVariantText(state: AgentState): string {
-	for (let index = state.history.length - 1; index >= 0; index--) {
-		const entry = state.history[index]
-		if (entry.type !== 'conversational-trace') continue
-		const trace = entry.payload as ConversationalTrace
-		if (trace.eventName === 'finished-main-streaming-step' || trace.eventName === 'sent-message') {
-			return String(trace.content ?? '')
-		}
-	}
+	const traceEntry = [...state.history].reverse().find((entry) => {
+		if (entry.type !== 'conversational-trace') return false
+		const trace = entry.payload
+		return trace.eventName === 'finished-main-streaming-step' || trace.eventName === 'sent-message'
+	})
+	if (traceEntry) return String((traceEntry.payload as ConversationalTrace).content ?? '')
 
-	for (let index = state.history.length - 1; index >= 0; index--) {
-		const entry = state.history[index]
-		if (entry.type !== 'message') continue
-		const payload = entry.payload as Message
-		if (payload.sender === 'agent') return String(payload.text ?? '')
-	}
+	const messageEntry = [...state.history]
+		.reverse()
+		.find((entry) => entry.type === 'message' && entry.payload.sender === 'agent')
+	if (messageEntry) return String((messageEntry.payload as Message).text ?? '')
 
 	return ''
 }
@@ -171,10 +160,9 @@ async function generateSingleReplyVariant(props: {
 	context: Record<string, any>
 	prevState: AgentState | null
 	statesById: Map<string, AgentState>
-	rawUserMessage: string
 	message: Message
 }): Promise<GeneratedVariant> {
-	const { index, factory, region, context, prevState, statesById, rawUserMessage, message } = props
+	const { index, factory, region, context, prevState, statesById, message } = props
 
 	const state = createVariantState(prevState?.id ?? null, index)
 	const scheduleSeed = clone(prevState?.schedule ?? {})
@@ -212,9 +200,6 @@ async function generateSingleReplyVariant(props: {
 	}
 	const conversationalTracer: ConversationalTracer = {
 		addConversationalTrace: (trace: ConversationalTrace) => {
-			;(trace as any).bosunStateId = state.id
-			;(trace as any).bosunPrevStateId = state.prev
-			;(trace as any).bosunUserMessage = rawUserMessage
 			state.history.push({
 				type: 'conversational-trace',
 				payload: trace
@@ -228,9 +213,6 @@ async function generateSingleReplyVariant(props: {
 		const current = (ctx.get() as Record<string, unknown> | null | undefined) || {}
 		ctx.set({ ...current, recombineRegion: region })
 	}
-	if (context && typeof context === 'object') {
-		ctx.swap(region ? { ...context, recombineRegion: region } : context)
-	}
 
 	const getMessages = () => [
 		...collectMessagesUpToState(statesById, state.prev),
@@ -263,6 +245,8 @@ async function generateSingleReplyVariant(props: {
 		}
 	})
 
+	ctx.swap(region ? { ...context, recombineRegion: region } : context)
+
 	scheduler.setState(scheduleSeed)
 
 	if (!state.prev && hasStart(agent)) {

package/src/backend/server.ts

@@ -9,12 +9,16 @@ import { createRegionConfigRouter } from './region/router'
 import { createContext, type TrpcContext } from './lib/trpc-context'
 import { cors } from './lib/middleware/cors'
 
-const WS_PORT = 8080
-const HTTP_PORT = 2022
+const WS_PORT = readPort(['BOSUN_WS_PORT', 'PUBLIC_WS_PORT'], 8080)
+const HTTP_PORT = readPort(['BOSUN_TRPC_PORT', 'PUBLIC_TRPC_PORT'], 2022)
 
 const wss = new WebSocketServer({ port: WS_PORT })
 
-wss.on('connection', oldServer)
+wss.on('connection', (ws) => {
+	void oldServer(ws).catch((error: unknown) => {
+		console.error(error)
+	})
+})
 
 console.log(`WebSocket server started on ws://localhost:${WS_PORT}`)
 
@@ -49,3 +53,18 @@ function shutdown() {
 	server.close()
 	wss.close()
 }
+
+function readPort(names: string[], fallback: number): number {
+	for (const name of names) {
+		const value = process.env[name]
+		if (!value) continue
+
+		const port = Number(value)
+		if (!Number.isInteger(port) || port < 1 || port > 65535) {
+			throw new Error(`Invalid port for ${name}: ${value}`)
+		}
+		return port
+	}
+
+	return fallback
+}

package/src/backend/tests/academy.ts

@@ -8,7 +8,7 @@ import { type Assertion as PFAssertion } from 'promptfoo'
 import { tryReadConfig } from './config'
 import { mkdir, rename, unlink, writeFile } from 'node:fs/promises'
 import path from 'node:path'
-import { readSuite } from './suite'
+import { readSuite, type TestRow as SuiteTestRow } from './suite'
 // Use types derived from promptfoo exports instead of incorrect default type imports
 
 import type { ApiProvider, CallApiContextParams, ProviderResponse } from 'promptfoo'
@@ -30,9 +30,13 @@ interface CsvRow {
 	expected_output: string
 	similarity_threshold: number | string
 	pinned: 'TRUE' | 'FALSE'
-	context: string
+	context: string | Record<string, unknown>
+	mocks?: string | Record<string, unknown>
 	timeout: string
+	capture_mode?: string
 	scenario?: string
+	step?: string
+	author?: string
 }
 
 export interface ResultEntry {
@@ -65,6 +69,7 @@ export interface WrittenRunReportConfig {
 }
 
 export interface WrittenRunReportRow extends TableRow {
+	test: SuiteTestRow
 	agent: string
 	test_id: string
 	run_timestamp: string
@@ -84,6 +89,7 @@ export interface WrittenRunReport {
 type PromptTestsScope = 'all' | 'pinned'
 
 type PromptTestsSelection = { scope: PromptTestsScope } | { testId: string }
+type CaptureMode = 'immediate' | 'final'
 
 const activeRunsBySuiteKey = new Map<string, AbortController>()
 
@@ -209,7 +215,7 @@ function toTestInputs(
 						value: row.assertion_value
 					},
 					{
-						type: 'similar' as 'similar',
+						type: 'similar' as const,
 						value: String(row.expected_output),
 						threshold: Number(row.similarity_threshold) / 100
 					}
@@ -217,7 +223,9 @@ function toTestInputs(
 				context: (() => {
 					const testCtx = makeContext(row.context)
 					return defaultContext ? mergeContexts(defaultContext, testCtx) : testCtx
-				})()
+				})(),
+				captureMode: row.capture_mode === 'final' ? 'final' : 'immediate',
+				mocks: typeof row.mocks === 'undefined' ? undefined : makeContext(row.mocks)
 			}
 		})
 }
@@ -248,13 +256,18 @@ export class RecombineAgentProvider implements ApiProvider {
 			conversation: Conversation
 			conversationParseError?: string
 			context: Context
+			mocks?: Context
+			captureMode?: CaptureMode | string
 		}
 
 		if (typeof vars.conversationParseError === 'string' && vars.conversationParseError.trim()) {
 			return { output: '', error: vars.conversationParseError }
 		}
 
+		const captureMode: CaptureMode = vars.captureMode === 'final' ? 'final' : 'immediate'
 		let captured: string = ''
+		let captureEnabled = false
+		const finalCaptureErrors: string[] = []
 		const capturedLogs: string[] = []
 		const capturedStepTraces: StepTrace[] = []
 
@@ -265,7 +278,7 @@ export class RecombineAgentProvider implements ApiProvider {
 		// Clone the conversation so parallel runs don't share the same array reference.
 		// Shallow-clone each message; messages are simple { sender, text } objects in this flow.
 		const conv: Conversation = Array.isArray(vars.conversation)
-			? (vars.conversation as Conversation).map((m) => ({ ...m }))
+			? vars.conversation.map((m) => ({ ...m }))
 			: []
 
 		// Prepare a streaming log writer so logs are written as they occur
@@ -307,6 +320,9 @@ export class RecombineAgentProvider implements ApiProvider {
 					line = line.slice(0, 5000) + '... [TRUNCATED]'
 				}
 				capturedLogs.push(line)
+				if (captureMode === 'final' && captureEnabled && level === 'error') {
+					finalCaptureErrors.push(line)
+				}
 				try {
 					logStream.write(line + '\n')
 				} catch {
@@ -331,14 +347,24 @@ export class RecombineAgentProvider implements ApiProvider {
 				getMessages: () => conv,
 				sendMessage: async (message: Message | string) => {
 					const m = typeof message === 'string' ? { sender: 'agent', text: message } : message
-					if (m.sender === 'agent') {
-						captured = typeof m.text === 'string' ? m.text : JSON.stringify(m)
-						logger.log('Agent message captured:', captured)
+					if (captureEnabled && m.sender === 'agent') {
+						const text = typeof m.text === 'string' ? m.text : JSON.stringify(m)
+						if (captureMode === 'final' || !captured) {
+							captured = text
+							logger.log('Agent message captured:', captured)
+						} else {
+							logger.log('Agent message ignored after immediate capture:', text)
+						}
 					}
 				},
+				onBeforeTestResponse: () => {
+					captured = ''
+					captureEnabled = true
+				},
 				stepTracer,
 				logger,
 				context: vars.context || {},
+				mocks: vars.mocks,
 				abortSignal: this.abortSignal
 			})
 
@@ -378,6 +404,19 @@ export class RecombineAgentProvider implements ApiProvider {
 				? capturedStepTraces.map(sanitizeTrace)
 				: []
 
+			if (captureMode === 'final' && finalCaptureErrors.length > 0) {
+				const detail = finalCaptureErrors.at(-1) ?? 'Unknown final capture error'
+				return {
+					output: captured,
+					error: `Final capture failed because a subsequent step emitted an error: ${detail}`,
+					metadata: {
+						logs: capturedLogs,
+						stepTraces: sanitizedStepTraces,
+						rawStepTraces: capturedStepTraces
+					}
+				}
+			}
+
 			if (captured && captured.trim()) {
 				return {
 					output: captured,
@@ -531,7 +570,7 @@ export async function runPromptTests(
 	const onlyTestId = 'testId' in selection ? selection.testId : undefined
 	const suiteKey = suite.suiteKey
 	const loadedSuite = await readSuite(suiteKey)
-	const rows = loadedSuite.tests as unknown as CsvRow[]
+	const rows = loadedSuite.tests
 	const cfg = (await tryReadConfig(suiteKey)) || {}
 
 	const repeats = Number(cfg?.repeats ?? 1)
@@ -539,7 +578,7 @@ export async function runPromptTests(
 	const baseContext = loadedSuite?.meta?.defaultContext || {}
 
 	// Add an index field to each row for mapping
-	const typedRows = rows as unknown as CsvRow[]
+	const typedRows = rows
 	const indexedRows = typedRows.map((row, index) => ({ ...row, index }))
 
 	if (onlyTestId && !indexedRows.some((row) => String(row.id ?? '') === String(onlyTestId))) {
@@ -578,6 +617,8 @@ export async function runPromptTests(
 					agentId: suite.agentId,
 					...(suite.region ? { region: suite.region } : {}),
 					context: test.context,
+					...(typeof test.mocks !== 'undefined' ? { mocks: test.mocks } : {}),
+					captureMode: test.captureMode,
 					conversation: test.conversation,
 					...(test.conversationParseError
 						? { conversationParseError: test.conversationParseError }
@@ -639,7 +680,7 @@ export async function runPromptTests(
 
 	const byTestId = new Map<string, WrittenRunReportRow>()
 
-	const originalRowById = new Map<string, CsvRow>()
+	const originalRowById = new Map<string, SuiteTestRow>()
 	for (const r of typedRows) {
 		const id = String((r as any)?.id ?? '').trim()
 		if (id) originalRowById.set(id, r)
@@ -685,6 +726,7 @@ export async function runPromptTests(
 					String((originalRow as any).pinned || '').toUpperCase() === 'TRUE' ? 'TRUE' : 'FALSE'
 				const nextRow: WrittenRunReportRow = {
 					...(originalRow as any),
+					test: originalRow,
 					prompts: `suite://${suiteKey}?testId=${testId}`,
 					success_rate_percent: 0,
 					success_rate: '0/0',
@@ -742,6 +784,7 @@ export async function runPromptTests(
 						(rowToWrite.pinned || '').toUpperCase() === 'TRUE' ? 'TRUE' : 'FALSE'
 					byTestId.set(testId, {
 						...rowToWrite,
+						test: originalRow,
 						agent: suiteKey,
 						test_id: testId,
 						run_timestamp: iso,
@@ -778,6 +821,7 @@ export async function runPromptTests(
 						(rowToWrite.pinned || '').toUpperCase() === 'TRUE' ? 'TRUE' : 'FALSE'
 					byTestId.set(testId, {
 						...rowToWrite,
+						test: originalRow,
 						agent: suiteKey,
 						test_id: testId,
 						run_timestamp: iso,