@recombine-ai/bosun 0.3.7 → 0.3.9

package/src/backend/tests/analyze.ts

@@ -0,0 +1,357 @@
+import OpenAI from 'openai'
+import type { StepTrace } from '@recombine-ai/engine'
+
+import { getRunResultEntry } from './data'
+import { decodeHtmlEntities } from '../lib/decode-html-entities'
+
+const DEFAULT_ANALYZE_SYSTEM_MESSAGE =
+	'Stop roleplay. Analyze the last response. Respond with a JSON array of direct quotes from the system prompt that you used to generate it'
+
+export interface QuoteCount {
+	quote: string
+	count: number
+}
+
+export interface AnalyzeRunResult {
+	quotes: QuoteCount[]
+	renderedPrompt: string
+	stringifiedConversation: string
+	raw: Record<string, unknown>
+}
+
+export async function analyzeTraceQuotes(input: {
+	model: string | undefined
+	renderedPrompt: string
+	stringifiedConversation: string
+	response: string
+	stepTrace?: StepTrace
+	rawInput?: Record<string, unknown>
+}): Promise<AnalyzeRunResult> {
+	const apiKey = process.env.OPENAI_API_KEY
+	if (!apiKey) {
+		throw new Error('OPENAI_API_KEY is not set')
+	}
+
+	const analyzeSystemMessage = envTextOrDefault(
+		'BOSUN_ANALYZE_SYSTEM_MESSAGE',
+		DEFAULT_ANALYZE_SYSTEM_MESSAGE,
+	)
+
+	const renderedPrompt = decodeHtmlEntities(String(input.renderedPrompt ?? ''))
+	if (!renderedPrompt) {
+		throw new Error('Missing rendered prompt')
+	}
+
+	const conversation = decodeHtmlEntities(String(input.stringifiedConversation ?? ''))
+	if (!conversation) {
+		throw new Error('Missing conversation')
+	}
+
+	const resolvedModel = resolveModel(input.model)
+	if (!resolvedModel) {
+		throw new Error('Missing model')
+	}
+
+	const agentLabel = inferAgentLabel(conversation)
+	const decodedResponse = decodeHtmlEntities(String(input.response ?? ''))
+	const conversationWithResponse = appendResponseToConversation(conversation, decodedResponse, agentLabel)
+
+	const client = new OpenAI({ apiKey })
+	const calls = Array.from({ length: 10 }, () =>
+		analyzeOnce(client, resolvedModel, renderedPrompt, conversationWithResponse, analyzeSystemMessage),
+	)
+
+	const settled = await Promise.allSettled(calls)
+	const attemptDetails: Array<Record<string, unknown>> = []
+	const counts = new Map<string, number>()
+	let succeeded = 0
+	let failed = 0
+	for (let i = 0; i < settled.length; i++) {
+		const res = settled[i]
+		if (res.status === 'fulfilled') {
+			succeeded += 1
+			const uniqueQuotesThisAttempt = [
+				...new Set(res.value.map((q) => normalizeQuote(String(q))).filter((q) => q.length > 0))
+			]
+			attemptDetails.push({ attempt: i + 1, status: 'ok', quotes: uniqueQuotesThisAttempt })
+			for (const quote of uniqueQuotesThisAttempt) {
+				counts.set(quote, (counts.get(quote) || 0) + 1)
+			}
+			continue
+		}
+		failed += 1
+		attemptDetails.push({ attempt: i + 1, status: 'error', error: toErrorString(res.reason) })
+	}
+
+	if (succeeded === 0) {
+		throw new Error('All analysis calls failed')
+	}
+
+	const quotes = [...counts.entries()]
+		.map(([quote, count]) => ({ quote, count }))
+		.sort((a, b) => b.count - a.count || a.quote.localeCompare(b.quote))
+
+	const sanitizedStepTrace = input.stepTrace ? sanitizeAnalyzedStepTrace(input.stepTrace) : null
+
+	return {
+		quotes,
+		renderedPrompt,
+		stringifiedConversation: conversation,
+		raw: {
+			input: input.rawInput ?? null,
+			calls: {
+				requested: 10,
+				succeeded,
+				failed
+			},
+			stepTrace: sanitizedStepTrace,
+			analysis: {
+				parallelCalls: 10,
+				systemMessage: analyzeSystemMessage,
+				userMessage: {
+					conversationChars: conversation.length,
+					appendedAgentLabel: agentLabel,
+					appendedResponseChars: String(input.response ?? '').length,
+					finalConversationChars: conversationWithResponse.length
+				},
+				systemPrompt: {
+					renderedPromptChars: renderedPrompt.length
+				}
+			},
+			attempts: attemptDetails
+		}
+	}
+}
+
+export async function analyzeRunQuotes(input: {
+	agent: string
+	testId: string
+	index: number
+	timestamp?: string
+}): Promise<AnalyzeRunResult> {
+	const run = await getRunResultEntry(input.agent, input.testId, input.index, input.timestamp)
+
+	const decodedRenderedPrompt = decodeHtmlEntities(String(run.rendered_prompt ?? ''))
+	if (!decodedRenderedPrompt || decodedRenderedPrompt.startsWith('NO RENDERED PROMPT')) {
+		throw new Error('Missing main step rendered prompt for this run')
+	}
+
+	const decodedConversation = decodeHtmlEntities(String(run.stringified_conversation ?? ''))
+	if (!decodedConversation || decodedConversation.startsWith('NO STRINGIFIED CONVERSATION')) {
+		throw new Error('Missing main step conversation for this run')
+	}
+
+	const mainTrace = findMainTrace(run.traces)
+	if (!mainTrace) {
+		throw new Error('Main step trace not found for this run')
+	}
+
+	const resolvedModel = resolveModel(mainTrace.model)
+	if (!resolvedModel) {
+		throw new Error('Missing main step model for this run')
+	}
+
+	return analyzeTraceQuotes({
+		model: resolvedModel,
+		renderedPrompt: decodedRenderedPrompt,
+		stringifiedConversation: decodedConversation,
+		response: String(run.result || ''),
+		stepTrace: mainTrace,
+		rawInput: {
+			agent: input.agent,
+			testId: input.testId,
+			index: input.index,
+			timestamp: input.timestamp ?? null
+		}
+	})
+}
+
+async function analyzeOnce(
+	client: OpenAI,
+	model: string,
+	renderedPrompt: string,
+	conversationWithResponse: string,
+	analyzeSystemMessage: string,
+): Promise<string[]> {
+	const completion = await client.chat.completions.create({
+		model,
+		messages: [
+			{ role: 'system', content: renderedPrompt },
+			{ role: 'user', content: conversationWithResponse },
+			{ role: 'system', content: analyzeSystemMessage },
+		],
+	})
+	const content = completion.choices?.[0]?.message?.content
+	if (!content) throw new Error('No analysis response from the LLM')
+	return parseJsonArrayOfStrings(content)
+}
+
+function normalizeQuote(raw: string): string {
+	let q = decodeHtmlEntities(raw).trim()
+	q = trimWrappingQuotes(q)
+	return q.trim()
+}
+
+function sanitizeAnalyzedStepTrace(trace: StepTrace): Record<string, unknown> {
+	let clone: any
+	try {
+		clone = JSON.parse(JSON.stringify(trace))
+	} catch {
+		clone = { ...(trace as any) }
+	}
+	if (clone && typeof clone === 'object') {
+		if ('renderedPrompt' in clone) delete clone.renderedPrompt
+		if ('receivedPrompt' in clone) delete clone.receivedPrompt
+	}
+	return clone as Record<string, unknown>
+}
+
+function toErrorString(err: unknown): string {
+	if (err instanceof Error) return `${err.name}: ${err.message}`
+	try {
+		return JSON.stringify(err)
+	} catch {
+		return String(err)
+	}
+}
+
+function trimWrappingQuotes(text: string): string {
+	const quotePairs: Array<[string, string]> = [
+		['"', '"'],
+		["'", "'"],
+		['“', '”'],
+		['‘', '’'],
+		['«', '»'],
+		['„', '“']
+	]
+
+	let out = text
+	for (let i = 0; i < 5; i++) {
+		const trimmed = out.trim()
+		let changed = false
+		for (const [open, close] of quotePairs) {
+			if (trimmed.startsWith(open) && trimmed.endsWith(close)) {
+				const next = trimmed.slice(open.length, trimmed.length - close.length)
+				if (next === trimmed) continue
+				out = next
+				changed = true
+				break
+			}
+		}
+		if (!changed) {
+			out = trimmed
+			break
+		}
+	}
+	return out
+}
+
+function findMainTrace(traces: StepTrace[] | undefined): StepTrace | undefined {
+	if (!Array.isArray(traces) || traces.length === 0) return undefined
+	for (let i = traces.length - 1; i >= 0; i--) {
+		const t = traces[i]
+		if (!t) continue
+		if (t.name === 'main' || t.name === 'mainReply') return t
+	}
+	return undefined
+}
+
+function resolveModel(model: string | undefined): string | null {
+	if (!model) return null
+	if (model === 'default') return 'gpt-4o-2024-08-06'
+
+	// Some traces stringify adapter options instead of a bare model name.
+	const trimmed = model.trim()
+	if (trimmed.startsWith('{') && trimmed.endsWith('}')) {
+		try {
+			const parsed = JSON.parse(trimmed) as unknown
+			if (parsed && typeof parsed === 'object' && 'model' in parsed) {
+				const m = (parsed as any).model
+				if (typeof m === 'string' && m.trim().length > 0) return m.trim()
+			}
+		} catch {
+			// ignore
+		}
+	}
+
+	return trimmed.length > 0 ? trimmed : null
+}
+
+function appendResponseToConversation(conversation: string, response: string, agentLabel: string): string {
+	const base = conversation.trimEnd()
+	const prefix = base.length > 0 ? base + '\n' : ''
+	return `${prefix}${agentLabel}: ${response}`
+}
+
+function inferAgentLabel(conversation: string): string {
+	const lines = conversation.split(/\r?\n/)
+	const counts = new Map<string, number>()
+	const seen: string[] = []
+
+	for (const line of lines) {
+		const m = line.match(/^([^:\n]{1,50}):\s/)
+		if (!m) continue
+		const label = m[1]
+		seen.push(label)
+		counts.set(label, (counts.get(label) || 0) + 1)
+	}
+
+	const userLabel = counts.has('User') ? 'User' : seen[0]
+	const sorted = [...counts.entries()].sort((a, b) => b[1] - a[1])
+	for (const [label] of sorted) {
+		if (!label) continue
+		if (label === userLabel) continue
+		if (label.toLowerCase() === 'system') continue
+		if (label.toLowerCase().startsWith('proposed')) continue
+		return label
+	}
+
+	return 'Agent'
+}
+
+function parseJsonArrayOfStrings(raw: string): string[] {
+	const text = stripCodeFences(raw).trim()
+
+	const parsed = tryParseJson(text) ?? tryParseJson(extractJsonArray(text))
+	if (!parsed) {
+		throw new Error('Failed to parse analysis response as JSON array')
+	}
+
+	if (Array.isArray(parsed)) {
+		return parsed.map((v) => String(v))
+	}
+
+	if (parsed && typeof parsed === 'object') {
+		const quotes = (parsed as any).quotes
+		if (Array.isArray(quotes)) return quotes.map((v: unknown) => String(v))
+	}
+
+	throw new Error('Analysis response was not a JSON array')
+}
+
+function stripCodeFences(text: string): string {
+	const trimmed = text.trim()
+	const m = trimmed.match(/^```(?:json)?\s*([\s\S]*?)\s*```$/i)
+	return m ? m[1] : trimmed
+}
+
+function tryParseJson(text: string): unknown | null {
+	try {
+		return JSON.parse(text)
+	} catch {
+		return null
+	}
+}
+
+function extractJsonArray(text: string): string {
+	const start = text.indexOf('[')
+	const end = text.lastIndexOf(']')
+	if (start === -1 || end === -1 || end < start) return text
+	return text.slice(start, end + 1)
+}
+
+function envTextOrDefault(key: string, defaultValue: string): string {
+	const raw = (process.env[key] ?? '').trim()
+	if (!raw) return defaultValue
+	return raw.replace(/\\n/g, '\n')
+}

package/src/backend/tests/data.ts

@@ -39,6 +39,12 @@ export interface RunResult {
 	logs: string[]
 }
 
+export interface PreviousTestResultSummary {
+	success_count: number
+	total_runs: number
+	report_timestamp: string // ISO
+}
+
 export interface TestResult {
 	test_id: string
 	success_rate_percent?: number
@@ -46,6 +52,7 @@ export interface TestResult {
 	results: RunResult[]
 	report_timestamp: string // ISO
 	is_old: boolean
+	previous?: PreviousTestResultSummary
 }
 export interface TestTable {
 	rows: TestRow[]
@@ -89,6 +96,7 @@ export async function loadTests(agent: string): Promise<TestTable> {
 		'id',
 		'pinned',
 		'scenario',
+		'step',
 		'author',
 		'category',
 		'assertion_type',
@@ -273,7 +281,8 @@ export async function loadLatestResultsByTest(agent: string): Promise<Record<str
 		.replace(/-/g, ':')
 
 	const results: Record<string, TestResult> = {}
-	for (const name of files) {
+	for (let i = files.length - 1; i >= 0; i--) {
+		const name = files[i]
 		const full = path.join(outDir, name)
 		const raw = await readFile(full, 'utf8')
 		const parsed = JSON.parse(raw) as any
@@ -288,18 +297,28 @@ export async function loadLatestResultsByTest(agent: string): Promise<Record<str
 			}
 			const par = String((r as any).pinned_at_run ?? 'TRUE').toUpperCase()
 			if (par === 'FALSE') continue
-			// Always prefer newer timestamp
-			const prev = results[test_id]
-			const rec: TestResult = {
-				test_id,
-				success_rate_percent: (r as any).success_rate_percent,
-				success_rate: (r as any).success_rate,
-				results: Array.isArray((r as any).results) ? ((r as any).results as RunResult[]) : [],
-				report_timestamp: ts,
-				is_old: ts < latestTs
+
+			const runResults: RunResult[] = Array.isArray((r as any).results) ? ((r as any).results as RunResult[]) : []
+			const summary: PreviousTestResultSummary = {
+				success_count: runResults.reduce((acc, rr) => acc + (rr?.success ? 1 : 0), 0),
+				total_runs: runResults.length,
+				report_timestamp: ts
+			}
+
+			const current = results[test_id]
+			if (!current) {
+				results[test_id] = {
+					test_id,
+					success_rate_percent: (r as any).success_rate_percent,
+					success_rate: (r as any).success_rate,
+					results: runResults,
+					report_timestamp: ts,
+					is_old: ts < latestTs
+				}
+				continue
 			}
-			if (!prev || prev.report_timestamp < ts) {
-				results[test_id] = rec
+			if (!current.previous && ts < current.report_timestamp) {
+				current.previous = summary
 			}
 		}
 	}
@@ -338,6 +357,23 @@ async function getRun(agent: string, ts?: string) {
 
 type ResultKey = keyof Omit<ResultEntry, 'success'>
 
+export async function getRunResultEntry(
+	agent: string,
+	testId: string,
+	index = 0,
+	ts?: string
+): Promise<ResultEntry> {
+	const result = await getRun(agent, ts)
+
+	const row = result.rows.find((r) => r.test_id === testId)
+	if (!row) throw new Error('Row not found in report')
+
+	const run = row.results[index]
+	if (!run) throw new Error(`Run not found for test ${testId}, index: ${index}`)
+
+	return run
+}
+
 export async function getResultEntry(
 	agent: string,
 	testId: string,

package/src/backend/tests/emulate-agents.ts

@@ -17,6 +17,7 @@ export async function loadAgents(props: {
 	logger?: Logger
 	context?: Record<string, any>
 	timeoutSeconds: number
+	abortSignal?: AbortSignal
 }): Promise<void> {
 	const {
 		agentId,
@@ -25,7 +26,8 @@ export async function loadAgents(props: {
 		stepTracer: stepTracer,
 		logger: providedLogger,
 		context,
-		timeoutSeconds
+		timeoutSeconds,
+		abortSignal
 	} = props
 	const { factories } = await getAgentFactories(await getRootPath())
 	const factory = factories[agentId]
@@ -87,13 +89,46 @@ export async function loadAgents(props: {
 		console.error('UNHANDLED', r)
 	})
 
+	const withAbort = async <T>(p: Promise<T>): Promise<T> => {
+		if (!abortSignal) return p
+		if (abortSignal.aborted) throw new Error('Operation cancelled')
+		return await new Promise<T>((resolve, reject) => {
+			const onAbort = () => {
+				cleanup()
+				reject(new Error('Operation cancelled'))
+			}
+			const cleanup = () => {
+				try {
+					abortSignal.removeEventListener('abort', onAbort)
+				} catch {
+					/* noop */
+				}
+			}
+			try {
+				abortSignal.addEventListener('abort', onAbort, { once: true })
+			} catch {
+				/* noop */
+			}
+			p.then(
+				(v) => {
+					cleanup()
+					resolve(v)
+				},
+				(e) => {
+					cleanup()
+					reject(e)
+				}
+			)
+		})
+	}
+
 	try {
 		if (isVoiceAgent(agent)) {
-			await agent.streamResponse()
+			await withAbort(agent.streamResponse())
 		} else {
-			await agent.respondToMessage()
+			await withAbort(agent.respondToMessage())
 		}
-		await new Promise((resolve) => setTimeout(resolve, timeoutSeconds * 1000))
+		await withAbort(new Promise((resolve) => setTimeout(resolve, timeoutSeconds * 1000)))
 	} catch (err) {
 		const msg = err instanceof Error ? `${err.name}: ${err.message}` : String(err)
 		const stack = err instanceof Error ? err.stack : undefined