@recombine-ai/bosun 0.5.1 → 0.6.0

package/src/backend/roleplay/variants.ts

@@ -0,0 +1,325 @@
+import {
+	createContext,
+	makeAction,
+	type Action,
+	type ApiCallTrace,
+	type ApiCallTracer,
+	type ConversationalTrace,
+	type ConversationalTracer,
+	type Message,
+	type StepTracer,
+	type TestAgent,
+	type TestVoiceAgent,
+	type StepTrace
+} from '@recombine-ai/engine'
+
+import type { LogEntry } from '../../lib/ws-message'
+import { getAgentFactories } from '../lib/get-agent-factories'
+import { getImportNonce } from '../lib/import-nonce'
+import { getRootPath } from '../lib/get-root-path'
+import type { AgentState } from './agents'
+import { createLogger } from './logger'
+import { createScheduler } from './scheduler'
+import { createStepRegistry } from './step-registry'
+
+export type GeneratedVariant = {
+	text: string
+	state: AgentState
+}
+
+export async function generateReplyVariants(props: {
+	agentId: string
+	region?: string | null
+	states: AgentState[]
+	prevStateId: string | null
+	rawUserMessage: string
+	count: number
+	context: Record<string, any>
+}): Promise<GeneratedVariant[]> {
+	const { agentId, region, states, prevStateId, rawUserMessage, count, context } = props
+	if (!Number.isFinite(count) || count <= 0) return []
+
+	const rootPath = await getRootPath()
+	const { factories } = await getAgentFactories(rootPath, getImportNonce())
+	const factory = factories[agentId]
+	if (!factory) throw new Error(`Cannot find agent ${agentId}`)
+
+	const statesById = new Map<string, AgentState>()
+	for (const state of states) statesById.set(state.id, clone(state))
+
+	const prevState = prevStateId ? (statesById.get(prevStateId) ?? null) : null
+	if (prevStateId && !prevState) {
+		throw new Error('Could not find the previous roleplay state for this reply')
+	}
+
+	const message = parseRawUserMessage(rawUserMessage)
+	return Promise.all(
+		Array.from({ length: count }, (_, index) =>
+			generateSingleReplyVariant({
+				index,
+				factory,
+				region,
+				context,
+				prevState,
+				statesById,
+				rawUserMessage,
+				message
+			})
+		)
+	)
+}
+
+export function replaceSavedState(
+	states: AgentState[],
+	targetStateId: string,
+	replacement: Pick<AgentState, 'history' | 'schedule'>
+): AgentState[] {
+	const targetIndex = states.findIndex((state) => state.id === targetStateId)
+	if (targetIndex === -1) throw new Error('Could not find the target state to replace')
+
+	const target = states[targetIndex]
+	const next = states.slice()
+	next[targetIndex] = {
+		...target,
+		history: clone(replacement.history),
+		schedule: clone(replacement.schedule)
+	}
+	return next
+}
+
+function parseRawUserMessage(raw: string): Message {
+	const trimmed = String(raw ?? '').trim()
+	if (!trimmed) throw new Error('Could not reconstruct the user message for variants')
+
+	const imageMatch = /^\[image\]\s+(.+)$/u.exec(trimmed)
+	if (imageMatch) {
+		const imageUrl = (imageMatch[1] ?? '').trim()
+		if (!imageUrl) throw new Error('Image variant message is missing its URL')
+		return { sender: 'user', text: '', imageUrl }
+	}
+
+	return { sender: 'user', text: trimmed }
+}
+
+function collectMessagesUpToState(
+	statesById: Map<string, AgentState>,
+	stateId: string | null
+): Message[] {
+	if (!stateId) return []
+
+	const chain: AgentState[] = []
+	let currentId: string | null = stateId
+
+	while (currentId) {
+		const state = statesById.get(currentId)
+		if (!state) break
+		chain.push(state)
+		currentId = state.prev
+	}
+
+	chain.reverse()
+
+	const messages: Message[] = []
+	for (const state of chain) {
+		messages.push(...collectMessagesFromHistory(state.history))
+	}
+	return messages
+}
+
+function collectMessagesFromHistory(history: AgentState['history']): Message[] {
+	const messages: Message[] = []
+	for (const entry of history) {
+		if (entry.type !== 'message') continue
+		messages.push(entry.payload as Message)
+	}
+	return messages
+}
+
+function extractVariantText(state: AgentState): string {
+	for (let index = state.history.length - 1; index >= 0; index--) {
+		const entry = state.history[index]
+		if (entry.type !== 'conversational-trace') continue
+		const trace = entry.payload as ConversationalTrace
+		if (trace.eventName === 'finished-main-streaming-step' || trace.eventName === 'sent-message') {
+			return String(trace.content ?? '')
+		}
+	}
+
+	for (let index = state.history.length - 1; index >= 0; index--) {
+		const entry = state.history[index]
+		if (entry.type !== 'message') continue
+		const payload = entry.payload as Message
+		if (payload.sender === 'agent') return String(payload.text ?? '')
+	}
+
+	return ''
+}
+
+function createVariantState(prev: string | null, index: number): AgentState {
+	const now = new Date()
+	return {
+		id: `${now.toISOString()}::variant-${index + 1}-${Math.random().toString(36).slice(2, 8)}`,
+		name: `${now.toLocaleString()} (Variant ${index + 1})`,
+		history: [],
+		schedule: {},
+		prev,
+		next: null
+	}
+}
+
+async function generateSingleReplyVariant(props: {
+	index: number
+	factory: Awaited<ReturnType<typeof getAgentFactories>>['factories'][string]
+	region: string | null | undefined
+	context: Record<string, any>
+	prevState: AgentState | null
+	statesById: Map<string, AgentState>
+	rawUserMessage: string
+	message: Message
+}): Promise<GeneratedVariant> {
+	const { index, factory, region, context, prevState, statesById, rawUserMessage, message } = props
+
+	const state = createVariantState(prevState?.id ?? null, index)
+	const scheduleSeed = clone(prevState?.schedule ?? {})
+
+	const log = (entry: LogEntry) => {
+		state.history.push({
+			type: 'log',
+			payload: entry
+		})
+	}
+	const logger = createLogger(log)
+
+	const sendAction = async (action: Action) => {
+		state.history.push({
+			type: 'action',
+			payload: action
+		})
+	}
+
+	const scheduler = createScheduler({
+		logger,
+		sendAction,
+		onUpdate: () => {}
+	})
+	const stepRegistry = createStepRegistry(logger)
+	const stepTracer: StepTracer = {
+		addStepTrace: (trace: StepTrace) => {
+			state.history.push({
+				type: 'trace',
+				payload: trace
+			})
+		},
+		async flush() {}
+	}
+	const apiCallTracer: ApiCallTracer = {
+		addApiCallTrace: (trace: ApiCallTrace) => {
+			state.history.push({
+				type: 'api-call-trace',
+				payload: trace
+			})
+		},
+		async flush() {}
+	}
+	const conversationalTracer: ConversationalTracer = {
+		addConversationalTrace: (trace: ConversationalTrace) => {
+			;(trace as any).bosunStateId = state.id
+			;(trace as any).bosunPrevStateId = state.prev
+			;(trace as any).bosunUserMessage = rawUserMessage
+			state.history.push({
+				type: 'conversational-trace',
+				payload: trace
+			})
+		},
+		async flush() {}
+	}
+
+	const ctx = createContext({})
+	if (region) {
+		const current = (ctx.get() as Record<string, unknown> | null | undefined) || {}
+		ctx.set({ ...current, recombineRegion: region })
+	}
+	if (context && typeof context === 'object') {
+		ctx.swap(region ? { ...context, recombineRegion: region } : context)
+	}
+
+	const getMessages = () => [
+		...collectMessagesUpToState(statesById, state.prev),
+		...collectMessagesFromHistory(state.history)
+	]
+
+	const agent = await factory({
+		stepTracer,
+		tracer: stepRegistry,
+		stepRegistry,
+		logger,
+		apiCallTracer,
+		conversationalTracer,
+		scheduler,
+		getMessages,
+		sendMessage: async (nextMessage: string | Message) => {
+			const payload =
+				typeof nextMessage === 'string'
+					? ({ sender: 'agent', text: nextMessage } as Message)
+					: nextMessage
+
+			state.history.push({
+				type: 'message',
+				payload
+			})
+		},
+		sendAction,
+		ctx
+	})
+
+	scheduler.setState(scheduleSeed)
+
+	if (!state.prev && hasStart(agent)) {
+		await Promise.resolve().then(() => agent.start())
+	}
+
+	state.history.push({
+		type: 'message',
+		payload: message
+	})
+
+	if (isVoiceAgent(agent)) {
+		await agent.streamResponse()
+	} else if (await agent.isAssigned()) {
+		try {
+			const reactAct = makeAction(sendAction, 'Agent', 'React on message')
+			await reactAct('started')
+			await agent.reactOnMessage()
+			await reactAct('completed')
+
+			const respondAct = makeAction(sendAction, 'Agent', 'Compose reply')
+			await respondAct('started')
+			await agent.respondToMessage()
+			await respondAct('completed')
+		} catch (error) {
+			if (typeof (agent as any)?.onFatalError === 'function') {
+				await (agent as any).onFatalError(error)
+			} else {
+				throw error
+			}
+		}
+	}
+
+	state.schedule = clone(scheduler.serialize())
+	return {
+		text: extractVariantText(state),
+		state
+	}
+}
+
+function isVoiceAgent(agent: TestAgent): agent is TestVoiceAgent {
+	return 'streamResponse' in agent
+}
+
+function hasStart(agent: TestAgent): agent is TestAgent & { start: () => Promise<void> | void } {
+	return typeof (agent as { start?: unknown }).start === 'function'
+}
+
+function clone<T>(value: T): T {
+	return JSON.parse(JSON.stringify(value))
+}

package/src/backend/tests/analyze.ts

@@ -10,6 +10,9 @@ const DEFAULT_ANALYZE_SYSTEM_MESSAGE =
 const DEFAULT_ANALYZE_RATIONALE_SYSTEM_MESSAGE =
 	'Stop roleplay. Analyze the last response. Extremely briefly and concisely, explain why you decieded to respond like that.'
 
+const DEFAULT_SUGGESTED_USER_RESPONSE_PROMPT =
+	'Given this conversation:\n\n{{stringifiedConversation}}\n\nPropose a realistic plausible response from the user. Be brief'
+
 export interface QuoteCount {
 	quote: string
 	count: number
@@ -30,6 +33,49 @@ export interface AnalyzeRunResult {
 	raw: Record<string, unknown>
 }
 
+export async function generateSuggestedUserResponse(input: {
+	model: string | undefined
+	stringifiedConversation: string
+	response: string
+}): Promise<{ text: string }> {
+	const apiKey = process.env.OPENAI_API_KEY
+	if (!apiKey) {
+		throw new Error('OPENAI_API_KEY is not set')
+	}
+
+	const conversation = decodeHtmlEntities(String(input.stringifiedConversation ?? ''))
+	if (!conversation) {
+		throw new Error('Missing conversation')
+	}
+
+	const decodedResponse = decodeHtmlEntities(String(input.response ?? ''))
+	if (!decodedResponse.trim()) {
+		throw new Error('Missing response')
+	}
+
+	const resolvedModel = resolveModel(input.model)
+	if (!resolvedModel) {
+		throw new Error('Missing model')
+	}
+
+	const agentLabel = inferAgentLabel(conversation)
+	const fullConversation = appendResponseToConversation(conversation, decodedResponse, agentLabel)
+	const client = new OpenAI({ apiKey })
+
+	const completion = await client.chat.completions.create({
+		model: resolvedModel,
+		messages: [{ role: 'user', content: buildSuggestedUserResponsePrompt(fullConversation) }]
+	})
+
+	const content = completion.choices?.[0]?.message?.content
+	if (!content) throw new Error('No suggested user response from the LLM')
+
+	const text = normalizeSuggestedUserResponse(content)
+	if (!text) throw new Error('Suggested user response was empty')
+
+	return { text }
+}
+
 export async function analyzeTraceQuotes(input: {
 	model: string | undefined
 	renderedPrompt: string
@@ -280,12 +326,27 @@ async function analyzeRationaleOnce(
 	return content
 }
 
+export function buildSuggestedUserResponsePrompt(stringifiedConversation: string): string {
+	return DEFAULT_SUGGESTED_USER_RESPONSE_PROMPT.replace(
+		'{{stringifiedConversation}}',
+		stringifiedConversation
+	)
+}
+
 function normalizeQuote(raw: string): string {
 	let q = decodeHtmlEntities(raw).trim()
 	q = trimWrappingQuotes(q)
 	return q.trim()
 }
 
+export function normalizeSuggestedUserResponse(raw: string): string {
+	let text = decodeHtmlEntities(String(raw ?? ''))
+	text = stripCodeFences(text).trim()
+	text = text.replace(/^user:\s*/i, '')
+	text = trimWrappingQuotes(text)
+	return text.trim()
+}
+
 function normalizeRationale(raw: string): string {
 	const text = decodeHtmlEntities(String(raw ?? '')).trim()
 	return stripCodeFences(text).trim()
@@ -434,7 +495,7 @@ function parseJsonArrayOfStrings(raw: string): string[] {
 
 function stripCodeFences(text: string): string {
 	const trimmed = text.trim()
-	const m = trimmed.match(/^```(?:json)?\s*([\s\S]*?)\s*```$/i)
+	const m = trimmed.match(/^```(?:[a-z0-9_-]+)?\s*([\s\S]*?)\s*```$/i)
 	return m ? m[1] : trimmed
 }
 

package/src/backend/tests/data.ts

@@ -6,6 +6,7 @@ import {
 	readSuite,
 	type TestSuite,
 	upsertSuiteMeta,
+	withSuiteMutationLock,
 	writeSuite,
 	type TestRow as SuiteRow
 } from './suite'
@@ -76,39 +77,41 @@ const defaultTestRow: Omit<TestRow, 'id'> = {
 
 // Utility: ensure each row has a unique id; persist if any are missing/duplicated.
 export async function loadTests(agent: string): Promise<TestTable> {
-	const suite = await readSuite(agent)
-	let changed = false
-	const seenIds = new Set<string>()
-	const rows: TestRow[] = suite.tests.map((r) => {
-		let id = String(r.id || '').trim()
-		if (!id || seenIds.has(id)) {
-			changed = true
-			id = globalThis.crypto?.randomUUID?.() || randomUUID()
+	return withSuiteMutationLock(agent, async () => {
+		const suite = await readSuite(agent)
+		let changed = false
+		const seenIds = new Set<string>()
+		const rows: TestRow[] = suite.tests.map((r) => {
+			let id = String(r.id || '').trim()
+			if (!id || seenIds.has(id)) {
+				changed = true
+				id = globalThis.crypto?.randomUUID?.() || randomUUID()
+			}
+			seenIds.add(id)
+			// previously tests were storing context as a string...
+			const context = typeof r.context === 'string' ? JSON.parse(r.context) : r.context
+			return { ...r, context, id }
+		})
+		if (changed) {
+			await writeSuite(agent, { ...suite, tests: rows })
 		}
-		seenIds.add(id)
-		// previously tests were storing context as a string...
-		const context = typeof r.context === 'string' ? JSON.parse(r.context) : r.context
-		return { ...r, context, id }
+		const headers = [
+			'id',
+			'pinned',
+			'scenario',
+			'step',
+			'author',
+			'category',
+			'assertion_type',
+			'assertion_value',
+			'conversation',
+			'expected_output',
+			'timeout',
+			'similarity_threshold',
+			'context'
+		]
+		return { rows, headers }
 	})
-	if (changed) {
-		await writeSuite(agent, { ...suite, tests: rows })
-	}
-	const headers = [
-		'id',
-		'pinned',
-		'scenario',
-		'step',
-		'author',
-		'category',
-		'assertion_type',
-		'assertion_value',
-		'conversation',
-		'expected_output',
-		'timeout',
-		'similarity_threshold',
-		'context'
-	]
-	return { rows, headers }
 }
 
 // Utility: update a single row by id and write the CSV
@@ -117,14 +120,16 @@ export async function patchTest(
 	id: string,
 	patch: Partial<TestRow>
 ): Promise<TestRow | null> {
-	const suite = await readSuite(agent)
-	const idx = suite.tests.findIndex((r) => r.id === id)
-	if (idx < 0) return null
-	const current = suite.tests[idx]
-	const updated = preparePatch(current, patch)
-	const nextRows = suite.tests.map((r, i) => (i === idx ? updated : r))
-	await writeSuite(agent, { ...suite, tests: nextRows })
-	return updated
+	return withSuiteMutationLock(agent, async () => {
+		const suite = await readSuite(agent)
+		const idx = suite.tests.findIndex((r) => r.id === id)
+		if (idx < 0) return null
+		const current = suite.tests[idx]
+		const updated = preparePatch(current, patch)
+		const nextRows = suite.tests.map((r, i) => (i === idx ? updated : r))
+		await writeSuite(agent, { ...suite, tests: nextRows })
+		return updated
+	})
 }
 
 function preparePatch(current: TestRow, patch: Partial<TestRow>) {
@@ -142,19 +147,21 @@ export async function patchTestsBatch(
 	updates: Array<{ id: string; patch: Partial<TestRow> }>
 ): Promise<TestRow[]> {
 	if (!Array.isArray(updates) || updates.length === 0) return []
-	const suite = await readSuite(agent)
-	const byId = new Map(suite.tests.map((r) => [r.id, r]))
-	const updatedRows: TestRow[] = []
-	for (const { id, patch } of updates) {
-		const current = byId.get(id)
-		if (!current) continue
-		const updated = preparePatch(current, patch)
-		byId.set(id, updated)
-		updatedRows.push(updated)
-	}
-	const nextRows: TestRow[] = suite.tests.map((r) => byId.get(r.id) ?? r)
-	await writeSuite(agent, { ...suite, tests: nextRows })
-	return updatedRows
+	return withSuiteMutationLock(agent, async () => {
+		const suite = await readSuite(agent)
+		const byId = new Map(suite.tests.map((r) => [r.id, r]))
+		const updatedRows: TestRow[] = []
+		for (const { id, patch } of updates) {
+			const current = byId.get(id)
+			if (!current) continue
+			const updated = preparePatch(current, patch)
+			byId.set(id, updated)
+			updatedRows.push(updated)
+		}
+		const nextRows: TestRow[] = suite.tests.map((r) => byId.get(r.id) ?? r)
+		await writeSuite(agent, { ...suite, tests: nextRows })
+		return updatedRows
+	})
 }
 
 // Add a new row at the top
@@ -163,21 +170,23 @@ export async function addTest(
 	mergeContexts: boolean,
 	seed?: Partial<TestRow>
 ): Promise<TestRow> {
-	await ensureConfigExists(agent)
-	const suite = await loadSuiteOrEmpty(agent)
-	const id = globalThis.crypto?.randomUUID?.() || randomUUID()
-
-	const newRow: TestRow = {
-		...defaultTestRow,
-		id,
-		...seed,
-		context: mergeContexts
-			? makeContextDiff(seed?.context, suite.meta?.defaultContext)
-			: makeContext(seed?.context)
-	}
-	await writeSuite(agent, { ...(suite as any), tests: [newRow, ...suite.tests] })
-	await ensureTestExpandedByDefault(agent, newRow.id)
-	return newRow
+	return withSuiteMutationLock(agent, async () => {
+		await ensureConfigExists(agent)
+		const suite = await loadSuiteOrEmpty(agent)
+		const id = globalThis.crypto?.randomUUID?.() || randomUUID()
+
+		const newRow: TestRow = {
+			...defaultTestRow,
+			id,
+			...seed,
+			context: mergeContexts
+				? makeContextDiff(seed?.context, suite.meta?.defaultContext)
+				: makeContext(seed?.context)
+		}
+		await writeSuite(agent, { ...(suite as any), tests: [newRow, ...suite.tests] })
+		await ensureTestExpandedByDefault(agent, newRow.id)
+		return newRow
+	})
 }
 
 async function ensureTestExpandedByDefault(agent: string, id: string): Promise<void> {
@@ -221,27 +230,31 @@ async function loadSuiteOrEmpty(agent: string): Promise<TestSuite> {
 }
 
 export async function deleteTest(agent: string, id: string): Promise<boolean> {
-	const suite = await readSuite(agent)
-	const filtered = suite.tests.filter((r) => r.id !== id)
-	if (filtered.length === suite.tests.length) return false
-	await writeSuite(agent, { ...suite, tests: filtered })
-	return true
+	return withSuiteMutationLock(agent, async () => {
+		const suite = await readSuite(agent)
+		const filtered = suite.tests.filter((r) => r.id !== id)
+		if (filtered.length === suite.tests.length) return false
+		await writeSuite(agent, { ...suite, tests: filtered })
+		return true
+	})
 }
 
 export async function reorderTests(agent: string, order: string[]): Promise<void> {
-	const suite = await readSuite(agent)
-	const map = new Map(suite.tests.map((r) => [r.id, r]))
-	const next: TestRow[] = []
-	for (const id of order) {
-		const r = map.get(id)
-		if (r) {
-			next.push(r)
-			map.delete(id)
+	await withSuiteMutationLock(agent, async () => {
+		const suite = await readSuite(agent)
+		const map = new Map(suite.tests.map((r) => [r.id, r]))
+		const next: TestRow[] = []
+		for (const id of order) {
+			const r = map.get(id)
+			if (r) {
+				next.push(r)
+				map.delete(id)
+			}
 		}
-	}
-	// Append any rows that were not included in order (safety)
-	for (const r of map.values()) next.push(r)
-	await writeSuite(agent, { ...suite, tests: next })
+		// Append any rows that were not included in order (safety)
+		for (const r of map.values()) next.push(r)
+		await writeSuite(agent, { ...suite, tests: next })
+	})
 }
 
 export async function loadConfig(agent: string): Promise<RunConfig> {