@recombine-ai/bosun 0.16.0 → 0.16.2

package/src/backend/roleplay/step-registry.ts

@@ -1,11 +1,22 @@
 import type { Logger, StepDef } from '@recombine-ai/engine'
 
-export function createStepRegistry(logger: Logger) {
+export type PromptOverrides = Record<string, string>
+
+export function createStepRegistry(logger: Logger, initialOverrides: PromptOverrides = {}) {
 	const steps: StepDef[] = []
+	let promptOverrides = { ...initialOverrides }
 	return {
 		addStep(def: StepDef) {
+			const originalContent = def.prompt.content.bind(def.prompt)
+			def.prompt.content = async () => {
+				const override = promptOverrides[def.name]
+				return typeof override === 'string' ? override : originalContent()
+			}
 			steps.push(def)
 		},
-		getSteps: () => steps
+		getSteps: () => steps,
+		setPromptOverrides(next: PromptOverrides) {
+			promptOverrides = { ...next }
+		}
 	}
 }

package/src/backend/roleplay/suggest-prompt-edit.ts

@@ -0,0 +1,163 @@
+import OpenAI from 'openai'
+
+import { applyV4ADiff } from '../lib/apply-v4a-diff'
+import type { PromptEditProgressPhase } from './prompt-edit-progress'
+
+const MODEL = 'gpt-5.4'
+const VIRTUAL_PROMPT_PATH = 'prompt.md'
+const MAX_PATCH_ROUNDS = 8
+
+type ApplyPatchCall = {
+	type: 'apply_patch_call'
+	call_id: string
+	operation: {
+		type: 'create_file' | 'update_file' | 'delete_file'
+		path: string
+		diff?: string
+	}
+}
+
+export async function suggestPromptEdit(input: {
+	request: string
+	originalPrompt: string
+	onProgress?: (progress: {
+		phase: PromptEditProgressPhase
+		activity: number
+		summary: string
+	}) => void
+}): Promise<string> {
+	const apiKey = process.env.OPENAI_API_KEY
+	if (!apiKey) throw new Error('OPENAI_API_KEY is not set')
+
+	const client = new OpenAI({ apiKey })
+	let virtualPrompt = input.originalPrompt
+	let activity = 0
+	let modelRound = 0
+	let reasoningSummary = ''
+	const handleEvent = (event: any) => {
+		activity += 1
+		if (
+			event?.type === 'response.reasoning_summary_text.delta' &&
+			typeof event.delta === 'string'
+		) {
+			reasoningSummary += event.delta
+		}
+		input.onProgress?.({
+			phase: progressPhase(event, modelRound),
+			activity,
+			summary: reasoningSummary.trim()
+		})
+	}
+	let response = await createResponse(client, {
+		input: buildPrompt(input.request, input.originalPrompt),
+		onEvent: handleEvent
+	})
+
+	for (let patchRound = 0; patchRound < MAX_PATCH_ROUNDS; patchRound += 1) {
+		const patchCalls = (response.output as unknown as ApplyPatchCall[]).filter(
+			(item) => item?.type === 'apply_patch_call'
+		)
+		if (!patchCalls.length) return virtualPrompt
+
+		const outputs = patchCalls.map((call) => {
+			try {
+				virtualPrompt = applyPromptOperation(virtualPrompt, call.operation)
+				return {
+					type: 'apply_patch_call_output',
+					call_id: call.call_id,
+					status: 'completed',
+					output: `Updated ${VIRTUAL_PROMPT_PATH} in memory`
+				}
+			} catch (error) {
+				return {
+					type: 'apply_patch_call_output',
+					call_id: call.call_id,
+					status: 'failed',
+					output: error instanceof Error ? error.message : String(error)
+				}
+			}
+		})
+
+		reasoningSummary = ''
+		input.onProgress?.({ phase: 'reviewing', activity, summary: '' })
+		modelRound += 1
+		response = await createResponse(client, {
+			previousResponseId: response.id,
+			input: outputs,
+			onEvent: handleEvent
+		})
+	}
+
+	throw new Error('The model did not finish editing the prompt')
+}
+
+function buildPrompt(request: string, originalPrompt: string): string {
+	return `The user is asking you to modify an LLM prompt given below. Try and make minimal changes. You are not allowed to change the shape of variables, their names or their set - this is set outside.
+
+This is their request:
+${request}
+
+This is the original prompt:
+${originalPrompt}
+`
+}
+
+async function createResponse(
+	client: OpenAI,
+	input: { input: unknown; previousResponseId?: string; onEvent: (event: any) => void }
+): Promise<any> {
+	const stream = await (client.responses.create as any)({
+		model: MODEL,
+		instructions:
+			'The original prompt is the existing virtual file prompt.md. Use the Apply Patch tool to make the requested edit to that file. Do not create, delete, or modify any other file.',
+		input: input.input,
+		...(input.previousResponseId ? { previous_response_id: input.previousResponseId } : {}),
+		tools: [{ type: 'apply_patch' }],
+		reasoning: { effort: 'xhigh', summary: 'auto' },
+		text: { verbosity: 'low' },
+		stream: true
+	})
+
+	let completedResponse: any = null
+	for await (const event of stream as AsyncIterable<any>) {
+		input.onEvent(event)
+		if (event?.type === 'response.completed') completedResponse = event.response
+		if (event?.type === 'response.failed') {
+			throw new Error(event.response?.error?.message || 'OpenAI failed to suggest prompt edits')
+		}
+		if (event?.type === 'response.incomplete') {
+			throw new Error('OpenAI returned an incomplete prompt edit')
+		}
+		if (event?.type === 'error') throw new Error(event.message || 'OpenAI streaming failed')
+	}
+
+	if (!completedResponse) throw new Error('OpenAI stream ended before the prompt edit completed')
+	return completedResponse
+}
+
+function progressPhase(event: any, round: number): PromptEditProgressPhase {
+	const eventType = String(event?.type || '')
+	const itemType = String(event?.item?.type || '')
+	if (eventType.includes('apply_patch') || itemType === 'apply_patch_call') return 'drafting'
+	return round > 0 ? 'reviewing' : 'thinking'
+}
+
+function applyPromptOperation(
+	currentPrompt: string,
+	operation: ApplyPatchCall['operation']
+): string {
+	const normalizedPath = String(operation.path || '')
+		.replace(/\\/g, '/')
+		.replace(/^\.\//, '')
+		.replace(/^\//, '')
+	if (normalizedPath !== VIRTUAL_PROMPT_PATH) {
+		throw new Error(`Only ${VIRTUAL_PROMPT_PATH} may be edited`)
+	}
+	if (operation.type !== 'update_file') {
+		throw new Error(`${VIRTUAL_PROMPT_PATH} already exists and may only be updated`)
+	}
+	if (typeof operation.diff !== 'string' || !operation.diff.trim()) {
+		throw new Error('Apply Patch returned an empty diff')
+	}
+	return applyV4ADiff(currentPrompt, operation.diff)
+}

package/src/backend/roleplay/variants.ts

@@ -18,6 +18,7 @@ import type { AgentState } from './agents'
 import { createLogger } from './logger'
 import { createScheduler } from './scheduler'
 import { createStepRegistry } from './step-registry'
+import type { PromptOverrides } from './step-registry'
 
 export type GeneratedVariant = {
 	text: string
@@ -30,8 +31,9 @@ export async function generateReplyVariants(props: {
 	states: AgentState[]
 	count: number
 	context: Record<string, any>
+	promptOverrides?: PromptOverrides
 }): Promise<GeneratedVariant[]> {
-	const { agentId, region, states, count, context } = props
+	const { agentId, region, states, count, context, promptOverrides = {} } = props
 	if (!Number.isFinite(count) || count <= 0) return []
 	const targetState = states.at(-1)
 	if (!targetState) throw new Error('No saved roleplay state found for variants')
@@ -61,7 +63,8 @@ export async function generateReplyVariants(props: {
 				context,
 				prevState,
 				statesById,
-				message
+				message,
+				promptOverrides
 			})
 		)
 	)
@@ -161,8 +164,9 @@ async function generateSingleReplyVariant(props: {
 	prevState: AgentState | null
 	statesById: Map<string, AgentState>
 	message: Message
+	promptOverrides: PromptOverrides
 }): Promise<GeneratedVariant> {
-	const { index, factory, region, context, prevState, statesById, message } = props
+	const { index, factory, region, context, prevState, statesById, message, promptOverrides } = props
 
 	const state = createVariantState(prevState?.id ?? null, index)
 	const scheduleSeed = clone(prevState?.schedule ?? {})
@@ -179,7 +183,7 @@ async function generateSingleReplyVariant(props: {
 		logger,
 		onUpdate: () => {}
 	})
-	const stepRegistry = createStepRegistry(logger)
+	const stepRegistry = createStepRegistry(logger, promptOverrides)
 	const stepTracer: StepTracer = {
 		addStepTrace: (trace: StepTrace) => {
 			state.history.push({

package/src/backend/tests/academy-json.spec.ts

@@ -3,6 +3,38 @@ import { describe, expect, it } from 'vitest'
 import { gradeJsonExpectedFields } from './json-field-match'
 
 describe('gradeJsonExpectedFields', () => {
+	it('compares string values case-insensitively by default', () => {
+		expect(
+			gradeJsonExpectedFields(
+				'{"result":{"outcome":"NO_ANSWER","message":"Call Back Later"}}',
+				'{"result":{"outcome":"no_answer","message":"call back later"}}'
+			)
+		).toMatchObject({ pass: true, score: 1 })
+	})
+
+	it('compares string values case-sensitively when enabled', () => {
+		expect(
+			gradeJsonExpectedFields(
+				'{"result":{"outcome":"NO_ANSWER"}}',
+				'{"result":{"outcome":"no_answer"}}',
+				true
+			)
+		).toMatchObject({
+			pass: false,
+			score: 0,
+			reason: expect.stringContaining('$.result.outcome')
+		})
+	})
+
+	it('always treats object field names as case-sensitive', () => {
+		expect(
+			gradeJsonExpectedFields('{"Result":{"outcome":"NO_ANSWER"}}', '{"result":{}}')
+		).toMatchObject({
+			pass: false,
+			reason: expect.stringContaining('Missing expected field $.result')
+		})
+	})
+
 	it('ignores object key order and formatting', () => {
 		expect(
 			gradeJsonExpectedFields(

package/src/backend/tests/academy.ts

@@ -36,6 +36,7 @@ interface CsvRow {
 	conversation: string
 	expected_output: string
 	evaluation_mode?: string
+	case_sensitive?: boolean
 	context_mode?: string
 	similarity_threshold: number | string
 	pinned: 'TRUE' | 'FALSE'
@@ -139,13 +140,15 @@ export function cancelPromptTests(suiteKey: string): boolean {
 export async function runPromptTestsManaged(
 	suite: SuiteSelection,
 	selection: PromptTestsSelection,
-	requestedConfig: PromptTestsRunConfig
+	requestedConfig: PromptTestsRunConfig,
+	promptOverrides: Record<string, string> = {}
 ): Promise<WrittenRunReport> {
 	const run = activeRuns.start(suite.suiteKey)
 	if (!run) throw new Error('Tests already running for this agent')
 	try {
 		return await runPromptTests(suite, selection, requestedConfig, {
 			signal: run.controller.signal,
+			promptOverrides,
 			setProgress: (progress) => activeRuns.setProgress(suite.suiteKey, run, progress)
 		})
 	} finally {
@@ -242,7 +245,8 @@ function toTestInputs(
 					? [
 							{
 								type: 'javascript',
-								value: (output: string) => gradeJsonExpectedFields(output, row.expected_output)
+								value: (output: string) =>
+									gradeJsonExpectedFields(output, row.expected_output, row.case_sensitive === true)
 							}
 						]
 					: [
@@ -301,6 +305,7 @@ export class RecombineAgentProvider implements ApiProvider {
 			context: Context
 			mocks?: Context
 			step?: string
+			promptOverrides?: Record<string, string>
 		}
 		const testId = String(vars.testId ?? '')
 		if (testId) this.onTestStarted?.(testId)
@@ -413,6 +418,7 @@ export class RecombineAgentProvider implements ApiProvider {
 				logger,
 				context: vars.context || {},
 				mocks: vars.mocks,
+				promptOverrides: vars.promptOverrides,
 				abortSignal: this.abortSignal,
 				...(isMainStepName(targetStep)
 					? {}
@@ -612,6 +618,7 @@ export async function runPromptTests(
 	requestedConfig: PromptTestsRunConfig,
 	opts?: {
 		signal?: AbortSignal
+		promptOverrides?: Record<string, string>
 		setProgress?: (progress: MutablePromptTestsProgress) => boolean
 	}
 ): Promise<WrittenRunReport> {
@@ -684,6 +691,7 @@ export async function runPromptTests(
 				...(suite.region ? { region: suite.region } : {}),
 				context: test.context,
 				step: test.step,
+				promptOverrides: opts?.promptOverrides ?? {},
 				...(typeof test.mocks !== 'undefined' ? { mocks: test.mocks } : {}),
 				conversation: test.conversation,
 				stringifiedConversation: test.stringifiedConversation,

package/src/backend/tests/data.ts

@@ -143,6 +143,7 @@ export async function loadTests(agent: string): Promise<TestTable> {
 			'conversation',
 			'expected_output',
 			'evaluation_mode',
+			'case_sensitive',
 			'context_mode',
 			'timeout',
 			'similarity_threshold',

package/src/backend/tests/emulate-agents.spec.ts

@@ -0,0 +1,106 @@
+import type { TestAgentFactory, TestAgentFactoryProps } from '@recombine-ai/bosun-lib'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const getAgentFactories = vi.hoisted(() => vi.fn())
+
+vi.mock('../lib/get-agent-factories', () => ({ getAgentFactories }))
+vi.mock('../lib/get-import-nonce', () => ({ getImportNonce: () => 1 }))
+vi.mock('../lib/get-root-path', () => ({ getRootPath: async () => '/project/workflows.ts' }))
+
+import { loadAgents } from './emulate-agents'
+
+function textAgent(start: () => Promise<void>) {
+	return {
+		start,
+		respondToMessage: async () => {},
+		reactOnMessage: async () => {},
+		isAssigned: async () => true,
+		onFatalError: async () => {}
+	}
+}
+
+async function run(factory: TestAgentFactory, props: Partial<Parameters<typeof loadAgents>[0]>) {
+	getAgentFactories.mockResolvedValue({ factories: { agent: factory } })
+	await loadAgents({
+		agentId: 'agent',
+		getMessages: () => [{ sender: 'user', text: 'Hello' }],
+		sendMessage: async () => {},
+		stepTracer: { addStepTrace: () => {}, flush: async () => {} },
+		...props
+	})
+}
+
+describe('test agent context lifecycle', () => {
+	beforeEach(() => getAgentFactories.mockReset())
+
+	it('seeds trace context before creating the adapter factory', async () => {
+		let factoryContext: Record<string, unknown> | undefined
+		let lifecycleContext: Record<string, unknown> | undefined
+		const traceContext = {
+			meta: { bookingId: 'trace-booking' },
+			state: { customerName: 'Trace customer' }
+		}
+		const factory: TestAgentFactory = async (props) => {
+			factoryContext = structuredClone(props.ctx.get())
+			props.ctx.set({ meta: { bookingId: 'adapter-default' } })
+			return textAgent(async () => {
+				lifecycleContext = structuredClone(props.ctx.get())
+			})
+		}
+
+		await run(factory, { context: traceContext, region: 'en-gb' })
+
+		expect(factoryContext).toEqual({
+			...traceContext,
+			recombineRegion: 'en-gb'
+		})
+		expect(lifecycleContext).toEqual(factoryContext)
+	})
+
+	it('restores test agent data after factory initialization and retains adapter mocks', async () => {
+		let lifecycleContext: Record<string, unknown> | undefined
+		const adapterMocks = { getJob: { status: 200 } }
+		const traceContext = {
+			state: {
+				jobCard: { 'Mobile or Base booking': 'M' },
+				currentDate: '2026-08-07'
+			}
+		}
+		const factory: TestAgentFactory = async (props: TestAgentFactoryProps<Record<string, any>>) => {
+			props.ctx.set(['state', 'jobCard', 'Mobile or Base booking'], 'adapter-mutated')
+			props.ctx.set({
+				state: { jobCard: {}, currentDate: '2025-10-26' },
+				mocks: adapterMocks
+			})
+			return textAgent(async () => {
+				lifecycleContext = props.ctx.get()
+			})
+		}
+
+		await run(factory, { context: traceContext })
+
+		expect(lifecycleContext).toEqual({ ...traceContext, mocks: adapterMocks })
+	})
+
+	it('keeps explicit test mocks authoritative in split context', async () => {
+		let factoryContext: Record<string, unknown> | undefined
+		let lifecycleContext: Record<string, unknown> | undefined
+		const traceContext = { meta: { bookingId: 'trace-booking' } }
+		const testMocks = { getBooking: { id: 'trace-booking' } }
+		const factory: TestAgentFactory = async (props) => {
+			factoryContext = structuredClone(props.ctx.get())
+			props.ctx.set({
+				context: { meta: { bookingId: 'adapter-default' } },
+				mocks: { getBooking: { id: 'adapter-default' } }
+			})
+			return textAgent(async () => {
+				lifecycleContext = structuredClone(props.ctx.get())
+			})
+		}
+
+		await run(factory, { context: traceContext, mocks: testMocks })
+
+		expect(factoryContext).toEqual({ context: traceContext, mocks: testMocks })
+		expect(lifecycleContext).toEqual(factoryContext)
+	})
+})

package/src/backend/tests/emulate-agents.ts

@@ -21,6 +21,7 @@ export async function loadAgents(props: {
 	logger?: Logger
 	context?: Record<string, unknown>
 	mocks?: Record<string, unknown>
+	promptOverrides?: Record<string, string>
 	abortSignal?: AbortSignal
 	onBeforeTestResponse?: () => void
 	testStep?: {
@@ -38,6 +39,7 @@ export async function loadAgents(props: {
 		logger: providedLogger,
 		context,
 		mocks,
+		promptOverrides = {},
 		abortSignal,
 		onBeforeTestResponse,
 		testStep
@@ -61,7 +63,13 @@ export async function loadAgents(props: {
 
 	const testSteps = new Map<string, RegisteredTestStep>()
 	const stepRegistry: TestStepRegistry = {
-		addStep: () => {},
+		addStep(step) {
+			const originalContent = step.prompt.content.bind(step.prompt)
+			step.prompt.content = async () => {
+				const override = promptOverrides[step.name]
+				return typeof override === 'string' ? override : originalContent()
+			}
+		},
 		addTestStep(step: RegisteredTestStep) {
 			// Step names are the current test identity. Preserve the first registration until
 			// workflow IDs become part of the persisted target.
@@ -74,14 +82,19 @@ export async function loadAgents(props: {
 		mocks !== null &&
 		typeof mocks === 'object' &&
 		!Array.isArray(mocks)
-	const providedContext = context && typeof context === 'object' ? context : {}
-	const runtimeContext = region
+	const providedContext = isRecord(context) ? clone(context) : {}
+	const testContext = region
 		? {
 				...providedContext,
 				recombineRegion: providedContext.recombineRegion ?? region
 			}
 		: providedContext
-	const ctx = createContext(hasSplitContext ? { context: runtimeContext, mocks } : runtimeContext)
+	const testMocks = hasSplitContext ? clone(mocks) : undefined
+	// The factory may mutate nested values in place, so seed it with a separate clone from the
+	// authoritative test snapshot that will be restored before the agent lifecycle starts.
+	const ctx = createContext(
+		hasSplitContext ? { context: clone(testContext), mocks: clone(testMocks) } : clone(testContext)
+	)
 
 	// Maintain in-run history so sendMessage appends to it, mirroring backend behavior
 	let history: Message[] = []
@@ -112,6 +125,23 @@ export async function loadAgents(props: {
 		})
 	)
 
+	// Project factories need the test context while they are being created (for example, to
+	// resolve regional config), but some factories also install their own default context. Keep
+	// only an adapter-provided mocks namespace and make the test case authoritative for agent data.
+	const factoryRuntime = isRecord(ctx.get()) ? ctx.get() : {}
+	if (hasSplitContext) {
+		ctx.swap({ context: clone(testContext), mocks: clone(testMocks) })
+	} else if (isRecord(factoryRuntime.mocks)) {
+		const restoredContext = clone(testContext)
+		ctx.swap(
+			isRecord(factoryRuntime.context)
+				? { context: restoredContext, mocks: factoryRuntime.mocks }
+				: { ...restoredContext, mocks: factoryRuntime.mocks }
+		)
+	} else {
+		ctx.swap(clone(testContext))
+	}
+
 	const withAbort = wrapAbort(abortSignal)
 
 	try {
@@ -170,6 +200,14 @@ function isVoiceAgent(agent: TestAgent): agent is TestVoiceAgent {
 	return 'streamResponse' in agent
 }
 
+function isRecord(value: unknown): value is Record<string, unknown> {
+	return value !== null && typeof value === 'object' && !Array.isArray(value)
+}
+
+function clone<T>(value: T): T {
+	return structuredClone(value)
+}
+
 function wrapAbort(abortSignal: AbortSignal | undefined) {
 	return async <T>(p: Promise<T>): Promise<T> => {
 		if (!abortSignal) return p

package/src/backend/tests/json-field-match.ts

@@ -6,7 +6,12 @@ function isRecord(value: unknown): value is Record<string, unknown> {
 	return typeof value === 'object' && value !== null && !Array.isArray(value)
 }
 
-function matchExpectedJsonFields(actual: unknown, expected: unknown, path = '$'): MatchResult {
+function matchExpectedJsonFields(
+	actual: unknown,
+	expected: unknown,
+	caseSensitive: boolean,
+	path = '$'
+): MatchResult {
 	if (isRecord(expected)) {
 		if (!isRecord(actual)) return { pass: false, reason: `Expected an object at ${path}` }
 
@@ -14,7 +19,12 @@ function matchExpectedJsonFields(actual: unknown, expected: unknown, path = '$')
 			if (!Object.prototype.hasOwnProperty.call(actual, key)) {
 				return { pass: false, reason: `Missing expected field ${path}.${key}` }
 			}
-			const result = matchExpectedJsonFields(actual[key], expectedValue, `${path}.${key}`)
+			const result = matchExpectedJsonFields(
+				actual[key],
+				expectedValue,
+				caseSensitive,
+				`${path}.${key}`
+			)
 			if (!result.pass) return result
 		}
 		return { pass: true }
@@ -29,18 +39,29 @@ function matchExpectedJsonFields(actual: unknown, expected: unknown, path = '$')
 			}
 		}
 		for (let index = 0; index < expected.length; index += 1) {
-			const result = matchExpectedJsonFields(actual[index], expected[index], `${path}[${index}]`)
+			const result = matchExpectedJsonFields(
+				actual[index],
+				expected[index],
+				caseSensitive,
+				`${path}[${index}]`
+			)
 			if (!result.pass) return result
 		}
 		return { pass: true }
 	}
 
-	return isDeepStrictEqual(actual, expected)
-		? { pass: true }
-		: { pass: false, reason: `Value differs at ${path}` }
+	const valuesMatch =
+		typeof actual === 'string' && typeof expected === 'string' && !caseSensitive
+			? actual.toLowerCase() === expected.toLowerCase()
+			: isDeepStrictEqual(actual, expected)
+	return valuesMatch ? { pass: true } : { pass: false, reason: `Value differs at ${path}` }
 }
 
-export function gradeJsonExpectedFields(output: string, expectedOutput: string) {
+export function gradeJsonExpectedFields(
+	output: string,
+	expectedOutput: string,
+	caseSensitive = false
+) {
 	let actual: unknown
 	let expected: unknown
 	try {
@@ -62,7 +83,7 @@ export function gradeJsonExpectedFields(output: string, expectedOutput: string)
 		}
 	}
 
-	const result = matchExpectedJsonFields(actual, expected)
+	const result = matchExpectedJsonFields(actual, expected, caseSensitive)
 	return {
 		pass: result.pass,
 		score: result.pass ? 1 : 0,