@recombine-ai/bosun 0.16.2 → 0.17.1

package/src/backend/roleplay/suggest-prompt-edit.ts

@@ -1,21 +1,7 @@
-import OpenAI from 'openai'
-
-import { applyV4ADiff } from '../lib/apply-v4a-diff'
 import type { PromptEditProgressPhase } from './prompt-edit-progress'
+import { suggestFileEdit } from './suggest-file-edit'
 
-const MODEL = 'gpt-5.4'
 const VIRTUAL_PROMPT_PATH = 'prompt.md'
-const MAX_PATCH_ROUNDS = 8
-
-type ApplyPatchCall = {
-	type: 'apply_patch_call'
-	call_id: string
-	operation: {
-		type: 'create_file' | 'update_file' | 'delete_file'
-		path: string
-		diff?: string
-	}
-}
 
 export async function suggestPromptEdit(input: {
 	request: string
@@ -26,73 +12,18 @@ export async function suggestPromptEdit(input: {
 		summary: string
 	}) => void
 }): Promise<string> {
-	const apiKey = process.env.OPENAI_API_KEY
-	if (!apiKey) throw new Error('OPENAI_API_KEY is not set')
-
-	const client = new OpenAI({ apiKey })
-	let virtualPrompt = input.originalPrompt
-	let activity = 0
-	let modelRound = 0
-	let reasoningSummary = ''
-	const handleEvent = (event: any) => {
-		activity += 1
-		if (
-			event?.type === 'response.reasoning_summary_text.delta' &&
-			typeof event.delta === 'string'
-		) {
-			reasoningSummary += event.delta
-		}
-		input.onProgress?.({
-			phase: progressPhase(event, modelRound),
-			activity,
-			summary: reasoningSummary.trim()
-		})
-	}
-	let response = await createResponse(client, {
-		input: buildPrompt(input.request, input.originalPrompt),
-		onEvent: handleEvent
+	return suggestFileEdit({
+		request: buildPrompt(input.request, input.originalPrompt),
+		originalContent: input.originalPrompt,
+		virtualPath: VIRTUAL_PROMPT_PATH,
+		instructions:
+			'The original prompt is the existing virtual file prompt.md. Use the Apply Patch tool to make the requested edit to that file. Do not create, delete, or modify any other file.',
+		onProgress: input.onProgress
 	})
-
-	for (let patchRound = 0; patchRound < MAX_PATCH_ROUNDS; patchRound += 1) {
-		const patchCalls = (response.output as unknown as ApplyPatchCall[]).filter(
-			(item) => item?.type === 'apply_patch_call'
-		)
-		if (!patchCalls.length) return virtualPrompt
-
-		const outputs = patchCalls.map((call) => {
-			try {
-				virtualPrompt = applyPromptOperation(virtualPrompt, call.operation)
-				return {
-					type: 'apply_patch_call_output',
-					call_id: call.call_id,
-					status: 'completed',
-					output: `Updated ${VIRTUAL_PROMPT_PATH} in memory`
-				}
-			} catch (error) {
-				return {
-					type: 'apply_patch_call_output',
-					call_id: call.call_id,
-					status: 'failed',
-					output: error instanceof Error ? error.message : String(error)
-				}
-			}
-		})
-
-		reasoningSummary = ''
-		input.onProgress?.({ phase: 'reviewing', activity, summary: '' })
-		modelRound += 1
-		response = await createResponse(client, {
-			previousResponseId: response.id,
-			input: outputs,
-			onEvent: handleEvent
-		})
-	}
-
-	throw new Error('The model did not finish editing the prompt')
 }
 
 function buildPrompt(request: string, originalPrompt: string): string {
-	return `The user is asking you to modify an LLM prompt given below. Try and make minimal changes. You are not allowed to change the shape of variables, their names or their set - this is set outside.
+	return `The user is asking you to modify an LLM prompt given below. Try and make minimal changes. You are not allowed to change the shape of variables, their names or their set - this is set outside. Note that this is a Nunjucks template and it will be rendered according to context. You can change/append the template logic if necessary.
 
 This is their request:
 ${request}
@@ -101,63 +32,3 @@ This is the original prompt:
 ${originalPrompt}
 `
 }
-
-async function createResponse(
-	client: OpenAI,
-	input: { input: unknown; previousResponseId?: string; onEvent: (event: any) => void }
-): Promise<any> {
-	const stream = await (client.responses.create as any)({
-		model: MODEL,
-		instructions:
-			'The original prompt is the existing virtual file prompt.md. Use the Apply Patch tool to make the requested edit to that file. Do not create, delete, or modify any other file.',
-		input: input.input,
-		...(input.previousResponseId ? { previous_response_id: input.previousResponseId } : {}),
-		tools: [{ type: 'apply_patch' }],
-		reasoning: { effort: 'xhigh', summary: 'auto' },
-		text: { verbosity: 'low' },
-		stream: true
-	})
-
-	let completedResponse: any = null
-	for await (const event of stream as AsyncIterable<any>) {
-		input.onEvent(event)
-		if (event?.type === 'response.completed') completedResponse = event.response
-		if (event?.type === 'response.failed') {
-			throw new Error(event.response?.error?.message || 'OpenAI failed to suggest prompt edits')
-		}
-		if (event?.type === 'response.incomplete') {
-			throw new Error('OpenAI returned an incomplete prompt edit')
-		}
-		if (event?.type === 'error') throw new Error(event.message || 'OpenAI streaming failed')
-	}
-
-	if (!completedResponse) throw new Error('OpenAI stream ended before the prompt edit completed')
-	return completedResponse
-}
-
-function progressPhase(event: any, round: number): PromptEditProgressPhase {
-	const eventType = String(event?.type || '')
-	const itemType = String(event?.item?.type || '')
-	if (eventType.includes('apply_patch') || itemType === 'apply_patch_call') return 'drafting'
-	return round > 0 ? 'reviewing' : 'thinking'
-}
-
-function applyPromptOperation(
-	currentPrompt: string,
-	operation: ApplyPatchCall['operation']
-): string {
-	const normalizedPath = String(operation.path || '')
-		.replace(/\\/g, '/')
-		.replace(/^\.\//, '')
-		.replace(/^\//, '')
-	if (normalizedPath !== VIRTUAL_PROMPT_PATH) {
-		throw new Error(`Only ${VIRTUAL_PROMPT_PATH} may be edited`)
-	}
-	if (operation.type !== 'update_file') {
-		throw new Error(`${VIRTUAL_PROMPT_PATH} already exists and may only be updated`)
-	}
-	if (typeof operation.diff !== 'string' || !operation.diff.trim()) {
-		throw new Error('Apply Patch returned an empty diff')
-	}
-	return applyV4ADiff(currentPrompt, operation.diff)
-}

package/src/backend/roleplay/suggest-test-fix.spec.ts

@@ -0,0 +1,50 @@
+import { describe, expect, it, vi } from 'vitest'
+
+import { buildTestFixRequest } from '../../lib/edit/test-ai-edit'
+import { suggestTestFix } from './suggest-test-fix'
+
+describe('suggestTestFix', () => {
+	it('uses the corrected history before suggesting a prompt edit', async () => {
+		const originalConversation = 'User: original'
+		const correctedConversation = 'User: corrected'
+		let finishHistory!: (conversation: string) => void
+		const historyResult = new Promise<string>((resolve) => {
+			finishHistory = resolve
+		})
+		const suggestHistory = vi.fn(() => historyResult)
+		const suggestPrompt = vi.fn(
+			async (_input: { request: string; originalPrompt: string }) => 'Updated prompt'
+		)
+		const requestInput = {
+			context: '{}',
+			conversation: originalConversation,
+			expectedOutput: 'Agent: expected',
+			assertionLabel: 'LLM',
+			assertionValue: 'Matches the expected behaviour',
+			currentIncorrectOutcome: 'Agent: wrong'
+		}
+
+		const pending = suggestTestFix(
+			{
+				...requestInput,
+				request: buildTestFixRequest(requestInput),
+				originalPrompt: 'Original prompt'
+			},
+			{ suggestHistory, suggestPrompt }
+		)
+
+		expect(suggestHistory).toHaveBeenCalledOnce()
+		expect(suggestPrompt).not.toHaveBeenCalled()
+
+		finishHistory(correctedConversation)
+		await expect(pending).resolves.toEqual({
+			suggestedConversation: correctedConversation,
+			suggestedPrompt: 'Updated prompt'
+		})
+
+		expect(suggestPrompt).toHaveBeenCalledOnce()
+		const promptInput = suggestPrompt.mock.calls[0][0]
+		expect(promptInput.request).toContain(`Conversation:\n${correctedConversation}`)
+		expect(promptInput.request).not.toContain(`Conversation:\n${originalConversation}`)
+	})
+})

package/src/backend/roleplay/suggest-test-fix.ts

@@ -0,0 +1,65 @@
+import { buildPromptFixRequest } from '../../lib/edit/test-ai-edit'
+import { suggestTestHistoryEdit } from '../tests/suggest-test-history-edit'
+import type { PromptEditProgressPhase } from './prompt-edit-progress'
+import { suggestPromptEdit } from './suggest-prompt-edit'
+
+type Progress = {
+	phase: PromptEditProgressPhase
+	activity: number
+	summary: string
+}
+
+interface SuggestTestFixInput {
+	request: string
+	originalPrompt: string
+	context: string
+	conversation: string
+	expectedOutput: string
+	assertionLabel: string
+	assertionValue: string
+	currentIncorrectOutcome: string
+	onProgress?: (progress: Progress) => void
+}
+
+interface SuggestTestFixDependencies {
+	suggestHistory: typeof suggestTestHistoryEdit
+	suggestPrompt: typeof suggestPromptEdit
+}
+
+const defaultDependencies: SuggestTestFixDependencies = {
+	suggestHistory: suggestTestHistoryEdit,
+	suggestPrompt: suggestPromptEdit
+}
+
+export async function suggestTestFix(
+	input: SuggestTestFixInput,
+	dependencies: SuggestTestFixDependencies = defaultDependencies
+) {
+	const suggestedConversation = await dependencies.suggestHistory({
+		request: input.request,
+		prompt: input.originalPrompt,
+		context: input.context,
+		conversation: input.conversation,
+		expectedOutput: input.expectedOutput,
+		onProgress: input.onProgress
+	})
+	const promptRequest = buildPromptFixRequest(
+		input.request,
+		{
+			context: input.context,
+			conversation: suggestedConversation,
+			expectedOutput: input.expectedOutput,
+			assertionLabel: input.assertionLabel,
+			assertionValue: input.assertionValue,
+			currentIncorrectOutcome: input.currentIncorrectOutcome
+		},
+		input.conversation
+	)
+	const suggestedPrompt = await dependencies.suggestPrompt({
+		request: promptRequest,
+		originalPrompt: input.originalPrompt,
+		onProgress: input.onProgress
+	})
+
+	return { suggestedConversation, suggestedPrompt }
+}

package/src/backend/roleplay/test-fix-proposal-acceptance.spec.ts

@@ -0,0 +1,253 @@
+import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
+import os from 'node:os'
+import path from 'node:path'
+import { initTRPC } from '@trpc/server'
+import { afterEach, describe, expect, it, vi } from 'vitest'
+
+const environment = vi.hoisted(() => ({
+	projectBasePath: '',
+	getAgentFactories: vi.fn(),
+	suggestTestFix: vi.fn()
+}))
+
+vi.mock('../lib/get-root-path', () => ({
+	getRootPath: async () => '/project/workflows.ts'
+}))
+
+vi.mock('../lib/get-agent-factories', () => ({
+	getAgentCatalog: vi.fn(),
+	getAgentFactories: environment.getAgentFactories
+}))
+
+vi.mock('../lib/project-api', () => ({
+	getProjectBasePath: async () => environment.projectBasePath,
+	getProjectBosunApi: async () => null
+}))
+
+vi.mock('./suggest-test-fix', () => ({
+	suggestTestFix: environment.suggestTestFix
+}))
+
+import type { TrpcContext } from '../lib/trpc-context'
+import { buildTestFixRequest } from '../../lib/edit/test-ai-edit'
+import { assertionLabel } from '../../lib/tests/assertion-types'
+import { createPromptEditProposal, deletePromptEditProposal } from './prompt-edit-proposals'
+import { createRolePlayRouter } from './router'
+import { patchTest } from '../tests/data'
+import { readSuite, writeSuite, type TestRow } from '../tests/suite'
+import { testEditSourceFingerprint } from '../tests/test-edit-source'
+import {
+	createTestHistoryEditProposal,
+	deleteTestHistoryEditProposal
+} from '../tests/test-history-edit-proposals'
+import { createTestsRouter } from '../tests/router'
+
+const temporaryDirectories: string[] = []
+const promptProposalIds: string[] = []
+const historyProposalIds: string[] = []
+
+afterEach(async () => {
+	for (const id of promptProposalIds.splice(0)) deletePromptEditProposal(id)
+	for (const id of historyProposalIds.splice(0)) deleteTestHistoryEditProposal(id)
+	await Promise.all(
+		temporaryDirectories
+			.splice(0)
+			.map((directory) => rm(directory, { recursive: true, force: true }))
+	)
+	environment.getAgentFactories.mockReset()
+	environment.suggestTestFix.mockReset()
+})
+
+describe('test Fix proposal acceptance', () => {
+	it.each(['history first', 'prompt first'] as const)(
+		'accepts the paired changes with %s',
+		async (order) => {
+			const fixture = await createFixture()
+			const proposals = createPairedProposals(fixture.test)
+
+			if (order === 'history first') {
+				await fixture.tests.acceptHistoryEdit({ id: proposals.historyId })
+				await fixture.agents.acceptPromptEdit({ id: proposals.promptId })
+			} else {
+				await fixture.agents.acceptPromptEdit({ id: proposals.promptId })
+				await fixture.tests.acceptHistoryEdit({ id: proposals.historyId })
+			}
+
+			expect(await readFile(fixture.promptPath, 'utf8')).toBe('suggested prompt')
+			expect((await readSuite('agent')).tests[0].conversation).toBe('User: corrected')
+		}
+	)
+
+	it('rejects both paired changes after another test source field changes', async () => {
+		const fixture = await createFixture()
+		const proposals = createPairedProposals(fixture.test)
+		await patchTest('agent', fixture.test.id, { expected_output: 'Agent: changed' })
+
+		await expect(fixture.agents.acceptPromptEdit({ id: proposals.promptId })).rejects.toThrow(
+			'This test changed while the suggestion was open'
+		)
+		await expect(fixture.tests.acceptHistoryEdit({ id: proposals.historyId })).rejects.toThrow(
+			'This test changed while the suggestion was open'
+		)
+
+		expect(await readFile(fixture.promptPath, 'utf8')).toBe('original prompt')
+		const current = (await readSuite('agent')).tests[0]
+		expect(current.conversation).toBe('User: original')
+		expect(current.expected_output).toBe('Agent: changed')
+	})
+
+	it('saves a pending prompt and sends the same canonical test source as the browser', async () => {
+		const fixture = await createFixture()
+		const currentIncorrectOutcome = 'Agent: wrong'
+		const requestInput = {
+			context: JSON.stringify(fixture.test.context),
+			conversation: fixture.test.conversation,
+			expectedOutput: fixture.test.expected_output,
+			assertionLabel: assertionLabel(fixture.test.assertion_type),
+			assertionValue: fixture.test.assertion_value,
+			currentIncorrectOutcome
+		}
+		environment.suggestTestFix.mockResolvedValue({
+			suggestedConversation: fixture.test.conversation,
+			suggestedPrompt: 'draft prompt'
+		})
+
+		await fixture.agents.suggestTestFix({
+			agentId: 'agent',
+			region: null,
+			testId: fixture.test.id,
+			step: 'main',
+			request: buildTestFixRequest(requestInput),
+			prompt: 'draft prompt',
+			basePrompt: 'original prompt',
+			currentIncorrectOutcome,
+			progressId: '00000000-0000-4000-8000-000000000000'
+		})
+
+		expect(await readFile(fixture.promptPath, 'utf8')).toBe('draft prompt')
+		expect(environment.suggestTestFix).toHaveBeenCalledWith(
+			expect.objectContaining({
+				...requestInput,
+				originalPrompt: 'draft prompt',
+				request: buildTestFixRequest(requestInput)
+			})
+		)
+	})
+
+	it('does not replace a prompt that differs from both submitted snapshots', async () => {
+		const fixture = await createFixture()
+		await writeFile(fixture.promptPath, 'newer prompt', 'utf8')
+
+		await expect(
+			fixture.agents.suggestTestFix({
+				agentId: 'agent',
+				region: null,
+				testId: fixture.test.id,
+				step: 'main',
+				request: 'Fix this test',
+				prompt: 'draft prompt',
+				basePrompt: 'original prompt',
+				currentIncorrectOutcome: 'Agent: wrong',
+				progressId: '00000000-0000-4000-8000-000000000000'
+			})
+		).rejects.toThrow('The prompt changed before Bosun could start this Fix')
+
+		expect(environment.suggestTestFix).not.toHaveBeenCalled()
+		expect(await readFile(fixture.promptPath, 'utf8')).toBe('newer prompt')
+	})
+})
+
+async function createFixture() {
+	const root = await mkdtemp(path.join(os.tmpdir(), 'bosun-test-fix-acceptance-'))
+	temporaryDirectories.push(root)
+	environment.projectBasePath = path.join(root, 'agents')
+	const suiteDirectory = path.join(environment.projectBasePath, 'support', 'bosun-tests', 'agent')
+	await mkdir(suiteDirectory, { recursive: true })
+	await writeSuite('agent', { tests: [testRow()] })
+	const test = (await readSuite('agent')).tests[0]
+
+	const promptBasePath = path.join(root, 'prompts')
+	await mkdir(promptBasePath, { recursive: true })
+	const promptPath = path.join(promptBasePath, 'prompt.md')
+	await writeFile(promptPath, 'original prompt', 'utf8')
+	environment.getAgentFactories.mockResolvedValue({
+		basePath: promptBasePath,
+		factories: {
+			agent: async ({ stepRegistry }: any) => {
+				stepRegistry.addStep({
+					name: 'main',
+					prompt: {
+						type: 'file',
+						path: 'prompt.md',
+						content: () => readFile(promptPath, 'utf8')
+					}
+				})
+			}
+		}
+	})
+
+	const t = initTRPC.context<TrpcContext>().create()
+	return {
+		test,
+		promptPath,
+		agents: createRolePlayRouter(t).createCaller({}),
+		tests: createTestsRouter(t).createCaller({})
+	}
+}
+
+function createPairedProposals(test: TestRow) {
+	const originalFingerprint = testEditSourceFingerprint(test)
+	const historyFingerprint = testEditSourceFingerprint(test, {
+		conversation: 'User: corrected'
+	})
+	const history = createTestHistoryEditProposal({
+		agentId: 'agent',
+		region: null,
+		testId: test.id,
+		step: 'main',
+		originalConversation: test.conversation,
+		suggestedConversation: 'User: corrected',
+		expectedOutput: test.expected_output,
+		sourceGuard: {
+			suiteKey: 'agent',
+			allowedTestFingerprints: [originalFingerprint],
+			allowedPromptContents: ['original prompt', 'suggested prompt']
+		}
+	})
+	const prompt = createPromptEditProposal({
+		agentId: 'agent',
+		step: 'main',
+		originalPrompt: 'original prompt',
+		suggestedPrompt: 'suggested prompt',
+		testSourceGuard: {
+			suiteKey: 'agent',
+			testId: test.id,
+			allowedFingerprints: [originalFingerprint, historyFingerprint]
+		}
+	})
+	historyProposalIds.push(history.id)
+	promptProposalIds.push(prompt.id)
+	return { historyId: history.id, promptId: prompt.id }
+}
+
+function testRow(): TestRow {
+	return {
+		id: 'test-1',
+		order: 0,
+		pinned: 'TRUE',
+		scenario: 'Scenario',
+		step: 'main',
+		author: '',
+		category: '',
+		assertion_type: 'llm-rubric',
+		assertion_value: 'Be correct',
+		conversation: 'User: original',
+		expected_output: 'Agent: expected',
+		evaluation_mode: 'assertion',
+		context_mode: 'merge',
+		timeout: '0',
+		similarity_threshold: '50',
+		context: { customer: 'Sam' },
+		mocks: { booking: { id: '123' } }
+	}
+}