@recombine-ai/bosun 0.16.2 → 0.17.1

package/src/backend/roleplay/prompt-file.ts

@@ -0,0 +1,59 @@
+import fs from 'node:fs'
+import path from 'node:path'
+
+import { loadAgentSteps } from './agent-loader'
+
+export interface LockedPromptFile {
+	path: string
+	read: () => Promise<string>
+	write: (prompt: string) => Promise<void>
+}
+
+const promptMutationQueue = new Map<string, Promise<void>>()
+
+async function resolveEditablePrompt(agentId: string, stepName: string) {
+	const { basePath, steps } = await loadAgentSteps(agentId)
+	const targetStep = steps.find((step) => step?.name === stepName)
+	if (!targetStep || targetStep?.prompt?.type !== 'file') {
+		throw new Error(`Step '${stepName}' does not have an editable file prompt`)
+	}
+	return {
+		promptPath: path.resolve(basePath, targetStep.prompt.path),
+		readContent: () => targetStep.prompt.content() as Promise<string>
+	}
+}
+
+export async function readEditablePrompt(agentId: string, stepName: string): Promise<string> {
+	const { readContent } = await resolveEditablePrompt(agentId, stepName)
+	return readContent()
+}
+
+export async function withPromptFileLock<T>(
+	agentId: string,
+	stepName: string,
+	operation: (file: LockedPromptFile) => Promise<T>
+): Promise<T> {
+	const { promptPath } = await resolveEditablePrompt(agentId, stepName)
+	const canonicalPath = await fs.promises.realpath(promptPath)
+	const previous = promptMutationQueue.get(canonicalPath) ?? Promise.resolve()
+	let release!: () => void
+	const gate = new Promise<void>((resolve) => {
+		release = resolve
+	})
+	const queued = previous.catch(() => {}).then(() => gate)
+	promptMutationQueue.set(canonicalPath, queued)
+	await previous.catch(() => {})
+
+	try {
+		return await operation({
+			path: canonicalPath,
+			read: () => fs.promises.readFile(canonicalPath, 'utf-8'),
+			write: (prompt) => fs.promises.writeFile(canonicalPath, prompt, 'utf-8')
+		})
+	} finally {
+		release()
+		if (promptMutationQueue.get(canonicalPath) === queued) {
+			promptMutationQueue.delete(canonicalPath)
+		}
+	}
+}

package/src/backend/roleplay/router.spec.ts

@@ -0,0 +1,75 @@
+import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
+import os from 'node:os'
+import path from 'node:path'
+import { initTRPC } from '@trpc/server'
+import { afterEach, describe, expect, it, vi } from 'vitest'
+
+const mocks = vi.hoisted(() => ({
+	getAgentFactories: vi.fn()
+}))
+
+vi.mock('../lib/get-root-path', () => ({
+	getRootPath: async () => '/project/workflows.ts'
+}))
+
+vi.mock('../lib/get-agent-factories', () => ({
+	getAgentCatalog: vi.fn(),
+	getAgentFactories: mocks.getAgentFactories
+}))
+
+import type { TrpcContext } from '../lib/trpc-context'
+import { createPromptEditProposal, deletePromptEditProposal } from './prompt-edit-proposals'
+import { createRolePlayRouter } from './router'
+
+const temporaryDirectories: string[] = []
+const proposalIds: string[] = []
+
+afterEach(async () => {
+	for (const id of proposalIds.splice(0)) deletePromptEditProposal(id)
+	await Promise.all(
+		temporaryDirectories
+			.splice(0)
+			.map((directory) => rm(directory, { recursive: true, force: true }))
+	)
+	mocks.getAgentFactories.mockReset()
+})
+
+describe('acceptPromptEdit', () => {
+	it('does not overwrite a prompt that changed after the proposal was created', async () => {
+		const basePath = await mkdtemp(path.join(os.tmpdir(), 'bosun-prompt-edit-router-'))
+		temporaryDirectories.push(basePath)
+		const promptPath = path.join(basePath, 'prompt.md')
+		await writeFile(promptPath, 'newer prompt', 'utf8')
+
+		mocks.getAgentFactories.mockResolvedValue({
+			basePath,
+			factories: {
+				agent: async ({ stepRegistry }: any) => {
+					stepRegistry.addStep({
+						name: 'main',
+						prompt: {
+							type: 'file',
+							path: 'prompt.md',
+							content: () => readFile(promptPath, 'utf8')
+						}
+					})
+				}
+			}
+		})
+
+		const proposal = createPromptEditProposal({
+			agentId: 'agent',
+			step: 'main',
+			originalPrompt: 'original prompt',
+			suggestedPrompt: 'suggested prompt'
+		})
+		proposalIds.push(proposal.id)
+		const t = initTRPC.context<TrpcContext>().create()
+		const caller = createRolePlayRouter(t).createCaller({})
+
+		await expect(caller.acceptPromptEdit({ id: proposal.id })).rejects.toThrow(
+			'The prompt changed while this suggestion was open'
+		)
+		expect(await readFile(promptPath, 'utf8')).toBe('newer prompt')
+	})
+})

package/src/backend/roleplay/router.ts

@@ -11,16 +11,25 @@ import { getRootPath } from '../lib/get-root-path'
 import type { TrpcContext } from '../lib/trpc-context'
 import { analyzeTraceQuotes, generateSuggestedUserResponse } from '../tests/analyze'
 import type { AgentState } from './agents'
+import { getAgentFactory, loadAgentSteps, noop, noopAsync } from './agent-loader'
 import { fsContextPersistence, fsStatePersistence } from './fs-persistence'
+import { readEditablePrompt, withPromptFileLock } from './prompt-file'
 import { validateAndReplaceSchemaFile } from './schema-file'
 import { generateReplyVariants, replaceSavedState } from './variants'
 import {
 	createPromptEditProposal,
 	deletePromptEditProposal,
-	getPromptEditProposal
+	getPromptEditProposal,
+	getPromptEditProposalForAcceptance
 } from './prompt-edit-proposals'
 import { suggestPromptEdit } from './suggest-prompt-edit'
+import { suggestTestFix } from './suggest-test-fix'
 import { getPromptEditProgress, updatePromptEditProgress } from './prompt-edit-progress'
+import { readSuite, withSuiteMutationLock } from '../tests/suite'
+import { resolveSuiteSelection } from '../tests/test-selection'
+import { testEditSourceFingerprint } from '../tests/test-edit-source'
+import { createTestHistoryEditProposal } from '../tests/test-history-edit-proposals'
+import { assertionLabel as getAssertionLabel } from '../../lib/tests/assertion-types'
 
 const AgentIdSchema = z.string().trim().min(1).max(512)
 const promptMappersCache = new Map<string, (ctx: object) => object>()
@@ -91,7 +100,7 @@ export function createRolePlayRouter(t: TRPCRootObject<TrpcContext, any, any>) {
 				})
 			)
 			.mutation(async ({ input }) => {
-				await persistPromptUpdate(input.agentId, input.step, input.prompt)
+				await withPromptFileLock(input.agentId, input.step, (file) => file.write(input.prompt))
 				return { ok: true }
 			}),
 		suggestPromptEdit: t.procedure
@@ -120,6 +129,94 @@ export function createRolePlayRouter(t: TRPCRootObject<TrpcContext, any, any>) {
 				})
 				return { status: 'changed' as const, proposal }
 			}),
+		suggestTestFix: t.procedure
+			.input(
+				z.object({
+					agentId: AgentIdSchema,
+					region: z.string().nullish(),
+					testId: z.string(),
+					step: z.string().trim().min(1).max(512),
+					request: z.string().trim().min(1).max(100_000),
+					prompt: z.string().max(1_000_000),
+					basePrompt: z.string().max(1_000_000),
+					currentIncorrectOutcome: z.string().max(1_000_000),
+					progressId: z.string().uuid()
+				})
+			)
+			.mutation(async ({ input }) => {
+				const { suiteKey } = await resolveSuiteSelection(input.agentId, input.region)
+				const test = (await readSuite(suiteKey)).tests.find((row) => row.id === input.testId)
+				if (!test) throw new Error('This test is no longer available.')
+				await withPromptFileLock(input.agentId, input.step, async (file) => {
+					const currentPrompt = await file.read()
+					if (currentPrompt !== input.basePrompt && currentPrompt !== input.prompt) {
+						throw new Error(
+							'The prompt changed before Bosun could start this Fix. Check the prompt, then try again.'
+						)
+					}
+					if (currentPrompt !== input.prompt) await file.write(input.prompt)
+				})
+				const originalPrompt = input.prompt
+				const context = JSON.stringify(test.context ?? {})
+				const assertionLabel =
+					test.evaluation_mode === 'json-deep-equal' ? '' : getAssertionLabel(test.assertion_type)
+				const assertionValue =
+					test.evaluation_mode === 'json-deep-equal' ? '' : test.assertion_value
+				const suggestion = await suggestTestFix({
+					request: input.request,
+					originalPrompt,
+					context,
+					conversation: test.conversation,
+					expectedOutput: test.expected_output,
+					assertionLabel,
+					assertionValue,
+					currentIncorrectOutcome: input.currentIncorrectOutcome,
+					onProgress: (progress) => updatePromptEditProgress(input.progressId, progress)
+				})
+				const originalTestFingerprint = testEditSourceFingerprint(test)
+				const historyTestFingerprint = testEditSourceFingerprint(test, {
+					conversation: suggestion.suggestedConversation
+				})
+				const history =
+					suggestion.suggestedConversation === test.conversation
+						? { status: 'unchanged' as const }
+						: {
+								status: 'changed' as const,
+								proposal: createTestHistoryEditProposal({
+									agentId: input.agentId,
+									region: input.region ?? null,
+									testId: input.testId,
+									step: input.step,
+									originalConversation: test.conversation,
+									suggestedConversation: suggestion.suggestedConversation,
+									expectedOutput: test.expected_output,
+									sourceGuard: {
+										suiteKey,
+										allowedTestFingerprints: [originalTestFingerprint],
+										allowedPromptContents: [originalPrompt, suggestion.suggestedPrompt]
+									}
+								})
+							}
+				const prompt =
+					suggestion.suggestedPrompt === originalPrompt
+						? { status: 'unchanged' as const }
+						: {
+								status: 'changed' as const,
+								proposal: createPromptEditProposal({
+									agentId: input.agentId,
+									step: input.step,
+									originalPrompt,
+									suggestedPrompt: suggestion.suggestedPrompt,
+									testSourceGuard: {
+										suiteKey,
+										testId: input.testId,
+										allowedFingerprints: [originalTestFingerprint, historyTestFingerprint]
+									}
+								})
+							}
+
+				return { history, prompt }
+			}),
 		promptEditProgress: t.procedure
 			.input(z.object({ id: z.string().uuid() }))
 			.query(({ input }) => getPromptEditProgress(input.id)),
@@ -129,9 +226,31 @@ export function createRolePlayRouter(t: TRPCRootObject<TrpcContext, any, any>) {
 		acceptPromptEdit: t.procedure
 			.input(z.object({ id: z.string().uuid() }))
 			.mutation(async ({ input }) => {
-				const proposal = getPromptEditProposal(input.id)
+				const proposal = getPromptEditProposalForAcceptance(input.id)
 				if (!proposal) throw new Error('This prompt suggestion is no longer available')
-				await persistPromptUpdate(proposal.agentId, proposal.step, proposal.suggestedPrompt)
+				await withPromptFileLock(proposal.agentId, proposal.step, async (file) => {
+					if ((await file.read()) !== proposal.originalPrompt) {
+						throw new Error(
+							'The prompt changed while this suggestion was open. Reject this suggestion, then ask Bosun to create a new one.'
+						)
+					}
+					const guard = proposal.testSourceGuard
+					if (!guard) {
+						await file.write(proposal.suggestedPrompt)
+						return
+					}
+					await withSuiteMutationLock(guard.suiteKey, async () => {
+						const test = (await readSuite(guard.suiteKey)).tests.find(
+							(row) => row.id === guard.testId
+						)
+						if (!test || !guard.allowedFingerprints.includes(testEditSourceFingerprint(test))) {
+							throw new Error(
+								'This test changed while the suggestion was open. Reject this suggestion, then ask Bosun to create a new one.'
+							)
+						}
+						await file.write(proposal.suggestedPrompt)
+					})
+				})
 				deletePromptEditProposal(proposal.id)
 				return { ok: true, prompt: proposal.suggestedPrompt }
 			}),
@@ -306,27 +425,6 @@ async function loadJsonArray(filePath: string): Promise<unknown[]> {
 	}
 }
 
-const noop = () => void 0
-const noopAsync = () => Promise.resolve()
-
-async function persistPromptUpdate(agentId: string, stepName: string, prompt: string) {
-	const { basePath, steps } = await loadAgentSteps(agentId)
-	const targetStep = steps.find((step) => step?.name === stepName)
-	if (!targetStep || targetStep?.prompt?.type !== 'file') return
-
-	const promptPath = path.resolve(basePath, targetStep.prompt.path)
-	await fs.promises.writeFile(promptPath, prompt, 'utf-8')
-}
-
-async function readEditablePrompt(agentId: string, stepName: string): Promise<string> {
-	const { steps } = await loadAgentSteps(agentId)
-	const targetStep = steps.find((step) => step?.name === stepName)
-	if (!targetStep || targetStep?.prompt?.type !== 'file') {
-		throw new Error(`Step '${stepName}' does not have an editable file prompt`)
-	}
-	return targetStep.prompt.content()
-}
-
 async function persistSchemaUpdate(agentId: string, stepName: string, schema: string) {
 	const { basePath, steps } = await loadAgentSteps(agentId)
 	const targetStep = steps.find((step) => step?.name === stepName)
@@ -340,50 +438,6 @@ async function persistSchemaUpdate(agentId: string, stepName: string, schema: st
 	await validateAndReplaceSchemaFile(schemaPath, schema)
 }
 
-async function loadAgentSteps(agentId: string) {
-	const { factory, basePath } = await getAgentFactory(agentId)
-	const ctx = createContext({})
-	const steps: any[] = []
-	const stepRegistry = {
-		addStep: (step: any) => {
-			steps.push(step)
-		},
-		getSteps: () => steps
-	}
-
-	await factory({
-		ctx,
-		getMessages: () => [],
-		logger: console,
-		scheduler: {
-			clearAllPendingActions: noopAsync,
-			registerAction: () => noopAsync
-		},
-		sendMessage: noopAsync,
-		stepRegistry,
-		stepTracer: {
-			addStepTrace: noop,
-			flush: noopAsync
-		},
-		config: {
-			enableTimers: false,
-			isTest: false
-		}
-	})
-
-	return { basePath, steps }
-}
-
-async function getAgentFactory(agentId: string) {
-	const rootPath = await getRootPath()
-	const agentFactories = await getAgentFactories(rootPath)
-	const factory = agentFactories.factories[agentId]
-	if (!factory) {
-		throw new Error(`Cannot find agent ${agentId}`)
-	}
-	return { factory, basePath: agentFactories.basePath }
-}
-
 async function getPromptMapper(agentId: string) {
 	const { factory } = await getAgentFactory(agentId)
 	const ctx = createContext({})

package/src/backend/roleplay/suggest-file-edit.ts

@@ -0,0 +1,160 @@
+import OpenAI from 'openai'
+
+import { applyV4ADiff } from '../lib/apply-v4a-diff'
+import type { PromptEditProgressPhase } from './prompt-edit-progress'
+
+const MODEL = 'gpt-5.4'
+const MAX_PATCH_ROUNDS = 8
+
+type ApplyPatchCall = {
+	type: 'apply_patch_call'
+	call_id: string
+	operation: {
+		type: 'create_file' | 'update_file' | 'delete_file'
+		path: string
+		diff?: string
+	}
+}
+
+export async function suggestFileEdit(input: {
+	request: string
+	originalContent: string
+	virtualPath: string
+	instructions: string
+	onProgress?: (progress: {
+		phase: PromptEditProgressPhase
+		activity: number
+		summary: string
+	}) => void
+}): Promise<string> {
+	const apiKey = process.env.OPENAI_API_KEY
+	if (!apiKey) throw new Error('OPENAI_API_KEY is not set')
+
+	const client = new OpenAI({ apiKey })
+	let virtualContent = input.originalContent
+	let activity = 0
+	let modelRound = 0
+	let reasoningSummary = ''
+	const handleEvent = (event: any) => {
+		activity += 1
+		if (
+			event?.type === 'response.reasoning_summary_text.delta' &&
+			typeof event.delta === 'string'
+		) {
+			reasoningSummary += event.delta
+		}
+		input.onProgress?.({
+			phase: progressPhase(event, modelRound),
+			activity,
+			summary: reasoningSummary.trim()
+		})
+	}
+	let response = await createResponse(client, {
+		input: input.request,
+		instructions: input.instructions,
+		onEvent: handleEvent
+	})
+
+	for (let patchRound = 0; patchRound < MAX_PATCH_ROUNDS; patchRound += 1) {
+		const patchCalls = (response.output as unknown as ApplyPatchCall[]).filter(
+			(item) => item?.type === 'apply_patch_call'
+		)
+		if (!patchCalls.length) return virtualContent
+
+		const outputs = patchCalls.map((call) => {
+			try {
+				virtualContent = applyFileOperation(virtualContent, input.virtualPath, call.operation)
+				return {
+					type: 'apply_patch_call_output',
+					call_id: call.call_id,
+					status: 'completed',
+					output: `Updated ${input.virtualPath} in memory`
+				}
+			} catch (error) {
+				return {
+					type: 'apply_patch_call_output',
+					call_id: call.call_id,
+					status: 'failed',
+					output: error instanceof Error ? error.message : String(error)
+				}
+			}
+		})
+
+		reasoningSummary = ''
+		input.onProgress?.({ phase: 'reviewing', activity, summary: '' })
+		modelRound += 1
+		response = await createResponse(client, {
+			previousResponseId: response.id,
+			input: outputs,
+			instructions: input.instructions,
+			onEvent: handleEvent
+		})
+	}
+
+	throw new Error(`The model did not finish editing ${input.virtualPath}`)
+}
+
+async function createResponse(
+	client: OpenAI,
+	input: {
+		input: unknown
+		instructions: string
+		previousResponseId?: string
+		onEvent: (event: any) => void
+	}
+): Promise<any> {
+	const stream = await (client.responses.create as any)({
+		model: MODEL,
+		instructions: input.instructions,
+		input: input.input,
+		...(input.previousResponseId ? { previous_response_id: input.previousResponseId } : {}),
+		tools: [{ type: 'apply_patch' }],
+		reasoning: { effort: 'high', summary: 'auto' },
+		text: { verbosity: 'low' },
+		stream: true
+	})
+
+	let completedResponse: any = null
+	for await (const event of stream as AsyncIterable<any>) {
+		input.onEvent(event)
+		if (event?.type === 'response.completed') completedResponse = event.response
+		if (event?.type === 'response.failed') {
+			throw new Error(event.response?.error?.message || 'OpenAI failed to suggest edits')
+		}
+		if (event?.type === 'response.incomplete') {
+			throw new Error('OpenAI returned an incomplete edit')
+		}
+		if (event?.type === 'error') throw new Error(event.message || 'OpenAI streaming failed')
+	}
+
+	if (!completedResponse) throw new Error('OpenAI stream ended before the edit completed')
+	return completedResponse
+}
+
+function progressPhase(event: any, round: number): PromptEditProgressPhase {
+	const eventType = String(event?.type || '')
+	const itemType = String(event?.item?.type || '')
+	if (eventType.includes('apply_patch') || itemType === 'apply_patch_call') return 'drafting'
+	return round > 0 ? 'reviewing' : 'thinking'
+}
+
+function applyFileOperation(
+	currentContent: string,
+	virtualPath: string,
+	operation: ApplyPatchCall['operation']
+): string {
+	const normalizedPath = String(operation.path || '')
+		.replace(/\\/g, '/')
+		.replace(/^\.\//, '')
+		.replace(/^\//, '')
+	if (normalizedPath !== virtualPath) {
+		throw new Error(`Only ${virtualPath} may be edited`)
+	}
+	if (operation.type !== 'update_file') {
+		throw new Error(`${virtualPath} already exists and may only be updated`)
+	}
+	if (typeof operation.diff !== 'string' || !operation.diff.trim()) {
+		throw new Error('Apply Patch returned an empty diff')
+	}
+	return applyV4ADiff(currentContent, operation.diff)
+}