@recombine-ai/bosun 0.16.2 → 0.18.0

package/src/backend/tests/academy.ts

@@ -77,6 +77,17 @@ export interface WrittenRunReportConfig {
 }
 
 export type PromptTestsRunConfig = Pick<WrittenRunReportConfig, 'repeats' | 'concurrency'>
+export type TestHistoryOverrides = Record<string, { conversation: string }>
+
+export function applyTestHistoryOverrides<T extends { id?: unknown; conversation: string }>(
+	rows: T[],
+	overrides: TestHistoryOverrides
+): T[] {
+	return rows.map((row) => {
+		const override = overrides[typeof row.id === 'string' ? row.id : '']
+		return override ? { ...row, conversation: override.conversation } : row
+	})
+}
 
 export interface WrittenRunReportRow extends TableRow {
 	test: SuiteTestRow
@@ -141,7 +152,8 @@ export async function runPromptTestsManaged(
 	suite: SuiteSelection,
 	selection: PromptTestsSelection,
 	requestedConfig: PromptTestsRunConfig,
-	promptOverrides: Record<string, string> = {}
+	promptOverrides: Record<string, string> = {},
+	testHistoryOverrides: TestHistoryOverrides = {}
 ): Promise<WrittenRunReport> {
 	const run = activeRuns.start(suite.suiteKey)
 	if (!run) throw new Error('Tests already running for this agent')
@@ -149,6 +161,7 @@ export async function runPromptTestsManaged(
 		return await runPromptTests(suite, selection, requestedConfig, {
 			signal: run.controller.signal,
 			promptOverrides,
+			testHistoryOverrides,
 			setProgress: (progress) => activeRuns.setProgress(suite.suiteKey, run, progress)
 		})
 	} finally {
@@ -619,6 +632,7 @@ export async function runPromptTests(
 	opts?: {
 		signal?: AbortSignal
 		promptOverrides?: Record<string, string>
+		testHistoryOverrides?: TestHistoryOverrides
 		setProgress?: (progress: MutablePromptTestsProgress) => boolean
 	}
 ): Promise<WrittenRunReport> {
@@ -627,7 +641,7 @@ export async function runPromptTests(
 	const onlyTestId = 'testId' in selection ? selection.testId : undefined
 	const suiteKey = suite.suiteKey
 	const loadedSuite = await readSuite(suiteKey)
-	const rows = loadedSuite.tests
+	const rows = applyTestHistoryOverrides(loadedSuite.tests, opts?.testHistoryOverrides ?? {})
 
 	const repeats = Math.min(500, Math.max(1, Math.floor(Number(requestedConfig.repeats) || 1)))
 	const concurrency = Math.min(

package/src/backend/tests/data.spec.ts

@@ -11,8 +11,16 @@ vi.mock('../lib/get-root-path', () => ({
 }))
 vi.mock('../lib/project-api', () => ({ getProjectBasePath: async () => project.basePath }))
 
-import { getRunResultEntry, loadLatestResultsByTest } from './data'
+import {
+	addTest,
+	getRunResultEntry,
+	loadLatestResultsByTest,
+	loadTests,
+	patchTestConversationIfCurrent,
+	type TestRow
+} from './data'
 import { compressReportFile } from './report-files'
+import { writeSuite } from './suite'
 
 const temporaryDirectories: string[] = []
 
@@ -20,7 +28,7 @@ afterEach(async () => {
 	await Promise.all(temporaryDirectories.splice(0).map((dir) => rm(dir, { recursive: true })))
 })
 
-function testRow(id: string) {
+function testRow(id: string): TestRow {
 	return {
 		id,
 		order: 0,
@@ -36,6 +44,47 @@ function testRow(id: string) {
 	}
 }
 
+describe('addTest', () => {
+	it('inserts before a test without changing existing test files', async () => {
+		const projectDir = await mkdtemp(path.join(os.tmpdir(), 'bosun-add-test-'))
+		temporaryDirectories.push(projectDir)
+		root.path = path.join(projectDir, 'src', 'lib', 'bosun', 'index.ts')
+		project.basePath = path.join(projectDir, 'src', 'agents')
+		const agent = `agent-${Date.now()}-${Math.random().toString(16).slice(2)}`
+		const suiteDirectory = path.join(project.basePath, 'support', 'bosun-tests', agent)
+		await mkdir(suiteDirectory, { recursive: true })
+
+		const rows: TestRow[] = [
+			{ ...testRow('main-first'), step: 'main', order: 0 },
+			{ ...testRow('other-first'), step: 'other', order: 0 },
+			{ ...testRow('a-other-second'), step: 'other', order: 1024 },
+			{ ...testRow('z-main-second'), step: 'main', order: 1024 }
+		]
+		await writeSuite(agent, { tests: rows })
+		const originalFiles = new Map(
+			await Promise.all(
+				rows.map(
+					async (row) =>
+						[row.id, await readFile(path.join(suiteDirectory, `${row.id}.yaml`), 'utf8')] as const
+				)
+			)
+		)
+
+		const created = await addTest(
+			agent,
+			{ scenario: 'duplicate', step: 'main' },
+			{ beforeTestId: 'z-main-second' }
+		)
+
+		expect(created.order).toBe(512)
+		for (const row of rows) {
+			expect(await readFile(path.join(suiteDirectory, `${row.id}.yaml`), 'utf8')).toBe(
+				originalFiles.get(row.id)
+			)
+		}
+	})
+})
+
 async function writeTests(projectDir: string, agent: string, rows: ReturnType<typeof testRow>[]) {
 	project.basePath = path.join(projectDir, 'src', 'agents')
 	const suiteDirectory = path.join(project.basePath, 'support', 'bosun-tests', agent)
@@ -144,3 +193,33 @@ describe('loadLatestResultsByTest', () => {
 		expect((await getRunResultEntry(agent, 'a')).success).toBe(true)
 	})
 })
+
+describe('patchTestConversationIfCurrent', () => {
+	it('does not overwrite a conversation that changed after the suggestion was created', async () => {
+		const projectDir = await mkdtemp(path.join(os.tmpdir(), 'bosun-history-suggestion-'))
+		temporaryDirectories.push(projectDir)
+		root.path = path.join(projectDir, 'src', 'lib', 'bosun', 'index.ts')
+		const agent = `agent-${Date.now()}-${Math.random().toString(16).slice(2)}`
+		await writeTests(projectDir, agent, [testRow('test-1')])
+
+		const applied = await patchTestConversationIfCurrent(
+			agent,
+			'test-1',
+			'user: hello',
+			'user: corrected'
+		)
+		expect(applied).toMatchObject({ status: 'changed' })
+
+		const stale = await patchTestConversationIfCurrent(
+			agent,
+			'test-1',
+			'user: hello',
+			'user: stale suggestion'
+		)
+		expect(stale).toEqual({
+			status: 'stale',
+			currentConversation: 'user: corrected'
+		})
+		expect((await loadTests(agent)).rows[0].conversation).toBe('user: corrected')
+	})
+})

package/src/backend/tests/data.ts

@@ -17,6 +17,7 @@ import type {
 } from './academy'
 import { makeContext } from './context-utils'
 import { listAgentReportFiles, readReportFile, reportTimestamp } from './report-files'
+import { testEditSourceFingerprint } from './test-edit-source'
 
 export type CsvRow = Record<string, string>
 
@@ -174,6 +175,38 @@ export async function patchTest(
 	})
 }
 
+export async function patchTestConversationIfCurrent(
+	agent: string,
+	id: string,
+	expectedConversation: string,
+	conversation: string,
+	allowedSourceFingerprints?: string[]
+): Promise<
+	| { status: 'changed'; test: TestRow }
+	| { status: 'stale'; currentConversation: string }
+	| { status: 'not-found' }
+> {
+	return withSuiteMutationLock(agent, async () => {
+		const suite = await readSuite(agent)
+		const current = suite.tests.find((row) => row.id === id)
+		if (!current) return { status: 'not-found' as const }
+		if (
+			current.conversation !== expectedConversation ||
+			(allowedSourceFingerprints &&
+				!allowedSourceFingerprints.includes(testEditSourceFingerprint(current)))
+		) {
+			return { status: 'stale' as const, currentConversation: current.conversation }
+		}
+
+		const updated = preparePatch(current, { conversation })
+		await writeSuite(agent, {
+			...suite,
+			tests: suite.tests.map((row) => (row.id === id ? updated : row))
+		})
+		return { status: 'changed' as const, test: updated }
+	})
+}
+
 function preparePatch(current: TestRow, patch: Partial<TestRow>) {
 	const updated: TestRow = {
 		...defaultTestRow,
@@ -210,8 +243,16 @@ export async function patchTestsBatch(
 	})
 }
 
-// Add a new row at the top
-export async function addTest(agent: string, seed?: Partial<TestRow>): Promise<TestRow> {
+interface AddTestOptions {
+	beforeTestId?: string
+}
+
+// Add a new row at the top or directly before an existing row.
+export async function addTest(
+	agent: string,
+	seed?: Partial<TestRow>,
+	options: AddTestOptions = {}
+): Promise<TestRow> {
 	return withSuiteMutationLock(agent, async () => {
 		const suite = await loadSuiteOrEmpty(agent)
 		const id = randomUUID()
@@ -219,18 +260,47 @@ export async function addTest(agent: string, seed?: Partial<TestRow>): Promise<T
 			(minimum, row) => Math.min(minimum, row.order),
 			Number.POSITIVE_INFINITY
 		)
+		let existingTests = suite.tests
+		let order = Number.isFinite(firstOrder) ? firstOrder - 1024 : 0
+
+		if (options.beforeTestId) {
+			const beforeTest = suite.tests.find((row) => row.id === options.beforeTestId)
+			if (!beforeTest) throw new Error(`Test '${options.beforeTestId}' was not found`)
+
+			const stepTests = suite.tests.filter((row) => row.step === beforeTest.step)
+			const beforeIndex = stepTests.findIndex((row) => row.id === beforeTest.id)
+			const previousTest = stepTests[beforeIndex - 1]
+			const candidate = previousTest
+				? previousTest.order + (beforeTest.order - previousTest.order) / 2
+				: beforeTest.order - 1024
+
+			if (
+				Number.isFinite(candidate) &&
+				candidate < beforeTest.order &&
+				(!previousTest || candidate > previousTest.order)
+			) {
+				order = candidate
+			} else {
+				const rebalancedOrders = new Map(stepTests.map((row, index) => [row.id, index * 1024]))
+				existingTests = suite.tests.map((row) => {
+					const rebalancedOrder = rebalancedOrders.get(row.id)
+					return typeof rebalancedOrder === 'number' ? { ...row, order: rebalancedOrder } : row
+				})
+				order = beforeIndex === 0 ? -1024 : beforeIndex * 1024 - 512
+			}
+		}
 
 		const newRow: TestRow = {
 			...defaultTestRow,
 			id,
-			order: Number.isFinite(firstOrder) ? firstOrder - 1024 : 0,
 			...seed,
+			order,
 			context: makeContext(seed?.context)
 		}
 		if (typeof seed?.mocks !== 'undefined') {
 			newRow.mocks = makeContext(seed.mocks)
 		}
-		await writeSuite(agent, { tests: [newRow, ...suite.tests] })
+		await writeSuite(agent, { tests: [newRow, ...existingTests] })
 		return newRow
 	})
 }

package/src/backend/tests/router.ts

@@ -10,6 +10,7 @@ import {
 	loadLatestResultsByTest,
 	loadTests,
 	patchTest,
+	patchTestConversationIfCurrent,
 	patchTestsBatch,
 	reorderTests
 } from './data'
@@ -18,7 +19,6 @@ import { cancelPromptTests, getPromptTestsProgress, runPromptTestsManaged } from
 import { analyzeRunQuotes } from './analyze'
 import { findPromptIssues } from './prompt-issues'
 import type { TrpcContext } from '../lib/trpc-context'
-import { getProjectBosunApi } from '../lib/project-api'
 import {
 	TraceTestReviewContentSchema,
 	TraceTestSeedSchema
@@ -28,38 +28,16 @@ import {
 	createTestFromPIISanitizedTestDraft
 } from './pii-sanitized-test-drafts'
 import { PIISanitizerResponseError } from './pii-sanitizer'
-
-type SuiteSelection = {
-	suiteKey: string
-	agentId: string
-	region?: string
-}
-
-async function resolveSuiteSelection(
-	agentId: string,
-	region?: string | null
-): Promise<SuiteSelection> {
-	// Convention:
-	// - `agentId` is the canonical factory key (workflow export key).
-	// - `suiteKey` is the filesystem/report key:
-	//    - non-regional projects: `${agentId}`
-	//    - regional projects: `${agentId}.${region}`
-	const bosunApi = await getProjectBosunApi()
-	const meta = bosunApi ? await bosunApi.getAgentMeta(agentId) : null
-	const isRegionScoped = meta?.testScope === 'region'
-	if (!isRegionScoped) return { suiteKey: agentId, agentId }
-
-	if (!bosunApi) {
-		throw new Error(`Project does not export bosunApi, cannot resolve suite for '${agentId}'.`)
-	}
-
-	const resolved = await bosunApi.resolveTestSuite({ agentId, region })
-	return {
-		suiteKey: resolved.suiteKey,
-		agentId,
-		region: resolved.resolvedRegion ?? undefined
-	}
-}
+import { suggestTestHistoryEdit } from './suggest-test-history-edit'
+import { updatePromptEditProgress } from '../roleplay/prompt-edit-progress'
+import { resolveSuiteSelection } from './test-selection'
+import { testEditSourceFingerprint } from './test-edit-source'
+import {
+	createTestHistoryEditProposal,
+	deleteTestHistoryEditProposal,
+	getTestHistoryEditProposalForAcceptance
+} from './test-history-edit-proposals'
+import { withPromptFileLock } from '../roleplay/prompt-file'
 
 export function createTestsRouter(t: TRPCRootObject<TrpcContext, any, any>) {
 	return t.router({
@@ -151,6 +129,51 @@ export function createTestsRouter(t: TRPCRootObject<TrpcContext, any, any>) {
 			.mutation(async ({ input }) => {
 				return findPromptIssues(input)
 			}),
+		suggestHistoryEdit: t.procedure
+			.input(
+				z.object({
+					agent: z.string(),
+					region: z.string().nullish(),
+					testId: z.string(),
+					step: z.string().trim().min(1).max(512),
+					request: z.string().trim().min(1).max(100_000),
+					prompt: z.string().max(1_000_000),
+					basePrompt: z.string().max(1_000_000),
+					progressId: z.string().uuid()
+				})
+			)
+			.mutation(async ({ input }) => {
+				const { suiteKey } = await resolveSuiteSelection(input.agent, input.region)
+				const test = (await readSuite(suiteKey)).tests.find((row) => row.id === input.testId)
+				if (!test) throw new Error('This test is no longer available.')
+				const suggestedConversation = await suggestTestHistoryEdit({
+					request: input.request,
+					prompt: input.prompt,
+					context: JSON.stringify(test.context ?? {}, null, 2),
+					conversation: test.conversation,
+					expectedOutput: test.expected_output,
+					onProgress: (progress) => updatePromptEditProgress(input.progressId, progress)
+				})
+				return suggestedConversation === test.conversation
+					? { status: 'unchanged' as const }
+					: {
+							status: 'changed' as const,
+							proposal: createTestHistoryEditProposal({
+								agentId: input.agent,
+								region: input.region ?? null,
+								testId: input.testId,
+								step: input.step,
+								originalConversation: test.conversation,
+								suggestedConversation,
+								expectedOutput: test.expected_output,
+								sourceGuard: {
+									suiteKey,
+									allowedTestFingerprints: [testEditSourceFingerprint(test)],
+									allowedPromptContents: [input.basePrompt, input.prompt]
+								}
+							})
+						}
+			}),
 		suite: t.procedure
 			.input(z.object({ agent: z.string(), region: z.string().nullish() }))
 			.query(async ({ input }) => {
@@ -162,12 +185,13 @@ export function createTestsRouter(t: TRPCRootObject<TrpcContext, any, any>) {
 				z.object({
 					agent: z.string(),
 					region: z.string().nullish(),
+					beforeTestId: z.string().optional(),
 					seed: TestRowSchema.omit({ id: true, order: true }).partial()
 				})
 			)
 			.mutation(async ({ input }) => {
 				const { suiteKey } = await resolveSuiteSelection(input.agent, input.region)
-				return addTest(suiteKey, input.seed)
+				return addTest(suiteKey, input.seed, { beforeTestId: input.beforeTestId })
 			}),
 		createPIISanitizedTestDraft: t.procedure
 			.input(
@@ -221,6 +245,39 @@ export function createTestsRouter(t: TRPCRootObject<TrpcContext, any, any>) {
 				const { suiteKey } = await resolveSuiteSelection(input.agent, input.region)
 				return patchTest(suiteKey, input.id, input.patch)
 			}),
+		acceptHistoryEdit: t.procedure
+			.input(z.object({ id: z.string().uuid() }))
+			.mutation(async ({ input }) => {
+				const proposal = getTestHistoryEditProposalForAcceptance(input.id)
+				if (!proposal) throw new Error('This test history suggestion is no longer available.')
+				const result = await withPromptFileLock(proposal.agentId, proposal.step, async (file) => {
+					if (!proposal.sourceGuard.allowedPromptContents.includes(await file.read())) {
+						throw new Error(
+							'The prompt changed while this suggestion was open. Reject this suggestion, then ask Bosun to create a new one.'
+						)
+					}
+					return patchTestConversationIfCurrent(
+						proposal.sourceGuard.suiteKey,
+						proposal.testId,
+						proposal.originalConversation,
+						proposal.suggestedConversation,
+						proposal.sourceGuard.allowedTestFingerprints
+					)
+				})
+				if (result.status === 'not-found') {
+					throw new Error('This test is no longer available.')
+				}
+				if (result.status === 'stale') {
+					throw new Error(
+						'This test changed while the suggestion was open. Reject this suggestion, then ask Bosun to create a new one.'
+					)
+				}
+				deleteTestHistoryEditProposal(proposal.id)
+				return { ok: true as const, test: result.test }
+			}),
+		rejectHistoryEdit: t.procedure
+			.input(z.object({ id: z.string().uuid() }))
+			.mutation(({ input }) => ({ ok: deleteTestHistoryEditProposal(input.id) })),
 		patchTestsBatch: t.procedure
 			.input(
 				z.object({
@@ -262,6 +319,9 @@ export function createTestsRouter(t: TRPCRootObject<TrpcContext, any, any>) {
 							step: z.string(),
 							scope: z.enum(['pinned', 'all']).optional(),
 							promptOverrides: z.record(z.string(), z.string().max(1_000_000)).optional(),
+							testHistoryOverrides: z
+								.record(z.string(), z.object({ conversation: z.string().max(1_000_000) }).strict())
+								.optional(),
 							repeats: z.number().int().min(1).max(500),
 							concurrency: z.number().int().min(1).max(75)
 						})
@@ -272,6 +332,9 @@ export function createTestsRouter(t: TRPCRootObject<TrpcContext, any, any>) {
 							region: z.string().nullish(),
 							testId: z.string(),
 							promptOverrides: z.record(z.string(), z.string().max(1_000_000)).optional(),
+							testHistoryOverrides: z
+								.record(z.string(), z.object({ conversation: z.string().max(1_000_000) }).strict())
+								.optional(),
 							repeats: z.number().int().min(1).max(500),
 							concurrency: z.number().int().min(1).max(75)
 						})
@@ -286,7 +349,8 @@ export function createTestsRouter(t: TRPCRootObject<TrpcContext, any, any>) {
 						selection,
 						{ testId: input.testId },
 						runConfig,
-						input.promptOverrides
+						input.promptOverrides,
+						input.testHistoryOverrides
 					)
 				}
 				return runPromptTestsManaged(
@@ -296,7 +360,8 @@ export function createTestsRouter(t: TRPCRootObject<TrpcContext, any, any>) {
 						step: input.step
 					},
 					runConfig,
-					input.promptOverrides
+					input.promptOverrides,
+					input.testHistoryOverrides
 				)
 			}),
 		cancelRun: t.procedure

package/src/backend/tests/suggest-test-history-edit.spec.ts

@@ -0,0 +1,20 @@
+import { describe, expect, it } from 'vitest'
+
+import { buildTestHistoryPrompt } from './suggest-test-history-edit'
+
+describe('buildTestHistoryPrompt', () => {
+	it('supplies the prompt, history, and immutable expected outcome', () => {
+		const prompt = buildTestHistoryPrompt({
+			request: 'Make the history valid.',
+			prompt: 'Always ask for a name.',
+			context: '{ "customer": { "name": "Sam" } }',
+			conversation: 'User: Hello\nAgent: Welcome back, Sam.',
+			expectedOutput: 'What is your name?'
+		})
+
+		expect(prompt).toContain('Always ask for a name.')
+		expect(prompt).toContain('{ "customer": { "name": "Sam" } }')
+		expect(prompt).toContain('User: Hello\nAgent: Welcome back, Sam.')
+		expect(prompt).toContain('immutable expected outcome:\nWhat is your name?')
+	})
+})

package/src/backend/tests/suggest-test-history-edit.ts

@@ -0,0 +1,52 @@
+import type { PromptEditProgressPhase } from '../roleplay/prompt-edit-progress'
+import { suggestFileEdit } from '../roleplay/suggest-file-edit'
+
+const VIRTUAL_HISTORY_PATH = 'history.md'
+
+export async function suggestTestHistoryEdit(input: {
+	request: string
+	prompt: string
+	context: string
+	conversation: string
+	expectedOutput: string
+	onProgress?: (progress: {
+		phase: PromptEditProgressPhase
+		activity: number
+		summary: string
+	}) => void
+}): Promise<string> {
+	return suggestFileEdit({
+		request: buildTestHistoryPrompt(input),
+		originalContent: input.conversation,
+		virtualPath: VIRTUAL_HISTORY_PATH,
+		instructions:
+			'The conversation history is the existing virtual file history.md. Decide whether it conflicts with the current LLM prompt or contains agent behaviour that the current prompt would not produce. If it does, use the Apply Patch tool to update history.md minimally. Otherwise, do not call Apply Patch. Preserve the test intent and never add or change the expected outcome. Do not create, delete, or modify any other file.',
+		onProgress: input.onProgress
+	})
+}
+
+export function buildTestHistoryPrompt(input: {
+	request: string
+	prompt: string
+	context: string
+	conversation: string
+	expectedOutput: string
+}): string {
+	return `The user wants you to update an LLM test's conversation history when needed. The history must reflect behaviour that is valid under the current prompt. Make the smallest useful change. The expected outcome is immutable and is provided only so you can preserve the test's intent.
+
+This is the user's request:
+${input.request}
+
+This is the current LLM prompt:
+${input.prompt}
+
+This is the test context:
+${input.context}
+
+This is the conversation history in history.md:
+${input.conversation}
+
+This is the immutable expected outcome:
+${input.expectedOutput}
+`
+}

package/src/backend/tests/test-edit-source.spec.ts

@@ -0,0 +1,61 @@
+import { describe, expect, it } from 'vitest'
+
+import type { TestRow } from './suite'
+import { testEditSourceFingerprint } from './test-edit-source'
+
+const sourceChanges: Array<[string, Partial<TestRow>]> = [
+	['step', { step: 'follow-up' }],
+	['evaluation mode', { evaluation_mode: 'json-deep-equal' }],
+	['assertion type', { assertion_type: 'contains' }],
+	['assertion value', { assertion_value: 'A different assertion' }],
+	['case sensitivity', { case_sensitive: true }],
+	['context mode', { context_mode: 'replace' }],
+	['conversation', { conversation: 'User: changed' }],
+	['expected output', { expected_output: 'Agent: changed' }],
+	['timeout', { timeout: '10' }],
+	['similarity threshold', { similarity_threshold: '80' }],
+	['context', { context: { customer: 'Alex' } }],
+	['mocks', { mocks: { booking: { id: '456' } } }]
+]
+
+describe('testEditSourceFingerprint', () => {
+	it.each(sourceChanges)('changes when the test %s changes', (_name, change) => {
+		const test = testRow()
+
+		expect(testEditSourceFingerprint({ ...test, ...change })).not.toBe(
+			testEditSourceFingerprint(test)
+		)
+	})
+
+	it('uses the proposed conversation without changing the other source fields', () => {
+		const test = testRow()
+		const conversation = 'User: corrected'
+
+		expect(testEditSourceFingerprint(test, { conversation })).toBe(
+			testEditSourceFingerprint({ ...test, conversation })
+		)
+	})
+})
+
+function testRow(): TestRow {
+	return {
+		id: 'test-1',
+		order: 0,
+		pinned: 'TRUE',
+		scenario: 'Scenario',
+		step: 'main',
+		author: '',
+		category: '',
+		assertion_type: 'llm-rubric',
+		assertion_value: 'Be correct',
+		conversation: 'User: original',
+		expected_output: 'Agent: expected',
+		evaluation_mode: 'assertion',
+		case_sensitive: false,
+		context_mode: 'merge',
+		timeout: '0',
+		similarity_threshold: '50',
+		context: { customer: 'Sam' },
+		mocks: { booking: { id: '123' } }
+	}
+}

package/src/backend/tests/test-edit-source.ts

@@ -0,0 +1,34 @@
+import { createHash } from 'node:crypto'
+
+import type { TestRow } from './suite'
+
+function canonicalize(value: unknown): unknown {
+	if (Array.isArray(value)) return value.map(canonicalize)
+	if (!value || typeof value !== 'object') return value
+	return Object.fromEntries(
+		Object.entries(value as Record<string, unknown>)
+			.sort(([left], [right]) => left.localeCompare(right))
+			.map(([key, entry]) => [key, canonicalize(entry)])
+	)
+}
+
+export function testEditSourceFingerprint(
+	test: TestRow,
+	overrides?: { conversation?: string }
+): string {
+	const source = canonicalize({
+		step: test.step,
+		evaluationMode: test.evaluation_mode,
+		assertionType: test.assertion_type,
+		assertionValue: test.assertion_value,
+		caseSensitive: test.case_sensitive,
+		contextMode: test.context_mode,
+		conversation: overrides?.conversation ?? test.conversation,
+		expectedOutput: test.expected_output,
+		timeout: test.timeout,
+		similarityThreshold: test.similarity_threshold,
+		context: test.context,
+		mocks: test.mocks ?? null
+	})
+	return createHash('sha256').update(JSON.stringify(source)).digest('hex')
+}