@recombine-ai/bosun 0.15.0 → 0.16.0

package/src/backend/tests/data.ts

@@ -1,39 +1,27 @@
-import { tryReadConfig, writeConfig } from './config'
-import { access, readdir, readFile, stat, writeFile } from 'node:fs/promises'
+import { createHash, randomUUID } from 'node:crypto'
+import { readdir, readFile, stat, writeFile } from 'node:fs/promises'
 import path from 'node:path'
 import {
-	getSuiteFilePath,
 	readSuite,
 	type TestSuite,
-	upsertSuiteMeta,
 	withSuiteMutationLock,
 	writeSuite,
 	type TestRow as SuiteRow
 } from './suite'
 import { getRootPath } from '../lib/get-root-path'
-import z from 'zod'
 import type {
 	ResultEntry,
 	WrittenRunReport,
 	WrittenRunReportConfig,
 	WrittenRunResult
 } from './academy'
-import { makeContext, makeContextDiff } from './context-utils'
+import { makeContext } from './context-utils'
 import { listAgentReportFiles, readReportFile, reportTimestamp } from './report-files'
 
 export type CsvRow = Record<string, string>
 
 export type TestRow = SuiteRow
 
-export const RunConfigSchema = z.object({
-	repeats: z.number().optional(),
-	concurrency: z.number().optional(),
-	defaultContext: z.record(z.unknown()).optional(),
-	expandedTestIds: z.array(z.string()).optional()
-})
-
-export type RunConfig = z.infer<typeof RunConfigSchema>
-
 export interface RunResult {
 	success: boolean
 	result: string
@@ -67,6 +55,7 @@ export type LatestRunResultSummary = Pick<RunResult, 'success' | 'result' | 'eva
 export interface TestTable {
 	rows: TestRow[]
 	headers: string[]
+	settingsKey: string
 }
 
 export interface TestRunHistorySummary {
@@ -104,6 +93,7 @@ export interface TestRunHistoryDetail {
 }
 
 const defaultTestRow: Omit<TestRow, 'id'> = {
+	order: 0,
 	pinned: 'FALSE',
 	scenario: 'New scenario',
 	author: '',
@@ -130,7 +120,7 @@ export async function loadTests(agent: string): Promise<TestTable> {
 			let id = String(r.id || '').trim()
 			if (!id || seenIds.has(id)) {
 				changed = true
-				id = globalThis.crypto?.randomUUID?.() || randomUUID()
+				id = randomUUID()
 			}
 			seenIds.add(id)
 			// previously tests were storing context as a string...
@@ -159,7 +149,9 @@ export async function loadTests(agent: string): Promise<TestTable> {
 			'context',
 			'mocks'
 		]
-		return { rows, headers }
+		const projectPath = path.dirname(await getRootPath())
+		const workspaceId = createHash('sha256').update(projectPath).digest('hex').slice(0, 16)
+		return { rows, headers, settingsKey: `bosun:test-settings:${workspaceId}:${agent}` }
 	})
 }
 
@@ -218,71 +210,32 @@ export async function patchTestsBatch(
 }
 
 // Add a new row at the top
-export async function addTest(
-	agent: string,
-	mergeContexts: boolean,
-	seed?: Partial<TestRow>
-): Promise<TestRow> {
+export async function addTest(agent: string, seed?: Partial<TestRow>): Promise<TestRow> {
 	return withSuiteMutationLock(agent, async () => {
-		await ensureConfigExists(agent)
 		const suite = await loadSuiteOrEmpty(agent)
-		const id = globalThis.crypto?.randomUUID?.() || randomUUID()
+		const id = randomUUID()
+		const firstOrder = suite.tests.reduce(
+			(minimum, row) => Math.min(minimum, row.order),
+			Number.POSITIVE_INFINITY
+		)
 
 		const newRow: TestRow = {
 			...defaultTestRow,
 			id,
+			order: Number.isFinite(firstOrder) ? firstOrder - 1024 : 0,
 			...seed,
-			context: mergeContexts
-				? makeContextDiff(seed?.context, suite.meta?.defaultContext)
-				: makeContext(seed?.context)
+			context: makeContext(seed?.context)
 		}
 		if (typeof seed?.mocks !== 'undefined') {
 			newRow.mocks = makeContext(seed.mocks)
 		}
-		await writeSuite(agent, { ...(suite as any), tests: [newRow, ...suite.tests] })
-		await ensureTestExpandedByDefault(agent, newRow.id)
+		await writeSuite(agent, { tests: [newRow, ...suite.tests] })
 		return newRow
 	})
 }
 
-async function ensureTestExpandedByDefault(agent: string, id: string): Promise<void> {
-	const normalizedId = String(id ?? '').trim()
-	if (!normalizedId) return
-
-	const cfg = (await tryReadConfig(agent)) || {}
-	const existingIds = Array.isArray((cfg as any).expandedTestIds)
-		? (cfg as any).expandedTestIds
-				.map((v: unknown) => (typeof v === 'string' ? v.trim() : ''))
-				.filter((v: string) => v.length > 0)
-		: []
-
-	if (existingIds.includes(normalizedId)) return
-	await writeConfig(agent, { ...(cfg as any), expandedTestIds: [...existingIds, normalizedId] })
-}
-
-async function ensureConfigExists(agent: string): Promise<void> {
-	const existing = await tryReadConfig(agent)
-	if (existing) return
-	await writeConfig(agent, { repeats: 1, concurrency: 75, expandedTestIds: [] })
-}
-
 async function loadSuiteOrEmpty(agent: string): Promise<TestSuite> {
-	const jsonlPath = await getSuiteFilePath(agent)
-	const legacyPath = jsonlPath.replace(/\.jsonl$/, '.json')
-
-	const exists = async (p: string) => {
-		try {
-			await access(p)
-			return true
-		} catch {
-			return false
-		}
-	}
-
-	if (await exists(jsonlPath)) return readSuite(agent)
-	if (await exists(legacyPath)) return readSuite(agent)
-
-	return { version: 2, meta: {}, tests: [] }
+	return readSuite(agent)
 }
 
 export async function deleteTest(agent: string, id: string): Promise<boolean> {
@@ -303,63 +256,16 @@ export async function reorderTests(agent: string, order: string[]): Promise<void
 		for (const id of order) {
 			const r = map.get(id)
 			if (r) {
-				next.push(r)
+				next.push({ ...r, order: next.length * 1024 })
 				map.delete(id)
 			}
 		}
 		// Append any rows that were not included in order (safety)
-		for (const r of map.values()) next.push(r)
+		for (const r of map.values()) next.push({ ...r, order: next.length * 1024 })
 		await writeSuite(agent, { ...suite, tests: next })
 	})
 }
 
-export async function loadConfig(agent: string): Promise<RunConfig> {
-	const cfg = (await tryReadConfig(agent)) || {}
-	// Always source defaultContext from the suite meta, not config
-	let defaultContext: Record<string, unknown> = {}
-	try {
-		const suite = await readSuite(agent)
-		defaultContext = (suite.meta?.defaultContext as any) || {}
-	} catch {
-		// ignore
-	}
-
-	const expandedTestIds = Array.isArray((cfg as any).expandedTestIds)
-		? (cfg as any).expandedTestIds.filter((id: unknown) => typeof id === 'string')
-		: []
-	return {
-		repeats: typeof (cfg as any).repeats === 'number' ? (cfg as any).repeats : 1,
-		concurrency: typeof (cfg as any).concurrency === 'number' ? (cfg as any).concurrency : 75,
-		defaultContext,
-		expandedTestIds
-	}
-}
-
-export async function saveConfig(agent: string, cfg: RunConfig): Promise<void> {
-	// Merge patch into existing config
-	const current = (await tryReadConfig(agent)) || {}
-	const expandedTestIds = Array.isArray(cfg.expandedTestIds)
-		? Array.from(
-				new Set(
-					cfg.expandedTestIds.map((id) => String(id || '').trim()).filter((id) => id.length > 0)
-				)
-			)
-		: Array.isArray((current as any).expandedTestIds)
-			? (current as any).expandedTestIds
-			: undefined
-	const next = {
-		repeats: typeof cfg.repeats === 'number' ? cfg.repeats : (current as any).repeats,
-		concurrency:
-			typeof cfg.concurrency === 'number' ? cfg.concurrency : (current as any).concurrency,
-		expandedTestIds
-	}
-	await writeConfig(agent, next)
-	// Persist defaultContext exclusively in the suite meta (do not write to config)
-	if (cfg.defaultContext) {
-		await upsertSuiteMeta(agent, { defaultContext: cfg.defaultContext })
-	}
-}
-
 // We no longer migrate legacy reports automatically. Please update old reports manually.
 
 type ReportFileState = {
@@ -750,15 +656,6 @@ export async function getRunHistory(
 	}
 }
 
-// Fallback UUID for Node environments without global crypto
-function randomUUID() {
-	const rnd = () =>
-		Math.floor(Math.random() * 0xffffffff)
-			.toString(16)
-			.padStart(8, '0')
-	return `${rnd()}-${rnd().slice(0, 4)}-${rnd().slice(0, 4)}-${rnd().slice(0, 4)}-${rnd()}${rnd()}`
-}
-
 async function getRun(agent: string, ts?: string) {
 	const projectRoot = path.dirname(await getRootPath())
 	const outDir = path.join(projectRoot, 'data', 'test_runs')

package/src/backend/tests/pii-sanitized-test-drafts.spec.ts

@@ -41,6 +41,7 @@ function sanitized(input: TraceTestSeed, replacement = 'Maya Turner'): PIISaniti
 function testRow(input: TraceTestSeed): TestRow {
 	return {
 		id: 'created-test',
+		order: 0,
 		scenario: '',
 		author: '',
 		category: '',
@@ -51,7 +52,7 @@ function testRow(input: TraceTestSeed): TestRow {
 describe('PII-sanitised test drafts', () => {
 	it('persists an unchanged server-created draft exactly once', async () => {
 		const sanitize = vi.fn(async (input: TraceTestSeed) => sanitized(input))
-		const persist = vi.fn(async (_suiteKey, _mergeContexts, input: TraceTestSeed) => testRow(input))
+		const persist = vi.fn(async (_suiteKey, input: TraceTestSeed) => testRow(input))
 		const draft = await createPIISanitizedTestDraft(
 			{ suiteKey: 'agent.en-gb', seed: seed() },
 			{ sanitize, now: () => 1_000 }
@@ -73,12 +74,12 @@ describe('PII-sanitised test drafts', () => {
 		expect(result.status).toBe('created')
 		expect(sanitize).toHaveBeenCalledTimes(1)
 		expect(persist).toHaveBeenCalledOnce()
-		expect(JSON.stringify(persist.mock.calls[0][2])).not.toContain('Sarah')
+		expect(JSON.stringify(persist.mock.calls[0][1])).not.toContain('Sarah')
 	})
 
 	it('persists reviewed edits immediately without sanitizing them again', async () => {
 		const sanitize = vi.fn(async (input: TraceTestSeed) => sanitized(input))
-		const persist = vi.fn(async (_suiteKey, _mergeContexts, input: TraceTestSeed) => testRow(input))
+		const persist = vi.fn(async (_suiteKey, input: TraceTestSeed) => testRow(input))
 		const draft = await createPIISanitizedTestDraft(
 			{ suiteKey: 'agent', seed: seed() },
 			{ sanitize, now: () => 1_000 }
@@ -100,7 +101,7 @@ describe('PII-sanitised test drafts', () => {
 		expect(result.status).toBe('created')
 		expect(sanitize).toHaveBeenCalledOnce()
 		expect(persist).toHaveBeenCalledOnce()
-		expect(persist.mock.calls[0][2].conversation).toBe('User: I am Sarah Khan.')
+		expect(persist.mock.calls[0][1].conversation).toBe('User: I am Sarah Khan.')
 	})
 
 	it('rejects a draft used for a different suite', async () => {
@@ -128,7 +129,7 @@ describe('PII-sanitised test drafts', () => {
 	it('allows only one create operation to consume a draft', async () => {
 		let finishPersist!: (row: TestRow) => void
 		const persist = vi.fn(
-			async (_suiteKey, _mergeContexts, _input: TraceTestSeed) =>
+			async (_suiteKey, _input: TraceTestSeed) =>
 				new Promise<TestRow>((resolve) => {
 					finishPersist = resolve
 				})

package/src/backend/tests/pii-sanitized-test-drafts.ts

@@ -26,7 +26,7 @@ type StoredDraft = {
 const drafts = new Map<string, StoredDraft>()
 
 type Sanitize = (seed: TraceTestSeed) => Promise<PIISanitizationResult>
-type Persist = (suiteKey: string, mergeContexts: boolean, seed: TraceTestSeed) => Promise<TestRow>
+type Persist = (suiteKey: string, seed: TraceTestSeed) => Promise<TestRow>
 
 export async function createPIISanitizedTestDraft(
 	input: { suiteKey: string; seed: TraceTestSeed },
@@ -76,7 +76,7 @@ export async function createTestFromPIISanitizedTestDraft(
 		})
 
 		const persist = dependencies.persist ?? addTest
-		const test = await persist(stored.suiteKey, false, reviewedSeed)
+		const test = await persist(stored.suiteKey, reviewedSeed)
 		drafts.delete(input.draftId)
 		return { status: 'created', test }
 	} catch (error) {

package/src/backend/tests/router.ts

@@ -1,7 +1,5 @@
 import type { TRPCRootObject } from '@trpc/server/unstable-core-do-not-import'
 import { z } from 'zod'
-import { access, copyFile, mkdir, readFile, writeFile } from 'node:fs/promises'
-import path from 'node:path'
 
 import {
 	addTest,
@@ -9,22 +7,17 @@ import {
 	getResultEntry,
 	getRunHistory,
 	listRunHistory,
-	loadConfig,
 	loadLatestResultsByTest,
 	loadTests,
 	patchTest,
 	patchTestsBatch,
-	reorderTests,
-	RunConfigSchema,
-	saveConfig
+	reorderTests
 } from './data'
 import { readSuite, TestRowSchema } from './suite'
 import { cancelPromptTests, getPromptTestsProgress, runPromptTestsManaged } from './academy'
 import { analyzeRunQuotes } from './analyze'
 import { findPromptIssues } from './prompt-issues'
 import type { TrpcContext } from '../lib/trpc-context'
-import { getRootPath } from '../lib/get-root-path'
-import { type BosunAgentMeta } from '../lib/get-agent-factories'
 import { getProjectBosunApi } from '../lib/project-api'
 import {
 	TraceTestReviewContentSchema,
@@ -35,68 +28,6 @@ import {
 	createTestFromPIISanitizedTestDraft
 } from './pii-sanitized-test-drafts'
 
-async function fileExists(p: string): Promise<boolean> {
-	try {
-		await access(p)
-		return true
-	} catch {
-		return false
-	}
-}
-
-async function suiteOrConfigExists(baseDir: string, suiteKey: string): Promise<boolean> {
-	const jsonl = path.resolve(baseDir, `${suiteKey}.tests.jsonl`)
-	const legacyJson = path.resolve(baseDir, `${suiteKey}.tests.json`)
-	const cfg = path.resolve(baseDir, `${suiteKey}.config.json`)
-	return (await fileExists(jsonl)) || (await fileExists(legacyJson)) || (await fileExists(cfg))
-}
-
-async function migrateLegacySuiteToDefaultRegion(
-	baseDir: string,
-	agent: string,
-	meta: BosunAgentMeta | null
-): Promise<void> {
-	if (!meta || meta.availableRegions.length === 0) return
-
-	const defaultRegion =
-		meta.defaultRegion && meta.availableRegions.includes(meta.defaultRegion)
-			? meta.defaultRegion
-			: null
-	if (!defaultRegion) return
-
-	const defaultSuiteKey = `${agent}.${defaultRegion}`
-	if (await suiteOrConfigExists(baseDir, defaultSuiteKey)) return
-	if (!(await suiteOrConfigExists(baseDir, agent))) return
-
-	// Non-trivial: one-time migration so the first region-aware run does NOT silently fall back to the
-	// legacy suite/config (which would incorrectly show the same tests for every region).
-	const legacyJsonl = path.resolve(baseDir, `${agent}.tests.jsonl`)
-	const legacyJson = path.resolve(baseDir, `${agent}.tests.json`)
-	const legacyCfg = path.resolve(baseDir, `${agent}.config.json`)
-
-	const targetJsonl = path.resolve(baseDir, `${defaultSuiteKey}.tests.jsonl`)
-	const targetCfg = path.resolve(baseDir, `${defaultSuiteKey}.config.json`)
-
-	await mkdir(baseDir, { recursive: true })
-
-	if (!(await fileExists(targetJsonl)) && (await fileExists(legacyJsonl))) {
-		await copyFile(legacyJsonl, targetJsonl)
-	} else if (!(await fileExists(targetJsonl)) && (await fileExists(legacyJson))) {
-		// Convert legacy JSON suite to JSONL so the new file matches the expected layout.
-		const raw = await readFile(legacyJson, 'utf8')
-		const parsed = JSON.parse(raw)
-		const metaObj = parsed?.meta && typeof parsed.meta === 'object' ? parsed.meta : {}
-		const tests = Array.isArray(parsed?.tests) ? parsed.tests : []
-		const header = { version: 1, meta: metaObj }
-		const lines = [JSON.stringify(header), ...tests.map((t: any) => JSON.stringify(t))]
-		await writeFile(targetJsonl, lines.join('\n') + '\n', 'utf8')
-	}
-
-	if (!(await fileExists(targetCfg)) && (await fileExists(legacyCfg))) {
-		await copyFile(legacyCfg, targetCfg)
-	}
-}
-
 type SuiteSelection = {
 	suiteKey: string
 	agentId: string
@@ -112,16 +43,11 @@ async function resolveSuiteSelection(
 	// - `suiteKey` is the filesystem/report key:
 	//    - non-regional projects: `${agentId}`
 	//    - regional projects: `${agentId}.${region}`
-	// We never substitute legacy tests for the wrong region.
-	const baseDir = path.dirname(await getRootPath())
-
 	const bosunApi = await getProjectBosunApi()
 	const meta = bosunApi ? await bosunApi.getAgentMeta(agentId) : null
 	const isRegionScoped = meta?.testScope === 'region'
 	if (!isRegionScoped) return { suiteKey: agentId, agentId }
 
-	await migrateLegacySuiteToDefaultRegion(baseDir, agentId, meta)
-
 	if (!bosunApi) {
 		throw new Error(`Project does not export bosunApi, cannot resolve suite for '${agentId}'.`)
 	}
@@ -142,18 +68,6 @@ export function createTestsRouter(t: TRPCRootObject<TrpcContext, any, any>) {
 				const { suiteKey } = await resolveSuiteSelection(input.agent, input.region)
 				return loadTests(suiteKey)
 			}),
-		config: t.procedure
-			.input(z.object({ agent: z.string(), region: z.string().nullish() }))
-			.query(async ({ input }) => {
-				const { suiteKey } = await resolveSuiteSelection(input.agent, input.region)
-				return loadConfig(suiteKey)
-			}),
-		updateConfig: t.procedure
-			.input(z.object({ agent: z.string(), region: z.string().nullish(), cfg: RunConfigSchema }))
-			.mutation(async ({ input }) => {
-				const { suiteKey } = await resolveSuiteSelection(input.agent, input.region)
-				return saveConfig(suiteKey, input.cfg)
-			}),
 		latestResult: t.procedure
 			.input(z.object({ agent: z.string(), region: z.string().nullish() }))
 			.query(async ({ input }) => {
@@ -247,13 +161,12 @@ export function createTestsRouter(t: TRPCRootObject<TrpcContext, any, any>) {
 				z.object({
 					agent: z.string(),
 					region: z.string().nullish(),
-					mergeContexts: z.boolean(),
-					seed: TestRowSchema.partial()
+					seed: TestRowSchema.omit({ id: true, order: true }).partial()
 				})
 			)
 			.mutation(async ({ input }) => {
 				const { suiteKey } = await resolveSuiteSelection(input.agent, input.region)
-				return addTest(suiteKey, input.mergeContexts, input.seed)
+				return addTest(suiteKey, input.seed)
 			}),
 		createPIISanitizedTestDraft: t.procedure
 			.input(
@@ -290,7 +203,7 @@ export function createTestsRouter(t: TRPCRootObject<TrpcContext, any, any>) {
 					agent: z.string(),
 					region: z.string().nullish(),
 					id: z.string(),
-					patch: TestRowSchema.partial()
+					patch: TestRowSchema.omit({ id: true, order: true }).partial()
 				})
 			)
 			.mutation(async ({ input }) => {
@@ -302,7 +215,12 @@ export function createTestsRouter(t: TRPCRootObject<TrpcContext, any, any>) {
 				z.object({
 					agent: z.string(),
 					region: z.string().nullish(),
-					updates: z.array(z.object({ id: z.string(), patch: TestRowSchema.partial() }))
+					updates: z.array(
+						z.object({
+							id: z.string(),
+							patch: TestRowSchema.omit({ id: true, order: true }).partial()
+						})
+					)
 				})
 			)
 			.mutation(async ({ input }) => {
@@ -331,27 +249,36 @@ export function createTestsRouter(t: TRPCRootObject<TrpcContext, any, any>) {
 							agent: z.string(),
 							region: z.string().nullish(),
 							step: z.string(),
-							scope: z.enum(['pinned', 'all']).optional()
+							scope: z.enum(['pinned', 'all']).optional(),
+							repeats: z.number().int().min(1).max(500),
+							concurrency: z.number().int().min(1).max(75)
 						})
 						.strict(),
 					z
 						.object({
 							agent: z.string(),
 							region: z.string().nullish(),
-							testId: z.string()
+							testId: z.string(),
+							repeats: z.number().int().min(1).max(500),
+							concurrency: z.number().int().min(1).max(75)
 						})
 						.strict()
 				])
 			)
 			.mutation(async ({ input }) => {
 				const selection = await resolveSuiteSelection(input.agent, input.region)
+				const runConfig = { repeats: input.repeats, concurrency: input.concurrency }
 				if ('testId' in input) {
-					return runPromptTestsManaged(selection, { testId: input.testId })
+					return runPromptTestsManaged(selection, { testId: input.testId }, runConfig)
 				}
-				return runPromptTestsManaged(selection, {
-					scope: input.scope ?? 'pinned',
-					step: input.step
-				})
+				return runPromptTestsManaged(
+					selection,
+					{
+						scope: input.scope ?? 'pinned',
+						step: input.step
+					},
+					runConfig
+				)
 			}),
 		cancelRun: t.procedure
 			.input(z.object({ agent: z.string(), region: z.string().nullish() }))

package/src/backend/tests/suite.spec.ts

@@ -1,13 +1,14 @@
-import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
+import { mkdtemp, mkdir, readFile, readdir, rm, writeFile } from 'node:fs/promises'
 import os from 'node:os'
 import path from 'node:path'
 import { afterEach, describe, expect, it, vi } from 'vitest'
 
-const root = vi.hoisted(() => ({ path: '' }))
+const project = vi.hoisted(() => ({ basePath: '', rootPath: '' }))
 
-vi.mock('../lib/get-root-path', () => ({ getRootPath: async () => root.path }))
+vi.mock('../lib/project-api', () => ({ getProjectBasePath: async () => project.basePath }))
+vi.mock('../lib/get-root-path', () => ({ getRootPath: async () => project.rootPath }))
 
-import { readSuite } from './suite'
+import { LEGACY_TEST_SUITE_ERROR_PREFIX, readSuite, writeSuite, type TestRow } from './suite'
 
 const temporaryDirectories: string[] = []
 
@@ -15,40 +16,87 @@ afterEach(async () => {
 	await Promise.all(temporaryDirectories.splice(0).map((dir) => rm(dir, { recursive: true })))
 })
 
-describe('test suite migration', () => {
-	it('moves every version-one test to main and removes final capture mode', async () => {
-		const dir = await mkdtemp(path.join(os.tmpdir(), 'bosun-suite-migration-'))
-		temporaryDirectories.push(dir)
-		root.path = path.join(dir, 'index.ts')
-		const suitePath = path.join(dir, 'agent.tests.jsonl')
+function row(id: string, order: number): TestRow {
+	return {
+		id,
+		order,
+		pinned: 'TRUE',
+		scenario: id,
+		step: 'main',
+		author: '',
+		category: '',
+		assertion_type: 'llm-rubric',
+		assertion_value: '',
+		conversation: 'User: hello',
+		expected_output: 'Hello',
+		evaluation_mode: 'assertion',
+		context_mode: 'merge',
+		timeout: '0',
+		similarity_threshold: '50',
+		context: {}
+	}
+}
+
+async function makeSuite(suiteKey = 'agent') {
+	const dir = await mkdtemp(path.join(os.tmpdir(), 'bosun-yaml-suite-'))
+	temporaryDirectories.push(dir)
+	project.basePath = path.join(dir, 'agents')
+	project.rootPath = path.join(dir, 'bosun', 'workflows.ts')
+	const suiteDirectory = path.join(project.basePath, 'support', 'bosun-tests', suiteKey)
+	await mkdir(suiteDirectory, { recursive: true })
+	return suiteDirectory
+}
+
+describe('YAML test suites', () => {
+	it('uses file names as ids and sorts tests by their stored order', async () => {
+		const suiteDirectory = await makeSuite()
+		await writeFile(
+			path.join(suiteDirectory, 'second.yaml'),
+			'order: 1024\npinned: "TRUE"\nscenario: second\nassertion_type: llm-rubric\nassertion_value: ""\nconversation: ""\nexpected_output: ""\ntimeout: "0"\nsimilarity_threshold: "50"\ncontext: {}\n'
+		)
 		await writeFile(
-			suitePath,
-			[
-				JSON.stringify({ version: 1, meta: {} }),
-				JSON.stringify({
-					id: 'old',
-					pinned: 'TRUE',
-					scenario: 'Old test',
-					step: 'experimental-step',
-					assertion_type: 'llm-rubric',
-					assertion_value: '',
-					conversation: 'User: hello',
-					expected_output: 'Hello',
-					capture_mode: 'final',
-					timeout: '0',
-					similarity_threshold: '50',
-					context: {}
-				})
-			].join('\n') + '\n'
+			path.join(suiteDirectory, 'first.yaml'),
+			'order: 0\npinned: "TRUE"\nscenario: first\nassertion_type: llm-rubric\nassertion_value: ""\nconversation: ""\nexpected_output: ""\ntimeout: "0"\nsimilarity_threshold: "50"\ncontext: {}\n'
 		)
 
 		const suite = await readSuite('agent')
 
-		expect(suite.version).toBe(2)
-		expect(suite.tests[0]).toMatchObject({ step: 'main', evaluation_mode: 'assertion' })
-		expect(suite.tests[0]).not.toHaveProperty('capture_mode')
-		const persisted = await readFile(suitePath, 'utf8')
-		expect(JSON.parse(persisted.split('\n')[0])).toMatchObject({ version: 2 })
-		expect(persisted).not.toContain('capture_mode')
+		expect(suite.tests.map((test) => test.id)).toEqual(['first', 'second'])
 	})
+
+	it('adds and edits independent files and deletes removed tests', async () => {
+		const suiteDirectory = await makeSuite()
+		await writeSuite('agent', { tests: [row('one', 0), row('two', 1024)] })
+		const twoBefore = await readFile(path.join(suiteDirectory, 'two.yaml'), 'utf8')
+
+		await writeSuite('agent', {
+			tests: [{ ...row('one', 0), scenario: 'edited' }, row('two', 1024), row('three', 2048)]
+		})
+
+		expect(await readFile(path.join(suiteDirectory, 'two.yaml'), 'utf8')).toBe(twoBefore)
+		expect((await readdir(suiteDirectory)).sort()).toEqual(['one.yaml', 'three.yaml', 'two.yaml'])
+
+		await writeSuite('agent', { tests: [row('two', 1024), row('three', 2048)] })
+		expect((await readdir(suiteDirectory)).sort()).toEqual(['three.yaml', 'two.yaml'])
+	})
+
+	it.each(['tests.jsonl', 'tests.json', 'config.json'])(
+		'reports that a legacy %s suite requires migration',
+		async (suffix) => {
+			const dir = await mkdtemp(path.join(os.tmpdir(), 'bosun-legacy-suite-'))
+			temporaryDirectories.push(dir)
+			project.basePath = path.join(dir, 'agents')
+			project.rootPath = path.join(dir, 'bosun', 'workflows.ts')
+			await mkdir(project.basePath, { recursive: true })
+			await mkdir(path.dirname(project.rootPath), { recursive: true })
+			await mkdir(path.join(project.basePath, 'support', 'bosun-tests', 'agent'), {
+				recursive: true
+			})
+			const legacyPath = path.join(path.dirname(project.rootPath), `agent.${suffix}`)
+			await writeFile(legacyPath, '{}\n')
+
+			await expect(readSuite('agent')).rejects.toThrow(LEGACY_TEST_SUITE_ERROR_PREFIX)
+			expect(await readFile(legacyPath, 'utf8')).toBe('{}\n')
+		}
+	)
 })