@recombine-ai/bosun 0.16.0 → 0.16.2-cloud-1

package/src/backend/tests/report-files.ts

@@ -1,120 +0,0 @@
-import { createReadStream, createWriteStream } from 'node:fs'
-import { readFile, rename, unlink } from 'node:fs/promises'
-import path from 'node:path'
-import { pipeline } from 'node:stream/promises'
-import { promisify } from 'node:util'
-import { createGzip, gunzip } from 'node:zlib'
-
-export const REPORT_JSON_SUFFIX = '.report.json'
-export const REPORT_GZIP_SUFFIX = `${REPORT_JSON_SUFFIX}.gz`
-
-const gunzipAsync = promisify(gunzip)
-
-function logicalReportName(name: string): string | null {
-	if (name.endsWith(REPORT_GZIP_SUFFIX)) return name.slice(0, -'.gz'.length)
-	if (name.endsWith(REPORT_JSON_SUFFIX)) return name
-	return null
-}
-
-export function listAgentReportFiles(entries: string[], agent: string): string[] {
-	const prefix = `${agent}.`
-	const filesByLogicalName = new Map<string, string>()
-
-	for (const name of entries) {
-		const logicalName = logicalReportName(name)
-		if (!logicalName?.startsWith(prefix)) continue
-
-		const existing = filesByLogicalName.get(logicalName)
-		// Once the complete gzip file has been atomically installed, prefer it during the
-		// brief period before its plain JSON counterpart is removed.
-		if (!existing || name.endsWith(REPORT_GZIP_SUFFIX)) {
-			filesByLogicalName.set(logicalName, name)
-		}
-	}
-
-	return [...filesByLogicalName.entries()]
-		.sort(([left], [right]) => left.localeCompare(right))
-		.map(([, name]) => name)
-}
-
-export function reportTimestamp(name: string, agent: string): string {
-	const logicalName = logicalReportName(name)
-	const prefix = `${agent}.`
-	if (!logicalName?.startsWith(prefix)) throw new Error(`Invalid report filename: ${name}`)
-	return logicalName.slice(prefix.length, -REPORT_JSON_SUFFIX.length).replace(/-/g, ':')
-}
-
-function alternateReportPath(filePath: string): string | null {
-	if (filePath.endsWith(REPORT_GZIP_SUFFIX)) return filePath.slice(0, -'.gz'.length)
-	if (filePath.endsWith(REPORT_JSON_SUFFIX)) return `${filePath}.gz`
-	return null
-}
-
-function isMissingFileError(error: unknown): error is NodeJS.ErrnoException {
-	return error instanceof Error && 'code' in error && error.code === 'ENOENT'
-}
-
-export async function readReportFile<T>(filePath: string): Promise<T> {
-	let resolvedPath = filePath
-	let raw: Buffer
-	try {
-		raw = await readFile(resolvedPath)
-	} catch (error) {
-		const alternatePath = alternateReportPath(resolvedPath)
-		if (!alternatePath || !isMissingFileError(error)) throw error
-		resolvedPath = alternatePath
-		raw = await readFile(resolvedPath)
-	}
-	const json = resolvedPath.endsWith(REPORT_GZIP_SUFFIX) ? await gunzipAsync(raw) : raw
-	return JSON.parse(json.toString('utf8')) as T
-}
-
-export async function compressReportFile(filePath: string): Promise<string> {
-	if (!filePath.endsWith(REPORT_JSON_SUFFIX)) {
-		throw new Error(`Cannot gzip a non-JSON report: ${filePath}`)
-	}
-
-	const gzipPath = `${filePath}.gz`
-	const tmpPath = path.join(
-		path.dirname(filePath),
-		`.${path.basename(gzipPath)}.${process.pid}.${Date.now()}.${Math.random().toString(16).slice(2)}.tmp`
-	)
-
-	try {
-		await pipeline(
-			createReadStream(filePath),
-			createGzip(),
-			createWriteStream(tmpPath, { flags: 'wx' })
-		)
-		try {
-			await rename(tmpPath, gzipPath)
-		} catch {
-			// Windows does not allow atomic rename over an existing file.
-			try {
-				await unlink(gzipPath)
-			} catch {
-				// ignore
-			}
-			await rename(tmpPath, gzipPath)
-		}
-		await unlink(filePath)
-		return gzipPath
-	} finally {
-		try {
-			await unlink(tmpPath)
-		} catch {
-			// The temporary file was renamed or never created.
-		}
-	}
-}
-
-let compressionQueue: Promise<void> = Promise.resolve()
-
-export function queueReportCompression(filePath: string): Promise<string> {
-	const compression = compressionQueue.catch(() => {}).then(() => compressReportFile(filePath))
-	compressionQueue = compression.then(
-		() => undefined,
-		() => undefined
-	)
-	return compression
-}

package/src/backend/tests/router.ts

@@ -1,290 +0,0 @@
-import type { TRPCRootObject } from '@trpc/server/unstable-core-do-not-import'
-import { z } from 'zod'
-
-import {
-	addTest,
-	deleteTest,
-	getResultEntry,
-	getRunHistory,
-	listRunHistory,
-	loadLatestResultsByTest,
-	loadTests,
-	patchTest,
-	patchTestsBatch,
-	reorderTests
-} from './data'
-import { readSuite, TestRowSchema } from './suite'
-import { cancelPromptTests, getPromptTestsProgress, runPromptTestsManaged } from './academy'
-import { analyzeRunQuotes } from './analyze'
-import { findPromptIssues } from './prompt-issues'
-import type { TrpcContext } from '../lib/trpc-context'
-import { getProjectBosunApi } from '../lib/project-api'
-import {
-	TraceTestReviewContentSchema,
-	TraceTestSeedSchema
-} from '../../lib/tests/pii-sanitized-test-draft'
-import {
-	createPIISanitizedTestDraft,
-	createTestFromPIISanitizedTestDraft
-} from './pii-sanitized-test-drafts'
-
-type SuiteSelection = {
-	suiteKey: string
-	agentId: string
-	region?: string
-}
-
-async function resolveSuiteSelection(
-	agentId: string,
-	region?: string | null
-): Promise<SuiteSelection> {
-	// Convention:
-	// - `agentId` is the canonical factory key (workflow export key).
-	// - `suiteKey` is the filesystem/report key:
-	//    - non-regional projects: `${agentId}`
-	//    - regional projects: `${agentId}.${region}`
-	const bosunApi = await getProjectBosunApi()
-	const meta = bosunApi ? await bosunApi.getAgentMeta(agentId) : null
-	const isRegionScoped = meta?.testScope === 'region'
-	if (!isRegionScoped) return { suiteKey: agentId, agentId }
-
-	if (!bosunApi) {
-		throw new Error(`Project does not export bosunApi, cannot resolve suite for '${agentId}'.`)
-	}
-
-	const resolved = await bosunApi.resolveTestSuite({ agentId, region })
-	return {
-		suiteKey: resolved.suiteKey,
-		agentId,
-		region: resolved.resolvedRegion ?? undefined
-	}
-}
-
-export function createTestsRouter(t: TRPCRootObject<TrpcContext, any, any>) {
-	return t.router({
-		list: t.procedure
-			.input(z.object({ agent: z.string(), region: z.string().nullish() }))
-			.query(async ({ input }) => {
-				const { suiteKey } = await resolveSuiteSelection(input.agent, input.region)
-				return loadTests(suiteKey)
-			}),
-		latestResult: t.procedure
-			.input(z.object({ agent: z.string(), region: z.string().nullish() }))
-			.query(async ({ input }) => {
-				const { suiteKey } = await resolveSuiteSelection(input.agent, input.region)
-				return loadLatestResultsByTest(suiteKey)
-			}),
-		runProgress: t.procedure
-			.input(z.object({ agent: z.string(), region: z.string().nullish() }))
-			.query(async ({ input }) => {
-				const { suiteKey } = await resolveSuiteSelection(input.agent, input.region)
-				return getPromptTestsProgress(suiteKey)
-			}),
-		historyList: t.procedure
-			.input(z.object({ agent: z.string(), region: z.string().nullish() }))
-			.query(async ({ input }) => {
-				const { suiteKey } = await resolveSuiteSelection(input.agent, input.region)
-				return listRunHistory(suiteKey)
-			}),
-		historyRun: t.procedure
-			.input(
-				z.object({
-					agent: z.string(),
-					region: z.string().nullish(),
-					timestamp: z.string()
-				})
-			)
-			.query(async ({ input }) => {
-				const { suiteKey } = await resolveSuiteSelection(input.agent, input.region)
-				return getRunHistory(suiteKey, input.timestamp)
-			}),
-		resultEntry: t.procedure
-			.input(
-				z.object({
-					agent: z.string(),
-					region: z.string().nullish(),
-					testId: z.string(),
-					// TODO resultKey should be keyof of ResultEntry
-					resultKey: z.string(),
-					index: z.number().optional(),
-					timestamp: z.string().optional()
-				})
-			)
-			.query(async ({ input }) => {
-				const { suiteKey } = await resolveSuiteSelection(input.agent, input.region)
-				return getResultEntry(
-					suiteKey,
-					input.testId,
-					input.resultKey as any,
-					input.index,
-					input.timestamp
-				)
-			}),
-		analyzeRun: t.procedure
-			.input(
-				z.object({
-					agent: z.string(),
-					region: z.string().nullish(),
-					testId: z.string(),
-					index: z.number().int().nonnegative(),
-					timestamp: z.string().optional()
-				})
-			)
-			.mutation(async ({ input }) => {
-				const { suiteKey } = await resolveSuiteSelection(input.agent, input.region)
-				return analyzeRunQuotes({
-					agent: suiteKey,
-					testId: input.testId,
-					index: input.index,
-					timestamp: input.timestamp
-				})
-			}),
-		findPromptIssues: t.procedure
-			.input(
-				z.object({
-					agent: z.string(),
-					step: z.string(),
-					prompt: z.string()
-				})
-			)
-			.mutation(async ({ input }) => {
-				return findPromptIssues(input)
-			}),
-		suite: t.procedure
-			.input(z.object({ agent: z.string(), region: z.string().nullish() }))
-			.query(async ({ input }) => {
-				const { suiteKey } = await resolveSuiteSelection(input.agent, input.region)
-				return readSuite(suiteKey)
-			}),
-		addTest: t.procedure
-			.input(
-				z.object({
-					agent: z.string(),
-					region: z.string().nullish(),
-					seed: TestRowSchema.omit({ id: true, order: true }).partial()
-				})
-			)
-			.mutation(async ({ input }) => {
-				const { suiteKey } = await resolveSuiteSelection(input.agent, input.region)
-				return addTest(suiteKey, input.seed)
-			}),
-		createPIISanitizedTestDraft: t.procedure
-			.input(
-				z.object({
-					agent: z.string(),
-					region: z.string().nullish(),
-					seed: TraceTestSeedSchema
-				})
-			)
-			.mutation(async ({ input }) => {
-				const { suiteKey } = await resolveSuiteSelection(input.agent, input.region)
-				return createPIISanitizedTestDraft({ suiteKey, seed: input.seed })
-			}),
-		createTestFromPIISanitizedTestDraft: t.procedure
-			.input(
-				z.object({
-					agent: z.string(),
-					region: z.string().nullish(),
-					draftId: z.string().uuid(),
-					reviewed: TraceTestReviewContentSchema
-				})
-			)
-			.mutation(async ({ input }) => {
-				const { suiteKey } = await resolveSuiteSelection(input.agent, input.region)
-				return createTestFromPIISanitizedTestDraft({
-					suiteKey,
-					draftId: input.draftId,
-					reviewed: input.reviewed
-				})
-			}),
-		patchTest: t.procedure
-			.input(
-				z.object({
-					agent: z.string(),
-					region: z.string().nullish(),
-					id: z.string(),
-					patch: TestRowSchema.omit({ id: true, order: true }).partial()
-				})
-			)
-			.mutation(async ({ input }) => {
-				const { suiteKey } = await resolveSuiteSelection(input.agent, input.region)
-				return patchTest(suiteKey, input.id, input.patch)
-			}),
-		patchTestsBatch: t.procedure
-			.input(
-				z.object({
-					agent: z.string(),
-					region: z.string().nullish(),
-					updates: z.array(
-						z.object({
-							id: z.string(),
-							patch: TestRowSchema.omit({ id: true, order: true }).partial()
-						})
-					)
-				})
-			)
-			.mutation(async ({ input }) => {
-				const { suiteKey } = await resolveSuiteSelection(input.agent, input.region)
-				return patchTestsBatch(suiteKey, input.updates)
-			}),
-		deleteTest: t.procedure
-			.input(z.object({ agent: z.string(), region: z.string().nullish(), id: z.string() }))
-			.mutation(async ({ input }) => {
-				const { suiteKey } = await resolveSuiteSelection(input.agent, input.region)
-				return deleteTest(suiteKey, input.id)
-			}),
-		reorder: t.procedure
-			.input(
-				z.object({ agent: z.string(), region: z.string().nullish(), order: z.array(z.string()) })
-			)
-			.mutation(async ({ input }) => {
-				const { suiteKey } = await resolveSuiteSelection(input.agent, input.region)
-				return reorderTests(suiteKey, input.order)
-			}),
-		runTests: t.procedure
-			.input(
-				z.union([
-					z
-						.object({
-							agent: z.string(),
-							region: z.string().nullish(),
-							step: z.string(),
-							scope: z.enum(['pinned', 'all']).optional(),
-							repeats: z.number().int().min(1).max(500),
-							concurrency: z.number().int().min(1).max(75)
-						})
-						.strict(),
-					z
-						.object({
-							agent: z.string(),
-							region: z.string().nullish(),
-							testId: z.string(),
-							repeats: z.number().int().min(1).max(500),
-							concurrency: z.number().int().min(1).max(75)
-						})
-						.strict()
-				])
-			)
-			.mutation(async ({ input }) => {
-				const selection = await resolveSuiteSelection(input.agent, input.region)
-				const runConfig = { repeats: input.repeats, concurrency: input.concurrency }
-				if ('testId' in input) {
-					return runPromptTestsManaged(selection, { testId: input.testId }, runConfig)
-				}
-				return runPromptTestsManaged(
-					selection,
-					{
-						scope: input.scope ?? 'pinned',
-						step: input.step
-					},
-					runConfig
-				)
-			}),
-		cancelRun: t.procedure
-			.input(z.object({ agent: z.string(), region: z.string().nullish() }))
-			.mutation(async ({ input }) => {
-				const { suiteKey } = await resolveSuiteSelection(input.agent, input.region)
-				return cancelPromptTests(suiteKey)
-			})
-	})
-}

package/src/backend/tests/suite.spec.ts

@@ -1,102 +0,0 @@
-import { mkdtemp, mkdir, readFile, readdir, rm, writeFile } from 'node:fs/promises'
-import os from 'node:os'
-import path from 'node:path'
-import { afterEach, describe, expect, it, vi } from 'vitest'
-
-const project = vi.hoisted(() => ({ basePath: '', rootPath: '' }))
-
-vi.mock('../lib/project-api', () => ({ getProjectBasePath: async () => project.basePath }))
-vi.mock('../lib/get-root-path', () => ({ getRootPath: async () => project.rootPath }))
-
-import { LEGACY_TEST_SUITE_ERROR_PREFIX, readSuite, writeSuite, type TestRow } from './suite'
-
-const temporaryDirectories: string[] = []
-
-afterEach(async () => {
-	await Promise.all(temporaryDirectories.splice(0).map((dir) => rm(dir, { recursive: true })))
-})
-
-function row(id: string, order: number): TestRow {
-	return {
-		id,
-		order,
-		pinned: 'TRUE',
-		scenario: id,
-		step: 'main',
-		author: '',
-		category: '',
-		assertion_type: 'llm-rubric',
-		assertion_value: '',
-		conversation: 'User: hello',
-		expected_output: 'Hello',
-		evaluation_mode: 'assertion',
-		context_mode: 'merge',
-		timeout: '0',
-		similarity_threshold: '50',
-		context: {}
-	}
-}
-
-async function makeSuite(suiteKey = 'agent') {
-	const dir = await mkdtemp(path.join(os.tmpdir(), 'bosun-yaml-suite-'))
-	temporaryDirectories.push(dir)
-	project.basePath = path.join(dir, 'agents')
-	project.rootPath = path.join(dir, 'bosun', 'workflows.ts')
-	const suiteDirectory = path.join(project.basePath, 'support', 'bosun-tests', suiteKey)
-	await mkdir(suiteDirectory, { recursive: true })
-	return suiteDirectory
-}
-
-describe('YAML test suites', () => {
-	it('uses file names as ids and sorts tests by their stored order', async () => {
-		const suiteDirectory = await makeSuite()
-		await writeFile(
-			path.join(suiteDirectory, 'second.yaml'),
-			'order: 1024\npinned: "TRUE"\nscenario: second\nassertion_type: llm-rubric\nassertion_value: ""\nconversation: ""\nexpected_output: ""\ntimeout: "0"\nsimilarity_threshold: "50"\ncontext: {}\n'
-		)
-		await writeFile(
-			path.join(suiteDirectory, 'first.yaml'),
-			'order: 0\npinned: "TRUE"\nscenario: first\nassertion_type: llm-rubric\nassertion_value: ""\nconversation: ""\nexpected_output: ""\ntimeout: "0"\nsimilarity_threshold: "50"\ncontext: {}\n'
-		)
-
-		const suite = await readSuite('agent')
-
-		expect(suite.tests.map((test) => test.id)).toEqual(['first', 'second'])
-	})
-
-	it('adds and edits independent files and deletes removed tests', async () => {
-		const suiteDirectory = await makeSuite()
-		await writeSuite('agent', { tests: [row('one', 0), row('two', 1024)] })
-		const twoBefore = await readFile(path.join(suiteDirectory, 'two.yaml'), 'utf8')
-
-		await writeSuite('agent', {
-			tests: [{ ...row('one', 0), scenario: 'edited' }, row('two', 1024), row('three', 2048)]
-		})
-
-		expect(await readFile(path.join(suiteDirectory, 'two.yaml'), 'utf8')).toBe(twoBefore)
-		expect((await readdir(suiteDirectory)).sort()).toEqual(['one.yaml', 'three.yaml', 'two.yaml'])
-
-		await writeSuite('agent', { tests: [row('two', 1024), row('three', 2048)] })
-		expect((await readdir(suiteDirectory)).sort()).toEqual(['three.yaml', 'two.yaml'])
-	})
-
-	it.each(['tests.jsonl', 'tests.json', 'config.json'])(
-		'reports that a legacy %s suite requires migration',
-		async (suffix) => {
-			const dir = await mkdtemp(path.join(os.tmpdir(), 'bosun-legacy-suite-'))
-			temporaryDirectories.push(dir)
-			project.basePath = path.join(dir, 'agents')
-			project.rootPath = path.join(dir, 'bosun', 'workflows.ts')
-			await mkdir(project.basePath, { recursive: true })
-			await mkdir(path.dirname(project.rootPath), { recursive: true })
-			await mkdir(path.join(project.basePath, 'support', 'bosun-tests', 'agent'), {
-				recursive: true
-			})
-			const legacyPath = path.join(path.dirname(project.rootPath), `agent.${suffix}`)
-			await writeFile(legacyPath, '{}\n')
-
-			await expect(readSuite('agent')).rejects.toThrow(LEGACY_TEST_SUITE_ERROR_PREFIX)
-			expect(await readFile(legacyPath, 'utf8')).toBe('{}\n')
-		}
-	)
-})