@recombine-ai/bosun 0.1.9 → 0.2.0-draft

package/build/academy/csv/index.ts

@@ -0,0 +1,155 @@
+import { readFile, writeFile, access } from 'node:fs/promises'
+import path from 'node:path'
+import { readdir } from 'node:fs/promises'
+
+// CSV functionality removed; this module only handles project paths and config JSON.
+
+// Resolve the base path for a given project (folder containing workflows.ts)
+export async function getProjectBasePath(project: string): Promise<string> {
+	const projectsDir = 'projects'
+	console.log('[academy] getProjectBasePath', { project, cwd: process.cwd(), projectsDir })
+
+	// Fast path: direct read (works with symlinks)
+	const directDir = path.join(projectsDir, project)
+	const directPkg = path.join(directDir, 'package.json')
+	try {
+		const raw = await readFile(directPkg, 'utf8')
+		const pkg = JSON.parse(raw)
+		const agentsRoot = pkg?.bosun?.root ?? 'src/bosun/workflows.ts'
+		const agentsRootPath = path.resolve(directDir, agentsRoot)
+		const basePath = path.dirname(agentsRootPath)
+		console.log('[academy] basePath (direct)', { directPkg, agentsRoot, basePath })
+		return basePath
+	} catch (e: any) {
+		console.warn('[academy] direct package.json read failed', { directPkg, error: e?.message })
+	}
+
+	// Fallback: scan projects directory (include symlinks)
+	let entries: any[] = []
+	try {
+		entries = await readdir(projectsDir, { withFileTypes: true } as any)
+		console.log(
+			'[academy] projects entries',
+			entries.map((ent: any) => ({
+				name: ent.name,
+				isDir: ent.isDirectory?.(),
+				isLink: ent.isSymbolicLink?.()
+			}))
+		)
+	} catch (err: any) {
+		console.warn('[academy] projects folder not found; using CWD as fallback', {
+			projectsDir,
+			error: err?.message
+		})
+		// Attempt to derive base path from cwd package.json (if present)
+		const cwd = process.cwd()
+		const cwdPkg = path.join(cwd, 'package.json')
+		try {
+			const raw = await readFile(cwdPkg, 'utf8')
+			const pkg = JSON.parse(raw)
+			const agentsRoot = pkg?.bosun?.root ?? 'src/bosun/workflows.ts'
+			const agentsRootPath = path.resolve(cwd, agentsRoot)
+			const basePath = path.dirname(agentsRootPath)
+			console.log('[academy] basePath (cwd fallback with pkg)', { cwdPkg, agentsRoot, basePath })
+			return basePath
+		} catch {
+			console.log('[academy] returning CWD as basePath (no pkg fallback)')
+			return cwd
+		}
+	}
+
+	for (const ent of entries) {
+		const name = ent.name
+		if (name !== project) continue
+		const dir = path.join(projectsDir, name)
+		const pkgPath = path.join(dir, 'package.json')
+		try {
+			const raw = await readFile(pkgPath, 'utf8')
+			const pkg = JSON.parse(raw)
+			const agentsRoot = pkg?.bosun?.root ?? 'src/bosun/workflows.ts'
+			const agentsRootPath = path.resolve(dir, agentsRoot)
+			const basePath = path.dirname(agentsRootPath)
+			console.log('[academy] basePath (scanned)', { pkgPath, agentsRoot, basePath })
+			return basePath
+		} catch (e: any) {
+			console.warn('[academy] failed reading candidate package.json', {
+				pkgPath,
+				error: e?.message
+			})
+		}
+	}
+
+	console.error('[academy] project not found (after scan)', { project, projectsDir })
+	throw new Error(`Project not found: ${project}`)
+}
+
+// Resolve the project root directory. Supports two modes:
+// - Bosun dev (projects/<project>)
+// - CLI from inside a project repo (cwd)
+export async function getProjectRootPath(project: string): Promise<string> {
+	const projectsDir = 'projects'
+	// 1) Direct projects/<project>
+	const directDir = path.join(projectsDir, project)
+	const directPkg = path.join(directDir, 'package.json')
+	try {
+		await access(directPkg)
+		return directDir
+	} catch {}
+
+	// 2) If projects folder missing or inaccessible, fallback to CWD if it looks like a project
+	try {
+		const cwd = process.cwd()
+		const cwdPkg = path.join(cwd, 'package.json')
+		await access(cwdPkg)
+		return cwd
+	} catch {}
+
+	// 3) Scan projects folder for named entry
+	try {
+		const entries = (await readdir(projectsDir, { withFileTypes: true } as any)) as any[]
+		for (const ent of entries as any[]) {
+			const entryName = (ent as any).name ?? String(ent)
+			if (entryName !== project) continue
+			const dir = path.join(projectsDir, entryName)
+			const pkgPath = path.join(dir, 'package.json')
+			try {
+				await access(pkgPath)
+				return dir
+			} catch {}
+		}
+	} catch {}
+
+	throw new Error(`Project not found: ${project}`)
+}
+
+// Removed: all CSV helpers (read/write)
+
+export async function tryReadConfig(
+	project: string,
+	agent: string
+): Promise<{
+	repeats?: number
+	concurrency?: number
+} | null> {
+	const base = await getProjectBasePath(project)
+	const jsonPath = path.join(base, `${agent}.config.json`)
+	try {
+		await access(jsonPath)
+		const raw = await readFile(jsonPath, 'utf8')
+		const obj = JSON.parse(raw)
+		return obj
+	} catch {
+		return null
+	}
+}
+
+export async function writeConfig(
+	project: string,
+	agent: string,
+	config: { repeats?: number; concurrency?: number }
+): Promise<void> {
+	const base = await getProjectBasePath(project)
+	const jsonPath = path.join(base, `${agent}.config.json`)
+	const body = JSON.stringify(config ?? {}, null, 2)
+	await writeFile(jsonPath, body, 'utf8')
+}

package/build/academy/data.ts

@@ -0,0 +1,299 @@
+import { tryReadConfig, writeConfig, getProjectRootPath } from './csv'
+import { readdir, readFile } from 'node:fs/promises'
+import path from 'node:path'
+import {
+	readSuite,
+	writeSuite,
+	getSuiteFilePath,
+	upsertSuiteMeta,
+	type TestRow as SuiteRow
+} from './suite'
+
+export type CsvRow = Record<string, string>
+
+export type TestRow = SuiteRow
+
+export type RunConfig = {
+	repeats?: number
+	concurrency?: number
+	defaultContext?: Record<string, unknown>
+}
+
+export type RunResult = {
+	success: boolean
+	result: string
+	evaluation: string
+	rendered_prompt: string
+	received_prompt: string
+	stringified_conversation: string
+	received_context: string
+	traces: any[]
+	errors: string
+	logs: string[]
+}
+
+export type TestResult = {
+	test_id: string
+	success_rate_percent?: number
+	success_rate?: string
+	results: RunResult[]
+	report_timestamp: string // ISO
+	is_old: boolean
+}
+
+// Utility: ensure each row has an id; persist if any are missing.
+export async function loadTests(
+	project: string,
+	agent: string
+): Promise<{ rows: TestRow[]; headers: string[] }> {
+	const suite = await readSuite(project, agent)
+	// Ensure ids exist; write back if any are missing
+	let changed = false
+	const rows: TestRow[] = suite.tests.map((r) => {
+		const id = (r.id || '').trim()
+		if (!id) changed = true
+		return { ...r, id: id || globalThis.crypto?.randomUUID?.() || randomUUID() }
+	})
+	if (changed) {
+		await writeSuite(project, agent, { ...suite, tests: rows })
+	}
+	const headers = [
+		'id',
+		'pinned',
+		'scenario',
+		'author',
+		'category',
+		'assertion_type',
+		'assertion_value',
+		'conversation',
+		'expected_output',
+		'timeout',
+		'similarity_threshold',
+		'context'
+	]
+	return { rows, headers }
+}
+
+// Utility: update a single row by id and write the CSV
+export async function patchTest(
+	project: string,
+	agent: string,
+	id: string,
+	patch: Partial<TestRow>
+): Promise<TestRow | null> {
+	const suite = await readSuite(project, agent)
+	const idx = suite.tests.findIndex((r) => r.id === id)
+	if (idx < 0) return null
+	const current = suite.tests[idx]
+	const updated: TestRow = {
+		id: id,
+		pinned:
+			String(patch.pinned ?? current.pinned ?? 'TRUE').toUpperCase() === 'TRUE' ? 'TRUE' : 'FALSE',
+		scenario: String(patch.scenario ?? current.scenario ?? ''),
+		author: String(patch.author ?? (current as any).author ?? ''),
+		category: String(patch.category ?? (current as any).category ?? ''),
+		assertion_type: String(patch.assertion_type ?? current.assertion_type ?? 'llm-rubric'),
+		assertion_value: String(patch.assertion_value ?? current.assertion_value ?? ''),
+		conversation: String(patch.conversation ?? current.conversation ?? ''),
+		expected_output: String(patch.expected_output ?? current.expected_output ?? ''),
+		timeout: String(patch.timeout ?? current.timeout ?? '0'),
+		similarity_threshold: String(
+			patch.similarity_threshold ?? current.similarity_threshold ?? '60'
+		),
+		context: typeof patch.context === 'string' ? patch.context : (current.context ?? '')
+	}
+	const nextRows = suite.tests.map((r, i) => (i === idx ? updated : r))
+	await writeSuite(project, agent, { ...suite, tests: nextRows })
+	return updated
+}
+
+// Batch update multiple rows in a single read/write to avoid race conditions
+export async function patchTestsBatch(
+	project: string,
+	agent: string,
+	updates: Array<{ id: string; patch: Partial<TestRow> }>
+): Promise<TestRow[]> {
+	if (!Array.isArray(updates) || updates.length === 0) return []
+	const suite = await readSuite(project, agent)
+	const byId = new Map(suite.tests.map((r) => [r.id, r]))
+	const updatedRows: TestRow[] = []
+	for (const { id, patch } of updates) {
+		const current = byId.get(id)
+		if (!current) continue
+		const updated: TestRow = {
+			id: id,
+			pinned:
+				String(patch.pinned ?? current.pinned ?? 'TRUE').toUpperCase() === 'TRUE'
+					? 'TRUE'
+					: 'FALSE',
+			scenario: String(patch.scenario ?? current.scenario ?? ''),
+			author: String(patch.author ?? (current as any).author ?? ''),
+			category: String(patch.category ?? (current as any).category ?? ''),
+			assertion_type: String(patch.assertion_type ?? current.assertion_type ?? 'llm-rubric'),
+			assertion_value: String(patch.assertion_value ?? current.assertion_value ?? ''),
+			conversation: String(patch.conversation ?? current.conversation ?? ''),
+			expected_output: String(patch.expected_output ?? current.expected_output ?? ''),
+			timeout: String(patch.timeout ?? current.timeout ?? '0'),
+			similarity_threshold: String(
+				patch.similarity_threshold ?? current.similarity_threshold ?? '60'
+			),
+			context: typeof patch.context === 'string' ? patch.context : (current.context ?? '')
+		}
+		byId.set(id, updated)
+		updatedRows.push(updated)
+	}
+	const nextRows: TestRow[] = suite.tests.map((r) => byId.get(r.id) ?? r)
+	await writeSuite(project, agent, { ...suite, tests: nextRows })
+	return updatedRows
+}
+
+// Add a new row at the top
+export async function addTest(
+	project: string,
+	agent: string,
+	seed?: Partial<TestRow>
+): Promise<TestRow> {
+	const suite = await readSuite(project, agent).catch(() => ({ version: 1, tests: [] }))
+	const id = globalThis.crypto?.randomUUID?.() || randomUUID()
+	const row: TestRow = {
+		id,
+		pinned:
+			((seed?.pinned || 'TRUE') as any).toString().toUpperCase() === 'TRUE' ? 'TRUE' : 'FALSE',
+		scenario: seed?.scenario ?? 'New scenario',
+		author: seed?.author ?? '',
+		category: seed?.category ?? '',
+		assertion_type: seed?.assertion_type ?? 'llm-rubric',
+		assertion_value: seed?.assertion_value ?? '',
+		conversation: seed?.conversation ?? 'user: ...',
+		expected_output: seed?.expected_output ?? '',
+		timeout: String((seed as any)?.timeout ?? '0'),
+		similarity_threshold: String((seed as any)?.similarity_threshold ?? '60'),
+		context: typeof seed?.context === 'string' ? (seed?.context as string) : '{}'
+	}
+	const nextRows = [row, ...suite.tests]
+	await writeSuite(project, agent, { ...(suite as any), tests: nextRows })
+	return row
+}
+
+export async function deleteTest(project: string, agent: string, id: string): Promise<boolean> {
+	const suite = await readSuite(project, agent)
+	const filtered = suite.tests.filter((r) => r.id !== id)
+	if (filtered.length === suite.tests.length) return false
+	await writeSuite(project, agent, { ...suite, tests: filtered })
+	return true
+}
+
+export async function reorderTests(project: string, agent: string, order: string[]): Promise<void> {
+	const suite = await readSuite(project, agent)
+	const map = new Map(suite.tests.map((r) => [r.id, r]))
+	const next: TestRow[] = []
+	for (const id of order) {
+		const r = map.get(id)
+		if (r) {
+			next.push(r)
+			map.delete(id)
+		}
+	}
+	// Append any rows that were not included in order (safety)
+	for (const r of map.values()) next.push(r)
+	await writeSuite(project, agent, { ...suite, tests: next })
+}
+
+export async function loadConfig(project: string, agent: string): Promise<RunConfig> {
+	const cfg = (await tryReadConfig(project, agent)) || {}
+	// Always source defaultContext from the suite meta, not config
+	let defaultContext: Record<string, unknown> = {}
+	try {
+		const suite = await (await import('./suite')).readSuite(project, agent)
+		defaultContext = (suite.meta?.defaultContext as any) || {}
+	} catch {
+		// ignore
+	}
+	return {
+		repeats: typeof (cfg as any).repeats === 'number' ? (cfg as any).repeats : 1,
+		concurrency: typeof (cfg as any).concurrency === 'number' ? (cfg as any).concurrency : 1,
+		defaultContext
+	}
+}
+
+export async function saveConfig(project: string, agent: string, cfg: RunConfig): Promise<void> {
+	// Merge patch into existing config, persisting only repeats/concurrency
+	const current = (await tryReadConfig(project, agent)) || {}
+	const next = {
+		repeats: typeof cfg.repeats === 'number' ? cfg.repeats : (current as any).repeats,
+		concurrency:
+			typeof cfg.concurrency === 'number' ? cfg.concurrency : (current as any).concurrency
+	}
+	await writeConfig(project, agent, next)
+	// Persist defaultContext exclusively in the suite meta (do not write to config)
+	if (cfg && typeof cfg.defaultContext !== 'undefined') {
+		await upsertSuiteMeta(project, agent, { defaultContext: cfg.defaultContext })
+	}
+}
+
+// We no longer migrate legacy reports automatically. Please update old reports manually.
+
+export async function loadLatestResultsByTest(
+	project: string,
+	agent: string
+): Promise<Record<string, TestResult>> {
+	const projectRoot = await getProjectRootPath(project)
+	const outDir = path.join(projectRoot, 'data', 'test_runs')
+	let entries: string[] = []
+	try {
+		entries = await readdir(outDir)
+	} catch {
+		return {}
+	}
+	const prefix = `${agent}.`
+	const suffix = `.report.json`
+	const files = entries.filter((n) => n.startsWith(prefix) && n.endsWith(suffix)).sort()
+	if (!files.length) return {}
+
+	const latestFile = files[files.length - 1]
+	const latestTs = latestFile
+		.slice(prefix.length, latestFile.length - suffix.length)
+		.replace(/-/g, ':')
+
+	const results: Record<string, TestResult> = {}
+	for (const name of files) {
+		const full = path.join(outDir, name)
+		const raw = await readFile(full, 'utf8')
+		const parsed = JSON.parse(raw) as any
+		const rows = Array.isArray(parsed) ? parsed : Array.isArray(parsed?.rows) ? parsed.rows : []
+		const ts = name.slice(prefix.length, name.length - suffix.length).replace(/-/g, ':')
+		for (const r of rows) {
+			const test_id = (r as any).test_id
+			if (!test_id) {
+				throw new Error(
+					`Missing test_id in report ${name}. Please add 'test_id' and provenance columns manually.`
+				)
+			}
+			const par = String((r as any).pinned_at_run ?? 'TRUE').toUpperCase()
+			if (par === 'FALSE') continue
+			// Always prefer newer timestamp
+			const prev = results[test_id]
+			const rec: TestResult = {
+				test_id,
+				success_rate_percent: (r as any).success_rate_percent,
+				success_rate: (r as any).success_rate,
+				results: Array.isArray((r as any).results) ? ((r as any).results as RunResult[]) : [],
+				report_timestamp: ts,
+				is_old: ts < latestTs
+			}
+			if (!prev || prev.report_timestamp < ts) {
+				results[test_id] = rec
+			}
+		}
+	}
+	return results
+}
+
+// Fallback UUID for Node environments without global crypto
+function randomUUID() {
+	const rnd = () =>
+		Math.floor(Math.random() * 0xffffffff)
+			.toString(16)
+			.padStart(8, '0')
+	return `${rnd()}-${rnd().slice(0, 4)}-${rnd().slice(0, 4)}-${rnd().slice(0, 4)}-${rnd()}${rnd()}`
+}

package/build/academy/emulate-agents.ts

@@ -0,0 +1,129 @@
+import {
+	createContext,
+	type Message,
+	type TestAgent,
+	type TestTextAgent,
+	type TestVoiceAgent,
+	type StepTracer
+} from '@recombine-ai/engine'
+import { loadProjects } from '../../ws-server/projects'
+
+export async function loadAgents(props: {
+	agentId: string
+	getMessages: () => Message[]
+	sendMessage: (message: Message | string) => Promise<void>
+	stepTracer: StepTracer
+	logger?: {
+		log: (...args: any[]) => void
+		debug: (...args: any[]) => void
+		warn: (...args: any[]) => void
+		error: (...args: any[]) => void
+	}
+	context?: Record<string, any>
+	timeoutSeconds: number
+}): Promise<void> {
+	const {
+		agentId,
+		getMessages,
+		sendMessage,
+		stepTracer: stepTracer,
+		logger: providedLogger,
+		context,
+		timeoutSeconds
+	} = props
+	const projects = await loadProjects()
+	if (!projects.length) throw new Error('No projects found')
+
+	// Pick target agent
+	let proj = projects[0]
+	let name = Object.keys(proj.factories)[0]
+	if (agentId) {
+		if (agentId.includes('/')) {
+			const [p, n] = agentId.split('/')
+			const found = projects.find((x) => x.project === p)
+			if (!found) throw new Error(`Project not found: ${p}`)
+			if (!found.factories[n]) throw new Error(`Agent not found: ${agentId}`)
+			proj = found
+			name = n
+		} else {
+			if (!proj.factories[agentId])
+				throw new Error(`Agent not found in default project: ${agentId}`)
+			name = agentId
+		}
+	}
+
+	const factory = proj.factories[name]
+	if (!factory) throw new Error('Factory not found')
+
+	// Minimal stubs
+	const logger = providedLogger || {
+		log: () => {},
+		debug: () => {},
+		error: () => {},
+		warn: () => {}
+	}
+	const scheduler = {
+		clearAllPendingActions: async () => {},
+		registerAction: () => async () => {},
+		triggerAction: async () => {},
+		serialize: () => ({}) as any,
+		setState: (_: any) => {}
+	}
+	const ctx = createContext({})
+	if (context && typeof context === 'object') {
+		// Apply provided context before agent lifecycle
+		ctx.swap(context)
+	}
+
+	// Maintain in-run history so sendMessage appends to it, mirroring ws-server behavior
+	const history = getMessages()
+	const getMessagesWithHistory = () => history
+	const sendMessageWithHistory = async (message: Message | string) => {
+		const m =
+			typeof message === 'string' ? ({ sender: 'agent', text: message } as Message) : message
+		try {
+			history.push(m)
+		} catch {
+			// Best-effort: if history is immutable, ignore push errors
+			logger.log('Failed to push message to history:', m)
+		}
+		await sendMessage(m)
+	}
+
+	const agent = (await factory({
+		stepTracer: stepTracer,
+		tracer: {
+			addStep: (step) => {}
+		},
+		logger: logger,
+		scheduler: scheduler as any,
+		getMessages: getMessagesWithHistory,
+		sendMessage: sendMessageWithHistory,
+		sendAction: async () => {},
+		ctx
+	})) as TestTextAgent
+
+	process.on('unhandledRejection', (r) => {
+		console.error('UNHANDLED', r)
+	})
+
+	try {
+		await agent.respondToMessage()
+		await new Promise((resolve) => setTimeout(resolve, timeoutSeconds * 1000))
+	} catch (err) {
+		const msg = err instanceof Error ? `${err.name}: ${err.message}` : String(err)
+		const stack = err instanceof Error ? err.stack : undefined
+		try {
+			logger.error('Error in agent.respondToMessage():', msg)
+		} catch {
+			/* noop */
+		}
+		if (stack) {
+			try {
+				logger.error(stack)
+			} catch {
+				/* noop */
+			}
+		}
+	}
+}