@recombine-ai/bosun 0.1.10 → 0.2.0-draft

package/build/academy/academy.ts

@@ -0,0 +1,584 @@
+import { type ProviderOptions } from 'promptfoo'
+import * as promptfoo from 'promptfoo'
+import { type Message, type StepTrace, type StepTracer } from '@recombine-ai/engine'
+import { loadAgents as defaultLoadAgents } from './emulate-agents'
+import { type Assertion as PFAssertion } from 'promptfoo'
+// Backend: per-project JSON test suites (no CSV)
+import { tryReadConfig, getProjectRootPath } from './csv'
+import { mkdir, writeFile } from 'node:fs/promises'
+import path from 'node:path'
+import { readSuite, getSuiteFilePath } from './suite'
+// Use types derived from promptfoo exports instead of incorrect default type imports
+
+import type { ApiProvider, CallApiContextParams, ProviderResponse } from 'promptfoo'
+import deepmerge from 'deepmerge'
+
+// ----------------- Type aliases & helpers -----------------
+type Context = { [key: string]: unknown }
+
+type Conversation = Message[]
+
+type Assertion = PFAssertion
+
+type NunjucksCompatibleString = string
+
+type Step = {
+	name: string
+	prompt: { raw: string; model: ModelConfig }
+	output: 'string' | 'json_schema'
+}
+
+type ModelConfig = ProviderOptions
+
+interface CsvRow {
+	id?: string
+	assertion_type: string
+	assertion_value: string
+	conversation: string
+	expected_output: string
+	similarity_threshold: number | string
+	pinned: 'TRUE' | 'FALSE'
+	context: string
+	timeout: string
+	scenario?: string
+}
+
+export type ResultEntry = {
+	success: boolean
+	result: string
+	evaluation: string
+	rendered_prompt: string
+	received_prompt: string
+	stringified_conversation: string
+	received_context: string
+	// Raw traces array for this run (sanitized where necessary)
+	traces: any[]
+	errors: string
+	logs: string[]
+}
+
+export interface TableRow extends CsvRow {
+	// Link to details view for this row
+	prompts: string
+	success_rate_percent: number
+	success_rate: string
+	// Per-run results (new format)
+	results: ResultEntry[]
+}
+
+// Deep merge helper using 'deepmerge' with array replacement semantics
+const mergeContexts = (baseCtx: Record<string, any>, testCtx: Record<string, any>) =>
+	deepmerge(baseCtx || {}, testCtx || {}, { arrayMerge: (_d, s, _o) => s })
+
+// Parse a multi-line sheet conversation string into engine messages
+function parseConversationFromSheet(conversation: string | undefined | null): Conversation {
+	if (!conversation) return []
+	return conversation
+		.split('\n')
+		.filter((l) => l.trim())
+		.map((line) => {
+			const [sender, ...content] = line.split(': ')
+			const lower = (sender || '').toLowerCase()
+			// Treat common alias 'assistant' as 'agent' to avoid misclassification
+			const role: 'agent' | 'user' | 'system' =
+				lower === 'agent' || lower === 'assistant'
+					? 'agent'
+					: lower === 'system'
+						? 'system'
+						: 'user'
+			return { sender: role, text: content.join(': ') }
+		})
+		.filter((m) => m.sender === 'user' || m.sender === 'agent' || m.sender === 'system')
+}
+
+// Map pinned sheet rows to common test inputs
+function toTestInputs(
+	rows: (CsvRow & { index: number })[],
+	defaultContext?: Record<string, any>,
+	scope: 'all' | 'pinned' = 'pinned'
+) {
+	return rows
+		.filter((r) => (scope === 'all' ? true : r.pinned === 'TRUE'))
+		.map((row) => ({
+			id: (row as any).id,
+			conversation: parseConversationFromSheet(row.conversation),
+			assertions: [
+				{
+					type: row.assertion_type as PFAssertion['type'],
+					value: row.assertion_value
+				},
+				{
+					type: 'similar' as 'similar',
+					value: String(row.expected_output),
+					threshold: Number(row.similarity_threshold)
+				}
+			],
+			context: (() => {
+				const testCtx = row.context ? JSON.parse(row.context) : {}
+				return defaultContext ? mergeContexts(defaultContext, testCtx) : testCtx
+			})(),
+			timeoutSeconds: (() => {
+				const rawValue = (row as any).timeout
+				if (rawValue === undefined || rawValue === null || String(rawValue).trim() === '') {
+					throw new Error('Timeout value is required')
+				}
+				const raw = Number(rawValue)
+				if (isNaN(raw)) {
+					throw new Error(`Invalid timeout value: ${rawValue}`)
+				}
+				if (raw > 1000) {
+					throw new Error(`Timeout value exceeds maximum of 1000: ${raw}`)
+				}
+				if (raw < 0) {
+					throw new Error(`Timeout value must be non-negative: ${raw}`)
+				}
+				return raw
+			})()
+		}))
+}
+
+type LoadAgentsFn = (props: {
+	agentId: string
+	getMessages: () => Message[]
+	sendMessage: (message: Message | string) => Promise<void>
+	stepTracer: StepTracer
+	logger?: {
+		log: (...args: any[]) => void
+		debug: (...args: any[]) => void
+		warn: (...args: any[]) => void
+		error: (...args: any[]) => void
+	}
+	context?: Context
+	timeoutSeconds: number
+}) => Promise<void>
+
+export class RecombineAgentProvider implements ApiProvider {
+	protected providerId: string
+	public config: Record<string, unknown>
+	private runLoadAgents: LoadAgentsFn
+
+	constructor(options: ProviderOptions & { loadAgents?: LoadAgentsFn } = {}) {
+		this.providerId = options.id || 'recombine-agent'
+		this.config = options.config || {}
+		this.runLoadAgents = options.loadAgents || (defaultLoadAgents as unknown as LoadAgentsFn)
+	}
+
+	id() {
+		return this.providerId
+	}
+
+	async callApi(_prompt: string, context?: CallApiContextParams): Promise<ProviderResponse> {
+		const vars = (context?.vars || {}) as unknown as {
+			project: string
+			agent: string
+			conversation: Conversation
+			context: Context
+			timeoutSeconds?: number
+		}
+
+		let captured: string = ''
+		const capturedLogs: string[] = []
+		const capturedStepTraces: any[] = []
+
+		// Streaming log writer; initialized after basic validation
+		let logStream: any = null
+		let logsFilePath = ''
+
+		// Compute agent id and pre-load state with full conversation history
+		const agentCompositeId = `${vars.project}/${vars.agent}`
+		// Clone the conversation so parallel runs don't share the same array reference.
+		// Shallow-clone each message; messages are simple { sender, text } objects in this flow.
+		const conv: Conversation = Array.isArray(vars.conversation)
+			? (vars.conversation as Conversation).map((m) => ({ ...m }))
+			: []
+		const lastUserIndex = (() => {
+			for (let i = conv.length - 1; i >= 0; i--) {
+				if (conv[i]?.sender === 'user') return i
+			}
+			return -1
+		})()
+		if (lastUserIndex < 0) {
+			return { output: '', error: 'No user message found in conversation' }
+		}
+		const lastUser = conv[lastUserIndex]
+
+		// Prepare a streaming log writer so logs are written as they occur
+		{
+			const { createWriteStream } = await import('node:fs')
+			const runId = `${Date.now()}_${Math.random().toString(36).slice(2, 8)}`
+			const projectRoot = await getProjectRootPath(vars.project)
+			const logsDir = path.join(projectRoot, 'data', 'logs')
+			await mkdir(logsDir, { recursive: true })
+			logsFilePath = path.join(logsDir, `capturedLogs_${runId}.log`)
+			logStream = createWriteStream(logsFilePath, { flags: 'a' })
+		}
+
+		const stepTracer: StepTracer = {
+			addStepTrace: (stepTrace: StepTrace) => {
+				try {
+					console.log(`trace: ${stepTrace.name}`)
+					capturedStepTraces.push(stepTrace)
+				} catch {
+					// best-effort capture; ignore serialization issues
+				}
+			}
+		}
+
+		const push = (level: 'log' | 'debug' | 'warn' | 'error', args: any[]) => {
+			try {
+				const raw = args
+					.map((a) => {
+						try {
+							return typeof a === 'string' ? a : JSON.stringify(a)
+						} catch {
+							return String(a)
+						}
+					})
+					.join(' ')
+				const prefix = `[${level.toUpperCase()}] `
+				let line = `${prefix}${raw}`
+				// Enforce max 5000 chars per entry for logs
+				if (line.length > 5000) {
+					line = line.slice(0, 5000) + '... [TRUNCATED]'
+				}
+				capturedLogs.push(line)
+				try {
+					logStream.write(line + '\n')
+				} catch {
+					/* best-effort stream write */
+				}
+			} catch {
+				// best-effort capture
+			}
+		}
+		const logger = {
+			log: (...args: any[]) => push('log', args),
+			debug: (...args: any[]) => push('debug', args),
+			warn: (...args: any[]) => push('warn', args),
+			error: (...args: any[]) => push('error', args)
+		}
+
+		try {
+			// Run the agent via loadAgents using provided conversation
+			await this.runLoadAgents({
+				agentId: agentCompositeId,
+				getMessages: () => conv,
+				sendMessage: async (message: Message | string) => {
+					const m = typeof message === 'string' ? { sender: 'agent', text: message } : message
+					if (m.sender === 'agent') {
+						if (true) {
+							captured = typeof (m as any).text === 'string' ? (m as any).text : JSON.stringify(m)
+							logger.log('Agent message captured:', captured)
+						} else {
+							logger.log('Agent message ignored, as we already captured a message')
+						}
+					}
+				},
+				stepTracer,
+				logger,
+				context: vars.context || {},
+				timeoutSeconds: Number(vars.timeoutSeconds)
+			})
+
+			const fs = await import('fs/promises')
+			const projectRoot = await getProjectRootPath(vars.project)
+			const logsDir = path.join(projectRoot, 'data', 'logs')
+			await mkdir(logsDir, { recursive: true })
+			const filePath = path.join(logsDir, `capturedStepTraces_${Date.now()}.json`)
+			await fs.writeFile(filePath, JSON.stringify(capturedStepTraces, null, 2))
+
+			// Prepare sanitized traces for sheet output
+			const sanitizeTrace = (t: any): any => {
+				try {
+					const clone: any = { ...t }
+					const isMain = clone?.name === 'main' || clone?.name === 'mainReply'
+					// Truncate large prompt fields on non-main steps
+					const truncate = (s: any, max = 5000) => {
+						if (typeof s !== 'string') return s
+						return s.length > max ? s.slice(0, max) + '... [TRUNCATED]' : s
+					}
+					if (isMain) {
+						// Hide main step rendered/received prompts entirely in traces
+						if ('renderedPrompt' in clone) delete clone.renderedPrompt
+						if ('receivedPrompt' in clone) delete clone.receivedPrompt
+					} else {
+						if (typeof clone.renderedPrompt !== 'undefined') {
+							clone.renderedPrompt = truncate(clone.renderedPrompt)
+						}
+						if (typeof clone.receivedPrompt !== 'undefined') {
+							clone.receivedPrompt = truncate(clone.receivedPrompt)
+						}
+					}
+					return clone
+				} catch {
+					return t
+				}
+			}
+			const sanitizedStepTraces = Array.isArray(capturedStepTraces)
+				? capturedStepTraces.map(sanitizeTrace)
+				: []
+
+			if (captured && captured.trim()) {
+				return {
+					output: captured,
+					metadata: {
+						logs: capturedLogs,
+						stepTraces: sanitizedStepTraces,
+						rawStepTraces: capturedStepTraces
+					}
+				}
+			}
+
+			return {
+				output: '',
+				error: 'No agent reply captured',
+				metadata: {
+					logs: capturedLogs,
+					stepTraces: sanitizedStepTraces,
+					rawStepTraces: capturedStepTraces
+				}
+			}
+		} catch (err) {
+			const msg = err instanceof Error ? `${err.name}: ${err.message}` : String(err)
+			capturedLogs.push(`[ERROR] ${msg}`)
+			try {
+				logStream.write(`[ERROR] ${msg}\n`)
+			} catch {
+				/* best-effort */
+			}
+			return {
+				output: '',
+				error: msg,
+				metadata: { logs: capturedLogs, stepTraces: capturedStepTraces }
+			}
+		} finally {
+			// Ensure the stream flushes all data
+			if (logStream) {
+				await new Promise<void>((resolve) => {
+					try {
+						logStream.end(resolve)
+					} catch {
+						resolve()
+					}
+				})
+			}
+		}
+	}
+}
+
+// ----------------- Helpers -----------------
+// Derive the result type returned inside Eval from promptfoo.evaluate
+type PFEval = Awaited<ReturnType<typeof promptfoo.evaluate>>
+type PFEvalResult = PFEval['results'][number]
+
+function buildTableRows(
+	evals: Array<Awaited<ReturnType<typeof promptfoo.evaluate>>>,
+	csvRows: CsvRow[],
+	spreadsheetId: string
+): TableRow[] {
+	const bucket = new Map<string, PFEvalResult[]>()
+
+	for (const ev of evals) {
+		for (const res of ev.results) {
+			const testId = String(res.metadata?.testId ?? '')
+			;(bucket.get(testId) ?? bucket.set(testId, []).get(testId)!).push(res)
+		}
+	}
+
+	const toStringSafe = (v: unknown): string => {
+		if (typeof v === 'string') return v
+		try {
+			return JSON.stringify(v)
+		} catch {
+			return String(v)
+		}
+	}
+
+	return csvRows.map((row) => {
+		const rowId = (row as any).id
+		const evalResults = bucket.get(String(rowId)) ?? []
+		const passCount = evalResults.filter((r) => r.success).length
+		const totalRuns = evalResults.length
+		const pct = totalRuns ? Math.round((passCount / totalRuns) * 100) : 0
+		const rateStr = `${passCount}/${totalRuns}`
+
+		const results: ResultEntry[] = evalResults.map((r) => {
+			const rawTraces = Array.isArray((r as any).metadata?.rawStepTraces)
+				? ((r as any).metadata?.rawStepTraces as any[])
+				: Array.isArray((r as any).metadata?.stepTraces)
+					? ((r as any).metadata?.stepTraces as any[])
+					: []
+			// Pick the last trace whose name === 'main'
+			const lastMain = (() => {
+				for (let i = rawTraces.length - 1; i >= 0; i--) {
+					const t = rawTraces[i]
+					if (t && t.name === 'main') return t
+				}
+				return undefined
+			})()
+			const rendered_prompt =
+				lastMain && typeof lastMain.renderedPrompt !== 'undefined'
+					? toStringSafe(lastMain.renderedPrompt)
+					: 'NO RENDERED PROMPT'
+			const received_prompt =
+				lastMain && typeof lastMain.receivedPrompt !== 'undefined'
+					? toStringSafe(lastMain.receivedPrompt)
+					: 'NO RECEIVED PROMPT'
+			const stringified_conversation =
+				lastMain && typeof lastMain.stringifiedConversation !== 'undefined'
+					? toStringSafe(lastMain.stringifiedConversation)
+					: 'NO STRINGIFIED CONVERSATION'
+			const received_context =
+				lastMain && typeof lastMain.receivedContext !== 'undefined'
+					? toStringSafe(lastMain.receivedContext)
+					: 'NO RECEIVED CONTEXT'
+
+			const logs = Array.isArray((r as any).metadata?.logs)
+				? ((r as any).metadata?.logs as string[])
+				: []
+			const traces = Array.isArray((r as any).metadata?.stepTraces)
+				? ((r as any).metadata?.stepTraces as any[])
+				: []
+			return {
+				success: !!r.success,
+				result: String(r.response?.output ?? ''),
+				evaluation: String(r.gradingResult?.reason ?? ''),
+				rendered_prompt,
+				received_prompt,
+				stringified_conversation,
+				received_context,
+				traces,
+				errors: String(r.error ?? ''),
+				logs
+			}
+		})
+
+		return {
+			...row,
+			prompts: `suite://${spreadsheetId}?testId=${rowId}`,
+			success_rate_percent: pct,
+			success_rate: rateStr,
+			results
+		}
+	})
+}
+
+// legacy viewDetails removed; details are now per-run via /api/view endpoint
+async function runPromptTests(
+	project: string,
+	agent: string,
+	opts?: { scope?: 'all' | 'pinned' },
+	deps?: { loadAgents?: LoadAgentsFn }
+): Promise<void> {
+	const backendId = `${project}/${agent}`
+	const suite = await readSuite(project, agent)
+	const rows = suite.tests as unknown as CsvRow[]
+	const cfg = await tryReadConfig(project, agent)
+	if (!cfg) {
+		const suitePath = await getSuiteFilePath(project, agent)
+		const expectedPath = path.join(path.dirname(suitePath), `${agent}.config.json`)
+		throw new Error(`Config not found at ${expectedPath}`)
+	}
+
+	const repeats = Number((cfg as any)?.repeats ?? 1)
+	const concurrency = Number((cfg as any)?.concurrency ?? 1)
+	const baseContext = (suite?.meta?.defaultContext as any) || {}
+
+	// Add an index field to each row for mapping
+	const typedRows = rows as unknown as CsvRow[]
+	const indexedRows = typedRows.map((row, index) => ({ ...(row as any), index }))
+
+	const tests = toTestInputs(indexedRows, baseContext, opts?.scope || 'pinned')
+
+	// Provider function with stable label
+	const providerInstance = new RecombineAgentProvider({
+		id: 'recombine-agent',
+		loadAgents: deps?.loadAgents
+	})
+	const providerFn = Object.assign(
+		(_promptStr: string, ctx?: CallApiContextParams) => providerInstance.callApi(_promptStr, ctx),
+		{ label: 'recombine-agent' }
+	)
+
+	const testSuites = tests.map((test) => ({
+		prompts: ['agent-run'], // placeholder; provider ignores
+		tests: [
+			{
+				metadata: { testId: test.id },
+				assert: test.assertions,
+				model: 'not used!',
+				vars: {
+					project,
+					agent,
+					context: test.context,
+					conversation: test.conversation,
+					timeoutSeconds: (test as any).timeoutSeconds
+				}
+			}
+		],
+		providers: [providerFn]
+	}))
+
+	const results: Array<Awaited<ReturnType<typeof promptfoo.evaluate>>> = []
+	for (const suite of testSuites) {
+		try {
+			const ev = await promptfoo.evaluate(suite, {
+				repeat: repeats,
+				cache: false,
+				maxConcurrency: concurrency
+			})
+			results.push(ev)
+		} catch (err) {
+			const msg = err instanceof Error ? `${err.name}: ${err.message}` : String(err)
+			results.push({
+				results: [
+					{
+						success: false,
+						response: { output: '' },
+						error: msg,
+						metadata: { testId: (suite.tests?.[0] as any)?.metadata?.testId }
+					} as any
+				]
+			} as any)
+		}
+	}
+
+	// Build rows with per-run results, including last-steptrace derived fields.
+	const rowsToWrite = buildTableRows(results, typedRows, backendId)
+
+	const projectRoot = await getProjectRootPath(project)
+	const outDir = path.join(projectRoot, 'data', 'test_runs')
+	await mkdir(outDir, { recursive: true })
+	// Use full ISO timestamp and sanitize ':' for filesystem safety
+	const iso = new Date().toISOString()
+	const isoFile = iso.replace(/:/g, '-')
+	const outPath = path.join(outDir, `${agent}.${isoFile}.report.json`)
+	// Attach provenance columns to each row before writing
+	const withProvenance = rowsToWrite.map((row, idx) => {
+		const test_id = (row as any).id ?? String(idx)
+		const pinned_at_run =
+			String((row as any).pinned || '').toUpperCase() === 'TRUE' ? 'TRUE' : 'FALSE'
+		const run_scope = opts?.scope || 'pinned'
+		return {
+			...row,
+			project,
+			agent,
+			test_id,
+			run_timestamp: iso,
+			run_config: JSON.stringify({ repeats, concurrency, defaultContext: baseContext }),
+			pinned_at_run,
+			run_scope
+		} as any
+	})
+	const body = {
+		project,
+		agent,
+		run_timestamp: iso,
+		run_scope: opts?.scope || 'pinned',
+		run_config: { repeats, concurrency, defaultContext: baseContext },
+		rows: withProvenance
+	}
+	await writeFile(outPath, JSON.stringify(body, null, 2), 'utf8')
+}
+
+export { runPromptTests }