@recombine-ai/bosun 0.11.1 → 0.11.2

package/src/backend/tests/data.ts

@@ -1,5 +1,5 @@
 import { tryReadConfig, writeConfig } from './config'
-import { access, readdir, readFile } from 'node:fs/promises'
+import { access, readdir, readFile, stat, writeFile } from 'node:fs/promises'
 import path from 'node:path'
 import {
 	getSuiteFilePath,
@@ -19,6 +19,7 @@ import type {
 	WrittenRunResult
 } from './academy'
 import { makeContext, makeContextDiff } from './context-utils'
+import { listAgentReportFiles, readReportFile, reportTimestamp } from './report-files'
 
 export type CsvRow = Record<string, string>
 
@@ -56,11 +57,13 @@ export interface TestResult {
 	test_id: string
 	success_rate_percent?: number
 	success_rate?: string
-	results: RunResult[]
+	results: LatestRunResultSummary[]
 	report_timestamp: string // ISO
 	is_old: boolean
 	previous?: PreviousTestResultSummary
 }
+
+export type LatestRunResultSummary = Pick<RunResult, 'success' | 'result' | 'evaluation' | 'errors'>
 export interface TestTable {
 	rows: TestRow[]
 	headers: string[]
@@ -256,7 +259,7 @@ async function ensureTestExpandedByDefault(agent: string, id: string): Promise<v
 async function ensureConfigExists(agent: string): Promise<void> {
 	const existing = await tryReadConfig(agent)
 	if (existing) return
-	await writeConfig(agent, { repeats: 1, concurrency: 1, expandedTestIds: [] })
+	await writeConfig(agent, { repeats: 1, concurrency: 75, expandedTestIds: [] })
 }
 
 async function loadSuiteOrEmpty(agent: string): Promise<TestSuite> {
@@ -322,7 +325,7 @@ export async function loadConfig(agent: string): Promise<RunConfig> {
 		: []
 	return {
 		repeats: typeof (cfg as any).repeats === 'number' ? (cfg as any).repeats : 1,
-		concurrency: typeof (cfg as any).concurrency === 'number' ? (cfg as any).concurrency : 1,
+		concurrency: typeof (cfg as any).concurrency === 'number' ? (cfg as any).concurrency : 75,
 		defaultContext,
 		expandedTestIds
 	}
@@ -355,70 +358,297 @@ export async function saveConfig(agent: string, cfg: RunConfig): Promise<void> {
 
 // We no longer migrate legacy reports automatically. Please update old reports manually.
 
-export async function loadLatestResultsByTest(agent: string): Promise<Record<string, TestResult>> {
-	const projectRoot = path.dirname(await getRootPath())
-	const outDir = path.join(projectRoot, 'data', 'test_runs')
-	let entries: string[] = []
+type ReportFileState = {
+	name: string
+	size: number
+	mtimeMs: number
+}
+
+type LatestResultsCacheEntry = {
+	files: ReportFileState[]
+	results: Record<string, TestResult>
+}
+
+type PersistedLatestResultsCache = LatestResultsCacheEntry & {
+	version: 1
+}
+
+const latestResultsCache = new Map<string, LatestResultsCacheEntry>()
+const latestResultsLoads = new Map<string, Promise<Record<string, TestResult>>>()
+
+function latestResultsCachePath(outDir: string, agent: string): string {
+	return path.join(outDir, `.bosun-latest-results-${encodeURIComponent(agent)}.json`)
+}
+
+async function reportFileStates(outDir: string, files: string[]): Promise<ReportFileState[]> {
+	return Promise.all(
+		files.map(async (name) => {
+			const info = await stat(path.join(outDir, name))
+			return { name, size: info.size, mtimeMs: info.mtimeMs }
+		})
+	)
+}
+
+function sameReportFile(a: ReportFileState, b: ReportFileState): boolean {
+	return a.name === b.name && a.size === b.size && a.mtimeMs === b.mtimeMs
+}
+
+function incrementalReportStart(
+	previous: ReportFileState[],
+	current: ReportFileState[]
+): number | null {
+	if (current.length < previous.length) return -1
+
+	const sharedLength = Math.min(previous.length, current.length)
+	for (let index = 0; index < sharedLength; index += 1) {
+		if (sameReportFile(previous[index], current[index])) continue
+		return index >= Math.max(0, previous.length - 1) ? index : -1
+	}
+
+	return current.length === previous.length ? null : previous.length
+}
+
+function isPersistedLatestResultsCache(value: unknown): value is PersistedLatestResultsCache {
+	if (!value || typeof value !== 'object') return false
+	const candidate = value as Partial<PersistedLatestResultsCache>
+	return (
+		candidate.version === 1 &&
+		Array.isArray(candidate.files) &&
+		candidate.files.every(
+			(file) =>
+				file &&
+				typeof file.name === 'string' &&
+				typeof file.size === 'number' &&
+				Number.isFinite(file.size) &&
+				typeof file.mtimeMs === 'number' &&
+				Number.isFinite(file.mtimeMs)
+		) &&
+		!!candidate.results &&
+		typeof candidate.results === 'object' &&
+		!Array.isArray(candidate.results)
+	)
+}
+
+async function readPersistedLatestResultsCache(
+	outDir: string,
+	agent: string
+): Promise<LatestResultsCacheEntry | null> {
 	try {
-		entries = await readdir(outDir)
+		const raw = await readFile(latestResultsCachePath(outDir, agent), 'utf8')
+		const parsed: unknown = JSON.parse(raw)
+		if (!isPersistedLatestResultsCache(parsed)) return null
+		return { files: parsed.files, results: parsed.results }
 	} catch {
-		return {}
+		return null
 	}
-	const prefix = `${agent}.`
-	const suffix = `.report.json`
-	const files = entries.filter((n) => n.startsWith(prefix) && n.endsWith(suffix)).sort()
-	if (!files.length) return {}
+}
+
+async function persistLatestResultsCache(
+	outDir: string,
+	agent: string,
+	entry: LatestResultsCacheEntry
+): Promise<void> {
+	try {
+		await writeFile(
+			latestResultsCachePath(outDir, agent),
+			JSON.stringify({ version: 1, ...entry } satisfies PersistedLatestResultsCache),
+			'utf8'
+		)
+	} catch {
+		// The in-memory cache still avoids repeat work for the current Bosun process.
+	}
+}
 
-	const latestFile = files[files.length - 1]
-	const latestTs = latestFile
-		.slice(prefix.length, latestFile.length - suffix.length)
-		.replace(/-/g, ':')
+function summarizeLatestRun(run: RunResult): LatestRunResultSummary {
+	return {
+		success: !!run?.success,
+		result: String(run?.result ?? ''),
+		evaluation: String(run?.evaluation ?? ''),
+		errors: String(run?.errors ?? '')
+	}
+}
 
-	const results: Record<string, TestResult> = {}
-	for (let i = files.length - 1; i >= 0; i--) {
-		const name = files[i]
-		const full = path.join(outDir, name)
-		const raw = await readFile(full, 'utf8')
-		const parsed = JSON.parse(raw)
-		const rows = Array.isArray(parsed) ? parsed : Array.isArray(parsed?.rows) ? parsed.rows : []
-		const ts = name.slice(prefix.length, name.length - suffix.length).replace(/-/g, ':')
-		for (const r of rows) {
-			const test_id = r.test_id
-			if (!test_id) {
+function reportRows(parsed: any): any[] {
+	return Array.isArray(parsed) ? parsed : Array.isArray(parsed?.rows) ? parsed.rows : []
+}
+
+function resultSummary(result: TestResult): PreviousTestResultSummary {
+	return {
+		success_count: result.results.reduce((count, run) => count + (run.success ? 1 : 0), 0),
+		total_runs: result.results.length,
+		report_timestamp: result.report_timestamp
+	}
+}
+
+function resultFromReportRow(
+	row: any,
+	timestamp: string,
+	latestTimestamp: string,
+	previous?: PreviousTestResultSummary
+): TestResult {
+	return {
+		test_id: String(row.test_id),
+		success_rate_percent: row.success_rate_percent,
+		success_rate: row.success_rate,
+		results: (Array.isArray(row.results) ? (row.results as RunResult[]) : []).map(
+			summarizeLatestRun
+		),
+		report_timestamp: timestamp,
+		is_old: timestamp < latestTimestamp,
+		...(previous ? { previous } : {})
+	}
+}
+
+function cloneLatestResults(results: Record<string, TestResult>): Record<string, TestResult> {
+	return Object.fromEntries(
+		Object.entries(results).map(([testId, result]) => [
+			testId,
+			{
+				...result,
+				results: result.results.map((run) => ({ ...run })),
+				...(result.previous ? { previous: { ...result.previous } } : {})
+			}
+		])
+	)
+}
+
+async function mergeIncrementalReports(
+	outDir: string,
+	files: ReportFileState[],
+	startIndex: number,
+	baseResults: Record<string, TestResult>,
+	agent: string
+): Promise<Record<string, TestResult>> {
+	const results = cloneLatestResults(baseResults)
+	const latestTimestamp = reportTimestamp(files[files.length - 1].name, agent)
+
+	for (let index = startIndex; index < files.length; index += 1) {
+		const file = files[index]
+		const timestamp = reportTimestamp(file.name, agent)
+		const parsed = await readReportFile<any>(path.join(outDir, file.name))
+
+		for (const row of reportRows(parsed)) {
+			const testId = String(row.test_id ?? '').trim()
+			if (!testId) {
 				throw new Error(
-					`Missing test_id in report ${name}. Please add 'test_id' and provenance columns manually.`
+					`Missing test_id in report ${file.name}. Please add 'test_id' and provenance columns manually.`
 				)
 			}
-			const par = String(r.pinned_at_run ?? 'TRUE').toUpperCase()
-			if (par === 'FALSE') continue
-
-			const runResults: RunResult[] = Array.isArray(r.results) ? (r.results as RunResult[]) : []
-			const summary: PreviousTestResultSummary = {
-				success_count: runResults.reduce((acc, rr) => acc + (rr?.success ? 1 : 0), 0),
-				total_runs: runResults.length,
-				report_timestamp: ts
+			if (String(row.pinned_at_run ?? 'TRUE').toUpperCase() === 'FALSE') continue
+
+			const current = results[testId]
+			if (!current || timestamp > current.report_timestamp) {
+				results[testId] = resultFromReportRow(
+					row,
+					timestamp,
+					latestTimestamp,
+					current ? resultSummary(current) : undefined
+				)
+			} else if (timestamp === current.report_timestamp) {
+				results[testId] = resultFromReportRow(row, timestamp, latestTimestamp, current.previous)
 			}
+		}
+	}
 
-			const current = results[test_id]
-			if (!current) {
-				results[test_id] = {
-					test_id,
-					success_rate_percent: r.success_rate_percent,
-					success_rate: r.success_rate,
-					results: runResults,
-					report_timestamp: ts,
-					is_old: ts < latestTs
-				}
-				continue
+	for (const result of Object.values(results)) {
+		result.is_old = result.report_timestamp < latestTimestamp
+	}
+	return results
+}
+
+async function rebuildLatestResults(
+	agent: string,
+	outDir: string,
+	files: ReportFileState[]
+): Promise<Record<string, TestResult>> {
+	const latestTimestamp = reportTimestamp(files[files.length - 1].name, agent)
+	const wantedTestIds = new Set(
+		(await readSuite(agent)).tests.map((test) => String(test.id ?? '').trim()).filter(Boolean)
+	)
+	if (!wantedTestIds.size) return {}
+
+	const unresolvedLatest = new Set(wantedTestIds)
+	const unresolvedPrevious = new Set<string>()
+	const results: Record<string, TestResult> = {}
+
+	for (let index = files.length - 1; index >= 0; index -= 1) {
+		const file = files[index]
+		const timestamp = reportTimestamp(file.name, agent)
+		const parsed = await readReportFile<any>(path.join(outDir, file.name))
+
+		for (const row of reportRows(parsed)) {
+			const testId = String(row.test_id ?? '').trim()
+			if (!testId) {
+				throw new Error(
+					`Missing test_id in report ${file.name}. Please add 'test_id' and provenance columns manually.`
+				)
 			}
-			if (!current.previous && ts < current.report_timestamp) {
-				current.previous = summary
+			if (
+				!wantedTestIds.has(testId) ||
+				String(row.pinned_at_run ?? 'TRUE').toUpperCase() === 'FALSE'
+			)
+				continue
+
+			const current = results[testId]
+			if (!current) {
+				results[testId] = resultFromReportRow(row, timestamp, latestTimestamp)
+				unresolvedLatest.delete(testId)
+				if (timestamp === latestTimestamp) unresolvedPrevious.add(testId)
+			} else if (unresolvedPrevious.has(testId) && timestamp < current.report_timestamp) {
+				current.previous = resultSummary(resultFromReportRow(row, timestamp, latestTimestamp))
+				unresolvedPrevious.delete(testId)
 			}
 		}
+
+		if (!unresolvedLatest.size && !unresolvedPrevious.size) break
 	}
+
 	return results
 }
 
+async function loadLatestResultsByTestUncached(agent: string): Promise<Record<string, TestResult>> {
+	const projectRoot = path.dirname(await getRootPath())
+	const outDir = path.join(projectRoot, 'data', 'test_runs')
+	let entries: string[] = []
+	try {
+		entries = await readdir(outDir)
+	} catch {
+		return {}
+	}
+	const files = listAgentReportFiles(entries, agent)
+	if (!files.length) return {}
+	const states = await reportFileStates(outDir, files)
+	const cached =
+		latestResultsCache.get(agent) ?? (await readPersistedLatestResultsCache(outDir, agent))
+	const incrementalStart = cached ? incrementalReportStart(cached.files, states) : -1
+	if (cached && incrementalStart === null) {
+		latestResultsCache.set(agent, cached)
+		return cached.results
+	}
+
+	let results: Record<string, TestResult>
+	if (cached && incrementalStart !== null && incrementalStart >= 0) {
+		results = await mergeIncrementalReports(outDir, states, incrementalStart, cached.results, agent)
+	} else {
+		results = await rebuildLatestResults(agent, outDir, states)
+	}
+	const entry = { files: states, results }
+	latestResultsCache.set(agent, entry)
+	await persistLatestResultsCache(outDir, agent, entry)
+	return results
+}
+
+export function loadLatestResultsByTest(agent: string): Promise<Record<string, TestResult>> {
+	const pending = latestResultsLoads.get(agent)
+	if (pending) return pending
+
+	const load = loadLatestResultsByTestUncached(agent).finally(() => {
+		if (latestResultsLoads.get(agent) === load) latestResultsLoads.delete(agent)
+	})
+	latestResultsLoads.set(agent, load)
+	return load
+}
+
 function summarizeReport(report: WrittenRunReport, timestamp: string): TestRunHistorySummary {
 	requireTestSnapshots(report)
 	const rows = report.rows
@@ -458,9 +688,7 @@ async function listRunReportFiles(agent: string): Promise<string[]> {
 	} catch {
 		return []
 	}
-	const prefix = `${agent}.`
-	const suffix = `.report.json`
-	return entries.filter((n) => n.startsWith(prefix) && n.endsWith(suffix)).sort()
+	return listAgentReportFiles(entries, agent)
 }
 
 function requireTestSnapshots(report: WrittenRunReport): void {
@@ -478,8 +706,7 @@ export async function listRunHistory(agent: string): Promise<TestRunHistorySumma
 
 	for (const name of files.slice().reverse()) {
 		try {
-			const raw = await readFile(path.join(outDir, name), 'utf8')
-			const parsed = JSON.parse(raw) as WrittenRunReport
+			const parsed = await readReportFile<WrittenRunReport>(path.join(outDir, name))
 			summaries.push(summarizeReport(parsed, parsed.run_timestamp))
 		} catch (e) {
 			console.error(`Failed to read test report ${name}`, e)
@@ -531,22 +758,26 @@ function randomUUID() {
 async function getRun(agent: string, ts?: string) {
 	const projectRoot = path.dirname(await getRootPath())
 	const outDir = path.join(projectRoot, 'data', 'test_runs')
-	const file = `${agent}.${(ts || '').replace(/:/g, '-')}.report.json`
-	const reportPath = ts ? path.join(outDir, file) : null
+	const logicalFile = `${agent}.${(ts || '').replace(/:/g, '-')}.report.json`
 
 	async function readLatestPath(): Promise<string> {
 		const entries = await readdir(outDir)
-		const prefix = `${agent}.`
-		const suffix = `.report.json`
-		const candidates = entries.filter((n) => n.startsWith(prefix) && n.endsWith(suffix)).sort()
+		const candidates = listAgentReportFiles(entries, agent)
 		if (!candidates.length) throw new Error('No reports found')
 		return path.join(outDir, candidates[candidates.length - 1])
 	}
 
-	const chosen = reportPath || (await readLatestPath())
-	const raw = await readFile(chosen, 'utf8')
-	const parsed = JSON.parse(raw) as WrittenRunResult
-	return parsed
+	async function readTimestampPath(): Promise<string> {
+		const entries = await readdir(outDir)
+		const candidate = listAgentReportFiles(entries, agent).find((name) =>
+			name.startsWith(logicalFile)
+		)
+		if (!candidate) throw new Error(`Report not found: ${ts}`)
+		return path.join(outDir, candidate)
+	}
+
+	const chosen = ts ? await readTimestampPath() : await readLatestPath()
+	return readReportFile<WrittenRunResult>(chosen)
 }
 
 type ResultKey = keyof Omit<ResultEntry, 'success'>

package/src/backend/tests/emulate-agents.ts

@@ -5,6 +5,7 @@ import { getAgentFactories } from '../lib/get-agent-factories'
 import { getImportNonce } from '../lib/import-nonce'
 import { getRootPath } from '../lib/get-root-path'
 import { sendCallStartMessage } from '../lib/voice-agent-lifecycle'
+import { withTestPromptCacheBust } from './openai-prompt-cache'
 
 export async function loadAgents(props: {
 	agentId: string
@@ -63,29 +64,31 @@ export async function loadAgents(props: {
 		ctx.set({ ...current, recombineRegion: region })
 	}
 
-	const agent = await factory({
-		stepTracer: stepTracer,
-		stepRegistry,
-		logger: logger,
-		scheduler: scheduler,
-		getMessages: () => history,
-		async sendMessage(message) {
-			const m =
-				typeof message === 'string' ? ({ sender: 'agent', text: message } as Message) : message
-			try {
-				history.push(m)
-			} catch {
-				// Best-effort: if history is immutable, ignore push errors
-				logger.log('Failed to push message to history:', m)
+	const agent = await withTestPromptCacheBust(() =>
+		factory({
+			stepTracer: stepTracer,
+			stepRegistry,
+			logger: logger,
+			scheduler: scheduler,
+			getMessages: () => history,
+			async sendMessage(message) {
+				const m =
+					typeof message === 'string' ? ({ sender: 'agent', text: message } as Message) : message
+				try {
+					history.push(m)
+				} catch {
+					// Best-effort: if history is immutable, ignore push errors
+					logger.log('Failed to push message to history:', m)
+				}
+				await sendMessage(m)
+			},
+			ctx,
+			config: {
+				isTest: true,
+				enableTimers: false
 			}
-			await sendMessage(m)
-		},
-		ctx,
-		config: {
-			isTest: true,
-			enableTimers: false
-		}
-	})
+		})
+	)
 
 	const hasSplitContext =
 		typeof mocks !== 'undefined' &&
@@ -100,32 +103,29 @@ export async function loadAgents(props: {
 		ctx.swap(context)
 	}
 
-	process.on('unhandledRejection', (r) => {
-		console.error('UNHANDLED', r)
-	})
-
 	const withAbort = wrapAbort(abortSignal)
 
-	process.env.IS_TEST = 'true'
 	try {
-		if (isVoiceAgent(agent)) {
-			// preparation
-			history = []
-			const callStartResult = await withAbort(agent.onCallStart())
-			await withAbort(sendCallStartMessage(callStartResult, sendMessage))
-			await withAbort(agent.waitForIdle())
+		await withTestPromptCacheBust(async () => {
+			if (isVoiceAgent(agent)) {
+				// preparation
+				history = []
+				const callStartResult = await withAbort(agent.onCallStart())
+				await withAbort(sendCallStartMessage(callStartResult, sendMessage))
+				await withAbort(agent.waitForIdle())
 
-			// actual test
-			history = getMessages()
-			onBeforeTestResponse?.()
-			await withAbort(agent.streamResponse())
-			await withAbort(agent.waitForIdle())
-		} else {
-			history = getMessages()
-			onBeforeTestResponse?.()
-			await withAbort(agent.start())
-			await withAbort(agent.respondToMessage())
-		}
+				// actual test
+				history = getMessages()
+				onBeforeTestResponse?.()
+				await withAbort(agent.streamResponse())
+				await withAbort(agent.waitForIdle())
+			} else {
+				history = getMessages()
+				onBeforeTestResponse?.()
+				await withAbort(agent.start())
+				await withAbort(agent.respondToMessage())
+			}
+		})
 	} catch (err) {
 		const error = err instanceof Error ? err : new Error(String(err))
 		try {
@@ -134,8 +134,8 @@ export async function loadAgents(props: {
 		} catch {
 			/* noop */
 		}
+		if (abortSignal?.aborted) throw error
 	}
-	process.env.IS_TEST = undefined
 }
 
 function isVoiceAgent(agent: TestAgent): agent is TestVoiceAgent {

package/src/backend/tests/openai-prompt-cache.ts

@@ -0,0 +1,68 @@
+import { AsyncLocalStorage } from 'node:async_hooks'
+import { randomUUID } from 'node:crypto'
+import { createRequire } from 'node:module'
+
+const cacheBustContext = new AsyncLocalStorage<boolean>()
+const patchedPrototypes = new WeakSet<object>()
+
+type OpenAIResourceConstructor = {
+	prototype: {
+		create: (body: Record<string, unknown>, ...args: unknown[]) => unknown
+	}
+}
+
+/**
+ * OpenAI clients are constructed by the project under test, outside Bosun's control. Patch the
+ * request-resource prototypes once, then use async-local state to alter only requests made by a
+ * Tests run. Roleplay/Playground requests execute outside this context and remain unchanged.
+ */
+export function installTestPromptCacheBust(projectEntryPath: string): void {
+	const projectRequire = createRequire(projectEntryPath)
+	patchOpenAIResources(projectRequire)
+
+	try {
+		const enginePackagePath = projectRequire.resolve('@recombine-ai/engine/package.json')
+		patchOpenAIResources(createRequire(enginePackagePath))
+	} catch {
+		// Some projects may use OpenAI directly without @recombine-ai/engine.
+	}
+}
+
+export function withTestPromptCacheBust<T>(run: () => Promise<T>): Promise<T> {
+	return cacheBustContext.run(true, run)
+}
+
+function patchOpenAIResources(requireFromProject: NodeJS.Require): void {
+	patchResource(requireFromProject, 'openai/resources/chat/completions', 'Completions')
+	patchResource(requireFromProject, 'openai/resources/responses/responses', 'Responses')
+}
+
+function patchResource(
+	requireFromProject: NodeJS.Require,
+	moduleId: string,
+	exportName: string
+): void {
+	let Resource: OpenAIResourceConstructor | undefined
+	try {
+		Resource = requireFromProject(moduleId)?.[exportName] as OpenAIResourceConstructor | undefined
+	} catch {
+		return
+	}
+
+	const prototype = Resource?.prototype
+	if (!prototype || patchedPrototypes.has(prototype) || typeof prototype.create !== 'function')
+		return
+
+	const originalCreate = prototype.create
+	prototype.create = function (body: Record<string, unknown>, ...args: unknown[]) {
+		if (!cacheBustContext.getStore() || !body || typeof body !== 'object') {
+			return Reflect.apply(originalCreate, this, [body, ...args])
+		}
+
+		return Reflect.apply(originalCreate, this, [
+			{ ...body, prompt_cache_key: `bosun-test-${randomUUID()}` },
+			...args
+		])
+	}
+	patchedPrototypes.add(prototype)
+}

package/src/backend/tests/promptfoo-runtime.spec.ts

@@ -0,0 +1,41 @@
+import { createRequire } from 'node:module'
+import path from 'node:path'
+import { afterEach, describe, expect, it, vi } from 'vitest'
+
+import { loadPromptfooForTests } from './promptfoo-runtime'
+
+const requireFromTest = createRequire(import.meta.url)
+const originalLogLevel = process.env.LOG_LEVEL
+
+afterEach(() => {
+	vi.restoreAllMocks()
+	if (originalLogLevel === undefined) delete process.env.LOG_LEVEL
+	else process.env.LOG_LEVEL = originalLogLevel
+})
+
+describe('loadPromptfooForTests', () => {
+	it('keeps Promptfoo quiet without changing the project log level', async () => {
+		process.env.LOG_LEVEL = 'debug'
+
+		const runtime = loadPromptfooForTests()
+
+		expect(typeof runtime.evaluate).toBe('function')
+		expect(process.env.LOG_LEVEL).toBe('debug')
+
+		const promptfooEntry = requireFromTest.resolve('promptfoo')
+		const promptfooLogger = requireFromTest(
+			path.join(path.dirname(promptfooEntry), 'logger.js')
+		) as {
+			getLogLevel(): string
+			default: { debug(message: string): void }
+		}
+		expect(promptfooLogger.getLogLevel()).toBe('warn')
+
+		const stdoutWrite = vi.spyOn(process.stdout, 'write')
+		const marker = 'OpenAI embeddings API response: [should-not-reach-stdout]'
+		promptfooLogger.default.debug(marker)
+		await new Promise<void>((resolve) => setImmediate(resolve))
+
+		expect(stdoutWrite.mock.calls.some(([chunk]) => String(chunk).includes(marker))).toBe(false)
+	})
+})