@recombine-ai/bosun 0.16.1 → 0.16.2-cloud-2

package/src/backend/tests/prompt-issues.ts

@@ -1,207 +0,0 @@
-import OpenAI from 'openai'
-import { z } from 'zod'
-
-import { decodeHtmlEntities } from '../lib/decode-html-entities'
-
-const MODEL = 'gpt-5.2-2025-12-11'
-
-const DEFAULT_SYSTEM_INSTRUCTIONS = [
-	'Find all logical inconsistencies and contradictions in the following LLM prompt.',
-	'As a result, produce a JSON object with a single field `issues` which is an array of objects of the shape:',
-	'{"reason": string, "contradictingQuotes": string[], "type": "contradiction"}',
-	'',
-	'Rules:',
-	'- Output must be valid JSON that matches the schema exactly (no markdown).',
-	'- Each entry must include direct quotes copied verbatim from the prompt (no paraphrases).',
-	'- If there are no issues, return {"issues":[]}.'
-].join('\n')
-
-const PromptIssueSchema = z.object({
-	reason: z.string(),
-	contradictingQuotes: z.array(z.string()),
-	type: z.literal('contradiction')
-})
-
-const PromptIssuesEnvelopeSchema = z.object({
-	issues: z.array(PromptIssueSchema)
-})
-
-export type PromptIssue = z.infer<typeof PromptIssueSchema>
-
-export interface FindPromptIssuesResult {
-	prompt: string
-	issues: PromptIssue[]
-	raw: Record<string, unknown>
-}
-
-export async function findPromptIssues(input: {
-	agent: string
-	step: string
-	prompt: string
-}): Promise<FindPromptIssuesResult> {
-	const apiKey = process.env.OPENAI_API_KEY
-	if (!apiKey) {
-		throw new Error('OPENAI_API_KEY is not set')
-	}
-
-	const prompt = decodeHtmlEntities(String(input.prompt ?? ''))
-	const instructions = envTextOrDefault(
-		'BOSUN_PROMPT_ISSUES_INSTRUCTIONS',
-		DEFAULT_SYSTEM_INSTRUCTIONS
-	)
-
-	const client = new OpenAI({ apiKey })
-	const response = await client.responses.create({
-		model: MODEL,
-		instructions,
-		input: prompt,
-		reasoning: { effort: 'low' },
-		text: {
-			format: {
-				type: 'json_schema',
-				name: 'prompt_issues',
-				strict: true,
-				schema: {
-					type: 'object',
-					additionalProperties: false,
-					required: ['issues'],
-					properties: {
-						issues: {
-							type: 'array',
-							items: {
-								type: 'object',
-								additionalProperties: false,
-								required: ['reason', 'contradictingQuotes', 'type'],
-								properties: {
-									reason: { type: 'string' },
-									contradictingQuotes: { type: 'array', items: { type: 'string' } },
-									type: { type: 'string', enum: ['contradiction'] }
-								}
-							}
-						}
-					}
-				}
-			}
-		}
-	})
-
-	const issues = parseIssues(response.output_text)
-	const normalized = normalizeIssues(issues)
-
-	return {
-		prompt,
-		issues: normalized,
-		raw: {
-			input: {
-				agent: input.agent,
-				step: input.step,
-				promptChars: prompt.length
-			},
-			instructions: {
-				env: 'BOSUN_PROMPT_ISSUES_INSTRUCTIONS',
-				overridden: instructions !== DEFAULT_SYSTEM_INSTRUCTIONS,
-				chars: instructions.length
-			},
-			model: MODEL,
-			reasoning: { effort: 'low' },
-			response: {
-				id: response.id,
-				status: response.status,
-				outputTextChars: response.output_text.length
-			},
-			result: {
-				issues: normalized.length
-			}
-		}
-	}
-}
-
-function parseIssues(raw: string): PromptIssue[] {
-	const text = stripCodeFences(String(raw || '')).trim()
-	const parsed = tryParseJson(text) ?? tryParseJson(extractJsonArray(text))
-	if (!parsed) throw new Error('Failed to parse issues response as JSON')
-	if (Array.isArray(parsed)) {
-		return PromptIssuesEnvelopeSchema.parse({ issues: parsed }).issues
-	}
-	return PromptIssuesEnvelopeSchema.parse(parsed).issues
-}
-
-function normalizeIssues(issues: PromptIssue[]): PromptIssue[] {
-	return (issues || [])
-		.map((issue) => {
-			const reason = String(issue.reason || '').trim()
-			const quotes = [
-				...new Set(
-					(issue.contradictingQuotes || [])
-						.map((q) => normalizeQuote(String(q)))
-						.filter((q) => q.length > 0)
-				)
-			]
-			return { type: 'contradiction' as const, reason, contradictingQuotes: quotes }
-		})
-		.filter((i) => i.reason.length > 0 || i.contradictingQuotes.length > 0)
-}
-
-function normalizeQuote(raw: string): string {
-	let q = decodeHtmlEntities(raw).trim()
-	q = trimWrappingQuotes(q)
-	return q.trim()
-}
-
-function trimWrappingQuotes(text: string): string {
-	const quotePairs: Array<[string, string]> = [
-		['"', '"'],
-		["'", "'"],
-		['“', '”'],
-		['‘', '’'],
-		['«', '»'],
-		['„', '“']
-	]
-
-	let out = text
-	for (let i = 0; i < 5; i++) {
-		const trimmed = out.trim()
-		let changed = false
-		for (const [open, close] of quotePairs) {
-			if (trimmed.startsWith(open) && trimmed.endsWith(close)) {
-				const next = trimmed.slice(open.length, trimmed.length - close.length)
-				if (next === trimmed) continue
-				out = next
-				changed = true
-				break
-			}
-		}
-		if (!changed) {
-			out = trimmed
-			break
-		}
-	}
-	return out
-}
-
-function stripCodeFences(text: string): string {
-	const trimmed = text.trim()
-	const m = trimmed.match(/^```(?:json)?\s*([\s\S]*?)\s*```$/i)
-	return m ? m[1] : trimmed
-}
-
-function tryParseJson(text: string): unknown | null {
-	try {
-		return JSON.parse(text)
-	} catch {
-		return null
-	}
-}
-
-function extractJsonArray(text: string): string {
-	const start = text.indexOf('[')
-	const end = text.lastIndexOf(']')
-	if (start === -1 || end === -1 || end < start) return text
-	return text.slice(start, end + 1)
-}
-
-function envTextOrDefault(key: string, defaultValue: string): string {
-	const raw = (process.env[key] ?? '').trim()
-	if (!raw) return defaultValue
-	return raw.replace(/\\n/g, '\n')
-}

package/src/backend/tests/promptfoo-runtime.spec.ts

@@ -1,41 +0,0 @@
-import { createRequire } from 'node:module'
-import path from 'node:path'
-import { afterEach, describe, expect, it, vi } from 'vitest'
-
-import { loadPromptfooForTests } from './promptfoo-runtime'
-
-const requireFromTest = createRequire(import.meta.url)
-const originalLogLevel = process.env.LOG_LEVEL
-
-afterEach(() => {
-	vi.restoreAllMocks()
-	if (originalLogLevel === undefined) delete process.env.LOG_LEVEL
-	else process.env.LOG_LEVEL = originalLogLevel
-})
-
-describe('loadPromptfooForTests', () => {
-	it('keeps Promptfoo quiet without changing the project log level', async () => {
-		process.env.LOG_LEVEL = 'debug'
-
-		const runtime = loadPromptfooForTests()
-
-		expect(typeof runtime.evaluate).toBe('function')
-		expect(process.env.LOG_LEVEL).toBe('debug')
-
-		const promptfooEntry = requireFromTest.resolve('promptfoo')
-		const promptfooLogger = requireFromTest(
-			path.join(path.dirname(promptfooEntry), 'logger.js')
-		) as {
-			getLogLevel(): string
-			default: { debug(message: string): void }
-		}
-		expect(promptfooLogger.getLogLevel()).toBe('warn')
-
-		const stdoutWrite = vi.spyOn(process.stdout, 'write')
-		const marker = 'OpenAI embeddings API response: [should-not-reach-stdout]'
-		promptfooLogger.default.debug(marker)
-		await new Promise<void>((resolve) => setImmediate(resolve))
-
-		expect(stdoutWrite.mock.calls.some(([chunk]) => String(chunk).includes(marker))).toBe(false)
-	})
-})

package/src/backend/tests/promptfoo-runtime.ts

@@ -1,27 +0,0 @@
-import { createRequire } from 'node:module'
-
-type PromptfooRuntime = typeof import('promptfoo')
-
-const requireFromBosun = createRequire(import.meta.url)
-
-/**
- * Promptfoo uses the generic LOG_LEVEL environment variable for its own console logger. Projects
- * commonly set LOG_LEVEL=debug for their application, which otherwise makes Promptfoo dump full
- * model and embedding responses while Bosun tests run.
- *
- * Promptfoo is CommonJS, so require it synchronously while its logger sees a quiet level, then
- * immediately restore the project's environment. This changes only Promptfoo's singleton logger;
- * project logging and Bosun's per-result captured logger keep their configured behavior.
- */
-export function loadPromptfooForTests(): PromptfooRuntime {
-	const previousLogLevel = process.env.LOG_LEVEL
-	try {
-		process.env.LOG_LEVEL = 'warn'
-		return requireFromBosun('promptfoo') as PromptfooRuntime
-	} finally {
-		if (previousLogLevel === undefined) delete process.env.LOG_LEVEL
-		else process.env.LOG_LEVEL = previousLogLevel
-	}
-}
-
-export const promptfoo = loadPromptfooForTests()

package/src/backend/tests/report-files.spec.ts

@@ -1,44 +0,0 @@
-import { access, mkdtemp, rm, writeFile } from 'node:fs/promises'
-import os from 'node:os'
-import path from 'node:path'
-import { afterEach, describe, expect, it } from 'vitest'
-import {
-	compressReportFile,
-	listAgentReportFiles,
-	readReportFile,
-	reportTimestamp
-} from './report-files'
-
-const temporaryDirectories: string[] = []
-
-afterEach(async () => {
-	await Promise.all(temporaryDirectories.splice(0).map((dir) => rm(dir, { recursive: true })))
-})
-
-describe('test report files', () => {
-	it('compresses reports and reads them transparently', async () => {
-		const dir = await mkdtemp(path.join(os.tmpdir(), 'bosun-report-files-'))
-		temporaryDirectories.push(dir)
-		const name = 'support.2026-07-16T10-00-00.000Z.report.json'
-		const reportPath = path.join(dir, name)
-		const report = { rows: [{ test_id: 'a', results: [{ success: true }] }] }
-		await writeFile(reportPath, JSON.stringify(report), 'utf8')
-
-		const gzipPath = await compressReportFile(reportPath)
-
-		await expect(access(reportPath)).rejects.toThrow()
-		expect(gzipPath).toBe(`${reportPath}.gz`)
-		expect(await readReportFile(gzipPath)).toEqual(report)
-		// A reader may retain the plain path after compression atomically replaces it.
-		expect(await readReportFile(reportPath)).toEqual(report)
-		expect(listAgentReportFiles([path.basename(gzipPath)], 'support')).toEqual([
-			path.basename(gzipPath)
-		])
-		expect(reportTimestamp(path.basename(gzipPath), 'support')).toBe('2026:07:16T10:00:00.000Z')
-	})
-
-	it('prefers the complete gzip file while both formats exist', () => {
-		const plain = 'support.2026-07-16T10-00-00.000Z.report.json'
-		expect(listAgentReportFiles([plain, `${plain}.gz`], 'support')).toEqual([`${plain}.gz`])
-	})
-})

package/src/backend/tests/report-files.ts

@@ -1,120 +0,0 @@
-import { createReadStream, createWriteStream } from 'node:fs'
-import { readFile, rename, unlink } from 'node:fs/promises'
-import path from 'node:path'
-import { pipeline } from 'node:stream/promises'
-import { promisify } from 'node:util'
-import { createGzip, gunzip } from 'node:zlib'
-
-export const REPORT_JSON_SUFFIX = '.report.json'
-export const REPORT_GZIP_SUFFIX = `${REPORT_JSON_SUFFIX}.gz`
-
-const gunzipAsync = promisify(gunzip)
-
-function logicalReportName(name: string): string | null {
-	if (name.endsWith(REPORT_GZIP_SUFFIX)) return name.slice(0, -'.gz'.length)
-	if (name.endsWith(REPORT_JSON_SUFFIX)) return name
-	return null
-}
-
-export function listAgentReportFiles(entries: string[], agent: string): string[] {
-	const prefix = `${agent}.`
-	const filesByLogicalName = new Map<string, string>()
-
-	for (const name of entries) {
-		const logicalName = logicalReportName(name)
-		if (!logicalName?.startsWith(prefix)) continue
-
-		const existing = filesByLogicalName.get(logicalName)
-		// Once the complete gzip file has been atomically installed, prefer it during the
-		// brief period before its plain JSON counterpart is removed.
-		if (!existing || name.endsWith(REPORT_GZIP_SUFFIX)) {
-			filesByLogicalName.set(logicalName, name)
-		}
-	}
-
-	return [...filesByLogicalName.entries()]
-		.sort(([left], [right]) => left.localeCompare(right))
-		.map(([, name]) => name)
-}
-
-export function reportTimestamp(name: string, agent: string): string {
-	const logicalName = logicalReportName(name)
-	const prefix = `${agent}.`
-	if (!logicalName?.startsWith(prefix)) throw new Error(`Invalid report filename: ${name}`)
-	return logicalName.slice(prefix.length, -REPORT_JSON_SUFFIX.length).replace(/-/g, ':')
-}
-
-function alternateReportPath(filePath: string): string | null {
-	if (filePath.endsWith(REPORT_GZIP_SUFFIX)) return filePath.slice(0, -'.gz'.length)
-	if (filePath.endsWith(REPORT_JSON_SUFFIX)) return `${filePath}.gz`
-	return null
-}
-
-function isMissingFileError(error: unknown): error is NodeJS.ErrnoException {
-	return error instanceof Error && 'code' in error && error.code === 'ENOENT'
-}
-
-export async function readReportFile<T>(filePath: string): Promise<T> {
-	let resolvedPath = filePath
-	let raw: Buffer
-	try {
-		raw = await readFile(resolvedPath)
-	} catch (error) {
-		const alternatePath = alternateReportPath(resolvedPath)
-		if (!alternatePath || !isMissingFileError(error)) throw error
-		resolvedPath = alternatePath
-		raw = await readFile(resolvedPath)
-	}
-	const json = resolvedPath.endsWith(REPORT_GZIP_SUFFIX) ? await gunzipAsync(raw) : raw
-	return JSON.parse(json.toString('utf8')) as T
-}
-
-export async function compressReportFile(filePath: string): Promise<string> {
-	if (!filePath.endsWith(REPORT_JSON_SUFFIX)) {
-		throw new Error(`Cannot gzip a non-JSON report: ${filePath}`)
-	}
-
-	const gzipPath = `${filePath}.gz`
-	const tmpPath = path.join(
-		path.dirname(filePath),
-		`.${path.basename(gzipPath)}.${process.pid}.${Date.now()}.${Math.random().toString(16).slice(2)}.tmp`
-	)
-
-	try {
-		await pipeline(
-			createReadStream(filePath),
-			createGzip(),
-			createWriteStream(tmpPath, { flags: 'wx' })
-		)
-		try {
-			await rename(tmpPath, gzipPath)
-		} catch {
-			// Windows does not allow atomic rename over an existing file.
-			try {
-				await unlink(gzipPath)
-			} catch {
-				// ignore
-			}
-			await rename(tmpPath, gzipPath)
-		}
-		await unlink(filePath)
-		return gzipPath
-	} finally {
-		try {
-			await unlink(tmpPath)
-		} catch {
-			// The temporary file was renamed or never created.
-		}
-	}
-}
-
-let compressionQueue: Promise<void> = Promise.resolve()
-
-export function queueReportCompression(filePath: string): Promise<string> {
-	const compression = compressionQueue.catch(() => {}).then(() => compressReportFile(filePath))
-	compressionQueue = compression.then(
-		() => undefined,
-		() => undefined
-	)
-	return compression
-}