@recombine-ai/bosun 0.16.1 → 0.16.2-cloud-2

package/src/backend/tests/emulate-agents.ts

@@ -1,206 +0,0 @@
-import { createContext, type Message, type Logger, type Scheduler } from '@recombine-ai/engine'
-import {
-	type RegisteredTestStep,
-	type TestAgent,
-	type TestStepRegistry,
-	type TestVoiceAgent
-} from '@recombine-ai/bosun-lib'
-import { type StepTracer } from '@recombine-ai/telescope'
-import { getAgentFactories } from '../lib/get-agent-factories'
-import { getImportNonce } from '../lib/import-nonce'
-import { getRootPath } from '../lib/get-root-path'
-import { sendCallStartMessage } from '../lib/voice-agent-lifecycle'
-import { withTestPromptCacheBust } from './openai-prompt-cache'
-
-export async function loadAgents(props: {
-	agentId: string
-	region?: string | null
-	getMessages: () => Message[]
-	sendMessage: (message: Message | string) => Promise<void>
-	stepTracer: StepTracer
-	logger?: Logger
-	context?: Record<string, unknown>
-	mocks?: Record<string, unknown>
-	abortSignal?: AbortSignal
-	onBeforeTestResponse?: () => void
-	testStep?: {
-		name: string
-		context: Record<string, unknown>
-		stringifiedConversation: string
-	}
-}): Promise<void> {
-	const {
-		agentId,
-		region,
-		getMessages,
-		sendMessage,
-		stepTracer: stepTracer,
-		logger: providedLogger,
-		context,
-		mocks,
-		abortSignal,
-		onBeforeTestResponse,
-		testStep
-	} = props
-	const importNonce = getImportNonce()
-	const { factories } = await getAgentFactories(await getRootPath(), importNonce)
-	const factory = factories[agentId]
-	if (!factory) throw new Error('Factory not found')
-
-	// Minimal stubs
-	const logger = providedLogger || {
-		log: () => {},
-		debug: () => {},
-		error: () => {},
-		warn: () => {}
-	}
-	const scheduler: Scheduler = {
-		clearAllPendingActions: async () => {},
-		registerAction: () => async () => {}
-	}
-
-	const testSteps = new Map<string, RegisteredTestStep>()
-	const stepRegistry: TestStepRegistry = {
-		addStep: () => {},
-		addTestStep(step: RegisteredTestStep) {
-			// Step names are the current test identity. Preserve the first registration until
-			// workflow IDs become part of the persisted target.
-			if (!testSteps.has(step.name)) testSteps.set(step.name, step)
-		}
-	}
-
-	const hasSplitContext =
-		typeof mocks !== 'undefined' &&
-		mocks !== null &&
-		typeof mocks === 'object' &&
-		!Array.isArray(mocks)
-	const providedContext = context && typeof context === 'object' ? context : {}
-	const runtimeContext = region
-		? {
-				...providedContext,
-				recombineRegion: providedContext.recombineRegion ?? region
-			}
-		: providedContext
-	const ctx = createContext(hasSplitContext ? { context: runtimeContext, mocks } : runtimeContext)
-
-	// Maintain in-run history so sendMessage appends to it, mirroring backend behavior
-	let history: Message[] = []
-
-	const agent = await withTestPromptCacheBust(() =>
-		factory({
-			stepTracer: stepTracer,
-			stepRegistry,
-			logger: logger,
-			scheduler: scheduler,
-			getMessages: () => history,
-			async sendMessage(message) {
-				const m =
-					typeof message === 'string' ? ({ sender: 'agent', text: message } as Message) : message
-				try {
-					history.push(m)
-				} catch {
-					// Best-effort: if history is immutable, ignore push errors
-					logger.log('Failed to push message to history:', m)
-				}
-				await sendMessage(m)
-			},
-			ctx,
-			config: {
-				isTest: true,
-				enableTimers: false
-			}
-		})
-	)
-
-	const withAbort = wrapAbort(abortSignal)
-
-	try {
-		await withTestPromptCacheBust(async () => {
-			if (testStep) {
-				const registered = testSteps.get(testStep.name)
-				if (!registered) {
-					throw new Error(
-						`Step "${testStep.name}" is not registered for isolated testing. ` +
-							"Update the project's Bosun adapter to register its real AI steps."
-					)
-				}
-				onBeforeTestResponse?.()
-				await withAbort(
-					registered.runForTest({
-						context: testStep.context,
-						stringifiedConversation: testStep.stringifiedConversation
-					})
-				)
-				return
-			}
-
-			if (isVoiceAgent(agent)) {
-				// preparation
-				history = []
-				const callStartResult = await withAbort(agent.onCallStart())
-				await withAbort(sendCallStartMessage(callStartResult, sendMessage))
-				await withAbort(agent.waitForIdle())
-
-				// actual test
-				history = getMessages()
-				onBeforeTestResponse?.()
-				await withAbort(agent.streamResponse())
-				await withAbort(agent.waitForIdle())
-			} else {
-				history = getMessages()
-				onBeforeTestResponse?.()
-				await withAbort(agent.start())
-				await withAbort(agent.respondToMessage())
-			}
-		})
-	} catch (err) {
-		const error = err instanceof Error ? err : new Error(String(err))
-		if (abortSignal?.aborted) throw error
-		if (testStep) throw error
-		try {
-			logger.error('Error in agent.respondToMessage():', error.message)
-			if (error.stack) logger.error(error.stack)
-		} catch {
-			/* noop */
-		}
-	}
-}
-
-function isVoiceAgent(agent: TestAgent): agent is TestVoiceAgent {
-	return 'streamResponse' in agent
-}
-
-function wrapAbort(abortSignal: AbortSignal | undefined) {
-	return async <T>(p: Promise<T>): Promise<T> => {
-		if (!abortSignal) return p
-		if (abortSignal.aborted) throw new Error('Operation cancelled')
-		return await new Promise<T>((resolve, reject) => {
-			const onAbort = () => {
-				cleanup()
-				reject(new Error('Operation cancelled'))
-			}
-			const cleanup = () => {
-				try {
-					abortSignal.removeEventListener('abort', onAbort)
-				} catch {
-					/* noop */
-				}
-			}
-			try {
-				abortSignal.addEventListener('abort', onAbort, { once: true })
-			} catch {
-				/* noop */
-			}
-			p.then(
-				(v) => {
-					cleanup()
-					resolve(v)
-				},
-				(e) => {
-					cleanup()
-					reject(e as Error)
-				}
-			)
-		})
-	}
-}

package/src/backend/tests/json-field-match.ts

@@ -1,94 +0,0 @@
-import { isDeepStrictEqual } from 'node:util'
-
-type MatchResult = { pass: true } | { pass: false; reason: string }
-
-function isRecord(value: unknown): value is Record<string, unknown> {
-	return typeof value === 'object' && value !== null && !Array.isArray(value)
-}
-
-function matchExpectedJsonFields(
-	actual: unknown,
-	expected: unknown,
-	caseSensitive: boolean,
-	path = '$'
-): MatchResult {
-	if (isRecord(expected)) {
-		if (!isRecord(actual)) return { pass: false, reason: `Expected an object at ${path}` }
-
-		for (const [key, expectedValue] of Object.entries(expected)) {
-			if (!Object.prototype.hasOwnProperty.call(actual, key)) {
-				return { pass: false, reason: `Missing expected field ${path}.${key}` }
-			}
-			const result = matchExpectedJsonFields(
-				actual[key],
-				expectedValue,
-				caseSensitive,
-				`${path}.${key}`
-			)
-			if (!result.pass) return result
-		}
-		return { pass: true }
-	}
-
-	if (Array.isArray(expected)) {
-		if (!Array.isArray(actual)) return { pass: false, reason: `Expected an array at ${path}` }
-		if (actual.length !== expected.length) {
-			return {
-				pass: false,
-				reason: `Expected ${expected.length} array items at ${path}, received ${actual.length}`
-			}
-		}
-		for (let index = 0; index < expected.length; index += 1) {
-			const result = matchExpectedJsonFields(
-				actual[index],
-				expected[index],
-				caseSensitive,
-				`${path}[${index}]`
-			)
-			if (!result.pass) return result
-		}
-		return { pass: true }
-	}
-
-	const valuesMatch =
-		typeof actual === 'string' && typeof expected === 'string' && !caseSensitive
-			? actual.toLowerCase() === expected.toLowerCase()
-			: isDeepStrictEqual(actual, expected)
-	return valuesMatch ? { pass: true } : { pass: false, reason: `Value differs at ${path}` }
-}
-
-export function gradeJsonExpectedFields(
-	output: string,
-	expectedOutput: string,
-	caseSensitive = false
-) {
-	let actual: unknown
-	let expected: unknown
-	try {
-		expected = JSON.parse(expectedOutput)
-	} catch (error) {
-		return {
-			pass: false,
-			score: 0,
-			reason: `Expected step output is not valid JSON: ${error instanceof Error ? error.message : String(error)}`
-		}
-	}
-	try {
-		actual = JSON.parse(output)
-	} catch (error) {
-		return {
-			pass: false,
-			score: 0,
-			reason: `Actual step output is not valid JSON: ${error instanceof Error ? error.message : String(error)}`
-		}
-	}
-
-	const result = matchExpectedJsonFields(actual, expected, caseSensitive)
-	return {
-		pass: result.pass,
-		score: result.pass ? 1 : 0,
-		reason: result.pass
-			? 'All expected JSON fields match the step output'
-			: `Step output does not match the expected JSON fields: ${result.reason}`
-	}
-}

package/src/backend/tests/openai-prompt-cache.ts

@@ -1,68 +0,0 @@
-import { AsyncLocalStorage } from 'node:async_hooks'
-import { randomUUID } from 'node:crypto'
-import { createRequire } from 'node:module'
-
-const cacheBustContext = new AsyncLocalStorage<boolean>()
-const patchedPrototypes = new WeakSet<object>()
-
-type OpenAIResourceConstructor = {
-	prototype: {
-		create: (body: Record<string, unknown>, ...args: unknown[]) => unknown
-	}
-}
-
-/**
- * OpenAI clients are constructed by the project under test, outside Bosun's control. Patch the
- * request-resource prototypes once, then use async-local state to alter only requests made by a
- * Tests run. Roleplay/Playground requests execute outside this context and remain unchanged.
- */
-export function installTestPromptCacheBust(projectEntryPath: string): void {
-	const projectRequire = createRequire(projectEntryPath)
-	patchOpenAIResources(projectRequire)
-
-	try {
-		const enginePackagePath = projectRequire.resolve('@recombine-ai/engine/package.json')
-		patchOpenAIResources(createRequire(enginePackagePath))
-	} catch {
-		// Some projects may use OpenAI directly without @recombine-ai/engine.
-	}
-}
-
-export function withTestPromptCacheBust<T>(run: () => Promise<T>): Promise<T> {
-	return cacheBustContext.run(true, run)
-}
-
-function patchOpenAIResources(requireFromProject: NodeJS.Require): void {
-	patchResource(requireFromProject, 'openai/resources/chat/completions', 'Completions')
-	patchResource(requireFromProject, 'openai/resources/responses/responses', 'Responses')
-}
-
-function patchResource(
-	requireFromProject: NodeJS.Require,
-	moduleId: string,
-	exportName: string
-): void {
-	let Resource: OpenAIResourceConstructor | undefined
-	try {
-		Resource = requireFromProject(moduleId)?.[exportName] as OpenAIResourceConstructor | undefined
-	} catch {
-		return
-	}
-
-	const prototype = Resource?.prototype
-	if (!prototype || patchedPrototypes.has(prototype) || typeof prototype.create !== 'function')
-		return
-
-	const originalCreate = prototype.create
-	prototype.create = function (body: Record<string, unknown>, ...args: unknown[]) {
-		if (!cacheBustContext.getStore() || !body || typeof body !== 'object') {
-			return Reflect.apply(originalCreate, this, [body, ...args])
-		}
-
-		return Reflect.apply(originalCreate, this, [
-			{ ...body, prompt_cache_key: `bosun-test-${randomUUID()}` },
-			...args
-		])
-	}
-	patchedPrototypes.add(prototype)
-}

package/src/backend/tests/pii-sanitized-test-drafts.spec.ts

@@ -1,163 +0,0 @@
-import { describe, expect, it, vi } from 'vitest'
-
-import type { TraceTestSeed } from '../../lib/tests/pii-sanitized-test-draft'
-import {
-	createPIISanitizedTestDraft,
-	createTestFromPIISanitizedTestDraft
-} from './pii-sanitized-test-drafts'
-import type { PIISanitizationResult } from './pii-sanitizer'
-import type { TestRow } from './suite'
-
-function seed(overrides: Partial<TraceTestSeed> = {}): TraceTestSeed {
-	return {
-		pinned: 'TRUE',
-		step: 'main',
-		evaluation_mode: 'assertion',
-		context_mode: 'replace',
-		assertion_type: 'llm-rubric',
-		assertion_value: '',
-		conversation: 'User: I am Sarah Khan.',
-		expected_output: 'Hello Sarah.',
-		similarity_threshold: '60',
-		timeout: '0',
-		context: { customer: { name: 'Sarah Khan' } },
-		...overrides
-	}
-}
-
-function sanitized(input: TraceTestSeed, replacement = 'Maya Turner'): PIISanitizationResult {
-	return {
-		seed: {
-			...input,
-			conversation: input.conversation.replace('Sarah Khan', replacement),
-			expected_output: input.expected_output.replace('Sarah', replacement.split(' ')[0]),
-			context: { customer: { name: replacement } }
-		},
-		findings: [{ category: 'person', paths: ['$.conversation', '$.context.customer.name'] }],
-		model: 'test-model'
-	}
-}
-
-function testRow(input: TraceTestSeed): TestRow {
-	return {
-		id: 'created-test',
-		order: 0,
-		scenario: '',
-		author: '',
-		category: '',
-		...input
-	}
-}
-
-describe('PII-sanitised test drafts', () => {
-	it('persists an unchanged server-created draft exactly once', async () => {
-		const sanitize = vi.fn(async (input: TraceTestSeed) => sanitized(input))
-		const persist = vi.fn(async (_suiteKey, input: TraceTestSeed) => testRow(input))
-		const draft = await createPIISanitizedTestDraft(
-			{ suiteKey: 'agent.en-gb', seed: seed() },
-			{ sanitize, now: () => 1_000 }
-		)
-
-		const result = await createTestFromPIISanitizedTestDraft(
-			{
-				suiteKey: 'agent.en-gb',
-				draftId: draft.draftId,
-				reviewed: {
-					conversation: draft.seed.conversation,
-					expected_output: draft.seed.expected_output,
-					context: draft.seed.context
-				}
-			},
-			{ persist, now: () => 2_000 }
-		)
-
-		expect(result.status).toBe('created')
-		expect(sanitize).toHaveBeenCalledTimes(1)
-		expect(persist).toHaveBeenCalledOnce()
-		expect(JSON.stringify(persist.mock.calls[0][1])).not.toContain('Sarah')
-	})
-
-	it('persists reviewed edits immediately without sanitizing them again', async () => {
-		const sanitize = vi.fn(async (input: TraceTestSeed) => sanitized(input))
-		const persist = vi.fn(async (_suiteKey, input: TraceTestSeed) => testRow(input))
-		const draft = await createPIISanitizedTestDraft(
-			{ suiteKey: 'agent', seed: seed() },
-			{ sanitize, now: () => 1_000 }
-		)
-
-		const result = await createTestFromPIISanitizedTestDraft(
-			{
-				suiteKey: 'agent',
-				draftId: draft.draftId,
-				reviewed: {
-					conversation: 'User: I am Sarah Khan.',
-					expected_output: draft.seed.expected_output,
-					context: draft.seed.context
-				}
-			},
-			{ persist, now: () => 2_000 }
-		)
-
-		expect(result.status).toBe('created')
-		expect(sanitize).toHaveBeenCalledOnce()
-		expect(persist).toHaveBeenCalledOnce()
-		expect(persist.mock.calls[0][1].conversation).toBe('User: I am Sarah Khan.')
-	})
-
-	it('rejects a draft used for a different suite', async () => {
-		const draft = await createPIISanitizedTestDraft(
-			{ suiteKey: 'agent.en-gb', seed: seed() },
-			{ sanitize: async (input) => sanitized(input), now: () => 1_000 }
-		)
-
-		await expect(
-			createTestFromPIISanitizedTestDraft(
-				{
-					suiteKey: 'other-agent',
-					draftId: draft.draftId,
-					reviewed: {
-						conversation: draft.seed.conversation,
-						expected_output: draft.seed.expected_output,
-						context: draft.seed.context
-					}
-				},
-				{ now: () => 2_000 }
-			)
-		).rejects.toThrow('does not match this suite')
-	})
-
-	it('allows only one create operation to consume a draft', async () => {
-		let finishPersist!: (row: TestRow) => void
-		const persist = vi.fn(
-			async (_suiteKey, _input: TraceTestSeed) =>
-				new Promise<TestRow>((resolve) => {
-					finishPersist = resolve
-				})
-		)
-		const draft = await createPIISanitizedTestDraft(
-			{ suiteKey: 'agent', seed: seed() },
-			{ sanitize: async (input) => sanitized(input), now: () => 1_000 }
-		)
-		const input = {
-			suiteKey: 'agent',
-			draftId: draft.draftId,
-			reviewed: {
-				conversation: draft.seed.conversation,
-				expected_output: draft.seed.expected_output,
-				context: draft.seed.context
-			}
-		}
-
-		const first = createTestFromPIISanitizedTestDraft(input, {
-			persist,
-			now: () => 2_000
-		})
-		await expect(
-			createTestFromPIISanitizedTestDraft(input, { persist, now: () => 2_000 })
-		).rejects.toThrow('already being created')
-
-		finishPersist(testRow(draft.seed))
-		await expect(first).resolves.toMatchObject({ status: 'created' })
-		expect(persist).toHaveBeenCalledOnce()
-	})
-})

package/src/backend/tests/pii-sanitized-test-drafts.ts

@@ -1,106 +0,0 @@
-import { randomUUID } from 'node:crypto'
-
-import {
-	TraceTestReviewContentSchema,
-	TraceTestSeedSchema,
-	type TraceTestReviewContent,
-	type TraceTestSeed
-} from '../../lib/tests/pii-sanitized-test-draft'
-import { addTest } from './data'
-import {
-	assertSameDataShape,
-	sanitizeTraceTestSeed,
-	type PIISanitizationResult
-} from './pii-sanitizer'
-import type { TestRow } from './suite'
-
-const DRAFT_TTL_MS = 30 * 60 * 1000
-
-type StoredDraft = {
-	suiteKey: string
-	seed: TraceTestSeed
-	expiresAt: number
-	consuming: boolean
-}
-
-const drafts = new Map<string, StoredDraft>()
-
-type Sanitize = (seed: TraceTestSeed) => Promise<PIISanitizationResult>
-type Persist = (suiteKey: string, seed: TraceTestSeed) => Promise<TestRow>
-
-export async function createPIISanitizedTestDraft(
-	input: { suiteKey: string; seed: TraceTestSeed },
-	dependencies: { sanitize?: Sanitize; now?: () => number } = {}
-) {
-	pruneExpiredDrafts(dependencies.now?.() ?? Date.now())
-	const seed = TraceTestSeedSchema.parse(input.seed)
-	const sanitized = await (dependencies.sanitize ?? sanitizeTraceTestSeed)(seed)
-	const draftId = randomUUID()
-	const expiresAt = (dependencies.now?.() ?? Date.now()) + DRAFT_TTL_MS
-
-	drafts.set(draftId, {
-		suiteKey: input.suiteKey,
-		seed: sanitized.seed,
-		expiresAt,
-		consuming: false
-	})
-
-	return {
-		draftId,
-		seed: sanitized.seed,
-		findings: sanitized.findings,
-		expiresAt,
-		model: sanitized.model
-	}
-}
-
-export async function createTestFromPIISanitizedTestDraft(
-	input: { suiteKey: string; draftId: string; reviewed: TraceTestReviewContent },
-	dependencies: { persist?: Persist; now?: () => number } = {}
-): Promise<{ status: 'created'; test: TestRow }> {
-	const now = dependencies.now?.() ?? Date.now()
-	pruneExpiredDrafts(now)
-	const stored = drafts.get(input.draftId)
-	if (!stored || stored.expiresAt <= now) throw new Error('PII-sanitised test draft has expired')
-	if (stored.suiteKey !== input.suiteKey)
-		throw new Error('PII-sanitised test draft does not match this suite')
-	if (stored.consuming) throw new Error('PII-sanitised test draft is already being created')
-	stored.consuming = true
-
-	try {
-		const reviewed = TraceTestReviewContentSchema.parse(input.reviewed)
-		assertReviewShape(stored.seed, reviewed)
-		const reviewedSeed = TraceTestSeedSchema.parse({
-			...stored.seed,
-			...reviewed
-		})
-
-		const persist = dependencies.persist ?? addTest
-		const test = await persist(stored.suiteKey, reviewedSeed)
-		drafts.delete(input.draftId)
-		return { status: 'created', test }
-	} catch (error) {
-		const current = drafts.get(input.draftId)
-		if (current === stored) current.consuming = false
-		throw error
-	}
-}
-
-function assertReviewShape(seed: TraceTestSeed, reviewed: TraceTestReviewContent) {
-	assertSameDataShape(seed.context, reviewed.context, '$.context')
-	if (typeof seed.mocks === 'undefined' && typeof reviewed.mocks !== 'undefined') {
-		throw new Error('Reviewed test draft added mocks that were not present')
-	}
-	if (typeof seed.mocks !== 'undefined' && typeof reviewed.mocks === 'undefined') {
-		throw new Error('Reviewed test draft removed mocks')
-	}
-	if (typeof seed.mocks !== 'undefined') {
-		assertSameDataShape(seed.mocks, reviewed.mocks, '$.mocks')
-	}
-}
-
-function pruneExpiredDrafts(now: number) {
-	for (const [draftId, draft] of drafts) {
-		if (draft.expiresAt <= now) drafts.delete(draftId)
-	}
-}