@recombine-ai/bosun 0.16.2 → 0.17.1

package/src/backend/tests/suggest-test-history-edit.ts

@@ -0,0 +1,52 @@
+import type { PromptEditProgressPhase } from '../roleplay/prompt-edit-progress'
+import { suggestFileEdit } from '../roleplay/suggest-file-edit'
+
+const VIRTUAL_HISTORY_PATH = 'history.md'
+
+export async function suggestTestHistoryEdit(input: {
+	request: string
+	prompt: string
+	context: string
+	conversation: string
+	expectedOutput: string
+	onProgress?: (progress: {
+		phase: PromptEditProgressPhase
+		activity: number
+		summary: string
+	}) => void
+}): Promise<string> {
+	return suggestFileEdit({
+		request: buildTestHistoryPrompt(input),
+		originalContent: input.conversation,
+		virtualPath: VIRTUAL_HISTORY_PATH,
+		instructions:
+			'The conversation history is the existing virtual file history.md. Decide whether it conflicts with the current LLM prompt or contains agent behaviour that the current prompt would not produce. If it does, use the Apply Patch tool to update history.md minimally. Otherwise, do not call Apply Patch. Preserve the test intent and never add or change the expected outcome. Do not create, delete, or modify any other file.',
+		onProgress: input.onProgress
+	})
+}
+
+export function buildTestHistoryPrompt(input: {
+	request: string
+	prompt: string
+	context: string
+	conversation: string
+	expectedOutput: string
+}): string {
+	return `The user wants you to update an LLM test's conversation history when needed. The history must reflect behaviour that is valid under the current prompt. Make the smallest useful change. The expected outcome is immutable and is provided only so you can preserve the test's intent.
+
+This is the user's request:
+${input.request}
+
+This is the current LLM prompt:
+${input.prompt}
+
+This is the test context:
+${input.context}
+
+This is the conversation history in history.md:
+${input.conversation}
+
+This is the immutable expected outcome:
+${input.expectedOutput}
+`
+}

package/src/backend/tests/test-edit-source.spec.ts

@@ -0,0 +1,61 @@
+import { describe, expect, it } from 'vitest'
+
+import type { TestRow } from './suite'
+import { testEditSourceFingerprint } from './test-edit-source'
+
+const sourceChanges: Array<[string, Partial<TestRow>]> = [
+	['step', { step: 'follow-up' }],
+	['evaluation mode', { evaluation_mode: 'json-deep-equal' }],
+	['assertion type', { assertion_type: 'contains' }],
+	['assertion value', { assertion_value: 'A different assertion' }],
+	['case sensitivity', { case_sensitive: true }],
+	['context mode', { context_mode: 'replace' }],
+	['conversation', { conversation: 'User: changed' }],
+	['expected output', { expected_output: 'Agent: changed' }],
+	['timeout', { timeout: '10' }],
+	['similarity threshold', { similarity_threshold: '80' }],
+	['context', { context: { customer: 'Alex' } }],
+	['mocks', { mocks: { booking: { id: '456' } } }]
+]
+
+describe('testEditSourceFingerprint', () => {
+	it.each(sourceChanges)('changes when the test %s changes', (_name, change) => {
+		const test = testRow()
+
+		expect(testEditSourceFingerprint({ ...test, ...change })).not.toBe(
+			testEditSourceFingerprint(test)
+		)
+	})
+
+	it('uses the proposed conversation without changing the other source fields', () => {
+		const test = testRow()
+		const conversation = 'User: corrected'
+
+		expect(testEditSourceFingerprint(test, { conversation })).toBe(
+			testEditSourceFingerprint({ ...test, conversation })
+		)
+	})
+})
+
+function testRow(): TestRow {
+	return {
+		id: 'test-1',
+		order: 0,
+		pinned: 'TRUE',
+		scenario: 'Scenario',
+		step: 'main',
+		author: '',
+		category: '',
+		assertion_type: 'llm-rubric',
+		assertion_value: 'Be correct',
+		conversation: 'User: original',
+		expected_output: 'Agent: expected',
+		evaluation_mode: 'assertion',
+		case_sensitive: false,
+		context_mode: 'merge',
+		timeout: '0',
+		similarity_threshold: '50',
+		context: { customer: 'Sam' },
+		mocks: { booking: { id: '123' } }
+	}
+}

package/src/backend/tests/test-edit-source.ts

@@ -0,0 +1,34 @@
+import { createHash } from 'node:crypto'
+
+import type { TestRow } from './suite'
+
+function canonicalize(value: unknown): unknown {
+	if (Array.isArray(value)) return value.map(canonicalize)
+	if (!value || typeof value !== 'object') return value
+	return Object.fromEntries(
+		Object.entries(value as Record<string, unknown>)
+			.sort(([left], [right]) => left.localeCompare(right))
+			.map(([key, entry]) => [key, canonicalize(entry)])
+	)
+}
+
+export function testEditSourceFingerprint(
+	test: TestRow,
+	overrides?: { conversation?: string }
+): string {
+	const source = canonicalize({
+		step: test.step,
+		evaluationMode: test.evaluation_mode,
+		assertionType: test.assertion_type,
+		assertionValue: test.assertion_value,
+		caseSensitive: test.case_sensitive,
+		contextMode: test.context_mode,
+		conversation: overrides?.conversation ?? test.conversation,
+		expectedOutput: test.expected_output,
+		timeout: test.timeout,
+		similarityThreshold: test.similarity_threshold,
+		context: test.context,
+		mocks: test.mocks ?? null
+	})
+	return createHash('sha256').update(JSON.stringify(source)).digest('hex')
+}

package/src/backend/tests/test-history-edit-proposals.ts

@@ -0,0 +1,69 @@
+import { randomUUID } from 'node:crypto'
+
+export interface TestHistoryEditProposal {
+	id: string
+	agentId: string
+	region: string | null
+	testId: string
+	step: string
+	originalConversation: string
+	suggestedConversation: string
+	expectedOutput: string
+	createdAt: number
+}
+
+export interface TestHistoryEditSourceGuard {
+	suiteKey: string
+	allowedTestFingerprints: string[]
+	allowedPromptContents: string[]
+}
+
+type StoredTestHistoryEditProposal = TestHistoryEditProposal & {
+	sourceGuard: TestHistoryEditSourceGuard
+}
+
+const proposals = new Map<string, StoredTestHistoryEditProposal>()
+const PROPOSAL_TTL_MS = 12 * 60 * 60 * 1000
+
+export function createTestHistoryEditProposal(
+	proposal: Omit<TestHistoryEditProposal, 'id' | 'createdAt'> & {
+		sourceGuard: TestHistoryEditSourceGuard
+	}
+): TestHistoryEditProposal {
+	removeExpiredTestHistoryEditProposals()
+	const stored = { ...proposal, id: randomUUID(), createdAt: Date.now() }
+	proposals.set(stored.id, stored)
+	return publicProposal(stored)
+}
+
+export function getTestHistoryEditProposalForAcceptance(
+	id: string
+): StoredTestHistoryEditProposal | null {
+	removeExpiredTestHistoryEditProposals()
+	const proposal = proposals.get(id)
+	return proposal
+		? {
+				...proposal,
+				sourceGuard: {
+					suiteKey: proposal.sourceGuard.suiteKey,
+					allowedTestFingerprints: [...proposal.sourceGuard.allowedTestFingerprints],
+					allowedPromptContents: [...proposal.sourceGuard.allowedPromptContents]
+				}
+			}
+		: null
+}
+
+export function deleteTestHistoryEditProposal(id: string): boolean {
+	return proposals.delete(id)
+}
+
+function removeExpiredTestHistoryEditProposals(now = Date.now()) {
+	for (const [id, proposal] of proposals) {
+		if (proposal.createdAt + PROPOSAL_TTL_MS <= now) proposals.delete(id)
+	}
+}
+
+function publicProposal(proposal: StoredTestHistoryEditProposal): TestHistoryEditProposal {
+	const { sourceGuard: _sourceGuard, ...result } = proposal
+	return { ...result }
+}

package/src/backend/tests/test-history-overrides.spec.ts

@@ -0,0 +1,24 @@
+import { describe, expect, it } from 'vitest'
+
+import { applyTestHistoryOverrides } from './academy'
+
+describe('applyTestHistoryOverrides', () => {
+	it('changes only the targeted test in memory', () => {
+		const first = { id: 'first', conversation: 'User: old', expected_output: 'Expected one' }
+		const second = {
+			id: 'second',
+			conversation: 'User: untouched',
+			expected_output: 'Expected two'
+		}
+
+		const result = applyTestHistoryOverrides([first, second], {
+			first: { conversation: 'User: proposed' }
+		})
+
+		expect(result).toEqual([
+			{ id: 'first', conversation: 'User: proposed', expected_output: 'Expected one' },
+			second
+		])
+		expect(first.conversation).toBe('User: old')
+	})
+})

package/src/backend/tests/test-selection.ts

@@ -0,0 +1,27 @@
+import { getProjectBosunApi } from '../lib/project-api'
+
+export interface SuiteSelection {
+	suiteKey: string
+	agentId: string
+	region?: string
+}
+
+export async function resolveSuiteSelection(
+	agentId: string,
+	region?: string | null
+): Promise<SuiteSelection> {
+	const bosunApi = await getProjectBosunApi()
+	const meta = bosunApi ? await bosunApi.getAgentMeta(agentId) : null
+	if (meta?.testScope !== 'region') return { suiteKey: agentId, agentId }
+
+	if (!bosunApi) {
+		throw new Error(`Project does not export bosunApi, cannot resolve suite for '${agentId}'.`)
+	}
+
+	const resolved = await bosunApi.resolveTestSuite({ agentId, region })
+	return {
+		suiteKey: resolved.suiteKey,
+		agentId,
+		region: resolved.resolvedRegion ?? undefined
+	}
+}

package/src/lib/api-telescope/source-selection.spec.ts

@@ -0,0 +1,44 @@
+import { describe, expect, it } from 'vitest'
+import {
+	getTelescopeSourceIdFromUrl,
+	resolveTelescopeSourceId,
+	withTelescopeSourceId
+} from './source-selection'
+
+const sources = [{ id: 'prod' }, { id: 'staging' }]
+
+describe('Telescope source URLs', () => {
+	it('uses a valid source from the URL before the persisted source', () => {
+		expect(resolveTelescopeSourceId(sources, 'staging', 'prod')).toBe('staging')
+	})
+
+	it('keeps links without a source on the persisted source', () => {
+		expect(resolveTelescopeSourceId(sources, null, 'staging')).toBe('staging')
+	})
+
+	it('uses the first source when a link and persisted source have no valid source', () => {
+		expect(resolveTelescopeSourceId(sources, null, null)).toBe('prod')
+		expect(resolveTelescopeSourceId(sources, 'unknown', 'missing')).toBe('prod')
+	})
+
+	it('reads only non-empty source IDs from URLs', () => {
+		expect(getTelescopeSourceIdFromUrl(new URL('http://localhost/view?source=staging'))).toBe(
+			'staging'
+		)
+		expect(getTelescopeSourceIdFromUrl(new URL('http://localhost/view?source=%20'))).toBeNull()
+	})
+
+	it('adds the source without changing the trace path or other query parameters', () => {
+		const original = new URL(
+			'http://localhost/view/conversations/call-1/traces?agent=booking&region=uk'
+		)
+		const result = withTelescopeSourceId(original, 'staging/eu')
+
+		expect(result.pathname).toBe('/view/conversations/call-1/traces')
+		expect(result.searchParams.get('agent')).toBe('booking')
+		expect(result.searchParams.get('region')).toBe('uk')
+		expect(result.searchParams.get('source')).toBe('staging/eu')
+		expect(result.search).toContain('source=staging%2Feu')
+		expect(original.searchParams.has('source')).toBe(false)
+	})
+})

package/src/lib/api-telescope/source-selection.ts

@@ -1,4 +1,32 @@
 export const TELESCOPE_SOURCE_STORAGE_KEY = 'bosun.traces.telescopeSourceId'
+export const TELESCOPE_SOURCE_QUERY_PARAM = 'source'
+
+type TelescopeSource = { id: string }
+
+export function getTelescopeSourceIdFromUrl(url: URL): string | null {
+	const sourceId = url.searchParams.get(TELESCOPE_SOURCE_QUERY_PARAM)?.trim()
+	return sourceId || null
+}
+
+export function resolveTelescopeSourceId(
+	sources: readonly TelescopeSource[],
+	requestedSourceId: string | null,
+	persistedSourceId: string | null
+): string | null {
+	const requested = requestedSourceId?.trim()
+	if (requested && sources.some((source) => source.id === requested)) return requested
+
+	const persisted = persistedSourceId?.trim()
+	if (persisted && sources.some((source) => source.id === persisted)) return persisted
+
+	return sources[0]?.id ?? null
+}
+
+export function withTelescopeSourceId(url: URL, sourceId: string): URL {
+	const nextUrl = new URL(url)
+	nextUrl.searchParams.set(TELESCOPE_SOURCE_QUERY_PARAM, sourceId)
+	return nextUrl
+}
 
 export function getPersistedTelescopeSourceId(): string | null {
 	if (typeof window === 'undefined') return null

package/src/lib/components/Modal.svelte

@@ -47,7 +47,7 @@
 			</div>
 			{#if underlayOpen && underlay}
 				<div
-					class="modal-underlay absolute left-0 right-0 top-full -z-10 overflow-hidden rounded-b-lg border border-t-0 border-rc-gray-600 bg-rc-gray-800 px-4 py-2.5 shadow-xl"
+					class="modal-underlay absolute left-0 right-0 top-full -z-10 -mt-2 overflow-hidden rounded-b-lg border border-t-0 border-rc-gray-600 bg-rc-gray-800 px-4 pb-1 pt-3 shadow-xl"
 				>
 					{@render underlay()}
 				</div>

package/src/lib/edit/edit-workspace-context.ts

@@ -1,7 +1,8 @@
 import { createContext } from 'svelte'
 import type { Message } from '@recombine-ai/engine'
 
-import type { AgentSateHeader, AppEvent, StepDef } from '$lib/ws-message'
+import type { AgentSateHeader, ClientAppEvent, StepDef } from '$lib/ws-message'
+import type { GitFlushTransportOptions } from '$lib/git-mode/client-admission'
 import type { EditState, PopoverKind, QuoteCount } from './edit-vm'
 
 export interface PendingPromptEdit {
@@ -13,6 +14,32 @@ export interface PendingPromptEdit {
 	createdAt: number
 }
 
+export interface PendingTestHistoryEdit {
+	id: string
+	agentId: string
+	region: string | null
+	testId: string
+	step: string
+	originalConversation: string
+	suggestedConversation: string
+	expectedOutput: string
+	createdAt: number
+}
+
+export interface TestAiEditTarget {
+	kind: 'test-history' | 'test-fix'
+	testId: string
+	step: string
+	prompt: string
+	basePrompt: string
+	conversation: string
+	expectedOutput: string
+	context: string
+	assertionLabel: string
+	assertionValue: string
+	currentIncorrectOutcome?: string
+}
+
 export interface EditWorkspaceState {
 	agentId: string
 	region: string | null
@@ -26,6 +53,7 @@ export interface EditWorkspaceState {
 	currentStep: StepDef | null
 	promptDrafts: Record<string, string>
 	promptEdits: Record<string, PendingPromptEdit>
+	testHistoryEdits: Record<string, PendingTestHistoryEdit>
 	context: Record<string, any> | null
 	promptContext: Record<string, any> | null
 	agentStates: AgentSateHeader[]
@@ -48,13 +76,15 @@ export interface EditWorkspaceActions {
 	openContextPopover: () => void
 	closeContextPopover: () => void
 	toggleContextPopover: () => void
+	openAiEdit: (request: string, target?: TestAiEditTarget) => void
 
 	// WS actions
-	sendAppEvent: (event: AppEvent) => void
-	sendChatMessage: (message: Message) => void
+	sendAppEvent: (event: ClientAppEvent, options?: GitFlushTransportOptions) => void
+	sendChatMessage: (message: Message) => boolean
 	selectState: (stateId: string | null) => void
 	flushPromptSave: () => Promise<void>
 	getPromptOverrides: () => Record<string, string>
+	getTestHistoryOverrides: () => Record<string, { conversation: string }>
 	getActivePromptEditId: () => string | null
 	hangup: () => void
 }

package/src/lib/edit/test-ai-edit.spec.ts

@@ -0,0 +1,31 @@
+import { describe, expect, it } from 'vitest'
+
+import { buildPromptFixRequest, buildTestFixRequest } from './test-ai-edit'
+
+const input = {
+	context: '{}',
+	conversation: 'User: corrected',
+	expectedOutput: 'Agent: expected',
+	assertionLabel: 'LLM',
+	assertionValue: 'Matches the expected behaviour',
+	currentIncorrectOutcome: 'Agent: wrong'
+}
+
+describe('buildPromptFixRequest', () => {
+	it('rebuilds the standard request with the corrected conversation', () => {
+		const entered = buildTestFixRequest({ ...input, conversation: 'User: original' })
+
+		const result = buildPromptFixRequest(entered, input, 'User: original')
+
+		expect(result).toContain('Conversation:\nUser: corrected')
+		expect(result).not.toContain('Conversation:\nUser: original')
+	})
+
+	it('retains a customized request and appends the authoritative corrected test', () => {
+		const result = buildPromptFixRequest('Keep this custom instruction.', input, 'User: original')
+
+		expect(result).toContain('Keep this custom instruction.')
+		expect(result).toContain('Use the following test case as authoritative')
+		expect(result).toContain('Conversation:\nUser: corrected')
+	})
+})

package/src/lib/edit/test-ai-edit.ts

@@ -0,0 +1,52 @@
+export interface TestFixRequestInput {
+	context: string
+	conversation: string
+	expectedOutput: string
+	assertionLabel: string
+	assertionValue: string
+	currentIncorrectOutcome: string
+}
+
+export const UPDATE_HISTORY_REQUEST = `Update the conversation history so it reflects behaviour that is valid under the current prompt. Preserve the test's intent and expected outcome, and make only the smallest necessary changes. Possible roles: "Agent", "User", "System"`
+
+export function formatTestFixAssertion(label: string, value: string): string {
+	const normalizedLabel = label.trim() === 'LLM' ? '' : label.trim()
+	return [normalizedLabel, value.trim()].filter(Boolean).join(' ')
+}
+
+export function buildTestFixRequest(input: TestFixRequestInput): string {
+	const assertion = formatTestFixAssertion(input.assertionLabel, input.assertionValue)
+
+	return `You will be given the context, the conversation and the desired output and assertion about it. First, find why the prompt is not producing the correct result. It's usually one concrete instruction within the relevant section, or a contradiction. Then, change the prompt minimally so this will work, but keeping the prompt clean and comprehensive is more important. Try to provide as general solution as possible. Try to change just one part of the prompt, not multiple parts. Avoid adding exceptions and overrides. Avoid duplicating information that already exists in the prompt.
+
+Context:
+${input.context.trim()}
+
+Conversation:
+${input.conversation.trim()}
+
+Desired outcome:
+${input.expectedOutput.trim()}
+
+Assertion about outcome:
+${assertion}
+
+Current incorrect outcome:
+${input.currentIncorrectOutcome.trim()}`
+}
+
+export function buildPromptFixRequest(
+	enteredRequest: string,
+	input: TestFixRequestInput,
+	originalConversation: string
+): string {
+	const canonicalOriginal = buildTestFixRequest({ ...input, conversation: originalConversation })
+	const canonicalCorrected = buildTestFixRequest(input)
+	if (enteredRequest.trim() === canonicalOriginal.trim()) return canonicalCorrected
+
+	return `${enteredRequest.trim()}
+
+The conversation history has now been checked and may have been corrected. Use the following test case as authoritative instead of any earlier version included above:
+
+${canonicalCorrected}`
+}

package/src/lib/git-mode/BlockingGitDialog.svelte

@@ -0,0 +1,118 @@
+<script lang="ts">
+	import { onDestroy, onMount, type Snippet } from 'svelte'
+
+	interface Props {
+		open: boolean
+		label: string
+		stacked?: boolean
+		class?: string
+		children: Snippet
+	}
+
+	let { open, label, stacked = false, class: className = '', children }: Props = $props()
+
+	let dialog: HTMLDialogElement
+	let entered = $state(false)
+	let closeTimer: ReturnType<typeof setTimeout> | null = null
+	let frame = 0
+
+	function clearPending() {
+		if (closeTimer) clearTimeout(closeTimer)
+		closeTimer = null
+		if (frame) cancelAnimationFrame(frame)
+		frame = 0
+	}
+
+	function show() {
+		clearPending()
+		if (!dialog.open) dialog.showModal()
+		entered = false
+		frame = requestAnimationFrame(() => {
+			frame = requestAnimationFrame(() => {
+				entered = true
+				frame = 0
+			})
+		})
+	}
+
+	function hide() {
+		clearPending()
+		if (!dialog.open) return
+		entered = false
+		closeTimer = setTimeout(() => {
+			dialog.close()
+			closeTimer = null
+		}, 200)
+	}
+
+	$effect(() => {
+		if (!dialog) return
+		if (open) show()
+		else hide()
+	})
+
+	onMount(() => {
+		if (open) show()
+	})
+
+	onDestroy(clearPending)
+</script>
+
+<dialog
+	bind:this={dialog}
+	aria-label={label}
+	oncancel={(event) => event.preventDefault()}
+	onclose={(event) => {
+		if (open) {
+			event.preventDefault()
+			show()
+		}
+	}}
+	class:entered
+	class:stacked
+	class={`git-blocking-dialog m-auto max-h-[90vh] w-[min(92vw,32rem)] overflow-visible rounded-xl border border-rc-gray-600 bg-rc-gray-800 p-0 text-rc-gray-100 shadow-2xl outline-none ${className}`}
+>
+	<div class="p-6">
+		{@render children()}
+	</div>
+</dialog>
+
+<style>
+	.git-blocking-dialog {
+		opacity: 0;
+		transform: translateY(4px) scale(0.99);
+		transition:
+			opacity 200ms ease,
+			transform 200ms ease;
+	}
+
+	.git-blocking-dialog.entered {
+		opacity: 1;
+		transform: translateY(0) scale(1);
+	}
+
+	.git-blocking-dialog::backdrop {
+		background: rgb(0 0 0 / 0%);
+		backdrop-filter: blur(0);
+		transition:
+			background-color 200ms ease,
+			backdrop-filter 200ms ease;
+	}
+
+	.git-blocking-dialog.entered::backdrop {
+		background: rgb(0 0 0 / 60%);
+		backdrop-filter: blur(2px);
+	}
+
+	.git-blocking-dialog.stacked.entered::backdrop {
+		background: rgb(0 0 0 / 25%);
+		backdrop-filter: blur(1px);
+	}
+
+	@media (prefers-reduced-motion: reduce) {
+		.git-blocking-dialog,
+		.git-blocking-dialog::backdrop {
+			transition-duration: 0.01ms;
+		}
+	}
+</style>