@resolveio/server-lib 22.3.219 → 22.3.221

package/src/util/ai-run-evidence-eval.ts

@@ -0,0 +1,1057 @@
+import {
+	AIRun,
+	AIRunOutcomeLabel,
+	AIRunSource,
+	AIRunTrainingExample,
+	buildAIRun,
+	buildAIQaRun,
+	redactAIRunTrainingExample
+} from './ai-run-evidence';
+
+export type AIRunFailureClass =
+	| 'none'
+	| 'infra'
+	| 'compile'
+	| 'route'
+	| 'journey_contract'
+	| 'business'
+	| 'release'
+	| 'false_pass'
+	| 'assistant_missing_evidence'
+	| 'assistant_query_error'
+	| 'assistant_permission'
+	| 'assistant_date_window'
+	| 'assistant_query_shape'
+	| 'assistant_correctness'
+	| 'manual'
+	| 'stopped'
+	| 'unknown';
+
+export type AIRunRetryAction =
+	| 'none'
+	| 'stop'
+	| 'retry_infra_repair'
+	| 'retry_build_repair'
+	| 'retry_route_repair'
+	| 'retry_journey_contract'
+	| 'retry_business_repair'
+	| 'retry_release_repair'
+	| 'retry_assistant_answer_quality'
+	| 'retry_assistant_query'
+	| 'request_assistant_permission'
+	| 'retry_assistant_date_window'
+	| 'retry_assistant_query_shape'
+	| 'manual_handoff';
+
+export interface AIRunEvalExpectedGate {
+	key: string;
+	status?: string;
+}
+
+export interface AIRunEvalExpected {
+	outcome: AIRunOutcomeLabel;
+	failureClass?: AIRunFailureClass;
+	nextActionIncludes?: string;
+	gates?: AIRunEvalExpectedGate[];
+}
+
+export interface AIRunEvalFixture {
+	id: string;
+	title: string;
+	source: AIRunSource;
+	run: AIRun;
+	expected: AIRunEvalExpected;
+	tags: string[];
+}
+
+export interface AIRunEvalResult {
+	id: string;
+	title: string;
+	source: AIRunSource;
+	passed: boolean;
+	actualOutcome: AIRunOutcomeLabel;
+	expectedOutcome: AIRunOutcomeLabel;
+	actualFailureClass: AIRunFailureClass;
+	expectedFailureClass?: AIRunFailureClass;
+	failures: string[];
+	tags: string[];
+}
+
+export interface AIRunEvalSummary {
+	passed: boolean;
+	total: number;
+	passedCount: number;
+	failedCount: number;
+	results: AIRunEvalResult[];
+	failuresByTag: Record<string, number>;
+	outcomes: Record<string, number>;
+}
+
+export interface AIRunDatasetExport {
+	generatedAt: string;
+	totalRuns: number;
+	summary: {
+		bySource: Record<string, number>;
+		byOutcome: Record<string, number>;
+		byFailureClass: Record<string, number>;
+		estimatedUsd: number;
+		totalTokens: number;
+	};
+	runs: AIRun[];
+	trainingExamples: AIRunTrainingExample[];
+}
+
+export interface AIRunDatasetExportOptions {
+	redact?: boolean;
+	includeRuns?: boolean;
+	maxEvidenceItems?: number;
+	now?: Date | string;
+}
+
+export interface AIRunRetryPolicyInput {
+	currentRun: AIRun;
+	previousRun?: AIRun;
+	newEvidenceCount?: number;
+	unchangedFailureLimit?: number;
+	sameFailureCount?: number;
+	historicalAcceptanceRate?: number;
+	estimatedNextRetryCostUsd?: number;
+	minRetryExpectedValue?: number;
+}
+
+export interface AIRunRetryDecision {
+	action: AIRunRetryAction;
+	failureClass: AIRunFailureClass;
+	repeatedFailure: boolean;
+	reason: string;
+	nextAction: string;
+	retryExpectedValue?: number;
+}
+
+function cleanText(value: any, max = 1000): string {
+	return String(value || '').replace(/\s+/g, ' ').trim().slice(0, max);
+}
+
+function countBy<T extends string>(values: T[]): Record<string, number> {
+	const result: Record<string, number> = {};
+	for (const value of values) {
+		result[value] = (result[value] || 0) + 1;
+	}
+	return result;
+}
+
+function isoNow(value?: Date | string): string {
+	if (value instanceof Date) {
+		return value.toISOString();
+	}
+	const parsed = value ? new Date(value) : new Date();
+	if (Number.isFinite(parsed.getTime())) {
+		return parsed.toISOString();
+	}
+	return new Date().toISOString();
+}
+
+function addUnique(values: string[], value: any, max = 1000): void {
+	const normalized = cleanText(value, max);
+	if (normalized && !values.includes(normalized)) {
+		values.push(normalized);
+	}
+}
+
+function normalizedStatus(value: any): string {
+	return cleanText(value, 160).toLowerCase().replace(/[\s-]+/g, '_');
+}
+
+function plainObject(value: any): Record<string, any> {
+	return value && typeof value === 'object' && !Array.isArray(value) ? value : {};
+}
+
+function assistantAnswerQualityMetadata(run: AIRun): Record<string, any> {
+	const gate = (run.gates || []).find((entry) => entry.key === 'assistant_answer_quality');
+	const gateMetadata = plainObject(gate?.metadata);
+	const runMetadata = plainObject(run.metadata);
+	const answerQuality = plainObject(runMetadata.answerQuality || runMetadata.answer_quality);
+	return {
+		...gateMetadata,
+		...answerQuality
+	};
+}
+
+function classifyAssistantFailureClass(run: AIRun): AIRunFailureClass {
+	const metadata = assistantAnswerQualityMetadata(run);
+	const status = normalizedStatus(metadata.status || metadata.answerQualityStatus || metadata.answer_quality_status);
+	const queryStatus = normalizedStatus(metadata.queryStatus || metadata.query_status);
+	const missingEvidenceStatuses = new Set([
+		'missing_data_source',
+		'missing_query_status',
+		'missing_query_proof',
+		'missing_citations',
+		'missing_citation',
+		'low_confidence',
+		'missing_next_action',
+		'no_data_unverified',
+		'blocked'
+	]);
+	if (queryStatus === 'permission_error' || status === 'permission_error') {
+		return 'assistant_permission';
+	}
+	if (queryStatus === 'query_error' || status === 'query_error') {
+		return 'assistant_query_error';
+	}
+	if (status === 'date_incorrect' || status === 'missing_date_window') {
+		return 'assistant_date_window';
+	}
+	if (status === 'illegal_query_shape') {
+		return 'assistant_query_shape';
+	}
+	if (missingEvidenceStatuses.has(status) || !status) {
+		return 'assistant_missing_evidence';
+	}
+	return 'assistant_correctness';
+}
+
+export function classifyAIRunFailureClass(run: AIRun): AIRunFailureClass {
+	if (run.outcome === 'accepted') {
+		return 'none';
+	}
+	if (run.outcome === 'qa_infra_failed') {
+		return 'infra';
+	}
+	if (run.outcome === 'build_failed') {
+		return 'compile';
+	}
+	if (run.outcome === 'release_blocked') {
+		return 'release';
+	}
+	if (run.source === 'aicoder_app' && (run.gates || []).some((gate) => gate.key === 'aicoder_journey_contract' && (gate.status === 'blocked' || gate.status === 'fail'))) {
+		return 'journey_contract';
+	}
+	if (run.source === 'ai_assistant' && run.outcome === 'false_pass') {
+		return classifyAssistantFailureClass(run);
+	}
+	if (run.outcome === 'false_pass') {
+		return 'false_pass';
+	}
+	if (run.outcome === 'manual_handoff') {
+		return 'manual';
+	}
+	if (run.outcome === 'stopped') {
+		return 'stopped';
+	}
+	if (run.source === 'ai_assistant' && (run.outcome === 'rejected' || run.outcome === 'qa_incomplete')) {
+		return classifyAssistantFailureClass(run);
+	}
+	if (run.qa?.outcome === 'compile_failed') {
+		return 'compile';
+	}
+	if (run.qa?.outcome === 'route_failed') {
+		return 'route';
+	}
+	if (run.qa?.outcome === 'route_only_pass' || run.qa?.outcome === 'incomplete') {
+		return 'business';
+	}
+	if (run.qa?.outcome === 'business_assertion_failed') {
+		return 'business';
+	}
+	if (run.outcome === 'qa_incomplete') {
+		return 'business';
+	}
+	if (run.outcome === 'rejected') {
+		return 'business';
+	}
+	return 'unknown';
+}
+
+export function evaluateAIRunFixture(fixture: AIRunEvalFixture): AIRunEvalResult {
+	const failures: string[] = [];
+	const actualFailureClass = classifyAIRunFailureClass(fixture.run);
+	if (fixture.run.outcome !== fixture.expected.outcome) {
+		failures.push(`Expected outcome ${fixture.expected.outcome}, got ${fixture.run.outcome}.`);
+	}
+	if (fixture.expected.failureClass && actualFailureClass !== fixture.expected.failureClass) {
+		failures.push(`Expected failure class ${fixture.expected.failureClass}, got ${actualFailureClass}.`);
+	}
+	if (fixture.expected.nextActionIncludes) {
+		const expectedText = fixture.expected.nextActionIncludes.toLowerCase();
+		const actualText = cleanText(fixture.run.nextAction, 2000).toLowerCase();
+		if (!actualText.includes(expectedText)) {
+			failures.push(`Expected next action to include "${fixture.expected.nextActionIncludes}", got "${fixture.run.nextAction || ''}".`);
+		}
+	}
+	const gatesByKey: Record<string, string> = {};
+	for (const gate of fixture.run.gates || []) {
+		gatesByKey[gate.key] = gate.status;
+	}
+	for (const expectedGate of fixture.expected.gates || []) {
+		if (!gatesByKey[expectedGate.key]) {
+			failures.push(`Expected gate ${expectedGate.key} to be present.`);
+		}
+		else if (expectedGate.status && gatesByKey[expectedGate.key] !== expectedGate.status) {
+			failures.push(`Expected gate ${expectedGate.key} status ${expectedGate.status}, got ${gatesByKey[expectedGate.key]}.`);
+		}
+	}
+	return {
+		id: fixture.id,
+		title: fixture.title,
+		source: fixture.source,
+		passed: failures.length === 0,
+		actualOutcome: fixture.run.outcome,
+		expectedOutcome: fixture.expected.outcome,
+		actualFailureClass,
+		expectedFailureClass: fixture.expected.failureClass,
+		failures,
+		tags: fixture.tags
+	};
+}
+
+export function evaluateAIRunFixtures(fixtures: AIRunEvalFixture[] = AI_RUN_SEED_EVAL_FIXTURES): AIRunEvalSummary {
+	const results = fixtures.map((fixture) => evaluateAIRunFixture(fixture));
+	const failuresByTag: Record<string, number> = {};
+	for (const result of results) {
+		if (result.passed) {
+			continue;
+		}
+		for (const tag of result.tags) {
+			failuresByTag[tag] = (failuresByTag[tag] || 0) + 1;
+		}
+	}
+	const outcomes = countBy(results.map((result) => result.actualOutcome));
+	const passedCount = results.filter((result) => result.passed).length;
+	return {
+		passed: passedCount === results.length,
+		total: results.length,
+		passedCount,
+		failedCount: results.length - passedCount,
+		results,
+		failuresByTag,
+		outcomes
+	};
+}
+
+export function buildAIRunTrainingExampleFromRun(run: AIRun, maxEvidenceItems = 30): AIRunTrainingExample {
+	const evidence: string[] = [];
+	for (const gate of run.gates || []) {
+		addUnique(evidence, `${gate.key}:${gate.status}: ${gate.reason}`, 1600);
+	}
+	for (const event of run.events || []) {
+		addUnique(evidence, `${event.type}:${event.category || ''}: ${event.message || ''}`, 1400);
+		if (evidence.length >= maxEvidenceItems) {
+			break;
+		}
+	}
+	if (evidence.length < maxEvidenceItems) {
+		for (const assertion of run.qa?.businessAssertions || []) {
+			addUnique(evidence, `assertion:${assertion.status}: ${assertion.assertion} observed=${assertion.observed || assertion.dataProof || assertion.message || ''}`, 1600);
+			if (evidence.length >= maxEvidenceItems) {
+				break;
+			}
+		}
+	}
+	return {
+		runId: run.id || run.sourceIds.jobId || run.sourceIds.ticketId || run.sourceIds.appId || run.sourceIds.conversationId || `${run.source}:unknown`,
+		source: run.source,
+		inputSummary: [
+			`source=${run.source}`,
+			`title=${run.title || ''}`,
+			`status=${run.status || ''}`,
+			`phase=${run.phase || ''}`,
+			`ids=${JSON.stringify(run.sourceIds || {})}`
+		].join(' | '),
+		actionSummary: [
+			`outcome=${run.outcome}`,
+			`failureClass=${classifyAIRunFailureClass(run)}`,
+			`nextAction=${run.nextAction || ''}`
+		].join(' | '),
+		outcome: run.outcome,
+		evidence: evidence.slice(0, maxEvidenceItems),
+		cost: run.cost,
+		metadata: {
+			warnings: run.warnings || [],
+			gateCount: (run.gates || []).length,
+			eventCount: (run.events || []).length,
+			qaOutcome: run.qa?.outcome
+		}
+	};
+}
+
+export function buildAIRunDatasetExport(runs: AIRun[], options: AIRunDatasetExportOptions = {}): AIRunDatasetExport {
+	const generatedAt = isoNow(options.now);
+	const trainingExamples = runs.map((run) => {
+		const example = buildAIRunTrainingExampleFromRun(run, options.maxEvidenceItems || 30);
+		return options.redact === false ? example : redactAIRunTrainingExample(example);
+	});
+	const estimatedUsd = runs.reduce((sum, run) => sum + (Number(run.cost?.estimatedUsd) || 0), 0);
+	const totalTokens = runs.reduce((sum, run) => sum + (Number(run.cost?.totalTokens) || 0), 0);
+	return {
+		generatedAt,
+		totalRuns: runs.length,
+		summary: {
+			bySource: countBy(runs.map((run) => run.source)),
+			byOutcome: countBy(runs.map((run) => run.outcome)),
+			byFailureClass: countBy(runs.map((run) => classifyAIRunFailureClass(run))),
+			estimatedUsd: Number(estimatedUsd.toFixed(6)),
+			totalTokens
+		},
+		runs: options.includeRuns === false ? [] : runs,
+		trainingExamples
+	};
+}
+
+export function decideAIRunRetry(input: AIRunRetryPolicyInput): AIRunRetryDecision {
+	const failureClass = classifyAIRunFailureClass(input.currentRun);
+	const previousFailureClass = input.previousRun ? classifyAIRunFailureClass(input.previousRun) : undefined;
+	const sameFailureCount = Number(input.sameFailureCount || 0);
+	const unchangedFailureLimit = Number(input.unchangedFailureLimit || 2);
+	const repeatedFailure = previousFailureClass === failureClass && (Number(input.newEvidenceCount || 0) <= 0 || sameFailureCount >= unchangedFailureLimit);
+	let retryExpectedValue: number | undefined;
+	if (input.historicalAcceptanceRate !== undefined && input.estimatedNextRetryCostUsd !== undefined) {
+		retryExpectedValue = Number((Number(input.historicalAcceptanceRate) - Number(input.estimatedNextRetryCostUsd)).toFixed(6));
+	}
+	if (input.currentRun.outcome === 'accepted') {
+		return {
+			action: 'none',
+			failureClass,
+			repeatedFailure: false,
+			reason: 'Run is already accepted.',
+			nextAction: 'No retry required.',
+			retryExpectedValue
+		};
+	}
+	if (failureClass === 'manual') {
+		return {
+			action: 'manual_handoff',
+			failureClass,
+			repeatedFailure,
+			reason: 'Run is marked for manual handoff.',
+			nextAction: input.currentRun.nextAction || 'Ask an operator to resolve the run.',
+			retryExpectedValue
+		};
+	}
+	if (failureClass === 'stopped') {
+		return {
+			action: 'stop',
+			failureClass,
+			repeatedFailure,
+			reason: 'Run was stopped before accepted evidence was recorded.',
+			nextAction: input.currentRun.nextAction || 'Do not spend another model cycle unless the request is resumed.',
+			retryExpectedValue
+		};
+	}
+	if (repeatedFailure) {
+		return {
+			action: 'stop',
+			failureClass,
+			repeatedFailure,
+			reason: 'The same failure class repeated with no new evidence. Stop model loops until the evidence changes.',
+			nextAction: input.currentRun.nextAction || 'Inspect the failing gate before retrying.',
+			retryExpectedValue
+		};
+	}
+	if (retryExpectedValue !== undefined && retryExpectedValue < Number(input.minRetryExpectedValue || 0)) {
+		return {
+			action: 'stop',
+			failureClass,
+			repeatedFailure,
+			reason: 'Historical retry expected value is below the configured threshold.',
+			nextAction: input.currentRun.nextAction || 'Do not retry without new evidence or a cheaper targeted repair.',
+			retryExpectedValue
+		};
+	}
+	if (failureClass === 'infra') {
+		return {
+			action: 'retry_infra_repair',
+			failureClass,
+			repeatedFailure,
+			reason: 'QA infrastructure failed before business validation.',
+			nextAction: input.currentRun.nextAction || 'Repair browser, settings, Mongo, ports, or startup before model repair.',
+			retryExpectedValue
+		};
+	}
+	if (failureClass === 'compile') {
+		return {
+			action: 'retry_build_repair',
+			failureClass,
+			repeatedFailure,
+			reason: 'Compile/build evidence failed or was stale.',
+			nextAction: input.currentRun.nextAction || 'Repair build before browser QA.',
+			retryExpectedValue
+		};
+	}
+	if (failureClass === 'route') {
+		return {
+			action: 'retry_route_repair',
+			failureClass,
+			repeatedFailure,
+			reason: 'Route probe failed before business proof.',
+			nextAction: input.currentRun.nextAction || 'Repair route/auth/startup and rerun the exact probe.',
+			retryExpectedValue
+		};
+	}
+	if (failureClass === 'journey_contract') {
+		return {
+			action: 'retry_journey_contract',
+			failureClass,
+			repeatedFailure,
+			reason: 'AICoder Journey Contract is missing or invalid before workflow build/proof.',
+			nextAction: input.currentRun.nextAction || 'Generate or repair docs/APP_JOURNEY_CONTRACT.md, validate the north-star workflow, then regenerate workflow QA rows.',
+			retryExpectedValue
+		};
+	}
+	if (failureClass === 'assistant_missing_evidence') {
+		return {
+			action: 'retry_assistant_answer_quality',
+			failureClass,
+			repeatedFailure,
+			reason: 'Assistant answer is missing structured answerQuality, query proof, citations, confidence, or next-action evidence.',
+			nextAction: input.currentRun.nextAction || 'Rebuild the answerQuality record from the exact query/tool evidence before another broad assistant response.',
+			retryExpectedValue
+		};
+	}
+	if (failureClass === 'assistant_query_error') {
+		return {
+			action: 'retry_assistant_query',
+			failureClass,
+			repeatedFailure,
+			reason: 'Assistant data access failed with a query error.',
+			nextAction: input.currentRun.nextAction || 'Repair and rerun the failing query/tool call, then answer only from the new query evidence.',
+			retryExpectedValue
+		};
+	}
+	if (failureClass === 'assistant_permission') {
+		return {
+			action: 'request_assistant_permission',
+			failureClass,
+			repeatedFailure,
+			reason: 'Assistant data access was blocked by permission or authorization.',
+			nextAction: input.currentRun.nextAction || 'Ask for permission, escalate to an authorized user, or answer with the permission blocker instead of claiming no data.',
+			retryExpectedValue
+		};
+	}
+	if (failureClass === 'assistant_date_window') {
+		return {
+			action: 'retry_assistant_date_window',
+			failureClass,
+			repeatedFailure,
+			reason: 'Assistant date-sensitive answer used the wrong current date or omitted the concrete date window.',
+			nextAction: input.currentRun.nextAction || 'Rerun the query using the actual current date and record exact start/end date-window evidence.',
+			retryExpectedValue
+		};
+	}
+	if (failureClass === 'assistant_query_shape') {
+		return {
+			action: 'retry_assistant_query_shape',
+			failureClass,
+			repeatedFailure,
+			reason: 'Assistant query shape is illegal or unsafe for Mongo.',
+			nextAction: input.currentRun.nextAction || 'Rewrite the query with a legal projection or aggregation shape before retrying the data answer.',
+			retryExpectedValue
+		};
+	}
+	if (failureClass === 'release') {
+		return {
+			action: 'retry_release_repair',
+			failureClass,
+			repeatedFailure,
+			reason: 'Business proof exists but deploy, publish, or sample-data gates block release.',
+			nextAction: input.currentRun.nextAction || 'Repair release gates without another broad model planning loop.',
+			retryExpectedValue
+		};
+	}
+	if (failureClass === 'business' || failureClass === 'false_pass' || failureClass === 'assistant_correctness') {
+		return {
+			action: 'retry_business_repair',
+			failureClass,
+			repeatedFailure,
+			reason: 'Business or assistant correctness proof failed or is missing.',
+			nextAction: input.currentRun.nextAction || 'Repair the exact failing workflow/query and rerun the deterministic assertion.',
+			retryExpectedValue
+		};
+	}
+	return {
+		action: 'manual_handoff',
+		failureClass,
+		repeatedFailure,
+		reason: 'Run lacks enough normalized evidence to choose a safe retry lane.',
+		nextAction: input.currentRun.nextAction || 'Ingest more run evidence before retrying.',
+		retryExpectedValue
+	};
+}
+
+const supportAccepted004131 = buildAIRun({
+	id: 'support-004131',
+	source: 'support_ticket',
+	sourceIds: { ticketNumber: '004131', ticketId: 'ticket-004131' },
+	title: '004131 accepted support fix',
+	status: 'Resolved',
+	phase: 'COMPLETE',
+	events: [{ type: 'git_commit', message: 'Fix support workflow and verified before/action/after proof.' }],
+	qa: buildAIQaRun({
+		compile: { status: 'pass', artifactPath: '.build-output/build-server.log' },
+		routeProbes: [{ route: '/support/004131', status: 'pass', screenshot: 'qa-artifacts/004131-after.jpg' }],
+		businessAssertions: [{
+			assertion: 'Before/action/after workflow proves the ticket fix.',
+			status: 'pass',
+			action: 'Run the ticket workflow against seeded data.',
+			expected: 'The corrected state is visible.',
+			observed: 'The corrected state is visible after the action.',
+			artifactPaths: ['qa-artifacts/004131-proof.json']
+		}]
+	}),
+	explicitAccepted: true,
+	terminal: true
+});
+
+const supportBuildFailed004333 = buildAIRun({
+	id: 'support-004333',
+	source: 'support_ticket',
+	sourceIds: { ticketNumber: '004333', ticketId: 'ticket-004333' },
+	title: '004333 compile failure',
+	status: 'Running',
+	phase: 'QA',
+	events: [{ type: 'log', message: 'Angular build failed before browser QA.' }],
+	qa: buildAIQaRun({
+		compile: { status: 'fail', artifactPath: '.build-output/build-client.log', message: 'TypeScript compile failed.' }
+	})
+});
+
+const supportInfraFailed004341 = buildAIRun({
+	id: 'support-004341',
+	source: 'support_ticket',
+	sourceIds: { ticketNumber: '004341', ticketId: 'ticket-004341' },
+	title: '004341 missing Puppeteer',
+	status: 'Running',
+	phase: 'QA',
+	events: [{ type: 'qa_infra', message: 'Puppeteer module could not be loaded.' }],
+	qa: buildAIQaRun({
+		infraChecks: [{ name: 'puppeteer_require', status: 'blocked', message: 'Cannot find module puppeteer.' }]
+	})
+});
+
+const supportFalsePass004421 = buildAIRun({
+	id: 'support-004421',
+	source: 'support_ticket',
+	sourceIds: { ticketNumber: '004421', ticketId: 'ticket-004421' },
+	title: '004421 route-only support QA',
+	status: 'Passed',
+	phase: 'QA',
+	events: [{ type: 'scorecard', message: 'Scorecard passed after route loaded.' }],
+	qa: buildAIQaRun({
+		infraChecks: [{ name: 'chrome_launch', status: 'pass' }],
+		compile: { status: 'pass' },
+		routeProbes: [{ route: '/support/004421', status: 'pass', screenshot: 'qa-artifacts/004421-route.jpg' }]
+	}),
+	scorecardPassed: true
+});
+
+const supportBusinessFailed004423 = buildAIRun({
+	id: 'support-004423',
+	source: 'support_ticket',
+	sourceIds: { ticketNumber: '004423', ticketId: 'ticket-004423' },
+	title: '004423 business assertion failed',
+	status: 'Running',
+	phase: 'QA',
+	events: [{ type: 'qa_business_assertion', message: 'The expected workflow state did not change.' }],
+	qa: buildAIQaRun({
+		compile: { status: 'pass' },
+		routeProbes: [{ route: '/support/004423', status: 'pass' }],
+		businessAssertions: [{
+			assertion: 'Ticket workflow changes the requested field.',
+			status: 'fail',
+			expected: 'Field updates after save.',
+			observed: 'Field stayed unchanged.'
+		}]
+	})
+});
+
+const supportStaleBuild004430 = buildAIRun({
+	id: 'support-004430',
+	source: 'support_ticket',
+	sourceIds: { ticketNumber: '004430', ticketId: 'ticket-004430' },
+	title: '004430 stale build evidence',
+	status: 'Running',
+	phase: 'QA',
+	events: [{ type: 'qa_compile', message: 'Build log predates the final code diff.' }],
+	qa: buildAIQaRun({
+		compile: { status: 'stale', staleEvidence: true, artifactPath: '.build-output/build-client.log' }
+	})
+});
+
+const cryptoPortfolioAccepted = buildAIRun({
+	id: 'aicoder-crypto-v6-golden',
+	source: 'aicoder_app',
+	sourceIds: { appId: 'crypto-v6-golden', jobId: 'crypto-job-golden' },
+	title: 'Crypto Portfolio Tracker V6 Golden',
+	status: 'Passed',
+	phase: 'COMPLETE',
+	events: [{ type: 'scorecard', message: 'Quality scorecard passed with workflow proof.' }],
+	qa: buildAIQaRun({
+		compile: { status: 'pass' },
+		routeProbes: [{ route: '/portfolio', status: 'pass', screenshot: 'qa-artifacts/crypto-dashboard.jpg' }],
+		businessAssertions: [{
+			assertion: 'Portfolio dashboard calculates seeded crypto holdings.',
+			status: 'pass',
+			dataProof: 'Seeded BTC and ETH holdings render totals, allocation, and change.'
+		}]
+	}),
+	scorecardPassed: true,
+	deployStatus: 'pass',
+	publishStatus: 'published',
+	sampleDataStatus: 'pass',
+	terminal: true
+});
+
+const layflatReleaseBlocked = buildAIRun({
+	id: 'aicoder-layflat-pressure-optimizer',
+	source: 'aicoder_app',
+	sourceIds: { appId: 'layflat-pressure-optimizer', jobId: 'layflat-job' },
+	title: 'Layflat Pressure Optimizer',
+	status: 'Passed',
+	phase: 'COMPLETE',
+	events: [{ type: 'scorecard', message: 'Scorecard passed but sample data/publish gate failed.' }],
+	qa: buildAIQaRun({
+		compile: { status: 'pass' },
+		routeProbes: [{ route: '/optimizer', status: 'pass', screenshot: 'qa-artifacts/layflat-route.jpg' }],
+		businessAssertions: [{
+			assertion: 'Pressure optimizer computes a layflat hose result.',
+			status: 'pass',
+			dataProof: 'A seeded hose diameter and flow rate produce PSI loss.'
+		}]
+	}),
+	scorecardPassed: true,
+	deployStatus: 'pass',
+	publishStatus: 'publish failed',
+	sampleDataStatus: 'empty seed collections',
+	terminal: true
+});
+
+const assistantWrongDateWindow = buildAIRun({
+	id: 'assistant-wrong-month-window',
+	source: 'ai_assistant',
+	sourceIds: { conversationId: 'assistant-date-window' },
+	title: 'Assistant wrong month/date-window answer',
+	status: 'Closed',
+	phase: 'ANSWERED',
+	events: [{ type: 'assistant_message', message: 'Answered with the wrong month for today-sensitive invoice data.' }],
+	qa: buildAIQaRun({
+		businessAssertions: [{
+			assertion: 'Assistant date window uses the actual current date.',
+			status: 'fail',
+			expected: 'Use June 2026 for current-month windows.',
+			observed: 'Used a stale month.'
+		}]
+	}),
+	gates: [{
+		key: 'assistant_answer_quality',
+		label: 'Assistant answer quality',
+		status: 'fail',
+		reason: 'Assistant answer did not prove it used the actual current date.',
+		evidenceRefs: [],
+		recordedAt: '2026-06-16T00:00:00.000Z',
+		metadata: {
+			status: 'date_incorrect',
+			queryStatus: 'ok',
+			dateBasis: '2026-05-01',
+			expectedCurrentDate: '2026-06-16'
+		}
+	}],
+	metadata: {
+		answerQuality: {
+			status: 'date_incorrect',
+			queryStatus: 'ok',
+			dateBasis: '2026-05-01',
+			expectedCurrentDate: '2026-06-16'
+		}
+	},
+	rejected: true,
+	terminal: true
+});
+
+const assistantIllegalProjection = buildAIRun({
+	id: 'assistant-illegal-mongo-projection',
+	source: 'ai_assistant',
+	sourceIds: { conversationId: 'assistant-positional-projection' },
+	title: 'Assistant illegal Mongo positional projection',
+	status: 'Closed',
+	phase: 'ANSWERED',
+	events: [{ type: 'assistant_message', message: 'Mongo query used an illegal positional projection shape.' }],
+	qa: buildAIQaRun({
+		businessAssertions: [{
+			assertion: 'Assistant avoids illegal Mongo positional projection.',
+			status: 'fail',
+			expected: 'Use legal aggregation or field projection shape.',
+			observed: 'Used invalid positional projection.'
+		}]
+	}),
+	gates: [{
+		key: 'assistant_answer_quality',
+		label: 'Assistant answer quality',
+		status: 'fail',
+		reason: 'Assistant must use legal Mongo query/projection shapes before answering.',
+		evidenceRefs: [],
+		recordedAt: '2026-06-16T00:00:00.000Z',
+		metadata: {
+			status: 'illegal_query_shape',
+			queryStatus: 'ok',
+			legalQueryShape: false
+		}
+	}],
+	metadata: {
+		answerQuality: {
+			status: 'illegal_query_shape',
+			queryStatus: 'ok',
+			legalQueryShape: false
+		}
+	},
+	rejected: true,
+	terminal: true
+});
+
+const qaMissingPuppeteer = buildAIRun({
+	id: 'qa-missing-puppeteer',
+	source: 'qa_runner',
+	sourceIds: { jobId: 'qa-missing-puppeteer' },
+	title: 'QA runner missing Puppeteer',
+	status: 'Running',
+	phase: 'PREFLIGHT',
+	events: [{ type: 'qa_infra', message: 'Puppeteer failed to require.' }],
+	qa: buildAIQaRun({
+		infraChecks: [{ name: 'puppeteer_require', status: 'blocked', message: 'Cannot find module puppeteer.' }]
+	})
+});
+
+const qaBadChromePath = buildAIRun({
+	id: 'qa-bad-chrome-path',
+	source: 'qa_runner',
+	sourceIds: { jobId: 'qa-bad-chrome-path' },
+	title: 'QA runner bad Chrome path',
+	status: 'Running',
+	phase: 'PREFLIGHT',
+	events: [{ type: 'qa_infra', message: 'Chrome executable path was missing.' }],
+	qa: buildAIQaRun({
+		infraChecks: [{ name: 'chrome_executable', status: 'fail', message: 'Chrome executable not found.', path: '/bad/chrome' }]
+	})
+});
+
+const qaCompileFailure = buildAIRun({
+	id: 'qa-compile-failure',
+	source: 'qa_runner',
+	sourceIds: { jobId: 'qa-compile-failure' },
+	title: 'QA runner compile failure',
+	status: 'Running',
+	phase: 'COMPILE',
+	events: [{ type: 'qa_compile', message: 'Compile failed before browser QA.' }],
+	qa: buildAIQaRun({
+		compile: { status: 'fail', command: 'npm run build', artifactPath: '.build-output/build.log' }
+	})
+});
+
+const qaRouteOnlyPass = buildAIRun({
+	id: 'qa-route-only-pass',
+	source: 'qa_runner',
+	sourceIds: { jobId: 'qa-route-only-pass' },
+	title: 'QA runner route-only pass',
+	status: 'Passed',
+	phase: 'BROWSER_QA',
+	events: [{ type: 'qa_route_probe', message: 'Route loaded but no business proof exists.' }],
+	qa: buildAIQaRun({
+		compile: { status: 'pass' },
+		routeProbes: [{ route: '/dashboard', status: 'pass', screenshot: 'qa-artifacts/dashboard.jpg' }]
+	}),
+	scorecardPassed: true
+});
+
+const qaShellOnlyPage = buildAIRun({
+	id: 'qa-shell-only-page',
+	source: 'qa_runner',
+	sourceIds: { jobId: 'qa-shell-only-page' },
+	title: 'QA runner shell-only page',
+	status: 'Running',
+	phase: 'BROWSER_QA',
+	events: [{ type: 'qa_route_probe', message: 'Shell-only route loaded.' }],
+	qa: buildAIQaRun({
+		compile: { status: 'pass' },
+		routeProbes: [{ route: '/dashboard', status: 'pass', shellOnly: true, screenshot: 'qa-artifacts/shell.jpg' }]
+	})
+});
+
+const qaEmptyData = buildAIRun({
+	id: 'qa-empty-data',
+	source: 'qa_runner',
+	sourceIds: { jobId: 'qa-empty-data' },
+	title: 'QA runner empty data',
+	status: 'Failed',
+	phase: 'BROWSER_QA',
+	events: [{ type: 'qa_business_assertion', message: 'Workflow rendered empty data.' }],
+	qa: buildAIQaRun({
+		compile: { status: 'pass' },
+		routeProbes: [{ route: '/invoices', status: 'pass' }],
+		businessAssertions: [{
+			assertion: 'Invoice dashboard renders non-empty meaningful data.',
+			status: 'fail',
+			expected: 'At least one meaningful invoice row.',
+			observed: 'Empty table.'
+		}]
+	}),
+	terminal: true
+});
+
+const qaPublishFailureAfterScorecard = buildAIRun({
+	id: 'qa-scorecard-pass-publish-failure',
+	source: 'qa_runner',
+	sourceIds: { jobId: 'qa-publish-failure' },
+	title: 'QA scorecard pass followed by publish failure',
+	status: 'Passed',
+	phase: 'RELEASE',
+	events: [{ type: 'publish', message: 'Scorecard passed but publish failed.' }],
+	qa: buildAIQaRun({
+		compile: { status: 'pass' },
+		routeProbes: [{ route: '/app', status: 'pass' }],
+		businessAssertions: [{
+			assertion: 'Primary app workflow runs.',
+			status: 'pass',
+			dataProof: 'Workflow action returned expected state.'
+		}]
+	}),
+	scorecardPassed: true,
+	publishStatus: 'publish failed',
+	sampleDataStatus: 'pass'
+});
+
+export const AI_RUN_SEED_EVAL_FIXTURES: AIRunEvalFixture[] = [
+	{
+		id: 'support-004131',
+		title: 'Support 004131 accepted proof',
+		source: 'support_ticket',
+		run: supportAccepted004131,
+		expected: { outcome: 'accepted', failureClass: 'none', gates: [{ key: 'qa_business_assertion', status: 'pass' }] },
+		tags: ['support', 'accepted', 'business_assertion']
+	},
+	{
+		id: 'support-004333',
+		title: 'Support 004333 compile failure',
+		source: 'support_ticket',
+		run: supportBuildFailed004333,
+		expected: { outcome: 'build_failed', failureClass: 'compile', nextActionIncludes: 'compile' },
+		tags: ['support', 'compile']
+	},
+	{
+		id: 'support-004341',
+		title: 'Support 004341 Puppeteer infra failure',
+		source: 'support_ticket',
+		run: supportInfraFailed004341,
+		expected: { outcome: 'qa_infra_failed', failureClass: 'infra', nextActionIncludes: 'browser' },
+		tags: ['support', 'infra', 'puppeteer']
+	},
+	{
+		id: 'support-004421',
+		title: 'Support 004421 route-only false pass',
+		source: 'support_ticket',
+		run: supportFalsePass004421,
+		expected: { outcome: 'false_pass', failureClass: 'false_pass', gates: [{ key: 'qa_business_assertion', status: 'blocked' }] },
+		tags: ['support', 'route_only', 'false_pass']
+	},
+	{
+		id: 'support-004423',
+		title: 'Support 004423 business assertion failed',
+		source: 'support_ticket',
+		run: supportBusinessFailed004423,
+		expected: { outcome: 'qa_incomplete', failureClass: 'business', nextActionIncludes: 'assertion' },
+		tags: ['support', 'business_assertion']
+	},
+	{
+		id: 'support-004430',
+		title: 'Support 004430 stale build evidence',
+		source: 'support_ticket',
+		run: supportStaleBuild004430,
+		expected: { outcome: 'build_failed', failureClass: 'compile', nextActionIncludes: 'build' },
+		tags: ['support', 'compile', 'stale_build']
+	},
+	{
+		id: 'aicoder-crypto-v6-golden',
+		title: 'AICoder Crypto Portfolio Tracker V6 Golden',
+		source: 'aicoder_app',
+		run: cryptoPortfolioAccepted,
+		expected: { outcome: 'accepted', failureClass: 'none', gates: [{ key: 'qa_business_assertion', status: 'pass' }] },
+		tags: ['aicoder', 'crypto', 'accepted']
+	},
+	{
+		id: 'aicoder-layflat-pressure-optimizer',
+		title: 'AICoder Layflat Pressure Optimizer release blocked',
+		source: 'aicoder_app',
+		run: layflatReleaseBlocked,
+		expected: { outcome: 'release_blocked', failureClass: 'release', gates: [{ key: 'release', status: 'fail' }] },
+		tags: ['aicoder', 'layflat', 'release_blocked', 'sample_data']
+	},
+	{
+		id: 'assistant-wrong-month-window',
+		title: 'Assistant wrong month/date-window answer',
+		source: 'ai_assistant',
+		run: assistantWrongDateWindow,
+		expected: { outcome: 'rejected', failureClass: 'assistant_date_window' },
+		tags: ['assistant', 'date_window']
+	},
+	{
+		id: 'assistant-illegal-mongo-projection',
+		title: 'Assistant illegal Mongo positional projection',
+		source: 'ai_assistant',
+		run: assistantIllegalProjection,
+		expected: { outcome: 'rejected', failureClass: 'assistant_query_shape' },
+		tags: ['assistant', 'mongo_query']
+	},
+	{
+		id: 'qa-missing-puppeteer',
+		title: 'QA missing Puppeteer',
+		source: 'qa_runner',
+		run: qaMissingPuppeteer,
+		expected: { outcome: 'qa_infra_failed', failureClass: 'infra' },
+		tags: ['qa_runner', 'infra', 'puppeteer']
+	},
+	{
+		id: 'qa-bad-chrome-path',
+		title: 'QA bad Chrome path',
+		source: 'qa_runner',
+		run: qaBadChromePath,
+		expected: { outcome: 'qa_infra_failed', failureClass: 'infra' },
+		tags: ['qa_runner', 'infra', 'chrome']
+	},
+	{
+		id: 'qa-compile-failure',
+		title: 'QA compile failure',
+		source: 'qa_runner',
+		run: qaCompileFailure,
+		expected: { outcome: 'build_failed', failureClass: 'compile' },
+		tags: ['qa_runner', 'compile']
+	},
+	{
+		id: 'qa-route-only-pass',
+		title: 'QA route-only pass',
+		source: 'qa_runner',
+		run: qaRouteOnlyPass,
+		expected: { outcome: 'false_pass', failureClass: 'false_pass' },
+		tags: ['qa_runner', 'route_only', 'false_pass']
+	},
+	{
+		id: 'qa-shell-only-page',
+		title: 'QA shell-only page',
+		source: 'qa_runner',
+		run: qaShellOnlyPage,
+		expected: { outcome: 'qa_incomplete', failureClass: 'route' },
+		tags: ['qa_runner', 'route', 'shell_only']
+	},
+	{
+		id: 'qa-empty-data',
+		title: 'QA empty data',
+		source: 'qa_runner',
+		run: qaEmptyData,
+		expected: { outcome: 'rejected', failureClass: 'business' },
+		tags: ['qa_runner', 'empty_data', 'business_assertion']
+	},
+	{
+		id: 'qa-scorecard-pass-publish-failure',
+		title: 'QA scorecard pass followed by publish failure',
+		source: 'qa_runner',
+		run: qaPublishFailureAfterScorecard,
+		expected: { outcome: 'release_blocked', failureClass: 'release' },
+		tags: ['qa_runner', 'release_blocked', 'publish']
+	}
+];