@resolveio/server-lib 22.3.221 → 22.3.223

package/src/util/ai-run-evidence-eval.ts

@@ -1,1057 +0,0 @@
-import {
-	AIRun,
-	AIRunOutcomeLabel,
-	AIRunSource,
-	AIRunTrainingExample,
-	buildAIRun,
-	buildAIQaRun,
-	redactAIRunTrainingExample
-} from './ai-run-evidence';
-
-export type AIRunFailureClass =
-	| 'none'
-	| 'infra'
-	| 'compile'
-	| 'route'
-	| 'journey_contract'
-	| 'business'
-	| 'release'
-	| 'false_pass'
-	| 'assistant_missing_evidence'
-	| 'assistant_query_error'
-	| 'assistant_permission'
-	| 'assistant_date_window'
-	| 'assistant_query_shape'
-	| 'assistant_correctness'
-	| 'manual'
-	| 'stopped'
-	| 'unknown';
-
-export type AIRunRetryAction =
-	| 'none'
-	| 'stop'
-	| 'retry_infra_repair'
-	| 'retry_build_repair'
-	| 'retry_route_repair'
-	| 'retry_journey_contract'
-	| 'retry_business_repair'
-	| 'retry_release_repair'
-	| 'retry_assistant_answer_quality'
-	| 'retry_assistant_query'
-	| 'request_assistant_permission'
-	| 'retry_assistant_date_window'
-	| 'retry_assistant_query_shape'
-	| 'manual_handoff';
-
-export interface AIRunEvalExpectedGate {
-	key: string;
-	status?: string;
-}
-
-export interface AIRunEvalExpected {
-	outcome: AIRunOutcomeLabel;
-	failureClass?: AIRunFailureClass;
-	nextActionIncludes?: string;
-	gates?: AIRunEvalExpectedGate[];
-}
-
-export interface AIRunEvalFixture {
-	id: string;
-	title: string;
-	source: AIRunSource;
-	run: AIRun;
-	expected: AIRunEvalExpected;
-	tags: string[];
-}
-
-export interface AIRunEvalResult {
-	id: string;
-	title: string;
-	source: AIRunSource;
-	passed: boolean;
-	actualOutcome: AIRunOutcomeLabel;
-	expectedOutcome: AIRunOutcomeLabel;
-	actualFailureClass: AIRunFailureClass;
-	expectedFailureClass?: AIRunFailureClass;
-	failures: string[];
-	tags: string[];
-}
-
-export interface AIRunEvalSummary {
-	passed: boolean;
-	total: number;
-	passedCount: number;
-	failedCount: number;
-	results: AIRunEvalResult[];
-	failuresByTag: Record<string, number>;
-	outcomes: Record<string, number>;
-}
-
-export interface AIRunDatasetExport {
-	generatedAt: string;
-	totalRuns: number;
-	summary: {
-		bySource: Record<string, number>;
-		byOutcome: Record<string, number>;
-		byFailureClass: Record<string, number>;
-		estimatedUsd: number;
-		totalTokens: number;
-	};
-	runs: AIRun[];
-	trainingExamples: AIRunTrainingExample[];
-}
-
-export interface AIRunDatasetExportOptions {
-	redact?: boolean;
-	includeRuns?: boolean;
-	maxEvidenceItems?: number;
-	now?: Date | string;
-}
-
-export interface AIRunRetryPolicyInput {
-	currentRun: AIRun;
-	previousRun?: AIRun;
-	newEvidenceCount?: number;
-	unchangedFailureLimit?: number;
-	sameFailureCount?: number;
-	historicalAcceptanceRate?: number;
-	estimatedNextRetryCostUsd?: number;
-	minRetryExpectedValue?: number;
-}
-
-export interface AIRunRetryDecision {
-	action: AIRunRetryAction;
-	failureClass: AIRunFailureClass;
-	repeatedFailure: boolean;
-	reason: string;
-	nextAction: string;
-	retryExpectedValue?: number;
-}
-
-function cleanText(value: any, max = 1000): string {
-	return String(value || '').replace(/\s+/g, ' ').trim().slice(0, max);
-}
-
-function countBy<T extends string>(values: T[]): Record<string, number> {
-	const result: Record<string, number> = {};
-	for (const value of values) {
-		result[value] = (result[value] || 0) + 1;
-	}
-	return result;
-}
-
-function isoNow(value?: Date | string): string {
-	if (value instanceof Date) {
-		return value.toISOString();
-	}
-	const parsed = value ? new Date(value) : new Date();
-	if (Number.isFinite(parsed.getTime())) {
-		return parsed.toISOString();
-	}
-	return new Date().toISOString();
-}
-
-function addUnique(values: string[], value: any, max = 1000): void {
-	const normalized = cleanText(value, max);
-	if (normalized && !values.includes(normalized)) {
-		values.push(normalized);
-	}
-}
-
-function normalizedStatus(value: any): string {
-	return cleanText(value, 160).toLowerCase().replace(/[\s-]+/g, '_');
-}
-
-function plainObject(value: any): Record<string, any> {
-	return value && typeof value === 'object' && !Array.isArray(value) ? value : {};
-}
-
-function assistantAnswerQualityMetadata(run: AIRun): Record<string, any> {
-	const gate = (run.gates || []).find((entry) => entry.key === 'assistant_answer_quality');
-	const gateMetadata = plainObject(gate?.metadata);
-	const runMetadata = plainObject(run.metadata);
-	const answerQuality = plainObject(runMetadata.answerQuality || runMetadata.answer_quality);
-	return {
-		...gateMetadata,
-		...answerQuality
-	};
-}
-
-function classifyAssistantFailureClass(run: AIRun): AIRunFailureClass {
-	const metadata = assistantAnswerQualityMetadata(run);
-	const status = normalizedStatus(metadata.status || metadata.answerQualityStatus || metadata.answer_quality_status);
-	const queryStatus = normalizedStatus(metadata.queryStatus || metadata.query_status);
-	const missingEvidenceStatuses = new Set([
-		'missing_data_source',
-		'missing_query_status',
-		'missing_query_proof',
-		'missing_citations',
-		'missing_citation',
-		'low_confidence',
-		'missing_next_action',
-		'no_data_unverified',
-		'blocked'
-	]);
-	if (queryStatus === 'permission_error' || status === 'permission_error') {
-		return 'assistant_permission';
-	}
-	if (queryStatus === 'query_error' || status === 'query_error') {
-		return 'assistant_query_error';
-	}
-	if (status === 'date_incorrect' || status === 'missing_date_window') {
-		return 'assistant_date_window';
-	}
-	if (status === 'illegal_query_shape') {
-		return 'assistant_query_shape';
-	}
-	if (missingEvidenceStatuses.has(status) || !status) {
-		return 'assistant_missing_evidence';
-	}
-	return 'assistant_correctness';
-}
-
-export function classifyAIRunFailureClass(run: AIRun): AIRunFailureClass {
-	if (run.outcome === 'accepted') {
-		return 'none';
-	}
-	if (run.outcome === 'qa_infra_failed') {
-		return 'infra';
-	}
-	if (run.outcome === 'build_failed') {
-		return 'compile';
-	}
-	if (run.outcome === 'release_blocked') {
-		return 'release';
-	}
-	if (run.source === 'aicoder_app' && (run.gates || []).some((gate) => gate.key === 'aicoder_journey_contract' && (gate.status === 'blocked' || gate.status === 'fail'))) {
-		return 'journey_contract';
-	}
-	if (run.source === 'ai_assistant' && run.outcome === 'false_pass') {
-		return classifyAssistantFailureClass(run);
-	}
-	if (run.outcome === 'false_pass') {
-		return 'false_pass';
-	}
-	if (run.outcome === 'manual_handoff') {
-		return 'manual';
-	}
-	if (run.outcome === 'stopped') {
-		return 'stopped';
-	}
-	if (run.source === 'ai_assistant' && (run.outcome === 'rejected' || run.outcome === 'qa_incomplete')) {
-		return classifyAssistantFailureClass(run);
-	}
-	if (run.qa?.outcome === 'compile_failed') {
-		return 'compile';
-	}
-	if (run.qa?.outcome === 'route_failed') {
-		return 'route';
-	}
-	if (run.qa?.outcome === 'route_only_pass' || run.qa?.outcome === 'incomplete') {
-		return 'business';
-	}
-	if (run.qa?.outcome === 'business_assertion_failed') {
-		return 'business';
-	}
-	if (run.outcome === 'qa_incomplete') {
-		return 'business';
-	}
-	if (run.outcome === 'rejected') {
-		return 'business';
-	}
-	return 'unknown';
-}
-
-export function evaluateAIRunFixture(fixture: AIRunEvalFixture): AIRunEvalResult {
-	const failures: string[] = [];
-	const actualFailureClass = classifyAIRunFailureClass(fixture.run);
-	if (fixture.run.outcome !== fixture.expected.outcome) {
-		failures.push(`Expected outcome ${fixture.expected.outcome}, got ${fixture.run.outcome}.`);
-	}
-	if (fixture.expected.failureClass && actualFailureClass !== fixture.expected.failureClass) {
-		failures.push(`Expected failure class ${fixture.expected.failureClass}, got ${actualFailureClass}.`);
-	}
-	if (fixture.expected.nextActionIncludes) {
-		const expectedText = fixture.expected.nextActionIncludes.toLowerCase();
-		const actualText = cleanText(fixture.run.nextAction, 2000).toLowerCase();
-		if (!actualText.includes(expectedText)) {
-			failures.push(`Expected next action to include "${fixture.expected.nextActionIncludes}", got "${fixture.run.nextAction || ''}".`);
-		}
-	}
-	const gatesByKey: Record<string, string> = {};
-	for (const gate of fixture.run.gates || []) {
-		gatesByKey[gate.key] = gate.status;
-	}
-	for (const expectedGate of fixture.expected.gates || []) {
-		if (!gatesByKey[expectedGate.key]) {
-			failures.push(`Expected gate ${expectedGate.key} to be present.`);
-		}
-		else if (expectedGate.status && gatesByKey[expectedGate.key] !== expectedGate.status) {
-			failures.push(`Expected gate ${expectedGate.key} status ${expectedGate.status}, got ${gatesByKey[expectedGate.key]}.`);
-		}
-	}
-	return {
-		id: fixture.id,
-		title: fixture.title,
-		source: fixture.source,
-		passed: failures.length === 0,
-		actualOutcome: fixture.run.outcome,
-		expectedOutcome: fixture.expected.outcome,
-		actualFailureClass,
-		expectedFailureClass: fixture.expected.failureClass,
-		failures,
-		tags: fixture.tags
-	};
-}
-
-export function evaluateAIRunFixtures(fixtures: AIRunEvalFixture[] = AI_RUN_SEED_EVAL_FIXTURES): AIRunEvalSummary {
-	const results = fixtures.map((fixture) => evaluateAIRunFixture(fixture));
-	const failuresByTag: Record<string, number> = {};
-	for (const result of results) {
-		if (result.passed) {
-			continue;
-		}
-		for (const tag of result.tags) {
-			failuresByTag[tag] = (failuresByTag[tag] || 0) + 1;
-		}
-	}
-	const outcomes = countBy(results.map((result) => result.actualOutcome));
-	const passedCount = results.filter((result) => result.passed).length;
-	return {
-		passed: passedCount === results.length,
-		total: results.length,
-		passedCount,
-		failedCount: results.length - passedCount,
-		results,
-		failuresByTag,
-		outcomes
-	};
-}
-
-export function buildAIRunTrainingExampleFromRun(run: AIRun, maxEvidenceItems = 30): AIRunTrainingExample {
-	const evidence: string[] = [];
-	for (const gate of run.gates || []) {
-		addUnique(evidence, `${gate.key}:${gate.status}: ${gate.reason}`, 1600);
-	}
-	for (const event of run.events || []) {
-		addUnique(evidence, `${event.type}:${event.category || ''}: ${event.message || ''}`, 1400);
-		if (evidence.length >= maxEvidenceItems) {
-			break;
-		}
-	}
-	if (evidence.length < maxEvidenceItems) {
-		for (const assertion of run.qa?.businessAssertions || []) {
-			addUnique(evidence, `assertion:${assertion.status}: ${assertion.assertion} observed=${assertion.observed || assertion.dataProof || assertion.message || ''}`, 1600);
-			if (evidence.length >= maxEvidenceItems) {
-				break;
-			}
-		}
-	}
-	return {
-		runId: run.id || run.sourceIds.jobId || run.sourceIds.ticketId || run.sourceIds.appId || run.sourceIds.conversationId || `${run.source}:unknown`,
-		source: run.source,
-		inputSummary: [
-			`source=${run.source}`,
-			`title=${run.title || ''}`,
-			`status=${run.status || ''}`,
-			`phase=${run.phase || ''}`,
-			`ids=${JSON.stringify(run.sourceIds || {})}`
-		].join(' | '),
-		actionSummary: [
-			`outcome=${run.outcome}`,
-			`failureClass=${classifyAIRunFailureClass(run)}`,
-			`nextAction=${run.nextAction || ''}`
-		].join(' | '),
-		outcome: run.outcome,
-		evidence: evidence.slice(0, maxEvidenceItems),
-		cost: run.cost,
-		metadata: {
-			warnings: run.warnings || [],
-			gateCount: (run.gates || []).length,
-			eventCount: (run.events || []).length,
-			qaOutcome: run.qa?.outcome
-		}
-	};
-}
-
-export function buildAIRunDatasetExport(runs: AIRun[], options: AIRunDatasetExportOptions = {}): AIRunDatasetExport {
-	const generatedAt = isoNow(options.now);
-	const trainingExamples = runs.map((run) => {
-		const example = buildAIRunTrainingExampleFromRun(run, options.maxEvidenceItems || 30);
-		return options.redact === false ? example : redactAIRunTrainingExample(example);
-	});
-	const estimatedUsd = runs.reduce((sum, run) => sum + (Number(run.cost?.estimatedUsd) || 0), 0);
-	const totalTokens = runs.reduce((sum, run) => sum + (Number(run.cost?.totalTokens) || 0), 0);
-	return {
-		generatedAt,
-		totalRuns: runs.length,
-		summary: {
-			bySource: countBy(runs.map((run) => run.source)),
-			byOutcome: countBy(runs.map((run) => run.outcome)),
-			byFailureClass: countBy(runs.map((run) => classifyAIRunFailureClass(run))),
-			estimatedUsd: Number(estimatedUsd.toFixed(6)),
-			totalTokens
-		},
-		runs: options.includeRuns === false ? [] : runs,
-		trainingExamples
-	};
-}
-
-export function decideAIRunRetry(input: AIRunRetryPolicyInput): AIRunRetryDecision {
-	const failureClass = classifyAIRunFailureClass(input.currentRun);
-	const previousFailureClass = input.previousRun ? classifyAIRunFailureClass(input.previousRun) : undefined;
-	const sameFailureCount = Number(input.sameFailureCount || 0);
-	const unchangedFailureLimit = Number(input.unchangedFailureLimit || 2);
-	const repeatedFailure = previousFailureClass === failureClass && (Number(input.newEvidenceCount || 0) <= 0 || sameFailureCount >= unchangedFailureLimit);
-	let retryExpectedValue: number | undefined;
-	if (input.historicalAcceptanceRate !== undefined && input.estimatedNextRetryCostUsd !== undefined) {
-		retryExpectedValue = Number((Number(input.historicalAcceptanceRate) - Number(input.estimatedNextRetryCostUsd)).toFixed(6));
-	}
-	if (input.currentRun.outcome === 'accepted') {
-		return {
-			action: 'none',
-			failureClass,
-			repeatedFailure: false,
-			reason: 'Run is already accepted.',
-			nextAction: 'No retry required.',
-			retryExpectedValue
-		};
-	}
-	if (failureClass === 'manual') {
-		return {
-			action: 'manual_handoff',
-			failureClass,
-			repeatedFailure,
-			reason: 'Run is marked for manual handoff.',
-			nextAction: input.currentRun.nextAction || 'Ask an operator to resolve the run.',
-			retryExpectedValue
-		};
-	}
-	if (failureClass === 'stopped') {
-		return {
-			action: 'stop',
-			failureClass,
-			repeatedFailure,
-			reason: 'Run was stopped before accepted evidence was recorded.',
-			nextAction: input.currentRun.nextAction || 'Do not spend another model cycle unless the request is resumed.',
-			retryExpectedValue
-		};
-	}
-	if (repeatedFailure) {
-		return {
-			action: 'stop',
-			failureClass,
-			repeatedFailure,
-			reason: 'The same failure class repeated with no new evidence. Stop model loops until the evidence changes.',
-			nextAction: input.currentRun.nextAction || 'Inspect the failing gate before retrying.',
-			retryExpectedValue
-		};
-	}
-	if (retryExpectedValue !== undefined && retryExpectedValue < Number(input.minRetryExpectedValue || 0)) {
-		return {
-			action: 'stop',
-			failureClass,
-			repeatedFailure,
-			reason: 'Historical retry expected value is below the configured threshold.',
-			nextAction: input.currentRun.nextAction || 'Do not retry without new evidence or a cheaper targeted repair.',
-			retryExpectedValue
-		};
-	}
-	if (failureClass === 'infra') {
-		return {
-			action: 'retry_infra_repair',
-			failureClass,
-			repeatedFailure,
-			reason: 'QA infrastructure failed before business validation.',
-			nextAction: input.currentRun.nextAction || 'Repair browser, settings, Mongo, ports, or startup before model repair.',
-			retryExpectedValue
-		};
-	}
-	if (failureClass === 'compile') {
-		return {
-			action: 'retry_build_repair',
-			failureClass,
-			repeatedFailure,
-			reason: 'Compile/build evidence failed or was stale.',
-			nextAction: input.currentRun.nextAction || 'Repair build before browser QA.',
-			retryExpectedValue
-		};
-	}
-	if (failureClass === 'route') {
-		return {
-			action: 'retry_route_repair',
-			failureClass,
-			repeatedFailure,
-			reason: 'Route probe failed before business proof.',
-			nextAction: input.currentRun.nextAction || 'Repair route/auth/startup and rerun the exact probe.',
-			retryExpectedValue
-		};
-	}
-	if (failureClass === 'journey_contract') {
-		return {
-			action: 'retry_journey_contract',
-			failureClass,
-			repeatedFailure,
-			reason: 'AICoder Journey Contract is missing or invalid before workflow build/proof.',
-			nextAction: input.currentRun.nextAction || 'Generate or repair docs/APP_JOURNEY_CONTRACT.md, validate the north-star workflow, then regenerate workflow QA rows.',
-			retryExpectedValue
-		};
-	}
-	if (failureClass === 'assistant_missing_evidence') {
-		return {
-			action: 'retry_assistant_answer_quality',
-			failureClass,
-			repeatedFailure,
-			reason: 'Assistant answer is missing structured answerQuality, query proof, citations, confidence, or next-action evidence.',
-			nextAction: input.currentRun.nextAction || 'Rebuild the answerQuality record from the exact query/tool evidence before another broad assistant response.',
-			retryExpectedValue
-		};
-	}
-	if (failureClass === 'assistant_query_error') {
-		return {
-			action: 'retry_assistant_query',
-			failureClass,
-			repeatedFailure,
-			reason: 'Assistant data access failed with a query error.',
-			nextAction: input.currentRun.nextAction || 'Repair and rerun the failing query/tool call, then answer only from the new query evidence.',
-			retryExpectedValue
-		};
-	}
-	if (failureClass === 'assistant_permission') {
-		return {
-			action: 'request_assistant_permission',
-			failureClass,
-			repeatedFailure,
-			reason: 'Assistant data access was blocked by permission or authorization.',
-			nextAction: input.currentRun.nextAction || 'Ask for permission, escalate to an authorized user, or answer with the permission blocker instead of claiming no data.',
-			retryExpectedValue
-		};
-	}
-	if (failureClass === 'assistant_date_window') {
-		return {
-			action: 'retry_assistant_date_window',
-			failureClass,
-			repeatedFailure,
-			reason: 'Assistant date-sensitive answer used the wrong current date or omitted the concrete date window.',
-			nextAction: input.currentRun.nextAction || 'Rerun the query using the actual current date and record exact start/end date-window evidence.',
-			retryExpectedValue
-		};
-	}
-	if (failureClass === 'assistant_query_shape') {
-		return {
-			action: 'retry_assistant_query_shape',
-			failureClass,
-			repeatedFailure,
-			reason: 'Assistant query shape is illegal or unsafe for Mongo.',
-			nextAction: input.currentRun.nextAction || 'Rewrite the query with a legal projection or aggregation shape before retrying the data answer.',
-			retryExpectedValue
-		};
-	}
-	if (failureClass === 'release') {
-		return {
-			action: 'retry_release_repair',
-			failureClass,
-			repeatedFailure,
-			reason: 'Business proof exists but deploy, publish, or sample-data gates block release.',
-			nextAction: input.currentRun.nextAction || 'Repair release gates without another broad model planning loop.',
-			retryExpectedValue
-		};
-	}
-	if (failureClass === 'business' || failureClass === 'false_pass' || failureClass === 'assistant_correctness') {
-		return {
-			action: 'retry_business_repair',
-			failureClass,
-			repeatedFailure,
-			reason: 'Business or assistant correctness proof failed or is missing.',
-			nextAction: input.currentRun.nextAction || 'Repair the exact failing workflow/query and rerun the deterministic assertion.',
-			retryExpectedValue
-		};
-	}
-	return {
-		action: 'manual_handoff',
-		failureClass,
-		repeatedFailure,
-		reason: 'Run lacks enough normalized evidence to choose a safe retry lane.',
-		nextAction: input.currentRun.nextAction || 'Ingest more run evidence before retrying.',
-		retryExpectedValue
-	};
-}
-
-const supportAccepted004131 = buildAIRun({
-	id: 'support-004131',
-	source: 'support_ticket',
-	sourceIds: { ticketNumber: '004131', ticketId: 'ticket-004131' },
-	title: '004131 accepted support fix',
-	status: 'Resolved',
-	phase: 'COMPLETE',
-	events: [{ type: 'git_commit', message: 'Fix support workflow and verified before/action/after proof.' }],
-	qa: buildAIQaRun({
-		compile: { status: 'pass', artifactPath: '.build-output/build-server.log' },
-		routeProbes: [{ route: '/support/004131', status: 'pass', screenshot: 'qa-artifacts/004131-after.jpg' }],
-		businessAssertions: [{
-			assertion: 'Before/action/after workflow proves the ticket fix.',
-			status: 'pass',
-			action: 'Run the ticket workflow against seeded data.',
-			expected: 'The corrected state is visible.',
-			observed: 'The corrected state is visible after the action.',
-			artifactPaths: ['qa-artifacts/004131-proof.json']
-		}]
-	}),
-	explicitAccepted: true,
-	terminal: true
-});
-
-const supportBuildFailed004333 = buildAIRun({
-	id: 'support-004333',
-	source: 'support_ticket',
-	sourceIds: { ticketNumber: '004333', ticketId: 'ticket-004333' },
-	title: '004333 compile failure',
-	status: 'Running',
-	phase: 'QA',
-	events: [{ type: 'log', message: 'Angular build failed before browser QA.' }],
-	qa: buildAIQaRun({
-		compile: { status: 'fail', artifactPath: '.build-output/build-client.log', message: 'TypeScript compile failed.' }
-	})
-});
-
-const supportInfraFailed004341 = buildAIRun({
-	id: 'support-004341',
-	source: 'support_ticket',
-	sourceIds: { ticketNumber: '004341', ticketId: 'ticket-004341' },
-	title: '004341 missing Puppeteer',
-	status: 'Running',
-	phase: 'QA',
-	events: [{ type: 'qa_infra', message: 'Puppeteer module could not be loaded.' }],
-	qa: buildAIQaRun({
-		infraChecks: [{ name: 'puppeteer_require', status: 'blocked', message: 'Cannot find module puppeteer.' }]
-	})
-});
-
-const supportFalsePass004421 = buildAIRun({
-	id: 'support-004421',
-	source: 'support_ticket',
-	sourceIds: { ticketNumber: '004421', ticketId: 'ticket-004421' },
-	title: '004421 route-only support QA',
-	status: 'Passed',
-	phase: 'QA',
-	events: [{ type: 'scorecard', message: 'Scorecard passed after route loaded.' }],
-	qa: buildAIQaRun({
-		infraChecks: [{ name: 'chrome_launch', status: 'pass' }],
-		compile: { status: 'pass' },
-		routeProbes: [{ route: '/support/004421', status: 'pass', screenshot: 'qa-artifacts/004421-route.jpg' }]
-	}),
-	scorecardPassed: true
-});
-
-const supportBusinessFailed004423 = buildAIRun({
-	id: 'support-004423',
-	source: 'support_ticket',
-	sourceIds: { ticketNumber: '004423', ticketId: 'ticket-004423' },
-	title: '004423 business assertion failed',
-	status: 'Running',
-	phase: 'QA',
-	events: [{ type: 'qa_business_assertion', message: 'The expected workflow state did not change.' }],
-	qa: buildAIQaRun({
-		compile: { status: 'pass' },
-		routeProbes: [{ route: '/support/004423', status: 'pass' }],
-		businessAssertions: [{
-			assertion: 'Ticket workflow changes the requested field.',
-			status: 'fail',
-			expected: 'Field updates after save.',
-			observed: 'Field stayed unchanged.'
-		}]
-	})
-});
-
-const supportStaleBuild004430 = buildAIRun({
-	id: 'support-004430',
-	source: 'support_ticket',
-	sourceIds: { ticketNumber: '004430', ticketId: 'ticket-004430' },
-	title: '004430 stale build evidence',
-	status: 'Running',
-	phase: 'QA',
-	events: [{ type: 'qa_compile', message: 'Build log predates the final code diff.' }],
-	qa: buildAIQaRun({
-		compile: { status: 'stale', staleEvidence: true, artifactPath: '.build-output/build-client.log' }
-	})
-});
-
-const cryptoPortfolioAccepted = buildAIRun({
-	id: 'aicoder-crypto-v6-golden',
-	source: 'aicoder_app',
-	sourceIds: { appId: 'crypto-v6-golden', jobId: 'crypto-job-golden' },
-	title: 'Crypto Portfolio Tracker V6 Golden',
-	status: 'Passed',
-	phase: 'COMPLETE',
-	events: [{ type: 'scorecard', message: 'Quality scorecard passed with workflow proof.' }],
-	qa: buildAIQaRun({
-		compile: { status: 'pass' },
-		routeProbes: [{ route: '/portfolio', status: 'pass', screenshot: 'qa-artifacts/crypto-dashboard.jpg' }],
-		businessAssertions: [{
-			assertion: 'Portfolio dashboard calculates seeded crypto holdings.',
-			status: 'pass',
-			dataProof: 'Seeded BTC and ETH holdings render totals, allocation, and change.'
-		}]
-	}),
-	scorecardPassed: true,
-	deployStatus: 'pass',
-	publishStatus: 'published',
-	sampleDataStatus: 'pass',
-	terminal: true
-});
-
-const layflatReleaseBlocked = buildAIRun({
-	id: 'aicoder-layflat-pressure-optimizer',
-	source: 'aicoder_app',
-	sourceIds: { appId: 'layflat-pressure-optimizer', jobId: 'layflat-job' },
-	title: 'Layflat Pressure Optimizer',
-	status: 'Passed',
-	phase: 'COMPLETE',
-	events: [{ type: 'scorecard', message: 'Scorecard passed but sample data/publish gate failed.' }],
-	qa: buildAIQaRun({
-		compile: { status: 'pass' },
-		routeProbes: [{ route: '/optimizer', status: 'pass', screenshot: 'qa-artifacts/layflat-route.jpg' }],
-		businessAssertions: [{
-			assertion: 'Pressure optimizer computes a layflat hose result.',
-			status: 'pass',
-			dataProof: 'A seeded hose diameter and flow rate produce PSI loss.'
-		}]
-	}),
-	scorecardPassed: true,
-	deployStatus: 'pass',
-	publishStatus: 'publish failed',
-	sampleDataStatus: 'empty seed collections',
-	terminal: true
-});
-
-const assistantWrongDateWindow = buildAIRun({
-	id: 'assistant-wrong-month-window',
-	source: 'ai_assistant',
-	sourceIds: { conversationId: 'assistant-date-window' },
-	title: 'Assistant wrong month/date-window answer',
-	status: 'Closed',
-	phase: 'ANSWERED',
-	events: [{ type: 'assistant_message', message: 'Answered with the wrong month for today-sensitive invoice data.' }],
-	qa: buildAIQaRun({
-		businessAssertions: [{
-			assertion: 'Assistant date window uses the actual current date.',
-			status: 'fail',
-			expected: 'Use June 2026 for current-month windows.',
-			observed: 'Used a stale month.'
-		}]
-	}),
-	gates: [{
-		key: 'assistant_answer_quality',
-		label: 'Assistant answer quality',
-		status: 'fail',
-		reason: 'Assistant answer did not prove it used the actual current date.',
-		evidenceRefs: [],
-		recordedAt: '2026-06-16T00:00:00.000Z',
-		metadata: {
-			status: 'date_incorrect',
-			queryStatus: 'ok',
-			dateBasis: '2026-05-01',
-			expectedCurrentDate: '2026-06-16'
-		}
-	}],
-	metadata: {
-		answerQuality: {
-			status: 'date_incorrect',
-			queryStatus: 'ok',
-			dateBasis: '2026-05-01',
-			expectedCurrentDate: '2026-06-16'
-		}
-	},
-	rejected: true,
-	terminal: true
-});
-
-const assistantIllegalProjection = buildAIRun({
-	id: 'assistant-illegal-mongo-projection',
-	source: 'ai_assistant',
-	sourceIds: { conversationId: 'assistant-positional-projection' },
-	title: 'Assistant illegal Mongo positional projection',
-	status: 'Closed',
-	phase: 'ANSWERED',
-	events: [{ type: 'assistant_message', message: 'Mongo query used an illegal positional projection shape.' }],
-	qa: buildAIQaRun({
-		businessAssertions: [{
-			assertion: 'Assistant avoids illegal Mongo positional projection.',
-			status: 'fail',
-			expected: 'Use legal aggregation or field projection shape.',
-			observed: 'Used invalid positional projection.'
-		}]
-	}),
-	gates: [{
-		key: 'assistant_answer_quality',
-		label: 'Assistant answer quality',
-		status: 'fail',
-		reason: 'Assistant must use legal Mongo query/projection shapes before answering.',
-		evidenceRefs: [],
-		recordedAt: '2026-06-16T00:00:00.000Z',
-		metadata: {
-			status: 'illegal_query_shape',
-			queryStatus: 'ok',
-			legalQueryShape: false
-		}
-	}],
-	metadata: {
-		answerQuality: {
-			status: 'illegal_query_shape',
-			queryStatus: 'ok',
-			legalQueryShape: false
-		}
-	},
-	rejected: true,
-	terminal: true
-});
-
-const qaMissingPuppeteer = buildAIRun({
-	id: 'qa-missing-puppeteer',
-	source: 'qa_runner',
-	sourceIds: { jobId: 'qa-missing-puppeteer' },
-	title: 'QA runner missing Puppeteer',
-	status: 'Running',
-	phase: 'PREFLIGHT',
-	events: [{ type: 'qa_infra', message: 'Puppeteer failed to require.' }],
-	qa: buildAIQaRun({
-		infraChecks: [{ name: 'puppeteer_require', status: 'blocked', message: 'Cannot find module puppeteer.' }]
-	})
-});
-
-const qaBadChromePath = buildAIRun({
-	id: 'qa-bad-chrome-path',
-	source: 'qa_runner',
-	sourceIds: { jobId: 'qa-bad-chrome-path' },
-	title: 'QA runner bad Chrome path',
-	status: 'Running',
-	phase: 'PREFLIGHT',
-	events: [{ type: 'qa_infra', message: 'Chrome executable path was missing.' }],
-	qa: buildAIQaRun({
-		infraChecks: [{ name: 'chrome_executable', status: 'fail', message: 'Chrome executable not found.', path: '/bad/chrome' }]
-	})
-});
-
-const qaCompileFailure = buildAIRun({
-	id: 'qa-compile-failure',
-	source: 'qa_runner',
-	sourceIds: { jobId: 'qa-compile-failure' },
-	title: 'QA runner compile failure',
-	status: 'Running',
-	phase: 'COMPILE',
-	events: [{ type: 'qa_compile', message: 'Compile failed before browser QA.' }],
-	qa: buildAIQaRun({
-		compile: { status: 'fail', command: 'npm run build', artifactPath: '.build-output/build.log' }
-	})
-});
-
-const qaRouteOnlyPass = buildAIRun({
-	id: 'qa-route-only-pass',
-	source: 'qa_runner',
-	sourceIds: { jobId: 'qa-route-only-pass' },
-	title: 'QA runner route-only pass',
-	status: 'Passed',
-	phase: 'BROWSER_QA',
-	events: [{ type: 'qa_route_probe', message: 'Route loaded but no business proof exists.' }],
-	qa: buildAIQaRun({
-		compile: { status: 'pass' },
-		routeProbes: [{ route: '/dashboard', status: 'pass', screenshot: 'qa-artifacts/dashboard.jpg' }]
-	}),
-	scorecardPassed: true
-});
-
-const qaShellOnlyPage = buildAIRun({
-	id: 'qa-shell-only-page',
-	source: 'qa_runner',
-	sourceIds: { jobId: 'qa-shell-only-page' },
-	title: 'QA runner shell-only page',
-	status: 'Running',
-	phase: 'BROWSER_QA',
-	events: [{ type: 'qa_route_probe', message: 'Shell-only route loaded.' }],
-	qa: buildAIQaRun({
-		compile: { status: 'pass' },
-		routeProbes: [{ route: '/dashboard', status: 'pass', shellOnly: true, screenshot: 'qa-artifacts/shell.jpg' }]
-	})
-});
-
-const qaEmptyData = buildAIRun({
-	id: 'qa-empty-data',
-	source: 'qa_runner',
-	sourceIds: { jobId: 'qa-empty-data' },
-	title: 'QA runner empty data',
-	status: 'Failed',
-	phase: 'BROWSER_QA',
-	events: [{ type: 'qa_business_assertion', message: 'Workflow rendered empty data.' }],
-	qa: buildAIQaRun({
-		compile: { status: 'pass' },
-		routeProbes: [{ route: '/invoices', status: 'pass' }],
-		businessAssertions: [{
-			assertion: 'Invoice dashboard renders non-empty meaningful data.',
-			status: 'fail',
-			expected: 'At least one meaningful invoice row.',
-			observed: 'Empty table.'
-		}]
-	}),
-	terminal: true
-});
-
-const qaPublishFailureAfterScorecard = buildAIRun({
-	id: 'qa-scorecard-pass-publish-failure',
-	source: 'qa_runner',
-	sourceIds: { jobId: 'qa-publish-failure' },
-	title: 'QA scorecard pass followed by publish failure',
-	status: 'Passed',
-	phase: 'RELEASE',
-	events: [{ type: 'publish', message: 'Scorecard passed but publish failed.' }],
-	qa: buildAIQaRun({
-		compile: { status: 'pass' },
-		routeProbes: [{ route: '/app', status: 'pass' }],
-		businessAssertions: [{
-			assertion: 'Primary app workflow runs.',
-			status: 'pass',
-			dataProof: 'Workflow action returned expected state.'
-		}]
-	}),
-	scorecardPassed: true,
-	publishStatus: 'publish failed',
-	sampleDataStatus: 'pass'
-});
-
-export const AI_RUN_SEED_EVAL_FIXTURES: AIRunEvalFixture[] = [
-	{
-		id: 'support-004131',
-		title: 'Support 004131 accepted proof',
-		source: 'support_ticket',
-		run: supportAccepted004131,
-		expected: { outcome: 'accepted', failureClass: 'none', gates: [{ key: 'qa_business_assertion', status: 'pass' }] },
-		tags: ['support', 'accepted', 'business_assertion']
-	},
-	{
-		id: 'support-004333',
-		title: 'Support 004333 compile failure',
-		source: 'support_ticket',
-		run: supportBuildFailed004333,
-		expected: { outcome: 'build_failed', failureClass: 'compile', nextActionIncludes: 'compile' },
-		tags: ['support', 'compile']
-	},
-	{
-		id: 'support-004341',
-		title: 'Support 004341 Puppeteer infra failure',
-		source: 'support_ticket',
-		run: supportInfraFailed004341,
-		expected: { outcome: 'qa_infra_failed', failureClass: 'infra', nextActionIncludes: 'browser' },
-		tags: ['support', 'infra', 'puppeteer']
-	},
-	{
-		id: 'support-004421',
-		title: 'Support 004421 route-only false pass',
-		source: 'support_ticket',
-		run: supportFalsePass004421,
-		expected: { outcome: 'false_pass', failureClass: 'false_pass', gates: [{ key: 'qa_business_assertion', status: 'blocked' }] },
-		tags: ['support', 'route_only', 'false_pass']
-	},
-	{
-		id: 'support-004423',
-		title: 'Support 004423 business assertion failed',
-		source: 'support_ticket',
-		run: supportBusinessFailed004423,
-		expected: { outcome: 'qa_incomplete', failureClass: 'business', nextActionIncludes: 'assertion' },
-		tags: ['support', 'business_assertion']
-	},
-	{
-		id: 'support-004430',
-		title: 'Support 004430 stale build evidence',
-		source: 'support_ticket',
-		run: supportStaleBuild004430,
-		expected: { outcome: 'build_failed', failureClass: 'compile', nextActionIncludes: 'build' },
-		tags: ['support', 'compile', 'stale_build']
-	},
-	{
-		id: 'aicoder-crypto-v6-golden',
-		title: 'AICoder Crypto Portfolio Tracker V6 Golden',
-		source: 'aicoder_app',
-		run: cryptoPortfolioAccepted,
-		expected: { outcome: 'accepted', failureClass: 'none', gates: [{ key: 'qa_business_assertion', status: 'pass' }] },
-		tags: ['aicoder', 'crypto', 'accepted']
-	},
-	{
-		id: 'aicoder-layflat-pressure-optimizer',
-		title: 'AICoder Layflat Pressure Optimizer release blocked',
-		source: 'aicoder_app',
-		run: layflatReleaseBlocked,
-		expected: { outcome: 'release_blocked', failureClass: 'release', gates: [{ key: 'release', status: 'fail' }] },
-		tags: ['aicoder', 'layflat', 'release_blocked', 'sample_data']
-	},
-	{
-		id: 'assistant-wrong-month-window',
-		title: 'Assistant wrong month/date-window answer',
-		source: 'ai_assistant',
-		run: assistantWrongDateWindow,
-		expected: { outcome: 'rejected', failureClass: 'assistant_date_window' },
-		tags: ['assistant', 'date_window']
-	},
-	{
-		id: 'assistant-illegal-mongo-projection',
-		title: 'Assistant illegal Mongo positional projection',
-		source: 'ai_assistant',
-		run: assistantIllegalProjection,
-		expected: { outcome: 'rejected', failureClass: 'assistant_query_shape' },
-		tags: ['assistant', 'mongo_query']
-	},
-	{
-		id: 'qa-missing-puppeteer',
-		title: 'QA missing Puppeteer',
-		source: 'qa_runner',
-		run: qaMissingPuppeteer,
-		expected: { outcome: 'qa_infra_failed', failureClass: 'infra' },
-		tags: ['qa_runner', 'infra', 'puppeteer']
-	},
-	{
-		id: 'qa-bad-chrome-path',
-		title: 'QA bad Chrome path',
-		source: 'qa_runner',
-		run: qaBadChromePath,
-		expected: { outcome: 'qa_infra_failed', failureClass: 'infra' },
-		tags: ['qa_runner', 'infra', 'chrome']
-	},
-	{
-		id: 'qa-compile-failure',
-		title: 'QA compile failure',
-		source: 'qa_runner',
-		run: qaCompileFailure,
-		expected: { outcome: 'build_failed', failureClass: 'compile' },
-		tags: ['qa_runner', 'compile']
-	},
-	{
-		id: 'qa-route-only-pass',
-		title: 'QA route-only pass',
-		source: 'qa_runner',
-		run: qaRouteOnlyPass,
-		expected: { outcome: 'false_pass', failureClass: 'false_pass' },
-		tags: ['qa_runner', 'route_only', 'false_pass']
-	},
-	{
-		id: 'qa-shell-only-page',
-		title: 'QA shell-only page',
-		source: 'qa_runner',
-		run: qaShellOnlyPage,
-		expected: { outcome: 'qa_incomplete', failureClass: 'route' },
-		tags: ['qa_runner', 'route', 'shell_only']
-	},
-	{
-		id: 'qa-empty-data',
-		title: 'QA empty data',
-		source: 'qa_runner',
-		run: qaEmptyData,
-		expected: { outcome: 'rejected', failureClass: 'business' },
-		tags: ['qa_runner', 'empty_data', 'business_assertion']
-	},
-	{
-		id: 'qa-scorecard-pass-publish-failure',
-		title: 'QA scorecard pass followed by publish failure',
-		source: 'qa_runner',
-		run: qaPublishFailureAfterScorecard,
-		expected: { outcome: 'release_blocked', failureClass: 'release' },
-		tags: ['qa_runner', 'release_blocked', 'publish']
-	}
-];