@resolveio/server-lib 22.3.220 → 22.3.221

package/tests/ai-assistant-eval-triage.ts

@@ -0,0 +1,831 @@
+import * as fs from 'fs';
+import * as path from 'path';
+
+type OpenAiE2EResult = {
+	id: string;
+	prompt: string;
+	pass: boolean;
+	error?: string;
+	userProfile?: string;
+};
+
+type OpenAiE2EOutput = {
+	now?: string;
+	total?: number;
+	passed?: number;
+	failed?: number;
+	results?: OpenAiE2EResult[];
+};
+
+type SnfLiveEvalResult = {
+	id: string;
+	prompt: string;
+	pass: boolean;
+	rowCount?: number;
+	minRows?: number;
+	error?: string;
+	classification?: {
+		pass?: boolean;
+		type?: string;
+		dataQuestion?: boolean;
+		source?: string;
+	};
+};
+
+type SnfLiveEvalOutput = {
+	now?: string;
+	results?: SnfLiveEvalResult[];
+	summary?: {
+		total?: number;
+		passed?: number;
+		failed?: number;
+	};
+};
+
+type DataParityEvalResult = {
+	id: string;
+	prompt: string;
+	pass: boolean;
+	errors?: string[];
+	directive?: {
+		type?: string;
+		collection?: string;
+		source?: string;
+	};
+	tool?: {
+		type?: string;
+		collection?: string;
+		rowCount?: number;
+		total?: number | null;
+		columns?: string[];
+	};
+};
+
+type DataParityEvalOutput = {
+	now?: string;
+	total?: number;
+	passed?: number;
+	failed?: number;
+	results?: DataParityEvalResult[];
+};
+
+type CorpusReplayEvalResult = {
+	id: string;
+	prompt: string;
+	pass: boolean;
+	errors?: string[];
+	tool?: {
+		type?: string;
+		collection?: string;
+		rowCount?: number;
+		total?: number | null;
+		columns?: string[];
+		source?: string;
+	};
+};
+
+type CorpusReplayEvalOutput = {
+	now?: string;
+	total?: number;
+	passed?: number;
+	failed?: number;
+	results?: CorpusReplayEvalResult[];
+};
+
+type FailureBucket = {
+	name: string;
+	count: number;
+	caseIds: string[];
+};
+
+type FailureEntry = {
+	id: string;
+	prompt: string;
+	error: string;
+	category: string;
+	suite: 'openai_e2e' | 'snf_live' | 'data_parity' | 'corpus_replay';
+	intent: string;
+	collectionFamily: string;
+};
+
+type PriorityCluster = {
+	name: string;
+	count: number;
+	score: number;
+	category: string;
+	intent: string;
+	suites: string[];
+	collectionFamilies: string[];
+	caseIds: string[];
+	examplePrompts: string[];
+};
+
+function readJsonFile<T>(filePath: string): T | null {
+	if (!fs.existsSync(filePath)) {
+		return null;
+	}
+	try {
+		const raw = fs.readFileSync(filePath, 'utf8');
+		return JSON.parse(raw) as T;
+	}
+	catch (error: any) {
+		throw new Error(`Failed to parse JSON file ${filePath}: ${String(error?.message || error || 'unknown error')}`);
+	}
+}
+
+function categorizeOpenAiFailure(errorText: string): string {
+	const text = String(errorText || '').toLowerCase();
+	if (!text) {
+		return 'unknown';
+	}
+	if (text.includes('timed out waiting for assistant completion')) {
+		return 'timeout';
+	}
+	if (text.includes('expected markdown table')) {
+		return 'missing_table';
+	}
+	if (text.includes('expected data/mixed classification') || text.includes('expected dataquestion=true')) {
+		return 'classification';
+	}
+	if (text.includes('expected first column to be metric')) {
+		return 'metric_grid_shape';
+	}
+	if (text.includes('expected first column to include customer')) {
+		return 'customer_grid_shape';
+	}
+	if (text.includes('expected day/date header')
+		|| text.includes('expected week header')
+		|| text.includes('expected month header')
+		|| text.includes('expected quarter header')
+		|| text.includes('expected year header')) {
+		return 'time_grain_header';
+	}
+	if (text.includes('expected fast_path=')
+		|| text.includes('fast_path_change_generic')) {
+		return 'fast_path';
+	}
+	if (text.includes('expected response to include route')
+		|| text.includes('expected response to exclude route')) {
+		return 'routing_permission';
+	}
+	return 'other';
+}
+
+function categorizeSnfFailure(result: SnfLiveEvalResult): string {
+	if (!result) {
+		return 'unknown';
+	}
+	if (result.error) {
+		return 'query_error';
+	}
+	if (result.classification?.pass === false) {
+		return 'classification';
+	}
+	const rowCount = Number(result.rowCount || 0);
+	const minRows = Number(result.minRows || 0);
+	if (rowCount < minRows) {
+		return 'row_count_below_expectation';
+	}
+	if (result.pass === false) {
+		return 'other';
+	}
+	return 'pass';
+}
+
+function categorizeParityFailure(errorText: string): string {
+	const text = String(errorText || '').toLowerCase();
+	if (!text) {
+		return 'unknown';
+	}
+	if (text.includes('collection is not configured for report builder')
+		|| text.includes('no tool_result found')) {
+		return 'collection_or_directive';
+	}
+	if (text.includes('truncated')) {
+		return 'display_truncation';
+	}
+	if (text.includes('could not parse')
+		|| text.includes('empty markdown table')) {
+		return 'display_shape';
+	}
+	if (text.includes('rowcount mismatch')
+		|| text.includes('replay display mismatch')
+		|| text.includes('replay row length mismatch')) {
+		return 'replay_mismatch';
+	}
+	if (text.includes('mismatch for')
+		|| text.includes('missing key')
+		|| text.includes('unexpected key')
+		|| text.includes('expected multiple named customers')) {
+		return 'data_mismatch';
+	}
+	return 'other';
+}
+
+function categorizeCorpusFailure(errorText: string): string {
+	const text = String(errorText || '').toLowerCase();
+	if (!text) {
+		return 'unknown';
+	}
+	if (text.includes('internal error response')) {
+		return 'assistant_runtime';
+	}
+	if (text.includes('no tool_result found')
+		|| text.includes('did not produce a usable tool_result')
+		|| text.includes('directive')) {
+		return 'directive_or_tool_result';
+	}
+	if (text.includes('replay rowcount mismatch')
+		|| text.includes('replay columns mismatch')
+		|| text.includes('replay display mismatch')) {
+		return 'replay_mismatch';
+	}
+	if (text.includes('timed out waiting for final assistant message')) {
+		return 'timeout';
+	}
+	return 'other';
+}
+
+function normalizePromptText(value: string): string {
+	return String(value || '').toLowerCase().replace(/\s+/g, ' ').trim();
+}
+
+function categorizePromptIntent(promptText: string): string {
+	const prompt = normalizePromptText(promptText);
+	if (!prompt) {
+		return 'unknown';
+	}
+	if (/when was the last time|what changed|last update|system update|updated/i.test(prompt)) {
+		return 'change_history';
+	}
+	if (/greyed out|grayed out|dropdown|submit button|won't|cant |can't |not working|error|bug/i.test(prompt)) {
+		return 'ui_bug_triage';
+	}
+	if (/where do i go|route|which page|which screen|how do i get to|navigate/i.test(prompt)) {
+		return 'navigation';
+	}
+	if (/\btop\s+\d+\b/i.test(prompt)) {
+		return 'ranking_top_n';
+	}
+	if (/grouped by|\bby each\b|\bby\b|\bper\b|break down|break up/i.test(prompt)) {
+		return 'grouping_breakdown';
+	}
+	if (/how many|count|right now|snapshot/i.test(prompt)) {
+		return 'count_snapshot';
+	}
+	if (/last|this|today|yesterday|month|week|day|quarter|year|date range/i.test(prompt)) {
+		return 'time_window';
+	}
+	return 'other';
+}
+
+function categorizeCollectionFamily(value: any): string {
+	const collection = String(value || '').trim().toLowerCase();
+	if (!collection) {
+		return 'unknown';
+	}
+	if (collection.includes('invoice') || collection.includes('billing') || collection.includes('sales-tax')) {
+		return 'invoice';
+	}
+	if (collection.includes('support') || collection.includes('ticket')) {
+		return 'support_ticket';
+	}
+	if (collection.includes('blend') || collection.includes('chemical-blend')) {
+		return 'blend';
+	}
+	if (collection.includes('deliver') || collection.includes('driver')) {
+		return 'delivery';
+	}
+	if (collection.includes('work-order') || collection.includes('order') || collection.includes('job')) {
+		return 'work_order';
+	}
+	if (collection.includes('customer') || collection.includes('client')) {
+		return 'customer_client';
+	}
+	return collection;
+}
+
+function bucketFailures(entries: FailureEntry[], keyResolver: (entry: FailureEntry) => string): FailureBucket[] {
+	const map = new Map<string, Set<string>>();
+	entries.forEach((entry) => {
+		const key = keyResolver(entry);
+		if (!entry || !key) {
+			return;
+		}
+		if (!map.has(key)) {
+			map.set(key, new Set<string>());
+		}
+		map.get(key)?.add(entry.id);
+	});
+	return Array.from(map.entries())
+		.map(([name, caseIds]) => ({
+			name,
+			count: caseIds.size,
+			caseIds: Array.from(caseIds).sort()
+		}))
+		.sort((a, b) => {
+			if (a.count !== b.count) {
+				return b.count - a.count;
+			}
+			return a.name.localeCompare(b.name);
+		});
+}
+
+function buildPriorityClusters(entries: FailureEntry[]): PriorityCluster[] {
+	if (!entries.length) {
+		return [];
+	}
+	const categoryWeight: Record<string, number> = {
+		assistant_runtime: 9,
+		directive_or_tool_result: 8,
+		collection_or_directive: 8,
+		replay_mismatch: 7,
+		data_mismatch: 6,
+		query_error: 6,
+		timeout: 6,
+		classification: 5,
+		display_shape: 4,
+		display_truncation: 4,
+		row_count_below_expectation: 4,
+		other: 2,
+		unknown: 1
+	};
+	const intentWeight: Record<string, number> = {
+		grouping_breakdown: 6,
+		ranking_top_n: 5,
+		count_snapshot: 4,
+		time_window: 4,
+		navigation: 3,
+		change_history: 3,
+		ui_bug_triage: 3,
+		other: 2,
+		unknown: 1
+	};
+	const clusters = new Map<string, {
+		category: string;
+		intent: string;
+		caseIds: Set<string>;
+		suites: Set<string>;
+		collectionFamilies: Set<string>;
+		prompts: Set<string>;
+	}>();
+	entries.forEach((entry) => {
+		const key = `${entry.category}__${entry.intent}`;
+		if (!clusters.has(key)) {
+			clusters.set(key, {
+				category: entry.category,
+				intent: entry.intent,
+				caseIds: new Set<string>(),
+				suites: new Set<string>(),
+				collectionFamilies: new Set<string>(),
+				prompts: new Set<string>()
+			});
+		}
+		const cluster = clusters.get(key);
+		cluster?.caseIds.add(entry.id);
+		cluster?.suites.add(entry.suite);
+		cluster?.collectionFamilies.add(entry.collectionFamily || 'unknown');
+		if (entry.prompt) {
+			cluster?.prompts.add(entry.prompt);
+		}
+	});
+	return Array.from(clusters.values())
+		.map((cluster) => {
+			const count = cluster.caseIds.size;
+			const suiteSpread = Math.max(cluster.suites.size - 1, 0);
+			const collectionSpread = Math.max(cluster.collectionFamilies.size - 1, 0);
+			const score = (
+				(categoryWeight[cluster.category] || categoryWeight.other) * count
+				+ (intentWeight[cluster.intent] || intentWeight.other) * count
+				+ suiteSpread * 3
+				+ collectionSpread
+			);
+			return {
+				name: `${cluster.category} + ${cluster.intent}`,
+				count,
+				score,
+				category: cluster.category,
+				intent: cluster.intent,
+				suites: Array.from(cluster.suites).sort(),
+				collectionFamilies: Array.from(cluster.collectionFamilies).sort(),
+				caseIds: Array.from(cluster.caseIds).sort(),
+				examplePrompts: Array.from(cluster.prompts).slice(0, 3)
+			};
+		})
+		.sort((a, b) => {
+			if (a.score !== b.score) {
+				return b.score - a.score;
+			}
+			if (a.count !== b.count) {
+				return b.count - a.count;
+			}
+			return a.name.localeCompare(b.name);
+		});
+}
+
+function toMarkdownPriorityClusters(clusters: PriorityCluster[]): string {
+	if (!clusters.length) {
+		return '_None_';
+	}
+	const lines = [
+		'| Priority Cluster | Score | Cases | Suites | Collections | Case IDs |',
+		'| --- | --- | --- | --- | --- | --- |'
+	];
+	clusters.forEach((cluster) => {
+		lines.push(
+			`| ${cluster.name} | ${cluster.score} | ${cluster.count} | ${cluster.suites.join(', ')} | ${cluster.collectionFamilies.join(', ')} | ${cluster.caseIds.join(', ')} |`
+		);
+	});
+	return lines.join('\n');
+}
+
+function toMarkdownBuckets(buckets: FailureBucket[]): string {
+	if (!buckets.length) {
+		return '_None_';
+	}
+	const lines = [
+		'| Bucket | Cases | Case IDs |',
+		'| --- | --- | --- |'
+	];
+	buckets.forEach((bucket) => {
+		lines.push(`| ${bucket.name} | ${bucket.count} | ${bucket.caseIds.join(', ')} |`);
+	});
+	return lines.join('\n');
+}
+
+function collectOpenAiFailures(data: OpenAiE2EOutput | null): FailureEntry[] {
+	const results = Array.isArray(data?.results) ? data.results : [];
+	return results
+		.filter(result => result && result.pass === false)
+		.map((result) => {
+			const prompt = String(result.prompt || '');
+			const error = String(result.error || 'Unknown failure');
+			return {
+				id: String(result.id || ''),
+				prompt,
+				error,
+				category: categorizeOpenAiFailure(error),
+				suite: 'openai_e2e',
+				intent: categorizePromptIntent(prompt),
+				collectionFamily: 'unknown'
+			};
+		});
+}
+
+function collectSnfFailures(data: SnfLiveEvalOutput | null): FailureEntry[] {
+	const results = Array.isArray(data?.results) ? data.results : [];
+	return results
+		.filter(result => result && result.pass === false)
+		.map((result) => {
+			const prompt = String(result.prompt || '');
+			const error = String(result.error || `rowCount=${result.rowCount || 0}, minRows=${result.minRows || 0}`);
+			return {
+				id: String(result.id || ''),
+				prompt,
+				error,
+				category: categorizeSnfFailure(result),
+				suite: 'snf_live',
+				intent: categorizePromptIntent(prompt),
+				collectionFamily: 'unknown'
+			};
+		});
+}
+
+function resolveParityCollection(result: DataParityEvalResult): string {
+	return String(
+		result?.tool?.collection
+		|| result?.directive?.collection
+		|| ''
+	).trim();
+}
+
+function resolveCorpusCollection(result: CorpusReplayEvalResult): string {
+	return String(result?.tool?.collection || '').trim();
+}
+
+function collectParityFailures(data: DataParityEvalOutput | null): FailureEntry[] {
+	const results = Array.isArray(data?.results) ? data.results : [];
+	return results
+		.filter(result => result && result.pass === false)
+		.map((result) => {
+			const prompt = String(result.prompt || '');
+			const errorList = Array.isArray(result.errors) ? result.errors : [];
+			const error = errorList.length ? errorList[0] : 'Unknown failure';
+			const collectionFamily = categorizeCollectionFamily(resolveParityCollection(result));
+			return {
+				id: String(result.id || ''),
+				prompt,
+				error,
+				category: categorizeParityFailure(error),
+				suite: 'data_parity',
+				intent: categorizePromptIntent(prompt),
+				collectionFamily
+			};
+		});
+}
+
+function collectCorpusFailures(data: CorpusReplayEvalOutput | null): FailureEntry[] {
+	const results = Array.isArray(data?.results) ? data.results : [];
+	return results
+		.filter(result => result && result.pass === false)
+		.map((result) => {
+			const prompt = String(result.prompt || '');
+			const errorList = Array.isArray(result.errors) ? result.errors : [];
+			const error = errorList.length ? errorList[0] : 'Unknown failure';
+			const collectionFamily = categorizeCollectionFamily(resolveCorpusCollection(result));
+			return {
+				id: String(result.id || ''),
+				prompt,
+				error,
+				category: categorizeCorpusFailure(error),
+				suite: 'corpus_replay',
+				intent: categorizePromptIntent(prompt),
+				collectionFamily
+			};
+		});
+}
+
+function resolveEvalOutputPath(repoRoot: string, envValue: string | undefined, fallbackRelativePath: string): string {
+	const value = String(envValue || '').trim();
+	if (!value) {
+		return path.join(repoRoot, fallbackRelativePath);
+	}
+	return path.isAbsolute(value) ? value : path.join(repoRoot, value);
+}
+
+function run(): void {
+	const repoRoot = process.cwd();
+	const tmpDir = path.join(repoRoot, 'tmp');
+	const openAiPath = resolveEvalOutputPath(
+		repoRoot,
+		process.env.AI_ASSISTANT_EVAL_OPENAI_OUTPUT,
+		path.join('tmp', 'ai-assistant-openai-e2e-output.json')
+	);
+	const snfLivePath = resolveEvalOutputPath(
+		repoRoot,
+		process.env.AI_ASSISTANT_EVAL_SNF_OUTPUT,
+		path.join('tmp', 'ai-assistant-snf-live-eval-output.json')
+	);
+	const parityPath = resolveEvalOutputPath(
+		repoRoot,
+		process.env.AI_ASSISTANT_EVAL_PARITY_OUTPUT,
+		path.join('tmp', 'ai-assistant-data-parity-e2e-output.json')
+	);
+	const corpusPath = resolveEvalOutputPath(
+		repoRoot,
+		process.env.AI_ASSISTANT_EVAL_CORPUS_OUTPUT,
+		path.join('tmp', 'ai-assistant-corpus-replay-e2e-output.json')
+	);
+	const triageJsonPath = path.join(tmpDir, 'ai-assistant-eval-triage.json');
+	const triageMdPath = path.join(tmpDir, 'ai-assistant-eval-triage.md');
+
+	const openAiOutput = readJsonFile<OpenAiE2EOutput>(openAiPath);
+	const snfLiveOutput = readJsonFile<SnfLiveEvalOutput>(snfLivePath);
+	const parityOutput = readJsonFile<DataParityEvalOutput>(parityPath);
+	const corpusOutput = readJsonFile<CorpusReplayEvalOutput>(corpusPath);
+
+	if (!openAiOutput && !snfLiveOutput && !parityOutput && !corpusOutput) {
+		throw new Error(`No eval outputs found. Expected at least one of:\n- ${openAiPath}\n- ${snfLivePath}\n- ${parityPath}\n- ${corpusPath}`);
+	}
+
+	const openAiFailures = collectOpenAiFailures(openAiOutput);
+	const snfFailures = collectSnfFailures(snfLiveOutput);
+	const parityFailures = collectParityFailures(parityOutput);
+	const corpusFailures = collectCorpusFailures(corpusOutput);
+	const openAiBuckets = bucketFailures(openAiFailures, entry => entry.category);
+	const snfBuckets = bucketFailures(snfFailures, entry => entry.category);
+	const parityBuckets = bucketFailures(parityFailures, entry => entry.category);
+	const corpusBuckets = bucketFailures(corpusFailures, entry => entry.category);
+	const allFailures = [
+		...openAiFailures,
+		...snfFailures,
+		...parityFailures,
+		...corpusFailures
+	];
+	const intentBuckets = bucketFailures(allFailures, entry => entry.intent);
+	const collectionBuckets = bucketFailures(allFailures, entry => entry.collectionFamily);
+	const priorityClusters = buildPriorityClusters(allFailures);
+
+	const openAiFailIds = openAiFailures.map(entry => entry.id).filter(Boolean).sort();
+	const snfFailIds = snfFailures.map(entry => entry.id).filter(Boolean).sort();
+	const parityFailIds = parityFailures.map(entry => entry.id).filter(Boolean).sort();
+	const corpusFailIds = corpusFailures.map(entry => entry.id).filter(Boolean).sort();
+	const suiteFailSets = [openAiFailIds, snfFailIds, parityFailIds, corpusFailIds].filter(ids => ids.length);
+	const overlapCounter = new Map<string, number>();
+	suiteFailSets.forEach((ids) => {
+		new Set(ids).forEach((id) => {
+			overlapCounter.set(id, (overlapCounter.get(id) || 0) + 1);
+		});
+	});
+	const overlapFailIds = Array.from(overlapCounter.entries())
+		.filter(([, count]) => count >= 2)
+		.map(([id]) => id)
+		.sort();
+
+	const triage = {
+		generatedAt: new Date().toISOString(),
+		files: {
+			openAiPath,
+			snfLivePath,
+			parityPath,
+			corpusPath
+		},
+		openAi: openAiOutput ? {
+			total: Number(openAiOutput.total || (openAiOutput.results || []).length || 0),
+			passed: Number(openAiOutput.passed || 0),
+			failed: Number(openAiOutput.failed || openAiFailures.length || 0),
+			failureBuckets: openAiBuckets,
+			failingCaseIds: openAiFailIds
+		} : null,
+		snfLive: snfLiveOutput ? {
+			total: Number(
+				snfLiveOutput.summary?.total
+				|| (snfLiveOutput.results || []).length
+				|| 0
+			),
+			passed: Number(snfLiveOutput.summary?.passed || 0),
+			failed: Number(snfLiveOutput.summary?.failed || snfFailures.length || 0),
+			failureBuckets: snfBuckets,
+			failingCaseIds: snfFailIds
+		} : null,
+		parity: parityOutput ? {
+			total: Number(parityOutput.total || (parityOutput.results || []).length || 0),
+			passed: Number(parityOutput.passed || 0),
+			failed: Number(parityOutput.failed || parityFailures.length || 0),
+			failureBuckets: parityBuckets,
+			failingCaseIds: parityFailIds
+		} : null,
+		corpusReplay: corpusOutput ? {
+			total: Number(corpusOutput.total || (corpusOutput.results || []).length || 0),
+			passed: Number(corpusOutput.passed || 0),
+			failed: Number(corpusOutput.failed || corpusFailures.length || 0),
+			failureBuckets: corpusBuckets,
+			failingCaseIds: corpusFailIds
+		} : null,
+		overlapFailIds,
+		allFailures: {
+			total: allFailures.length,
+			intentBuckets,
+			collectionBuckets,
+			priorityClusters: priorityClusters.slice(0, 20)
+		}
+	};
+
+	fs.mkdirSync(tmpDir, { recursive: true });
+	fs.writeFileSync(triageJsonPath, JSON.stringify(triage, null, 2));
+
+	const markdownSections: string[] = [];
+	markdownSections.push('# AI Assistant Eval Triage');
+	markdownSections.push('');
+	markdownSections.push(`Generated: ${triage.generatedAt}`);
+	markdownSections.push('');
+
+	if (openAiOutput) {
+		markdownSections.push('## OpenAI E2E');
+		markdownSections.push('');
+		markdownSections.push(`- Total: ${triage.openAi?.total || 0}`);
+		markdownSections.push(`- Passed: ${triage.openAi?.passed || 0}`);
+		markdownSections.push(`- Failed: ${triage.openAi?.failed || 0}`);
+		markdownSections.push('');
+		markdownSections.push(toMarkdownBuckets(openAiBuckets));
+		markdownSections.push('');
+		markdownSections.push('Rerun failed E2E cases:');
+		if (openAiFailIds.length) {
+			markdownSections.push('');
+			markdownSections.push('```bash');
+			markdownSections.push(`AI_ASSISTANT_E2E_CASES="${openAiFailIds.join(',')}" npm run ai-assistant-openai-e2e`);
+			markdownSections.push('```');
+		}
+		else {
+			markdownSections.push('');
+			markdownSections.push('_No E2E failures detected._');
+		}
+		markdownSections.push('');
+	}
+
+	if (snfLiveOutput) {
+		markdownSections.push('## SNF Live Eval');
+		markdownSections.push('');
+		markdownSections.push(`- Total: ${triage.snfLive?.total || 0}`);
+		markdownSections.push(`- Passed: ${triage.snfLive?.passed || 0}`);
+		markdownSections.push(`- Failed: ${triage.snfLive?.failed || 0}`);
+		markdownSections.push('');
+		markdownSections.push(toMarkdownBuckets(snfBuckets));
+		markdownSections.push('');
+		markdownSections.push('Rerun failed SNF live cases:');
+		if (snfFailIds.length) {
+			markdownSections.push('');
+			markdownSections.push('```bash');
+			markdownSections.push(`AI_ASSISTANT_SNF_EVAL_CASES="${snfFailIds.join(',')}" npm run ai-assistant-snf-live-eval`);
+			markdownSections.push('```');
+		}
+		else {
+			markdownSections.push('');
+			markdownSections.push('_No SNF live failures detected._');
+		}
+		markdownSections.push('');
+	}
+
+	if (parityOutput) {
+		markdownSections.push('## Data Parity E2E');
+		markdownSections.push('');
+		markdownSections.push(`- Total: ${triage.parity?.total || 0}`);
+		markdownSections.push(`- Passed: ${triage.parity?.passed || 0}`);
+		markdownSections.push(`- Failed: ${triage.parity?.failed || 0}`);
+		markdownSections.push('');
+		markdownSections.push(toMarkdownBuckets(parityBuckets));
+		markdownSections.push('');
+		markdownSections.push('Rerun failed data parity cases:');
+		if (parityFailIds.length) {
+			markdownSections.push('');
+			markdownSections.push('```bash');
+			markdownSections.push(`AI_ASSISTANT_PARITY_CASES="${parityFailIds.join(',')}" npm run ai-assistant-data-parity-e2e`);
+			markdownSections.push('```');
+		}
+		else {
+			markdownSections.push('');
+			markdownSections.push('_No data parity failures detected._');
+		}
+		markdownSections.push('');
+	}
+
+	if (corpusOutput) {
+		markdownSections.push('## Corpus Replay E2E');
+		markdownSections.push('');
+		markdownSections.push(`- Total: ${triage.corpusReplay?.total || 0}`);
+		markdownSections.push(`- Passed: ${triage.corpusReplay?.passed || 0}`);
+		markdownSections.push(`- Failed: ${triage.corpusReplay?.failed || 0}`);
+		markdownSections.push('');
+		markdownSections.push(toMarkdownBuckets(corpusBuckets));
+		markdownSections.push('');
+		markdownSections.push('Rerun failed corpus replay cases:');
+		if (corpusFailIds.length) {
+			markdownSections.push('');
+			markdownSections.push('```bash');
+			markdownSections.push(`AI_ASSISTANT_CORPUS_CASES="${corpusFailIds.join(',')}" npm run ai-assistant-corpus-replay-e2e`);
+			markdownSections.push('```');
+		}
+		else {
+			markdownSections.push('');
+			markdownSections.push('_No corpus replay failures detected._');
+		}
+		markdownSections.push('');
+	}
+
+	markdownSections.push('## Prioritized Fix Clusters');
+	markdownSections.push('');
+	markdownSections.push(`Total failures across suites: ${allFailures.length}`);
+	markdownSections.push('');
+	markdownSections.push(toMarkdownPriorityClusters(priorityClusters.slice(0, 10)));
+	markdownSections.push('');
+
+	markdownSections.push('## Failure Intent Buckets');
+	markdownSections.push('');
+	markdownSections.push(toMarkdownBuckets(intentBuckets));
+	markdownSections.push('');
+
+	markdownSections.push('## Failure Collection Buckets');
+	markdownSections.push('');
+	markdownSections.push(toMarkdownBuckets(collectionBuckets));
+	markdownSections.push('');
+
+	markdownSections.push('## Cross-Suite Overlap');
+	markdownSections.push('');
+	if (overlapFailIds.length) {
+		markdownSections.push(`Failing in both suites (${overlapFailIds.length}): ${overlapFailIds.join(', ')}`);
+	}
+	else {
+		markdownSections.push('_No overlapping failures between suites._');
+	}
+	markdownSections.push('');
+
+	fs.writeFileSync(triageMdPath, markdownSections.join('\n'));
+
+	console.log(`wrote ${triageJsonPath}`);
+	console.log(`wrote ${triageMdPath}`);
+	if (openAiFailIds.length) {
+		console.log(`rerun-e2e: AI_ASSISTANT_E2E_CASES="${openAiFailIds.join(',')}" npm run ai-assistant-openai-e2e`);
+	}
+	if (snfFailIds.length) {
+		console.log(`rerun-snf: AI_ASSISTANT_SNF_EVAL_CASES="${snfFailIds.join(',')}" npm run ai-assistant-snf-live-eval`);
+	}
+	if (parityFailIds.length) {
+		console.log(`rerun-parity: AI_ASSISTANT_PARITY_CASES="${parityFailIds.join(',')}" npm run ai-assistant-data-parity-e2e`);
+	}
+	if (corpusFailIds.length) {
+		console.log(`rerun-corpus: AI_ASSISTANT_CORPUS_CASES="${corpusFailIds.join(',')}" npm run ai-assistant-corpus-replay-e2e`);
+	}
+	if (priorityClusters.length) {
+		const topCluster = priorityClusters[0];
+		console.log(`top-priority-cluster: ${topCluster.name} (score=${topCluster.score}, cases=${topCluster.caseIds.join(',')})`);
+	}
+}
+
+try {
+	run();
+	process.exit(process.exitCode ?? 0);
+}
+catch (error) {
+	console.error(error);
+	process.exit(1);
+}