@resolveio/server-lib 22.3.221 → 22.3.222

package/tests/ai-assistant-eval-triage.ts

@@ -1,831 +0,0 @@
-import * as fs from 'fs';
-import * as path from 'path';
-
-type OpenAiE2EResult = {
-	id: string;
-	prompt: string;
-	pass: boolean;
-	error?: string;
-	userProfile?: string;
-};
-
-type OpenAiE2EOutput = {
-	now?: string;
-	total?: number;
-	passed?: number;
-	failed?: number;
-	results?: OpenAiE2EResult[];
-};
-
-type SnfLiveEvalResult = {
-	id: string;
-	prompt: string;
-	pass: boolean;
-	rowCount?: number;
-	minRows?: number;
-	error?: string;
-	classification?: {
-		pass?: boolean;
-		type?: string;
-		dataQuestion?: boolean;
-		source?: string;
-	};
-};
-
-type SnfLiveEvalOutput = {
-	now?: string;
-	results?: SnfLiveEvalResult[];
-	summary?: {
-		total?: number;
-		passed?: number;
-		failed?: number;
-	};
-};
-
-type DataParityEvalResult = {
-	id: string;
-	prompt: string;
-	pass: boolean;
-	errors?: string[];
-	directive?: {
-		type?: string;
-		collection?: string;
-		source?: string;
-	};
-	tool?: {
-		type?: string;
-		collection?: string;
-		rowCount?: number;
-		total?: number | null;
-		columns?: string[];
-	};
-};
-
-type DataParityEvalOutput = {
-	now?: string;
-	total?: number;
-	passed?: number;
-	failed?: number;
-	results?: DataParityEvalResult[];
-};
-
-type CorpusReplayEvalResult = {
-	id: string;
-	prompt: string;
-	pass: boolean;
-	errors?: string[];
-	tool?: {
-		type?: string;
-		collection?: string;
-		rowCount?: number;
-		total?: number | null;
-		columns?: string[];
-		source?: string;
-	};
-};
-
-type CorpusReplayEvalOutput = {
-	now?: string;
-	total?: number;
-	passed?: number;
-	failed?: number;
-	results?: CorpusReplayEvalResult[];
-};
-
-type FailureBucket = {
-	name: string;
-	count: number;
-	caseIds: string[];
-};
-
-type FailureEntry = {
-	id: string;
-	prompt: string;
-	error: string;
-	category: string;
-	suite: 'openai_e2e' | 'snf_live' | 'data_parity' | 'corpus_replay';
-	intent: string;
-	collectionFamily: string;
-};
-
-type PriorityCluster = {
-	name: string;
-	count: number;
-	score: number;
-	category: string;
-	intent: string;
-	suites: string[];
-	collectionFamilies: string[];
-	caseIds: string[];
-	examplePrompts: string[];
-};
-
-function readJsonFile<T>(filePath: string): T | null {
-	if (!fs.existsSync(filePath)) {
-		return null;
-	}
-	try {
-		const raw = fs.readFileSync(filePath, 'utf8');
-		return JSON.parse(raw) as T;
-	}
-	catch (error: any) {
-		throw new Error(`Failed to parse JSON file ${filePath}: ${String(error?.message || error || 'unknown error')}`);
-	}
-}
-
-function categorizeOpenAiFailure(errorText: string): string {
-	const text = String(errorText || '').toLowerCase();
-	if (!text) {
-		return 'unknown';
-	}
-	if (text.includes('timed out waiting for assistant completion')) {
-		return 'timeout';
-	}
-	if (text.includes('expected markdown table')) {
-		return 'missing_table';
-	}
-	if (text.includes('expected data/mixed classification') || text.includes('expected dataquestion=true')) {
-		return 'classification';
-	}
-	if (text.includes('expected first column to be metric')) {
-		return 'metric_grid_shape';
-	}
-	if (text.includes('expected first column to include customer')) {
-		return 'customer_grid_shape';
-	}
-	if (text.includes('expected day/date header')
-		|| text.includes('expected week header')
-		|| text.includes('expected month header')
-		|| text.includes('expected quarter header')
-		|| text.includes('expected year header')) {
-		return 'time_grain_header';
-	}
-	if (text.includes('expected fast_path=')
-		|| text.includes('fast_path_change_generic')) {
-		return 'fast_path';
-	}
-	if (text.includes('expected response to include route')
-		|| text.includes('expected response to exclude route')) {
-		return 'routing_permission';
-	}
-	return 'other';
-}
-
-function categorizeSnfFailure(result: SnfLiveEvalResult): string {
-	if (!result) {
-		return 'unknown';
-	}
-	if (result.error) {
-		return 'query_error';
-	}
-	if (result.classification?.pass === false) {
-		return 'classification';
-	}
-	const rowCount = Number(result.rowCount || 0);
-	const minRows = Number(result.minRows || 0);
-	if (rowCount < minRows) {
-		return 'row_count_below_expectation';
-	}
-	if (result.pass === false) {
-		return 'other';
-	}
-	return 'pass';
-}
-
-function categorizeParityFailure(errorText: string): string {
-	const text = String(errorText || '').toLowerCase();
-	if (!text) {
-		return 'unknown';
-	}
-	if (text.includes('collection is not configured for report builder')
-		|| text.includes('no tool_result found')) {
-		return 'collection_or_directive';
-	}
-	if (text.includes('truncated')) {
-		return 'display_truncation';
-	}
-	if (text.includes('could not parse')
-		|| text.includes('empty markdown table')) {
-		return 'display_shape';
-	}
-	if (text.includes('rowcount mismatch')
-		|| text.includes('replay display mismatch')
-		|| text.includes('replay row length mismatch')) {
-		return 'replay_mismatch';
-	}
-	if (text.includes('mismatch for')
-		|| text.includes('missing key')
-		|| text.includes('unexpected key')
-		|| text.includes('expected multiple named customers')) {
-		return 'data_mismatch';
-	}
-	return 'other';
-}
-
-function categorizeCorpusFailure(errorText: string): string {
-	const text = String(errorText || '').toLowerCase();
-	if (!text) {
-		return 'unknown';
-	}
-	if (text.includes('internal error response')) {
-		return 'assistant_runtime';
-	}
-	if (text.includes('no tool_result found')
-		|| text.includes('did not produce a usable tool_result')
-		|| text.includes('directive')) {
-		return 'directive_or_tool_result';
-	}
-	if (text.includes('replay rowcount mismatch')
-		|| text.includes('replay columns mismatch')
-		|| text.includes('replay display mismatch')) {
-		return 'replay_mismatch';
-	}
-	if (text.includes('timed out waiting for final assistant message')) {
-		return 'timeout';
-	}
-	return 'other';
-}
-
-function normalizePromptText(value: string): string {
-	return String(value || '').toLowerCase().replace(/\s+/g, ' ').trim();
-}
-
-function categorizePromptIntent(promptText: string): string {
-	const prompt = normalizePromptText(promptText);
-	if (!prompt) {
-		return 'unknown';
-	}
-	if (/when was the last time|what changed|last update|system update|updated/i.test(prompt)) {
-		return 'change_history';
-	}
-	if (/greyed out|grayed out|dropdown|submit button|won't|cant |can't |not working|error|bug/i.test(prompt)) {
-		return 'ui_bug_triage';
-	}
-	if (/where do i go|route|which page|which screen|how do i get to|navigate/i.test(prompt)) {
-		return 'navigation';
-	}
-	if (/\btop\s+\d+\b/i.test(prompt)) {
-		return 'ranking_top_n';
-	}
-	if (/grouped by|\bby each\b|\bby\b|\bper\b|break down|break up/i.test(prompt)) {
-		return 'grouping_breakdown';
-	}
-	if (/how many|count|right now|snapshot/i.test(prompt)) {
-		return 'count_snapshot';
-	}
-	if (/last|this|today|yesterday|month|week|day|quarter|year|date range/i.test(prompt)) {
-		return 'time_window';
-	}
-	return 'other';
-}
-
-function categorizeCollectionFamily(value: any): string {
-	const collection = String(value || '').trim().toLowerCase();
-	if (!collection) {
-		return 'unknown';
-	}
-	if (collection.includes('invoice') || collection.includes('billing') || collection.includes('sales-tax')) {
-		return 'invoice';
-	}
-	if (collection.includes('support') || collection.includes('ticket')) {
-		return 'support_ticket';
-	}
-	if (collection.includes('blend') || collection.includes('chemical-blend')) {
-		return 'blend';
-	}
-	if (collection.includes('deliver') || collection.includes('driver')) {
-		return 'delivery';
-	}
-	if (collection.includes('work-order') || collection.includes('order') || collection.includes('job')) {
-		return 'work_order';
-	}
-	if (collection.includes('customer') || collection.includes('client')) {
-		return 'customer_client';
-	}
-	return collection;
-}
-
-function bucketFailures(entries: FailureEntry[], keyResolver: (entry: FailureEntry) => string): FailureBucket[] {
-	const map = new Map<string, Set<string>>();
-	entries.forEach((entry) => {
-		const key = keyResolver(entry);
-		if (!entry || !key) {
-			return;
-		}
-		if (!map.has(key)) {
-			map.set(key, new Set<string>());
-		}
-		map.get(key)?.add(entry.id);
-	});
-	return Array.from(map.entries())
-		.map(([name, caseIds]) => ({
-			name,
-			count: caseIds.size,
-			caseIds: Array.from(caseIds).sort()
-		}))
-		.sort((a, b) => {
-			if (a.count !== b.count) {
-				return b.count - a.count;
-			}
-			return a.name.localeCompare(b.name);
-		});
-}
-
-function buildPriorityClusters(entries: FailureEntry[]): PriorityCluster[] {
-	if (!entries.length) {
-		return [];
-	}
-	const categoryWeight: Record<string, number> = {
-		assistant_runtime: 9,
-		directive_or_tool_result: 8,
-		collection_or_directive: 8,
-		replay_mismatch: 7,
-		data_mismatch: 6,
-		query_error: 6,
-		timeout: 6,
-		classification: 5,
-		display_shape: 4,
-		display_truncation: 4,
-		row_count_below_expectation: 4,
-		other: 2,
-		unknown: 1
-	};
-	const intentWeight: Record<string, number> = {
-		grouping_breakdown: 6,
-		ranking_top_n: 5,
-		count_snapshot: 4,
-		time_window: 4,
-		navigation: 3,
-		change_history: 3,
-		ui_bug_triage: 3,
-		other: 2,
-		unknown: 1
-	};
-	const clusters = new Map<string, {
-		category: string;
-		intent: string;
-		caseIds: Set<string>;
-		suites: Set<string>;
-		collectionFamilies: Set<string>;
-		prompts: Set<string>;
-	}>();
-	entries.forEach((entry) => {
-		const key = `${entry.category}__${entry.intent}`;
-		if (!clusters.has(key)) {
-			clusters.set(key, {
-				category: entry.category,
-				intent: entry.intent,
-				caseIds: new Set<string>(),
-				suites: new Set<string>(),
-				collectionFamilies: new Set<string>(),
-				prompts: new Set<string>()
-			});
-		}
-		const cluster = clusters.get(key);
-		cluster?.caseIds.add(entry.id);
-		cluster?.suites.add(entry.suite);
-		cluster?.collectionFamilies.add(entry.collectionFamily || 'unknown');
-		if (entry.prompt) {
-			cluster?.prompts.add(entry.prompt);
-		}
-	});
-	return Array.from(clusters.values())
-		.map((cluster) => {
-			const count = cluster.caseIds.size;
-			const suiteSpread = Math.max(cluster.suites.size - 1, 0);
-			const collectionSpread = Math.max(cluster.collectionFamilies.size - 1, 0);
-			const score = (
-				(categoryWeight[cluster.category] || categoryWeight.other) * count
-				+ (intentWeight[cluster.intent] || intentWeight.other) * count
-				+ suiteSpread * 3
-				+ collectionSpread
-			);
-			return {
-				name: `${cluster.category} + ${cluster.intent}`,
-				count,
-				score,
-				category: cluster.category,
-				intent: cluster.intent,
-				suites: Array.from(cluster.suites).sort(),
-				collectionFamilies: Array.from(cluster.collectionFamilies).sort(),
-				caseIds: Array.from(cluster.caseIds).sort(),
-				examplePrompts: Array.from(cluster.prompts).slice(0, 3)
-			};
-		})
-		.sort((a, b) => {
-			if (a.score !== b.score) {
-				return b.score - a.score;
-			}
-			if (a.count !== b.count) {
-				return b.count - a.count;
-			}
-			return a.name.localeCompare(b.name);
-		});
-}
-
-function toMarkdownPriorityClusters(clusters: PriorityCluster[]): string {
-	if (!clusters.length) {
-		return '_None_';
-	}
-	const lines = [
-		'| Priority Cluster | Score | Cases | Suites | Collections | Case IDs |',
-		'| --- | --- | --- | --- | --- | --- |'
-	];
-	clusters.forEach((cluster) => {
-		lines.push(
-			`| ${cluster.name} | ${cluster.score} | ${cluster.count} | ${cluster.suites.join(', ')} | ${cluster.collectionFamilies.join(', ')} | ${cluster.caseIds.join(', ')} |`
-		);
-	});
-	return lines.join('\n');
-}
-
-function toMarkdownBuckets(buckets: FailureBucket[]): string {
-	if (!buckets.length) {
-		return '_None_';
-	}
-	const lines = [
-		'| Bucket | Cases | Case IDs |',
-		'| --- | --- | --- |'
-	];
-	buckets.forEach((bucket) => {
-		lines.push(`| ${bucket.name} | ${bucket.count} | ${bucket.caseIds.join(', ')} |`);
-	});
-	return lines.join('\n');
-}
-
-function collectOpenAiFailures(data: OpenAiE2EOutput | null): FailureEntry[] {
-	const results = Array.isArray(data?.results) ? data.results : [];
-	return results
-		.filter(result => result && result.pass === false)
-		.map((result) => {
-			const prompt = String(result.prompt || '');
-			const error = String(result.error || 'Unknown failure');
-			return {
-				id: String(result.id || ''),
-				prompt,
-				error,
-				category: categorizeOpenAiFailure(error),
-				suite: 'openai_e2e',
-				intent: categorizePromptIntent(prompt),
-				collectionFamily: 'unknown'
-			};
-		});
-}
-
-function collectSnfFailures(data: SnfLiveEvalOutput | null): FailureEntry[] {
-	const results = Array.isArray(data?.results) ? data.results : [];
-	return results
-		.filter(result => result && result.pass === false)
-		.map((result) => {
-			const prompt = String(result.prompt || '');
-			const error = String(result.error || `rowCount=${result.rowCount || 0}, minRows=${result.minRows || 0}`);
-			return {
-				id: String(result.id || ''),
-				prompt,
-				error,
-				category: categorizeSnfFailure(result),
-				suite: 'snf_live',
-				intent: categorizePromptIntent(prompt),
-				collectionFamily: 'unknown'
-			};
-		});
-}
-
-function resolveParityCollection(result: DataParityEvalResult): string {
-	return String(
-		result?.tool?.collection
-		|| result?.directive?.collection
-		|| ''
-	).trim();
-}
-
-function resolveCorpusCollection(result: CorpusReplayEvalResult): string {
-	return String(result?.tool?.collection || '').trim();
-}
-
-function collectParityFailures(data: DataParityEvalOutput | null): FailureEntry[] {
-	const results = Array.isArray(data?.results) ? data.results : [];
-	return results
-		.filter(result => result && result.pass === false)
-		.map((result) => {
-			const prompt = String(result.prompt || '');
-			const errorList = Array.isArray(result.errors) ? result.errors : [];
-			const error = errorList.length ? errorList[0] : 'Unknown failure';
-			const collectionFamily = categorizeCollectionFamily(resolveParityCollection(result));
-			return {
-				id: String(result.id || ''),
-				prompt,
-				error,
-				category: categorizeParityFailure(error),
-				suite: 'data_parity',
-				intent: categorizePromptIntent(prompt),
-				collectionFamily
-			};
-		});
-}
-
-function collectCorpusFailures(data: CorpusReplayEvalOutput | null): FailureEntry[] {
-	const results = Array.isArray(data?.results) ? data.results : [];
-	return results
-		.filter(result => result && result.pass === false)
-		.map((result) => {
-			const prompt = String(result.prompt || '');
-			const errorList = Array.isArray(result.errors) ? result.errors : [];
-			const error = errorList.length ? errorList[0] : 'Unknown failure';
-			const collectionFamily = categorizeCollectionFamily(resolveCorpusCollection(result));
-			return {
-				id: String(result.id || ''),
-				prompt,
-				error,
-				category: categorizeCorpusFailure(error),
-				suite: 'corpus_replay',
-				intent: categorizePromptIntent(prompt),
-				collectionFamily
-			};
-		});
-}
-
-function resolveEvalOutputPath(repoRoot: string, envValue: string | undefined, fallbackRelativePath: string): string {
-	const value = String(envValue || '').trim();
-	if (!value) {
-		return path.join(repoRoot, fallbackRelativePath);
-	}
-	return path.isAbsolute(value) ? value : path.join(repoRoot, value);
-}
-
-function run(): void {
-	const repoRoot = process.cwd();
-	const tmpDir = path.join(repoRoot, 'tmp');
-	const openAiPath = resolveEvalOutputPath(
-		repoRoot,
-		process.env.AI_ASSISTANT_EVAL_OPENAI_OUTPUT,
-		path.join('tmp', 'ai-assistant-openai-e2e-output.json')
-	);
-	const snfLivePath = resolveEvalOutputPath(
-		repoRoot,
-		process.env.AI_ASSISTANT_EVAL_SNF_OUTPUT,
-		path.join('tmp', 'ai-assistant-snf-live-eval-output.json')
-	);
-	const parityPath = resolveEvalOutputPath(
-		repoRoot,
-		process.env.AI_ASSISTANT_EVAL_PARITY_OUTPUT,
-		path.join('tmp', 'ai-assistant-data-parity-e2e-output.json')
-	);
-	const corpusPath = resolveEvalOutputPath(
-		repoRoot,
-		process.env.AI_ASSISTANT_EVAL_CORPUS_OUTPUT,
-		path.join('tmp', 'ai-assistant-corpus-replay-e2e-output.json')
-	);
-	const triageJsonPath = path.join(tmpDir, 'ai-assistant-eval-triage.json');
-	const triageMdPath = path.join(tmpDir, 'ai-assistant-eval-triage.md');
-
-	const openAiOutput = readJsonFile<OpenAiE2EOutput>(openAiPath);
-	const snfLiveOutput = readJsonFile<SnfLiveEvalOutput>(snfLivePath);
-	const parityOutput = readJsonFile<DataParityEvalOutput>(parityPath);
-	const corpusOutput = readJsonFile<CorpusReplayEvalOutput>(corpusPath);
-
-	if (!openAiOutput && !snfLiveOutput && !parityOutput && !corpusOutput) {
-		throw new Error(`No eval outputs found. Expected at least one of:\n- ${openAiPath}\n- ${snfLivePath}\n- ${parityPath}\n- ${corpusPath}`);
-	}
-
-	const openAiFailures = collectOpenAiFailures(openAiOutput);
-	const snfFailures = collectSnfFailures(snfLiveOutput);
-	const parityFailures = collectParityFailures(parityOutput);
-	const corpusFailures = collectCorpusFailures(corpusOutput);
-	const openAiBuckets = bucketFailures(openAiFailures, entry => entry.category);
-	const snfBuckets = bucketFailures(snfFailures, entry => entry.category);
-	const parityBuckets = bucketFailures(parityFailures, entry => entry.category);
-	const corpusBuckets = bucketFailures(corpusFailures, entry => entry.category);
-	const allFailures = [
-		...openAiFailures,
-		...snfFailures,
-		...parityFailures,
-		...corpusFailures
-	];
-	const intentBuckets = bucketFailures(allFailures, entry => entry.intent);
-	const collectionBuckets = bucketFailures(allFailures, entry => entry.collectionFamily);
-	const priorityClusters = buildPriorityClusters(allFailures);
-
-	const openAiFailIds = openAiFailures.map(entry => entry.id).filter(Boolean).sort();
-	const snfFailIds = snfFailures.map(entry => entry.id).filter(Boolean).sort();
-	const parityFailIds = parityFailures.map(entry => entry.id).filter(Boolean).sort();
-	const corpusFailIds = corpusFailures.map(entry => entry.id).filter(Boolean).sort();
-	const suiteFailSets = [openAiFailIds, snfFailIds, parityFailIds, corpusFailIds].filter(ids => ids.length);
-	const overlapCounter = new Map<string, number>();
-	suiteFailSets.forEach((ids) => {
-		new Set(ids).forEach((id) => {
-			overlapCounter.set(id, (overlapCounter.get(id) || 0) + 1);
-		});
-	});
-	const overlapFailIds = Array.from(overlapCounter.entries())
-		.filter(([, count]) => count >= 2)
-		.map(([id]) => id)
-		.sort();
-
-	const triage = {
-		generatedAt: new Date().toISOString(),
-		files: {
-			openAiPath,
-			snfLivePath,
-			parityPath,
-			corpusPath
-		},
-		openAi: openAiOutput ? {
-			total: Number(openAiOutput.total || (openAiOutput.results || []).length || 0),
-			passed: Number(openAiOutput.passed || 0),
-			failed: Number(openAiOutput.failed || openAiFailures.length || 0),
-			failureBuckets: openAiBuckets,
-			failingCaseIds: openAiFailIds
-		} : null,
-		snfLive: snfLiveOutput ? {
-			total: Number(
-				snfLiveOutput.summary?.total
-				|| (snfLiveOutput.results || []).length
-				|| 0
-			),
-			passed: Number(snfLiveOutput.summary?.passed || 0),
-			failed: Number(snfLiveOutput.summary?.failed || snfFailures.length || 0),
-			failureBuckets: snfBuckets,
-			failingCaseIds: snfFailIds
-		} : null,
-		parity: parityOutput ? {
-			total: Number(parityOutput.total || (parityOutput.results || []).length || 0),
-			passed: Number(parityOutput.passed || 0),
-			failed: Number(parityOutput.failed || parityFailures.length || 0),
-			failureBuckets: parityBuckets,
-			failingCaseIds: parityFailIds
-		} : null,
-		corpusReplay: corpusOutput ? {
-			total: Number(corpusOutput.total || (corpusOutput.results || []).length || 0),
-			passed: Number(corpusOutput.passed || 0),
-			failed: Number(corpusOutput.failed || corpusFailures.length || 0),
-			failureBuckets: corpusBuckets,
-			failingCaseIds: corpusFailIds
-		} : null,
-		overlapFailIds,
-		allFailures: {
-			total: allFailures.length,
-			intentBuckets,
-			collectionBuckets,
-			priorityClusters: priorityClusters.slice(0, 20)
-		}
-	};
-
-	fs.mkdirSync(tmpDir, { recursive: true });
-	fs.writeFileSync(triageJsonPath, JSON.stringify(triage, null, 2));
-
-	const markdownSections: string[] = [];
-	markdownSections.push('# AI Assistant Eval Triage');
-	markdownSections.push('');
-	markdownSections.push(`Generated: ${triage.generatedAt}`);
-	markdownSections.push('');
-
-	if (openAiOutput) {
-		markdownSections.push('## OpenAI E2E');
-		markdownSections.push('');
-		markdownSections.push(`- Total: ${triage.openAi?.total || 0}`);
-		markdownSections.push(`- Passed: ${triage.openAi?.passed || 0}`);
-		markdownSections.push(`- Failed: ${triage.openAi?.failed || 0}`);
-		markdownSections.push('');
-		markdownSections.push(toMarkdownBuckets(openAiBuckets));
-		markdownSections.push('');
-		markdownSections.push('Rerun failed E2E cases:');
-		if (openAiFailIds.length) {
-			markdownSections.push('');
-			markdownSections.push('```bash');
-			markdownSections.push(`AI_ASSISTANT_E2E_CASES="${openAiFailIds.join(',')}" npm run ai-assistant-openai-e2e`);
-			markdownSections.push('```');
-		}
-		else {
-			markdownSections.push('');
-			markdownSections.push('_No E2E failures detected._');
-		}
-		markdownSections.push('');
-	}
-
-	if (snfLiveOutput) {
-		markdownSections.push('## SNF Live Eval');
-		markdownSections.push('');
-		markdownSections.push(`- Total: ${triage.snfLive?.total || 0}`);
-		markdownSections.push(`- Passed: ${triage.snfLive?.passed || 0}`);
-		markdownSections.push(`- Failed: ${triage.snfLive?.failed || 0}`);
-		markdownSections.push('');
-		markdownSections.push(toMarkdownBuckets(snfBuckets));
-		markdownSections.push('');
-		markdownSections.push('Rerun failed SNF live cases:');
-		if (snfFailIds.length) {
-			markdownSections.push('');
-			markdownSections.push('```bash');
-			markdownSections.push(`AI_ASSISTANT_SNF_EVAL_CASES="${snfFailIds.join(',')}" npm run ai-assistant-snf-live-eval`);
-			markdownSections.push('```');
-		}
-		else {
-			markdownSections.push('');
-			markdownSections.push('_No SNF live failures detected._');
-		}
-		markdownSections.push('');
-	}
-
-	if (parityOutput) {
-		markdownSections.push('## Data Parity E2E');
-		markdownSections.push('');
-		markdownSections.push(`- Total: ${triage.parity?.total || 0}`);
-		markdownSections.push(`- Passed: ${triage.parity?.passed || 0}`);
-		markdownSections.push(`- Failed: ${triage.parity?.failed || 0}`);
-		markdownSections.push('');
-		markdownSections.push(toMarkdownBuckets(parityBuckets));
-		markdownSections.push('');
-		markdownSections.push('Rerun failed data parity cases:');
-		if (parityFailIds.length) {
-			markdownSections.push('');
-			markdownSections.push('```bash');
-			markdownSections.push(`AI_ASSISTANT_PARITY_CASES="${parityFailIds.join(',')}" npm run ai-assistant-data-parity-e2e`);
-			markdownSections.push('```');
-		}
-		else {
-			markdownSections.push('');
-			markdownSections.push('_No data parity failures detected._');
-		}
-		markdownSections.push('');
-	}
-
-	if (corpusOutput) {
-		markdownSections.push('## Corpus Replay E2E');
-		markdownSections.push('');
-		markdownSections.push(`- Total: ${triage.corpusReplay?.total || 0}`);
-		markdownSections.push(`- Passed: ${triage.corpusReplay?.passed || 0}`);
-		markdownSections.push(`- Failed: ${triage.corpusReplay?.failed || 0}`);
-		markdownSections.push('');
-		markdownSections.push(toMarkdownBuckets(corpusBuckets));
-		markdownSections.push('');
-		markdownSections.push('Rerun failed corpus replay cases:');
-		if (corpusFailIds.length) {
-			markdownSections.push('');
-			markdownSections.push('```bash');
-			markdownSections.push(`AI_ASSISTANT_CORPUS_CASES="${corpusFailIds.join(',')}" npm run ai-assistant-corpus-replay-e2e`);
-			markdownSections.push('```');
-		}
-		else {
-			markdownSections.push('');
-			markdownSections.push('_No corpus replay failures detected._');
-		}
-		markdownSections.push('');
-	}
-
-	markdownSections.push('## Prioritized Fix Clusters');
-	markdownSections.push('');
-	markdownSections.push(`Total failures across suites: ${allFailures.length}`);
-	markdownSections.push('');
-	markdownSections.push(toMarkdownPriorityClusters(priorityClusters.slice(0, 10)));
-	markdownSections.push('');
-
-	markdownSections.push('## Failure Intent Buckets');
-	markdownSections.push('');
-	markdownSections.push(toMarkdownBuckets(intentBuckets));
-	markdownSections.push('');
-
-	markdownSections.push('## Failure Collection Buckets');
-	markdownSections.push('');
-	markdownSections.push(toMarkdownBuckets(collectionBuckets));
-	markdownSections.push('');
-
-	markdownSections.push('## Cross-Suite Overlap');
-	markdownSections.push('');
-	if (overlapFailIds.length) {
-		markdownSections.push(`Failing in both suites (${overlapFailIds.length}): ${overlapFailIds.join(', ')}`);
-	}
-	else {
-		markdownSections.push('_No overlapping failures between suites._');
-	}
-	markdownSections.push('');
-
-	fs.writeFileSync(triageMdPath, markdownSections.join('\n'));
-
-	console.log(`wrote ${triageJsonPath}`);
-	console.log(`wrote ${triageMdPath}`);
-	if (openAiFailIds.length) {
-		console.log(`rerun-e2e: AI_ASSISTANT_E2E_CASES="${openAiFailIds.join(',')}" npm run ai-assistant-openai-e2e`);
-	}
-	if (snfFailIds.length) {
-		console.log(`rerun-snf: AI_ASSISTANT_SNF_EVAL_CASES="${snfFailIds.join(',')}" npm run ai-assistant-snf-live-eval`);
-	}
-	if (parityFailIds.length) {
-		console.log(`rerun-parity: AI_ASSISTANT_PARITY_CASES="${parityFailIds.join(',')}" npm run ai-assistant-data-parity-e2e`);
-	}
-	if (corpusFailIds.length) {
-		console.log(`rerun-corpus: AI_ASSISTANT_CORPUS_CASES="${corpusFailIds.join(',')}" npm run ai-assistant-corpus-replay-e2e`);
-	}
-	if (priorityClusters.length) {
-		const topCluster = priorityClusters[0];
-		console.log(`top-priority-cluster: ${topCluster.name} (score=${topCluster.score}, cases=${topCluster.caseIds.join(',')})`);
-	}
-}
-
-try {
-	run();
-	process.exit(process.exitCode ?? 0);
-}
-catch (error) {
-	console.error(error);
-	process.exit(1);
-}