@resolveio/server-lib 22.3.221 → 22.3.223

package/src/util/ai-run-evidence-dashboard.ts

@@ -1,323 +0,0 @@
-import { AIRun, AIRunCost } from './ai-run-evidence';
-import { buildAIRunTrainingExampleFromRun, classifyAIRunFailureClass } from './ai-run-evidence-eval';
-
-export interface AIRunTimelineItem {
-	time?: string;
-	type: string;
-	label: string;
-	status?: string;
-	message?: string;
-	evidenceRefs: string[];
-	metadata?: Record<string, any>;
-}
-
-export interface AIRunQaHealth {
-	outcome?: string;
-	infra: {
-		pass: number;
-		fail: number;
-		blocked: number;
-		warn: number;
-		unknown: number;
-	};
-	compile?: {
-		status?: string;
-		stale?: boolean;
-		artifactPath?: string;
-		message?: string;
-	};
-	route: {
-		pass: number;
-		fail: number;
-		shellOnly: number;
-	};
-	business: {
-		pass: number;
-		fail: number;
-		blocked: number;
-		missing: boolean;
-	};
-}
-
-export interface AIRunCostPanel {
-	totalTokens: number;
-	estimatedUsd: number;
-	models: string[];
-	categories: Array<{
-		category: string;
-		totalTokens: number;
-		estimatedUsd: number;
-		count: number;
-	}>;
-	sources?: Array<{
-		source: string;
-		totalTokens: number;
-		estimatedUsd: number;
-		count: number;
-		manualCount: number;
-		untrackedCount: number;
-	}>;
-	surfaces?: Array<{
-		surface: string;
-		totalTokens: number;
-		estimatedUsd: number;
-		count: number;
-	}>;
-	manualEstimatedUsd?: number;
-	untrackedEstimatedUsd?: number;
-	untrackedWarnings?: string[];
-	observabilityStatus?: string;
-	missingLedger?: boolean;
-	externalCodexMonitorUsed?: boolean;
-	nextAction?: string;
-}
-
-export interface AIRunOperatorView {
-	runId?: string;
-	source: string;
-	title?: string;
-	outcome: string;
-	failureClass: string;
-	nextAction?: string;
-	warnings: string[];
-	timeline: AIRunTimelineItem[];
-	qaHealth: AIRunQaHealth;
-	costPanel: AIRunCostPanel;
-	trainingExample: ReturnType<typeof buildAIRunTrainingExampleFromRun>;
-}
-
-function cleanText(value: any, max = 1000): string {
-	return String(value || '').replace(/\s+/g, ' ').trim().slice(0, max);
-}
-
-function toIso(value: any): string | undefined {
-	if (value instanceof Date && Number.isFinite(value.getTime())) {
-		return value.toISOString();
-	}
-	if (value) {
-		const parsed = new Date(value);
-		if (Number.isFinite(parsed.getTime())) {
-			return parsed.toISOString();
-		}
-	}
-	return undefined;
-}
-
-function countStatuses(values: Array<{ status?: string }>): Record<string, number> {
-	const result: Record<string, number> = {
-		pass: 0,
-		fail: 0,
-		blocked: 0,
-		warn: 0,
-		unknown: 0
-	};
-	for (const value of values) {
-		const status = cleanText(value.status, 40).toLowerCase();
-		if (status === 'pass' || status === 'fail' || status === 'blocked' || status === 'warn') {
-			result[status] += 1;
-		}
-		else {
-			result.unknown += 1;
-		}
-	}
-	return result;
-}
-
-function sortTimeline(items: AIRunTimelineItem[]): AIRunTimelineItem[] {
-	return [...items].sort((left, right) => {
-		const leftTime = left.time ? new Date(left.time).getTime() : 0;
-		const rightTime = right.time ? new Date(right.time).getTime() : 0;
-		return leftTime - rightTime;
-	});
-}
-
-export function buildAIRunTimeline(run: AIRun): AIRunTimelineItem[] {
-	const items: AIRunTimelineItem[] = [];
-	for (const event of run.events || []) {
-		items.push({
-			time: toIso(event.recordedAt),
-			type: event.type,
-			label: event.category || event.type,
-			message: cleanText(event.message, 2000),
-			evidenceRefs: Array.isArray(event.artifactPaths) ? event.artifactPaths : [],
-			metadata: event.metadata
-		});
-	}
-	for (const gate of run.gates || []) {
-		items.push({
-			time: toIso(gate.recordedAt),
-			type: 'gate',
-			label: gate.label || gate.key,
-			status: gate.status,
-			message: gate.reason,
-			evidenceRefs: gate.evidenceRefs || [],
-			metadata: gate.metadata
-		});
-	}
-	for (const check of run.qa?.infraChecks || []) {
-		items.push({
-			time: toIso(check.checkedAt),
-			type: 'qa_infra',
-			label: check.name,
-			status: check.status,
-			message: check.message,
-			evidenceRefs: check.path ? [check.path] : [],
-			metadata: check.metadata
-		});
-	}
-	if (run.qa?.compile) {
-		items.push({
-			time: toIso(run.qa.compile.recordedAt),
-			type: 'qa_compile',
-			label: run.qa.compile.command || 'Compile/build',
-			status: run.qa.compile.status,
-			message: run.qa.compile.message,
-			evidenceRefs: run.qa.compile.artifactPath ? [run.qa.compile.artifactPath] : [],
-			metadata: run.qa.compile.metadata
-		});
-	}
-	for (const probe of run.qa?.routeProbes || []) {
-		items.push({
-			time: toIso(probe.recordedAt),
-			type: 'qa_route_probe',
-			label: probe.route,
-			status: probe.status,
-			message: probe.message || probe.caption,
-			evidenceRefs: probe.screenshot ? [probe.screenshot] : [],
-			metadata: probe.metadata
-		});
-	}
-	for (const assertion of run.qa?.businessAssertions || []) {
-		items.push({
-			time: toIso(assertion.recordedAt),
-			type: 'qa_business_assertion',
-			label: assertion.assertion,
-			status: assertion.status,
-			message: assertion.observed || assertion.dataProof || assertion.message,
-			evidenceRefs: assertion.artifactPaths || [],
-			metadata: assertion.metadata
-		});
-	}
-	return sortTimeline(items);
-}
-
-export function buildAIRunQaHealth(run: AIRun): AIRunQaHealth {
-	const infraCounts = countStatuses(run.qa?.infraChecks || []);
-	const routeProbes = run.qa?.routeProbes || [];
-	const routePass = routeProbes.filter((probe) => probe.status === 'pass' && probe.shellOnly !== true).length;
-	const routeFail = routeProbes.filter((probe) => probe.status === 'fail' || probe.status === 'blocked').length;
-	const shellOnly = routeProbes.filter((probe) => probe.shellOnly === true).length;
-	const assertions = run.qa?.businessAssertions || [];
-	const businessPass = assertions.filter((assertion) => assertion.status === 'pass').length;
-	const businessFail = assertions.filter((assertion) => assertion.status === 'fail').length;
-	const businessBlocked = assertions.filter((assertion) => assertion.status === 'blocked').length;
-	return {
-		outcome: run.qa?.outcome,
-		infra: {
-			pass: infraCounts.pass,
-			fail: infraCounts.fail,
-			blocked: infraCounts.blocked,
-			warn: infraCounts.warn,
-			unknown: infraCounts.unknown
-		},
-		compile: run.qa?.compile ? {
-			status: run.qa.compile.status,
-			stale: run.qa.compile.staleEvidence === true || run.qa.compile.status === 'stale',
-			artifactPath: run.qa.compile.artifactPath,
-			message: run.qa.compile.message
-		} : undefined,
-		route: {
-			pass: routePass,
-			fail: routeFail,
-			shellOnly
-		},
-		business: {
-			pass: businessPass,
-			fail: businessFail,
-			blocked: businessBlocked,
-			missing: assertions.length === 0
-		}
-	};
-}
-
-export function buildAIRunCostPanel(cost?: AIRunCost, run?: AIRun): AIRunCostPanel {
-	const categories = Object.keys(cost?.categories || {}).sort().map((category) => {
-		const entry = cost?.categories?.[category];
-		return {
-			category,
-			totalTokens: Number(entry?.totalTokens || 0),
-			estimatedUsd: Number(entry?.estimatedUsd || 0),
-			count: Number(entry?.count || 0)
-		};
-	});
-	const sources = Object.keys(cost?.sources || {}).sort().map((source) => {
-		const entry = cost?.sources?.[source];
-		return {
-			source,
-			totalTokens: Number(entry?.totalTokens || 0),
-			estimatedUsd: Number(entry?.estimatedUsd || 0),
-			count: Number(entry?.count || 0),
-			manualCount: Number(entry?.manualCount || 0),
-			untrackedCount: Number(entry?.untrackedCount || 0)
-		};
-	});
-	const surfaces = Object.keys(cost?.surfaces || {}).sort().map((surface) => {
-		const entry = cost?.surfaces?.[surface];
-		return {
-			surface,
-			totalTokens: Number(entry?.totalTokens || 0),
-			estimatedUsd: Number(entry?.estimatedUsd || 0),
-			count: Number(entry?.count || 0)
-		};
-	});
-	const costGate = run?.gates?.find((gate) => gate.key === 'cost_observability');
-	const observability = (run?.metadata as any)?.costObservability || costGate?.metadata || {};
-	return {
-		totalTokens: Number(cost?.totalTokens || 0),
-		estimatedUsd: Number(cost?.estimatedUsd || 0),
-		models: Array.isArray(cost?.models) ? cost.models : [],
-		categories,
-		sources,
-		surfaces,
-		manualEstimatedUsd: Number(cost?.manualEstimatedUsd || 0),
-		untrackedEstimatedUsd: Number(cost?.untrackedEstimatedUsd || 0),
-		untrackedWarnings: Array.isArray(cost?.untrackedWarnings) ? cost.untrackedWarnings : [],
-		observabilityStatus: cleanText(observability.status || costGate?.status, 80),
-		missingLedger: observability.missingLedger === true || costGate?.metadata?.missingLedger === true,
-		externalCodexMonitorUsed: observability.externalCodexMonitorUsed === true || costGate?.metadata?.externalCodexMonitorUsed === true,
-		nextAction: cleanText(observability.nextAction, 500)
-	};
-}
-
-export function buildAIRunFalsePassWarnings(run: AIRun): string[] {
-	const warnings = Array.isArray(run.warnings) ? [...run.warnings] : [];
-	if (run.outcome === 'false_pass' && !warnings.includes('false_pass_outcome')) {
-		warnings.push('false_pass_outcome');
-	}
-	if ((run.qa?.outcome === 'route_only_pass' || run.qa?.outcome === 'incomplete') && !warnings.includes('missing_business_assertion')) {
-		warnings.push('missing_business_assertion');
-	}
-	if (run.qa?.routeProbes?.some((probe) => probe.shellOnly === true) && !warnings.includes('shell_only_route')) {
-		warnings.push('shell_only_route');
-	}
-	if (run.outcome === 'release_blocked' && !warnings.includes('release_blocked_after_scorecard')) {
-		warnings.push('release_blocked_after_scorecard');
-	}
-	return warnings;
-}
-
-export function buildAIRunOperatorView(run: AIRun): AIRunOperatorView {
-	return {
-		runId: run.id || (run as any)._id,
-		source: run.source,
-		title: run.title,
-		outcome: run.outcome,
-		failureClass: classifyAIRunFailureClass(run),
-		nextAction: run.nextAction,
-		warnings: buildAIRunFalsePassWarnings(run),
-		timeline: buildAIRunTimeline(run),
-		qaHealth: buildAIRunQaHealth(run),
-		costPanel: buildAIRunCostPanel(run.cost, run),
-		trainingExample: buildAIRunTrainingExampleFromRun(run)
-	};
-}