@resolveio/server-lib 22.3.220 → 22.3.221

package/src/util/ai-run-evidence-dashboard.ts

@@ -0,0 +1,323 @@
+import { AIRun, AIRunCost } from './ai-run-evidence';
+import { buildAIRunTrainingExampleFromRun, classifyAIRunFailureClass } from './ai-run-evidence-eval';
+
+export interface AIRunTimelineItem {
+	time?: string;
+	type: string;
+	label: string;
+	status?: string;
+	message?: string;
+	evidenceRefs: string[];
+	metadata?: Record<string, any>;
+}
+
+export interface AIRunQaHealth {
+	outcome?: string;
+	infra: {
+		pass: number;
+		fail: number;
+		blocked: number;
+		warn: number;
+		unknown: number;
+	};
+	compile?: {
+		status?: string;
+		stale?: boolean;
+		artifactPath?: string;
+		message?: string;
+	};
+	route: {
+		pass: number;
+		fail: number;
+		shellOnly: number;
+	};
+	business: {
+		pass: number;
+		fail: number;
+		blocked: number;
+		missing: boolean;
+	};
+}
+
+export interface AIRunCostPanel {
+	totalTokens: number;
+	estimatedUsd: number;
+	models: string[];
+	categories: Array<{
+		category: string;
+		totalTokens: number;
+		estimatedUsd: number;
+		count: number;
+	}>;
+	sources?: Array<{
+		source: string;
+		totalTokens: number;
+		estimatedUsd: number;
+		count: number;
+		manualCount: number;
+		untrackedCount: number;
+	}>;
+	surfaces?: Array<{
+		surface: string;
+		totalTokens: number;
+		estimatedUsd: number;
+		count: number;
+	}>;
+	manualEstimatedUsd?: number;
+	untrackedEstimatedUsd?: number;
+	untrackedWarnings?: string[];
+	observabilityStatus?: string;
+	missingLedger?: boolean;
+	externalCodexMonitorUsed?: boolean;
+	nextAction?: string;
+}
+
+export interface AIRunOperatorView {
+	runId?: string;
+	source: string;
+	title?: string;
+	outcome: string;
+	failureClass: string;
+	nextAction?: string;
+	warnings: string[];
+	timeline: AIRunTimelineItem[];
+	qaHealth: AIRunQaHealth;
+	costPanel: AIRunCostPanel;
+	trainingExample: ReturnType<typeof buildAIRunTrainingExampleFromRun>;
+}
+
+function cleanText(value: any, max = 1000): string {
+	return String(value || '').replace(/\s+/g, ' ').trim().slice(0, max);
+}
+
+function toIso(value: any): string | undefined {
+	if (value instanceof Date && Number.isFinite(value.getTime())) {
+		return value.toISOString();
+	}
+	if (value) {
+		const parsed = new Date(value);
+		if (Number.isFinite(parsed.getTime())) {
+			return parsed.toISOString();
+		}
+	}
+	return undefined;
+}
+
+function countStatuses(values: Array<{ status?: string }>): Record<string, number> {
+	const result: Record<string, number> = {
+		pass: 0,
+		fail: 0,
+		blocked: 0,
+		warn: 0,
+		unknown: 0
+	};
+	for (const value of values) {
+		const status = cleanText(value.status, 40).toLowerCase();
+		if (status === 'pass' || status === 'fail' || status === 'blocked' || status === 'warn') {
+			result[status] += 1;
+		}
+		else {
+			result.unknown += 1;
+		}
+	}
+	return result;
+}
+
+function sortTimeline(items: AIRunTimelineItem[]): AIRunTimelineItem[] {
+	return [...items].sort((left, right) => {
+		const leftTime = left.time ? new Date(left.time).getTime() : 0;
+		const rightTime = right.time ? new Date(right.time).getTime() : 0;
+		return leftTime - rightTime;
+	});
+}
+
+export function buildAIRunTimeline(run: AIRun): AIRunTimelineItem[] {
+	const items: AIRunTimelineItem[] = [];
+	for (const event of run.events || []) {
+		items.push({
+			time: toIso(event.recordedAt),
+			type: event.type,
+			label: event.category || event.type,
+			message: cleanText(event.message, 2000),
+			evidenceRefs: Array.isArray(event.artifactPaths) ? event.artifactPaths : [],
+			metadata: event.metadata
+		});
+	}
+	for (const gate of run.gates || []) {
+		items.push({
+			time: toIso(gate.recordedAt),
+			type: 'gate',
+			label: gate.label || gate.key,
+			status: gate.status,
+			message: gate.reason,
+			evidenceRefs: gate.evidenceRefs || [],
+			metadata: gate.metadata
+		});
+	}
+	for (const check of run.qa?.infraChecks || []) {
+		items.push({
+			time: toIso(check.checkedAt),
+			type: 'qa_infra',
+			label: check.name,
+			status: check.status,
+			message: check.message,
+			evidenceRefs: check.path ? [check.path] : [],
+			metadata: check.metadata
+		});
+	}
+	if (run.qa?.compile) {
+		items.push({
+			time: toIso(run.qa.compile.recordedAt),
+			type: 'qa_compile',
+			label: run.qa.compile.command || 'Compile/build',
+			status: run.qa.compile.status,
+			message: run.qa.compile.message,
+			evidenceRefs: run.qa.compile.artifactPath ? [run.qa.compile.artifactPath] : [],
+			metadata: run.qa.compile.metadata
+		});
+	}
+	for (const probe of run.qa?.routeProbes || []) {
+		items.push({
+			time: toIso(probe.recordedAt),
+			type: 'qa_route_probe',
+			label: probe.route,
+			status: probe.status,
+			message: probe.message || probe.caption,
+			evidenceRefs: probe.screenshot ? [probe.screenshot] : [],
+			metadata: probe.metadata
+		});
+	}
+	for (const assertion of run.qa?.businessAssertions || []) {
+		items.push({
+			time: toIso(assertion.recordedAt),
+			type: 'qa_business_assertion',
+			label: assertion.assertion,
+			status: assertion.status,
+			message: assertion.observed || assertion.dataProof || assertion.message,
+			evidenceRefs: assertion.artifactPaths || [],
+			metadata: assertion.metadata
+		});
+	}
+	return sortTimeline(items);
+}
+
+export function buildAIRunQaHealth(run: AIRun): AIRunQaHealth {
+	const infraCounts = countStatuses(run.qa?.infraChecks || []);
+	const routeProbes = run.qa?.routeProbes || [];
+	const routePass = routeProbes.filter((probe) => probe.status === 'pass' && probe.shellOnly !== true).length;
+	const routeFail = routeProbes.filter((probe) => probe.status === 'fail' || probe.status === 'blocked').length;
+	const shellOnly = routeProbes.filter((probe) => probe.shellOnly === true).length;
+	const assertions = run.qa?.businessAssertions || [];
+	const businessPass = assertions.filter((assertion) => assertion.status === 'pass').length;
+	const businessFail = assertions.filter((assertion) => assertion.status === 'fail').length;
+	const businessBlocked = assertions.filter((assertion) => assertion.status === 'blocked').length;
+	return {
+		outcome: run.qa?.outcome,
+		infra: {
+			pass: infraCounts.pass,
+			fail: infraCounts.fail,
+			blocked: infraCounts.blocked,
+			warn: infraCounts.warn,
+			unknown: infraCounts.unknown
+		},
+		compile: run.qa?.compile ? {
+			status: run.qa.compile.status,
+			stale: run.qa.compile.staleEvidence === true || run.qa.compile.status === 'stale',
+			artifactPath: run.qa.compile.artifactPath,
+			message: run.qa.compile.message
+		} : undefined,
+		route: {
+			pass: routePass,
+			fail: routeFail,
+			shellOnly
+		},
+		business: {
+			pass: businessPass,
+			fail: businessFail,
+			blocked: businessBlocked,
+			missing: assertions.length === 0
+		}
+	};
+}
+
+export function buildAIRunCostPanel(cost?: AIRunCost, run?: AIRun): AIRunCostPanel {
+	const categories = Object.keys(cost?.categories || {}).sort().map((category) => {
+		const entry = cost?.categories?.[category];
+		return {
+			category,
+			totalTokens: Number(entry?.totalTokens || 0),
+			estimatedUsd: Number(entry?.estimatedUsd || 0),
+			count: Number(entry?.count || 0)
+		};
+	});
+	const sources = Object.keys(cost?.sources || {}).sort().map((source) => {
+		const entry = cost?.sources?.[source];
+		return {
+			source,
+			totalTokens: Number(entry?.totalTokens || 0),
+			estimatedUsd: Number(entry?.estimatedUsd || 0),
+			count: Number(entry?.count || 0),
+			manualCount: Number(entry?.manualCount || 0),
+			untrackedCount: Number(entry?.untrackedCount || 0)
+		};
+	});
+	const surfaces = Object.keys(cost?.surfaces || {}).sort().map((surface) => {
+		const entry = cost?.surfaces?.[surface];
+		return {
+			surface,
+			totalTokens: Number(entry?.totalTokens || 0),
+			estimatedUsd: Number(entry?.estimatedUsd || 0),
+			count: Number(entry?.count || 0)
+		};
+	});
+	const costGate = run?.gates?.find((gate) => gate.key === 'cost_observability');
+	const observability = (run?.metadata as any)?.costObservability || costGate?.metadata || {};
+	return {
+		totalTokens: Number(cost?.totalTokens || 0),
+		estimatedUsd: Number(cost?.estimatedUsd || 0),
+		models: Array.isArray(cost?.models) ? cost.models : [],
+		categories,
+		sources,
+		surfaces,
+		manualEstimatedUsd: Number(cost?.manualEstimatedUsd || 0),
+		untrackedEstimatedUsd: Number(cost?.untrackedEstimatedUsd || 0),
+		untrackedWarnings: Array.isArray(cost?.untrackedWarnings) ? cost.untrackedWarnings : [],
+		observabilityStatus: cleanText(observability.status || costGate?.status, 80),
+		missingLedger: observability.missingLedger === true || costGate?.metadata?.missingLedger === true,
+		externalCodexMonitorUsed: observability.externalCodexMonitorUsed === true || costGate?.metadata?.externalCodexMonitorUsed === true,
+		nextAction: cleanText(observability.nextAction, 500)
+	};
+}
+
+export function buildAIRunFalsePassWarnings(run: AIRun): string[] {
+	const warnings = Array.isArray(run.warnings) ? [...run.warnings] : [];
+	if (run.outcome === 'false_pass' && !warnings.includes('false_pass_outcome')) {
+		warnings.push('false_pass_outcome');
+	}
+	if ((run.qa?.outcome === 'route_only_pass' || run.qa?.outcome === 'incomplete') && !warnings.includes('missing_business_assertion')) {
+		warnings.push('missing_business_assertion');
+	}
+	if (run.qa?.routeProbes?.some((probe) => probe.shellOnly === true) && !warnings.includes('shell_only_route')) {
+		warnings.push('shell_only_route');
+	}
+	if (run.outcome === 'release_blocked' && !warnings.includes('release_blocked_after_scorecard')) {
+		warnings.push('release_blocked_after_scorecard');
+	}
+	return warnings;
+}
+
+export function buildAIRunOperatorView(run: AIRun): AIRunOperatorView {
+	return {
+		runId: run.id || (run as any)._id,
+		source: run.source,
+		title: run.title,
+		outcome: run.outcome,
+		failureClass: classifyAIRunFailureClass(run),
+		nextAction: run.nextAction,
+		warnings: buildAIRunFalsePassWarnings(run),
+		timeline: buildAIRunTimeline(run),
+		qaHealth: buildAIRunQaHealth(run),
+		costPanel: buildAIRunCostPanel(run.cost, run),
+		trainingExample: buildAIRunTrainingExampleFromRun(run)
+	};
+}