@resolveio/server-lib 22.3.125 → 22.3.126

package/tests/ai-run-eval.test.ts

@@ -0,0 +1,88 @@
+import * as assert from 'node:assert/strict';
+import {
+	AI_RUN_SEED_EVAL_FIXTURES,
+	buildAIRunDatasetExport,
+	buildAIRunTrainingExampleFromRun,
+	classifyAIRunFailureClass,
+	decideAIRunRetry,
+	evaluateAIRunFixtures
+} from '../src/util/ai-run-evidence-eval';
+import {
+	buildAIRunFalsePassWarnings,
+	buildAIRunOperatorView,
+	buildAIRunQaHealth,
+	buildAIRunTimeline
+} from '../src/util/ai-run-evidence-dashboard';
+
+const summary = evaluateAIRunFixtures();
+assert.equal(summary.passed, true);
+assert.equal(summary.failedCount, 0);
+assert.equal(summary.total, AI_RUN_SEED_EVAL_FIXTURES.length);
+assert.equal(summary.outcomes.accepted, 2);
+assert.ok(summary.outcomes.false_pass >= 2);
+assert.ok(summary.outcomes.qa_infra_failed >= 2);
+assert.ok(summary.outcomes.release_blocked >= 2);
+
+const routeOnlyFixture = AI_RUN_SEED_EVAL_FIXTURES.find((fixture) => fixture.id === 'qa-route-only-pass');
+assert.ok(routeOnlyFixture);
+assert.equal(routeOnlyFixture.run.outcome, 'false_pass');
+assert.equal(classifyAIRunFailureClass(routeOnlyFixture.run), 'false_pass');
+
+const repeatedRouteOnlyDecision = decideAIRunRetry({
+	currentRun: routeOnlyFixture.run,
+	previousRun: routeOnlyFixture.run,
+	newEvidenceCount: 0,
+	sameFailureCount: 2,
+	unchangedFailureLimit: 2
+});
+assert.equal(repeatedRouteOnlyDecision.action, 'stop');
+assert.equal(repeatedRouteOnlyDecision.repeatedFailure, true);
+
+const puppeteerFixture = AI_RUN_SEED_EVAL_FIXTURES.find((fixture) => fixture.id === 'qa-missing-puppeteer');
+assert.ok(puppeteerFixture);
+const infraDecision = decideAIRunRetry({
+	currentRun: puppeteerFixture.run,
+	newEvidenceCount: 1
+});
+assert.equal(infraDecision.action, 'retry_infra_repair');
+assert.equal(infraDecision.failureClass, 'infra');
+
+const layflatFixture = AI_RUN_SEED_EVAL_FIXTURES.find((fixture) => fixture.id === 'aicoder-layflat-pressure-optimizer');
+assert.ok(layflatFixture);
+const releaseDecision = decideAIRunRetry({
+	currentRun: layflatFixture.run,
+	newEvidenceCount: 1
+});
+assert.equal(releaseDecision.action, 'retry_release_repair');
+
+const example = buildAIRunTrainingExampleFromRun(routeOnlyFixture.run);
+assert.equal(example.outcome, 'false_pass');
+assert.match(example.actionSummary, /failureClass=false_pass/);
+assert.ok(example.evidence.some((entry) => /business/i.test(entry)));
+
+const routeOnlyTimeline = buildAIRunTimeline(routeOnlyFixture.run);
+assert.ok(routeOnlyTimeline.some((entry) => entry.type === 'qa_route_probe'));
+assert.ok(routeOnlyTimeline.some((entry) => entry.type === 'gate' && entry.status === 'blocked'));
+const routeOnlyHealth = buildAIRunQaHealth(routeOnlyFixture.run);
+assert.equal(routeOnlyHealth.business.missing, true);
+assert.equal(routeOnlyHealth.route.pass, 1);
+assert.ok(buildAIRunFalsePassWarnings(routeOnlyFixture.run).includes('false_pass_outcome'));
+const routeOnlyOperatorView = buildAIRunOperatorView(routeOnlyFixture.run);
+assert.equal(routeOnlyOperatorView.failureClass, 'false_pass');
+assert.equal(routeOnlyOperatorView.costPanel.totalTokens, 0);
+assert.equal(routeOnlyOperatorView.trainingExample.outcome, 'false_pass');
+
+const exportPayload = buildAIRunDatasetExport(AI_RUN_SEED_EVAL_FIXTURES.map((fixture) => fixture.run), {
+	includeRuns: false,
+	redact: true,
+	now: '2026-06-13T00:00:00.000Z'
+});
+assert.equal(exportPayload.generatedAt, '2026-06-13T00:00:00.000Z');
+assert.equal(exportPayload.runs.length, 0);
+assert.equal(exportPayload.trainingExamples.length, AI_RUN_SEED_EVAL_FIXTURES.length);
+assert.equal(exportPayload.summary.byFailureClass.false_pass, 2);
+assert.equal(exportPayload.summary.bySource.support_ticket, 6);
+assert.equal(exportPayload.summary.bySource.aicoder_app, 2);
+assert.equal(exportPayload.summary.bySource.ai_assistant, 2);
+
+console.log('ai run eval tests passed');

package/tests/ai-run-evidence.test.ts

@@ -0,0 +1,232 @@
+import * as assert from 'node:assert/strict';
+import {
+	buildAIRun,
+	buildAIRunCost,
+	buildAIQaRun,
+	classifyAIRunOutcome,
+	normalizeOpenAIUsageRowsForCosting,
+	redactAIRunTrainingExample
+} from '../src/util/ai-run-evidence';
+import {
+	buildAICoderAIRunFromEvidence,
+	buildAssistantAIRunFromEvidence,
+	buildSupportAIRunFromEvidence
+} from '../src/util/ai-run-evidence-adapters';
+import { estimateOpenAIUsageCost } from '../src/util/openai-usage-cost';
+
+const routeOnlyQa = buildAIQaRun({
+	infraChecks: [
+		{ name: 'puppeteer_require', status: 'pass' },
+		{ name: 'chrome_executable', status: 'pass' }
+	],
+	compile: { status: 'pass', artifactPath: '.build-output/build-client.log' },
+	routeProbes: [{
+		route: '/billing/dashboard',
+		status: 'pass',
+		screenshot: 'qa-artifacts/route-ready.jpg',
+		caption: 'Route loaded.'
+	}]
+});
+
+assert.equal(routeOnlyQa.outcome, 'route_only_pass');
+assert.equal(routeOnlyQa.gateResults.some((gate) => gate.key === 'qa_business_assertion' && gate.status === 'blocked'), true);
+assert.equal(classifyAIRunOutcome({ qa: routeOnlyQa }).outcome, 'qa_incomplete');
+assert.equal(classifyAIRunOutcome({ qa: routeOnlyQa, scorecardPassed: true }).outcome, 'false_pass');
+
+const infraFailedQa = buildAIQaRun({
+	infraChecks: [
+		{ name: 'puppeteer_require', status: 'blocked', message: 'Unable to require puppeteer.' }
+	]
+});
+assert.equal(infraFailedQa.outcome, 'infra_failed');
+assert.equal(classifyAIRunOutcome({ qa: infraFailedQa }).outcome, 'qa_infra_failed');
+
+const compileFailedQa = buildAIQaRun({
+	infraChecks: [{ name: 'chrome_executable', status: 'pass' }],
+	compile: { status: 'stale', staleEvidence: true, artifactPath: '.build-output/build-client.log' }
+});
+assert.equal(compileFailedQa.outcome, 'compile_failed');
+assert.equal(classifyAIRunOutcome({ qa: compileFailedQa }).outcome, 'build_failed');
+
+const businessPassedQa = buildAIQaRun({
+	infraChecks: [{ name: 'chrome_launch', status: 'pass' }],
+	compile: { status: 'pass' },
+	routeProbes: [{ route: '/report', status: 'pass' }],
+	businessAssertions: [{
+		assertion: 'Report exports include seeded rows.',
+		status: 'pass',
+		dataProof: '3 seeded rows rendered and exported.',
+		artifactPaths: ['qa-artifacts/report-proof.json']
+	}]
+});
+assert.equal(businessPassedQa.outcome, 'business_assertion_passed');
+assert.equal(classifyAIRunOutcome({
+	qa: businessPassedQa,
+	status: 'Passed',
+	phase: 'COMPLETE',
+	scorecardPassed: true,
+	deployStatus: 'pass',
+	publishStatus: 'published',
+	sampleDataStatus: 'pass'
+}).outcome, 'accepted');
+
+const releaseBlocked = classifyAIRunOutcome({
+	qa: businessPassedQa,
+	scorecardPassed: true,
+	deployStatus: 'Backend health smoke failed',
+	publishStatus: 'No such remote origin',
+	sampleDataStatus: 'empty collections detected'
+});
+assert.equal(releaseBlocked.outcome, 'release_blocked');
+assert.ok(releaseBlocked.gates.some((gate) => gate.key === 'release' && gate.status === 'fail'));
+
+const builtRun = buildAIRun({
+	source: 'aicoder_app',
+	sourceIds: { appId: 'app-1', jobId: 'job-1' },
+	title: 'Layflat Pressure Optimizer',
+	status: 'Passed',
+	phase: 'COMPLETE',
+	events: [],
+	qa: businessPassedQa,
+	scorecardPassed: true,
+	deployStatus: 'pass',
+	publishStatus: 'published',
+	sampleDataStatus: 'pass'
+});
+assert.equal(builtRun.outcome, 'accepted');
+assert.equal(builtRun.sourceIds.appId, 'app-1');
+
+const cost = buildAIRunCost([
+	{ model: 'gpt-5.3-codex', category: 'review', input_tokens: 1000, cached_input_tokens: 100, output_tokens: 250, cost_estimate: 1.23456789 },
+	{ model: 'gpt-5.4-mini', category: 'qa', inputTokens: 200, outputTokens: 50, estimatedUsd: 0.12 }
+]);
+assert.equal(cost.inputTokens, 1200);
+assert.equal(cost.cachedInputTokens, 100);
+assert.equal(cost.outputTokens, 300);
+assert.equal(cost.categories.review.count, 1);
+assert.deepEqual(cost.models, ['gpt-5.3-codex', 'gpt-5.4-mini']);
+
+const normalizedUsageRows = normalizeOpenAIUsageRowsForCosting([
+	{
+		model: 'gpt-5.5',
+		category: 'ai-dashboard-codex:master:file-body',
+		id_conversation: 'thread-1',
+		timestamp: '2026-06-13T10:00:00.000Z',
+		input_tokens: 1000,
+		cached_input_tokens: 800,
+		output_tokens: 100,
+		total_tokens: 1100,
+		cost_estimate: 1
+	},
+	{
+		model: 'gpt-5.5',
+		category: 'ai-dashboard-codex:master:file-body',
+		id_conversation: 'thread-1',
+		timestamp: '2026-06-13T10:01:00.000Z',
+		input_tokens: 1500,
+		cached_input_tokens: 1000,
+		output_tokens: 140,
+		total_tokens: 1640,
+		cost_estimate: 1.4
+	}
+]);
+assert.equal(normalizedUsageRows[1].input_tokens, 500);
+assert.equal(normalizedUsageRows[1].cached_input_tokens, 200);
+assert.equal(normalizedUsageRows[1].output_tokens, 40);
+assert.equal(normalizedUsageRows[1].cost_estimate, 0.4);
+
+const cumulativeCost = buildAIRunCost([
+	{
+		model: 'gpt-5.5',
+		category: 'ai-dashboard-codex:master:file-body',
+		id_conversation: 'thread-1',
+		timestamp: '2026-06-13T10:00:00.000Z',
+		input_tokens: 1000,
+		cached_input_tokens: 800,
+		output_tokens: 100,
+		total_tokens: 1100,
+		cost_estimate: 1
+	},
+	{
+		model: 'gpt-5.5',
+		category: 'ai-dashboard-codex:master:file-body',
+		id_conversation: 'thread-1',
+		timestamp: '2026-06-13T10:01:00.000Z',
+		input_tokens: 1500,
+		cached_input_tokens: 1000,
+		output_tokens: 140,
+		total_tokens: 1640,
+		cost_estimate: 1.4
+	}
+]);
+assert.equal(cumulativeCost.inputTokens, 1500);
+assert.equal(cumulativeCost.cachedInputTokens, 1000);
+assert.equal(cumulativeCost.outputTokens, 140);
+assert.equal(cumulativeCost.estimatedUsd, 1.4);
+assert.equal(estimateOpenAIUsageCost('gpt-5.4-2026-03-10', 1000, 100, 500), 0.002875);
+assert.equal(estimateOpenAIUsageCost('gpt-5.3-codex', 1000, 100, 500), 0.00575);
+assert.equal(estimateOpenAIUsageCost('gpt-5.4-codex', 1000, 100, 500), 0.002875);
+assert.equal(estimateOpenAIUsageCost('codex-mini-latest', 1000, 100, 500), 0.000863);
+
+const redacted = redactAIRunTrainingExample({
+	runId: 'run-1',
+	source: 'ai_assistant',
+	inputSummary: 'Mongo URL mongodb://user:password@host/db and email test@example.com',
+	actionSummary: 'Used key sk-12345678901234567890',
+	outcome: 'rejected',
+	evidence: ['ghp_12345678901234567890 appears here']
+});
+assert.match(redacted.inputSummary, /\[redacted-mongo-url\]/);
+assert.match(redacted.inputSummary, /\[redacted-email\]/);
+assert.match(redacted.actionSummary, /\[redacted-secret\]/);
+assert.match(redacted.evidence[0], /\[redacted-secret\]/);
+
+const supportRun = buildSupportAIRunFromEvidence({
+	ticket: { _id: 'ticket-004421', ticketNumber: '004421', title: 'QA route only false pass' },
+	job: { _id: 'support-job-1', status: 'Running', phase: 'QA' },
+	taskEvents: [{ message: 'Route loaded in browser.', phase: 'qa' }],
+	qaEvidence: {
+		infraChecks: [{ name: 'puppeteer_require', status: 'pass' }],
+		compile: { status: 'pass' },
+		routeProbes: [{ route: '/support/004421', status: 'pass', screenshot: 'qa-artifacts/route.jpg' }],
+		scorecardPassed: true
+	}
+});
+assert.equal(supportRun.source, 'support_ticket');
+assert.equal(supportRun.sourceIds.ticketNumber, '004421');
+assert.equal(supportRun.outcome, 'false_pass');
+
+const aicoderRun = buildAICoderAIRunFromEvidence({
+	app: { _id: 'layflat-app', name: 'Layflat Pressure Optimizer', domain: 'layflat.resolveio.app' },
+	job: { _id: 'aicoder-job-1', phase: 'COMPLETE', status: 'Passed' },
+	qaEvidence: {
+		scorecardPassed: true,
+		deployStatus: 'pass',
+		publishStatus: 'published',
+		sampleDataStatus: 'empty seed collections',
+		compile: { status: 'pass' },
+		routeProbes: [{ route: '/optimizer', status: 'pass' }],
+		businessAssertions: [{
+			assertion: 'Optimizer computes pressure drop from seeded hose data.',
+			status: 'pass',
+			dataProof: 'Seeded hose row produced a pressure result.'
+		}]
+	}
+});
+assert.equal(aicoderRun.source, 'aicoder_app');
+assert.equal(aicoderRun.outcome, 'release_blocked');
+
+const assistantRun = buildAssistantAIRunFromEvidence({
+	conversation: { _id: 'conversation-1', title: 'Invoice query', status: 'Closed', accepted: true },
+	messages: [{ role: 'assistant', content: 'Used the current date window.' }],
+	correctnessChecks: [{
+		assertion: 'Date window uses the actual current month.',
+		status: 'pass',
+		expected: 'June 2026',
+		observed: 'June 2026'
+	}]
+});
+assert.equal(assistantRun.source, 'ai_assistant');
+assert.equal(assistantRun.outcome, 'accepted');
+
+console.log('ai run evidence tests passed');