@resolveio/server-lib 22.3.150 → 22.3.152

package/tests/ai-run-evidence.test.ts

@@ -1,348 +0,0 @@
-import * as assert from 'node:assert/strict';
-import {
-	buildAIRun,
-	buildAIRunCost,
-	buildAIQaRun,
-	classifyAIRunOutcome,
-	normalizeOpenAIUsageRowsForCosting,
-	redactAIRunTrainingExample
-} from '../src/util/ai-run-evidence';
-import {
-	buildAICoderAIRunFromEvidence,
-	buildAssistantAIRunFromEvidence,
-	buildSupportAIRunFromEvidence
-} from '../src/util/ai-run-evidence-adapters';
-import { estimateOpenAIUsageCost } from '../src/util/openai-usage-cost';
-
-const routeOnlyQa = buildAIQaRun({
-	infraChecks: [
-		{ name: 'puppeteer_require', status: 'pass' },
-		{ name: 'chrome_executable', status: 'pass' }
-	],
-	compile: { status: 'pass', artifactPath: '.build-output/build-client.log' },
-	routeProbes: [{
-		route: '/billing/dashboard',
-		status: 'pass',
-		screenshot: 'qa-artifacts/route-ready.jpg',
-		caption: 'Route loaded.'
-	}]
-});
-
-assert.equal(routeOnlyQa.outcome, 'route_only_pass');
-assert.equal(routeOnlyQa.gateResults.some((gate) => gate.key === 'qa_business_assertion' && gate.status === 'blocked'), true);
-assert.equal(classifyAIRunOutcome({ qa: routeOnlyQa }).outcome, 'qa_incomplete');
-assert.equal(classifyAIRunOutcome({ qa: routeOnlyQa, scorecardPassed: true }).outcome, 'false_pass');
-
-const infraFailedQa = buildAIQaRun({
-	infraChecks: [
-		{ name: 'puppeteer_require', status: 'blocked', message: 'Unable to require puppeteer.' }
-	]
-});
-assert.equal(infraFailedQa.outcome, 'infra_failed');
-assert.equal(classifyAIRunOutcome({ qa: infraFailedQa }).outcome, 'qa_infra_failed');
-
-const compileFailedQa = buildAIQaRun({
-	infraChecks: [{ name: 'chrome_executable', status: 'pass' }],
-	compile: { status: 'stale', staleEvidence: true, artifactPath: '.build-output/build-client.log' }
-});
-assert.equal(compileFailedQa.outcome, 'compile_failed');
-assert.equal(classifyAIRunOutcome({ qa: compileFailedQa }).outcome, 'build_failed');
-
-const businessPassedQa = buildAIQaRun({
-	infraChecks: [{ name: 'chrome_launch', status: 'pass' }],
-	compile: { status: 'pass' },
-	routeProbes: [{ route: '/report', status: 'pass' }],
-	businessAssertions: [{
-		assertion: 'Report exports include seeded rows.',
-		status: 'pass',
-		dataProof: '3 seeded rows rendered and exported.',
-		artifactPaths: ['qa-artifacts/report-proof.json']
-	}]
-});
-assert.equal(businessPassedQa.outcome, 'business_assertion_passed');
-assert.equal(classifyAIRunOutcome({
-	qa: businessPassedQa,
-	status: 'Passed',
-	phase: 'COMPLETE',
-	scorecardPassed: true,
-	deployStatus: 'pass',
-	publishStatus: 'published',
-	sampleDataStatus: 'pass'
-}).outcome, 'accepted');
-
-const releaseBlocked = classifyAIRunOutcome({
-	qa: businessPassedQa,
-	scorecardPassed: true,
-	deployStatus: 'Backend health smoke failed',
-	publishStatus: 'No such remote origin',
-	sampleDataStatus: 'empty collections detected'
-});
-assert.equal(releaseBlocked.outcome, 'release_blocked');
-assert.ok(releaseBlocked.gates.some((gate) => gate.key === 'release' && gate.status === 'fail'));
-
-const builtRun = buildAIRun({
-	source: 'aicoder_app',
-	sourceIds: { appId: 'app-1', jobId: 'job-1' },
-	title: 'Layflat Pressure Optimizer',
-	status: 'Passed',
-	phase: 'COMPLETE',
-	events: [],
-	qa: businessPassedQa,
-	scorecardPassed: true,
-	deployStatus: 'pass',
-	publishStatus: 'published',
-	sampleDataStatus: 'pass'
-});
-assert.equal(builtRun.outcome, 'accepted');
-assert.equal(builtRun.sourceIds.appId, 'app-1');
-
-const cost = buildAIRunCost([
-	{ model: 'gpt-5.3-codex', category: 'review', input_tokens: 1000, cached_input_tokens: 100, output_tokens: 250, cost_estimate: 1.23456789 },
-	{ model: 'gpt-5.4-mini', category: 'qa', inputTokens: 200, outputTokens: 50, estimatedUsd: 0.12 },
-	{
-		model: 'gpt-5.3-codex',
-		category: 'manual-monitoring',
-		usage_source: 'manual_codex',
-		usage_surface: 'aicoder',
-		cost_basis: 'manual_estimate',
-		is_manual: true,
-		is_untracked: true,
-		untracked_reason: 'Sidecar Codex session watched the Layflat builder.',
-		input_tokens: 3000,
-		output_tokens: 700,
-		cost_estimate: 7.5
-	}
-]);
-assert.equal(cost.inputTokens, 4200);
-assert.equal(cost.cachedInputTokens, 100);
-assert.equal(cost.outputTokens, 1000);
-assert.equal(cost.categories.review.count, 1);
-assert.deepEqual(cost.models, ['gpt-5.3-codex', 'gpt-5.4-mini']);
-assert.equal(cost.sources?.manual_codex.estimatedUsd, 7.5);
-assert.equal(cost.sources?.manual_codex.manualCount, 1);
-assert.equal(cost.sources?.manual_codex.untrackedCount, 1);
-assert.equal(cost.surfaces?.aicoder.estimatedUsd, 7.5);
-assert.equal(cost.manualEstimatedUsd, 7.5);
-assert.equal(cost.untrackedEstimatedUsd, 7.5);
-assert.equal(cost.untrackedWarnings?.includes('Sidecar Codex session watched the Layflat builder.'), true);
-
-const costOnlyManual = buildAIRunCost([
-	{
-		model: 'codex-manual',
-		category: 'manual-codex',
-		usage_source: 'manual_codex',
-		usage_surface: 'support',
-		cost_basis: 'manual_estimate',
-		is_manual: true,
-		is_untracked: true,
-		untracked_reason: 'Codex sidecar monitored support runner 004421.',
-		input_tokens: 0,
-		output_tokens: 0,
-		total_tokens: 0,
-		cost_estimate: 23.75
-	}
-]);
-assert.equal(costOnlyManual.totalTokens, 0);
-assert.equal(costOnlyManual.estimatedUsd, 23.75);
-assert.equal(costOnlyManual.manualEstimatedUsd, 23.75);
-assert.equal(costOnlyManual.untrackedEstimatedUsd, 23.75);
-assert.equal(costOnlyManual.sources?.manual_codex.count, 1);
-assert.equal(costOnlyManual.surfaces?.support.estimatedUsd, 23.75);
-
-const normalizedUsageRows = normalizeOpenAIUsageRowsForCosting([
-	{
-		model: 'gpt-5.5',
-		category: 'ai-dashboard-codex:master:file-body',
-		id_conversation: 'thread-1',
-		timestamp: '2026-06-13T10:00:00.000Z',
-		input_tokens: 1000,
-		cached_input_tokens: 800,
-		output_tokens: 100,
-		total_tokens: 1100,
-		cost_estimate: 1
-	},
-	{
-		model: 'gpt-5.5',
-		category: 'ai-dashboard-codex:master:file-body',
-		id_conversation: 'thread-1',
-		timestamp: '2026-06-13T10:01:00.000Z',
-		input_tokens: 1500,
-		cached_input_tokens: 1000,
-		output_tokens: 140,
-		total_tokens: 1640,
-		cost_estimate: 1.4
-	}
-]);
-assert.equal(normalizedUsageRows[1].input_tokens, 500);
-assert.equal(normalizedUsageRows[1].cached_input_tokens, 200);
-assert.equal(normalizedUsageRows[1].output_tokens, 40);
-assert.equal(normalizedUsageRows[1].cost_estimate, 0.4);
-
-const cumulativeCost = buildAIRunCost([
-	{
-		model: 'gpt-5.5',
-		category: 'ai-dashboard-codex:master:file-body',
-		id_conversation: 'thread-1',
-		timestamp: '2026-06-13T10:00:00.000Z',
-		input_tokens: 1000,
-		cached_input_tokens: 800,
-		output_tokens: 100,
-		total_tokens: 1100,
-		cost_estimate: 1
-	},
-	{
-		model: 'gpt-5.5',
-		category: 'ai-dashboard-codex:master:file-body',
-		id_conversation: 'thread-1',
-		timestamp: '2026-06-13T10:01:00.000Z',
-		input_tokens: 1500,
-		cached_input_tokens: 1000,
-		output_tokens: 140,
-		total_tokens: 1640,
-		cost_estimate: 1.4
-	}
-]);
-assert.equal(cumulativeCost.inputTokens, 1500);
-assert.equal(cumulativeCost.cachedInputTokens, 1000);
-assert.equal(cumulativeCost.outputTokens, 140);
-assert.equal(cumulativeCost.estimatedUsd, 1.4);
-assert.equal(estimateOpenAIUsageCost('gpt-5.4-2026-03-10', 1000, 100, 500), 0.002875);
-assert.equal(estimateOpenAIUsageCost('gpt-5.3-codex', 1000, 100, 500), 0.00575);
-assert.equal(estimateOpenAIUsageCost('gpt-5.4-codex', 1000, 100, 500), 0.002875);
-assert.equal(estimateOpenAIUsageCost('codex-mini-latest', 1000, 100, 500), 0.000863);
-
-const redacted = redactAIRunTrainingExample({
-	runId: 'run-1',
-	source: 'ai_assistant',
-	inputSummary: 'Mongo URL mongodb://user:password@host/db and email test@example.com',
-	actionSummary: 'Used key sk-12345678901234567890',
-	outcome: 'rejected',
-	evidence: ['ghp_12345678901234567890 appears here']
-});
-assert.match(redacted.inputSummary, /\[redacted-mongo-url\]/);
-assert.match(redacted.inputSummary, /\[redacted-email\]/);
-assert.match(redacted.actionSummary, /\[redacted-secret\]/);
-assert.match(redacted.evidence[0], /\[redacted-secret\]/);
-
-const supportRun = buildSupportAIRunFromEvidence({
-	ticket: { _id: 'ticket-004421', ticketNumber: '004421', title: 'QA route only false pass' },
-	job: {
-		_id: 'support-job-1',
-		status: 'Running',
-		phase: 'QA',
-		supportV5DiagnosisGate: {
-			issue_class: 'filter_query_mismatch',
-			status: 'passed',
-			accepted_hypothesis: {
-				statement: 'The ticket list query ignores the customer filter.'
-			},
-			owner_files: ['server/src/publications/support-tickets.ts'],
-			proof_plan: {
-				before: 'Customer A ticket appears while filtered to Customer B.',
-				action: 'Apply the Customer B filter and reload the ticket list.',
-				after: 'Only Customer B tickets are listed.',
-				business_assertion: 'The wrong-customer ticket is absent after filtering.'
-			}
-		}
-	},
-	taskEvents: [{ message: 'Route loaded in browser.', phase: 'qa' }],
-	qaEvidence: {
-		infraChecks: [{ name: 'puppeteer_require', status: 'pass' }],
-		compile: { status: 'pass' },
-		routeProbes: [{ route: '/support/004421', status: 'pass', screenshot: 'qa-artifacts/route.jpg' }],
-		scorecardPassed: true
-	}
-});
-assert.equal(supportRun.source, 'support_ticket');
-assert.equal(supportRun.sourceIds.ticketNumber, '004421');
-assert.equal(supportRun.outcome, 'false_pass');
-assert.equal(supportRun.metadata?.diagnosis?.issueClass, 'filter_query_mismatch');
-assert.deepEqual(supportRun.metadata?.diagnosis?.ownerFiles, ['server/src/publications/support-tickets.ts']);
-assert.equal(supportRun.events.some((event) => event.category === 'diagnosis_gate'), true);
-
-const supportGenericBusinessProofRun = buildSupportAIRunFromEvidence({
-	ticket: { _id: 'ticket-004421', ticketNumber: '004421', title: 'Generic support proof' },
-	job: {
-		_id: 'support-job-generic-proof',
-		status: 'Closed',
-		phase: 'COMPLETE',
-		supportV5DiagnosisGate: supportRun.metadata?.diagnosis
-	},
-	qaEvidence: {
-		infraChecks: [{ name: 'puppeteer_require', status: 'pass' }],
-		compile: { status: 'pass' },
-		routeProbes: [{ route: '/support', status: 'pass' }],
-		scorecardPassed: true,
-		businessAssertions: [{
-			assertion: 'The route loads and the ticket page has visible controls.',
-			status: 'pass',
-			route: '/support',
-			action: 'Open the ticket page.',
-			expected: 'Visible controls render.',
-			observed: 'Visible controls rendered.',
-			dataProof: 'Screenshot showed page content.'
-		}]
-	}
-});
-assert.equal(supportGenericBusinessProofRun.outcome, 'false_pass');
-assert.equal(supportGenericBusinessProofRun.gates.some((gate) => gate.key === 'support_diagnosis_business_proof' && gate.status === 'blocked'), true);
-
-const supportBusinessProofRun = buildSupportAIRunFromEvidence({
-	ticket: { _id: 'ticket-004430', ticketNumber: '004430', title: 'Filtered support ticket list' },
-	job: {
-		_id: 'support-job-2',
-		status: 'Closed',
-		phase: 'COMPLETE',
-		supportV5DiagnosisGate: supportRun.metadata?.diagnosis
-	},
-	qaEvidence: {
-		infraChecks: [{ name: 'puppeteer_require', status: 'pass' }],
-		compile: { status: 'pass' },
-		routeProbes: [{ route: '/support', status: 'pass' }],
-		businessAssertions: [{
-			assertion: 'Wrong-customer ticket is absent after applying the customer filter.',
-			status: 'pass',
-			route: '/support',
-			action: 'Apply customer filter and reload.',
-			expected: 'Only Customer B tickets are visible.',
-			observed: 'Customer A ticket was not rendered.',
-			dataProof: 'DOM and query proof matched the diagnosis proof plan.'
-		}]
-	}
-});
-assert.equal(supportBusinessProofRun.outcome, 'accepted');
-
-const aicoderRun = buildAICoderAIRunFromEvidence({
-	app: { _id: 'layflat-app', name: 'Layflat Pressure Optimizer', domain: 'layflat.resolveio.app' },
-	job: { _id: 'aicoder-job-1', phase: 'COMPLETE', status: 'Passed' },
-	qaEvidence: {
-		scorecardPassed: true,
-		deployStatus: 'pass',
-		publishStatus: 'published',
-		sampleDataStatus: 'empty seed collections',
-		compile: { status: 'pass' },
-		routeProbes: [{ route: '/optimizer', status: 'pass' }],
-		businessAssertions: [{
-			assertion: 'Optimizer computes pressure drop from seeded hose data.',
-			status: 'pass',
-			dataProof: 'Seeded hose row produced a pressure result.'
-		}]
-	}
-});
-assert.equal(aicoderRun.source, 'aicoder_app');
-assert.equal(aicoderRun.outcome, 'release_blocked');
-
-const assistantRun = buildAssistantAIRunFromEvidence({
-	conversation: { _id: 'conversation-1', title: 'Invoice query', status: 'Closed', accepted: true },
-	messages: [{ role: 'assistant', content: 'Used the current date window.' }],
-	correctnessChecks: [{
-		assertion: 'Date window uses the actual current month.',
-		status: 'pass',
-		expected: 'June 2026',
-		observed: 'June 2026'
-	}]
-});
-assert.equal(assistantRun.source, 'ai_assistant');
-assert.equal(assistantRun.outcome, 'accepted');
-
-console.log('ai run evidence tests passed');