@resolveio/server-lib 22.3.221 → 22.3.223

package/tests/ai-run-eval.test.ts

@@ -1,132 +0,0 @@
-import * as assert from 'node:assert/strict';
-import {
-	AI_RUN_SEED_EVAL_FIXTURES,
-	buildAIRunDatasetExport,
-	buildAIRunTrainingExampleFromRun,
-	classifyAIRunFailureClass,
-	decideAIRunRetry,
-	evaluateAIRunFixtures
-} from '../src/util/ai-run-evidence-eval';
-import {
-	buildAIRunFalsePassWarnings,
-	buildAIRunOperatorView,
-	buildAIRunQaHealth,
-	buildAIRunTimeline
-} from '../src/util/ai-run-evidence-dashboard';
-import { buildSupportAIRunFromEvidence } from '../src/util/ai-run-evidence-adapters';
-
-const summary = evaluateAIRunFixtures();
-assert.equal(summary.passed, true);
-assert.equal(summary.failedCount, 0);
-assert.equal(summary.total, AI_RUN_SEED_EVAL_FIXTURES.length);
-assert.equal(summary.outcomes.accepted, 2);
-assert.ok(summary.outcomes.false_pass >= 2);
-assert.ok(summary.outcomes.qa_infra_failed >= 2);
-assert.ok(summary.outcomes.release_blocked >= 2);
-
-const routeOnlyFixture = AI_RUN_SEED_EVAL_FIXTURES.find((fixture) => fixture.id === 'qa-route-only-pass');
-assert.ok(routeOnlyFixture);
-assert.equal(routeOnlyFixture.run.outcome, 'false_pass');
-assert.equal(classifyAIRunFailureClass(routeOnlyFixture.run), 'false_pass');
-
-const repeatedRouteOnlyDecision = decideAIRunRetry({
-	currentRun: routeOnlyFixture.run,
-	previousRun: routeOnlyFixture.run,
-	newEvidenceCount: 0,
-	sameFailureCount: 2,
-	unchangedFailureLimit: 2
-});
-assert.equal(repeatedRouteOnlyDecision.action, 'stop');
-assert.equal(repeatedRouteOnlyDecision.repeatedFailure, true);
-
-const puppeteerFixture = AI_RUN_SEED_EVAL_FIXTURES.find((fixture) => fixture.id === 'qa-missing-puppeteer');
-assert.ok(puppeteerFixture);
-const infraDecision = decideAIRunRetry({
-	currentRun: puppeteerFixture.run,
-	newEvidenceCount: 1
-});
-assert.equal(infraDecision.action, 'retry_infra_repair');
-assert.equal(infraDecision.failureClass, 'infra');
-
-const layflatFixture = AI_RUN_SEED_EVAL_FIXTURES.find((fixture) => fixture.id === 'aicoder-layflat-pressure-optimizer');
-assert.ok(layflatFixture);
-const releaseDecision = decideAIRunRetry({
-	currentRun: layflatFixture.run,
-	newEvidenceCount: 1
-});
-assert.equal(releaseDecision.action, 'retry_release_repair');
-
-const example = buildAIRunTrainingExampleFromRun(routeOnlyFixture.run);
-assert.equal(example.outcome, 'false_pass');
-assert.match(example.actionSummary, /failureClass=false_pass/);
-assert.ok(example.evidence.some((entry) => /business/i.test(entry)));
-
-const routeOnlyTimeline = buildAIRunTimeline(routeOnlyFixture.run);
-assert.ok(routeOnlyTimeline.some((entry) => entry.type === 'qa_route_probe'));
-assert.ok(routeOnlyTimeline.some((entry) => entry.type === 'gate' && entry.status === 'blocked'));
-const routeOnlyHealth = buildAIRunQaHealth(routeOnlyFixture.run);
-assert.equal(routeOnlyHealth.business.missing, true);
-assert.equal(routeOnlyHealth.route.pass, 1);
-assert.ok(buildAIRunFalsePassWarnings(routeOnlyFixture.run).includes('false_pass_outcome'));
-const routeOnlyOperatorView = buildAIRunOperatorView(routeOnlyFixture.run);
-assert.equal(routeOnlyOperatorView.failureClass, 'false_pass');
-assert.equal(routeOnlyOperatorView.costPanel.totalTokens, 0);
-assert.equal(routeOnlyOperatorView.trainingExample.outcome, 'false_pass');
-
-const staleProofRun = buildSupportAIRunFromEvidence({
-	ticket: { ticketNumber: '004999', title: 'Stale support proof' },
-	job: {
-		jobId: 'job-stale-proof',
-		businessProofReadiness: {
-			ready: false,
-			status: 'stale',
-			reason: 'support_business_proof_same_fingerprint_as_previous',
-			blockers: ['Business proof did not produce a new proof/artifact fingerprint for this run.'],
-			requiredEvidence: ['fresh AIQaBusinessAssertion artifact'],
-			artifactPaths: ['qa-artifacts/stale-business-proof.json'],
-			proofFingerprint: 'ev-staleproof',
-			artifactFingerprint: 'ev-staleartifact',
-			proofFreshness: 'same_as_previous'
-		}
-	},
-	now: '2026-06-13T00:00:00.000Z'
-});
-const staleProofGate = staleProofRun.gates.find((gate) => gate.key === 'support_business_proof_readiness');
-assert.equal(staleProofGate?.status, 'blocked');
-assert.equal(staleProofGate?.metadata?.proofFingerprint, 'ev-staleproof');
-assert.equal(staleProofGate?.metadata?.proofFreshness, 'same_as_previous');
-
-const manualCodexCostRun = buildSupportAIRunFromEvidence({
-	ticket: { ticketNumber: '004998', title: 'Manual Codex cost visibility' },
-	job: { jobId: 'job-manual-cost', status: 'Running', phase: 'QA' },
-	usageLedger: [{
-		model: 'codex-manual',
-		category: 'manual-codex-monitor',
-		usage_source: 'manual_codex',
-		usage_surface: 'support',
-		is_manual: true,
-		is_untracked: true,
-		untracked_reason: 'Manual Codex session monitored support runner.',
-		cost_estimate: 11.5
-	}]
-});
-const manualCostOperatorView = buildAIRunOperatorView(manualCodexCostRun);
-assert.equal(manualCostOperatorView.costPanel.observabilityStatus, 'warn');
-assert.equal(manualCostOperatorView.costPanel.externalCodexMonitorUsed, true);
-assert.equal(manualCostOperatorView.costPanel.manualEstimatedUsd, 11.5);
-assert.match(String(manualCostOperatorView.costPanel.nextAction || ''), /replace external monitoring/i);
-
-const exportPayload = buildAIRunDatasetExport(AI_RUN_SEED_EVAL_FIXTURES.map((fixture) => fixture.run), {
-	includeRuns: false,
-	redact: true,
-	now: '2026-06-13T00:00:00.000Z'
-});
-assert.equal(exportPayload.generatedAt, '2026-06-13T00:00:00.000Z');
-assert.equal(exportPayload.runs.length, 0);
-assert.equal(exportPayload.trainingExamples.length, AI_RUN_SEED_EVAL_FIXTURES.length);
-assert.equal(exportPayload.summary.byFailureClass.false_pass, 2);
-assert.equal(exportPayload.summary.bySource.support_ticket, 6);
-assert.equal(exportPayload.summary.bySource.aicoder_app, 2);
-assert.equal(exportPayload.summary.bySource.ai_assistant, 2);
-
-console.log('ai run eval tests passed');