@resolveio/server-lib 22.3.148 → 22.3.150

package/tests/ai-manager-recovery-checkpoint.test.ts

@@ -0,0 +1,595 @@
+import * as assert from 'node:assert/strict';
+import {
+	buildResolveIOAIManagerRecoveryCheckpoint,
+	buildResolveIOAIManagerRecoveryActionPacket,
+	buildResolveIOAIManagerRecoveryEvidenceProbe,
+	buildResolveIOAIManagerRecoveryPlan,
+	appendResolveIOAIManagerRecoveryActionDispatch,
+	buildResolveIOAIManagerRecoveryExecutionDirective,
+	buildResolveIOAIManagerHotfixFirstReleasePolicy,
+	buildResolveIOAIManagerRecoveryReplayMatrix,
+	decideResolveIOAIManagerRecoveryActionDispatch,
+	decideResolveIOAIManagerRecoveryGate,
+	decideResolveIOAIManagerPolicy,
+	isResolveIOAIManagerSafeAutoDispatch
+} from '../src/util/ai-runner-manager-policy';
+
+const parkedPlan = buildResolveIOAIManagerRecoveryPlan({
+	action: 'park_repeated_failure',
+	reason: 'same blocker repeated without new proof',
+	failureClass: 'product_code',
+	lane: 'repair',
+	stepType: 'repair',
+	blocker: 'Save action still throws TypeError.',
+	artifactPaths: ['qa/save-before.log'],
+	maxSameFailureRepeats: 2,
+	productRepairFailure: true
+});
+
+const parkedCheckpoint = buildResolveIOAIManagerRecoveryCheckpoint({
+	plan: parkedPlan,
+	current: {
+		lane: 'repair',
+		stepType: 'repair',
+		failureClass: 'product_code',
+		blocker: 'Save action still throws TypeError.',
+		evidenceHash: 'save-typeerror-before',
+		artifactPaths: ['qa/save-before.log']
+	},
+	now: '2026-06-15T00:00:00.000Z'
+});
+
+assert.equal(parkedCheckpoint.status, 'parked');
+assert.equal(parkedCheckpoint.recoveryClass, 'blocked_until_new_evidence');
+assert.equal(parkedCheckpoint.allowedAction, 'collect_new_evidence_only');
+
+const parkedEvidenceProbe = buildResolveIOAIManagerRecoveryEvidenceProbe({
+	checkpoint: parkedCheckpoint
+});
+const parkedAction = buildResolveIOAIManagerRecoveryActionPacket({
+	checkpoint: parkedCheckpoint,
+	probe: parkedEvidenceProbe
+});
+
+assert.equal(parkedEvidenceProbe.evidenceOnly, true);
+assert.equal(parkedEvidenceProbe.productRepairAllowed, false);
+assert.equal(parkedEvidenceProbe.expensiveModelAllowed, false);
+assert.ok(parkedEvidenceProbe.steps.some((step) => step.kind === 'rerun_same_gate'));
+assert.ok(parkedEvidenceProbe.stopConditions.some((entry) => /product-code repair/i.test(entry)));
+assert.equal(parkedAction.mode, 'collect_evidence');
+assert.equal(parkedAction.evidenceOnly, true);
+assert.equal(parkedAction.productRepairAllowed, false);
+assert.equal(parkedAction.retryPolicy.requireNewEvidence, true);
+assert.ok(parkedAction.nextCommands.some((command) => /rerun_same_gate|snapshot_current_blocker/i.test(command)));
+
+assert.equal(isResolveIOAIManagerSafeAutoDispatch({ dispatchAction: 'run_evidence_probe' }), true);
+assert.equal(isResolveIOAIManagerSafeAutoDispatch({ dispatchAction: 'run_release_repair' }), true);
+assert.equal(isResolveIOAIManagerSafeAutoDispatch({ dispatchAction: 'run_journey_contract_repair' }), true);
+assert.equal(isResolveIOAIManagerSafeAutoDispatch({ dispatchAction: 'run_targeted_product_repair' }), false);
+assert.equal(isResolveIOAIManagerSafeAutoDispatch({ dispatchAction: 'run_business_assertion_repair' }), false);
+assert.equal(isResolveIOAIManagerSafeAutoDispatch({
+	dispatchAction: 'run_release_repair',
+	includeReleaseRepair: false
+}), false);
+
+const duplicateReleasePolicy = buildResolveIOAIManagerHotfixFirstReleasePolicy({
+	surface: 'aicoder_builder',
+	deployFingerprint: 'app-v6-build-sha',
+	lastDeployFingerprint: 'app-v6-build-sha',
+	deployStatus: 'completed',
+	publishStatus: 'published'
+});
+
+assert.equal(duplicateReleasePolicy.policy, 'hotfix_first');
+assert.equal(duplicateReleasePolicy.recommendedAction, 'block_duplicate_deploy');
+assert.equal(duplicateReleasePolicy.duplicateDeployBlocked, true);
+assert.equal(duplicateReleasePolicy.fullDeployAllowed, false);
+assert.equal(duplicateReleasePolicy.forceDeployRequiredToRepeat, true);
+assert.ok(duplicateReleasePolicy.allowedActions.includes('hotfix_release_artifact'));
+assert.ok(duplicateReleasePolicy.forbiddenActions.some((entry) => /duplicate deploy/i.test(entry)));
+assert.equal(duplicateReleasePolicy.hotfixPlan.fullDeployAvoided, true);
+assert.equal(duplicateReleasePolicy.hotfixPlan.recommendedChannel, 'static_ui');
+assert.ok(duplicateReleasePolicy.hotfixPlan.steps.some((step) => step.channel === 'static_ui' && step.requiresFullDeploy === false));
+assert.ok(duplicateReleasePolicy.hotfixPlan.steps.some((step) => step.channel === 'backend_js' && step.requiredEvidence.some((entry) => /compiled JS artifact/i.test(entry))));
+
+const successfulAICoderDeployPolicy = buildResolveIOAIManagerHotfixFirstReleasePolicy({
+	surface: 'aicoder_deploy_dashboard',
+	deployFingerprint: 'client:bucket:apps/layflat/:',
+	lastDeployFingerprint: 'client:bucket:apps/layflat/:',
+	deployStatus: 'Success',
+	publishStatus: 'Client deploy from s3://bucket/apps/layflat/'
+});
+
+assert.equal(successfulAICoderDeployPolicy.recommendedAction, 'block_duplicate_deploy');
+assert.equal(successfulAICoderDeployPolicy.duplicateDeployBlocked, true);
+assert.equal(successfulAICoderDeployPolicy.fullDeployAllowed, false);
+
+const duplicatePublishPolicy = buildResolveIOAIManagerHotfixFirstReleasePolicy({
+	surface: 'aicoder_publish_gate',
+	publishFingerprint: 'npm:@resolveio/client-lib-aicoder-dashboard:21.0.16:22c7d73a',
+	lastPublishFingerprint: 'npm:@resolveio/client-lib-aicoder-dashboard:21.0.16:22c7d73a',
+	publishStatus: 'failed: registry denied publish PUT'
+});
+
+assert.equal(duplicatePublishPolicy.recommendedAction, 'block_duplicate_publish');
+assert.equal(duplicatePublishPolicy.duplicatePublishBlocked, true);
+assert.equal(duplicatePublishPolicy.fullDeployAllowed, false);
+assert.equal(duplicatePublishPolicy.forcePublishRequiredToRepeat, true);
+assert.ok(duplicatePublishPolicy.allowedActions.includes('request_force_publish_with_reason'));
+assert.equal(duplicatePublishPolicy.hotfixPlan.label, 'Hotfix Or Force Publish');
+assert.equal(duplicatePublishPolicy.hotfixPlan.recommendedChannel, 'release_artifact');
+assert.ok(duplicatePublishPolicy.forbiddenActions.some((entry) => /duplicate publish/i.test(entry)));
+
+const completedPublishPolicy = buildResolveIOAIManagerHotfixFirstReleasePolicy({
+	surface: 'support_push_master',
+	publishFingerprint: 'support-pr-123-merge-sha',
+	lastPublishFingerprint: 'support-pr-123-merge-sha',
+	publishStatus: 'completed',
+	hasNewArtifact: true
+});
+
+assert.equal(completedPublishPolicy.duplicatePublishBlocked, false);
+assert.notEqual(completedPublishPolicy.recommendedAction, 'block_duplicate_publish');
+
+const forcedPublishPolicy = buildResolveIOAIManagerHotfixFirstReleasePolicy({
+	surface: 'aicoder_publish_gate',
+	publishFingerprint: 'npm:@resolveio/client-lib-aicoder-dashboard:21.0.16:22c7d73a',
+	lastPublishFingerprint: 'npm:@resolveio/client-lib-aicoder-dashboard:21.0.16:22c7d73a',
+	publishStatus: 'failed: registry denied publish PUT',
+	forcePublish: true
+});
+
+assert.equal(forcedPublishPolicy.recommendedAction, 'review_force_deploy');
+assert.equal(forcedPublishPolicy.duplicatePublishBlocked, false);
+assert.equal(forcedPublishPolicy.fullDeployAllowed, true);
+assert.match(forcedPublishPolicy.reason, /force_publish/);
+
+const forcedReleasePolicy = buildResolveIOAIManagerHotfixFirstReleasePolicy({
+	surface: 'support_runner',
+	deployFingerprint: 'support-pr-123-sha',
+	lastDeployFingerprint: 'support-pr-123-sha',
+	deployStatus: 'completed',
+	forceDeploy: true
+});
+
+assert.equal(forcedReleasePolicy.recommendedAction, 'review_force_deploy');
+assert.equal(forcedReleasePolicy.duplicateDeployBlocked, false);
+assert.equal(forcedReleasePolicy.fullDeployAllowed, true);
+assert.ok(forcedReleasePolicy.requiredEvidence.some((entry) => /force_deploy/i.test(entry)));
+
+const newArtifactReleasePolicy = buildResolveIOAIManagerHotfixFirstReleasePolicy({
+	surface: 'aicoder_app_dashboard_terminal',
+	deployFingerprint: 'new-app-build-sha',
+	lastDeployFingerprint: 'old-app-build-sha',
+	deployStatus: 'ready',
+	publishStatus: 'ready',
+	hasNewArtifact: true
+});
+
+assert.equal(newArtifactReleasePolicy.recommendedAction, 'deploy_new_artifact_after_release_gate');
+assert.equal(newArtifactReleasePolicy.fullDeployAllowed, true);
+assert.equal(newArtifactReleasePolicy.hotfixPreferred, false);
+assert.equal(newArtifactReleasePolicy.hotfixPlan.fullDeployAvoided, false);
+assert.equal(newArtifactReleasePolicy.hotfixPlan.recommendedChannel, 'new_artifact_deploy');
+
+const releasePlan = buildResolveIOAIManagerRecoveryPlan({
+	action: 'continue',
+	reason: 'domain publish failed after business proof',
+	failureClass: 'release',
+	lane: 'publish',
+	stepType: 'test_deploy',
+	blocker: 'CloudFront invalidation failed for the published app.'
+});
+const releaseCheckpoint = buildResolveIOAIManagerRecoveryCheckpoint({
+	plan: releasePlan,
+	current: {
+		lane: 'publish',
+		stepType: 'test_deploy',
+		failureClass: 'release',
+		blocker: 'CloudFront invalidation failed for the published app.',
+		evidenceHash: 'publish-cloudfront-failed',
+		artifactPaths: ['qa/release-publish.log']
+	}
+});
+const releaseProbe = buildResolveIOAIManagerRecoveryEvidenceProbe({ checkpoint: releaseCheckpoint });
+const releaseAction = buildResolveIOAIManagerRecoveryActionPacket({
+	checkpoint: releaseCheckpoint,
+	probe: releaseProbe
+});
+const releaseDirective = buildResolveIOAIManagerRecoveryExecutionDirective({
+	action: releaseAction,
+	current: {
+		lane: 'publish',
+		stepType: 'test_deploy',
+		failureClass: 'release',
+		blocker: 'CloudFront invalidation failed for the published app.',
+		evidenceHash: 'publish-cloudfront-failed',
+		artifactPaths: ['qa/release-publish.log']
+	},
+	surface: 'aicoder_builder'
+});
+
+assert.equal(releasePlan.recoveryClass, 'release_repair');
+assert.equal(releasePlan.productRepairAllowed, false);
+assert.equal(releaseAction.mode, 'repair_release');
+assert.equal(releaseAction.productRepairAllowed, false);
+assert.equal(releaseAction.releasePolicy?.policy, 'hotfix_first');
+assert.equal(releaseAction.releasePolicy?.recommendedAction, 'hotfix_release_artifact');
+assert.ok(releaseAction.nextCommands.includes('hotfix_live_runner_or_release_artifact'));
+assert.equal(releaseDirective.phase, 'release');
+assert.equal(releaseDirective.releasePolicy?.policy, 'hotfix_first');
+assert.ok(releaseDirective.forbiddenActions.some((entry) => /full builder loop|duplicate deploy/i.test(entry)));
+
+const supportReplayMatrix = buildResolveIOAIManagerRecoveryReplayMatrix({
+	surface: 'support_runner',
+	includeReleaseRepair: true,
+	includeJourneyContractRepair: false
+});
+const aicoderBuilderReplayMatrix = buildResolveIOAIManagerRecoveryReplayMatrix({
+	surface: 'aicoder_builder',
+	includeReleaseRepair: true,
+	includeJourneyContractRepair: true
+});
+const aicoderTerminalReplayMatrix = buildResolveIOAIManagerRecoveryReplayMatrix({
+	surface: 'aicoder_app_dashboard_terminal',
+	includeReleaseRepair: true,
+	includeJourneyContractRepair: true
+});
+
+assert.ok(supportReplayMatrix.every((entry) => entry.pass), JSON.stringify(supportReplayMatrix.filter((entry) => !entry.pass), null, 2));
+assert.ok(aicoderBuilderReplayMatrix.every((entry) => entry.pass), JSON.stringify(aicoderBuilderReplayMatrix.filter((entry) => !entry.pass), null, 2));
+assert.ok(aicoderTerminalReplayMatrix.every((entry) => entry.pass), JSON.stringify(aicoderTerminalReplayMatrix.filter((entry) => !entry.pass), null, 2));
+assert.equal(supportReplayMatrix.find((entry) => entry.caseId === 'journey_repair_is_safe_only_for_aicoder_surfaces')?.safeAutoDispatch, false);
+assert.equal(aicoderBuilderReplayMatrix.find((entry) => entry.caseId === 'journey_repair_is_safe_only_for_aicoder_surfaces')?.safeAutoDispatch, true);
+assert.equal(aicoderTerminalReplayMatrix.find((entry) => entry.caseId === 'journey_repair_is_safe_only_for_aicoder_surfaces')?.safeAutoDispatch, true);
+for (const matrix of [supportReplayMatrix, aicoderBuilderReplayMatrix, aicoderTerminalReplayMatrix]) {
+	assert.equal(matrix.find((entry) => entry.caseId === 'release_uses_hotfix_first_repair')?.hotfixFirst, true);
+	assert.equal(matrix.find((entry) => entry.caseId === 'release_uses_hotfix_first_repair')?.fullDeployAllowed, false);
+	assert.equal(matrix.find((entry) => entry.caseId === 'duplicate_release_fingerprint_blocks_full_deploy')?.fullDeployAllowed, false);
+	assert.equal(matrix.find((entry) => entry.caseId === 'product_repair_requires_explicit_operator_policy')?.safeAutoDispatch, false);
+	assert.equal(matrix.find((entry) => entry.caseId === 'weak_hash_only_evidence_stays_parked')?.evidenceStrength, 'weak');
+	assert.equal(matrix.find((entry) => entry.caseId === 'material_evidence_resets_loop_budget')?.loopBudgetShouldReset, true);
+}
+
+const firstDispatch = decideResolveIOAIManagerRecoveryActionDispatch({
+	action: parkedAction,
+	current: {
+		lane: 'repair',
+		stepType: 'repair',
+		failureClass: 'product_code',
+		blocker: 'Save action still throws TypeError.',
+		evidenceHash: 'save-typeerror-before',
+		artifactPaths: ['qa/save-before.log']
+	},
+	now: '2026-06-15T00:01:00.000Z'
+});
+
+assert.equal(firstDispatch.allowed, true);
+assert.equal(firstDispatch.dispatchAction, 'run_evidence_probe');
+assert.equal(firstDispatch.canRunProductRepair, false);
+assert.equal(firstDispatch.shouldRecordDispatch, true);
+assert.equal(firstDispatch.dispatchRecord?.status, 'started');
+
+const firstDirective = buildResolveIOAIManagerRecoveryExecutionDirective({
+	action: parkedAction,
+	dispatchDecision: firstDispatch,
+	current: {
+		lane: 'repair',
+		stepType: 'repair',
+		failureClass: 'product_code',
+		blocker: 'Save action still throws TypeError.',
+		evidenceHash: 'save-typeerror-before',
+		artifactPaths: ['qa/save-before.log']
+	},
+	surface: 'aicoder_builder',
+	now: '2026-06-15T00:01:00.000Z'
+});
+
+assert.equal(firstDirective.allowed, true);
+assert.equal(firstDirective.phase, 'evidence');
+assert.equal(firstDirective.canRunProductRepair, false);
+assert.match(firstDirective.rerunReason, /MANAGER_RECOVERY:run_evidence_probe/);
+assert.ok(firstDirective.forbiddenActions.some((entry) => /product-code repair/i.test(entry)));
+
+const completedDispatchHistory = appendResolveIOAIManagerRecoveryActionDispatch([], {
+	...firstDispatch.dispatchRecord!,
+	status: 'completed',
+	completedAt: '2026-06-15T00:02:00.000Z'
+});
+const repeatedDispatch = decideResolveIOAIManagerRecoveryActionDispatch({
+	action: parkedAction,
+	history: completedDispatchHistory,
+	current: {
+		lane: 'repair',
+		stepType: 'repair',
+		failureClass: 'product_code',
+		blocker: 'Save action still throws TypeError.',
+		evidenceHash: 'save-typeerror-before',
+		artifactPaths: ['qa/save-before.log']
+	},
+	now: '2026-06-15T00:03:00.000Z'
+});
+
+assert.equal(repeatedDispatch.allowed, false);
+assert.equal(repeatedDispatch.dispatchAction, 'park_manual');
+assert.equal(repeatedDispatch.reason, 'recovery_dispatch_same_evidence_already_attempted');
+assert.equal(repeatedDispatch.requiresNewEvidence, true);
+
+const repeatedDirective = buildResolveIOAIManagerRecoveryExecutionDirective({
+	action: parkedAction,
+	dispatchDecision: repeatedDispatch,
+	current: {
+		lane: 'repair',
+		stepType: 'repair',
+		failureClass: 'product_code',
+		blocker: 'Save action still throws TypeError.',
+		evidenceHash: 'save-typeerror-before',
+		artifactPaths: ['qa/save-before.log']
+	},
+	surface: 'aicoder_builder',
+	now: '2026-06-15T00:03:00.000Z'
+});
+
+assert.equal(repeatedDirective.allowed, false);
+assert.equal(repeatedDirective.phase, 'manual');
+assert.equal(repeatedDirective.dispatchAction, 'park_manual');
+assert.equal(repeatedDirective.requiresNewEvidence, true);
+
+const newEvidenceDispatch = decideResolveIOAIManagerRecoveryActionDispatch({
+	action: parkedAction,
+	history: completedDispatchHistory,
+	current: {
+		lane: 'repair',
+		stepType: 'repair',
+		failureClass: 'product_code',
+		blocker: 'Save action still throws TypeError.',
+		evidenceHash: 'save-typeerror-after-dom-trace',
+		artifactPaths: ['qa/save-before.log', 'qa/save-after-dom-trace.json']
+	},
+	now: '2026-06-15T00:04:00.000Z'
+});
+
+assert.equal(newEvidenceDispatch.allowed, true);
+assert.equal(newEvidenceDispatch.newEvidence, true);
+assert.equal(newEvidenceDispatch.dispatchAction, 'run_evidence_probe');
+
+const stillParked = decideResolveIOAIManagerRecoveryGate({
+	checkpoint: parkedCheckpoint,
+	current: {
+		lane: 'repair',
+		stepType: 'repair',
+		failureClass: 'product_code',
+		blocker: 'Save action still throws TypeError.',
+		evidenceHash: 'save-typeerror-before',
+		artifactPaths: ['qa/save-before.log']
+	}
+});
+
+assert.equal(stillParked.action, 'collect_new_evidence');
+assert.equal(stillParked.shouldResetLoopBudget, false);
+assert.equal(stillParked.canRunProductRepair, false);
+
+const runnerSpecificFingerprintCheckpoint = buildResolveIOAIManagerRecoveryCheckpoint({
+	plan: parkedPlan,
+	current: {
+		lane: 'repair',
+		stepType: 'repair',
+		failureClass: 'product_code',
+		blocker: 'Save action still throws TypeError.',
+		blockerFingerprint: 'aicoder-v6-runner-specific',
+		evidenceHash: 'save-typeerror-before',
+		artifactPaths: ['qa/save-before.log']
+	},
+	now: '2026-06-15T00:00:00.000Z'
+});
+const runnerSpecificStillParked = decideResolveIOAIManagerRecoveryGate({
+	checkpoint: runnerSpecificFingerprintCheckpoint,
+	current: {
+		lane: 'repair',
+		stepType: 'repair',
+		failureClass: 'product_code',
+		blocker: 'Save action still throws TypeError.',
+		evidenceHash: 'save-typeerror-before',
+		artifactPaths: ['qa/save-before.log']
+	}
+});
+
+assert.equal(runnerSpecificStillParked.action, 'collect_new_evidence');
+assert.equal(runnerSpecificStillParked.newEvidence, false);
+assert.equal(runnerSpecificStillParked.canRunProductRepair, false);
+
+const noCurrentStillParked = decideResolveIOAIManagerRecoveryGate({
+	checkpoint: runnerSpecificFingerprintCheckpoint
+});
+
+assert.equal(noCurrentStillParked.action, 'collect_new_evidence');
+assert.equal(noCurrentStillParked.newEvidence, false);
+
+const blindRepairRejected = decideResolveIOAIManagerRecoveryGate({
+	checkpoint: parkedCheckpoint,
+	proposedAction: 'targeted_product_repair',
+	current: {
+		lane: 'repair',
+		stepType: 'repair',
+		failureClass: 'product_code',
+		blocker: 'Save action still throws TypeError.',
+		evidenceHash: 'save-typeerror-before',
+		artifactPaths: ['qa/save-before.log']
+	}
+});
+
+assert.equal(blindRepairRejected.action, 'reject_action');
+assert.equal(blindRepairRejected.reason, 'recovery_gate_product_repair_not_allowed_for_checkpoint');
+
+const newEvidenceUnblocks = decideResolveIOAIManagerRecoveryGate({
+	checkpoint: parkedCheckpoint,
+	current: {
+		lane: 'repair',
+		stepType: 'repair',
+		failureClass: 'product_code',
+		blocker: 'Save action still throws TypeError.',
+		evidenceHash: 'save-typeerror-after-dom-trace',
+		artifactPaths: ['qa/save-before.log', 'qa/save-after-dom-trace.json']
+	}
+});
+
+assert.equal(newEvidenceUnblocks.action, 'allow');
+assert.equal(newEvidenceUnblocks.reason, 'recovery_gate_new_material_evidence_unblocks_retry');
+assert.equal(newEvidenceUnblocks.shouldResetLoopBudget, true);
+assert.equal(newEvidenceUnblocks.canRunProductRepair, true);
+assert.equal(newEvidenceUnblocks.newEvidence, true);
+assert.equal(newEvidenceUnblocks.materialEvidence, true);
+assert.equal(newEvidenceUnblocks.evidenceStrength, 'material');
+
+const weakHashOnlyStillParked = decideResolveIOAIManagerRecoveryGate({
+	checkpoint: parkedCheckpoint,
+	current: {
+		lane: 'repair',
+		stepType: 'repair',
+		failureClass: 'product_code',
+		blocker: 'Save action still throws TypeError.',
+		evidenceHash: 'save-typeerror-renamed-hash-only',
+		artifactPaths: ['qa/save-before.log']
+	}
+});
+
+assert.equal(weakHashOnlyStillParked.action, 'collect_new_evidence');
+assert.equal(weakHashOnlyStillParked.reason, 'recovery_gate_weak_evidence_requires_material_signal');
+assert.equal(weakHashOnlyStillParked.newEvidence, true);
+assert.equal(weakHashOnlyStillParked.materialEvidence, false);
+assert.equal(weakHashOnlyStillParked.evidenceStrength, 'weak');
+assert.equal(weakHashOnlyStillParked.shouldResetLoopBudget, false);
+assert.equal(weakHashOnlyStillParked.canRunProductRepair, false);
+
+const weakHashOnlyPolicy = decideResolveIOAIManagerPolicy({
+	history: [
+		{ lane: 'repair', stepType: 'repair', outcome: 'needs_repair', failureClass: 'product_code', blocker: 'Save action still throws TypeError.', evidenceHash: 'save-typeerror-before' },
+		{ lane: 'repair', stepType: 'repair', outcome: 'needs_repair', failureClass: 'product_code', blocker: 'Save action still throws TypeError.', evidenceHash: 'save-typeerror-renamed-hash-only' }
+	],
+	current: {
+		lane: 'repair',
+		stepType: 'repair',
+		outcome: 'needs_repair',
+		failureClass: 'product_code',
+		blocker: 'Save action still throws TypeError.',
+		evidenceHash: 'save-typeerror-renamed-hash-only'
+	},
+	maxSameFailureRepeats: 2
+});
+
+assert.equal(weakHashOnlyPolicy.action, 'park_repeated_failure');
+assert.equal(weakHashOnlyPolicy.reason, 'manager_policy_same_failure_with_weak_evidence');
+assert.equal(weakHashOnlyPolicy.newEvidence, true);
+assert.equal(weakHashOnlyPolicy.materialEvidence, false);
+assert.equal(weakHashOnlyPolicy.evidenceStrength, 'weak');
+assert.equal(weakHashOnlyPolicy.loopBudgetShouldReset, false);
+
+const materialEvidencePolicy = decideResolveIOAIManagerPolicy({
+	history: [
+		{ lane: 'repair', stepType: 'repair', outcome: 'needs_repair', failureClass: 'product_code', blocker: 'Save action still throws TypeError.', evidenceHash: 'save-typeerror-before', artifactPaths: ['qa/save-before.log'] },
+		{
+			lane: 'repair',
+			stepType: 'repair',
+			outcome: 'needs_repair',
+			failureClass: 'product_code',
+			blocker: 'Save action still throws TypeError.',
+			summary: 'New DOM trace proves the save click reaches the method with an undefined id.',
+			evidenceHash: 'save-typeerror-after-dom-trace',
+			artifactPaths: ['qa/save-before.log', 'qa/save-after-dom-trace.json']
+		}
+	],
+	current: {
+		lane: 'repair',
+		stepType: 'repair',
+		outcome: 'needs_repair',
+		failureClass: 'product_code',
+		blocker: 'Save action still throws TypeError.',
+		summary: 'New DOM trace proves the save click reaches the method with an undefined id.',
+		evidenceHash: 'save-typeerror-after-dom-trace',
+		artifactPaths: ['qa/save-before.log', 'qa/save-after-dom-trace.json']
+	},
+	maxSameFailureRepeats: 2
+});
+
+assert.equal(materialEvidencePolicy.action, 'continue');
+assert.equal(materialEvidencePolicy.reason, 'manager_policy_new_material_evidence_resets_loop_budget');
+assert.equal(materialEvidencePolicy.newEvidence, true);
+assert.equal(materialEvidencePolicy.materialEvidence, true);
+assert.equal(materialEvidencePolicy.evidenceStrength, 'material');
+assert.equal(materialEvidencePolicy.loopBudgetShouldReset, true);
+
+const infraPlan = buildResolveIOAIManagerRecoveryPlan({
+	action: 'retry_infra',
+	reason: 'puppeteer missing',
+	failureClass: 'infra',
+	lane: 'qa',
+	stepType: 'workflow_qa',
+	blocker: 'Chrome executable missing.'
+});
+const infraCheckpoint = buildResolveIOAIManagerRecoveryCheckpoint({
+	plan: infraPlan,
+	current: {
+		lane: 'qa',
+		stepType: 'workflow_qa',
+		failureClass: 'infra',
+		blocker: 'Chrome executable missing.',
+		evidenceHash: 'chrome-missing'
+	}
+});
+const productRepairOnInfra = decideResolveIOAIManagerRecoveryGate({
+	checkpoint: infraCheckpoint,
+	proposedAction: 'targeted_product_repair',
+	current: {
+		lane: 'qa',
+		stepType: 'workflow_qa',
+		failureClass: 'infra',
+		blocker: 'Chrome executable missing.',
+		evidenceHash: 'chrome-missing'
+	}
+});
+
+assert.equal(productRepairOnInfra.action, 'reject_action');
+assert.equal(productRepairOnInfra.canRunProductRepair, false);
+
+const infraEvidenceProbe = buildResolveIOAIManagerRecoveryEvidenceProbe({
+	checkpoint: infraCheckpoint
+});
+const infraAction = buildResolveIOAIManagerRecoveryActionPacket({
+	checkpoint: infraCheckpoint,
+	probe: infraEvidenceProbe
+});
+
+assert.equal(infraEvidenceProbe.evidenceOnly, true);
+assert.ok(infraEvidenceProbe.steps.some((step) => step.kind === 'infra_preflight'));
+assert.ok(infraEvidenceProbe.requiredArtifacts.some((entry) => /infra-preflight/i.test(entry)));
+assert.equal(infraAction.mode, 'repair_infra');
+assert.equal(infraAction.autoRunnable, true);
+assert.equal(infraAction.productRepairAllowed, false);
+
+const budgetPlan = buildResolveIOAIManagerRecoveryPlan({
+	action: 'budget_stop',
+	reason: 'hard budget guard',
+	failureClass: 'product_code',
+	lane: 'build',
+	stepType: 'build'
+});
+const budgetCheckpoint = buildResolveIOAIManagerRecoveryCheckpoint({
+	plan: budgetPlan,
+	current: {
+		lane: 'build',
+		stepType: 'build',
+		failureClass: 'product_code',
+		blocker: 'Budget guard tripped.',
+		evidenceHash: 'budget-stop'
+	}
+});
+
+assert.equal(decideResolveIOAIManagerRecoveryGate({
+	checkpoint: budgetCheckpoint
+}).action, 'manual_handoff');
+assert.equal(decideResolveIOAIManagerRecoveryGate({
+	checkpoint: budgetCheckpoint,
+	operatorApproved: true
+}).action, 'allow');
+
+console.log('ai manager recovery checkpoint tests passed');

package/tests/ai-run-eval.test.ts

@@ -0,0 +1,88 @@
+import * as assert from 'node:assert/strict';
+import {
+	AI_RUN_SEED_EVAL_FIXTURES,
+	buildAIRunDatasetExport,
+	buildAIRunTrainingExampleFromRun,
+	classifyAIRunFailureClass,
+	decideAIRunRetry,
+	evaluateAIRunFixtures
+} from '../src/util/ai-run-evidence-eval';
+import {
+	buildAIRunFalsePassWarnings,
+	buildAIRunOperatorView,
+	buildAIRunQaHealth,
+	buildAIRunTimeline
+} from '../src/util/ai-run-evidence-dashboard';
+
+const summary = evaluateAIRunFixtures();
+assert.equal(summary.passed, true);
+assert.equal(summary.failedCount, 0);
+assert.equal(summary.total, AI_RUN_SEED_EVAL_FIXTURES.length);
+assert.equal(summary.outcomes.accepted, 2);
+assert.ok(summary.outcomes.false_pass >= 2);
+assert.ok(summary.outcomes.qa_infra_failed >= 2);
+assert.ok(summary.outcomes.release_blocked >= 2);
+
+const routeOnlyFixture = AI_RUN_SEED_EVAL_FIXTURES.find((fixture) => fixture.id === 'qa-route-only-pass');
+assert.ok(routeOnlyFixture);
+assert.equal(routeOnlyFixture.run.outcome, 'false_pass');
+assert.equal(classifyAIRunFailureClass(routeOnlyFixture.run), 'false_pass');
+
+const repeatedRouteOnlyDecision = decideAIRunRetry({
+	currentRun: routeOnlyFixture.run,
+	previousRun: routeOnlyFixture.run,
+	newEvidenceCount: 0,
+	sameFailureCount: 2,
+	unchangedFailureLimit: 2
+});
+assert.equal(repeatedRouteOnlyDecision.action, 'stop');
+assert.equal(repeatedRouteOnlyDecision.repeatedFailure, true);
+
+const puppeteerFixture = AI_RUN_SEED_EVAL_FIXTURES.find((fixture) => fixture.id === 'qa-missing-puppeteer');
+assert.ok(puppeteerFixture);
+const infraDecision = decideAIRunRetry({
+	currentRun: puppeteerFixture.run,
+	newEvidenceCount: 1
+});
+assert.equal(infraDecision.action, 'retry_infra_repair');
+assert.equal(infraDecision.failureClass, 'infra');
+
+const layflatFixture = AI_RUN_SEED_EVAL_FIXTURES.find((fixture) => fixture.id === 'aicoder-layflat-pressure-optimizer');
+assert.ok(layflatFixture);
+const releaseDecision = decideAIRunRetry({
+	currentRun: layflatFixture.run,
+	newEvidenceCount: 1
+});
+assert.equal(releaseDecision.action, 'retry_release_repair');
+
+const example = buildAIRunTrainingExampleFromRun(routeOnlyFixture.run);
+assert.equal(example.outcome, 'false_pass');
+assert.match(example.actionSummary, /failureClass=false_pass/);
+assert.ok(example.evidence.some((entry) => /business/i.test(entry)));
+
+const routeOnlyTimeline = buildAIRunTimeline(routeOnlyFixture.run);
+assert.ok(routeOnlyTimeline.some((entry) => entry.type === 'qa_route_probe'));
+assert.ok(routeOnlyTimeline.some((entry) => entry.type === 'gate' && entry.status === 'blocked'));
+const routeOnlyHealth = buildAIRunQaHealth(routeOnlyFixture.run);
+assert.equal(routeOnlyHealth.business.missing, true);
+assert.equal(routeOnlyHealth.route.pass, 1);
+assert.ok(buildAIRunFalsePassWarnings(routeOnlyFixture.run).includes('false_pass_outcome'));
+const routeOnlyOperatorView = buildAIRunOperatorView(routeOnlyFixture.run);
+assert.equal(routeOnlyOperatorView.failureClass, 'false_pass');
+assert.equal(routeOnlyOperatorView.costPanel.totalTokens, 0);
+assert.equal(routeOnlyOperatorView.trainingExample.outcome, 'false_pass');
+
+const exportPayload = buildAIRunDatasetExport(AI_RUN_SEED_EVAL_FIXTURES.map((fixture) => fixture.run), {
+	includeRuns: false,
+	redact: true,
+	now: '2026-06-13T00:00:00.000Z'
+});
+assert.equal(exportPayload.generatedAt, '2026-06-13T00:00:00.000Z');
+assert.equal(exportPayload.runs.length, 0);
+assert.equal(exportPayload.trainingExamples.length, AI_RUN_SEED_EVAL_FIXTURES.length);
+assert.equal(exportPayload.summary.byFailureClass.false_pass, 2);
+assert.equal(exportPayload.summary.bySource.support_ticket, 6);
+assert.equal(exportPayload.summary.bySource.aicoder_app, 2);
+assert.equal(exportPayload.summary.bySource.ai_assistant, 2);
+
+console.log('ai run eval tests passed');