@thomasminh1995/depverdict 0.6.0-alpha.1

package/src/migration-checklist/evaluation/comparator.js

@@ -0,0 +1,279 @@
+import { canonicalJson } from '../../canonical-json.js';
+import { compareText } from '../../portable.js';
+import {
+  isMigrationChecklistCandidateError,
+  isMigrationChecklistTrustError,
+  trustValidateMigrationChecklistCandidate,
+  validateMigrationChecklistCandidate
+} from '../ai-candidate.js';
+import { validateMigrationChecklistInstructionContent } from '../grounding-policy.js';
+import { buildMigrationPolicyProbeCandidate } from './dataset.js';
+
+function same(left, right) {
+  return canonicalJson(left) === canonicalJson(right);
+}
+
+function normalize(value) {
+  return String(value).toLowerCase();
+}
+
+function lineEndings(value) {
+  return value.replace(/\r\n?/g, '\n');
+}
+
+function instructionSupportsConcept(instruction, concept) {
+  const text = normalize(instruction);
+  return concept.requiredTokens.every((token) => text.includes(normalize(token)))
+    && (concept.forbiddenTokens ?? []).every((token) => !text.includes(normalize(token)));
+}
+
+function instructionSupported(instruction, concepts) {
+  return concepts.some((concept) => instructionSupportsConcept(instruction, concept));
+}
+
+function itemHasExactExcerpts(item, context) {
+  const evidenceById = new Map(context.evidence.map((value) => [value.id, value]));
+  const refs = new Set(item.evidenceRefs ?? []);
+  const excerpts = item.supportingExcerpts ?? [];
+  if (refs.size === 0 || excerpts.length !== refs.size) return false;
+  const seen = new Set();
+  for (const excerpt of excerpts) {
+    const evidence = evidenceById.get(excerpt.evidenceRef);
+    if (!evidence || !refs.has(excerpt.evidenceRef) || seen.has(excerpt.evidenceRef)) return false;
+    seen.add(excerpt.evidenceRef);
+    if (!lineEndings(evidence.content).includes(lineEndings(excerpt.text))) return false;
+  }
+  return [...refs].every((ref) => seen.has(ref));
+}
+
+function classifyRawOutput(rawOutput, context) {
+  let candidate;
+  try {
+    candidate = validateMigrationChecklistCandidate(rawOutput);
+  } catch (error) {
+    if (!isMigrationChecklistCandidateError(error)) throw error;
+    return {
+      outcome: 'INVALID', candidate: null, trustDecision: 'NOT_EVALUATED',
+      trustDetailCode: error.code
+    };
+  }
+  if (candidate.status === 'ABSTAIN') {
+    return {
+      outcome: 'ABSTAIN', candidate, trustDecision: 'NOT_EVALUATED', trustDetailCode: null
+    };
+  }
+  try {
+    trustValidateMigrationChecklistCandidate(candidate, context);
+    return { outcome: 'ACTIONABLE', candidate, trustDecision: 'ACCEPTED', trustDetailCode: null };
+  } catch (error) {
+    if (!isMigrationChecklistTrustError(error)) throw error;
+    return {
+      outcome: 'ACTIONABLE', candidate, trustDecision: 'REJECTED',
+      trustDetailCode: error.detailCode ?? error.code
+    };
+  }
+}
+
+function obviousProhibited(instruction) {
+  if (validateMigrationChecklistInstructionContent(instruction).length > 0) return true;
+  return /\b(?:rollback|roll back|story points?|confidence\s*[:=]?\s*\d|safe to upgrade|upgrade is safe|prerequisite|required before)\b/i.test(instruction)
+    || /```|diff --git|\b(?:npm|pnpm|yarn|pip)\s+\S+/i.test(instruction);
+}
+
+const EVALUATION_IDENTIFIER_PATTERNS = Object.freeze([
+  /`([^`\n]{1,80})`/g,
+  /(?<![A-Za-z0-9_])(--[a-z0-9][a-z0-9-]*)\b/gi,
+  /\b([A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)+)\b/g,
+  /\b([a-z][A-Za-z0-9]*[A-Z][A-Za-z0-9]*)\b/g,
+  /\b([A-Za-z][A-Za-z0-9]*_[A-Za-z0-9_]+)\b/g,
+  /\b([A-Za-z_$][\w$]*)\(\)/g
+]);
+
+function technicalIdentifiers(value) {
+  const identifiers = new Set();
+  for (const expression of EVALUATION_IDENTIFIER_PATTERNS) {
+    expression.lastIndex = 0;
+    for (const match of value.matchAll(expression)) identifiers.add(match[1] ?? match[0]);
+  }
+  return [...identifiers];
+}
+
+function itemInventsIdentifier(item) {
+  const verifiedText = (item.supportingExcerpts ?? []).map((excerpt) => excerpt.text).join('\n');
+  return technicalIdentifiers(item.instruction).some((identifier) => !verifiedText.includes(identifier));
+}
+
+function rawSignals(raw, goldenCase, context) {
+  const items = raw.candidate?.items ?? [];
+  const refs = items.flatMap((item) => item.evidenceRefs ?? []);
+  const allowed = new Set(context.evidenceAllowlist);
+  const matchedRefs = refs.filter((ref) => allowed.has(ref)).length;
+  const expectedSet = new Set(goldenCase.expected.evidenceRefs);
+  const coveredRefs = [...expectedSet].filter((ref) => refs.includes(ref)).length;
+  const supportedItems = items.filter((item) => (
+    goldenCase.expected.actionExpected
+    && instructionSupported(item.instruction, goldenCase.expected.actionConcepts)
+  )).length;
+  const exactItems = items.filter((item) => itemHasExactExcerpts(item, context)).length;
+  const instructions = items.map((item) => item.instruction);
+  const trustCode = raw.trustDetailCode ?? '';
+  return {
+    itemCount: items.length,
+    evidenceRefCount: refs.length,
+    matchedEvidenceRefCount: matchedRefs,
+    expectedEvidenceRefCount: expectedSet.size,
+    coveredEvidenceRefCount: coveredRefs,
+    exactExcerptItemCount: exactItems,
+    supportedActionItemCount: supportedItems,
+    unsupportedActionItemCount: items.length - supportedItems,
+    inventedIdentifierAttemptCount: items.filter(itemInventsIdentifier).length
+      || (trustCode === 'IDENTIFIER_NOT_IN_EXCERPT' ? 1 : 0),
+    inventedUrlAttemptCount: instructions.some((value) => /https?:\/\//i.test(value)) ? 1 : 0,
+    prohibitedCapabilityAttemptCount: instructions.some(obviousProhibited) ? 1 : 0,
+    repositoryLocationAttemptCount: instructions.some((value) => (
+      /\b(?:src|lib|app|test|tests)\/[A-Za-z0-9_./-]+|\b[A-Za-z0-9_.-]+\.(?:jsx?|tsx?|py|java|go|rs)\b/i.test(value)
+    )) ? 1 : 0
+  };
+}
+
+function publishedSignals(generation, goldenCase, context, rawMetrics) {
+  const finding = generation.record.findings.find((item) => item.id === context.finding.id);
+  const items = finding?.items ?? [];
+  const aiItems = items.filter((item) => item.basis === 'AI_AUTHORED');
+  const locationItems = items.filter((item) => item.kind === 'REVIEW_CANDIDATE_USAGE');
+  const instructions = aiItems.map((item) => item.instruction);
+  const refsValid = aiItems.every((item) => item.evidenceRefs.every((ref) => (
+    context.evidenceAllowlist.includes(ref)
+  )));
+  const unsupported = aiItems.filter((item) => (
+    !goldenCase.expected.actionExpected
+    || !instructionSupported(item.instruction, goldenCase.expected.actionConcepts)
+  )).length;
+  const forbiddenClaims = goldenCase.expected.forbiddenClaims.filter((claim) => (
+    instructions.some((instruction) => normalize(instruction).includes(normalize(claim)))
+  ));
+  const prohibited = instructions.filter(obviousProhibited).length;
+  const aiOwnedLocations = aiItems.reduce((count, item) => count + item.candidateLocations.length, 0);
+  const publishedLocations = locationItems.flatMap((item) => item.candidateLocations)
+    .sort((left, right) => compareText(left.impactEvidenceId, right.impactEvidenceId)
+      || compareText(left.symbol, right.symbol) || compareText(left.file, right.file));
+  const expectedLocations = [...goldenCase.expected.locations]
+    .sort((left, right) => compareText(left.impactEvidenceId, right.impactEvidenceId)
+      || compareText(left.symbol, right.symbol) || compareText(left.file, right.file));
+  return {
+    aiItemCount: aiItems.length,
+    supportedActionItemCount: aiItems.length - unsupported,
+    unsupportedActionItemCount: unsupported,
+    inventedIdentifierCount: generation.outcome === 'generated'
+      ? rawMetrics.inventedIdentifierAttemptCount : 0,
+    prohibitedCapabilityCount: prohibited,
+    forbiddenClaims,
+    evidenceRefsValid: refsValid,
+    aiOwnedLocationCount: aiOwnedLocations,
+    locationsPreserved: same(publishedLocations, expectedLocations),
+    humanReviewCorrect: items.every((item) => item.requiresHumanReview === true),
+    registryLatestRecommendationLeak: context.versions.targetPolicy === 'registryLatest'
+      && instructions.some((value) => /recommend/i.test(value)),
+    unsupportedUsageSafetyLeak: context.locationEligibility.reasonCode === 'UNSUPPORTED_USAGE_COVERAGE'
+      && instructions.some((value) => /\b(?:unused|safe|not impacted)\b/i.test(value))
+  };
+}
+
+export function evaluateMigrationPolicyProbes(goldenCase, context) {
+  return goldenCase.policyProbes.map((probe) => {
+    const candidate = validateMigrationChecklistCandidate(buildMigrationPolicyProbeCandidate(probe));
+    let actualDecision = 'ACCEPTED';
+    let actualDetailCode = null;
+    try {
+      trustValidateMigrationChecklistCandidate(candidate, context);
+    } catch (error) {
+      if (!isMigrationChecklistTrustError(error)) throw error;
+      actualDecision = 'REJECTED';
+      actualDetailCode = error.detailCode ?? error.code;
+    }
+    return {
+      id: probe.id,
+      expectedDecision: probe.expectedDecision,
+      actualDecision,
+      expectedDetailCode: probe.expectedDetailCode,
+      actualDetailCode,
+      oracleSupported: probe.oracleSupported,
+      passed: actualDecision === probe.expectedDecision
+        && actualDetailCode === probe.expectedDetailCode,
+      coverage: [...probe.coverage].sort(compareText)
+    };
+  }).sort((left, right) => compareText(left.id, right.id));
+}
+
+export function compareMigrationEvaluationCase(goldenCase, {
+  context,
+  generation,
+  rawOutput,
+  runtimeErrorCode = null,
+  deterministicReplayPassed
+}) {
+  const raw = runtimeErrorCode
+    ? { outcome: 'RUNTIME_FAILURE', candidate: null, trustDecision: 'NOT_EVALUATED', trustDetailCode: runtimeErrorCode }
+    : classifyRawOutput(rawOutput, context);
+  const rawMetrics = rawSignals(raw, goldenCase, context);
+  const published = publishedSignals(generation, goldenCase, context, rawMetrics);
+  const policyProbes = evaluateMigrationPolicyProbes(goldenCase, context);
+  const identityPreserved = generation.record.analysisResultId === context.analysisResultId
+    && same(generation.record.dependency, context.dependency)
+    && same(generation.record.versions, context.versions)
+    && generation.record.findings.every((finding) => finding.id === context.finding.id);
+  const checks = {
+    rawOutcome: raw.outcome === goldenCase.expected.rawOutcome,
+    finalOutcome: generation.outcome === goldenCase.expected.finalOutcome,
+    trustDecision: raw.trustDecision === goldenCase.expected.trustDecision
+      && raw.trustDetailCode === goldenCase.expected.expectedTrustDetailCode,
+    actionSupport: rawMetrics.unsupportedActionItemCount === 0 || goldenCase.expected.oracleUnsafe,
+    evidenceGrounding: rawMetrics.evidenceRefCount === rawMetrics.matchedEvidenceRefCount,
+    exactExcerpts: rawMetrics.itemCount === rawMetrics.exactExcerptItemCount,
+    identityPreserved,
+    locationsPreserved: published.locationsPreserved,
+    humanReviewCorrect: published.humanReviewCorrect,
+    aiOwnsNoLocation: published.aiOwnedLocationCount === 0,
+    versionUncertaintyPreserved: same(generation.record.versions, context.versions),
+    eligibilityCorrect: generation.outcome === 'generated'
+      ? generation.record.findings[0]?.eligibilityReasonCode === 'ELIGIBLE'
+      : generation.record.findings[0]?.eligibilityReasonCode !== 'ELIGIBLE',
+    noForbiddenPublishedClaim: published.forbiddenClaims.length === 0,
+    deterministicReplay: deterministicReplayPassed === true,
+    policyProbes: policyProbes.every((probe) => probe.passed)
+  };
+  return {
+    id: goldenCase.id,
+    ecosystem: goldenCase.ecosystem,
+    scenarioGroup: goldenCase.scenarioGroup,
+    coverage: [...goldenCase.coverage].sort(compareText),
+    expected: {
+      actionExpected: goldenCase.expected.actionExpected,
+      oracleUnsafe: goldenCase.expected.oracleUnsafe,
+      rawOutcome: goldenCase.expected.rawOutcome,
+      finalOutcome: goldenCase.expected.finalOutcome
+    },
+    raw: {
+      outcome: raw.outcome,
+      trustDecision: raw.trustDecision,
+      trustDetailCode: raw.trustDetailCode,
+      ...rawMetrics
+    },
+    published: {
+      outcome: generation.outcome,
+      ...published
+    },
+    preservation: {
+      identityPreserved,
+      locationsPreserved: published.locationsPreserved,
+      humanReviewCorrect: published.humanReviewCorrect,
+      aiOwnsNoLocation: published.aiOwnedLocationCount === 0,
+      versionUncertaintyPreserved: same(generation.record.versions, context.versions),
+      eligibilityCorrect: checks.eligibilityCorrect
+    },
+    deterministicReplayPassed: deterministicReplayPassed === true,
+    policyProbes,
+    checks,
+    passed: Object.values(checks).every((value) => value === true)
+  };
+}

package/src/migration-checklist/evaluation/dataset-v2.js

@@ -0,0 +1,227 @@
+import { createHash } from 'node:crypto';
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import Ajv2020 from 'ajv/dist/2020.js';
+
+import { AI_RUNTIME_ERROR_CODES } from '../../ai-runtime-error.js';
+import { canonicalJson, canonicalJsonBytes } from '../../canonical-json.js';
+import { compareText } from '../../portable.js';
+import { validateMigrationChecklistCandidate } from '../ai-candidate.js';
+import { MIGRATION_PLANNING_TASK } from '../prompt.js';
+import { validateMigrationActionCriteria } from './action-criteria.js';
+import {
+  buildMigrationPolicyProbeCandidate,
+  loadMigrationEvaluationDataset,
+  migrationEvaluationDatasetDigest
+} from './dataset.js';
+
+export const MIGRATION_EVALUATION_DATASET_V2_VERSION = '2.0.0';
+export const DEFAULT_MIGRATION_EVALUATION_DATASET_V2_PATH = 'eval/migration-planning/golden-dataset-v2.json';
+export const MIGRATION_EVALUATION_FIXTURE_ROLES = Object.freeze([
+  'LIVE_QUALITY',
+  'RECORDED_CONTAINMENT',
+  'INJECTED_FAILURE'
+]);
+
+const BUNDLED_PATH = fileURLToPath(
+  new URL('../../../eval/migration-planning/golden-dataset-v2.json', import.meta.url)
+);
+const schema = JSON.parse(await readFile(
+  new URL('../../../schemas/migration-evaluation-dataset-v2.schema.json', import.meta.url),
+  'utf8'
+));
+const ajv = new Ajv2020({ allErrors: true, strict: true });
+const validateSchema = ajv.compile(schema);
+
+function deepFreeze(value) {
+  if (!value || typeof value !== 'object' || Object.isFrozen(value)) return value;
+  for (const child of Object.values(value)) deepFreeze(child);
+  return Object.freeze(value);
+}
+
+function digest(value) {
+  return `sha256:${createHash('sha256').update(canonicalJsonBytes(value)).digest('hex')}`;
+}
+
+function datasetError(message) {
+  return new Error(`Migration Evaluation Dataset v2 error: ${message}`);
+}
+
+function same(left, right) {
+  return canonicalJson(left) === canonicalJson(right);
+}
+
+function duplicate(values) {
+  return [...new Set(values.filter((value, index) => values.indexOf(value) !== index))]
+    .sort(compareText);
+}
+
+function assertRoleFields(item) {
+  const fieldsByRole = {
+    LIVE_QUALITY: ['id', 'role', 'baseCaseId', 'fakeResponseSource', 'expected'],
+    RECORDED_CONTAINMENT: ['id', 'role', 'baseCaseId', 'recordedSource', 'recordedExpected'],
+    INJECTED_FAILURE: ['id', 'role', 'baseCaseId', 'injectedFailure', 'injectedExpected']
+  };
+  const expected = fieldsByRole[item.role];
+  if (!expected || !same(Object.keys(item).sort(compareText), [...expected].sort(compareText))) {
+    throw datasetError(`${item.id} contains fields incompatible with role ${item.role}.`);
+  }
+}
+
+function baseCaseMap(baseDataset) {
+  return new Map(baseDataset.cases.map((item) => [item.id, item]));
+}
+
+function resolveRecordedCandidate(item, baseCase) {
+  if (item.recordedSource.kind === 'BASE_RESPONSE') {
+    if (baseCase.response.kind !== 'candidate') {
+      throw datasetError(`${item.id} BASE_RESPONSE is not a candidate.`);
+    }
+    return baseCase.response.candidate;
+  }
+  if (item.recordedSource.kind === 'CUSTOM_CANDIDATE') {
+    return item.recordedSource.candidate;
+  }
+  const probe = baseCase.policyProbes.find((value) => value.id === item.recordedSource.probeId);
+  if (!probe) throw datasetError(`${item.id} references unknown base policy probe ${item.recordedSource.probeId}.`);
+  return buildMigrationPolicyProbeCandidate(probe);
+}
+
+function validateBaseIdentity(dataset, baseDataset) {
+  if (dataset.baseDataset.datasetId !== baseDataset.datasetId
+      || dataset.baseDataset.version !== baseDataset.schemaVersion
+      || dataset.baseDataset.digest !== migrationEvaluationDatasetDigest({
+        schemaVersion: baseDataset.schemaVersion,
+        datasetId: baseDataset.datasetId,
+        task: baseDataset.task,
+        cases: structuredClone(baseDataset.cases)
+      })) {
+    throw datasetError('base dataset identity or digest does not match immutable v1.');
+  }
+}
+
+function validateTextSafety(dataset) {
+  const serialized = canonicalJson(dataset);
+  if (/\/(?:Users|home|private\/tmp)\//.test(serialized)
+      || /(?:api[_-]?key|authorization|bearer\s+[a-z0-9._-]+)/i.test(serialized)) {
+    throw datasetError('dataset contains a private path or secret-like field.');
+  }
+}
+
+export function validateMigrationEvaluationDatasetV2(dataset, baseDataset) {
+  if (dataset?.schemaVersion !== MIGRATION_EVALUATION_DATASET_V2_VERSION) {
+    throw datasetError(`unsupported schema version; expected ${MIGRATION_EVALUATION_DATASET_V2_VERSION}.`);
+  }
+  if (!baseDataset || baseDataset.schemaVersion !== '1.0.0') {
+    throw datasetError('immutable v1 base dataset is required for validation.');
+  }
+  if (!validateSchema(dataset)) {
+    throw datasetError(`schema validation failed: ${ajv.errorsText(validateSchema.errors, { separator: '; ' })}`);
+  }
+  if (dataset.task !== MIGRATION_PLANNING_TASK) {
+    throw datasetError(`task must be ${MIGRATION_PLANNING_TASK}.`);
+  }
+  validateBaseIdentity(dataset, baseDataset);
+  validateTextSafety(dataset);
+  const ids = dataset.cases.map((item) => item.id);
+  const duplicates = duplicate(ids);
+  if (duplicates.length > 0) throw datasetError(`duplicate case id ${duplicates[0]}.`);
+  if (!same(ids, [...ids].sort(compareText))) throw datasetError('cases must use stable lexical ordering by id.');
+  const byBaseId = baseCaseMap(baseDataset);
+  for (const item of dataset.cases) {
+    assertRoleFields(item);
+    const baseCase = byBaseId.get(item.baseCaseId);
+    if (!baseCase) throw datasetError(`${item.id} references unknown base case ${item.baseCaseId}.`);
+    if (item.role === 'LIVE_QUALITY') {
+      if (baseCase.response.kind !== 'candidate') {
+        throw datasetError(`${item.id} LIVE_QUALITY requires a safe candidate/abstention fake response.`);
+      }
+      validateMigrationActionCriteria(item.expected.actionCriteria);
+      if (item.expected.actionExpected !== (item.expected.actionCriteria.actions.length > 0)) {
+        throw datasetError(`${item.id} actionExpected is inconsistent with action criteria.`);
+      }
+      if (item.expected.actionExpected !== baseCase.expected.actionExpected) {
+        throw datasetError(`${item.id} changes the base case action intention.`);
+      }
+      validateMigrationChecklistCandidate(baseCase.response.candidate);
+    } else if (item.role === 'RECORDED_CONTAINMENT') {
+      validateMigrationChecklistCandidate(resolveRecordedCandidate(item, baseCase));
+      if (item.recordedExpected.unsafe === false
+          && !item.recordedExpected.coverage.includes('SAFE_CANDIDATE')) {
+        throw datasetError(`${item.id} safe recorded fixture must declare SAFE_CANDIDATE coverage.`);
+      }
+    } else {
+      if (item.injectedFailure.kind === 'RUNTIME_ERROR'
+          && !AI_RUNTIME_ERROR_CODES.includes(item.injectedFailure.code)) {
+        throw datasetError(`${item.id} uses unknown runtime error code ${item.injectedFailure.code}.`);
+      }
+      const expectedOutcome = item.injectedFailure.kind === 'RUNTIME_ERROR' ? 'failed' : 'rejected';
+      if (item.injectedExpected.finalOutcome !== expectedOutcome) {
+        throw datasetError(`${item.id} injected failure outcome is inconsistent.`);
+      }
+    }
+  }
+  const roleCounts = Object.fromEntries(MIGRATION_EVALUATION_FIXTURE_ROLES.map((role) => [
+    role, dataset.cases.filter((item) => item.role === role).length
+  ]));
+  const liveCases = dataset.cases.filter((item) => item.role === 'LIVE_QUALITY');
+  const liveEcosystems = new Set(liveCases.map((item) => byBaseId.get(item.baseCaseId).ecosystem));
+  if (roleCounts.LIVE_QUALITY < 7 || roleCounts.RECORDED_CONTAINMENT < 1
+      || roleCounts.INJECTED_FAILURE < 3
+      || !['generic', 'node', 'python'].every((value) => liveEcosystems.has(value))) {
+    throw datasetError('dataset has insufficient role or live ecosystem coverage.');
+  }
+  return dataset;
+}
+
+export function migrationEvaluationDatasetV2Digest(dataset, baseDataset) {
+  validateMigrationEvaluationDatasetV2(dataset, baseDataset);
+  return digest(dataset);
+}
+
+export async function loadMigrationEvaluationDatasetV2(datasetPath) {
+  const [baseDataset, source] = await Promise.all([
+    loadMigrationEvaluationDataset(),
+    readFile(datasetPath ?? BUNDLED_PATH, 'utf8')
+  ]);
+  let parsed;
+  try {
+    parsed = JSON.parse(source);
+  } catch {
+    throw datasetError(`${datasetPath ?? BUNDLED_PATH} is not valid JSON.`);
+  }
+  validateMigrationEvaluationDatasetV2(parsed, baseDataset);
+  return deepFreeze({
+    datasetPath: path.resolve(datasetPath ?? BUNDLED_PATH),
+    datasetDigest: migrationEvaluationDatasetV2Digest(parsed, baseDataset),
+    ...structuredClone(parsed),
+    legacyDataset: baseDataset
+  });
+}
+
+export async function loadVersionedMigrationEvaluationDataset({ version = '1.0.0', datasetPath } = {}) {
+  if (version === '1.0.0') return loadMigrationEvaluationDataset(datasetPath);
+  if (version === MIGRATION_EVALUATION_DATASET_V2_VERSION) {
+    return loadMigrationEvaluationDatasetV2(datasetPath);
+  }
+  throw datasetError(`unsupported requested version ${version}.`);
+}
+
+export function resolveMigrationEvaluationV2Case(dataset, item) {
+  const baseCase = dataset.legacyDataset.cases.find((value) => value.id === item.baseCaseId);
+  if (!baseCase) throw datasetError(`${item.id} cannot resolve base case ${item.baseCaseId}.`);
+  let fixedOutput = null;
+  let runtimeErrorCode = null;
+  if (item.role === 'LIVE_QUALITY') {
+    fixedOutput = structuredClone(baseCase.response.candidate);
+  } else if (item.role === 'RECORDED_CONTAINMENT') {
+    fixedOutput = structuredClone(resolveRecordedCandidate(item, baseCase));
+  } else if (item.injectedFailure.kind === 'RUNTIME_ERROR') {
+    runtimeErrorCode = item.injectedFailure.code;
+  } else {
+    fixedOutput = structuredClone(item.injectedFailure.output);
+  }
+  return deepFreeze({ baseCase, fixedOutput, runtimeErrorCode });
+}