@thomasminh1995/depverdict 0.6.0-alpha.1

package/src/metrics-engine.js

@@ -0,0 +1,205 @@
+import { createHash } from 'node:crypto';
+import { mkdir, open, readFile, rename, rm } from 'node:fs/promises';
+import path from 'node:path';
+
+import Ajv2020 from 'ajv/dist/2020.js';
+import addFormats from 'ajv-formats';
+
+import { ARTIFACT_GENERATOR_NAME, VERSION } from './constants.js';
+import { validateEvaluationReport } from './evaluation-report.js';
+import { compareText } from './portable.js';
+
+export const METRICS_SCHEMA_VERSION = '1.0.0';
+export const DEFAULT_METRICS_PATH = 'metrics.json';
+
+const schema = JSON.parse(await readFile(
+  new URL('../schemas/metrics.schema.json', import.meta.url),
+  'utf8'
+));
+const ajv = new Ajv2020({ allErrors: true, strict: true, strictRequired: false });
+addFormats(ajv);
+const validateSchema = ajv.compile(schema);
+
+function ratio(numerator, denominator, emptyValue = 1) {
+  return denominator === 0 ? emptyValue : numerator / denominator;
+}
+
+function metric(passed, total) {
+  return { value: ratio(passed, total), passed, total };
+}
+
+function boolCheck(caseResult, name) {
+  return caseResult.checks?.[name]?.passed === true;
+}
+
+function sortedUnique(values = []) {
+  return [...new Set(values)].sort(compareText);
+}
+
+function flattenRefs(refs = {}) {
+  return sortedUnique([
+    ...(refs.summary ?? []),
+    ...(refs.risk ?? []),
+    ...(refs.findings ?? [])
+  ]);
+}
+
+function evidenceRefCounts(caseResult) {
+  const expected = flattenRefs(caseResult.checks?.evidenceReferences?.expected);
+  const actual = flattenRefs(caseResult.checks?.evidenceReferences?.actual);
+  const actualSet = new Set(actual);
+  return {
+    expected: expected.length,
+    actual: actual.length,
+    matched: expected.filter((ref) => actualSet.has(ref)).length
+  };
+}
+
+function validationWarningCodes(caseResult) {
+  return sortedUnique(caseResult.checks?.validationState?.actual?.warningCodes ?? []);
+}
+
+function unsupportedClaimCount(caseResult) {
+  return validationWarningCodes(caseResult).includes('CLAIMS_DROPPED') ? 1 : 0;
+}
+
+function actualFindingCount(caseResult) {
+  return Array.isArray(caseResult.checks?.findings?.actual)
+    ? caseResult.checks.findings.actual.length
+    : 0;
+}
+
+function deterministicPassed(caseResult) {
+  return boolCheck(caseResult, 'schema')
+    && boolCheck(caseResult, 'evidenceReferences')
+    && boolCheck(caseResult, 'validationState')
+    && boolCheck(caseResult, 'findings');
+}
+
+function buildInput(report) {
+  return {
+    evaluationReport: {
+      schemaVersion: report.schemaVersion,
+      generatedAt: report.generatedAt,
+      datasetPath: report.evaluation.datasetPath,
+      datasetVersion: report.evaluation.datasetVersion,
+      promptVersion: report.evaluation.promptVersion,
+      model: structuredClone(report.model),
+      caseCount: report.evaluation.caseCount
+    }
+  };
+}
+
+function buildCaseMetrics(report) {
+  return report.cases
+    .map((caseResult) => ({
+      id: caseResult.id,
+      category: caseResult.category,
+      ecosystem: caseResult.ecosystem,
+      passed: caseResult.passed,
+      riskClassificationPassed: boolCheck(caseResult, 'riskLevel'),
+      humanReviewPassed: boolCheck(caseResult, 'humanReview'),
+      evidenceReferencesPassed: boolCheck(caseResult, 'evidenceReferences'),
+      validationPassed: boolCheck(caseResult, 'schema') && boolCheck(caseResult, 'validationState'),
+      deterministicPassed: deterministicPassed(caseResult),
+      unsupportedClaimCount: unsupportedClaimCount(caseResult),
+      actualFindingCount: actualFindingCount(caseResult)
+    }))
+    .sort((left, right) => compareText(left.id, right.id));
+}
+
+function metricsFromReport(report, cases) {
+  const total = report.cases.length;
+  const refCounts = report.cases.map(evidenceRefCounts);
+  const matchedReferenceCount = refCounts.reduce((sum, item) => sum + item.matched, 0);
+  const expectedReferenceCount = refCounts.reduce((sum, item) => sum + item.expected, 0);
+  const unsupportedClaimCountTotal = cases.reduce((sum, item) => sum + item.unsupportedClaimCount, 0);
+  const totalFindingCount = cases.reduce((sum, item) => sum + item.actualFindingCount, 0);
+  const unsupportedDenominator = totalFindingCount + unsupportedClaimCountTotal;
+
+  return {
+    casePassRate: metric(report.summary.passed, total),
+    riskClassificationAccuracy: metric(report.cases.filter((item) => boolCheck(item, 'riskLevel')).length, total),
+    humanReviewAccuracy: metric(report.cases.filter((item) => boolCheck(item, 'humanReview')).length, total),
+    humanReviewReasonAccuracy: metric(report.cases.filter((item) => boolCheck(item, 'humanReviewReasons')).length, total),
+    evidenceReferenceAccuracy: metric(report.cases.filter((item) => boolCheck(item, 'evidenceReferences')).length, total),
+    evidenceCoverageAccuracy: metric(report.cases.filter((item) => boolCheck(item, 'evidenceCoverage')).length, total),
+    evidenceReferenceCoverage: {
+      value: ratio(matchedReferenceCount, expectedReferenceCount),
+      matchedReferenceCount,
+      expectedReferenceCount
+    },
+    unsupportedClaimRate: {
+      value: ratio(unsupportedClaimCountTotal, unsupportedDenominator, 0),
+      unsupportedClaimCount: unsupportedClaimCountTotal,
+      totalFindingCount,
+      denominator: unsupportedDenominator
+    },
+    validationPassRate: metric(cases.filter((item) => item.validationPassed).length, total),
+    deterministicPassRate: metric(cases.filter((item) => item.deterministicPassed).length, total)
+  };
+}
+
+export function buildMetrics(report, { generatedAt = new Date() } = {}) {
+  const validated = validateEvaluationReport(report);
+  const cases = buildCaseMetrics(validated);
+  const metrics = {
+    schemaVersion: METRICS_SCHEMA_VERSION,
+    generatedAt: generatedAt instanceof Date ? generatedAt.toISOString() : generatedAt,
+    generator: { name: ARTIFACT_GENERATOR_NAME, version: VERSION },
+    input: buildInput(validated),
+    summary: structuredClone(validated.summary),
+    metrics: metricsFromReport(validated, cases),
+    cases
+  };
+  return validateMetrics(metrics);
+}
+
+export function validateMetrics(metrics) {
+  if (metrics?.schemaVersion !== METRICS_SCHEMA_VERSION) {
+    throw new Error(`Metrics validation error: unsupported schema version; expected ${METRICS_SCHEMA_VERSION}.`);
+  }
+  if (!validateSchema(metrics)) {
+    throw new Error(`Metrics validation error: schema validation failed: ${ajv.errorsText(validateSchema.errors, { separator: '; ' })}`);
+  }
+  return metrics;
+}
+
+export function serializeMetrics(metrics) {
+  validateMetrics(metrics);
+  return `${JSON.stringify(metrics, null, 2)}\n`;
+}
+
+export function metricsDigest(metrics) {
+  return `sha256:${createHash('sha256').update(serializeMetrics(metrics)).digest('hex')}`;
+}
+
+export async function loadEvaluationReportForMetrics(reportPath) {
+  try {
+    return validateEvaluationReport(JSON.parse(await readFile(reportPath, 'utf8')));
+  } catch (error) {
+    if (error instanceof SyntaxError) throw new Error(`Evaluation Report is not valid JSON: ${reportPath}.`);
+    throw error;
+  }
+}
+
+export async function writeMetrics(outputPath, metrics) {
+  const target = path.resolve(outputPath);
+  const temporary = `${target}.${process.pid}.tmp`;
+  const contents = serializeMetrics(metrics);
+  await mkdir(path.dirname(target), { recursive: true });
+  let handle;
+  try {
+    handle = await open(temporary, 'w', 0o600);
+    await handle.writeFile(contents, 'utf8');
+    await handle.sync();
+    await handle.close();
+    handle = null;
+    await rename(temporary, target);
+  } catch (error) {
+    await handle?.close().catch(() => {});
+    await rm(temporary, { force: true }).catch(() => {});
+    throw error;
+  }
+  return target;
+}

package/src/migration-checklist/ai-candidate.js

@@ -0,0 +1,320 @@
+import Ajv2020 from 'ajv/dist/2020.js';
+
+import { compareText } from '../portable.js';
+import { validateMigrationChecklistInstructionContent } from './grounding-policy.js';
+
+export const MIGRATION_CANDIDATE_STATUSES = Object.freeze(['ACTIONABLE', 'ABSTAIN']);
+export const MIGRATION_CANDIDATE_ABSTENTION_REASONS = Object.freeze([
+  'NO_EXPLICIT_ACTION',
+  'AMBIGUOUS_EVIDENCE',
+  'VERSION_SCOPE_UNCLEAR',
+  'CONFLICTING_EVIDENCE'
+]);
+export const MIGRATION_CHECKLIST_CANDIDATE_ERROR_CODES = Object.freeze([
+  'OUTPUT_JSON_INVALID',
+  'OUTPUT_SCHEMA_INVALID',
+  'OUTPUT_SEMANTICS_INVALID'
+]);
+export const MIGRATION_CHECKLIST_TRUST_ERROR_CODES = Object.freeze([
+  'EVIDENCE_REFERENCE_INVALID',
+  'SUPPORTING_EXCERPT_INVALID',
+  'PROHIBITED_CAPABILITY',
+  'UNSUPPORTED_IDENTIFIER',
+  'DUPLICATE_ITEM'
+]);
+
+const DIGEST_PATTERN = '^sha256:[a-f0-9]{64}$';
+const MAX_ITEMS = 4;
+const MAX_REFS = 6;
+const MAX_INSTRUCTION_CHARACTERS = 800;
+const MAX_EXCERPT_CHARACTERS = 500;
+
+export const MIGRATION_CHECKLIST_CANDIDATE_SCHEMA = deepFreeze({
+  $schema: 'https://json-schema.org/draft/2020-12/schema',
+  type: 'object',
+  additionalProperties: false,
+  required: ['status', 'items', 'abstentionReason'],
+  properties: {
+    status: { enum: [...MIGRATION_CANDIDATE_STATUSES] },
+    items: {
+      type: 'array',
+      maxItems: MAX_ITEMS,
+      items: {
+        type: 'object',
+        additionalProperties: false,
+        required: ['instruction', 'evidenceRefs', 'supportingExcerpts'],
+        properties: {
+          instruction: {
+            type: 'string',
+            minLength: 1,
+            maxLength: MAX_INSTRUCTION_CHARACTERS
+          },
+          evidenceRefs: {
+            type: 'array',
+            minItems: 1,
+            maxItems: MAX_REFS,
+            uniqueItems: true,
+            items: { type: 'string', pattern: DIGEST_PATTERN }
+          },
+          supportingExcerpts: {
+            type: 'array',
+            minItems: 1,
+            maxItems: MAX_REFS,
+            items: {
+              type: 'object',
+              additionalProperties: false,
+              required: ['evidenceRef', 'text'],
+              properties: {
+                evidenceRef: { type: 'string', pattern: DIGEST_PATTERN },
+                text: {
+                  type: 'string',
+                  minLength: 1,
+                  maxLength: MAX_EXCERPT_CHARACTERS
+                }
+              }
+            }
+          }
+        }
+      }
+    },
+    abstentionReason: {
+      enum: [null, ...MIGRATION_CANDIDATE_ABSTENTION_REASONS]
+    }
+  }
+});
+
+const ajv = new Ajv2020({ allErrors: true, strict: true });
+const validateSchema = ajv.compile(MIGRATION_CHECKLIST_CANDIDATE_SCHEMA);
+
+export class MigrationChecklistCandidateError extends Error {
+  constructor(code, message) {
+    if (!MIGRATION_CHECKLIST_CANDIDATE_ERROR_CODES.includes(code)) {
+      throw new TypeError('Unknown Migration Checklist candidate error code.');
+    }
+    super(message);
+    this.name = 'MigrationChecklistCandidateError';
+    this.code = code;
+  }
+}
+
+export class MigrationChecklistTrustError extends Error {
+  constructor(code, message, { detailCode } = {}) {
+    if (!MIGRATION_CHECKLIST_TRUST_ERROR_CODES.includes(code)) {
+      throw new TypeError('Unknown Migration Checklist trust error code.');
+    }
+    super(message);
+    this.name = 'MigrationChecklistTrustError';
+    this.code = code;
+    if (typeof detailCode === 'string') this.detailCode = detailCode;
+  }
+}
+
+export function isMigrationChecklistCandidateError(error) {
+  return error instanceof MigrationChecklistCandidateError;
+}
+
+export function isMigrationChecklistTrustError(error) {
+  return error instanceof MigrationChecklistTrustError;
+}
+
+function deepFreeze(value) {
+  if (!value || typeof value !== 'object' || Object.isFrozen(value)) return value;
+  for (const child of Object.values(value)) deepFreeze(child);
+  return Object.freeze(value);
+}
+
+function normalizeLineEndings(value) {
+  return value.replace(/\r\n?/g, '\n');
+}
+
+function parseCandidate(output) {
+  if (typeof output !== 'string') return output;
+  try {
+    return JSON.parse(output);
+  } catch {
+    throw new MigrationChecklistCandidateError(
+      'OUTPUT_JSON_INVALID',
+      'Migration Checklist candidate is not valid JSON.'
+    );
+  }
+}
+
+/** Parse and validate the strict model-owned candidate contract. */
+export function validateMigrationChecklistCandidate(output) {
+  const candidate = parseCandidate(output);
+  if (!validateSchema(candidate)) {
+    throw new MigrationChecklistCandidateError(
+      'OUTPUT_SCHEMA_INVALID',
+      `Migration Checklist candidate schema validation failed: ${ajv.errorsText(validateSchema.errors, { separator: '; ' })}`
+    );
+  }
+  if (candidate.status === 'ACTIONABLE'
+      && (candidate.items.length === 0 || candidate.abstentionReason !== null)) {
+    throw new MigrationChecklistCandidateError(
+      'OUTPUT_SEMANTICS_INVALID',
+      'ACTIONABLE candidates require items and a null abstentionReason.'
+    );
+  }
+  if (candidate.status === 'ABSTAIN'
+      && (candidate.items.length !== 0 || candidate.abstentionReason === null)) {
+    throw new MigrationChecklistCandidateError(
+      'OUTPUT_SEMANTICS_INVALID',
+      'ABSTAIN candidates require no items and a constrained abstentionReason.'
+    );
+  }
+  return deepFreeze(structuredClone(candidate));
+}
+
+const PROHIBITED_PATTERNS = Object.freeze([
+  ['ROLLBACK_PLAN', /\b(?:rollback|roll back|revert)\b/i],
+  ['EFFORT_ESTIMATE', /\b(?:(?:low|medium|high) effort|story points?|\d+(?:\.\d+)?\s*(?:minutes?|hours?|days?|weeks?)\b)/i],
+  ['NUMERIC_CONFIDENCE', /\b(?:confidence(?: score)?\s*(?:is|:|=)?\s*\d+(?:\.\d+)?\s*%?|\d+(?:\.\d+)?%\s+confiden)/i],
+  ['DEPENDENCY_ORDERING', /\b(?:before|after|first|then|next)\b[^.\n]{0,100}\b(?:install|migrate|update|upgrade)\b|\b(?:install|migrate|update|upgrade)\b[^.\n]{0,100}\b(?:before|after|first|then|next)\b|\b(?:prerequisite|required before|depends on)\b/i],
+  ['REPOSITORY_LOCATION', /(?:^|[\s("'])\.{0,2}\/[^\s)"']+|\b(?:src|lib|app|test|tests)\/[A-Za-z0-9_./-]+|\b[A-Za-z0-9_.-]+\.(?:jsx?|tsx?|py|java|go|rs|json|ya?ml|toml)\b/i],
+  ['SAFETY_OR_COMPLETION_CLAIM', /\b(?:safe to (?:upgrade|migrate)|ready to (?:upgrade|migrate|deploy)|(?:is|are|will be|now) (?:safe|ready|verified|complete)|upgrade is safe)\b/i],
+  ['COMMAND_OR_CODE', /(?:^|\n)\s*(?:\$|>|#)\s*\S|\b(?:npm|pnpm|yarn|npx|pip3?|poetry|uv|cargo|mvn|gradle|go|bash|zsh|curl|wget|git|rm|cp|mv|sed|awk)\s+\S+/i]
+]);
+
+const IDENTIFIER_PATTERNS = Object.freeze([
+  /`([^`\n]{1,80})`/g,
+  /\b--[a-z0-9][a-z0-9-]*\b/gi,
+  /\b[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)+\b/g,
+  /\b[a-z][A-Za-z0-9]*[A-Z][A-Za-z0-9]*\b/g,
+  /\b[A-Za-z][A-Za-z0-9]*_[A-Za-z0-9_]+\b/g,
+  /\b[A-Za-z_$][\w$]*\(\)/g
+]);
+
+const IDENTIFIER_STOPLIST = new Set([
+  'AI', 'API', 'CLI', 'CSS', 'HTML', 'HTTP', 'HTTPS', 'JSON', 'SDK', 'SQL', 'URL', 'XML'
+]);
+
+function technicalIdentifiers(instruction) {
+  const identifiers = new Set();
+  for (const expression of IDENTIFIER_PATTERNS) {
+    expression.lastIndex = 0;
+    for (const match of instruction.matchAll(expression)) {
+      const token = match[1] ?? match[0];
+      const normalized = token.endsWith('()') ? token.slice(0, -2) : token;
+      if (!IDENTIFIER_STOPLIST.has(normalized)) identifiers.add(normalized);
+    }
+  }
+  return [...identifiers].sort(compareText);
+}
+
+function trustError(code, message, detailCode) {
+  return new MigrationChecklistTrustError(code, message, { detailCode });
+}
+
+export function validateMigrationChecklistInstructionTrust(instruction, verifiedText) {
+  const inheritedViolations = validateMigrationChecklistInstructionContent(instruction);
+  if (inheritedViolations.length > 0) {
+    const violation = inheritedViolations[0];
+    throw trustError('PROHIBITED_CAPABILITY', violation.message, violation.code);
+  }
+  for (const [code, expression] of PROHIBITED_PATTERNS) {
+    if (expression.test(instruction)) {
+      throw trustError(
+        'PROHIBITED_CAPABILITY',
+        `Migration instruction contains prohibited capability ${code}.`,
+        code
+      );
+    }
+  }
+  for (const identifier of technicalIdentifiers(instruction)) {
+    if (!verifiedText.includes(identifier)) {
+      throw trustError(
+        'UNSUPPORTED_IDENTIFIER',
+        'Migration instruction contains an identifier absent from its verified supporting excerpts.',
+        'IDENTIFIER_NOT_IN_EXCERPT'
+      );
+    }
+  }
+}
+
+function compareExcerpt(left, right) {
+  return compareText(left.evidenceRef, right.evidenceRef) || compareText(left.text, right.text);
+}
+
+function compareItem(left, right) {
+  return compareText(left.instruction, right.instruction)
+    || compareText(JSON.stringify(left.evidenceRefs), JSON.stringify(right.evidenceRefs));
+}
+
+/**
+ * Fail-closed trust validation. One invalid item rejects the entire candidate;
+ * exact excerpt matching normalizes line endings only.
+ */
+export function trustValidateMigrationChecklistCandidate(candidate, context) {
+  const allowed = new Set(context.evidenceAllowlist);
+  const evidenceById = new Map(context.evidence.map((evidence) => [evidence.id, evidence]));
+  const normalizedItems = [];
+
+  for (const item of candidate.items) {
+    const refs = [...item.evidenceRefs].sort(compareText);
+    const refSet = new Set(refs);
+    for (const ref of refs) {
+      if (!allowed.has(ref) || !evidenceById.has(ref)) {
+        throw trustError(
+          'EVIDENCE_REFERENCE_INVALID',
+          'Migration instruction references evidence outside its exact context allowlist.',
+          'UNKNOWN_EVIDENCE_REFERENCE'
+        );
+      }
+    }
+
+    const excerptRefs = new Set();
+    const excerpts = [...item.supportingExcerpts].sort(compareExcerpt);
+    for (const excerpt of excerpts) {
+      if (!refSet.has(excerpt.evidenceRef)
+          || !allowed.has(excerpt.evidenceRef)
+          || !evidenceById.has(excerpt.evidenceRef)
+          || excerptRefs.has(excerpt.evidenceRef)) {
+        throw trustError(
+          'EVIDENCE_REFERENCE_INVALID',
+          'Each evidence reference must have exactly one excerpt from that same selected record.',
+          'EXCERPT_REFERENCE_MISMATCH'
+        );
+      }
+      excerptRefs.add(excerpt.evidenceRef);
+      const content = normalizeLineEndings(evidenceById.get(excerpt.evidenceRef).content);
+      const text = normalizeLineEndings(excerpt.text);
+      if (!content.includes(text)) {
+        throw trustError(
+          'SUPPORTING_EXCERPT_INVALID',
+          'Supporting excerpt is not an exact substring of its selected evidence record.',
+          'EXCERPT_NOT_EXACT'
+        );
+      }
+    }
+    if (excerptRefs.size !== refSet.size || refs.some((ref) => !excerptRefs.has(ref))) {
+      throw trustError(
+        'EVIDENCE_REFERENCE_INVALID',
+        'Every evidence reference must have exactly one supporting excerpt.',
+        'MISSING_SUPPORTING_EXCERPT'
+      );
+    }
+
+    const verifiedText = excerpts.map((excerpt) => normalizeLineEndings(excerpt.text)).join('\n');
+    validateMigrationChecklistInstructionTrust(item.instruction, verifiedText);
+    normalizedItems.push({
+      instruction: item.instruction,
+      evidenceRefs: refs,
+      supportingExcerpts: structuredClone(excerpts)
+    });
+  }
+
+  const itemKeys = normalizedItems.map((item) => JSON.stringify(item));
+  if (new Set(itemKeys).size !== itemKeys.length) {
+    throw trustError(
+      'DUPLICATE_ITEM',
+      'Migration candidate contains duplicate instructions.',
+      'DUPLICATE_INSTRUCTION'
+    );
+  }
+
+  return deepFreeze({
+    status: candidate.status,
+    items: normalizedItems.sort(compareItem),
+    abstentionReason: candidate.abstentionReason
+  });
+}

package/src/migration-checklist/assembler.js

@@ -0,0 +1,37 @@
+import { canonicalJson } from '../canonical-json.js';
+import { buildMigrationChecklist, validateMigrationChecklist } from './migration-checklist.js';
+
+function inputError(message) {
+  return new TypeError(`Migration Checklist assembly error: ${message}`);
+}
+
+/** Assemble normalized contexts plus versioned generator output through the MP-01 builder. */
+export function assembleMigrationChecklist({
+  prepared,
+  generation,
+  qualification,
+  generatedAt
+}) {
+  if (!prepared?.input || !Array.isArray(prepared.eligibleContexts)
+      || !Array.isArray(prepared.fallbackRecords)) {
+    throw inputError('prepared MP-02 contexts are required.');
+  }
+  if (!generation?.input || !Array.isArray(generation.records)
+      || !Array.isArray(generation.warnings)) {
+    throw inputError('normalized MP-03 generation output is required.');
+  }
+  if (canonicalJson(prepared.input) !== canonicalJson(generation.input)) {
+    throw inputError('MP-02 and MP-03 lineage inputs do not match.');
+  }
+  if (!qualification || !Array.isArray(qualification.limitations)) {
+    throw inputError('a qualification guard result is required.');
+  }
+  const checklist = buildMigrationChecklist({
+    input: prepared.input,
+    repository: prepared.input.projectManifest.repository,
+    dependencies: generation.records,
+    limitations: qualification.limitations,
+    generatedAt
+  });
+  return validateMigrationChecklist(checklist);
+}