@thomasminh1995/depverdict 0.6.0-alpha.1

package/src/artifact-root-compatibility.js

@@ -0,0 +1,91 @@
+import { access } from 'node:fs/promises';
+import path from 'node:path';
+
+import { DEFAULT_OUTPUT_DIRECTORY } from './constants.js';
+import { isPortableRelativePath } from './portable.js';
+
+export const LEGACY_OUTPUT_DIRECTORY = '.upgradelens';
+
+export class ArtifactRootConflictError extends Error {
+  constructor() {
+    super(
+      'Artifact root conflict: the required input chain is partial or split between '
+      + `${DEFAULT_OUTPUT_DIRECTORY}/ and ${LEGACY_OUTPUT_DIRECTORY}/.`
+    );
+    this.name = 'ArtifactRootConflictError';
+    this.code = 'ARTIFACT_ROOT_CONFLICT';
+  }
+}
+
+export function legacyArtifactPath(canonicalArtifact) {
+  const prefix = `${DEFAULT_OUTPUT_DIRECTORY}/`;
+  if (!isPortableRelativePath(canonicalArtifact) || !canonicalArtifact.startsWith(prefix)) {
+    throw new TypeError(
+      `Canonical artifact must be below ${DEFAULT_OUTPUT_DIRECTORY}/.`
+    );
+  }
+  return `${LEGACY_OUTPUT_DIRECTORY}/${canonicalArtifact.slice(prefix.length)}`;
+}
+
+async function exists(filename) {
+  try {
+    await access(filename);
+    return true;
+  } catch (error) {
+    if (error?.code === 'ENOENT' || error?.code === 'ENOTDIR') return false;
+    throw error;
+  }
+}
+
+/**
+ * Select one complete persisted artifact root. No returned chain ever mixes the
+ * canonical and legacy directories.
+ */
+export async function resolveArtifactRootChain(
+  repositoryRoot,
+  canonicalArtifacts,
+  options = {}
+) {
+  if (!Array.isArray(canonicalArtifacts) || canonicalArtifacts.length === 0) {
+    throw new TypeError('canonicalArtifacts must be a non-empty array.');
+  }
+  const canonical = [...canonicalArtifacts];
+  const legacy = canonical.map(legacyArtifactPath);
+  const [canonicalState, legacyState] = await Promise.all([
+    Promise.all(canonical.map((artifact) => exists(path.resolve(repositoryRoot, artifact)))),
+    Promise.all(legacy.map((artifact) => exists(path.resolve(repositoryRoot, artifact))))
+  ]);
+  const canonicalAll = canonicalState.every(Boolean);
+  const canonicalAny = canonicalState.some(Boolean);
+  const legacyAll = legacyState.every(Boolean);
+  const legacyAny = legacyState.some(Boolean);
+
+  if (canonicalAll) {
+    if (legacyAny) {
+      options.onDiagnostic?.(
+        `LEGACY_ARTIFACT_ROOT_IGNORED: using complete ${DEFAULT_OUTPUT_DIRECTORY}/ input chain.`
+      );
+    }
+    return Object.freeze({
+      root: DEFAULT_OUTPUT_DIRECTORY,
+      artifacts: Object.freeze(canonical)
+    });
+  }
+
+  if (!canonicalAny && legacyAll) {
+    options.onDiagnostic?.(
+      `LEGACY_ARTIFACT_ROOT_USED: using complete deprecated ${LEGACY_OUTPUT_DIRECTORY}/ input chain.`
+    );
+    return Object.freeze({
+      root: LEGACY_OUTPUT_DIRECTORY,
+      artifacts: Object.freeze(legacy)
+    });
+  }
+
+  if (canonicalAny || legacyAny) throw new ArtifactRootConflictError();
+
+  return Object.freeze({
+    root: DEFAULT_OUTPUT_DIRECTORY,
+    artifacts: Object.freeze(canonical)
+  });
+}

package/src/benchmark-report.js

@@ -0,0 +1,111 @@
+import { mkdir, open, readFile, rename, rm } from 'node:fs/promises';
+import path from 'node:path';
+
+import Ajv2020 from 'ajv/dist/2020.js';
+import addFormats from 'ajv-formats';
+
+import { ARTIFACT_GENERATOR_NAME, VERSION } from './constants.js';
+import { compareText } from './portable.js';
+
+export const BENCHMARK_REPORT_SCHEMA_VERSION = '1.0.0';
+export const DEFAULT_BENCHMARK_REPORT_PATH = 'benchmark-report.json';
+
+const schema = JSON.parse(await readFile(
+  new URL('../schemas/benchmark-report.schema.json', import.meta.url),
+  'utf8'
+));
+const ajv = new Ajv2020({ allErrors: true, strict: true, strictRequired: false });
+addFormats(ajv);
+const validateSchema = ajv.compile(schema);
+
+function nullableNumberCompare(left, right) {
+  if (left === null && right === null) return 0;
+  if (left === null) return 1;
+  if (right === null) return -1;
+  return left - right;
+}
+
+function compareRank(left, right) {
+  return right.metrics.overallScore - left.metrics.overallScore
+    || right.metrics.riskAccuracy - left.metrics.riskAccuracy
+    || right.metrics.evidenceQuality - left.metrics.evidenceQuality
+    || nullableNumberCompare(left.performance.latencyMs, right.performance.latencyMs)
+    || nullableNumberCompare(left.performance.estimatedCost, right.performance.estimatedCost)
+    || compareText(left.id, right.id);
+}
+
+function rankingFor(runs) {
+  return [...runs].sort(compareRank).map((run, index) => ({
+    rank: index + 1,
+    runId: run.id,
+    overallScore: run.metrics.overallScore,
+    riskAccuracy: run.metrics.riskAccuracy,
+    evidenceQuality: run.metrics.evidenceQuality,
+    latencyMs: run.performance.latencyMs,
+    estimatedCost: run.performance.estimatedCost
+  }));
+}
+
+export function buildBenchmarkReport({
+  benchmark,
+  runs,
+  generatedAt = new Date()
+}) {
+  const sortedRuns = [...runs].sort((left, right) => compareText(left.id, right.id));
+  const report = {
+    schemaVersion: BENCHMARK_REPORT_SCHEMA_VERSION,
+    generatedAt: generatedAt instanceof Date ? generatedAt.toISOString() : generatedAt,
+    generator: { name: ARTIFACT_GENERATOR_NAME, version: VERSION },
+    benchmark: {
+      name: benchmark.name,
+      configPath: benchmark.configPath ?? null,
+      datasetPath: benchmark.datasetPath,
+      runCount: sortedRuns.length
+    },
+    runs: sortedRuns,
+    ranking: rankingFor(sortedRuns)
+  };
+  return validateBenchmarkReport(report);
+}
+
+export function validateBenchmarkReport(report) {
+  if (report?.schemaVersion !== BENCHMARK_REPORT_SCHEMA_VERSION) {
+    throw new Error(`Benchmark Report validation error: unsupported schema version; expected ${BENCHMARK_REPORT_SCHEMA_VERSION}.`);
+  }
+  if (!validateSchema(report)) {
+    throw new Error(`Benchmark Report validation error: schema validation failed: ${ajv.errorsText(validateSchema.errors, { separator: '; ' })}`);
+  }
+  const runIds = new Set(report.runs.map((run) => run.id));
+  for (const entry of report.ranking) {
+    if (!runIds.has(entry.runId)) {
+      throw new Error(`Benchmark Report validation error: ranking references unknown run ${entry.runId}.`);
+    }
+  }
+  return report;
+}
+
+export function serializeBenchmarkReport(report) {
+  validateBenchmarkReport(report);
+  return `${JSON.stringify(report, null, 2)}\n`;
+}
+
+export async function writeBenchmarkReport(outputPath, report) {
+  const target = path.resolve(outputPath);
+  const temporary = `${target}.${process.pid}.tmp`;
+  const contents = serializeBenchmarkReport(report);
+  await mkdir(path.dirname(target), { recursive: true });
+  let handle;
+  try {
+    handle = await open(temporary, 'w', 0o600);
+    await handle.writeFile(contents, 'utf8');
+    await handle.sync();
+    await handle.close();
+    handle = null;
+    await rename(temporary, target);
+  } catch (error) {
+    await handle?.close().catch(() => {});
+    await rm(temporary, { force: true }).catch(() => {});
+    throw error;
+  }
+  return target;
+}

package/src/benchmark-runner.js

@@ -0,0 +1,191 @@
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+
+import Ajv2020 from 'ajv/dist/2020.js';
+
+import { buildAiScorecard } from './ai-scorecard.js';
+import {
+  buildBenchmarkReport,
+  serializeBenchmarkReport,
+  writeBenchmarkReport
+} from './benchmark-report.js';
+import { runEvaluation } from './evaluation-runner.js';
+import { buildMetrics } from './metrics-engine.js';
+import { compareText } from './portable.js';
+
+export const BENCHMARK_SCHEMA_VERSION = '1.0.0';
+export const DEFAULT_BENCHMARK_CONFIG_PATH = 'benchmark.json';
+
+const schema = JSON.parse(await readFile(
+  new URL('../schemas/benchmark.schema.json', import.meta.url),
+  'utf8'
+));
+const ajv = new Ajv2020({ allErrors: true, strict: true, strictRequired: false });
+const validateSchema = ajv.compile(schema);
+
+function benchmarkError(message) {
+  return new Error(`Benchmark config error: ${message}`);
+}
+
+function sortedUnique(values = []) {
+  return [...new Set(values)].sort(compareText);
+}
+
+function validateBenchmarkConfigInvariants(config) {
+  const ids = config.runs.map((run) => run.id);
+  const duplicateIds = sortedUnique(ids.filter((id, index) => ids.indexOf(id) !== index));
+  if (duplicateIds.length > 0) throw benchmarkError(`duplicate run id ${duplicateIds[0]}.`);
+  return config;
+}
+
+export function validateBenchmarkConfig(config) {
+  if (config?.schemaVersion !== BENCHMARK_SCHEMA_VERSION) {
+    throw benchmarkError(`unsupported schema version; expected ${BENCHMARK_SCHEMA_VERSION}.`);
+  }
+  if (!validateSchema(config)) {
+    throw benchmarkError(`schema validation failed: ${ajv.errorsText(validateSchema.errors, { separator: '; ' })}`);
+  }
+  return validateBenchmarkConfigInvariants(config);
+}
+
+export async function loadBenchmarkConfig(configPath = DEFAULT_BENCHMARK_CONFIG_PATH) {
+  try {
+    return validateBenchmarkConfig(JSON.parse(await readFile(configPath, 'utf8')));
+  } catch (error) {
+    if (error instanceof SyntaxError) throw benchmarkError(`${configPath} is not valid JSON.`);
+    throw error;
+  }
+}
+
+function numberOrNull(value) {
+  return Number.isFinite(value) && value >= 0 ? value : null;
+}
+
+function tokenUsageFrom(response) {
+  if (Number.isFinite(response?.tokenUsage)) return response.tokenUsage;
+  if (Number.isFinite(response?.usage?.totalTokens)) return response.usage.totalTokens;
+  if (Number.isFinite(response?.usage?.total_tokens)) return response.usage.total_tokens;
+  return null;
+}
+
+function costFrom(response) {
+  if (Number.isFinite(response?.estimatedCost)) return response.estimatedCost;
+  if (Number.isFinite(response?.cost?.estimated)) return response.cost.estimated;
+  return null;
+}
+
+function createPerformanceCollector() {
+  const values = {
+    latencyMs: [],
+    tokenUsage: [],
+    estimatedCost: []
+  };
+  return {
+    record(response) {
+      const latencyMs = numberOrNull(response?.latencyMs);
+      const tokenUsage = numberOrNull(tokenUsageFrom(response));
+      const estimatedCost = numberOrNull(costFrom(response));
+      if (latencyMs !== null) values.latencyMs.push(latencyMs);
+      if (tokenUsage !== null) values.tokenUsage.push(tokenUsage);
+      if (estimatedCost !== null) values.estimatedCost.push(estimatedCost);
+    },
+    summary() {
+      return {
+        latencyMs: values.latencyMs.length === 0
+          ? null
+          : values.latencyMs.reduce((sum, value) => sum + value, 0) / values.latencyMs.length,
+        tokenUsage: values.tokenUsage.length === 0
+          ? null
+          : values.tokenUsage.reduce((sum, value) => sum + value, 0),
+        estimatedCost: values.estimatedCost.length === 0
+          ? null
+          : values.estimatedCost.reduce((sum, value) => sum + value, 0)
+      };
+    }
+  };
+}
+
+function wrapRuntime(runtime, collector) {
+  if (!runtime) return null;
+  return {
+    async generateStructured(request) {
+      const response = await runtime.generateStructured(request);
+      collector.record(response);
+      return response;
+    }
+  };
+}
+
+function runMetrics(metrics, scorecard) {
+  return {
+    overallScore: scorecard.overallScore,
+    riskAccuracy: metrics.metrics.riskClassificationAccuracy.value,
+    humanReviewAccuracy: metrics.metrics.humanReviewAccuracy.value,
+    evidenceQuality: scorecard.categoryScores.evidenceQuality,
+    unsupportedClaimRate: metrics.metrics.unsupportedClaimRate.value,
+    deterministicPassRate: metrics.metrics.deterministicPassRate.value
+  };
+}
+
+function scorecardSummary(scorecard) {
+  return {
+    overallScore: scorecard.overallScore,
+    categoryScores: structuredClone(scorecard.categoryScores),
+    recommendationCodes: scorecard.recommendations.map((item) => item.code).sort(compareText)
+  };
+}
+
+async function defaultRuntimeFactory(run) {
+  if (run.runtime.type === 'goldenFake') return null;
+  throw new Error(`Benchmark runtime ${run.runtime.type} requires an injected runtimeFactory.`);
+}
+
+export async function runBenchmark(configInput, {
+  configPath = null,
+  runtimeFactory = defaultRuntimeFactory,
+  runEvaluationImplementation = runEvaluation,
+  buildMetricsImplementation = buildMetrics,
+  buildAiScorecardImplementation = buildAiScorecard,
+  generatedAt = new Date()
+} = {}) {
+  const config = validateBenchmarkConfig(configInput);
+  const runs = [];
+  for (const run of config.runs) {
+    const collector = createPerformanceCollector();
+    const runtime = await runtimeFactory(run, config);
+    const evaluationReport = await runEvaluationImplementation({
+      datasetPath: run.datasetPath ?? config.datasetPath,
+      runtime: wrapRuntime(runtime, collector),
+      model: run.model,
+      promptVersion: run.promptVersion,
+      generatedAt
+    });
+    const metrics = buildMetricsImplementation(evaluationReport, { generatedAt });
+    const scorecard = buildAiScorecardImplementation(metrics, { generatedAt });
+    runs.push({
+      id: run.id,
+      promptVersion: run.promptVersion,
+      model: structuredClone(run.model),
+      runtime: structuredClone(run.runtime),
+      datasetPath: path.resolve(run.datasetPath ?? config.datasetPath),
+      evaluation: structuredClone(evaluationReport.summary),
+      metrics: runMetrics(metrics, scorecard),
+      scorecard: scorecardSummary(scorecard),
+      performance: collector.summary()
+    });
+  }
+  return buildBenchmarkReport({
+    benchmark: {
+      name: config.name,
+      configPath,
+      datasetPath: path.resolve(config.datasetPath)
+    },
+    runs,
+    generatedAt
+  });
+}
+
+export {
+  serializeBenchmarkReport,
+  writeBenchmarkReport
+};

package/src/canonical-json.js

@@ -0,0 +1,69 @@
+import { compareText } from './portable.js';
+
+export class CanonicalJsonError extends Error {
+  constructor(message) {
+    super(message);
+    this.name = 'CanonicalJsonError';
+    this.code = 'KNOWLEDGE_CACHE_INVALID_BODY';
+  }
+}
+
+function fail(path, message) {
+  throw new CanonicalJsonError(`Invalid JSON value at ${path}: ${message}`);
+}
+
+/**
+ * Serialize a JSON-compatible value with code-unit-sorted object keys. Arrays
+ * retain their order, so their order remains part of the content digest.
+ */
+export function canonicalJson(value) {
+  const ancestors = new Set();
+
+  function serialize(current, currentPath) {
+    if (current === null) return 'null';
+
+    switch (typeof current) {
+      case 'string':
+        return JSON.stringify(current);
+      case 'boolean':
+        return current ? 'true' : 'false';
+      case 'number':
+        if (!Number.isFinite(current)) fail(currentPath, 'numbers must be finite.');
+        return JSON.stringify(current);
+      case 'undefined':
+      case 'function':
+      case 'symbol':
+      case 'bigint':
+        fail(currentPath, `${typeof current} values are not JSON-compatible.`);
+      case 'object':
+        break;
+      default:
+        fail(currentPath, 'unsupported value type.');
+    }
+
+    if (ancestors.has(current)) fail(currentPath, 'circular references are not supported.');
+    ancestors.add(current);
+    try {
+      if (Array.isArray(current)) {
+        return `[${current.map((item, index) => serialize(item, `${currentPath}[${index}]`)).join(',')}]`;
+      }
+
+      const prototype = Object.getPrototypeOf(current);
+      if (prototype !== Object.prototype && prototype !== null) {
+        fail(currentPath, 'objects must be plain JSON objects.');
+      }
+      const entries = Object.keys(current)
+        .sort(compareText)
+        .map((key) => `${JSON.stringify(key)}:${serialize(current[key], `${currentPath}.${key}`)}`);
+      return `{${entries.join(',')}}`;
+    } finally {
+      ancestors.delete(current);
+    }
+  }
+
+  return serialize(value, '$');
+}
+
+export function canonicalJsonBytes(value) {
+  return Buffer.from(canonicalJson(value), 'utf8');
+}