@thomasminh1995/depverdict 0.6.0-alpha.1

package/src/usage/runtime.js

@@ -0,0 +1,187 @@
+import { readFile, realpath, stat } from 'node:fs/promises';
+import path from 'node:path';
+
+import { DEFAULT_MANIFEST_PATH, DEFAULT_VERSION_ANALYSIS_PATH } from '../constants.js';
+import { createCooperativeScheduler } from '../cooperative-scheduler.js';
+import { relativePath } from '../files.js';
+import { createUsageAnalyzerRegistry } from './analyzer-registry.js';
+import { buildProjectUsageCoverage } from './coverage.js';
+import { loadUsageDiscoveryInputs } from './input-loader.js';
+import { createJavaScriptUsageAnalyzer } from './js/analyzer.js';
+import { createUsageDiscoveryScope } from './scope.js';
+import { collectUsageSourceFiles } from './source-files.js';
+import { buildUsageIndex } from './usage-index.js';
+
+async function repositoryDirectory(inputPath) {
+  const resolved = path.resolve(inputPath);
+  let details;
+  try {
+    details = await stat(resolved);
+  } catch (error) {
+    throw new Error(`Cannot access repository root "${inputPath}": ${error.message}`);
+  }
+  if (!details.isDirectory()) throw new Error(`Repository root is not a directory: ${inputPath}`);
+  return realpath(resolved);
+}
+
+function pathBelongsToProject(file, projectPath) {
+  if (projectPath === '.') return true;
+  return file === projectPath || file.startsWith(`${projectPath}/`);
+}
+
+function owningProject(file, scopes, registry) {
+  const candidates = scopes.filter((scope) => (
+    pathBelongsToProject(file, scope.projectPath) && registry.find(scope.ecosystem, file)
+  ));
+  return candidates.sort((left, right) => (
+    right.projectPath.split('/').length - left.projectPath.split('/').length
+    || left.projectId.localeCompare(right.projectId)
+  ))[0] ?? null;
+}
+
+function coverageOwner(warningPath, projects) {
+  return projects
+    .filter((project) => pathBelongsToProject(warningPath, project.projectPath))
+    .sort((left, right) => (
+      right.projectPath.split('/').length - left.projectPath.split('/').length
+      || left.projectId.localeCompare(right.projectId)
+    ))[0] ?? null;
+}
+
+export function createDefaultUsageAnalyzerRegistry() {
+  return createUsageAnalyzerRegistry([createJavaScriptUsageAnalyzer()]);
+}
+
+export async function discoverRepositoryUsage({
+  repositoryRoot,
+  projectManifest,
+  versionAnalysis,
+  input,
+  registry = createDefaultUsageAnalyzerRegistry(),
+  clock,
+  maxDepth,
+  signal,
+  cooperativeScheduler: injectedCooperativeScheduler,
+  cooperativeScheduling = true,
+  cooperativeBatchSize,
+  cooperativeMaxIntervalMs,
+  monotonicClock
+}) {
+  const cooperativeScheduler = injectedCooperativeScheduler ?? createCooperativeScheduler({
+    signal,
+    enabled: cooperativeScheduling,
+    batchSize: cooperativeBatchSize,
+    maxIntervalMs: cooperativeMaxIntervalMs,
+    monotonicClock
+  });
+  cooperativeScheduler.checkpoint();
+  const root = await repositoryDirectory(repositoryRoot);
+  const scopes = createUsageDiscoveryScope(projectManifest, versionAnalysis);
+  const projectScopes = projectManifest.projects.map((project) => ({
+    projectId: project.id,
+    projectPath: project.path,
+    ecosystem: project.ecosystem
+  }));
+  const dependenciesByProject = new Map();
+  for (const scope of scopes) {
+    if (!dependenciesByProject.has(scope.projectId)) dependenciesByProject.set(scope.projectId, []);
+    dependenciesByProject.get(scope.projectId).push(scope);
+    await cooperativeScheduler.boundary();
+  }
+  const collected = await collectUsageSourceFiles(root, registry.extensions(), {
+    maxDepth,
+    cooperativeScheduler
+  });
+  const warnings = [...collected.warnings];
+  const usages = [];
+  let scannedFileCount = 0;
+  let analyzedFileCount = 0;
+  const coverageCounts = new Map(projectScopes.map((project) => [project.projectId, {
+    scannedFileCount: 0,
+    analyzedFileCount: 0,
+    parseFailureCount: 0,
+    analyzerFailureCount: 0,
+    unreadableFileCount: 0,
+    scanFailureCount: 0
+  }]));
+  for (const warning of collected.warnings) {
+    const owner = coverageOwner(warning.path, projectScopes);
+    if (owner) coverageCounts.get(owner.projectId).scanFailureCount += 1;
+  }
+
+  for (const absoluteFile of collected.files) {
+    try {
+      const file = relativePath(root, absoluteFile);
+      const project = owningProject(file, projectScopes, registry);
+      if (!project) continue;
+      if (!dependenciesByProject.has(project.projectId)) continue;
+      const analyzer = registry.find(project.ecosystem, file);
+      if (!analyzer) continue;
+      scannedFileCount += 1;
+      const counts = coverageCounts.get(project.projectId);
+      counts.scannedFileCount += 1;
+      let source;
+      try {
+        source = await readFile(absoluteFile, 'utf8');
+      } catch (error) {
+        warnings.push({
+          code: 'FILE_UNREADABLE',
+          path: file,
+          message: `Unable to read source file (${error.code ?? 'unknown error'}).`
+        });
+        counts.unreadableFileCount += 1;
+        continue;
+      }
+      try {
+        const discovered = await analyzer.analyze({
+          source,
+          file,
+          projectId: project.projectId,
+          dependencies: dependenciesByProject.get(project.projectId) ?? []
+        });
+        usages.push(...discovered.map((usage) => ({ ...usage, projectId: project.projectId })));
+        analyzedFileCount += 1;
+        counts.analyzedFileCount += 1;
+      } catch (error) {
+        warnings.push({
+          code: error instanceof SyntaxError ? 'SOURCE_PARSE_FAILED' : 'ANALYZER_FAILED',
+          path: file,
+          message: error instanceof SyntaxError
+            ? 'Unable to parse source file.'
+            : `Analyzer failed (${error.code ?? error.name ?? 'unknown error'}).`
+        });
+        if (error instanceof SyntaxError) counts.parseFailureCount += 1;
+        else counts.analyzerFailureCount += 1;
+      }
+    } finally {
+      await cooperativeScheduler.boundary();
+    }
+  }
+
+  cooperativeScheduler.checkpoint();
+  const coverage = projectScopes.map((project) => buildProjectUsageCoverage({
+    ...project,
+    analyzers: registry.forEcosystem(project.ecosystem),
+    ...coverageCounts.get(project.projectId)
+  }));
+  return buildUsageIndex({
+    input,
+    usages,
+    scannedFileCount,
+    analyzedFileCount,
+    analyzers: registry.analyzers(),
+    coverage,
+    warnings,
+    generatedAt: clock?.() ?? new Date()
+  });
+}
+
+export async function runUsageDiscovery({ repositoryRoot = '.', sources, ...options } = {}) {
+  const root = path.resolve(repositoryRoot);
+  const effectiveSources = sources ?? {
+    projectManifest: path.join(root, DEFAULT_MANIFEST_PATH),
+    versionAnalysis: path.join(root, DEFAULT_VERSION_ANALYSIS_PATH)
+  };
+  const artifacts = await loadUsageDiscoveryInputs(effectiveSources, options);
+  return discoverRepositoryUsage({ repositoryRoot, ...artifacts, ...options });
+}

package/src/usage/scope.js

@@ -0,0 +1,44 @@
+import { compareText } from '../portable.js';
+
+function packageNameFromId(packageId) {
+  const separator = packageId.indexOf(':');
+  return separator === -1 ? packageId : packageId.slice(separator + 1);
+}
+
+export function createUsageDiscoveryScope(projectManifest, versionAnalysisManifest) {
+  const projects = new Map(projectManifest.projects.map((project) => [project.id, project]));
+  const scoped = new Map();
+
+  for (const result of versionAnalysisManifest.results) {
+    const project = projects.get(result.dependency.projectId);
+    if (!project) {
+      throw new Error(`Usage Discovery input error: unknown project ${result.dependency.projectId}.`);
+    }
+    if (project.ecosystem !== result.dependency.ecosystem) {
+      throw new Error(`Usage Discovery input error: ecosystem mismatch for ${project.id}.`);
+    }
+    const declared = project.dependencies.find((dependency) => (
+      dependency.normalizedName === result.dependency.normalizedName
+      && dependency.type === result.dependency.dependencyType
+      && dependency.manifest === result.dependency.manifest
+    ));
+    if (!declared) {
+      throw new Error(
+        `Usage Discovery input error: ${result.dependency.packageId} is not declared by ${project.id}.`
+      );
+    }
+    const key = `${project.id}\0${result.dependency.packageId}`;
+    scoped.set(key, {
+      projectId: project.id,
+      projectPath: project.path,
+      ecosystem: project.ecosystem,
+      packageId: result.dependency.packageId,
+      name: declared.name || packageNameFromId(result.dependency.packageId),
+      normalizedName: declared.normalizedName
+    });
+  }
+
+  return [...scoped.values()].sort((left, right) => (
+    compareText(left.projectId, right.projectId) || compareText(left.packageId, right.packageId)
+  ));
+}

package/src/usage/source-files.js

@@ -0,0 +1,50 @@
+import { readdir } from 'node:fs/promises';
+import path from 'node:path';
+
+import { DEFAULT_IGNORED_DIRECTORIES } from '../constants.js';
+import { relativePath } from '../files.js';
+
+export async function collectUsageSourceFiles(root, extensions, options = {}) {
+  const supportedExtensions = new Set(extensions.map((extension) => extension.toLowerCase()));
+  const maxDepth = options.maxDepth ?? Number.POSITIVE_INFINITY;
+  const cooperativeScheduler = options.cooperativeScheduler;
+  const files = [];
+  const warnings = [];
+
+  async function visit(directory, depth) {
+    cooperativeScheduler?.checkpoint();
+    let entries;
+    try {
+      entries = await readdir(directory, { withFileTypes: true });
+    } catch (error) {
+      warnings.push({
+        code: 'DIRECTORY_UNREADABLE',
+        path: relativePath(root, directory),
+        message: `Unable to read directory (${error.code ?? 'unknown error'}).`
+      });
+      return;
+    }
+
+    entries.sort((left, right) => left.name.localeCompare(right.name));
+    for (const entry of entries) {
+      const entryPath = path.join(directory, entry.name);
+      try {
+        if (entry.isSymbolicLink()) continue;
+        if (entry.isDirectory()) {
+          if (depth < maxDepth && !DEFAULT_IGNORED_DIRECTORIES.has(entry.name)) {
+            await visit(entryPath, depth + 1);
+          }
+        } else if (entry.isFile() && supportedExtensions.has(path.extname(entry.name).toLowerCase())) {
+          files.push(entryPath);
+        }
+      } finally {
+        await cooperativeScheduler?.boundary();
+      }
+    }
+  }
+
+  await visit(root, 0);
+  return { files, warnings };
+}
+
+export const collectRepositorySourceFiles = collectUsageSourceFiles;

package/src/usage/usage-index.js

@@ -0,0 +1,217 @@
+import { readFile } from 'node:fs/promises';
+
+import Ajv2020 from 'ajv/dist/2020.js';
+import addFormats from 'ajv-formats';
+
+import {
+  ARTIFACT_GENERATOR_NAME,
+  USAGE_INDEX_SCHEMA_VERSION,
+  VERSION
+} from '../constants.js';
+import { compareText, isSorted } from '../portable.js';
+import {
+  USAGE_COVERAGE_REASON_CODES,
+  USAGE_COVERAGE_STATUSES,
+  compareCoverage
+} from './coverage.js';
+
+const schema = JSON.parse(await readFile(
+  new URL('../../schemas/usage-index.schema.json', import.meta.url),
+  'utf8'
+));
+const ajv = new Ajv2020({ allErrors: true, strict: true });
+addFormats(ajv);
+const validateSchema = ajv.compile(schema);
+
+const compareDependencies = (left, right) => (
+  compareText(left.projectId, right.projectId) || compareText(left.packageId, right.packageId)
+);
+const compareSymbols = (left, right) => compareText(left.name, right.name);
+const compareWarnings = (left, right) => (
+  compareText(left.path, right.path) || compareText(left.code, right.code) || compareText(left.message, right.message)
+);
+const compareAnalyzers = (left, right) => compareText(left.id, right.id) || compareText(left.version, right.version);
+
+function indexUsages(usages) {
+  const dependencies = new Map();
+  for (const usage of usages) {
+    const dependencyKey = `${usage.projectId}\0${usage.packageId}`;
+    if (!dependencies.has(dependencyKey)) {
+      dependencies.set(dependencyKey, {
+        projectId: usage.projectId,
+        packageId: usage.packageId,
+        name: usage.dependency,
+        files: new Set(),
+        symbols: new Map()
+      });
+    }
+    const dependency = dependencies.get(dependencyKey);
+    dependency.files.add(usage.file);
+    if (usage.symbol === null) continue;
+    if (!dependency.symbols.has(usage.symbol)) dependency.symbols.set(usage.symbol, new Set());
+    dependency.symbols.get(usage.symbol).add(usage.file);
+  }
+
+  return [...dependencies.values()].map((dependency) => ({
+    projectId: dependency.projectId,
+    packageId: dependency.packageId,
+    name: dependency.name,
+    files: [...dependency.files].sort(compareText),
+    symbols: [...dependency.symbols].map(([name, files]) => ({
+      name,
+      files: [...files].sort(compareText)
+    })).sort(compareSymbols)
+  })).sort(compareDependencies);
+}
+
+function summary(dependencies, warningCount) {
+  const files = new Set(dependencies.flatMap((dependency) => dependency.files));
+  return {
+    dependencyCount: dependencies.length,
+    symbolCount: dependencies.reduce((count, dependency) => count + dependency.symbols.length, 0),
+    fileCount: files.size,
+    warningCount
+  };
+}
+
+export function validateUsageIndexInvariants(index) {
+  const errors = [];
+  if (!isSorted(index.dependencies, compareDependencies)) errors.push('dependencies must be sorted.');
+  if (!isSorted(index.analysis.analyzers, compareAnalyzers)) errors.push('analyzers must be sorted.');
+  if (index.analysis.coverage && !isSorted(index.analysis.coverage, compareCoverage)) {
+    errors.push('analysis.coverage must be sorted.');
+  }
+  if (new Set(index.analysis.analyzers.map((analyzer) => analyzer.id)).size !== index.analysis.analyzers.length) {
+    errors.push('analyzer ids must be unique.');
+  }
+  if (index.analysis.analyzedFileCount > index.analysis.scannedFileCount) {
+    errors.push('analysis.analyzedFileCount cannot exceed scannedFileCount.');
+  }
+  const coverageProjects = new Set();
+  let coverageScannedFileCount = 0;
+  let coverageAnalyzedFileCount = 0;
+  for (const coverage of index.analysis.coverage ?? []) {
+    if (coverageProjects.has(coverage.projectId)) {
+      errors.push(`duplicate coverage project ${coverage.projectId}.`);
+    }
+    coverageProjects.add(coverage.projectId);
+    if (!USAGE_COVERAGE_STATUSES.includes(coverage.status)) {
+      errors.push(`coverage ${coverage.projectId} has unsupported status.`);
+    }
+    if (!USAGE_COVERAGE_REASON_CODES.includes(coverage.reasonCode)) {
+      errors.push(`coverage ${coverage.projectId} has unsupported reasonCode.`);
+    }
+    if (coverage.analyzedFileCount > coverage.scannedFileCount) {
+      errors.push(`coverage ${coverage.projectId} analyzedFileCount cannot exceed scannedFileCount.`);
+    }
+    coverageScannedFileCount += coverage.scannedFileCount;
+    coverageAnalyzedFileCount += coverage.analyzedFileCount;
+    const processedFileCount = coverage.analyzedFileCount + coverage.parseFailureCount
+      + coverage.analyzerFailureCount + coverage.unreadableFileCount;
+    if (processedFileCount !== coverage.scannedFileCount) {
+      errors.push(`coverage ${coverage.projectId} file counts are inconsistent.`);
+    }
+    if ((coverage.status === 'unavailable') !== (coverage.analyzer === null)) {
+      errors.push(`coverage ${coverage.projectId} analyzer availability is inconsistent.`);
+    }
+    const failures = coverage.parseFailureCount + coverage.analyzerFailureCount
+      + coverage.unreadableFileCount + coverage.scanFailureCount;
+    if (coverage.status === 'complete' && failures !== 0) {
+      errors.push(`coverage ${coverage.projectId} complete status has failures.`);
+    }
+    if (coverage.status === 'complete' && coverage.reasonCode !== 'COVERAGE_COMPLETE') {
+      errors.push(`coverage ${coverage.projectId} complete reasonCode is inconsistent.`);
+    }
+    if (coverage.analyzer
+        && !index.analysis.analyzers.some((analyzer) => (
+          analyzer.id === coverage.analyzer.id && analyzer.version === coverage.analyzer.version
+        ))) {
+      errors.push(`coverage ${coverage.projectId} analyzer is not declared by analysis.`);
+    }
+  }
+  if ((index.analysis.coverage?.length ?? 0) > 0) {
+    if (coverageScannedFileCount !== index.analysis.scannedFileCount) {
+      errors.push('analysis.scannedFileCount does not equal project coverage total.');
+    }
+    if (coverageAnalyzedFileCount !== index.analysis.analyzedFileCount) {
+      errors.push('analysis.analyzedFileCount does not equal project coverage total.');
+    }
+  }
+  if (!isSorted(index.warnings, compareWarnings)) errors.push('warnings must be sorted.');
+  const identities = new Set();
+  for (const dependency of index.dependencies) {
+    const identity = `${dependency.projectId}\0${dependency.packageId}`;
+    if (identities.has(identity)) errors.push(`duplicate dependency ${dependency.packageId} in ${dependency.projectId}.`);
+    identities.add(identity);
+    if (!isSorted(dependency.files, compareText)) errors.push(`files for ${dependency.packageId} must be sorted.`);
+    if (new Set(dependency.files).size !== dependency.files.length) {
+      errors.push(`files for ${dependency.packageId} must be unique.`);
+    }
+    if (!isSorted(dependency.symbols, compareSymbols)) {
+      errors.push(`symbols for ${dependency.packageId} must be sorted.`);
+    }
+    if (new Set(dependency.symbols.map((symbol) => symbol.name)).size !== dependency.symbols.length) {
+      errors.push(`symbols for ${dependency.packageId} must be unique.`);
+    }
+    const dependencyFiles = new Set(dependency.files);
+    for (const symbol of dependency.symbols) {
+      if (!isSorted(symbol.files, compareText)) {
+        errors.push(`files for ${dependency.packageId}/${symbol.name} must be sorted.`);
+      }
+      if (new Set(symbol.files).size !== symbol.files.length) {
+        errors.push(`files for ${dependency.packageId}/${symbol.name} must be unique.`);
+      }
+      for (const file of symbol.files) {
+        if (!dependencyFiles.has(file)) {
+          errors.push(`file ${file} for ${dependency.packageId}/${symbol.name} is missing from dependency files.`);
+        }
+      }
+    }
+  }
+  const expected = summary(index.dependencies, index.warnings.length);
+  for (const [field, value] of Object.entries(expected)) {
+    if (index.summary[field] !== value) {
+      errors.push(`summary.${field} is ${index.summary[field]}; expected ${value}.`);
+    }
+  }
+  return errors.sort(compareText);
+}
+
+export function validateUsageIndex(index) {
+  if (!validateSchema(index)) {
+    throw new Error(`Usage Index validation error: ${ajv.errorsText(validateSchema.errors, { separator: '; ' })}`);
+  }
+  const errors = validateUsageIndexInvariants(index);
+  if (errors.length > 0) throw new Error(`Usage Index invariant error: ${errors.join(' ')}`);
+  return index;
+}
+
+export function buildUsageIndex({
+  input,
+  usages,
+  scannedFileCount,
+  analyzedFileCount,
+  analyzers,
+  coverage = [],
+  warnings,
+  generatedAt = new Date()
+}) {
+  const dependencies = indexUsages(usages);
+  const sortedWarnings = structuredClone(warnings).sort(compareWarnings);
+  const sortedAnalyzers = analyzers.map(({ id, version }) => ({ id, version })).sort(compareAnalyzers);
+  return validateUsageIndex({
+    schemaVersion: USAGE_INDEX_SCHEMA_VERSION,
+    generatedAt: generatedAt instanceof Date ? generatedAt.toISOString() : generatedAt,
+    generator: { name: ARTIFACT_GENERATOR_NAME, version: VERSION },
+    input: structuredClone(input),
+    analysis: {
+      analyzers: sortedAnalyzers,
+      scannedFileCount,
+      analyzedFileCount,
+      coverage: structuredClone(coverage).sort(compareCoverage)
+    },
+    summary: summary(dependencies, sortedWarnings.length),
+    dependencies,
+    warnings: sortedWarnings
+  });
+}

package/src/usage/writer.js

@@ -0,0 +1,31 @@
+import { mkdir, open, rename, rm } from 'node:fs/promises';
+import path from 'node:path';
+
+import { validateUsageIndex } from './usage-index.js';
+
+export function serializeUsageIndex(index) {
+  validateUsageIndex(index);
+  return `${JSON.stringify(index, null, 2)}\n`;
+}
+
+export async function writeUsageIndex(outputPath, index) {
+  const target = path.resolve(outputPath);
+  const temporary = `${target}.${process.pid}.tmp`;
+  const contents = serializeUsageIndex(index);
+  await mkdir(path.dirname(target), { recursive: true });
+  let handle;
+  try {
+    handle = await open(temporary, 'w', 0o600);
+    await handle.writeFile(contents, 'utf8');
+    await handle.sync();
+    await handle.close();
+    handle = null;
+    await rename(temporary, target);
+  } catch (error) {
+    await handle?.close().catch(() => {});
+    await rm(temporary, { force: true }).catch(() => {});
+    throw error;
+  }
+  return target;
+}
+