@thomasminh1995/depverdict 0.6.0-alpha.1

package/src/knowledge-evidence-producer.js

@@ -0,0 +1,233 @@
+import { createHash } from 'node:crypto';
+import { mkdir, open, rename, rm } from 'node:fs/promises';
+import path from 'node:path';
+
+import { canonicalJson } from './canonical-json.js';
+import { ARTIFACT_GENERATOR_NAME, VERSION } from './constants.js';
+import {
+  KNOWLEDGE_EVIDENCE_BUNDLE_SCHEMA_VERSION,
+  validateKnowledgeEvidenceBundle
+} from './knowledge-evidence-bundle.js';
+import { validateKnowledgeManifest } from './knowledge-manifest-builder.js';
+import { compareText } from './portable.js';
+
+function digestBytes(bytes) {
+  return `sha256:${createHash('sha256').update(bytes).digest('hex')}`;
+}
+
+function digestText(value) {
+  return digestBytes(Buffer.from(value, 'utf8'));
+}
+
+function digestJson(value) {
+  return digestText(canonicalJson(value));
+}
+
+function textValue(value) {
+  if (value === null || value === undefined) return 'unknown';
+  if (typeof value === 'boolean') return value ? 'true' : 'false';
+  return String(value);
+}
+
+function sourceFor(manifest, sourceId) {
+  return manifest.sources.find((source) => source.id === sourceId) ?? null;
+}
+
+function retrievedAtFor(manifest, source) {
+  return source?.snapshot?.retrievedAt ?? manifest.generatedAt;
+}
+
+function latestEvidenceContent(packageRecord) {
+  return [
+    `Package: ${packageRecord.id}`,
+    `Ecosystem: ${packageRecord.ecosystem}`,
+    `Registry: ${packageRecord.identity.registry}`,
+    `Normalized name: ${packageRecord.identity.normalizedName}`,
+    `Latest version: ${packageRecord.latest.version}`,
+    `Latest selection: ${packageRecord.latest.selection}`,
+    `Published at: ${textValue(packageRecord.latest.publishedAt)}`,
+    `Release URL: ${textValue(packageRecord.latest.releaseUrl)}`,
+    `Prerelease: ${textValue(packageRecord.latest.prerelease)}`,
+    `Yanked: ${textValue(packageRecord.latest.yanked)}`,
+    `Deprecated: ${textValue(packageRecord.latest.deprecated)}`,
+    `Package description: ${textValue(packageRecord.metadata.description)}`,
+    `License: ${textValue(packageRecord.metadata.license)}`,
+    `Deprecation message: ${textValue(packageRecord.metadata.deprecationMessage)}`,
+    `Project status: ${textValue(packageRecord.metadata.projectStatus)}`
+  ].join('\n');
+}
+
+function evidenceId(material) {
+  return digestJson(material);
+}
+
+function buildLatestEvidence(manifest, packageRecord) {
+  if (!packageRecord.latest?.version || !packageRecord.latest?.sourceId) return null;
+  const source = sourceFor(manifest, packageRecord.latest.sourceId);
+  if (!source || !['available', 'stale'].includes(source.status)) return null;
+  const content = latestEvidenceContent(packageRecord);
+  const contentDigest = digestText(content);
+  const releaseVersions = [packageRecord.latest.version].sort(compareText);
+  const locator = `registry-latest:${packageRecord.latest.version}`;
+  const material = {
+    packageId: packageRecord.id,
+    sourceId: packageRecord.latest.sourceId,
+    kind: 'registryFact',
+    locator,
+    releaseVersions,
+    contentDigest
+  };
+  return {
+    id: evidenceId(material),
+    packageId: packageRecord.id,
+    sourceId: packageRecord.latest.sourceId,
+    kind: 'registryFact',
+    contentDigest,
+    retrievedAt: retrievedAtFor(manifest, source),
+    mediaType: 'text/plain',
+    locator,
+    releaseVersions,
+    content
+  };
+}
+
+function compareEvidence(left, right) {
+  return compareText(left.id, right.id);
+}
+
+function compareWarnings(left, right) {
+  return compareText(left.packageId ?? '', right.packageId ?? '')
+    || compareText(left.sourceId ?? '', right.sourceId ?? '')
+    || compareText(left.code, right.code)
+    || compareText(left.message, right.message);
+}
+
+function bundleWarningFromManifest(warning) {
+  return {
+    code: warning.code,
+    ...(warning.packageId ? { packageId: warning.packageId } : {}),
+    ...(warning.sourceId ? { sourceId: warning.sourceId } : {}),
+    message: warning.message
+  };
+}
+
+function evidenceMissingWarning(packageRecord) {
+  return {
+    code: 'EVIDENCE_MISSING',
+    packageId: packageRecord.id,
+    message: `No portable evidence could be produced for ${packageRecord.id}.`
+  };
+}
+
+function sortedUniqueWarnings(warnings) {
+  const seen = new Set();
+  const unique = [];
+  for (const warning of warnings.sort(compareWarnings)) {
+    const key = [
+      warning.packageId ?? '',
+      warning.sourceId ?? '',
+      warning.code,
+      warning.message
+    ].join('\0');
+    if (!seen.has(key)) {
+      seen.add(key);
+      unique.push(warning);
+    }
+  }
+  return unique;
+}
+
+function summaryFor(evidence, warnings) {
+  return {
+    evidenceCount: evidence.length,
+    packageCount: new Set(evidence.map((item) => item.packageId)).size,
+    sourceCount: new Set(evidence.map((item) => item.sourceId)).size,
+    warningCount: warnings.length
+  };
+}
+
+export function buildKnowledgeEvidenceBundle(manifest, {
+  knowledgeManifestArtifact,
+  knowledgeManifestBytes,
+  generatedAt,
+  enrichedEvidence = []
+}) {
+  validateKnowledgeManifest(manifest);
+  if (!(knowledgeManifestBytes instanceof Uint8Array)) {
+    throw new Error('Knowledge Evidence Bundle producer requires Knowledge Manifest bytes.');
+  }
+  if (!knowledgeManifestArtifact) {
+    throw new Error('Knowledge Evidence Bundle producer requires Knowledge Manifest artifact path.');
+  }
+  if (!Array.isArray(enrichedEvidence)) {
+    throw new Error('Knowledge Evidence Bundle producer requires enrichedEvidence to be an array.');
+  }
+
+  const evidence = [];
+  const warnings = (manifest.warnings ?? []).map(bundleWarningFromManifest);
+  for (const packageRecord of manifest.packages) {
+    const item = buildLatestEvidence(manifest, packageRecord);
+    if (item) evidence.push(item);
+    else if (!['invalid', 'notFound'].includes(packageRecord.status)) warnings.push(evidenceMissingWarning(packageRecord));
+  }
+
+  const packageIds = new Set(manifest.packages.map((item) => item.id));
+  const sourceIds = new Set(manifest.sources.map((item) => item.id));
+  const seenContent = new Set(evidence.map((item) => `${item.packageId}\0${item.contentDigest}`));
+  for (const item of enrichedEvidence) {
+    if (!packageIds.has(item?.packageId) || !sourceIds.has(item?.sourceId)) {
+      throw new Error('Knowledge Evidence Bundle producer received enriched evidence with unknown provenance.');
+    }
+    const contentKey = `${item.packageId}\0${item.contentDigest}`;
+    if (seenContent.has(contentKey)) continue;
+    seenContent.add(contentKey);
+    evidence.push(structuredClone(item));
+  }
+
+  evidence.sort(compareEvidence);
+  const sortedWarnings = sortedUniqueWarnings(warnings);
+  const bundle = {
+    schemaVersion: KNOWLEDGE_EVIDENCE_BUNDLE_SCHEMA_VERSION,
+    generatedAt: generatedAt ?? manifest.generatedAt,
+    generator: { name: ARTIFACT_GENERATOR_NAME, version: VERSION },
+    input: {
+      knowledgeManifest: {
+        schemaVersion: manifest.schemaVersion,
+        artifact: knowledgeManifestArtifact,
+        artifactDigest: digestBytes(Buffer.from(knowledgeManifestBytes)),
+        researchId: manifest.research.researchId
+      }
+    },
+    summary: summaryFor(evidence, sortedWarnings),
+    evidence,
+    warnings: sortedWarnings
+  };
+  return validateKnowledgeEvidenceBundle(bundle);
+}
+
+export function serializeKnowledgeEvidenceBundle(bundle) {
+  validateKnowledgeEvidenceBundle(bundle);
+  return `${JSON.stringify(bundle, null, 2)}\n`;
+}
+
+export async function writeKnowledgeEvidenceBundle(outputPath, bundle) {
+  validateKnowledgeEvidenceBundle(bundle);
+  const target = path.resolve(outputPath);
+  const temporary = `${target}.${process.pid}.tmp`;
+  const contents = serializeKnowledgeEvidenceBundle(bundle);
+  await mkdir(path.dirname(target), { recursive: true });
+  let handle;
+  try {
+    handle = await open(temporary, 'w', 0o600);
+    await handle.writeFile(contents, 'utf8');
+    await handle.sync();
+    await handle.close();
+    handle = null;
+    await rename(temporary, target);
+  } catch (error) {
+    await handle?.close().catch(() => {});
+    await rm(temporary, { force: true }).catch(() => {});
+    throw error;
+  }
+  return target;
+}

package/src/knowledge-manifest-builder.js

@@ -0,0 +1,188 @@
+import { createHash } from 'node:crypto';
+import { readFile } from 'node:fs/promises';
+
+import Ajv2020 from 'ajv/dist/2020.js';
+import addFormats from 'ajv-formats';
+
+import { canonicalJson } from './canonical-json.js';
+import {
+  ARTIFACT_GENERATOR_NAME,
+  KNOWLEDGE_MANIFEST_SCHEMA_VERSION,
+  VERSION
+} from './constants.js';
+import { validateKnowledgeManifestInvariants } from './knowledge-manifest.js';
+
+const schema = JSON.parse(await readFile(
+  new URL('../schemas/knowledge-manifest.schema.json', import.meta.url), 'utf8'
+));
+const ajv = new Ajv2020({ allErrors: true, strict: true, strictRequired: false });
+addFormats(ajv);
+const validateSchema = ajv.compile(schema);
+
+function digest(value) {
+  return `sha256:${createHash('sha256').update(canonicalJson(value), 'utf8').digest('hex')}`;
+}
+
+function clone(value) {
+  return structuredClone(value);
+}
+
+function date(value, field) {
+  const parsed = new Date(value);
+  if (Number.isNaN(parsed.getTime())) throw new Error(`Knowledge Manifest builder requires a valid ${field}.`);
+  return parsed.toISOString();
+}
+
+function publicOccurrence(occurrence) {
+  return {
+    projectId: occurrence.projectId,
+    projectPath: occurrence.projectPath,
+    manifest: occurrence.manifest,
+    dependencyType: occurrence.dependencyType,
+    declaredName: occurrence.declaredName,
+    declaredVersion: occurrence.declaredVersion
+  };
+}
+
+function publicPackage(packageRecord) {
+  return {
+    id: packageRecord.id,
+    ecosystem: packageRecord.ecosystem,
+    status: packageRecord.status,
+    identity: clone(packageRecord.identity),
+    occurrences: packageRecord.occurrences.map(publicOccurrence),
+    metadata: clone(packageRecord.metadata),
+    latest: clone(packageRecord.latest),
+    releaseIndex: clone(packageRecord.releaseIndex),
+    sourceIds: clone(packageRecord.sourceIds),
+    warningCodes: clone(packageRecord.warningCodes)
+  };
+}
+
+function defaultPolicy(mode) {
+  return {
+    mode,
+    policyVersion: '1',
+    registryBases: { npm: 'https://registry.npmjs.org', pypi: 'https://pypi.org' },
+    ttlPolicyVersion: '1',
+    sourceAllowlistVersion: '1',
+    includePrereleases: false
+  };
+}
+
+function compareText(left = '', right = '') {
+  return left < right ? -1 : left > right ? 1 : 0;
+}
+
+function compareWarnings(left, right) {
+  return compareText(left.packageId ?? '', right.packageId ?? '')
+    || compareText(left.code, right.code)
+    || compareText(left.sourceId ?? '', right.sourceId ?? '')
+    || compareText(left.message, right.message);
+}
+
+function policyFor(value, mode) {
+  const policy = { ...defaultPolicy(mode), ...(value ?? {}) };
+  policy.mode = mode;
+  policy.registryBases = { ...defaultPolicy(mode).registryBases, ...(value?.registryBases ?? {}) };
+  return policy;
+}
+
+function sourceDigest(source) {
+  return source.snapshot?.contentDigest ?? digest({
+    id: source.id,
+    kind: source.kind,
+    authority: source.authority,
+    trust: source.trust,
+    url: source.url,
+    status: source.status,
+    supports: source.supports,
+    discoveredFrom: source.discoveredFrom,
+    trustEvidenceSourceIds: source.trustEvidenceSourceIds,
+    conflictsWith: source.conflictsWith ?? []
+  });
+}
+
+/** Deterministic identity for one input lineage, policy, and source set. */
+export function createResearchId(input, policy, sources) {
+  return digest({
+    projectManifestDigest: input.projectManifest.artifactDigest,
+    policyFingerprint: digest(policy),
+    sourceDigests: [...sources]
+      .sort((left, right) => left.id < right.id ? -1 : left.id > right.id ? 1 : 0)
+      .map(sourceDigest)
+  });
+}
+
+/** Schema and relational validation for the public Knowledge Manifest. */
+export function validateKnowledgeManifest(manifest) {
+  if (!validateSchema(manifest)) {
+    throw new Error(`Knowledge Manifest schema validation failed: ${ajv.errorsText(validateSchema.errors, { separator: '; ' })}`);
+  }
+  const invariantErrors = validateKnowledgeManifestInvariants(manifest);
+  if (invariantErrors.length > 0) {
+    throw new Error(`Knowledge Manifest runtime invariants failed: ${invariantErrors.join(' ')}`);
+  }
+  return manifest;
+}
+
+/** Convert one validated internal KnowledgeResearchResult into public schema 1.0.0. */
+export function buildKnowledgeManifest(researchResult, { policy, generatedAt } = {}) {
+  if (!researchResult || researchResult.resultVersion !== '1') {
+    throw new Error('Knowledge Manifest builder requires KnowledgeResearchResult version 1.');
+  }
+  const mode = policy?.mode ?? 'online';
+  const publicPolicy = policyFor(policy, mode);
+  const packages = researchResult.packages.map(publicPackage).sort((left, right) => compareText(left.id, right.id));
+  const sources = clone(researchResult.sources).sort((left, right) => compareText(left.id, right.id));
+  const warnings = clone(researchResult.warnings).sort(compareWarnings);
+  const completedAt = date(researchResult.execution.completedAt, 'execution completedAt');
+  const manifest = {
+    schemaVersion: KNOWLEDGE_MANIFEST_SCHEMA_VERSION,
+    generatedAt: generatedAt ? date(generatedAt, 'generatedAt') : completedAt,
+    generator: { name: ARTIFACT_GENERATOR_NAME, version: VERSION },
+    input: { projectManifest: clone(researchResult.input.projectManifest) },
+    policy: publicPolicy,
+    research: {
+      researchId: createResearchId({ projectManifest: researchResult.input.projectManifest }, publicPolicy, sources),
+      startedAt: date(researchResult.execution.startedAt, 'execution startedAt'),
+      completedAt,
+      durationMs: researchResult.execution.durationMs,
+      inputOccurrenceCount: researchResult.summary.inputOccurrenceCount,
+      inputPackageCount: packages.length,
+      researchedPackageCount: packages.filter((item) => item.status !== 'invalid').length,
+      sourceCount: sources.length,
+      cacheHitCount: researchResult.summary.cacheHitCount,
+      cacheMissCount: researchResult.summary.cacheMissCount,
+      cacheRevalidationCount: researchResult.summary.cacheRevalidationCount,
+      retryCount: researchResult.summary.retryCount,
+      partialFailureCount: researchResult.summary.partialFailureCount
+    },
+    summary: {
+      inputOccurrenceCount: researchResult.summary.inputOccurrenceCount,
+      packageCount: packages.length,
+      resolvedPackageCount: researchResult.summary.resolvedPackageCount,
+      partialPackageCount: researchResult.summary.partialPackageCount,
+      notFoundPackageCount: researchResult.summary.notFoundPackageCount,
+      invalidPackageCount: researchResult.summary.invalidPackageCount,
+      unavailablePackageCount: researchResult.summary.unavailablePackageCount,
+      sourceCount: sources.length,
+      warningCount: warnings.length,
+      cacheHitCount: researchResult.summary.cacheHitCount,
+      cacheMissCount: researchResult.summary.cacheMissCount,
+      staleSourceCount: sources.filter((source) => source.status === 'stale').length
+    },
+    packages,
+    sources,
+    cache: {
+      mode: publicPolicy.mode,
+      policyVersion: publicPolicy.ttlPolicyVersion,
+      hitCount: researchResult.summary.cacheHitCount,
+      missCount: researchResult.summary.cacheMissCount,
+      revalidationCount: researchResult.summary.cacheRevalidationCount,
+      staleEntryCount: sources.filter((source) => source.status === 'stale').length
+    },
+    warnings
+  };
+  return validateKnowledgeManifest(manifest);
+}

package/src/knowledge-manifest-writer.js

@@ -0,0 +1,32 @@
+import { mkdir, open, rename, rm } from 'node:fs/promises';
+import path from 'node:path';
+
+import { validateKnowledgeManifest } from './knowledge-manifest-builder.js';
+
+/** Serialize a validated public manifest as stable UTF-8 pretty JSON. */
+export function serializeKnowledgeManifest(manifest) {
+  return `${JSON.stringify(manifest, null, 2)}\n`;
+}
+
+/** Atomically replace a manifest using write, flush, close, and rename. */
+export async function writeKnowledgeManifest(outputPath, manifest) {
+  validateKnowledgeManifest(manifest);
+  const target = path.resolve(outputPath);
+  const temporary = `${target}.${process.pid}.tmp`;
+  const contents = serializeKnowledgeManifest(manifest);
+  await mkdir(path.dirname(target), { recursive: true });
+  let handle;
+  try {
+    handle = await open(temporary, 'w', 0o600);
+    await handle.writeFile(contents, 'utf8');
+    await handle.sync();
+    await handle.close();
+    handle = null;
+    await rename(temporary, target);
+  } catch (error) {
+    await handle?.close().catch(() => {});
+    await rm(temporary, { force: true }).catch(() => {});
+    throw error;
+  }
+  return target;
+}