@thomasminh1995/depverdict 0.6.0-alpha.1

package/src/registry/npm-packument.js

@@ -0,0 +1,256 @@
+import { compareText } from '../portable.js';
+
+const NPM_NAME = /^(?:@[a-z0-9][a-z0-9._-]*\/)?[a-z0-9][a-z0-9._-]*$/;
+const PRIVATE_KEYS = new Set([
+  'authorization', 'proxyauthorization', 'cookie', 'setcookie', 'etag',
+  'lastmodified', 'headers', 'cachekey', 'stack', 'password', 'credential',
+  'credentials', 'secret', 'token', 'accesstoken', 'apikey', 'clientsecret', 'privatekey'
+]);
+
+export class NpmPackumentError extends Error {
+  constructor(message) {
+    super(message);
+    this.name = 'NpmPackumentError';
+    this.code = 'NPM_PACKUMENT_INVALID';
+  }
+}
+
+function plainObject(value) {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) return false;
+  const prototype = Object.getPrototypeOf(value);
+  return prototype === Object.prototype || prototype === null;
+}
+
+function nullableString(value) {
+  return typeof value === 'string' && value.trim() !== '' ? value.trim() : null;
+}
+
+function normalizedHttpsUrl(value) {
+  if (typeof value !== 'string' || value.trim() === '') return null;
+  try {
+    const url = new URL(value.trim());
+    if (url.protocol !== 'https:' || url.username || url.password) return null;
+    url.search = '';
+    url.hash = '';
+    if (url.pathname === '/') return `${url.protocol}//${url.host}`;
+    return url.toString();
+  } catch {
+    return null;
+  }
+}
+
+function repositoryUrl(value) {
+  const raw = typeof value === 'string' ? value : plainObject(value) ? value.url : null;
+  if (typeof raw !== 'string') return null;
+  const candidate = raw.startsWith('git+') ? raw.slice(4) : raw;
+  const normalized = normalizedHttpsUrl(candidate);
+  if (!normalized) return null;
+  const url = new URL(normalized);
+  if (url.hostname.toLowerCase() === 'github.com') {
+    url.pathname = url.pathname.replace(/\.git$/i, '');
+    return url.pathname === '/' ? `${url.protocol}//${url.host}` : url.toString();
+  }
+  return normalized;
+}
+
+function issueUrl(value) {
+  const raw = typeof value === 'string' ? value : plainObject(value) ? value.url : null;
+  return normalizedHttpsUrl(raw);
+}
+
+function publishedAt(value) {
+  if (typeof value !== 'string') return null;
+  const date = new Date(value);
+  return Number.isNaN(date.getTime()) ? null : date.toISOString();
+}
+
+function explicitDeprecated(value) {
+  if (typeof value === 'string') return value.trim() === '' ? false : true;
+  if (typeof value === 'boolean') return value;
+  return null;
+}
+
+function npmPageUrl(name) {
+  return `https://www.npmjs.com/package/${name}`;
+}
+
+function npmReleaseUrl(name, version) {
+  return `${npmPageUrl(name)}/v/${encodeURIComponent(version)}`;
+}
+
+function sortedStrings(values) {
+  return [...new Set(values.filter((value) => typeof value === 'string' && value.length > 0))]
+    .sort(compareText);
+}
+
+function privateKey(key) {
+  return PRIVATE_KEYS.has(key.toLowerCase().replace(/[-_]/g, ''));
+}
+
+function unsafeStoredString(value) {
+  return value.startsWith('/')
+    || /^[A-Za-z]:[\\/]/.test(value)
+    || /[a-z][a-z0-9+.-]*:\/\/[^/?#]*@/i.test(value);
+}
+
+/**
+ * Keep cache envelopes portable even when publisher-controlled metadata
+ * contains a credentialed URL or request-shaped property.
+ */
+export function sanitizeNpmPackumentForCache(value) {
+  if (value === null || typeof value === 'boolean' || typeof value === 'number') return value;
+  if (typeof value === 'string') return unsafeStoredString(value) ? null : value;
+  if (Array.isArray(value)) return value.map(sanitizeNpmPackumentForCache);
+  if (!plainObject(value)) return null;
+  return Object.fromEntries(Object.keys(value).sort(compareText).flatMap((key) => {
+    if (privateKey(key)) return [];
+    return [[key, sanitizeNpmPackumentForCache(value[key])]];
+  }));
+}
+
+function packageMetadata(packument) {
+  const deprecated = packument.deprecated;
+  const deprecationMessage = nullableString(deprecated);
+  return {
+    description: nullableString(packument.description),
+    license: nullableString(packument.license) ?? nullableString(packument.license?.type),
+    homepageUrl: normalizedHttpsUrl(packument.homepage),
+    documentationUrl: null,
+    repositoryUrl: repositoryUrl(packument.repository),
+    issueUrl: issueUrl(packument.bugs),
+    deprecationMessage,
+    projectStatus: deprecated === undefined || deprecated === false ? null : 'deprecated'
+  };
+}
+
+export function normalizeNpmRegistryBaseUrl(value) {
+  if (typeof value !== 'string' || value.trim() === '') {
+    throw new NpmPackumentError('npm Registry base URL must be a non-empty HTTPS URL.');
+  }
+  let url;
+  try {
+    url = new URL(value);
+  } catch {
+    throw new NpmPackumentError('npm Registry base URL must be a valid HTTPS URL.');
+  }
+  if (url.protocol !== 'https:' || url.username || url.password || url.search || url.hash) {
+    throw new NpmPackumentError('npm Registry base URL must be HTTPS and must not contain credentials or query data.');
+  }
+  url.pathname = url.pathname.replace(/\/$/, '');
+  return url.toString().replace(/\/$/, '');
+}
+
+export function validateNpmResearchPackage(researchPackage) {
+  if (!plainObject(researchPackage)) {
+    throw new NpmPackumentError('npm research package must be an object.');
+  }
+  const name = researchPackage.normalizedName;
+  if (typeof name !== 'string' || !NPM_NAME.test(name)
+    || researchPackage.id !== `npm:${name}`
+    || researchPackage.registry !== 'npm'
+    || researchPackage.ecosystem !== 'node') {
+    throw new NpmPackumentError('Research package is not a valid npm registry identity.');
+  }
+  return {
+    id: researchPackage.id,
+    registry: 'npm',
+    ecosystem: 'node',
+    normalizedName: name,
+    observedDeclaredNames: sortedStrings(researchPackage.observedDeclaredNames ?? [name]),
+    occurrences: structuredClone(researchPackage.occurrences ?? [])
+  };
+}
+
+export function npmRegistryUrls(registryBaseUrl, normalizedName) {
+  const base = normalizeNpmRegistryBaseUrl(registryBaseUrl);
+  return {
+    registryBaseUrl: base,
+    packageUrl: npmPageUrl(normalizedName),
+    apiUrl: `${base}/${encodeURIComponent(normalizedName)}`
+  };
+}
+
+/** Validate the minimum packument shape before the caller writes it to cache. */
+export function validateNpmPackument(packument, normalizedName) {
+  if (!plainObject(packument)
+    || packument.name !== normalizedName
+    || !plainObject(packument['dist-tags'])
+    || !plainObject(packument.versions)
+    || !plainObject(packument.time)) {
+    throw new NpmPackumentError('npm Registry package metadata has an unsupported shape.');
+  }
+  if (Object.hasOwn(packument['dist-tags'], 'latest')
+    && typeof packument['dist-tags'].latest !== 'string') {
+    throw new NpmPackumentError('npm Registry latest distribution tag must be a string.');
+  }
+  return packument;
+}
+
+/**
+ * Translate a valid full npm packument into records that match the package and
+ * source shapes used by the Knowledge Manifest, without assembling a manifest.
+ */
+export function normalizeNpmPackument(researchPackage, packument, { snapshot, registryBaseUrl }) {
+  const input = validateNpmResearchPackage(researchPackage);
+  validateNpmPackument(packument, input.normalizedName);
+  const urls = npmRegistryUrls(registryBaseUrl, input.normalizedName);
+  const sourceId = `${input.id}:registry`;
+  const versions = packument.versions;
+  const time = packument.time;
+  const releases = Object.keys(versions).sort(compareText).map((version) => ({
+    version,
+    publishedAt: publishedAt(time[version]),
+    url: npmReleaseUrl(input.normalizedName, version),
+    prerelease: null,
+    yanked: null,
+    deprecated: explicitDeprecated(versions[version]?.deprecated),
+    sourceIds: [sourceId]
+  }));
+  const latestVersion = packument['dist-tags'].latest;
+  const latestRecord = typeof latestVersion === 'string' && Object.hasOwn(versions, latestVersion)
+    ? {
+        version: latestVersion,
+        selection: 'dist-tag:latest',
+        publishedAt: publishedAt(time[latestVersion]),
+        releaseUrl: npmReleaseUrl(input.normalizedName, latestVersion),
+        prerelease: null,
+        yanked: null,
+        deprecated: explicitDeprecated(versions[latestVersion]?.deprecated),
+        sourceId
+      }
+    : null;
+  const status = latestRecord ? 'resolved' : 'partial';
+
+  return {
+    package: {
+      id: input.id,
+      ecosystem: 'node',
+      status,
+      identity: {
+        observedDeclaredNames: input.observedDeclaredNames,
+        normalizedName: input.normalizedName,
+        registry: 'npm',
+        ...urls
+      },
+      occurrences: input.occurrences,
+      metadata: packageMetadata(packument),
+      latest: latestRecord,
+      releaseIndex: releases,
+      sourceIds: [sourceId],
+      warningCodes: []
+    },
+    source: {
+      id: sourceId,
+      kind: 'registry',
+      authority: 'registryAuthoritative',
+      trust: 'publisher',
+      url: urls.packageUrl,
+      apiUrl: urls.apiUrl,
+      status: 'available',
+      supports: ['identity', 'latest', 'metadata', 'releases'],
+      discoveredFrom: null,
+      trustEvidenceSourceIds: [],
+      snapshot
+    }
+  };
+}

package/src/registry/npm-registry-adapter.js

@@ -0,0 +1,262 @@
+import {
+  fetchNpmJson,
+  BoundedFetchError,
+  validateRegistryResponseLimit
+} from '../http/bounded-fetch.js';
+import { USER_AGENT } from '../constants.js';
+import { createCacheIdentity, createKnowledgeCache } from '../knowledge-cache.js';
+import {
+  normalizeNpmPackument,
+  normalizeNpmRegistryBaseUrl,
+  npmRegistryUrls,
+  sanitizeNpmPackumentForCache,
+  validateNpmPackument,
+  validateNpmResearchPackage
+} from './npm-packument.js';
+
+export const DEFAULT_NPM_REGISTRY_BASE_URL = 'https://registry.npmjs.org';
+export const DEFAULT_NPM_METADATA_TTL_MS = 24 * 60 * 60 * 1_000;
+export const DEFAULT_NPM_MAX_RESPONSE_BYTES = 16 * 1024 * 1024;
+
+function currentDate(clock) {
+  const value = clock();
+  const date = value instanceof Date ? new Date(value.getTime()) : new Date(value);
+  if (Number.isNaN(date.getTime())) throw new Error('npm Registry adapter clock returned an invalid time.');
+  return date;
+}
+
+function cacheIdentityFor(input) {
+  return createCacheIdentity({
+    adapter: 'npm',
+    resourceKind: 'registry-package',
+    packageId: input.id,
+    resourceVariant: 'full-packument',
+    adapterVersion: '1'
+  });
+}
+
+function snapshot(bodyDigest, retrievedAt) {
+  return {
+    contentDigest: bodyDigest,
+    mediaType: 'application/json',
+    retrievedAt,
+    freshness: 'fresh'
+  };
+}
+
+function cacheOutcome(readResult) {
+  if (readResult.status === 'missing') return { outcome: 'miss' };
+  if (readResult.status === 'expired') return { outcome: 'expired' };
+  if (readResult.status === 'corrupted') {
+    return { outcome: 'corrupted', reason: readResult.reason };
+  }
+  return { outcome: 'hit' };
+}
+
+function unavailableResult(input, registryBaseUrl, cache, {
+  status = 'unavailable',
+  warningCode,
+  message,
+  retryable,
+  errorCode
+}) {
+  const urls = npmRegistryUrls(registryBaseUrl, input.normalizedName);
+  const sourceId = `${input.id}:registry`;
+  const warning = {
+    code: warningCode,
+    packageId: input.id,
+    sourceId,
+    message,
+    retryable
+  };
+  return {
+    package: {
+      id: input.id,
+      ecosystem: 'node',
+      status,
+      identity: {
+        observedDeclaredNames: input.observedDeclaredNames,
+        normalizedName: input.normalizedName,
+        registry: 'npm',
+        ...urls
+      },
+      occurrences: input.occurrences,
+      metadata: {},
+      latest: null,
+      releaseIndex: [],
+      sourceIds: [sourceId],
+      warningCodes: [warningCode]
+    },
+    source: {
+      id: sourceId,
+      kind: 'registry',
+      authority: 'registryAuthoritative',
+      trust: 'publisher',
+      url: urls.packageUrl,
+      apiUrl: urls.apiUrl,
+      status: status === 'notFound' ? 'notFound' : 'unavailable',
+      supports: ['identity'],
+      discoveredFrom: null,
+      trustEvidenceSourceIds: [],
+      snapshot: null
+    },
+    cache,
+    warnings: [warning],
+    errorCode
+  };
+}
+
+function normalizedResult(researchPackage, body, registryBaseUrl, cache, cacheRecord) {
+  const normalized = normalizeNpmPackument(researchPackage, body, {
+    registryBaseUrl,
+    snapshot: snapshot(cacheRecord.bodyDigest, cacheRecord.storedAt)
+  });
+  return {
+    ...normalized,
+    cache,
+    warnings: [],
+    errorCode: null
+  };
+}
+
+/**
+ * Create the private npm-compatible Registry adapter. It resolves `npm:`
+ * identities only; npm, Yarn, pnpm, and Bun are intentionally not inputs to
+ * its request or cache behavior.
+ */
+export function createNpmRegistryAdapter({
+  registryBaseUrl = DEFAULT_NPM_REGISTRY_BASE_URL,
+  fetch: fetchImplementation = globalThis.fetch,
+  cache,
+  clock = () => new Date(),
+  timeoutMs = 10_000,
+  maxResponseBytes = DEFAULT_NPM_MAX_RESPONSE_BYTES,
+  ttlMs = DEFAULT_NPM_METADATA_TTL_MS,
+  offline = false,
+  userAgent = USER_AGENT
+} = {}) {
+  const normalizedRegistryBaseUrl = normalizeNpmRegistryBaseUrl(registryBaseUrl);
+  validateRegistryResponseLimit(maxResponseBytes, { errorPrefix: 'NPM' });
+  const knowledgeCache = cache ?? createKnowledgeCache({ clock });
+  if (!knowledgeCache || typeof knowledgeCache.read !== 'function' || typeof knowledgeCache.write !== 'function') {
+    throw new Error('npm Registry adapter requires a Knowledge Store cache.');
+  }
+
+  function requestFor(researchPackage) {
+    const input = validateNpmResearchPackage(researchPackage);
+    return npmRegistryUrls(normalizedRegistryBaseUrl, input.normalizedName).apiUrl;
+  }
+
+  return {
+    requestFor,
+    cacheIdentityFor(researchPackage) {
+      return cacheIdentityFor(validateNpmResearchPackage(researchPackage));
+    },
+    async researchPackage(researchPackage) {
+      const input = validateNpmResearchPackage(researchPackage);
+      const identity = cacheIdentityFor(input);
+      const cached = await knowledgeCache.read(identity);
+
+      if (cached.status === 'fresh') {
+        try {
+          return normalizedResult(input, cached.body, normalizedRegistryBaseUrl, cacheOutcome(cached), cached);
+        } catch {
+          return unavailableResult(input, normalizedRegistryBaseUrl, cacheOutcome(cached), {
+            warningCode: 'REGISTRY_RESPONSE_INVALID',
+            message: 'npm Registry package metadata is invalid.',
+            retryable: true,
+            errorCode: 'NPM_PACKUMENT_INVALID'
+          });
+        }
+      }
+
+      const previousCacheOutcome = cacheOutcome(cached);
+      if (offline) {
+        return unavailableResult(input, normalizedRegistryBaseUrl, previousCacheOutcome, {
+          warningCode: 'OFFLINE_CACHE_MISS',
+          message: 'Offline mode has no fresh npm Registry package metadata in cache.',
+          retryable: false,
+          errorCode: 'NPM_OFFLINE_CACHE_MISS'
+        });
+      }
+      let response;
+      try {
+        response = await fetchNpmJson(requestFor(input), {
+          fetchImplementation,
+          timeoutMs,
+          maxResponseBytes,
+          userAgent
+        });
+      } catch (error) {
+        if (error instanceof BoundedFetchError
+          && ['NPM_RESPONSE_INVALID', 'NPM_RESPONSE_TOO_LARGE'].includes(error.code)) {
+          return unavailableResult(input, normalizedRegistryBaseUrl, previousCacheOutcome, {
+            warningCode: 'REGISTRY_RESPONSE_INVALID',
+            message: 'npm Registry returned an invalid package metadata response.',
+            retryable: true,
+            errorCode: error.code
+          });
+        }
+        return unavailableResult(input, normalizedRegistryBaseUrl, previousCacheOutcome, {
+          warningCode: 'REGISTRY_UNAVAILABLE',
+          message: 'npm Registry package metadata is unavailable.',
+          retryable: true,
+          errorCode: error instanceof BoundedFetchError ? error.code : 'NPM_TRANSPORT_FAILED'
+        });
+      }
+
+      if (response.status === 404) {
+        return unavailableResult(input, normalizedRegistryBaseUrl, previousCacheOutcome, {
+          status: 'notFound',
+          warningCode: 'PACKAGE_NOT_FOUND',
+          message: 'Package was not found in the npm Registry.',
+          retryable: false,
+          errorCode: 'NPM_HTTP_404'
+        });
+      }
+      if (response.status === 429) {
+        return unavailableResult(input, normalizedRegistryBaseUrl, previousCacheOutcome, {
+          warningCode: 'REGISTRY_RATE_LIMITED',
+          message: 'npm Registry rate limited package metadata retrieval.',
+          retryable: true,
+          errorCode: 'NPM_HTTP_429'
+        });
+      }
+      if (response.status !== 200 && response.status >= 200 && response.status < 300) {
+        return unavailableResult(input, normalizedRegistryBaseUrl, previousCacheOutcome, {
+          warningCode: 'REGISTRY_RESPONSE_INVALID',
+          message: 'npm Registry returned an invalid package metadata response.',
+          retryable: true,
+          errorCode: `NPM_HTTP_${response.status}`
+        });
+      }
+      if (response.status !== 200) {
+        return unavailableResult(input, normalizedRegistryBaseUrl, previousCacheOutcome, {
+          warningCode: 'REGISTRY_UNAVAILABLE',
+          message: 'npm Registry package metadata is unavailable.',
+          retryable: response.status >= 500,
+          errorCode: `NPM_HTTP_${response.status}`
+        });
+      }
+
+      try {
+        validateNpmPackument(response.body, input.normalizedName);
+      } catch {
+        return unavailableResult(input, normalizedRegistryBaseUrl, previousCacheOutcome, {
+          warningCode: 'REGISTRY_RESPONSE_INVALID',
+          message: 'npm Registry returned an invalid package metadata response.',
+          retryable: true,
+          errorCode: 'NPM_PACKUMENT_INVALID'
+        });
+      }
+
+      const cacheBody = sanitizeNpmPackumentForCache(response.body);
+      const stored = await knowledgeCache.write(identity, cacheBody, { ttlMs });
+      const fetchedCache = {
+        outcome: cached.status === 'expired' ? 'revalidated' : cached.status === 'corrupted' ? 'corrupted-replaced' : 'miss'
+      };
+      if (cached.status === 'corrupted') fetchedCache.reason = cached.reason;
+      return normalizedResult(input, cacheBody, normalizedRegistryBaseUrl, fetchedCache, stored);
+    }
+  };
+}