@thomasminh1995/depverdict 0.6.0-alpha.1

package/src/registry/pypi-project.js

@@ -0,0 +1,300 @@
+import { normalizePythonPackageName } from '../python-requirements.js';
+import { compareText } from '../portable.js';
+
+const PYPI_NAME = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
+const PROJECT_URL_ROLES = {
+  documentationUrl: ['documentation', 'docs'],
+  repositoryUrl: ['source', 'repository', 'code'],
+  issueUrl: ['issues', 'tracker', 'bug tracker'],
+  homepageUrl: ['homepage', 'home']
+};
+const SOURCE_CANDIDATE_ROLES = {
+  changelog: ['changelog', 'changes'],
+  releaseNotes: ['release notes'],
+  releases: ['releases'],
+  migrationGuide: ['migration', 'migration guide', 'upgrade guide']
+};
+
+export class PypiProjectError extends Error {
+  constructor(message) {
+    super(message);
+    this.name = 'PypiProjectError';
+    this.code = 'PYPI_PROJECT_INVALID';
+  }
+}
+
+function plainObject(value) {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) return false;
+  const prototype = Object.getPrototypeOf(value);
+  return prototype === Object.prototype || prototype === null;
+}
+
+function nullableString(value) {
+  return typeof value === 'string' && value.trim() !== '' ? value.trim() : null;
+}
+
+function normalizedHttpsUrl(value) {
+  if (typeof value !== 'string' || value.trim() === '') return null;
+  try {
+    const url = new URL(value.trim());
+    if (url.protocol !== 'https:' || url.username || url.password) return null;
+    url.search = '';
+    url.hash = '';
+    if (url.pathname === '/') return `${url.protocol}//${url.host}`;
+    return url.toString();
+  } catch {
+    return null;
+  }
+}
+
+function repositoryUrl(value) {
+  const raw = typeof value === 'string' ? value : plainObject(value) ? value.url : null;
+  if (typeof raw !== 'string') return null;
+  const normalized = normalizedHttpsUrl(raw.startsWith('git+') ? raw.slice(4) : raw);
+  if (!normalized) return null;
+  const url = new URL(normalized);
+  if (url.hostname.toLowerCase() === 'github.com') {
+    url.pathname = url.pathname.replace(/\.git$/i, '');
+    return url.pathname === '/' ? `${url.protocol}//${url.host}` : url.toString();
+  }
+  return normalized;
+}
+
+function normalizedLabel(value) {
+  return typeof value === 'string'
+    ? value.trim().toLowerCase().replace(/\s+/g, ' ')
+    : '';
+}
+
+function classifiedProjectUrls(projectUrls) {
+  const entries = plainObject(projectUrls)
+    ? Object.entries(projectUrls)
+      .map(([label, url]) => ({ label: normalizedLabel(label), url }))
+      .sort((left, right) => compareText(left.label, right.label) || compareText(String(left.url), String(right.url)))
+    : [];
+  const result = {};
+  for (const [field, labels] of Object.entries(PROJECT_URL_ROLES)) {
+    for (const label of labels) {
+      const match = entries.find((entry) => entry.label === label);
+      if (match) {
+        const normalized = field === 'repositoryUrl'
+          ? repositoryUrl(match.url)
+          : normalizedHttpsUrl(match.url);
+        if (normalized) {
+          result[field] = normalized;
+          break;
+        }
+      }
+    }
+  }
+  return result;
+}
+
+function sourceCandidates(projectUrls) {
+  const entries = plainObject(projectUrls)
+    ? Object.entries(projectUrls)
+      .map(([label, url]) => ({ label: normalizedLabel(label), originalLabel: String(label), url }))
+      .sort((left, right) => compareText(left.label, right.label) || compareText(left.originalLabel, right.originalLabel))
+    : [];
+  return Object.entries(SOURCE_CANDIDATE_ROLES).flatMap(([role, labels]) => entries
+    .filter((entry) => labels.includes(entry.label))
+    .flatMap((entry) => {
+      const url = normalizedHttpsUrl(entry.url);
+      return url ? [{
+        role,
+        url,
+        discoveredFromField: `info.project_urls.${entry.originalLabel}`
+      }] : [];
+    }))
+    .sort((left, right) => compareText(left.role, right.role)
+      || compareText(left.url, right.url)
+      || compareText(left.discoveredFromField, right.discoveredFromField));
+}
+
+function normalizedDate(value) {
+  if (typeof value !== 'string') return null;
+  const date = new Date(value);
+  return Number.isNaN(date.getTime()) ? null : date.toISOString();
+}
+
+function filesFor(value) {
+  return Array.isArray(value) ? value.filter(plainObject) : [];
+}
+
+function earliestUpload(files) {
+  return files
+    .flatMap((file) => [normalizedDate(file.upload_time_iso_8601), normalizedDate(file.upload_time)])
+    .filter(Boolean)
+    .sort(compareText)[0] ?? null;
+}
+
+function yankedState(files) {
+  if (files.some((file) => file.yanked === false)) return false;
+  return files.length > 0 && files.every((file) => file.yanked === true) ? true : null;
+}
+
+function sortedStrings(values) {
+  return [...new Set(values.filter((value) => typeof value === 'string' && value.length > 0))]
+    .sort(compareText);
+}
+
+function projectStatus(classifiers) {
+  if (!Array.isArray(classifiers)) return null;
+  return classifiers
+    .filter((item) => typeof item === 'string' && item.startsWith('Development Status :: '))
+    .sort(compareText)[0] ?? null;
+}
+
+function pypiProjectUrl(base, name, version = null) {
+  const suffix = version === null ? '' : `${encodeURIComponent(version)}/`;
+  return `${base}/project/${encodeURIComponent(name)}/${suffix}`;
+}
+
+export function normalizePypiRegistryBaseUrl(value) {
+  if (typeof value !== 'string' || value.trim() === '') {
+    throw new PypiProjectError('PyPI registry base URL must be a non-empty HTTPS URL.');
+  }
+  let url;
+  try {
+    url = new URL(value);
+  } catch {
+    throw new PypiProjectError('PyPI registry base URL must be a valid HTTPS URL.');
+  }
+  if (url.protocol !== 'https:' || url.username || url.password || url.search || url.hash) {
+    throw new PypiProjectError('PyPI registry base URL must be HTTPS and must not contain credentials or query data.');
+  }
+  url.pathname = url.pathname.replace(/\/$/, '');
+  return url.toString().replace(/\/$/, '');
+}
+
+export function validatePypiResearchPackage(researchPackage) {
+  if (!plainObject(researchPackage)) throw new PypiProjectError('PyPI research package must be an object.');
+  const normalizedName = normalizePythonPackageName(researchPackage.normalizedName ?? '');
+  if (!PYPI_NAME.test(normalizedName)
+    || researchPackage.id !== `pypi:${normalizedName}`
+    || researchPackage.registry !== 'pypi'
+    || researchPackage.ecosystem !== 'python') {
+    throw new PypiProjectError('Research package is not a valid PyPI registry identity.');
+  }
+  return {
+    id: researchPackage.id,
+    registry: 'pypi',
+    ecosystem: 'python',
+    normalizedName,
+    observedDeclaredNames: sortedStrings(researchPackage.observedDeclaredNames ?? [normalizedName]),
+    occurrences: structuredClone(researchPackage.occurrences ?? [])
+  };
+}
+
+export function pypiRegistryUrls(registryBaseUrl, normalizedName) {
+  const base = normalizePypiRegistryBaseUrl(registryBaseUrl);
+  return {
+    registryBaseUrl: base,
+    packageUrl: pypiProjectUrl(base, normalizedName),
+    apiUrl: `${base}/pypi/${encodeURIComponent(normalizedName)}/json`
+  };
+}
+
+/** Validate the stable subset of PyPI project JSON used by this MVP. */
+export function validatePypiProject(project, requestedNormalizedName) {
+  if (!plainObject(project) || !plainObject(project.info)
+    || typeof project.info.name !== 'string'
+    || normalizePythonPackageName(project.info.name) !== requestedNormalizedName) {
+    throw new PypiProjectError('PyPI project metadata has an unsupported shape.');
+  }
+  if (project.releases !== undefined && !plainObject(project.releases)) {
+    throw new PypiProjectError('PyPI project releases must be an object when present.');
+  }
+  if (project.urls !== undefined && !Array.isArray(project.urls)) {
+    throw new PypiProjectError('PyPI project URLs must be an array when present.');
+  }
+  return project;
+}
+
+function metadata(info) {
+  const classified = classifiedProjectUrls(info.project_urls);
+  return {
+    description: nullableString(info.summary) ?? nullableString(info.description),
+    license: nullableString(info.license_expression) ?? nullableString(info.license),
+    homepageUrl: classified.homepageUrl ?? normalizedHttpsUrl(info.home_page),
+    documentationUrl: classified.documentationUrl ?? null,
+    repositoryUrl: classified.repositoryUrl ?? null,
+    issueUrl: classified.issueUrl ?? null,
+    deprecationMessage: null,
+    projectStatus: projectStatus(info.classifiers)
+  };
+}
+
+/**
+ * Translate validated PyPI project JSON into package and registry-source
+ * records compatible with later Knowledge Manifest assembly.
+ */
+export function normalizePypiProject(researchPackage, project, { snapshot, registryBaseUrl }) {
+  const input = validatePypiResearchPackage(researchPackage);
+  validatePypiProject(project, input.normalizedName);
+  const urls = pypiRegistryUrls(registryBaseUrl, input.normalizedName);
+  const sourceId = `${input.id}:registry`;
+  const releases = plainObject(project.releases) ? project.releases : {};
+  const latestVersion = nullableString(project.info.version);
+  const latestFiles = latestVersion && Object.hasOwn(releases, latestVersion)
+    ? filesFor(releases[latestVersion])
+    : latestVersion ? filesFor(project.urls) : [];
+  const releaseIndex = Object.keys(releases).sort(compareText).map((version) => {
+    const files = filesFor(releases[version]);
+    return {
+      version,
+      publishedAt: earliestUpload(files),
+      url: pypiProjectUrl(urls.registryBaseUrl, input.normalizedName, version),
+      prerelease: null,
+      yanked: yankedState(files),
+      deprecated: null,
+      sourceIds: [sourceId]
+    };
+  });
+  const latest = latestVersion === null ? null : {
+    version: latestVersion,
+    selection: 'project-info-version',
+    publishedAt: earliestUpload(latestFiles),
+    releaseUrl: pypiProjectUrl(urls.registryBaseUrl, input.normalizedName, latestVersion),
+    prerelease: null,
+    yanked: yankedState(latestFiles),
+    deprecated: null,
+    sourceId
+  };
+
+  return {
+    package: {
+      id: input.id,
+      ecosystem: 'python',
+      status: latest ? 'resolved' : 'partial',
+      identity: {
+        observedDeclaredNames: input.observedDeclaredNames,
+        normalizedName: input.normalizedName,
+        registry: 'pypi',
+        ...urls
+      },
+      occurrences: input.occurrences,
+      metadata: metadata(project.info),
+      latest,
+      releaseIndex,
+      sourceIds: [sourceId],
+      warningCodes: []
+    },
+    source: {
+      id: sourceId,
+      kind: 'registry',
+      authority: 'registryAuthoritative',
+      trust: 'publisher',
+      url: urls.packageUrl,
+      apiUrl: urls.apiUrl,
+      status: 'available',
+      supports: ['identity', 'latest', 'metadata', 'releases'],
+      discoveredFrom: null,
+      trustEvidenceSourceIds: [],
+      snapshot
+    },
+    // Internal only: the public metadata contract intentionally has no
+    // changelog/release-note fields. MVP-02-06 resolves these candidates.
+    sourceCandidates: sourceCandidates(project.info.project_urls)
+  };
+}

package/src/registry/pypi-registry-adapter.js

@@ -0,0 +1,235 @@
+import {
+  fetchRegistryJson,
+  BoundedFetchError,
+  validateRegistryResponseLimit
+} from '../http/bounded-fetch.js';
+import { USER_AGENT } from '../constants.js';
+import { createCacheIdentity, createKnowledgeCache } from '../knowledge-cache.js';
+import {
+  normalizePypiProject,
+  normalizePypiRegistryBaseUrl,
+  pypiRegistryUrls,
+  validatePypiProject,
+  validatePypiResearchPackage
+} from './pypi-project.js';
+import { sanitizeRegistryBodyForCache } from './sanitize-registry-body.js';
+
+export const DEFAULT_PYPI_REGISTRY_BASE_URL = 'https://pypi.org';
+export const DEFAULT_PYPI_INDEX_BASE_URL = 'https://pypi.org/simple';
+export const DEFAULT_PYPI_METADATA_TTL_MS = 24 * 60 * 60 * 1_000;
+export const DEFAULT_PYPI_MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
+
+function cacheIdentityFor(input) {
+  return createCacheIdentity({
+    adapter: 'pypi',
+    resourceKind: 'registry-package',
+    packageId: input.id,
+    resourceVariant: 'project-json',
+    adapterVersion: '1'
+  });
+}
+
+function snapshot(bodyDigest, retrievedAt) {
+  return {
+    contentDigest: bodyDigest,
+    mediaType: 'application/json',
+    retrievedAt,
+    freshness: 'fresh'
+  };
+}
+
+function cacheOutcome(readResult) {
+  if (readResult.status === 'missing') return { outcome: 'miss' };
+  if (readResult.status === 'expired') return { outcome: 'expired' };
+  if (readResult.status === 'corrupted') return { outcome: 'corrupted', reason: readResult.reason };
+  return { outcome: 'hit' };
+}
+
+function unavailableResult(input, registryBaseUrl, cache, {
+  status = 'unavailable', warningCode, message, retryable, errorCode
+}) {
+  const urls = pypiRegistryUrls(registryBaseUrl, input.normalizedName);
+  const sourceId = `${input.id}:registry`;
+  const warning = { code: warningCode, packageId: input.id, sourceId, message, retryable };
+  return {
+    package: {
+      id: input.id,
+      ecosystem: 'python',
+      status,
+      identity: {
+        observedDeclaredNames: input.observedDeclaredNames,
+        normalizedName: input.normalizedName,
+        registry: 'pypi',
+        ...urls
+      },
+      occurrences: input.occurrences,
+      metadata: {},
+      latest: null,
+      releaseIndex: [],
+      sourceIds: [sourceId],
+      warningCodes: [warningCode]
+    },
+    source: {
+      id: sourceId,
+      kind: 'registry',
+      authority: 'registryAuthoritative',
+      trust: 'publisher',
+      url: urls.packageUrl,
+      apiUrl: urls.apiUrl,
+      status: status === 'notFound' ? 'notFound' : 'unavailable',
+      supports: ['identity'],
+      discoveredFrom: null,
+      trustEvidenceSourceIds: [],
+      snapshot: null
+    },
+    cache,
+    warnings: [warning],
+    errorCode
+  };
+}
+
+function normalizedResult(researchPackage, body, registryBaseUrl, cache, cacheRecord) {
+  return {
+    ...normalizePypiProject(researchPackage, body, {
+      registryBaseUrl,
+      snapshot: snapshot(cacheRecord.bodyDigest, cacheRecord.storedAt)
+    }),
+    cache,
+    warnings: [],
+    errorCode: null
+  };
+}
+
+/** Private adapter for public PyPI project JSON, independent of Python installers. */
+export function createPypiRegistryAdapter({
+  registryBaseUrl = DEFAULT_PYPI_REGISTRY_BASE_URL,
+  indexBaseUrl = DEFAULT_PYPI_INDEX_BASE_URL,
+  fetch: fetchImplementation = globalThis.fetch,
+  cache,
+  clock = () => new Date(),
+  timeoutMs = 10_000,
+  maxResponseBytes = DEFAULT_PYPI_MAX_RESPONSE_BYTES,
+  ttlMs = DEFAULT_PYPI_METADATA_TTL_MS,
+  offline = false,
+  userAgent = USER_AGENT
+} = {}) {
+  const normalizedRegistryBaseUrl = normalizePypiRegistryBaseUrl(registryBaseUrl);
+  validateRegistryResponseLimit(maxResponseBytes, { errorPrefix: 'PYPI' });
+  // Validate future JSON Index policy input even though MVP-02-05 makes no index request.
+  normalizePypiRegistryBaseUrl(indexBaseUrl);
+  const knowledgeCache = cache ?? createKnowledgeCache({ clock });
+  if (!knowledgeCache || typeof knowledgeCache.read !== 'function' || typeof knowledgeCache.write !== 'function') {
+    throw new Error('PyPI Registry adapter requires a Knowledge Store cache.');
+  }
+
+  function requestFor(researchPackage) {
+    const input = validatePypiResearchPackage(researchPackage);
+    return pypiRegistryUrls(normalizedRegistryBaseUrl, input.normalizedName).apiUrl;
+  }
+
+  return {
+    requestFor,
+    cacheIdentityFor(researchPackage) {
+      return cacheIdentityFor(validatePypiResearchPackage(researchPackage));
+    },
+    async researchPackage(researchPackage) {
+      const input = validatePypiResearchPackage(researchPackage);
+      const identity = cacheIdentityFor(input);
+      const cached = await knowledgeCache.read(identity);
+      if (cached.status === 'fresh') {
+        try {
+          return normalizedResult(input, cached.body, normalizedRegistryBaseUrl, cacheOutcome(cached), cached);
+        } catch {
+          return unavailableResult(input, normalizedRegistryBaseUrl, cacheOutcome(cached), {
+            warningCode: 'REGISTRY_RESPONSE_INVALID',
+            message: 'PyPI project metadata is invalid.',
+            retryable: true,
+            errorCode: 'PYPI_PROJECT_INVALID'
+          });
+        }
+      }
+
+      const previousCacheOutcome = cacheOutcome(cached);
+      if (offline) {
+        return unavailableResult(input, normalizedRegistryBaseUrl, previousCacheOutcome, {
+          warningCode: 'OFFLINE_CACHE_MISS',
+          message: 'Offline mode has no fresh PyPI project metadata in cache.',
+          retryable: false,
+          errorCode: 'PYPI_OFFLINE_CACHE_MISS'
+        });
+      }
+      let response;
+      try {
+        response = await fetchRegistryJson(requestFor(input), {
+          fetchImplementation,
+          timeoutMs,
+          maxResponseBytes,
+          userAgent,
+          errorPrefix: 'PYPI',
+          serviceName: 'PyPI'
+        });
+      } catch (error) {
+        if (error instanceof BoundedFetchError
+          && ['PYPI_RESPONSE_INVALID', 'PYPI_RESPONSE_TOO_LARGE'].includes(error.code)) {
+          return unavailableResult(input, normalizedRegistryBaseUrl, previousCacheOutcome, {
+            warningCode: 'REGISTRY_RESPONSE_INVALID',
+            message: 'PyPI returned an invalid project metadata response.',
+            retryable: true,
+            errorCode: error.code
+          });
+        }
+        return unavailableResult(input, normalizedRegistryBaseUrl, previousCacheOutcome, {
+          warningCode: 'REGISTRY_UNAVAILABLE',
+          message: 'PyPI project metadata is unavailable.',
+          retryable: true,
+          errorCode: error instanceof BoundedFetchError ? error.code : 'PYPI_TRANSPORT_FAILED'
+        });
+      }
+
+      if (response.status === 404) {
+        return unavailableResult(input, normalizedRegistryBaseUrl, previousCacheOutcome, {
+          status: 'notFound', warningCode: 'PACKAGE_NOT_FOUND',
+          message: 'Package was not found in PyPI.', retryable: false, errorCode: 'PYPI_HTTP_404'
+        });
+      }
+      if (response.status === 429) {
+        return unavailableResult(input, normalizedRegistryBaseUrl, previousCacheOutcome, {
+          warningCode: 'REGISTRY_RATE_LIMITED',
+          message: 'PyPI rate limited project metadata retrieval.', retryable: true, errorCode: 'PYPI_HTTP_429'
+        });
+      }
+      if (response.status !== 200 && response.status >= 200 && response.status < 300) {
+        return unavailableResult(input, normalizedRegistryBaseUrl, previousCacheOutcome, {
+          warningCode: 'REGISTRY_RESPONSE_INVALID',
+          message: 'PyPI returned an invalid project metadata response.', retryable: true,
+          errorCode: `PYPI_HTTP_${response.status}`
+        });
+      }
+      if (response.status !== 200) {
+        return unavailableResult(input, normalizedRegistryBaseUrl, previousCacheOutcome, {
+          warningCode: 'REGISTRY_UNAVAILABLE',
+          message: 'PyPI project metadata is unavailable.', retryable: response.status >= 500,
+          errorCode: `PYPI_HTTP_${response.status}`
+        });
+      }
+
+      try {
+        validatePypiProject(response.body, input.normalizedName);
+      } catch {
+        return unavailableResult(input, normalizedRegistryBaseUrl, previousCacheOutcome, {
+          warningCode: 'REGISTRY_RESPONSE_INVALID',
+          message: 'PyPI returned an invalid project metadata response.', retryable: true,
+          errorCode: 'PYPI_PROJECT_INVALID'
+        });
+      }
+
+      const cacheBody = sanitizeRegistryBodyForCache(response.body);
+      const stored = await knowledgeCache.write(identity, cacheBody, { ttlMs });
+      const fetchedCache = {
+        outcome: cached.status === 'expired' ? 'revalidated' : cached.status === 'corrupted' ? 'corrupted-replaced' : 'miss'
+      };
+      if (cached.status === 'corrupted') fetchedCache.reason = cached.reason;
+      return normalizedResult(input, cacheBody, normalizedRegistryBaseUrl, fetchedCache, stored);
+    }
+  };
+}

package/src/registry/sanitize-registry-body.js

@@ -0,0 +1,51 @@
+import { compareText } from '../portable.js';
+
+const PRIVATE_KEYS = new Set([
+  'authorization', 'proxyauthorization', 'cookie', 'setcookie', 'etag',
+  'lastmodified', 'headers', 'cachekey', 'stack', 'password', 'credential',
+  'credentials', 'secret', 'token', 'accesstoken', 'apikey', 'clientsecret', 'privatekey'
+]);
+
+function plainObject(value) {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) return false;
+  const prototype = Object.getPrototypeOf(value);
+  return prototype === Object.prototype || prototype === null;
+}
+
+function privateKey(key) {
+  return PRIVATE_KEYS.has(key.toLowerCase().replace(/[-_]/g, ''));
+}
+
+function unsafeStoredString(value) {
+  return value.startsWith('/')
+    || /^[A-Za-z]:[\\/]/.test(value)
+    || /[a-z][a-z0-9+.-]*:\/\/[^/?#]*@/i.test(value);
+}
+
+function sanitizeStoredString(value) {
+  if (unsafeStoredString(value)) return null;
+  const prefix = value.startsWith('git+') ? 'git+' : '';
+  const candidate = prefix ? value.slice(prefix.length) : value;
+  try {
+    const url = new URL(candidate);
+    if (!['http:', 'https:', 'ssh:', 'git:'].includes(url.protocol)) return value;
+    if (url.username || url.password) return null;
+    url.search = '';
+    url.hash = '';
+    return `${prefix}${url.toString()}`;
+  } catch {
+    return value;
+  }
+}
+
+/** Keep publisher-controlled response bodies safe for private cache storage. */
+export function sanitizeRegistryBodyForCache(value) {
+  if (value === null || typeof value === 'boolean' || typeof value === 'number') return value;
+  if (typeof value === 'string') return sanitizeStoredString(value);
+  if (Array.isArray(value)) return value.map(sanitizeRegistryBodyForCache);
+  if (!plainObject(value)) return null;
+  return Object.fromEntries(Object.keys(value).sort(compareText).flatMap((key) => {
+    if (privateKey(key)) return [];
+    return [[key, sanitizeRegistryBodyForCache(value[key])]];
+  }));
+}