@aikidosec/safe-chain 1.4.3 → 1.4.7

package/src/registryProxy/interceptors/pip/modifyPipInfo.js

@@ -0,0 +1,167 @@
+import { ui } from "../../../environment/userInteraction.js";
+import { clearCachingHeaders } from "../../http-utils.js";
+import { normalizePipPackageName } from "../../../scanning/packageNameVariants.js";
+import { parsePipPackageFromUrl } from "./parsePipPackageUrl.js";
+export { parsePipMetadataUrl, isPipPackageInfoUrl } from "./parsePipPackageUrl.js";
+import { getPipMetadataContentType, logSuppressedVersion } from "./pipMetadataResponseUtils.js";
+import { modifyPipJsonResponse } from "./modifyPipJsonResponse.js";
+
+// Match simple-index anchor tags and capture their href so we can suppress
+// individual distribution links from PyPI HTML metadata responses.
+const HTML_ANCHOR_HREF_RE =
+  /<a\b[^>]*href\s*=\s*(["'])([^"']+)\1[^>]*>[\s\S]*?<\/a>/gi;
+
+/**
+ * @param {Buffer} body
+ * @param {NodeJS.Dict<string | string[]> | undefined} headers
+ * @param {string} metadataUrl
+ * @param {(packageName: string | undefined, version: string | undefined) => boolean} isNewlyReleasedPackage
+ * @param {string} packageName
+ * @returns {Buffer}
+ */
+export function modifyPipInfoResponse(
+  body,
+  headers,
+  metadataUrl,
+  isNewlyReleasedPackage,
+  packageName
+) {
+  try {
+    const contentType = getPipMetadataContentType(headers);
+
+    if (!contentType || body.byteLength === 0) {
+      return body;
+    }
+
+    if (
+      contentType.includes("html") ||
+      contentType.includes("application/vnd.pypi.simple.v1+html")
+    ) {
+      return modifyHtmlSimpleResponse(
+        body,
+        headers,
+        metadataUrl,
+        isNewlyReleasedPackage,
+        packageName
+      );
+    }
+
+    if (
+      contentType.includes("json") ||
+      contentType.includes("application/vnd.pypi.simple.v1+json")
+    ) {
+      return modifyJsonResponse(
+        body,
+        headers,
+        metadataUrl,
+        isNewlyReleasedPackage,
+        packageName
+      );
+    }
+
+    return body;
+  } catch (/** @type {any} */ err) {
+    ui.writeVerbose(
+      `Safe-chain: PyPI package metadata not in expected format - bypassing modification. Error: ${err.message}`
+    );
+    return body;
+  }
+}
+
+/**
+ * @param {Buffer} body
+ * @param {NodeJS.Dict<string | string[]> | undefined} headers
+ * @param {string} metadataUrl
+ * @param {(packageName: string | undefined, version: string | undefined) => boolean} isNewlyReleasedPackage
+ * @param {string} packageName
+ * @returns {Buffer}
+ */
+function modifyHtmlSimpleResponse(
+  body,
+  headers,
+  metadataUrl,
+  isNewlyReleasedPackage,
+  packageName
+) {
+  const html = body.toString("utf8");
+  let modified = false;
+  const rewriteHtmlAnchor = createHtmlAnchorRewriter(
+    metadataUrl,
+    isNewlyReleasedPackage,
+    packageName,
+    () => {
+      modified = true;
+    }
+  );
+  const updatedHtml = html.replace(HTML_ANCHOR_HREF_RE, rewriteHtmlAnchor);
+
+  if (!modified) return body;
+  const modifiedBuffer = Buffer.from(updatedHtml);
+  clearCachingHeaders(headers);
+  return modifiedBuffer;
+}
+
+/**
+ * @param {string} metadataUrl
+ * @param {(packageName: string | undefined, version: string | undefined) => boolean} isNewlyReleasedPackage
+ * @param {string} packageName
+ * @param {() => void} onModified
+ * @returns {(anchor: string, quote: string, href: string) => string}
+ */
+function createHtmlAnchorRewriter(
+  metadataUrl,
+  isNewlyReleasedPackage,
+  packageName,
+  onModified
+) {
+  return (anchor, _quote, href) => {
+    const resolvedHref = new URL(href, metadataUrl).toString();
+    const { packageName: hrefPackageName, version } = parsePipPackageFromUrl(
+      resolvedHref,
+      new URL(resolvedHref).host
+    );
+
+    if (
+      hrefPackageName &&
+      normalizePipPackageName(hrefPackageName) ===
+        normalizePipPackageName(packageName) &&
+      version &&
+      isNewlyReleasedPackage(packageName, version)
+    ) {
+      onModified();
+      logSuppressedVersion(packageName, version);
+      return "";
+    }
+
+    return anchor;
+  };
+}
+
+/**
+ * @param {Buffer} body
+ * @param {NodeJS.Dict<string | string[]> | undefined} headers
+ * @param {string} metadataUrl
+ * @param {(packageName: string | undefined, version: string | undefined) => boolean} isNewlyReleasedPackage
+ * @param {string} packageName
+ * @returns {Buffer}
+ */
+function modifyJsonResponse(
+  body,
+  headers,
+  metadataUrl,
+  isNewlyReleasedPackage,
+  packageName
+) {
+  const json = JSON.parse(body.toString("utf8"));
+  const modified = modifyPipJsonResponse(
+    json,
+    metadataUrl,
+    isNewlyReleasedPackage,
+    packageName
+  );
+
+  if (!modified) return body;
+  const modifiedBuffer = Buffer.from(JSON.stringify(json));
+  clearCachingHeaders(headers);
+  return modifiedBuffer;
+}

package/src/registryProxy/interceptors/pip/modifyPipJsonResponse.js

@@ -0,0 +1,176 @@
+import {
+  calculateLatestVersion,
+  getAvailableVersionsFromJson,
+  getPackageVersionFromMetadataFile,
+} from "./pipMetadataVersionUtils.js";
+import { logSuppressedVersion } from "./pipMetadataResponseUtils.js";
+
+/**
+ * @param {any} json
+ * @param {string} metadataUrl
+ * @param {(packageName: string | undefined, version: string | undefined) => boolean} isNewlyReleasedPackage
+ * @param {string} packageName
+ * @returns {boolean}
+ */
+export function modifyPipJsonResponse(
+  json,
+  metadataUrl,
+  isNewlyReleasedPackage,
+  packageName
+) {
+  const filesModified = filterJsonMetadataFiles(
+    json,
+    metadataUrl,
+    isNewlyReleasedPackage,
+    packageName
+  );
+  const releasesModified = removeJsonMetadataReleases(
+    json,
+    isNewlyReleasedPackage,
+    packageName
+  );
+  const urlsModified = filterJsonMetadataUrls(
+    json,
+    metadataUrl,
+    isNewlyReleasedPackage,
+    packageName
+  );
+  const versionModified = updateJsonInfoVersion(json, metadataUrl);
+
+  return filesModified || releasesModified || urlsModified || versionModified;
+}
+
+/**
+ * @param {any} json
+ * @param {string} metadataUrl
+ * @param {(packageName: string | undefined, version: string | undefined) => boolean} isNewlyReleasedPackage
+ * @param {string} packageName
+ * @returns {boolean}
+ */
+function filterJsonMetadataFiles(
+  json,
+  metadataUrl,
+  isNewlyReleasedPackage,
+  packageName
+) {
+  if (!Array.isArray(json.files)) {
+    return false;
+  }
+
+  let modified = false;
+  const loggedVersions = new Set();
+  json.files = json.files.filter((/** @type {any} */ file) => {
+    const version = getPackageVersionFromMetadataFile(file, metadataUrl);
+
+    if (version && isNewlyReleasedPackage(packageName, version)) {
+      modified = true;
+      if (!loggedVersions.has(version)) {
+        logSuppressedVersion(packageName, version);
+        loggedVersions.add(version);
+      }
+      return false;
+    }
+
+    return true;
+  });
+
+  return modified;
+}
+
+/**
+ * @param {any} json
+ * @param {(packageName: string | undefined, version: string | undefined) => boolean} isNewlyReleasedPackage
+ * @param {string} packageName
+ * @returns {boolean}
+ */
+function removeJsonMetadataReleases(json, isNewlyReleasedPackage, packageName) {
+  if (!json.releases || typeof json.releases !== "object") {
+    return false;
+  }
+
+  let modified = false;
+
+  for (const [version, files] of Object.entries(json.releases)) {
+    if (
+      Array.isArray(/** @type {unknown[]} */ (files)) &&
+      isNewlyReleasedPackage(packageName, version)
+    ) {
+      delete json.releases[version];
+      modified = true;
+      logSuppressedVersion(packageName, version);
+    }
+  }
+
+  return modified;
+}
+
+/**
+ * @param {any} json
+ * @param {string} metadataUrl
+ * @param {(packageName: string | undefined, version: string | undefined) => boolean} isNewlyReleasedPackage
+ * @param {string} packageName
+ * @returns {boolean}
+ */
+function filterJsonMetadataUrls(
+  json,
+  metadataUrl,
+  isNewlyReleasedPackage,
+  packageName
+) {
+  if (!Array.isArray(json.urls)) {
+    return false;
+  }
+
+  let modified = false;
+  const loggedVersions = new Set();
+  json.urls = json.urls.filter((/** @type {any} */ file) => {
+    const version = getPackageVersionFromMetadataFile(file, metadataUrl);
+
+    if (version && isNewlyReleasedPackage(packageName, version)) {
+      modified = true;
+      if (!loggedVersions.has(version)) {
+        logSuppressedVersion(packageName, version);
+        loggedVersions.add(version);
+      }
+      return false;
+    }
+
+    return true;
+  });
+
+  return modified;
+}
+
+/**
+ * @param {any} json
+ * @param {string} metadataUrl
+ * @returns {boolean}
+ */
+function updateJsonInfoVersion(json, metadataUrl) {
+  if (!json.info || typeof json.info !== "object") {
+    return false;
+  }
+
+  const replacementVersion = computeReplacementVersion(json, metadataUrl);
+
+  if (
+    typeof json.info.version !== "string" ||
+    !replacementVersion ||
+    json.info.version === replacementVersion
+  ) {
+    return false;
+  }
+
+  json.info.version = replacementVersion;
+  return true;
+}
+
+/**
+ * @param {any} json
+ * @param {string} metadataUrl
+ * @returns {string | undefined}
+ */
+function computeReplacementVersion(json, metadataUrl) {
+  const candidateVersions = getAvailableVersionsFromJson(json, metadataUrl);
+  return calculateLatestVersion(candidateVersions);
+}

package/src/registryProxy/interceptors/pip/parsePipPackageUrl.js

@@ -0,0 +1,162 @@
+/**
+ * Parses a PyPI metadata URL and returns the package name and API type.
+ *
+ * @example
+ * parsePipMetadataUrl("https://pypi.org/simple/requests/")
+ * // => { packageName: "requests", type: "simple" }
+ *
+ * parsePipMetadataUrl("https://pypi.org/pypi/requests/json")
+ * // => { packageName: "requests", type: "json" }
+ *
+ * parsePipMetadataUrl("https://pypi.org/pypi/requests/2.28.1/json")
+ * // => { packageName: "requests", type: "json" }
+ *
+ * parsePipMetadataUrl("https://files.pythonhosted.org/packages/requests-2.28.1.tar.gz")
+ * // => { packageName: undefined, type: undefined }
+ *
+ * @param {string} url
+ * @returns {{ packageName: string | undefined, type: "simple" | "json" | undefined }}
+ */
+export function parsePipMetadataUrl(url) {
+  if (typeof url !== "string") {
+    return { packageName: undefined, type: undefined };
+  }
+
+  let urlObj;
+  try {
+    urlObj = new URL(url);
+  } catch {
+    return { packageName: undefined, type: undefined };
+  }
+
+  const pathSegments = urlObj.pathname.split("/").filter(Boolean);
+  if (pathSegments[0] === "simple" && pathSegments[1]) {
+    return {
+      packageName: decodeURIComponent(pathSegments[1]),
+      type: "simple",
+    };
+  }
+
+  if (
+    pathSegments[0] === "pypi" &&
+    pathSegments[pathSegments.length - 1] === "json" &&
+    pathSegments[1]
+  ) {
+    return {
+      packageName: decodeURIComponent(pathSegments[1]),
+      type: "json",
+    };
+  }
+
+  return { packageName: undefined, type: undefined };
+}
+
+/**
+ * @param {string} url
+ * @returns {boolean}
+ */
+export function isPipPackageInfoUrl(url) {
+  return !!parsePipMetadataUrl(url).packageName;
+}
+
+/**
+ * Parse Python package artifact URLs from PyPI-style registries.
+ * Examples:
+ * - Wheel: https://files.pythonhosted.org/packages/.../requests-2.28.1-py3-none-any.whl
+ * - Wheel metadata: https://files.pythonhosted.org/packages/.../requests-2.28.1-py3-none-any.whl.metadata
+ * - Sdist: https://files.pythonhosted.org/packages/.../requests-2.28.1.tar.gz
+ *
+ * @param {string} url
+ * @param {string} registry
+ * @returns {{packageName: string | undefined, version: string | undefined}}
+ */
+export function parsePipPackageFromUrl(url, registry) {
+  if (!registry || typeof url !== "string") {
+    return { packageName: undefined, version: undefined };
+  }
+
+  let urlObj;
+  try {
+    urlObj = new URL(url);
+  } catch {
+    return { packageName: undefined, version: undefined };
+  }
+
+  const lastSegment = urlObj.pathname.split("/").filter(Boolean).pop();
+  if (!lastSegment) {
+    return { packageName: undefined, version: undefined };
+  }
+
+  const filename = decodeURIComponent(lastSegment);
+
+  const wheelExtRe = /\.whl(?:\.metadata)?$/;
+  if (wheelExtRe.test(filename)) {
+    return parseWheelFilename(filename, wheelExtRe);
+  }
+
+  const sdistExtWithMetadataRe = /\.(tar\.gz|zip|tar\.bz2|tar\.xz)(\.metadata)?$/i;
+  if (!sdistExtWithMetadataRe.test(filename)) {
+    return { packageName: undefined, version: undefined };
+  }
+
+  return parseSdistFilename(filename, sdistExtWithMetadataRe);
+}
+
+/**
+ * Parse wheel filenames and Poetry preflight metadata.
+ * Examples:
+ * - foo_bar-2.0.0-py3-none-any.whl
+ * - foo_bar-2.0.0-py3-none-any.whl.metadata
+ *
+ * @param {string} filename
+ * @param {RegExp} wheelExtRe
+ * @returns {{packageName: string | undefined, version: string | undefined}}
+ */
+function parseWheelFilename(filename, wheelExtRe) {
+  const base = filename.replace(wheelExtRe, "");
+  const firstDash = base.indexOf("-");
+  if (firstDash <= 0) {
+    return { packageName: undefined, version: undefined };
+  }
+
+  const packageName = base.slice(0, firstDash);
+  const rest = base.slice(firstDash + 1);
+  const secondDash = rest.indexOf("-");
+  const version = secondDash >= 0 ? rest.slice(0, secondDash) : rest;
+
+  // "latest" is a resolver-style token, not an actual published artifact version.
+  if (version === "latest" || !packageName || !version) {
+    return { packageName: undefined, version: undefined };
+  }
+
+  return { packageName, version };
+}
+
+/**
+ * Parse source distribution filenames, with optional metadata suffix.
+ * Examples:
+ * - requests-2.28.1.tar.gz
+ * - requests-2.28.1.zip
+ * - requests-2.28.1.tar.gz.metadata
+ *
+ * @param {string} filename
+ * @param {RegExp} sdistExtWithMetadataRe
+ * @returns {{packageName: string | undefined, version: string | undefined}}
+ */
+function parseSdistFilename(filename, sdistExtWithMetadataRe) {
+  const base = filename.replace(sdistExtWithMetadataRe, "");
+  const lastDash = base.lastIndexOf("-");
+  if (lastDash <= 0 || lastDash >= base.length - 1) {
+    return { packageName: undefined, version: undefined };
+  }
+
+  const packageName = base.slice(0, lastDash);
+  const version = base.slice(lastDash + 1);
+
+  // "latest" is a resolver-style token, not an actual published artifact version.
+  if (version === "latest" || !packageName || !version) {
+    return { packageName: undefined, version: undefined };
+  }
+
+  return { packageName, version };
+}

package/src/registryProxy/interceptors/pip/pipInterceptor.js

@@ -0,0 +1,122 @@
+import {
+  ECOSYSTEM_PY,
+  getPipCustomRegistries,
+  skipMinimumPackageAge,
+} from "../../../config/settings.js";
+import { isMalwarePackage } from "../../../scanning/audit/index.js";
+import { getEquivalentPackageNames } from "../../../scanning/packageNameVariants.js";
+import { openNewPackagesDatabase } from "../../../scanning/newPackagesListCache.js";
+import { interceptRequests } from "../interceptorBuilder.js";
+import { isExcludedFromMinimumPackageAge } from "../minimumPackageAgeExclusions.js";
+import {
+  modifyPipInfoResponse,
+  parsePipMetadataUrl,
+} from "./modifyPipInfo.js";
+import { parsePipPackageFromUrl } from "./parsePipPackageUrl.js";
+
+const knownPipRegistries = [
+  "files.pythonhosted.org",
+  "pypi.org",
+  "pypi.python.org",
+  "pythonhosted.org",
+];
+
+/**
+ * @param {string} url
+ * @returns {import("../interceptorBuilder.js").Interceptor | undefined}
+ */
+export function pipInterceptorForUrl(url) {
+  const customRegistries = getPipCustomRegistries();
+  const registries = [...knownPipRegistries, ...customRegistries];
+  const registry = registries.find((reg) => url.includes(reg));
+
+  if (registry) {
+    return buildPipInterceptor(registry);
+  }
+
+  return undefined;
+}
+
+/**
+ * @param {string} registry
+ * @returns {import("../interceptorBuilder.js").Interceptor | undefined}
+ */
+function buildPipInterceptor(registry) {
+  return interceptRequests(createPipRequestHandler(registry));
+}
+
+/**
+ * @param {string} registry
+ * @returns {(reqContext: import("../interceptorBuilder.js").RequestInterceptionContext) => Promise<void>}
+ */
+function createPipRequestHandler(registry) {
+  return async (reqContext) => {
+    const minimumAgeChecksEnabled = !skipMinimumPackageAge();
+    const metadataInfo = parsePipMetadataUrl(reqContext.targetUrl);
+    const metadataPackageName = metadataInfo.packageName;
+
+    if (
+      minimumAgeChecksEnabled &&
+      metadataPackageName &&
+      !isExcludedFromMinimumPackageAge(metadataPackageName)
+    ) {
+      const newPackagesDatabase = await openNewPackagesDatabase();
+      reqContext.modifyBody((body, headers) =>
+        modifyPipInfoResponse(
+          body,
+          headers,
+          reqContext.targetUrl,
+          newPackagesDatabase.isNewlyReleasedPackage,
+          metadataPackageName
+        )
+      );
+      return;
+    }
+
+    const { packageName, version } = parsePipPackageFromUrl(
+      reqContext.targetUrl,
+      registry
+    );
+
+    if (!packageName) {
+      return;
+    }
+
+    const equivalentPackageNames = getEquivalentPackageNames(
+      packageName,
+      ECOSYSTEM_PY
+    );
+    let isMalicious = false;
+    for (const equivalentPackageName of equivalentPackageNames) {
+      if (await isMalwarePackage(equivalentPackageName, version)) {
+        isMalicious = true;
+        break;
+      }
+    }
+
+    if (isMalicious) {
+      reqContext.blockMalware(packageName, version);
+      return;
+    }
+
+    if (
+      version &&
+      minimumAgeChecksEnabled &&
+      !isExcludedFromMinimumPackageAge(packageName)
+    ) {
+      const newPackagesDatabase = await openNewPackagesDatabase();
+      const isNewlyReleased = newPackagesDatabase.isNewlyReleasedPackage(
+        packageName,
+        version
+      );
+
+      if (isNewlyReleased) {
+        reqContext.blockMinimumAgeRequest(
+          packageName,
+          version,
+          `Forbidden - blocked by safe-chain direct download minimum package age (${packageName}@${version})`
+        );
+      }
+    }
+  };
+}

package/src/registryProxy/interceptors/pip/pipMetadataResponseUtils.js

@@ -0,0 +1,27 @@
+import { getMinimumPackageAgeHours } from "../../../config/settings.js";
+import { ui } from "../../../environment/userInteraction.js";
+import { getHeaderValueAsString } from "../../http-utils.js";
+import { recordSuppressedVersion } from "../suppressedVersionsState.js";
+
+/**
+ * @param {NodeJS.Dict<string | string[]> | undefined} headers
+ * @returns {string | undefined}
+ */
+export function getPipMetadataContentType(headers) {
+  return getHeaderValueAsString(headers, "content-type")
+    ?.toLowerCase()
+    .split(";")[0]
+    .trim();
+}
+
+/**
+ * @param {string} packageName
+ * @param {string} version
+ * @returns {void}
+ */
+export function logSuppressedVersion(packageName, version) {
+  recordSuppressedVersion();
+  ui.writeVerbose(
+    `Safe-chain: ${packageName}@${version} is newer than ${getMinimumPackageAgeHours()} hours and was removed (minimumPackageAgeInHours setting).`
+  );
+}