@aikidosec/safe-chain 0.0.1-immutable-releases-beta

package/src/registryProxy/interceptors/pip/modifyPipJsonResponse.js

@@ -0,0 +1,176 @@
+import {
+  calculateLatestVersion,
+  getAvailableVersionsFromJson,
+  getPackageVersionFromMetadataFile,
+} from "./pipMetadataVersionUtils.js";
+import { logSuppressedVersion } from "./pipMetadataResponseUtils.js";
+
+/**
+ * @param {any} json
+ * @param {string} metadataUrl
+ * @param {(packageName: string | undefined, version: string | undefined) => boolean} isNewlyReleasedPackage
+ * @param {string} packageName
+ * @returns {boolean}
+ */
+export function modifyPipJsonResponse(
+  json,
+  metadataUrl,
+  isNewlyReleasedPackage,
+  packageName
+) {
+  const filesModified = filterJsonMetadataFiles(
+    json,
+    metadataUrl,
+    isNewlyReleasedPackage,
+    packageName
+  );
+  const releasesModified = removeJsonMetadataReleases(
+    json,
+    isNewlyReleasedPackage,
+    packageName
+  );
+  const urlsModified = filterJsonMetadataUrls(
+    json,
+    metadataUrl,
+    isNewlyReleasedPackage,
+    packageName
+  );
+  const versionModified = updateJsonInfoVersion(json, metadataUrl);
+
+  return filesModified || releasesModified || urlsModified || versionModified;
+}
+
+/**
+ * @param {any} json
+ * @param {string} metadataUrl
+ * @param {(packageName: string | undefined, version: string | undefined) => boolean} isNewlyReleasedPackage
+ * @param {string} packageName
+ * @returns {boolean}
+ */
+function filterJsonMetadataFiles(
+  json,
+  metadataUrl,
+  isNewlyReleasedPackage,
+  packageName
+) {
+  if (!Array.isArray(json.files)) {
+    return false;
+  }
+
+  let modified = false;
+  const loggedVersions = new Set();
+  json.files = json.files.filter((/** @type {any} */ file) => {
+    const version = getPackageVersionFromMetadataFile(file, metadataUrl);
+
+    if (version && isNewlyReleasedPackage(packageName, version)) {
+      modified = true;
+      if (!loggedVersions.has(version)) {
+        logSuppressedVersion(packageName, version);
+        loggedVersions.add(version);
+      }
+      return false;
+    }
+
+    return true;
+  });
+
+  return modified;
+}
+
+/**
+ * @param {any} json
+ * @param {(packageName: string | undefined, version: string | undefined) => boolean} isNewlyReleasedPackage
+ * @param {string} packageName
+ * @returns {boolean}
+ */
+function removeJsonMetadataReleases(json, isNewlyReleasedPackage, packageName) {
+  if (!json.releases || typeof json.releases !== "object") {
+    return false;
+  }
+
+  let modified = false;
+
+  for (const [version, files] of Object.entries(json.releases)) {
+    if (
+      Array.isArray(/** @type {unknown[]} */ (files)) &&
+      isNewlyReleasedPackage(packageName, version)
+    ) {
+      delete json.releases[version];
+      modified = true;
+      logSuppressedVersion(packageName, version);
+    }
+  }
+
+  return modified;
+}
+
+/**
+ * @param {any} json
+ * @param {string} metadataUrl
+ * @param {(packageName: string | undefined, version: string | undefined) => boolean} isNewlyReleasedPackage
+ * @param {string} packageName
+ * @returns {boolean}
+ */
+function filterJsonMetadataUrls(
+  json,
+  metadataUrl,
+  isNewlyReleasedPackage,
+  packageName
+) {
+  if (!Array.isArray(json.urls)) {
+    return false;
+  }
+
+  let modified = false;
+  const loggedVersions = new Set();
+  json.urls = json.urls.filter((/** @type {any} */ file) => {
+    const version = getPackageVersionFromMetadataFile(file, metadataUrl);
+
+    if (version && isNewlyReleasedPackage(packageName, version)) {
+      modified = true;
+      if (!loggedVersions.has(version)) {
+        logSuppressedVersion(packageName, version);
+        loggedVersions.add(version);
+      }
+      return false;
+    }
+
+    return true;
+  });
+
+  return modified;
+}
+
+/**
+ * @param {any} json
+ * @param {string} metadataUrl
+ * @returns {boolean}
+ */
+function updateJsonInfoVersion(json, metadataUrl) {
+  if (!json.info || typeof json.info !== "object") {
+    return false;
+  }
+
+  const replacementVersion = computeReplacementVersion(json, metadataUrl);
+
+  if (
+    typeof json.info.version !== "string" ||
+    !replacementVersion ||
+    json.info.version === replacementVersion
+  ) {
+    return false;
+  }
+
+  json.info.version = replacementVersion;
+  return true;
+}
+
+/**
+ * @param {any} json
+ * @param {string} metadataUrl
+ * @returns {string | undefined}
+ */
+function computeReplacementVersion(json, metadataUrl) {
+  const candidateVersions = getAvailableVersionsFromJson(json, metadataUrl);
+  return calculateLatestVersion(candidateVersions);
+}

package/src/registryProxy/interceptors/pip/parsePipPackageUrl.js

@@ -0,0 +1,162 @@
+/**
+ * Parses a PyPI metadata URL and returns the package name and API type.
+ *
+ * @example
+ * parsePipMetadataUrl("https://pypi.org/simple/requests/")
+ * // => { packageName: "requests", type: "simple" }
+ *
+ * parsePipMetadataUrl("https://pypi.org/pypi/requests/json")
+ * // => { packageName: "requests", type: "json" }
+ *
+ * parsePipMetadataUrl("https://pypi.org/pypi/requests/2.28.1/json")
+ * // => { packageName: "requests", type: "json" }
+ *
+ * parsePipMetadataUrl("https://files.pythonhosted.org/packages/requests-2.28.1.tar.gz")
+ * // => { packageName: undefined, type: undefined }
+ *
+ * @param {string} url
+ * @returns {{ packageName: string | undefined, type: "simple" | "json" | undefined }}
+ */
+export function parsePipMetadataUrl(url) {
+  if (typeof url !== "string") {
+    return { packageName: undefined, type: undefined };
+  }
+
+  let urlObj;
+  try {
+    urlObj = new URL(url);
+  } catch {
+    return { packageName: undefined, type: undefined };
+  }
+
+  const pathSegments = urlObj.pathname.split("/").filter(Boolean);
+  if (pathSegments[0] === "simple" && pathSegments[1]) {
+    return {
+      packageName: decodeURIComponent(pathSegments[1]),
+      type: "simple",
+    };
+  }
+
+  if (
+    pathSegments[0] === "pypi" &&
+    pathSegments[pathSegments.length - 1] === "json" &&
+    pathSegments[1]
+  ) {
+    return {
+      packageName: decodeURIComponent(pathSegments[1]),
+      type: "json",
+    };
+  }
+
+  return { packageName: undefined, type: undefined };
+}
+
+/**
+ * @param {string} url
+ * @returns {boolean}
+ */
+export function isPipPackageInfoUrl(url) {
+  return !!parsePipMetadataUrl(url).packageName;
+}
+
+/**
+ * Parse Python package artifact URLs from PyPI-style registries.
+ * Examples:
+ * - Wheel: https://files.pythonhosted.org/packages/.../requests-2.28.1-py3-none-any.whl
+ * - Wheel metadata: https://files.pythonhosted.org/packages/.../requests-2.28.1-py3-none-any.whl.metadata
+ * - Sdist: https://files.pythonhosted.org/packages/.../requests-2.28.1.tar.gz
+ *
+ * @param {string} url
+ * @param {string} registry
+ * @returns {{packageName: string | undefined, version: string | undefined}}
+ */
+export function parsePipPackageFromUrl(url, registry) {
+  if (!registry || typeof url !== "string") {
+    return { packageName: undefined, version: undefined };
+  }
+
+  let urlObj;
+  try {
+    urlObj = new URL(url);
+  } catch {
+    return { packageName: undefined, version: undefined };
+  }
+
+  const lastSegment = urlObj.pathname.split("/").filter(Boolean).pop();
+  if (!lastSegment) {
+    return { packageName: undefined, version: undefined };
+  }
+
+  const filename = decodeURIComponent(lastSegment);
+
+  const wheelExtRe = /\.whl(?:\.metadata)?$/;
+  if (wheelExtRe.test(filename)) {
+    return parseWheelFilename(filename, wheelExtRe);
+  }
+
+  const sdistExtWithMetadataRe = /\.(tar\.gz|zip|tar\.bz2|tar\.xz)(\.metadata)?$/i;
+  if (!sdistExtWithMetadataRe.test(filename)) {
+    return { packageName: undefined, version: undefined };
+  }
+
+  return parseSdistFilename(filename, sdistExtWithMetadataRe);
+}
+
+/**
+ * Parse wheel filenames and Poetry preflight metadata.
+ * Examples:
+ * - foo_bar-2.0.0-py3-none-any.whl
+ * - foo_bar-2.0.0-py3-none-any.whl.metadata
+ *
+ * @param {string} filename
+ * @param {RegExp} wheelExtRe
+ * @returns {{packageName: string | undefined, version: string | undefined}}
+ */
+function parseWheelFilename(filename, wheelExtRe) {
+  const base = filename.replace(wheelExtRe, "");
+  const firstDash = base.indexOf("-");
+  if (firstDash <= 0) {
+    return { packageName: undefined, version: undefined };
+  }
+
+  const packageName = base.slice(0, firstDash);
+  const rest = base.slice(firstDash + 1);
+  const secondDash = rest.indexOf("-");
+  const version = secondDash >= 0 ? rest.slice(0, secondDash) : rest;
+
+  // "latest" is a resolver-style token, not an actual published artifact version.
+  if (version === "latest" || !packageName || !version) {
+    return { packageName: undefined, version: undefined };
+  }
+
+  return { packageName, version };
+}
+
+/**
+ * Parse source distribution filenames, with optional metadata suffix.
+ * Examples:
+ * - requests-2.28.1.tar.gz
+ * - requests-2.28.1.zip
+ * - requests-2.28.1.tar.gz.metadata
+ *
+ * @param {string} filename
+ * @param {RegExp} sdistExtWithMetadataRe
+ * @returns {{packageName: string | undefined, version: string | undefined}}
+ */
+function parseSdistFilename(filename, sdistExtWithMetadataRe) {
+  const base = filename.replace(sdistExtWithMetadataRe, "");
+  const lastDash = base.lastIndexOf("-");
+  if (lastDash <= 0 || lastDash >= base.length - 1) {
+    return { packageName: undefined, version: undefined };
+  }
+
+  const packageName = base.slice(0, lastDash);
+  const version = base.slice(lastDash + 1);
+
+  // "latest" is a resolver-style token, not an actual published artifact version.
+  if (version === "latest" || !packageName || !version) {
+    return { packageName: undefined, version: undefined };
+  }
+
+  return { packageName, version };
+}

package/src/registryProxy/interceptors/pip/pipInterceptor.js

@@ -0,0 +1,122 @@
+import {
+  ECOSYSTEM_PY,
+  getPipCustomRegistries,
+  skipMinimumPackageAge,
+} from "../../../config/settings.js";
+import { isMalwarePackage } from "../../../scanning/audit/index.js";
+import { getEquivalentPackageNames } from "../../../scanning/packageNameVariants.js";
+import { openNewPackagesDatabase } from "../../../scanning/newPackagesListCache.js";
+import { interceptRequests } from "../interceptorBuilder.js";
+import { isExcludedFromMinimumPackageAge } from "../minimumPackageAgeExclusions.js";
+import {
+  modifyPipInfoResponse,
+  parsePipMetadataUrl,
+} from "./modifyPipInfo.js";
+import { parsePipPackageFromUrl } from "./parsePipPackageUrl.js";
+
+const knownPipRegistries = [
+  "files.pythonhosted.org",
+  "pypi.org",
+  "pypi.python.org",
+  "pythonhosted.org",
+];
+
+/**
+ * @param {string} url
+ * @returns {import("../interceptorBuilder.js").Interceptor | undefined}
+ */
+export function pipInterceptorForUrl(url) {
+  const customRegistries = getPipCustomRegistries();
+  const registries = [...knownPipRegistries, ...customRegistries];
+  const registry = registries.find((reg) => url.includes(reg));
+
+  if (registry) {
+    return buildPipInterceptor(registry);
+  }
+
+  return undefined;
+}
+
+/**
+ * @param {string} registry
+ * @returns {import("../interceptorBuilder.js").Interceptor | undefined}
+ */
+function buildPipInterceptor(registry) {
+  return interceptRequests(createPipRequestHandler(registry));
+}
+
+/**
+ * @param {string} registry
+ * @returns {(reqContext: import("../interceptorBuilder.js").RequestInterceptionContext) => Promise<void>}
+ */
+function createPipRequestHandler(registry) {
+  return async (reqContext) => {
+    const minimumAgeChecksEnabled = !skipMinimumPackageAge();
+    const metadataInfo = parsePipMetadataUrl(reqContext.targetUrl);
+    const metadataPackageName = metadataInfo.packageName;
+
+    if (
+      minimumAgeChecksEnabled &&
+      metadataPackageName &&
+      !isExcludedFromMinimumPackageAge(metadataPackageName)
+    ) {
+      const newPackagesDatabase = await openNewPackagesDatabase();
+      reqContext.modifyBody((body, headers) =>
+        modifyPipInfoResponse(
+          body,
+          headers,
+          reqContext.targetUrl,
+          newPackagesDatabase.isNewlyReleasedPackage,
+          metadataPackageName
+        )
+      );
+      return;
+    }
+
+    const { packageName, version } = parsePipPackageFromUrl(
+      reqContext.targetUrl,
+      registry
+    );
+
+    if (!packageName) {
+      return;
+    }
+
+    const equivalentPackageNames = getEquivalentPackageNames(
+      packageName,
+      ECOSYSTEM_PY
+    );
+    let isMalicious = false;
+    for (const equivalentPackageName of equivalentPackageNames) {
+      if (await isMalwarePackage(equivalentPackageName, version)) {
+        isMalicious = true;
+        break;
+      }
+    }
+
+    if (isMalicious) {
+      reqContext.blockMalware(packageName, version);
+      return;
+    }
+
+    if (
+      version &&
+      minimumAgeChecksEnabled &&
+      !isExcludedFromMinimumPackageAge(packageName)
+    ) {
+      const newPackagesDatabase = await openNewPackagesDatabase();
+      const isNewlyReleased = newPackagesDatabase.isNewlyReleasedPackage(
+        packageName,
+        version
+      );
+
+      if (isNewlyReleased) {
+        reqContext.blockMinimumAgeRequest(
+          packageName,
+          version,
+          `Forbidden - blocked by safe-chain direct download minimum package age (${packageName}@${version})`
+        );
+      }
+    }
+  };
+}

package/src/registryProxy/interceptors/pip/pipMetadataResponseUtils.js

@@ -0,0 +1,27 @@
+import { getMinimumPackageAgeHours } from "../../../config/settings.js";
+import { ui } from "../../../environment/userInteraction.js";
+import { getHeaderValueAsString } from "../../http-utils.js";
+import { recordSuppressedVersion } from "../suppressedVersionsState.js";
+
+/**
+ * @param {NodeJS.Dict<string | string[]> | undefined} headers
+ * @returns {string | undefined}
+ */
+export function getPipMetadataContentType(headers) {
+  return getHeaderValueAsString(headers, "content-type")
+    ?.toLowerCase()
+    .split(";")[0]
+    .trim();
+}
+
+/**
+ * @param {string} packageName
+ * @param {string} version
+ * @returns {void}
+ */
+export function logSuppressedVersion(packageName, version) {
+  recordSuppressedVersion();
+  ui.writeVerbose(
+    `Safe-chain: ${packageName}@${version} is newer than ${getMinimumPackageAgeHours()} hours and was removed (minimumPackageAgeInHours setting).`
+  );
+}

package/src/registryProxy/interceptors/pip/pipMetadataVersionUtils.js

@@ -0,0 +1,131 @@
+import { parsePipPackageFromUrl } from "./parsePipPackageUrl.js";
+
+/**
+ * @param {any} file
+ * @param {string} metadataUrl
+ * @returns {string | undefined}
+ */
+export function getPackageVersionFromMetadataFile(file, metadataUrl) {
+  const href = typeof file?.url === "string" ? file.url : undefined;
+  const filename = typeof file?.filename === "string" ? file.filename : undefined;
+
+  if (href) {
+    const resolvedHref = new URL(href, metadataUrl).toString();
+    return parsePipPackageFromUrl(
+      resolvedHref,
+      new URL(resolvedHref).host
+    ).version;
+  }
+
+  if (filename) {
+    return parsePipPackageFromUrl(
+      new URL(filename, metadataUrl).toString(),
+      new URL(metadataUrl).host
+    ).version;
+  }
+
+  return undefined;
+}
+
+/**
+ * @param {any} json
+ * @param {string} metadataUrl
+ * @returns {string[]}
+ */
+export function getAvailableVersionsFromJson(json, metadataUrl) {
+  if (json.releases && typeof json.releases === "object") {
+    return Object.keys(json.releases);
+  }
+
+  if (!Array.isArray(json.files)) {
+    return [];
+  }
+
+  return [
+    ...new Set(
+      json.files
+        .map((/** @type {any} */ file) =>
+          getPackageVersionFromMetadataFile(file, metadataUrl)
+        )
+        .filter(isDefinedString)
+    ),
+  ];
+}
+
+/**
+ * @param {string | undefined} value
+ * @returns {value is string}
+ */
+function isDefinedString(value) {
+  return typeof value === "string";
+}
+
+/**
+ * @param {string[]} versions
+ * @returns {string | undefined}
+ */
+export function calculateLatestVersion(versions) {
+  const stableVersions = versions.filter((version) => !isPrerelease(version));
+  if (stableVersions.length > 0) {
+    return stableVersions.sort(comparePep440ishVersions).at(-1);
+  }
+
+  return versions.sort(comparePep440ishVersions).at(-1);
+}
+
+/**
+ * @param {string} left
+ * @param {string} right
+ * @returns {number}
+ */
+function comparePep440ishVersions(left, right) {
+  const leftParts = tokenizeVersion(left);
+  const rightParts = tokenizeVersion(right);
+  const maxLength = Math.max(leftParts.length, rightParts.length);
+
+  for (let index = 0; index < maxLength; index += 1) {
+    const leftPart = leftParts[index];
+    const rightPart = rightParts[index];
+
+    if (leftPart === undefined) return -1;
+    if (rightPart === undefined) return 1;
+
+    if (leftPart === rightPart) {
+      continue;
+    }
+
+    const leftNumeric = typeof leftPart === "number";
+    const rightNumeric = typeof rightPart === "number";
+
+    if (leftNumeric && rightNumeric) {
+      return leftPart - rightPart;
+    }
+
+    if (leftNumeric) return 1;
+    if (rightNumeric) return -1;
+
+    return String(leftPart).localeCompare(String(rightPart));
+  }
+
+  return 0;
+}
+
+/**
+ * @param {string} version
+ * @returns {(string | number)[]}
+ */
+function tokenizeVersion(version) {
+  return version
+    .toLowerCase()
+    .split(/[^a-z0-9]+/)
+    .flatMap((part) => part.match(/[a-z]+|\d+/g) || [])
+    .map((part) => (/^\d+$/.test(part) ? Number(part) : part));
+}
+
+/**
+ * @param {string} version
+ * @returns {boolean}
+ */
+function isPrerelease(version) {
+  return /(a|b|rc|dev)\d+/i.test(version);
+}

package/src/registryProxy/interceptors/suppressedVersionsState.js

@@ -0,0 +1,21 @@
+const state = {
+  hasSuppressedVersions: false,
+};
+
+/**
+ * Tracks whether any rewritten metadata response suppressed versions during the
+ * current process lifetime. This is intentional shared state used only for the
+ * end-of-run summary message exposed through the proxy API.
+ *
+ * @returns {void}
+ */
+export function recordSuppressedVersion() {
+  state.hasSuppressedVersions = true;
+}
+
+/**
+ * @returns {boolean}
+ */
+export function getHasSuppressedVersions() {
+  return state.hasSuppressedVersions;
+}

package/src/registryProxy/isImdsEndpoint.js

@@ -0,0 +1,13 @@
+// Instance Metadata Service (IMDS) endpoints used by cloud providers.
+// Cloud SDK tools probe these to detect environment and retrieve credentials.
+// When outside cloud environments, connections timeout - we reduce timeout (3s vs 30s)
+// and suppress error logging since this is expected behavior.
+const imdsEndpoints = [
+  "metadata.google.internal",
+  "metadata.goog",
+  "169.254.169.254", // AWS, Azure, Oracle Cloud, GCP
+];
+
+export function isImdsEndpoint(/** @type {string} */ host) {
+  return imdsEndpoints.includes(host);
+}