dependabot-python 0.385.0 → 0.386.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: edc98d373ce3e27a5db1080f1ddc5f2a7a60c8be26a734ac889abf8e2298ede0
-  data.tar.gz: fd9d7507adb7b29b972c2f318a785eea5efd4150e12e2a6a3fe5707463d38f8a
+  metadata.gz: d6743a3b5b5ae514de1fc09550af84d3285ae7fed69a8c74b8d95291b7cb2c1a
+  data.tar.gz: 1bde8ed1421e31bf67c7ca1fab04f7f0c2edd3fbd4afcb2eb38df5c0bba007d1
 SHA512:
-  metadata.gz: 9d328bd3c10cb36152a63a2c3975303e111a540367ec6e464ef7ab8208bd2ee642c12561f72928116069160a39cf1b9aa0527b4efb7411966714ff6e41032cdf
-  data.tar.gz: 2a00eef7b5ac804ee7389022190c84a933e9ae2afa89e37852ffb68729a591ceba986587f823ac1a0df460d6d42e72587979328104af12b2d53a05f99c16b150
+  metadata.gz: e484276f80b4690e9f41f471f18adb8f4e0adf1481c73c1912ead78274785d908cea245a50265a3f2e567ae643d5d4759fc1694fbc66ec91903f3c65136f95f7
+  data.tar.gz: f0b4c4092d96440d9e98acb64d750b380712cb456da886ecf4c8a6872ef5cb4b2f3d98551fb0e940878de37fe600dcc5981494e6d9eee978351f8ade1e584f99

data/lib/dependabot/python/dependency_grapher/requirements_layers.rb

@@ -0,0 +1,210 @@
+# typed: strong
+# frozen_string_literal: true
+
+require "pathname"
+require "sorbet-runtime"
+
+require "dependabot/dependency_file"
+require "dependabot/dependency_graphers/base"
+require "dependabot/python/shared_file_fetcher"
+
+module Dependabot
+  module Python
+    class DependencyGrapher < Dependabot::DependencyGraphers::Base
+      # Splits a directory of requirements files (pip / pip-compile) into one manifest group per "layer".
+      #
+      # Python is the reported case of "layered requirements": a single directory can hold multiple
+      # independent requirements files (e.g. base-requirements.txt, test-requirements.txt) cross-linked
+      # with `-r`/`-c`.
+      #
+      # Each 'layer' is its own manifest and must get its own snapshot, rather than the whole
+      # directory collapsing onto the first `.txt` alphabetically.
+      class RequirementsLayers
+        extend T::Sig
+
+        # Regex patterns for detecting Python requirements / dependencies .txt manifest variants.
+        # Used to filter out unrelated .txt files (e.g. README-style notes, tool output, etc.) from being
+        # treated as pip manifests.
+
+        # Matches "requirements" preceded by a hyphen, period, underscore, start-of-string, or slash,
+        # followed by non-whitespace chars and ".txt".
+        # Examples: requirements.txt, requirements.prod.txt, requirements/production.txt
+        REQUIREMENTS_TXT_REGEX = T.let(%r{(?:[-._]|^|/)requirements[^\s]*\.txt$}i, Regexp)
+
+        # More lenient: matches "require" at the start of the filename or after a hyphen/period/underscore/
+        # slash delimiter, with an optional hyphen/underscore/slash suffix. The leading delimiter prevents
+        # matching "require" as a substring of another word (e.g. "prequire.txt", "acquire.txt").
+        # Examples: require.txt, require-test.txt, py3-require.txt, pyenv_require_e2e.txt
+        REQUIRE_TXT_REGEX = T.let(%r{(?:[-._]|^|/)require(?:[-_/][^\s.]*)?\.txt$}i, Regexp)
+
+        # Matches "dependencies" / "dependency" preceded by a hyphen, period, underscore,
+        # start-of-string, or slash, followed by non-whitespace chars and ".txt".
+        # Examples: dependencies.txt, my-dependencies.txt, dependencies/python/ansible-lint.txt
+        DEPENDENCIES_TXT_REGEX = T.let(%r{(?:[-._]|^|/)dependenc(?:y|ies)[^\s]*\.txt$}i, Regexp)
+
+        # More lenient: matches "depend" / "depends" at the start of the filename or after a hyphen/period/
+        # underscore/slash delimiter, with an optional hyphen/underscore/slash suffix. The leading delimiter
+        # prevents matching "depend" as a substring of another word (e.g. "independ.txt").
+        # Examples: depend.txt, depends.txt, depend-test.txt, py3-depends.txt
+        DEPEND_TXT_REGEX = T.let(%r{(?:[-._]|^|/)depend(?:s)?(?:[-_/][^\s.]*)?\.txt$}i, Regexp)
+
+        # Whether a .txt filename looks like a real pip manifest rather than an unrelated .txt file.
+        sig { params(path: String).returns(T::Boolean) }
+        def self.manifest_txt_filename?(path)
+          path.match?(REQUIREMENTS_TXT_REGEX) ||
+            path.match?(REQUIRE_TXT_REGEX) ||
+            path.match?(DEPENDENCIES_TXT_REGEX) ||
+            path.match?(DEPEND_TXT_REGEX)
+        end
+
+        # Resolves the sibling paths a requirements file references via `-r`/`-c`, returning repo-relative
+        # cleanpaths (matching fetched DependencyFile names). Single home for reference resolution, shared by the
+        # grapher's bystander filter and the layering group builder so reference-syntax handling never drifts.
+        #
+        # Reuses SharedFileFetcher's reference regexes so callers keep exactly the children the file fetcher pulled in.
+        sig { params(file: Dependabot::DependencyFile).returns(T::Array[String]) }
+        def self.referenced_paths(file)
+          content = file.content
+          return [] if content.nil?
+
+          current_dir = File.dirname(file.name)
+          referenced =
+            content.scan(Dependabot::Python::SharedFileFetcher::CHILD_REQUIREMENT_REGEX).flatten +
+            content.scan(Dependabot::Python::SharedFileFetcher::CONSTRAINT_REGEX).flatten
+
+          referenced.map do |path|
+            resolved = current_dir == "." ? path : File.join(current_dir, path)
+            Pathname.new(resolved).cleanpath.to_path
+          end
+        end
+
+        sig { params(dependency_files: T::Array[Dependabot::DependencyFile]).void }
+        def initialize(dependency_files:)
+          @dependency_files = dependency_files
+        end
+
+        # Builds one manifest group per requirements layer. A layer's primary is its compiled `.txt` (the
+        # most specific file, akin to a lockfile) when present, otherwise its `.in`. Each group also includes
+        # the paired `.in`/`.txt`, any constraints files, and any sibling files referenced via `-r`/`-c` so
+        # the parser can resolve the layer in isolation.
+        sig { returns(T::Array[Dependabot::DependencyGraphers::ManifestGroup]) }
+        def groups
+          layer_primaries.map do |primary|
+            Dependabot::DependencyGraphers::ManifestGroup.new(
+              primary: primary,
+              files: files_for_layer(primary)
+            )
+          end
+        end
+
+        private
+
+        sig { returns(T::Array[Dependabot::DependencyFile]) }
+        attr_reader :dependency_files
+
+        # The set of files that act as the owning manifest for a layer: every requirements `.txt` that looks
+        # like a real manifest, plus any `.in` file that has no compiled `.txt` counterpart.
+        sig { returns(T::Array[Dependabot::DependencyFile]) }
+        def layer_primaries
+          txt_primaries = requirement_family_files.select do |f|
+            f.name.end_with?(".txt") && self.class.manifest_txt_filename?(f.name)
+          end
+          compiled_stems = txt_primaries.map { |f| requirements_stem(f.name) }
+
+          in_primaries = requirement_family_files.select do |f|
+            f.name.end_with?(".in") && !compiled_stems.include?(requirements_stem(f.name))
+          end
+
+          txt_primaries + in_primaries
+        end
+
+        # All the files a single layer needs to parse: the primary, its paired `.in`/`.txt`, constraints
+        # files and any `-r`/`-c` referenced siblings (added as support files so they never win attribution).
+        sig { params(primary: Dependabot::DependencyFile).returns(T::Array[Dependabot::DependencyFile]) }
+        def files_for_layer(primary)
+          stem = requirements_stem(primary.name)
+
+          paired = requirement_family_files.select do |f|
+            f != primary && requirements_stem(f.name) == stem
+          end
+
+          referenced = referenced_requirement_files(primary)
+
+          support = (paired + referenced + constraints_files).uniq.map do |f|
+            as_support_file(f)
+          end
+
+          [primary] + support
+        end
+
+        sig { returns(T::Array[Dependabot::DependencyFile]) }
+        def requirement_family_files
+          @requirement_family_files ||= T.let(
+            dependency_files.select { |f| f.name.end_with?(".txt", ".in") },
+            T.nilable(T::Array[Dependabot::DependencyFile])
+          )
+        end
+
+        sig { returns(T::Array[Dependabot::DependencyFile]) }
+        def constraints_files
+          requirement_family_files.select { |f| File.basename(f.name).include?("constraint") }
+        end
+
+        # Finds sibling files referenced by a requirements file via `-r`/`-c`, following the chain transitively.
+        #
+        # A referenced file can itself reference further files (e.g. `develop.in -r test.in`, `test.in -r base.in`),
+        # so we walk the whole closure to gather every sibling the layer needs on disk to parse in isolation.
+        sig { params(file: Dependabot::DependencyFile).returns(T::Array[Dependabot::DependencyFile]) }
+        def referenced_requirement_files(file)
+          by_name = requirement_family_files.to_h { |f| [f.name, f] }
+          seen = T.let(Set.new, T::Set[String])
+          queue = [file]
+          collected = T.let([], T::Array[Dependabot::DependencyFile])
+
+          until queue.empty?
+            current = T.must(queue.shift)
+            self.class.referenced_paths(current).each do |path|
+              next if seen.include?(path)
+
+              seen << path
+              referenced = by_name[path]
+              next if referenced.nil?
+
+              collected << referenced
+              queue << referenced
+            end
+          end
+
+          collected
+        end
+
+        # Returns a support-file copy of the given file so it can be parsed for cross-reference resolution
+        # without becoming an attribution target. Never mutates the shared DependencyFile.
+        sig { params(file: Dependabot::DependencyFile).returns(Dependabot::DependencyFile) }
+        def as_support_file(file)
+          return file if file.support_file?
+
+          Dependabot::DependencyFile.new(
+            name: file.name,
+            content: file.content,
+            directory: file.directory,
+            type: file.type,
+            support_file: true,
+            vendored_file: file.vendored_file,
+            symlink_target: file.symlink_target,
+            content_encoding: file.content_encoding,
+            deleted: file.deleted?,
+            operation: file.operation,
+            mode: file.mode
+          )
+        end
+
+        # The stem shared by a pip-compile `.in` and its compiled `.txt` (e.g. "base-requirements").
+        sig { params(name: String).returns(String) }
+        def requirements_stem(name)
+          name.sub(/\.(txt|in)\z/, "")
+        end
+      end
+    end
+  end
+end

data/lib/dependabot/python/dependency_grapher.rb

@@ -3,6 +3,7 @@
 
 require "json"
 require "sorbet-runtime"
+require "pathname"
 
 require "dependabot/dependency_graphers"
 require "dependabot/dependency_graphers/base"
@@ -11,36 +12,14 @@ require "dependabot/python/language_version_manager"
 require "dependabot/python/name_normaliser"
 require "dependabot/python/pip_compile_file_matcher"
 require "dependabot/python/pipenv_runner"
+require "dependabot/python/shared_file_fetcher"
 require "toml-rb"
 
 module Dependabot
   module Python
     class DependencyGrapher < Dependabot::DependencyGraphers::Base
       require_relative "dependency_grapher/lockfile_generator"
-
-      # Regex patterns for detecting Python requirements / dependencies .txt manifest variants.
-      # Used by the dependency grapher to filter out unrelated .txt files (e.g. README-style notes,
-      # tool output, etc.) from being treated as pip manifests.
-
-      # Matches "requirements" preceded by a hyphen, period, underscore, start-of-string, or slash,
-      # followed by non-whitespace chars and ".txt".
-      # Examples: requirements.txt, requirements.prod.txt, requirements/production.txt
-      REQUIREMENTS_TXT_REGEX = T.let(%r{(?:[-._]|^|/)requirements[^\s]*\.txt$}i, Regexp)
-
-      # More lenient: matches "require" with optional prefix (no dots/whitespace)
-      # and optional hyphen/underscore/slash suffix. Does not match "require" as a substring.
-      # Examples: require.txt, require-test.txt, py3-require.txt, pyenv_require_e2e.txt
-      REQUIRE_TXT_REGEX = T.let(%r{[^\s|.]*require(?:[-_/][^\s|.]*)?\.txt$}i, Regexp)
-
-      # Matches "dependencies" / "dependency" preceded by a hyphen, period, underscore,
-      # start-of-string, or slash, followed by non-whitespace chars and ".txt".
-      # Examples: dependencies.txt, my-dependencies.txt, dependencies/python/ansible-lint.txt
-      DEPENDENCIES_TXT_REGEX = T.let(%r{(?:[-._]|^|/)dependenc(?:y|ies)[^\s]*\.txt$}i, Regexp)
-
-      # More lenient: matches "depend" / "depends" with optional prefix (no dots/whitespace)
-      # and optional hyphen/underscore/slash suffix. Does not match "depend" as a substring.
-      # Examples: depend.txt, depends.txt, depend-test.txt, py3-depends.txt
-      DEPEND_TXT_REGEX = T.let(%r{[^\s|.]*depend(?:s)?(?:[-_/][^\s|.]*)?\.txt$}i, Regexp)
+      require_relative "dependency_grapher/requirements_layers"
 
       sig { override.returns(Dependabot::DependencyFile) }
       def relevant_dependency_file
@@ -59,11 +38,22 @@ module Dependabot
         relevant_file = candidates.compact.first
         return relevant_file if relevant_file
 
+        # If we do not have any dependencies to report, the absence of a relevant manifest file is tolerable, it
+        # means the file fetcher retrieved bystander `txt` files we have filtered out and the correct outcome
+        # is a blank snapshot for this path.
+        return empty_manifest_file if resolved_dependencies.empty?
+
+        # If dependencies resolved but no owning manifest could be identified, we are in an inconsistent state
+        # that we cannot represent.
         raise DependabotError, "No supported dependency file present."
       end
 
       sig { override.void }
       def prepare!
+        # Exclude .txt files that don't look like pip manifests before we parse anything to avoid risk of bystander
+        # files in non-pip projects.
+        filter_non_manifest_txt_files!
+
         if poetry_project_without_lockfile?
           # Generating an ephemeral lockfile requires executing `poetry lock`. Strictly speaking, that violates our
           # policy of refusing to run Python tooling when external code execution is disallowed, so fail fast.
@@ -76,8 +66,126 @@ module Dependabot
         super
       end
 
+      # Layering is specific to pip / pip-compile for Python.
+      sig { override.returns(T::Array[Dependabot::DependencyGraphers::ManifestGroup]) }
+      def manifest_groups
+        return super unless supports_layering?
+
+        groups = RequirementsLayers.new(dependency_files: dependency_files).groups
+        # If we try to apply grouping, but find there is only one group, we prefer
+        # to fallback to the base method (the whole directory parsed as one manifest,
+        # which naturally includes any setup.py/setup.cfg/pyproject.toml present).
+        return super if groups.length < 2
+
+        groups + non_requirements_manifest_groups
+      end
+
       private
 
+      # When a pip/pip-compile directory is split into requirements layers, any non-requirements manifest
+      # sharing that directory (setup.py, setup.cfg, pyproject.toml) would otherwise fall outside every layer
+      # group and have its dependencies dropped.
+      #
+      # We instead emit each as its own self-attributed group so its dependencies are preserved and attributed
+      # to the file itself. This matches existing static analysis behaviour.
+      #
+      # NOTE:
+      # This logic is only applied on the pip/pip-compile path, so poetry.lock/Pipfile.lock are absent as they would
+      # select the poetry/pipenv path which does not support layering.
+      #
+      # If a pyproject.toml is present, it is treated as a pip-context manifest.
+      sig { returns(T::Array[Dependabot::DependencyGraphers::ManifestGroup]) }
+      def non_requirements_manifest_groups
+        [setup_file, setup_cfg_file, pyproject_toml].compact.map do |file|
+          Dependabot::DependencyGraphers::ManifestGroup.new(primary: file, files: [file])
+        end
+      end
+
+      # An empty, nameless dependency file used to represent a directory that has no supported manifest and
+      # resolves to no dependencies. The submission layer treats a nameless manifest as "nothing to report",
+      # producing a valid, empty snapshot for the directory instead of a failure.
+      sig { returns(Dependabot::DependencyFile) }
+      def empty_manifest_file
+        Dependabot::DependencyFile.new(
+          name: "",
+          content: "",
+          directory: file_parser.source&.directory || "/"
+        )
+      end
+
+      # Non-.txt files (pyproject.toml, setup.py, Pipfile, lockfiles, .in files, etc.) are always retained.
+      # A .txt file is kept when its name looks like a pip requirements/dependencies manifest, the pip-compile
+      # matcher recognises it as a compiled lockfile, or it is referenced (transitively) via `-r`/`-c` from a
+      # retained requirements file. The last case preserves constraint/child files (e.g. `constraints.txt`) that
+      # the parser needs on disk to resolve a real manifest, while still dropping bystander `.txt` files.
+      sig { void }
+      def filter_non_manifest_txt_files!
+        keep = txt_files_to_keep
+
+        file_parser.dependency_files.reject! do |file|
+          next false unless file.name.end_with?(".txt")
+
+          !keep.include?(file.name)
+        end
+      end
+
+      # Names of the `.txt` files that must be retained: those that look like manifests or pip-compile lockfiles,
+      # plus any `.txt` reachable via `-r`/`-c` references from a retained requirements file (`.in` files are
+      # never dropped, so references originating from them are followed too).
+      sig { returns(T::Set[String]) }
+      def txt_files_to_keep
+        files_by_name = dependency_files.to_h { |file| [file.name, file] }
+
+        seeds = dependency_files.select do |file|
+          file.name.end_with?(".txt") &&
+            (RequirementsLayers.manifest_txt_filename?(file.name) ||
+              pip_compile_file_matcher.lockfile_for_pip_compile_file?(file))
+        end
+
+        # `.in` files are always retained, so a `.txt` they reference must be kept too.
+        roots = seeds + dependency_files.select { |file| file.name.end_with?(".in") }
+
+        reachable_txt_names(roots, files_by_name, Set.new(seeds.map(&:name)))
+      end
+
+      # Breadth-first closure over `-r`/`-c` references starting from `roots`, adding every referenced `.txt`
+      # file that exists in `files_by_name` to `keep`.
+      sig do
+        params(
+          roots: T::Array[Dependabot::DependencyFile],
+          files_by_name: T::Hash[String, Dependabot::DependencyFile],
+          keep: T::Set[String]
+        ).returns(T::Set[String])
+      end
+      def reachable_txt_names(roots, files_by_name, keep)
+        queue = roots.dup
+        until queue.empty?
+          file = T.must(queue.shift)
+          referenced_txt_names(file).each do |name|
+            referenced_file = files_by_name[name]
+            next if referenced_file.nil? || keep.include?(name)
+
+            keep << name
+            queue << referenced_file
+          end
+        end
+
+        keep
+      end
+
+      # Resolves the `.txt` files referenced from a requirements file via `-r`/`-c`, returning their names
+      # relative to the repo (matching the fetched DependencyFile names) so we can retain them. Delegates to
+      # RequirementsLayers.referenced_paths so grouping and the bystander filter resolve references identically.
+      sig { params(file: Dependabot::DependencyFile).returns(T::Array[String]) }
+      def referenced_txt_names(file)
+        RequirementsLayers.referenced_paths(file).select { |path| path.end_with?(".txt") }
+      end
+
+      sig { returns(T::Boolean) }
+      def supports_layering?
+        [PipPackageManager::NAME, PipCompilePackageManager::NAME].include?(python_package_manager)
+      end
+
       # Returns the poetry.lock only if it was committed to the repo,
       # not if it was generated ephemerally. This ensures that
       # relevant_dependency_file reports the real manifest (pyproject.toml).
@@ -380,19 +488,11 @@ module Dependabot
 
         @pip_requirements_file = T.let(
           dependency_files.find { |f| f.name == "requirements.txt" } ||
-            dependency_files.find { |f| f.name.end_with?(".txt") && python_manifest_txt_filename?(f.name) },
+            dependency_files.find { |f| f.name.end_with?(".txt") && RequirementsLayers.manifest_txt_filename?(f.name) },
           T.nilable(Dependabot::DependencyFile)
         )
       end
 
-      sig { params(path: String).returns(T::Boolean) }
-      def python_manifest_txt_filename?(path)
-        path.match?(REQUIREMENTS_TXT_REGEX) ||
-          path.match?(REQUIRE_TXT_REGEX) ||
-          path.match?(DEPENDENCIES_TXT_REGEX) ||
-          path.match?(DEPEND_TXT_REGEX)
-      end
-
       sig { params(filename: String).returns(T.nilable(Dependabot::DependencyFile)) }
       def dependency_file(filename)
         dependency_files.find { |file| file.name == filename }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dependabot-python
 version: !ruby/object:Gem::Version
-  version: 0.385.0
+  version: 0.386.0
 platform: ruby
 authors:
 - Dependabot
@@ -15,14 +15,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.385.0
+        version: 0.386.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.385.0
+        version: 0.386.0
 - !ruby/object:Gem::Dependency
   name: debug
   requirement: !ruby/object:Gem::Requirement
@@ -277,6 +277,7 @@ files:
 - lib/dependabot/python/authed_url_builder.rb
 - lib/dependabot/python/dependency_grapher.rb
 - lib/dependabot/python/dependency_grapher/lockfile_generator.rb
+- lib/dependabot/python/dependency_grapher/requirements_layers.rb
 - lib/dependabot/python/file_fetcher.rb
 - lib/dependabot/python/file_parser.rb
 - lib/dependabot/python/file_parser/pipfile_files_parser.rb
@@ -322,7 +323,7 @@ licenses:
 - MIT
 metadata:
   bug_tracker_uri: https://github.com/dependabot/dependabot-core/issues
-  changelog_uri: https://github.com/dependabot/dependabot-core/releases/tag/v0.385.0
+  changelog_uri: https://github.com/dependabot/dependabot-core/releases/tag/v0.386.0
 rdoc_options: []
 require_paths:
 - lib