dependabot-core 0.76.1

data/lib/dependabot/file_parsers/python/pip/setup_file_parser.rb

@@ -0,0 +1,160 @@
+# frozen_string_literal: true
+
+require "dependabot/dependency"
+require "dependabot/errors"
+require "dependabot/file_parsers/base/dependency_set"
+require "dependabot/file_parsers/python/pip"
+require "dependabot/shared_helpers"
+
+module Dependabot
+  module FileParsers
+    module Python
+      class Pip
+        class SetupFileParser
+          INSTALL_REQUIRES_REGEX =
+            /install_requires\s*=\s*(\[.*?\])[,)\s]/m.freeze
+          SETUP_REQUIRES_REGEX = /setup_requires\s*=\s*(\[.*?\])[,)\s]/m.freeze
+          TESTS_REQUIRE_REGEX = /tests_require\s*=\s*(\[.*?\])[,)\s]/m.freeze
+          EXTRAS_REQUIRE_REGEX = /extras_require\s*=\s*(\{.*?\})[,)\s]/m.freeze
+
+          def initialize(dependency_files:)
+            @dependency_files = dependency_files
+          end
+
+          def dependency_set
+            dependencies = Dependabot::FileParsers::Base::DependencySet.new
+
+            parsed_setup_file.each do |dep|
+              # If a requirement has a `<` or `<=` marker then updating it is
+              # probably blocked. Ignore it.
+              next if dep["markers"].include?("<")
+
+              dependencies <<
+                Dependency.new(
+                  name: normalised_name(dep["name"]),
+                  version: dep["version"]&.include?("*") ? nil : dep["version"],
+                  requirements: [{
+                    requirement: dep["requirement"],
+                    file: Pathname.new(dep["file"]).cleanpath.to_path,
+                    source: nil,
+                    groups: [dep["requirement_type"]]
+                  }],
+                  package_manager: "pip"
+                )
+            end
+            dependencies
+          end
+
+          private
+
+          attr_reader :dependency_files
+
+          def parsed_setup_file
+            SharedHelpers.in_a_temporary_directory do
+              write_temporary_dependency_files
+
+              requirements = SharedHelpers.run_helper_subprocess(
+                command: "pyenv exec python #{python_helper_path}",
+                function: "parse_setup",
+                args: [Dir.pwd]
+              )
+
+              check_requirements(requirements)
+              requirements
+            end
+          rescue SharedHelpers::HelperSubprocessFailed => error
+            if error.message.start_with?("InstallationError")
+              raise Dependabot::DependencyFileNotEvaluatable, error.message
+            end
+
+            parsed_sanitized_setup_file
+          end
+
+          def parsed_sanitized_setup_file
+            SharedHelpers.in_a_temporary_directory do
+              write_sanitized_setup_file
+
+              requirements = SharedHelpers.run_helper_subprocess(
+                command: "pyenv exec python #{python_helper_path}",
+                function: "parse_setup",
+                args: [Dir.pwd]
+              )
+
+              check_requirements(requirements)
+              requirements
+            end
+          rescue SharedHelpers::HelperSubprocessFailed
+            # Assume there are no dependencies in setup.py files that fail to
+            # parse. This isn't ideal, and we should continue to improve
+            # parsing, but there are a *lot* of things that can go wrong at
+            # the moment!
+            []
+          end
+
+          def check_requirements(requirements)
+            requirements.each do |dep|
+              next unless dep["requirement"]
+
+              Utils::Python::Requirement.new(dep["requirement"].split(","))
+            rescue Gem::Requirement::BadRequirementError => error
+              raise Dependabot::DependencyFileNotEvaluatable, error.message
+            end
+          end
+
+          def write_temporary_dependency_files
+            dependency_files.
+              reject { |f| f.name == ".python-version" }.
+              each do |file|
+                path = file.name
+                FileUtils.mkdir_p(Pathname.new(path).dirname)
+                File.write(path, file.content)
+              end
+          end
+
+          # Write a setup.py with only entries for the requires fields.
+          #
+          # This sanitization is far from perfect (it will fail if any of the
+          # entries are dynamic), but it is an alternative approach to the one
+          # used in parser.py which sometimes succeeds when that has failed.
+          def write_sanitized_setup_file
+            original_content = setup_file.content
+
+            install_requires =
+              original_content.match(INSTALL_REQUIRES_REGEX)&.captures&.first
+            setup_requires =
+              original_content.match(SETUP_REQUIRES_REGEX)&.captures&.first
+            tests_require =
+              original_content.match(TESTS_REQUIRE_REGEX)&.captures&.first
+            extras_require =
+              original_content.match(EXTRAS_REQUIRE_REGEX)&.captures&.first
+
+            tmp = "from setuptools import setup\n\n"\
+                  "setup(name=\"sanitized-package\",version=\"0.0.1\","
+
+            tmp += "install_requires=#{install_requires}," if install_requires
+            tmp += "setup_requires=#{setup_requires}," if setup_requires
+            tmp += "tests_require=#{tests_require}," if tests_require
+            tmp += "extras_require=#{extras_require}," if extras_require
+            tmp += ")"
+
+            File.write("setup.py", tmp)
+          end
+
+          def python_helper_path
+            project_root = File.join(File.dirname(__FILE__), "../../../../..")
+            File.join(project_root, "helpers/python/run.py")
+          end
+
+          # See https://www.python.org/dev/peps/pep-0503/#normalized-names
+          def normalised_name(name)
+            name.downcase.gsub(/[-_.]+/, "-")
+          end
+
+          def setup_file
+            dependency_files.find { |f| f.name == "setup.py" }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dependabot/file_parsers/ruby/bundler.rb

@@ -0,0 +1,295 @@
+# frozen_string_literal: true
+
+require "dependabot/dependency"
+require "dependabot/file_parsers/base"
+require "dependabot/file_updaters/ruby/bundler/lockfile_updater"
+require "dependabot/shared_helpers"
+require "dependabot/errors"
+
+module Dependabot
+  module FileParsers
+    module Ruby
+      class Bundler < Dependabot::FileParsers::Base
+        require "dependabot/file_parsers/base/dependency_set"
+        require "dependabot/file_parsers/ruby/bundler/file_preparer"
+        require "dependabot/file_parsers/ruby/bundler/gemfile_checker"
+
+        def parse
+          dependency_set = DependencySet.new
+          dependency_set += gemfile_dependencies
+          dependency_set += gemspec_dependencies
+          dependency_set += lockfile_dependencies
+          dependency_set.dependencies
+        end
+
+        private
+
+        # Can't be a constant because some of these don't exist in bundler
+        # 1.15, which Heroku uses, which causes an exception on boot.
+        def sources
+          [
+            NilClass,
+            ::Bundler::Source::Rubygems,
+            ::Bundler::Source::Git,
+            ::Bundler::Source::Path,
+            ::Bundler::Source::Gemspec,
+            ::Bundler::Source::Metadata
+          ]
+        end
+
+        def gemfile_dependencies
+          dependencies = DependencySet.new
+
+          return dependencies unless gemfile
+
+          [gemfile, *evaled_gemfiles].each do |file|
+            parsed_gemfile.each do |dep|
+              next unless dependency_in_gemfile?(gemfile: file, dependency: dep)
+
+              dependencies <<
+                Dependency.new(
+                  name: dep.name,
+                  version: dependency_version(dep.name)&.to_s,
+                  requirements: [{
+                    requirement: dep.requirement.to_s,
+                    groups: dep.groups,
+                    source: source_for(dep),
+                    file: file.name
+                  }],
+                  package_manager: "bundler"
+                )
+            end
+          end
+
+          dependencies
+        end
+
+        def gemspec_dependencies
+          dependencies = DependencySet.new
+
+          gemspecs.each do |gemspec|
+            parsed_gemspec(gemspec).dependencies.each do |dependency|
+              dependencies <<
+                Dependency.new(
+                  name: dependency.name,
+                  version: dependency_version(dependency.name)&.to_s,
+                  requirements: [{
+                    requirement: dependency.requirement.to_s,
+                    groups: dependency.runtime? ? ["runtime"] : ["development"],
+                    source: nil,
+                    file: gemspec.name
+                  }],
+                  package_manager: "bundler"
+                )
+            end
+          end
+
+          dependencies
+        end
+
+        def lockfile_dependencies
+          dependencies = DependencySet.new
+
+          return dependencies unless lockfile
+
+          # Create a DependencySet where each element has no requirement. Any
+          # requirements will be added when combining the DependencySet with
+          # other DependencySets.
+          parsed_lockfile.specs.each do |dependency|
+            next if dependency.source.is_a?(::Bundler::Source::Path)
+
+            dependencies <<
+              Dependency.new(
+                name: dependency.name,
+                version: dependency_version(dependency.name)&.to_s,
+                requirements: [],
+                package_manager: "bundler"
+              )
+          end
+
+          dependencies
+        end
+
+        def parsed_gemfile
+          base_directory = dependency_files.first.directory
+          @parsed_gemfile ||=
+            SharedHelpers.in_a_temporary_directory(base_directory) do
+              write_temporary_dependency_files
+
+              SharedHelpers.in_a_forked_process do
+                ::Bundler.instance_variable_set(:@root, Pathname.new(Dir.pwd))
+
+                ::Bundler::Definition.build(gemfile.name, nil, {}).
+                  dependencies.
+                  select(&:current_platform?).
+                  # We can't dump gemspec sources, and we wouldn't bump them
+                  # anyway, so we filter them out.
+                  reject { |dep| dep.source.is_a?(::Bundler::Source::Gemspec) }
+              end
+            end
+        rescue SharedHelpers::ChildProcessFailed => error
+          msg = error.error_class + " with message: " +
+                error.error_message.force_encoding("UTF-8").encode
+          raise Dependabot::DependencyFileNotEvaluatable, msg
+        end
+
+        def parsed_gemspec(file)
+          @parsed_gemspecs ||= {}
+          @parsed_gemspecs[file.name] ||=
+            SharedHelpers.in_a_temporary_directory do
+              File.write(file.name, file.content)
+              imported_ruby_files.each do |f|
+                path = f.name
+                FileUtils.mkdir_p(Pathname.new(path).dirname)
+                File.write(path, f.content)
+              end
+
+              SharedHelpers.in_a_forked_process do
+                ::Bundler.instance_variable_set(:@root, Pathname.new(Dir.pwd))
+                ::Bundler.load_gemspec_uncached(file.name)
+              end
+            end
+        rescue SharedHelpers::ChildProcessFailed => error
+          msg = error.error_class + " with message: " + error.error_message
+          raise Dependabot::DependencyFileNotEvaluatable, msg
+        end
+
+        def prepared_dependency_files
+          @prepared_dependency_files ||=
+            FilePreparer.new(dependency_files: dependency_files).
+            prepared_dependency_files
+        end
+
+        def write_temporary_dependency_files
+          prepared_dependency_files.each do |file|
+            path = file.name
+            FileUtils.mkdir_p(Pathname.new(path).dirname)
+            File.write(path, file.content)
+          end
+        end
+
+        def check_required_files
+          file_names = dependency_files.map(&:name)
+
+          return if file_names.any? do |name|
+            name.end_with?(".gemspec") && !name.include?("/")
+          end
+
+          return if gemfile
+
+          raise "A gemspec or Gemfile must be provided!"
+        end
+
+        def source_for(dependency)
+          source = dependency.source
+          if lockfile && default_rubygems?(source)
+            # If there's a lockfile and the Gemfile doesn't have anything
+            # interesting to say about the source, check that.
+            source = source_from_lockfile(dependency.name)
+          end
+          raise "Bad source: #{source}" unless sources.include?(source.class)
+
+          return nil if default_rubygems?(source)
+
+          details = { type: source.class.name.split("::").last.downcase }
+          if source.is_a?(::Bundler::Source::Git)
+            details.merge!(git_source_details(source))
+          end
+          if source.is_a?(::Bundler::Source::Rubygems)
+            details[:url] = source.remotes.first.to_s
+          end
+          details
+        end
+
+        def git_source_details(source)
+          {
+            url: source.uri,
+            branch: source.branch || "master",
+            ref: source.ref
+          }
+        end
+
+        def default_rubygems?(source)
+          return true if source.nil?
+          return false unless source.is_a?(::Bundler::Source::Rubygems)
+
+          source.remotes.any? { |r| r.to_s.include?("rubygems.org") }
+        end
+
+        def dependency_version(dependency_name)
+          return unless lockfile
+
+          spec = parsed_lockfile.specs.find { |s| s.name == dependency_name }
+
+          # Not all files in the Gemfile will appear in the Gemfile.lock. For
+          # instance, if a gem specifies `platform: [:windows]`, and the
+          # Gemfile.lock is generated on a Linux machine, the gem will be not
+          # appear in the lockfile.
+          return unless spec
+
+          # If the source is Git we're better off knowing the SHA-1 than the
+          # version.
+          if spec.source.instance_of?(::Bundler::Source::Git)
+            return spec.source.revision
+          end
+
+          spec.version
+        end
+
+        def source_from_lockfile(dependency_name)
+          parsed_lockfile.specs.find { |s| s.name == dependency_name }&.source
+        end
+
+        def dependency_in_gemfile?(gemfile:, dependency:)
+          GemfileChecker.new(
+            dependency: dependency,
+            gemfile: gemfile
+          ).includes_dependency?
+        end
+
+        def gemfile
+          @gemfile ||= get_original_file("Gemfile") ||
+                       get_original_file("gems.rb")
+        end
+
+        def evaled_gemfiles
+          dependency_files.
+            reject { |f| f.name.end_with?(".gemspec") }.
+            reject { |f| f.name.end_with?(".lock") }.
+            reject { |f| f.name.end_with?(".ruby-version") }.
+            reject { |f| f.name == "Gemfile" }.
+            reject { |f| f.name == "gems.rb" }.
+            reject { |f| f.name == "gems.locked" }
+        end
+
+        def lockfile
+          @lockfile ||= get_original_file("Gemfile.lock") ||
+                        get_original_file("gems.locked")
+        end
+
+        def parsed_lockfile
+          @parsed_lockfile ||=
+            ::Bundler::LockfileParser.new(sanitized_lockfile_content)
+        end
+
+        def sanitized_lockfile_content
+          regex = FileUpdaters::Ruby::Bundler::LockfileUpdater::LOCKFILE_ENDING
+          lockfile.content.gsub(regex, "")
+        end
+
+        def gemspecs
+          # The gemspecs for this project will be at the top level
+          @gemspecs ||= prepared_dependency_files.select do |file|
+            file.name.match?(%r{^[^/]*\.gemspec$})
+          end
+        end
+
+        def imported_ruby_files
+          dependency_files.
+            select { |f| f.name.end_with?(".rb") }.
+            reject { |f| f.name == "gems.rb" }
+        end
+      end
+    end
+  end
+end