dependabot-npm_and_yarn 0.213.0 → 0.215.0

data/lib/dependabot/npm_and_yarn/helpers.rb

@@ -16,29 +16,82 @@ module Dependabot
         6
       end
 
-      # Run any number of yarn commands while ensuring that `enableScripts` is
-      # set to false. Yarn commands should _not_ be ran outside of this helper
-      # to ensure that postinstall scripts are never executed, as they could
-      # contain malicious code.
-      def self.run_yarn_commands(*commands)
-        # We never want to execute postinstall scripts
-        SharedHelpers.run_shell_command("yarn config set enableScripts false")
+      def self.fetch_yarnrc_yml_value(key, default_value)
+        if File.exist?(".yarnrc.yml") && (yarnrc = YAML.load_file(".yarnrc.yml"))
+          yarnrc.fetch(key, default_value)
+        else
+          default_value
+        end
+      end
+
+      def self.yarn_berry?(yarn_lock)
+        yaml = YAML.safe_load(yarn_lock.content)
+        yaml.key?("__metadata")
+      rescue StandardError
+        false
+      end
+
+      def self.yarn_major_version
+        output = SharedHelpers.run_shell_command("yarn --version")
+        Version.new(output).major
+      end
+
+      def self.yarn_zero_install?
+        File.exist?(".pnp.cjs")
+      end
+
+      def self.yarn_offline_cache?
+        yarn_cache_dir = fetch_yarnrc_yml_value("cacheFolder", ".yarn/cache")
+        File.exist?(yarn_cache_dir) && (fetch_yarnrc_yml_value("nodeLinker", "") == "node-modules")
+      end
+
+      def self.yarn_berry_args
+        if yarn_major_version == 2
+          ""
+        elsif yarn_major_version >= 3 && (yarn_zero_install? || yarn_offline_cache?)
+          "--mode=skip-build"
+        else
+          # We only want this mode if the cache is not being updated/managed
+          # as this improperly leaves old versions in the cache
+          "--mode=update-lockfile"
+        end
+      end
+
+      def self.setup_yarn_berry
+        # Always disable immutable installs so yarn's CI detection doesn't prevent updates.
+        SharedHelpers.run_shell_command("yarn config set enableImmutableInstalls false")
+        # We never want to execute postinstall scripts, either set this config or mode=skip-build must be set
+        if yarn_major_version == 2 || !yarn_zero_install?
+          SharedHelpers.run_shell_command("yarn config set enableScripts false")
+        end
         if (http_proxy = ENV.fetch("HTTP_PROXY", false))
           SharedHelpers.run_shell_command("yarn config set httpProxy #{http_proxy}")
         end
         if (https_proxy = ENV.fetch("HTTPS_PROXY", false))
           SharedHelpers.run_shell_command("yarn config set httpsProxy #{https_proxy}")
         end
-        if (ca_file_path = ENV.fetch("NODE_EXTRA_CA_CERTS", false))
-          output = SharedHelpers.run_shell_command("yarn --version")
-          major_version = Version.new(output).major
-          if major_version >= 4
-            SharedHelpers.run_shell_command("yarn config set httpsCaFilePath #{ca_file_path}")
-          else
-            SharedHelpers.run_shell_command("yarn config set caFilePath #{ca_file_path}")
-          end
+        return unless (ca_file_path = ENV.fetch("NODE_EXTRA_CA_CERTS", false))
+
+        if yarn_major_version >= 4
+          SharedHelpers.run_shell_command("yarn config set httpsCaFilePath #{ca_file_path}")
+        else
+          SharedHelpers.run_shell_command("yarn config set caFilePath #{ca_file_path}")
         end
-        commands.each { |cmd| SharedHelpers.run_shell_command(cmd) }
+      end
+
+      # Run any number of yarn commands while ensuring that `enableScripts` is
+      # set to false. Yarn commands should _not_ be ran outside of this helper
+      # to ensure that postinstall scripts are never executed, as they could
+      # contain malicious code.
+      def self.run_yarn_commands(*commands)
+        setup_yarn_berry
+        commands.each { |cmd, fingerprint| SharedHelpers.run_shell_command(cmd, fingerprint: fingerprint) }
+      end
+
+      # Run a single yarn command returning stdout/stderr
+      def self.run_yarn_command(command, fingerprint: nil)
+        setup_yarn_berry
+        SharedHelpers.run_shell_command(command, fingerprint: fingerprint)
       end
 
       def self.dependencies_with_all_versions_metadata(dependency_set)

data/lib/dependabot/npm_and_yarn/native_helpers.rb

@@ -14,14 +14,14 @@ module Dependabot
         File.join(__dir__, "../../../helpers")
       end
 
-      def self.npm8_subdependency_update_command(dependency_names)
+      def self.run_npm8_subdependency_update_command(dependency_names)
         # NOTE: npm options
         # - `--force` ignores checks for platform (os, cpu) and engines
         # - `--dry-run=false` the updater sets a global .npmrc with dry-run: true to
         #   work around an issue in npm 6, we don't want that here
         # - `--ignore-scripts` disables prepare and prepack scripts which are run
         #   when installing git dependencies
-        [
+        command = [
           "npm",
           "update",
           *dependency_names,
@@ -31,6 +31,19 @@ module Dependabot
           "--ignore-scripts",
           "--package-lock-only"
         ].join(" ")
+
+        fingerprint = [
+          "npm",
+          "update",
+          "<dependency_names>",
+          "--force",
+          "--dry-run",
+          "false",
+          "--ignore-scripts",
+          "--package-lock-only"
+        ].join(" ")
+
+        SharedHelpers.run_shell_command(command, fingerprint: fingerprint)
       end
     end
   end

data/lib/dependabot/npm_and_yarn/update_checker/dependency_files_builder.rb

@@ -16,7 +16,12 @@ module Dependabot
         def write_temporary_dependency_files
           write_lock_files
 
-          File.write(".npmrc", npmrc_content)
+          if Helpers.yarn_berry?(yarn_locks.first)
+            File.write(".yarnrc.yml", yarnrc_yml_content) if yarnrc_yml_file
+          else
+            File.write(".npmrc", npmrc_content) unless Helpers.yarn_berry?(yarn_locks.first)
+            File.write(".yarnrc", yarnrc_content) if yarnrc_specifies_private_reg?
+          end
 
           package_files.each do |file|
             path = file.name
@@ -69,6 +74,24 @@ module Dependabot
           end
         end
 
+        def yarnrc_specifies_private_reg?
+          return false unless yarnrc_file
+
+          regex = UpdateChecker::RegistryFinder::YARN_GLOBAL_REGISTRY_REGEX
+          yarnrc_global_registry =
+            yarnrc_file.content.
+            lines.find { |line| line.match?(regex) }&.
+            match(regex)&.
+            named_captures&.
+            fetch("registry")
+
+          return false unless yarnrc_global_registry
+
+          UpdateChecker::RegistryFinder::CENTRAL_REGISTRIES.none? do |r|
+            r.include?(URI(yarnrc_global_registry).host)
+          end
+        end
+
         # Duplicated in NpmLockfileUpdater
         # Remove the dependency we want to update from the lockfile and let
         # yarn find the latest resolvable version and fix the lockfile
@@ -88,6 +111,25 @@ module Dependabot
             dependency_files: dependency_files
           ).npmrc_content
         end
+
+        def yarnrc_file
+          dependency_files.find { |f| f.name == ".yarnrc" }
+        end
+
+        def yarnrc_content
+          NpmAndYarn::FileUpdater::NpmrcBuilder.new(
+            credentials: credentials,
+            dependency_files: dependency_files
+          ).yarnrc_content
+        end
+
+        def yarnrc_yml_file
+          dependency_files.find { |f| f.name.end_with?(".yarnrc.yml") }
+        end
+
+        def yarnrc_yml_content
+          yarnrc_yml_file.content
+        end
       end
     end
   end

data/lib/dependabot/npm_and_yarn/update_checker/latest_version_finder.rb

@@ -130,10 +130,10 @@ module Dependabot
         end
 
         def filter_lower_versions(versions_array)
-          return versions_array unless dependency.version && version_class.correct?(dependency.version)
+          return versions_array unless dependency.numeric_version
 
           versions_array.
-            select { |version, _| version > version_class.new(dependency.version) }
+            select { |version, _| version > dependency.numeric_version }
         end
 
         def version_from_dist_tags
@@ -159,13 +159,10 @@ module Dependabot
           wants_latest_dist_tag?(latest) ? latest : nil
         end
 
-        # rubocop:disable Metrics/PerceivedComplexity
         def related_to_current_pre?(version)
-          current_version = dependency.version
-          if current_version &&
-             version_class.correct?(current_version) &&
-             version_class.new(current_version).prerelease? &&
-             version_class.new(current_version).release == version.release
+          current_version = dependency.numeric_version
+          if current_version&.prerelease? &&
+             current_version&.release == version.release
             return true
           end
 
@@ -181,7 +178,6 @@ module Dependabot
             false
           end
         end
-        # rubocop:enable Metrics/PerceivedComplexity
 
         def specified_dist_tag_requirement?
           dependency.requirements.any? do |req|
@@ -204,10 +200,9 @@ module Dependabot
         end
 
         def current_version_greater_than?(version)
-          return false unless dependency.version
-          return false unless version_class.correct?(dependency.version)
+          return false unless dependency.numeric_version
 
-          version_class.new(dependency.version) > version
+          dependency.numeric_version > version
         end
 
         def current_requirement_greater_than?(version)
@@ -292,7 +287,6 @@ module Dependabot
             url: dependency_url,
             headers: registry_auth_headers
           )
-
           return response unless response.status == 500
           return response unless registry_auth_headers["Authorization"]
 

data/lib/dependabot/npm_and_yarn/update_checker/registry_finder.rb

@@ -12,6 +12,7 @@ module Dependabot
           https://registry.npmjs.org
           http://registry.npmjs.org
           https://registry.yarnpkg.com
+          http://registry.yarnpkg.com
         ).freeze
         NPM_AUTH_TOKEN_REGEX = %r{//(?<registry>.*)/:_authToken=(?<token>.*)$}
         NPM_GLOBAL_REGISTRY_REGEX = /^registry\s*=\s*['"]?(?<registry>.*?)['"]?$/
@@ -145,10 +146,13 @@ module Dependabot
           npmrc_file.content.scan(NPM_AUTH_TOKEN_REGEX) do
             next if Regexp.last_match[:registry].include?("${")
 
+            registry = Regexp.last_match[:registry]
+            token = Regexp.last_match[:token]&.strip
+
             registries << {
               "type" => "npm_registry",
-              "registry" => Regexp.last_match[:registry],
-              "token" => Regexp.last_match[:token]&.strip
+              "registry" => registry.gsub(/\s+/, "%20"),
+              "token" => token
             }
           end
 
@@ -157,7 +161,8 @@ module Dependabot
 
             registry = Regexp.last_match[:registry].strip.
                        sub(%r{/+$}, "").
-                       sub(%r{^.*?//}, "")
+                       sub(%r{^.*?//}, "").
+                       gsub(/\s+/, "%20")
             next if registries.map { |r| r["registry"] }.include?(registry)
 
             registries << {
@@ -179,7 +184,8 @@ module Dependabot
 
             registry = Regexp.last_match[:registry].strip.
                        sub(%r{/+$}, "").
-                       sub(%r{^.*?//}, "")
+                       sub(%r{^.*?//}, "").
+                       gsub(/\s+/, "%20")
             registries << {
               "type" => "npm_registry",
               "registry" => registry,

data/lib/dependabot/npm_and_yarn/update_checker/subdependency_version_resolver.rb

@@ -19,19 +19,21 @@ module Dependabot
     class UpdateChecker
       class SubdependencyVersionResolver
         def initialize(dependency:, credentials:, dependency_files:,
-                       ignored_versions:, latest_allowable_version:)
+                       ignored_versions:, latest_allowable_version:, repo_contents_path:)
           @dependency = dependency
           @credentials = credentials
           @dependency_files = dependency_files
           @ignored_versions = ignored_versions
           @latest_allowable_version = latest_allowable_version
+          @repo_contents_path = repo_contents_path
         end
 
         def latest_resolvable_version
           raise "Not a subdependency!" if dependency.requirements.any?
           return if bundled_dependency?
 
-          SharedHelpers.in_a_temporary_directory do
+          base_dir = dependency_files.first.directory
+          SharedHelpers.in_a_temporary_repo_directory(base_dir, repo_contents_path) do
             dependency_files_builder.write_temporary_dependency_files
 
             updated_lockfiles = filtered_lockfiles.map do |lockfile|
@@ -53,13 +55,13 @@ module Dependabot
         private
 
         attr_reader :dependency, :credentials, :dependency_files,
-                    :ignored_versions, :latest_allowable_version
+                    :ignored_versions, :latest_allowable_version, :repo_contents_path
 
         def update_subdependency_in_lockfile(lockfile)
           lockfile_name = Pathname.new(lockfile.name).basename.to_s
           path = Pathname.new(lockfile.name).dirname.to_s
 
-          updated_files = if lockfile.name.end_with?("yarn.lock") && yarn_berry?(lockfile)
+          updated_files = if lockfile.name.end_with?("yarn.lock") && Helpers.yarn_berry?(lockfile)
                             run_yarn_berry_updater(path, lockfile_name)
                           elsif lockfile.name.end_with?("yarn.lock")
                             run_yarn_updater(path, lockfile_name)
@@ -70,15 +72,6 @@ module Dependabot
           updated_files.fetch(lockfile_name)
         end
 
-        def yarn_berry?(yarn_lock)
-          return false unless Experiments.enabled?(:yarn_berry)
-
-          yaml = YAML.safe_load(yarn_lock.content)
-          yaml.key?("__metadata")
-        rescue StandardError
-          false
-        end
-
         def version_from_updated_lockfiles(updated_lockfiles)
           updated_files = dependency_files -
                           dependency_files_builder.yarn_locks -
@@ -123,8 +116,9 @@ module Dependabot
         def run_yarn_berry_updater(path, lockfile_name)
           SharedHelpers.with_git_configured(credentials: credentials) do
             Dir.chdir(path) do
-              Helpers.run_yarn_commands(
-                "yarn up -R #{dependency.name}"
+              Helpers.run_yarn_command(
+                "yarn up -R #{dependency.name} #{Helpers.yarn_berry_args}".strip,
+                fingerprint: "yarn up -R <dependency_name> #{Helpers.yarn_berry_args}".strip
               )
               { lockfile_name => File.read(lockfile_name) }
             end
@@ -137,7 +131,7 @@ module Dependabot
               npm_version = Dependabot::NpmAndYarn::Helpers.npm_version(lockfile_content)
 
               if npm_version == "npm8"
-                SharedHelpers.run_shell_command(NativeHelpers.npm8_subdependency_update_command([dependency.name]))
+                NativeHelpers.run_npm8_subdependency_update_command([dependency.name])
                 { lockfile_name => File.read(lockfile_name) }
               else
                 SharedHelpers.run_helper_subprocess(

data/lib/dependabot/npm_and_yarn/update_checker/version_resolver.rb

@@ -38,6 +38,15 @@ module Dependabot
             "(?<required_dep>[^"]+)"
           /x
 
+        # Error message from yarn add:
+        # YN0060: │ eve-roster@workspace:. provides jest (p8d618) \
+        # with version 29.3.0, which doesn't satisfy \
+        # what ts-jest requests\n
+        YARN_BERRY_PEER_DEP_ERROR_REGEX =
+          /
+            YN0060:\s|\s.+\sprovides\s(?<required_dep>.+?)\s\((?<info_hash>\w+)\).+what\s(?<requiring_dep>.+?)\srequests
+          /x
+
         # Error message from npm install:
         # react-dom@15.2.0 requires a peer of react@^15.2.0 \
         # but none is installed. You must install peer dependencies yourself.
@@ -62,7 +71,7 @@ module Dependabot
           /x
 
         def initialize(dependency:, credentials:, dependency_files:,
-                       latest_allowable_version:, latest_version_finder:)
+                       latest_allowable_version:, latest_version_finder:, repo_contents_path:)
           @dependency               = dependency
           @credentials              = credentials
           @dependency_files         = dependency_files
@@ -70,6 +79,7 @@ module Dependabot
 
           @latest_version_finder = {}
           @latest_version_finder[dependency] = latest_version_finder
+          @repo_contents_path = repo_contents_path
         end
 
         def latest_resolvable_version
@@ -135,7 +145,7 @@ module Dependabot
         private
 
         attr_reader :dependency, :credentials, :dependency_files,
-                    :latest_allowable_version
+                    :latest_allowable_version, :repo_contents_path
 
         def latest_version_finder(dep)
           @latest_version_finder[dep] ||=
@@ -307,30 +317,15 @@ module Dependabot
           # TODO: Add all of the error handling that the FileUpdater does
           # here (since problematic repos will be resolved here before they're
           # seen by the FileUpdater)
-          SharedHelpers.in_a_temporary_directory do
+          base_dir = dependency_files.first.directory
+          SharedHelpers.in_a_temporary_repo_directory(base_dir, repo_contents_path) do
             dependency_files_builder.write_temporary_dependency_files
 
             filtered_package_files.flat_map do |file|
               path = Pathname.new(file.name).dirname
               run_checker(path: path, version: version)
             rescue SharedHelpers::HelperSubprocessFailed => e
-              errors = []
-              if e.message.match?(NPM6_PEER_DEP_ERROR_REGEX)
-                e.message.scan(NPM6_PEER_DEP_ERROR_REGEX) do
-                  errors << Regexp.last_match.named_captures
-                end
-              elsif e.message.match?(NPM8_PEER_DEP_ERROR_REGEX)
-                e.message.scan(NPM8_PEER_DEP_ERROR_REGEX) do
-                  errors << Regexp.last_match.named_captures
-                end
-              elsif e.message.match?(YARN_PEER_DEP_ERROR_REGEX)
-                e.message.scan(YARN_PEER_DEP_ERROR_REGEX) do
-                  errors << Regexp.last_match.named_captures
-                end
-              else
-                raise
-              end
-              errors
+              handle_peer_dependency_errors(e)
             end.compact
           end
         rescue SharedHelpers::HelperSubprocessFailed
@@ -340,6 +335,30 @@ module Dependabot
           []
         end
 
+        def handle_peer_dependency_errors(error)
+          errors = []
+          if error.message.match?(NPM6_PEER_DEP_ERROR_REGEX)
+            error.message.scan(NPM6_PEER_DEP_ERROR_REGEX) do
+              errors << Regexp.last_match.named_captures
+            end
+          elsif error.message.match?(NPM8_PEER_DEP_ERROR_REGEX)
+            error.message.scan(NPM8_PEER_DEP_ERROR_REGEX) do
+              errors << Regexp.last_match.named_captures
+            end
+          elsif error.message.match?(YARN_PEER_DEP_ERROR_REGEX)
+            error.message.scan(YARN_PEER_DEP_ERROR_REGEX) do
+              errors << Regexp.last_match.named_captures
+            end
+          elsif error.message.match?(YARN_BERRY_PEER_DEP_ERROR_REGEX)
+            error.message.scan(YARN_BERRY_PEER_DEP_ERROR_REGEX) do
+              errors << Regexp.last_match.named_captures
+            end
+          else
+            raise
+          end
+          errors
+        end
+
         def unmet_peer_dependencies
           peer_dependency_errors.
             map { |captures| error_details_from_captures(captures) }
@@ -351,13 +370,14 @@ module Dependabot
         end
 
         def error_details_from_captures(captures)
+          required_dep_captures  = captures.fetch("required_dep")
+          requiring_dep_captures = captures.fetch("requiring_dep")
+          return {} unless required_dep_captures && requiring_dep_captures
+
           {
-            requirement_name:
-              captures.fetch("required_dep").sub(/@[^@]+$/, ""),
-            requirement_version:
-              captures.fetch("required_dep").split("@").last.delete('"'),
-            requiring_dep_name:
-              captures.fetch("requiring_dep").sub(/@[^@]+$/, "")
+            requirement_name: required_dep_captures.sub(/@[^@]+$/, ""),
+            requirement_version: required_dep_captures.split("@").last.delete('"'),
+            requiring_dep_name: requiring_dep_captures.sub(/@[^@]+$/, "")
           }
         end
 
@@ -468,13 +488,37 @@ module Dependabot
         def run_checker(path:, version:)
           # If there are both yarn lockfiles and npm lockfiles only run the
           # yarn updater
-          if lockfiles_for_path(lockfiles: dependency_files_builder.yarn_locks, path: path).any?
+          lockfiles = lockfiles_for_path(lockfiles: dependency_files_builder.yarn_locks, path: path)
+          if lockfiles.any?
+            return run_yarn_berry_checker(path: path, version: version) if Helpers.yarn_berry?(lockfiles.first)
+
             return run_yarn_checker(path: path, version: version)
           end
 
           run_npm_checker(path: path, version: version)
         end
 
+        def run_yarn_berry_checker(path:, version:)
+          # This method mimics calling a native helper in order to comply with the caller's expectations
+          # Specifically we add the dependency at the specified updated version
+          # then check the output of the add command for Peer Dependency errors (Denoted by YN0060)
+          # If we find peer dependency issues, we raise HelperSubprocessFailed as
+          # the native helpers do.
+          SharedHelpers.with_git_configured(credentials: credentials) do
+            Dir.chdir(path) do
+              output = Helpers.run_yarn_command(
+                "yarn add #{dependency.name}@#{version} #{Helpers.yarn_berry_args}".strip
+              )
+              if output.include?("YN0060")
+                raise SharedHelpers::HelperSubprocessFailed.new(
+                  message: output,
+                  error_context: {}
+                )
+              end
+            end
+          end
+        end
+
         def run_yarn_checker(path:, version:)
           SharedHelpers.with_git_configured(credentials: credentials) do
             Dir.chdir(path) do

data/lib/dependabot/npm_and_yarn/update_checker.rb

@@ -22,7 +22,7 @@ module Dependabot
                         dependency.version &&
                         version_class.correct?(dependency.version) &&
                         vulnerable_versions.any? &&
-                        !vulnerable_versions.include?(version_class.new(dependency.version))
+                        !vulnerable_versions.include?(current_version)
 
         super
       end
@@ -279,9 +279,7 @@ module Dependabot
 
       def latest_version_for_git_dependency
         @latest_version_for_git_dependency ||=
-          if git_branch_or_ref_in_latest_release?
-            latest_released_version
-          elsif version_class.correct?(dependency.version)
+          if version_class.correct?(dependency.version)
             latest_git_version_details[:version] &&
               version_class.new(latest_git_version_details[:version])
           else
@@ -294,26 +292,9 @@ module Dependabot
           latest_version_finder.latest_version_from_registry
       end
 
-      def should_switch_source_from_git_to_registry?
-        return false unless git_dependency?
-        return false unless git_branch_or_ref_in_latest_release?
-        return false if latest_version_for_git_dependency.nil?
-
-        version_class.correct?(latest_version_for_git_dependency)
-      end
-
-      def git_branch_or_ref_in_latest_release?
-        return false unless latest_released_version
-
-        return @git_branch_or_ref_in_latest_release if defined?(@git_branch_or_ref_in_latest_release)
-
-        @git_branch_or_ref_in_latest_release ||=
-          git_commit_checker.branch_or_ref_in_release?(latest_released_version)
-      end
-
       def latest_version_details
         @latest_version_details ||=
-          if git_dependency? && !should_switch_source_from_git_to_registry?
+          if git_dependency?
             latest_git_version_details
           else
             { version: latest_released_version }
@@ -339,7 +320,8 @@ module Dependabot
             credentials: credentials,
             dependency_files: dependency_files,
             latest_allowable_version: latest_version,
-            latest_version_finder: latest_version_finder
+            latest_version_finder: latest_version_finder,
+            repo_contents_path: repo_contents_path
           )
       end
 
@@ -350,7 +332,8 @@ module Dependabot
             credentials: credentials,
             dependency_files: dependency_files,
             ignored_versions: ignored_versions,
-            latest_allowable_version: latest_version
+            latest_allowable_version: latest_version,
+            repo_contents_path: repo_contents_path
           )
       end
 
@@ -387,9 +370,6 @@ module Dependabot
         # Never need to update source, unless a git_dependency
         return dependency_source_details unless git_dependency?
 
-        # Source becomes `nil` if switching to default rubygems
-        return nil if should_switch_source_from_git_to_registry?
-
         # Update the git tag if updating a pinned version
         if git_commit_checker.pinned_ref_looks_like_version? &&
            !git_commit_checker.local_tag_for_latest_version.nil?

data/lib/dependabot/npm_and_yarn.rb

@@ -24,3 +24,5 @@ Dependabot::Dependency.register_production_check(
     groups.include?("dependencies")
   end
 )
+
+Dependabot::Utils.register_always_clone("npm_and_yarn")