dependabot-npm_and_yarn 0.212.0 → 0.214.0

data/lib/dependabot/npm_and_yarn/update_checker/registry_finder.rb

@@ -12,20 +12,21 @@ module Dependabot
           https://registry.npmjs.org
           http://registry.npmjs.org
           https://registry.yarnpkg.com
+          http://registry.yarnpkg.com
         ).freeze
-        NPM_AUTH_TOKEN_REGEX =
-          %r{//(?<registry>.*)/:_authToken=(?<token>.*)$}.freeze
-        NPM_GLOBAL_REGISTRY_REGEX =
-          /^registry\s*=\s*['"]?(?<registry>.*?)['"]?$/.freeze
-        YARN_GLOBAL_REGISTRY_REGEX =
-          /^(?:--)?registry\s+['"](?<registry>.*)['"]/.freeze
+        NPM_AUTH_TOKEN_REGEX = %r{//(?<registry>.*)/:_authToken=(?<token>.*)$}
+        NPM_GLOBAL_REGISTRY_REGEX = /^registry\s*=\s*['"]?(?<registry>.*?)['"]?$/
+        YARN_GLOBAL_REGISTRY_REGEX = /^(?:--)?registry\s+((['"](?<registry>.*)['"])|(?<registry>.*))/
+        NPM_SCOPED_REGISTRY_REGEX = /^(?<scope>@[^:]+)\s*:registry\s*=\s*['"]?(?<registry>.*?)['"]?$/
+        YARN_SCOPED_REGISTRY_REGEX = /['"](?<scope>@[^:]+):registry['"]\s((['"](?<registry>.*)['"])|(?<registry>.*))/
 
         def initialize(dependency:, credentials:, npmrc_file: nil,
-                       yarnrc_file: nil)
+                       yarnrc_file: nil, yarnrc_yml_file: nil)
           @dependency = dependency
           @credentials = credentials
           @npmrc_file = npmrc_file
           @yarnrc_file = yarnrc_file
+          @yarnrc_yml_file = yarnrc_yml_file
         end
 
         def registry
@@ -46,15 +47,24 @@ module Dependabot
           end
         end
 
+        def registry_from_rc(dependency_name)
+          return global_registry unless dependency_name.start_with?("@") && dependency_name.include?("/")
+
+          scope = dependency_name.split("/").first
+          scoped_registry(scope)
+        end
+
         private
 
-        attr_reader :dependency, :credentials, :npmrc_file, :yarnrc_file
+        attr_reader :dependency, :credentials, :npmrc_file, :yarnrc_file, :yarnrc_yml_file
 
         def first_registry_with_dependency_details
           @first_registry_with_dependency_details ||=
             known_registries.find do |details|
+              url = "#{details['registry'].gsub(%r{/+$}, '')}/#{escaped_dependency_name}"
+              url = "https://#{url}" unless url.start_with?("http")
               response = Dependabot::RegistryClient.get(
-                url: "https://#{details['registry'].gsub(%r{/+$}, '')}/#{escaped_dependency_name}",
+                url: url,
                 headers: auth_header_for(details["token"])
               )
               response.status < 400 && JSON.parse(response.body)
@@ -64,10 +74,12 @@ module Dependabot
               nil
             end&.fetch("registry")
 
-          @first_registry_with_dependency_details ||= global_registry
+          @first_registry_with_dependency_details ||= global_registry.sub(%r{/+$}, "").sub(%r{^.*?//}, "")
         end
 
         def registry_url
+          return registry if registry.start_with?("http")
+
           protocol =
             if registry_source_url
               registry_source_url.split("://").first
@@ -134,10 +146,13 @@ module Dependabot
           npmrc_file.content.scan(NPM_AUTH_TOKEN_REGEX) do
             next if Regexp.last_match[:registry].include?("${")
 
+            registry = Regexp.last_match[:registry]
+            token = Regexp.last_match[:token]&.strip
+
             registries << {
               "type" => "npm_registry",
-              "registry" => Regexp.last_match[:registry],
-              "token" => Regexp.last_match[:token]&.strip
+              "registry" => registry.gsub(/\s+/, "%20"),
+              "token" => token
             }
           end
 
@@ -146,7 +161,8 @@ module Dependabot
 
             registry = Regexp.last_match[:registry].strip.
                        sub(%r{/+$}, "").
-                       sub(%r{^.*?//}, "")
+                       sub(%r{^.*?//}, "").
+                       gsub(/\s+/, "%20")
             next if registries.map { |r| r["registry"] }.include?(registry)
 
             registries << {
@@ -168,7 +184,8 @@ module Dependabot
 
             registry = Regexp.last_match[:registry].strip.
                        sub(%r{/+$}, "").
-                       sub(%r{^.*?//}, "")
+                       sub(%r{^.*?//}, "").
+                       gsub(/\s+/, "%20")
             registries << {
               "type" => "npm_registry",
               "registry" => registry,
@@ -190,26 +207,56 @@ module Dependabot
           end
         end
 
+        # rubocop:disable Metrics/PerceivedComplexity
         def global_registry
+          return @global_registry if defined? @global_registry
+
           npmrc_file&.content.to_s.scan(NPM_GLOBAL_REGISTRY_REGEX) do
             next if Regexp.last_match[:registry].include?("${")
 
-            registry = Regexp.last_match[:registry].strip.
-                       sub(%r{/+$}, "").
-                       sub(%r{^.*?//}, "")
-            return registry
+            return @global_registry = Regexp.last_match[:registry].strip
           end
 
           yarnrc_file&.content.to_s.scan(YARN_GLOBAL_REGISTRY_REGEX) do
             next if Regexp.last_match[:registry].include?("${")
 
-            registry = Regexp.last_match[:registry].strip.
-                       sub(%r{/+$}, "").
-                       sub(%r{^.*?//}, "")
-            return registry
+            return @global_registry = Regexp.last_match[:registry].strip
+          end
+
+          if parsed_yarnrc_yml&.key?("npmRegistryServer")
+            return @global_registry = parsed_yarnrc_yml["npmRegistryServer"]
+          end
+
+          replaces_base = credentials.find { |cred| cred["type"] == "npm_registry" && cred["replaces-base"] == true }
+          if replaces_base
+            registry = replaces_base["registry"]
+            registry = "https://#{registry}" unless registry.start_with?("http")
+            return @global_registry = registry
           end
 
-          "registry.npmjs.org"
+          "https://registry.npmjs.org"
+        end
+        # rubocop:enable Metrics/PerceivedComplexity
+
+        def scoped_registry(scope)
+          npmrc_file&.content.to_s.scan(NPM_SCOPED_REGISTRY_REGEX) do
+            next if Regexp.last_match[:registry].include?("${") || Regexp.last_match[:scope] != scope
+
+            return Regexp.last_match[:registry].strip
+          end
+
+          yarnrc_file&.content.to_s.scan(YARN_SCOPED_REGISTRY_REGEX) do
+            next if Regexp.last_match[:registry].include?("${") || Regexp.last_match[:scope] != scope
+
+            return Regexp.last_match[:registry].strip
+          end
+
+          if parsed_yarnrc_yml
+            yarn_berry_registry = parsed_yarnrc_yml.dig("npmScopes", scope.delete_prefix("@"), "npmRegistryServer")
+            return yarn_berry_registry if yarn_berry_registry
+          end
+
+          global_registry
         end
 
         # npm registries expect slashes to be escaped
@@ -224,6 +271,13 @@ module Dependabot
 
           sources.find { |s| s[:type] == "registry" }&.fetch(:url)
         end
+
+        def parsed_yarnrc_yml
+          return unless yarnrc_yml_file
+          return @parsed_yarnrc_yml if defined? @parsed_yarnrc_yml
+
+          @parsed_yarnrc_yml = YAML.safe_load(yarnrc_yml_file.content)
+        end
       end
     end
   end

data/lib/dependabot/npm_and_yarn/update_checker/requirements_updater.rb

@@ -13,10 +13,9 @@ module Dependabot
   module NpmAndYarn
     class UpdateChecker
       class RequirementsUpdater
-        VERSION_REGEX = /[0-9]+(?:\.[A-Za-z0-9\-_]+)*/.freeze
-        SEPARATOR = /(?<=[a-zA-Z0-9*])[\s|]+(?![\s|-])/.freeze
-        ALLOWED_UPDATE_STRATEGIES =
-          %i(widen_ranges bump_versions bump_versions_if_necessary).freeze
+        VERSION_REGEX = /[0-9]+(?:\.[A-Za-z0-9\-_]+)*/
+        SEPARATOR = /(?<=[a-zA-Z0-9*])[\s|]+(?![\s|-])/
+        ALLOWED_UPDATE_STRATEGIES = %i(widen_ranges bump_versions bump_versions_if_necessary).freeze
 
         def initialize(requirements:, updated_source:, update_strategy:,
                        latest_resolvable_version:)

data/lib/dependabot/npm_and_yarn/update_checker/subdependency_version_resolver.rb

@@ -19,19 +19,21 @@ module Dependabot
     class UpdateChecker
       class SubdependencyVersionResolver
         def initialize(dependency:, credentials:, dependency_files:,
-                       ignored_versions:, latest_allowable_version:)
+                       ignored_versions:, latest_allowable_version:, repo_contents_path:)
           @dependency = dependency
           @credentials = credentials
           @dependency_files = dependency_files
           @ignored_versions = ignored_versions
           @latest_allowable_version = latest_allowable_version
+          @repo_contents_path = repo_contents_path
         end
 
         def latest_resolvable_version
           raise "Not a subdependency!" if dependency.requirements.any?
           return if bundled_dependency?
 
-          SharedHelpers.in_a_temporary_directory do
+          base_dir = dependency_files.first.directory
+          SharedHelpers.in_a_temporary_repo_directory(base_dir, repo_contents_path) do
             dependency_files_builder.write_temporary_dependency_files
 
             updated_lockfiles = filtered_lockfiles.map do |lockfile|
@@ -53,13 +55,15 @@ module Dependabot
         private
 
         attr_reader :dependency, :credentials, :dependency_files,
-                    :ignored_versions, :latest_allowable_version
+                    :ignored_versions, :latest_allowable_version, :repo_contents_path
 
         def update_subdependency_in_lockfile(lockfile)
           lockfile_name = Pathname.new(lockfile.name).basename.to_s
           path = Pathname.new(lockfile.name).dirname.to_s
 
-          updated_files = if lockfile.name.end_with?("yarn.lock")
+          updated_files = if lockfile.name.end_with?("yarn.lock") && Helpers.yarn_berry?(lockfile)
+                            run_yarn_berry_updater(path, lockfile_name)
+                          elsif lockfile.name.end_with?("yarn.lock")
                             run_yarn_updater(path, lockfile_name)
                           else
                             run_npm_updater(path, lockfile_name, lockfile.content)
@@ -109,6 +113,17 @@ module Dependabot
           sleep(rand(3.0..10.0)) && retry
         end
 
+        def run_yarn_berry_updater(path, lockfile_name)
+          SharedHelpers.with_git_configured(credentials: credentials) do
+            Dir.chdir(path) do
+              Helpers.run_yarn_commands(
+                "yarn up -R #{dependency.name} #{Helpers.yarn_berry_args}".strip
+              )
+              { lockfile_name => File.read(lockfile_name) }
+            end
+          end
+        end
+
         def run_npm_updater(path, lockfile_name, lockfile_content)
           SharedHelpers.with_git_configured(credentials: credentials) do
             Dir.chdir(path) do

data/lib/dependabot/npm_and_yarn/update_checker/version_resolver.rb

@@ -36,7 +36,16 @@ module Dependabot
             "\s>\s(?<requiring_dep>[^"]+)"\s
             has\s(incorrect|unmet)\speer\sdependency\s
             "(?<required_dep>[^"]+)"
-          /x.freeze
+          /x
+
+        # Error message from yarn add:
+        # YN0060: │ eve-roster@workspace:. provides jest (p8d618) \
+        # with version 29.3.0, which doesn't satisfy \
+        # what ts-jest requests\n
+        YARN_BERRY_PEER_DEP_ERROR_REGEX =
+          /
+            YN0060:\s|\s.+\sprovides\s(?<required_dep>.+?)\s\((?<info_hash>\w+)\).+what\s(?<requiring_dep>.+?)\srequests
+          /x
 
         # Error message from npm install:
         # react-dom@15.2.0 requires a peer of react@^15.2.0 \
@@ -46,7 +55,7 @@ module Dependabot
             (?<requiring_dep>[^\s]+)\s
             requires\sa\speer\sof\s
             (?<required_dep>.+?)\sbut\snone\sis\sinstalled.
-          /x.freeze
+          /x
 
         # Error message from npm install:
         # npm ERR! Could not resolve dependency:
@@ -59,10 +68,10 @@ module Dependabot
           /
             npm\s(?:WARN|ERR!)\sCould\snot\sresolve\sdependency:\n
             npm\s(?:WARN|ERR!)\speer\s(?<required_dep>\S+@\S+(\s\S+)?)\sfrom\s(?<requiring_dep>\S+@\S+)
-          /x.freeze
+          /x
 
         def initialize(dependency:, credentials:, dependency_files:,
-                       latest_allowable_version:, latest_version_finder:)
+                       latest_allowable_version:, latest_version_finder:, repo_contents_path:)
           @dependency               = dependency
           @credentials              = credentials
           @dependency_files         = dependency_files
@@ -70,6 +79,7 @@ module Dependabot
 
           @latest_version_finder = {}
           @latest_version_finder[dependency] = latest_version_finder
+          @repo_contents_path = repo_contents_path
         end
 
         def latest_resolvable_version
@@ -135,7 +145,7 @@ module Dependabot
         private
 
         attr_reader :dependency, :credentials, :dependency_files,
-                    :latest_allowable_version
+                    :latest_allowable_version, :repo_contents_path
 
         def latest_version_finder(dep)
           @latest_version_finder[dep] ||=
@@ -307,30 +317,15 @@ module Dependabot
           # TODO: Add all of the error handling that the FileUpdater does
           # here (since problematic repos will be resolved here before they're
           # seen by the FileUpdater)
-          SharedHelpers.in_a_temporary_directory do
+          base_dir = dependency_files.first.directory
+          SharedHelpers.in_a_temporary_repo_directory(base_dir, repo_contents_path) do
             dependency_files_builder.write_temporary_dependency_files
 
             filtered_package_files.flat_map do |file|
               path = Pathname.new(file.name).dirname
               run_checker(path: path, version: version)
             rescue SharedHelpers::HelperSubprocessFailed => e
-              errors = []
-              if e.message.match?(NPM6_PEER_DEP_ERROR_REGEX)
-                e.message.scan(NPM6_PEER_DEP_ERROR_REGEX) do
-                  errors << Regexp.last_match.named_captures
-                end
-              elsif e.message.match?(NPM8_PEER_DEP_ERROR_REGEX)
-                e.message.scan(NPM8_PEER_DEP_ERROR_REGEX) do
-                  errors << Regexp.last_match.named_captures
-                end
-              elsif e.message.match?(YARN_PEER_DEP_ERROR_REGEX)
-                e.message.scan(YARN_PEER_DEP_ERROR_REGEX) do
-                  errors << Regexp.last_match.named_captures
-                end
-              else
-                raise
-              end
-              errors
+              handle_peer_dependency_errors(e)
             end.compact
           end
         rescue SharedHelpers::HelperSubprocessFailed
@@ -340,6 +335,30 @@ module Dependabot
           []
         end
 
+        def handle_peer_dependency_errors(error)
+          errors = []
+          if error.message.match?(NPM6_PEER_DEP_ERROR_REGEX)
+            error.message.scan(NPM6_PEER_DEP_ERROR_REGEX) do
+              errors << Regexp.last_match.named_captures
+            end
+          elsif error.message.match?(NPM8_PEER_DEP_ERROR_REGEX)
+            error.message.scan(NPM8_PEER_DEP_ERROR_REGEX) do
+              errors << Regexp.last_match.named_captures
+            end
+          elsif error.message.match?(YARN_PEER_DEP_ERROR_REGEX)
+            error.message.scan(YARN_PEER_DEP_ERROR_REGEX) do
+              errors << Regexp.last_match.named_captures
+            end
+          elsif error.message.match?(YARN_BERRY_PEER_DEP_ERROR_REGEX)
+            error.message.scan(YARN_BERRY_PEER_DEP_ERROR_REGEX) do
+              errors << Regexp.last_match.named_captures
+            end
+          else
+            raise
+          end
+          errors
+        end
+
         def unmet_peer_dependencies
           peer_dependency_errors.
             map { |captures| error_details_from_captures(captures) }
@@ -351,13 +370,14 @@ module Dependabot
         end
 
         def error_details_from_captures(captures)
+          required_dep_captures  = captures.fetch("required_dep")
+          requiring_dep_captures = captures.fetch("requiring_dep")
+          return {} unless required_dep_captures && requiring_dep_captures
+
           {
-            requirement_name:
-              captures.fetch("required_dep").sub(/@[^@]+$/, ""),
-            requirement_version:
-              captures.fetch("required_dep").split("@").last.delete('"'),
-            requiring_dep_name:
-              captures.fetch("requiring_dep").sub(/@[^@]+$/, "")
+            requirement_name: required_dep_captures.sub(/@[^@]+$/, ""),
+            requirement_version: required_dep_captures.split("@").last.delete('"'),
+            requiring_dep_name: requiring_dep_captures.sub(/@[^@]+$/, "")
           }
         end
 
@@ -468,13 +488,37 @@ module Dependabot
         def run_checker(path:, version:)
           # If there are both yarn lockfiles and npm lockfiles only run the
           # yarn updater
-          if lockfiles_for_path(lockfiles: dependency_files_builder.yarn_locks, path: path).any?
+          lockfiles = lockfiles_for_path(lockfiles: dependency_files_builder.yarn_locks, path: path)
+          if lockfiles.any?
+            return run_yarn_berry_checker(path: path, version: version) if Helpers.yarn_berry?(lockfiles.first)
+
             return run_yarn_checker(path: path, version: version)
           end
 
           run_npm_checker(path: path, version: version)
         end
 
+        def run_yarn_berry_checker(path:, version:)
+          # This method mimics calling a native helper in order to comply with the caller's expectations
+          # Specifically we add the dependency at the specified updated version
+          # then check the output of the add command for Peer Dependency errors (Denoted by YN0060)
+          # If we find peer dependency issues, we raise HelperSubprocessFailed as
+          # the native helpers do.
+          SharedHelpers.with_git_configured(credentials: credentials) do
+            Dir.chdir(path) do
+              output = Helpers.run_yarn_command(
+                "yarn add #{dependency.name}@#{version} #{Helpers.yarn_berry_args}".strip
+              )
+              if output.include?("YN0060")
+                raise SharedHelpers::HelperSubprocessFailed.new(
+                  message: output,
+                  error_context: {}
+                )
+              end
+            end
+          end
+        end
+
         def run_yarn_checker(path:, version:)
           SharedHelpers.with_git_configured(credentials: credentials) do
             Dir.chdir(path) do

data/lib/dependabot/npm_and_yarn/update_checker/vulnerability_auditor.rb

@@ -21,6 +21,7 @@ module Dependabot
           @allow_removal = allow_removal
         end
 
+        # rubocop:disable Metrics/MethodLength
         # Finds any dependencies in the `package-lock.json` or `npm-shrinkwrap.json` that have
         # a subdependency on the given dependency that is locked to a vuln version range.
         #
@@ -41,7 +42,10 @@ module Dependabot
         #       dependency on the blocking dependency
         #   * :top_level_ancestors [Array<String>] the names of all top-level dependencies with a transitive
         #     dependency on the dependency
+        #   * :explanation [String] an explanation for why the project failed the vulnerability auditor run
         def audit(dependency:, security_advisories:)
+          Dependabot.logger.info("VulnerabilityAuditor: starting audit")
+
           fix_unavailable = {
             "dependency_name" => dependency.name,
             "fix_available" => false,
@@ -60,7 +64,10 @@ module Dependabot
             # `npm-shrinkwrap.js`, if present, takes precedence over `package-lock.js`.
             # Both files use the same format. See https://bit.ly/3lDIAJV for more.
             lockfile = (dependency_files_builder.shrinkwraps + dependency_files_builder.package_locks).first
-            return fix_unavailable unless lockfile
+            unless lockfile
+              Dependabot.logger.info("VulnerabilityAuditor: missing lockfile")
+              return fix_unavailable
+            end
 
             vuln_versions = security_advisories.map do |a|
               {
@@ -74,25 +81,37 @@ module Dependabot
               function: "npm:vulnerabilityAuditor",
               args: [Dir.pwd, vuln_versions]
             )
-            return fix_unavailable unless viable_audit_result?(audit_result, security_advisories)
 
+            validation_result = validate_audit_result(audit_result, security_advisories)
+            if validation_result != :viable
+              Dependabot.logger.info("VulnerabilityAuditor: audit result not viable: #{validation_result}")
+              fix_unavailable["explanation"] = explain_fix_unavailable(validation_result, dependency)
+              return fix_unavailable
+            end
+
+            Dependabot.logger.info("VulnerabilityAuditor: audit result viable")
             audit_result
           end
         rescue SharedHelpers::HelperSubprocessFailed => e
           log_helper_subprocess_failure(dependency, e)
           fix_unavailable
         end
+        # rubocop:enable Metrics/MethodLength
 
         private
 
         attr_reader :dependency_files, :credentials
 
-        def viable_audit_result?(audit_result, security_advisories)
-          validation_result = validate_audit_result(audit_result, security_advisories)
-          return true if validation_result == :viable
-
-          Dependabot.logger.info("VulnerabilityAuditor: audit result not viable: #{validation_result}")
-          false
+        def explain_fix_unavailable(validation_result, dependency)
+          case validation_result
+          when :fix_unavailable, :dependency_still_vulnerable, :downgrades_dependencies
+            "No patched version available for #{dependency.name}"
+          when :fix_incomplete
+            "The lockfile might be out of sync?"
+          when :vulnerable_dependency_removed
+            "#{dependency.name} was removed in the update. Dependabot is not able to " \
+            "deal with this yet, but you can still upgrade manually."
+          end
         end
 
         def validate_audit_result(audit_result, security_advisories)
@@ -100,6 +119,7 @@ module Dependabot
           return :vulnerable_dependency_removed if !@allow_removal && vulnerable_dependency_removed?(audit_result)
           return :dependency_still_vulnerable if dependency_still_vulnerable?(audit_result, security_advisories)
           return :downgrades_dependencies if downgrades_dependencies?(audit_result)
+          return :fix_incomplete if fix_incomplete?(audit_result)
 
           :viable
         end
@@ -132,6 +152,11 @@ module Dependabot
           current > target
         end
 
+        def fix_incomplete?(audit_result)
+          audit_result["fix_updates"].any? { |update| !update.key?("target_version") } ||
+            audit_result["fix_updates"].empty?
+        end
+
         def log_helper_subprocess_failure(dependency, error)
           # See `Dependabot::SharedHelpers.run_helper_subprocess` for details on error context
           context = error.error_context || {}