dependabot-npm_and_yarn 0.212.0 → 0.213.0

data/lib/dependabot/npm_and_yarn/file_updater.rb

@@ -1,7 +1,9 @@
 # frozen_string_literal: true
 
+require "dependabot/experiments"
 require "dependabot/file_updaters"
 require "dependabot/file_updaters/base"
+require "dependabot/file_updaters/vendor_updater"
 require "dependabot/npm_and_yarn/dependency_files_filterer"
 require "dependabot/npm_and_yarn/sub_dependency_files_filterer"
 
@@ -53,11 +55,62 @@ module Dependabot
           )
         end
 
+        base_dir = updated_files.first.directory
+        vendor_updater.updated_vendor_cache_files(base_directory: base_dir).each { |file| updated_files << file }
+        install_state_updater.updated_vendor_cache_files(base_directory: base_dir).each do |file|
+          updated_files << file
+        end
+        pnp_updater.updated_vendor_cache_files(base_directory: base_dir).each do |file|
+          updated_files << file if file.name == ".pnp.cjs" || file.name == ".pnp.data.json"
+        end
+
         updated_files
       end
 
       private
 
+      # Dynamically fetch the vendor cache folder from yarn
+      def vendor_cache_dir
+        return @vendor_cache_dir if defined?(@vendor_cache_dir)
+
+        @vendor_cache_dir = if File.exist?(".yarnrc.yml")
+                              YAML.load_file(".yarnrc.yml").fetch("cacheFolder", "./.yarn/cache")
+                            else
+                              "./.yarn/cache"
+                            end
+      end
+
+      def install_state_path
+        return @install_state_path if defined?(@install_state_path)
+
+        @install_state_path = if File.exist?(".yarnrc.yml")
+                                YAML.load_file(".yarnrc.yml").fetch("installStatePath", "./.yarn/install-state.gz")
+                              else
+                                "./.yarn/install-state.gz"
+                              end
+      end
+
+      def vendor_updater
+        Dependabot::FileUpdaters::VendorUpdater.new(
+          repo_contents_path: repo_contents_path,
+          vendor_dir: vendor_cache_dir
+        )
+      end
+
+      def install_state_updater
+        Dependabot::FileUpdaters::VendorUpdater.new(
+          repo_contents_path: repo_contents_path,
+          vendor_dir: install_state_path
+        )
+      end
+
+      def pnp_updater
+        Dependabot::FileUpdaters::VendorUpdater.new(
+          repo_contents_path: repo_contents_path,
+          vendor_dir: "./"
+        )
+      end
+
       def filtered_dependency_files
         @filtered_dependency_files ||=
           if dependencies.select(&:top_level?).any?
@@ -175,6 +228,7 @@ module Dependabot
           YarnLockfileUpdater.new(
             dependencies: dependencies,
             dependency_files: dependency_files,
+            repo_contents_path: repo_contents_path,
             credentials: credentials
           )
       end

data/lib/dependabot/npm_and_yarn/helpers.rb

@@ -15,6 +15,54 @@ module Dependabot
       rescue JSON::ParserError
         6
       end
+
+      # Run any number of yarn commands while ensuring that `enableScripts` is
+      # set to false. Yarn commands should _not_ be ran outside of this helper
+      # to ensure that postinstall scripts are never executed, as they could
+      # contain malicious code.
+      def self.run_yarn_commands(*commands)
+        # We never want to execute postinstall scripts
+        SharedHelpers.run_shell_command("yarn config set enableScripts false")
+        if (http_proxy = ENV.fetch("HTTP_PROXY", false))
+          SharedHelpers.run_shell_command("yarn config set httpProxy #{http_proxy}")
+        end
+        if (https_proxy = ENV.fetch("HTTPS_PROXY", false))
+          SharedHelpers.run_shell_command("yarn config set httpsProxy #{https_proxy}")
+        end
+        if (ca_file_path = ENV.fetch("NODE_EXTRA_CA_CERTS", false))
+          output = SharedHelpers.run_shell_command("yarn --version")
+          major_version = Version.new(output).major
+          if major_version >= 4
+            SharedHelpers.run_shell_command("yarn config set httpsCaFilePath #{ca_file_path}")
+          else
+            SharedHelpers.run_shell_command("yarn config set caFilePath #{ca_file_path}")
+          end
+        end
+        commands.each { |cmd| SharedHelpers.run_shell_command(cmd) }
+      end
+
+      def self.dependencies_with_all_versions_metadata(dependency_set)
+        working_set = Dependabot::NpmAndYarn::FileParser::DependencySet.new
+        dependencies = []
+
+        names = dependency_set.dependencies.map(&:name)
+        names.each do |name|
+          all_versions = dependency_set.all_versions_for_name(name)
+          all_versions.each do |dep|
+            metadata_versions = dep.metadata.fetch(:all_versions, [])
+            if metadata_versions.any?
+              metadata_versions.each { |a| working_set << a }
+            else
+              working_set << dep
+            end
+          end
+          dependency = working_set.dependency_for_name(name)
+          dependency.metadata[:all_versions] = working_set.all_versions_for_name(name)
+          dependencies << dependency
+        end
+
+        dependencies
+      end
     end
   end
 end

data/lib/dependabot/npm_and_yarn/package_name.rb

@@ -18,7 +18,7 @@ module Dependabot
             [a-z0-9\-\_\.\!\~\*\'\(\)]+               # URL-safe characters
           )
           \z                                          # end of string
-      }xi.freeze                                      # multi-line/case-insensitive
+      }xi                                             # multi-line/case-insensitive
 
       TYPES_PACKAGE_NAME_REGEX = %r{
           \A                                          # beginning of string
@@ -26,7 +26,7 @@ module Dependabot
           ((?<scope>.+)__)?                           # capture scope
           (?<name>.+)                                 # capture name
           \z                                          # end of string
-      }xi.freeze                                      # multi-line/case-insensitive
+      }xi                                             # multi-line/case-insensitive
 
       class InvalidPackageName < StandardError; end
 

data/lib/dependabot/npm_and_yarn/requirement.rb

@@ -6,8 +6,8 @@ require "dependabot/npm_and_yarn/version"
 module Dependabot
   module NpmAndYarn
     class Requirement < Gem::Requirement
-      AND_SEPARATOR = /(?<=[a-zA-Z0-9*])\s+(?:&+\s+)?(?!\s*[|-])/.freeze
-      OR_SEPARATOR = /(?<=[a-zA-Z0-9*])\s*\|+/.freeze
+      AND_SEPARATOR = /(?<=[a-zA-Z0-9*])\s+(?:&+\s+)?(?!\s*[|-])/
+      OR_SEPARATOR = /(?<=[a-zA-Z0-9*])\s*\|+/
       LATEST_REQUIREMENT = "latest"
 
       # Override the version pattern to allow a 'v' prefix
@@ -15,7 +15,7 @@ module Dependabot
       version_pattern = "v?#{NpmAndYarn::Version::VERSION_PATTERN}"
 
       PATTERN_RAW = "\\s*(#{quoted})?\\s*(#{version_pattern})\\s*"
-      PATTERN = /\A#{PATTERN_RAW}\z/.freeze
+      PATTERN = /\A#{PATTERN_RAW}\z/
 
       def self.parse(obj)
         return ["=", nil] if obj.is_a?(String) && obj.strip == LATEST_REQUIREMENT

data/lib/dependabot/npm_and_yarn/update_checker/latest_version_finder.rb

@@ -371,7 +371,8 @@ module Dependabot
             dependency: dependency,
             credentials: credentials,
             npmrc_file: npmrc_file,
-            yarnrc_file: yarnrc_file
+            yarnrc_file: yarnrc_file,
+            yarnrc_yml_file: yarnrc_yml_file
           )
         end
 
@@ -395,6 +396,10 @@ module Dependabot
           dependency_files.find { |f| f.name.end_with?(".yarnrc") }
         end
 
+        def yarnrc_yml_file
+          dependency_files.find { |f| f.name.end_with?(".yarnrc.yml") }
+        end
+
         # TODO: Remove need for me
         def git_dependency?
           # ignored_version/raise_on_ignored are irrelevant.

data/lib/dependabot/npm_and_yarn/update_checker/library_detector.rb

@@ -8,8 +8,10 @@ module Dependabot
   module NpmAndYarn
     class UpdateChecker
       class LibraryDetector
-        def initialize(package_json_file:)
+        def initialize(package_json_file:, credentials:, dependency_files:)
           @package_json_file = package_json_file
+          @credentials = credentials
+          @dependency_files = dependency_files
         end
 
         def library?
@@ -20,7 +22,7 @@ module Dependabot
 
         private
 
-        attr_reader :package_json_file
+        attr_reader :package_json_file, :credentials, :dependency_files
 
         def package_json_may_be_for_library?
           return false unless project_name
@@ -36,7 +38,8 @@ module Dependabot
           return false unless project_description
 
           # Check if the project is listed on npm. If it is, it's a library
-          @project_npm_response ||= Dependabot::RegistryClient.get(url: "https://registry.npmjs.org/#{escaped_project_name}")
+          url = "#{registry.chomp('/')}/#{escaped_project_name}"
+          @project_npm_response ||= Dependabot::RegistryClient.get(url: url)
           return false unless @project_npm_response.status == 200
 
           @project_npm_response.body.force_encoding("UTF-8").encode.
@@ -56,6 +59,16 @@ module Dependabot
         def parsed_package_json
           @parsed_package_json ||= JSON.parse(package_json_file.content)
         end
+
+        def registry
+          NpmAndYarn::UpdateChecker::RegistryFinder.new(
+            dependency: nil,
+            credentials: credentials,
+            npmrc_file: dependency_files.find { |f| f.name.end_with?(".npmrc") },
+            yarnrc_file: dependency_files.find { |f| f.name.end_with?(".yarnrc") },
+            yarnrc_yml_file: dependency_files.find { |f| f.name.end_with?(".yarnrc.yml") }
+          ).registry_from_rc(project_name)
+        end
       end
     end
   end

data/lib/dependabot/npm_and_yarn/update_checker/registry_finder.rb

@@ -13,19 +13,19 @@ module Dependabot
           http://registry.npmjs.org
           https://registry.yarnpkg.com
         ).freeze
-        NPM_AUTH_TOKEN_REGEX =
-          %r{//(?<registry>.*)/:_authToken=(?<token>.*)$}.freeze
-        NPM_GLOBAL_REGISTRY_REGEX =
-          /^registry\s*=\s*['"]?(?<registry>.*?)['"]?$/.freeze
-        YARN_GLOBAL_REGISTRY_REGEX =
-          /^(?:--)?registry\s+['"](?<registry>.*)['"]/.freeze
+        NPM_AUTH_TOKEN_REGEX = %r{//(?<registry>.*)/:_authToken=(?<token>.*)$}
+        NPM_GLOBAL_REGISTRY_REGEX = /^registry\s*=\s*['"]?(?<registry>.*?)['"]?$/
+        YARN_GLOBAL_REGISTRY_REGEX = /^(?:--)?registry\s+((['"](?<registry>.*)['"])|(?<registry>.*))/
+        NPM_SCOPED_REGISTRY_REGEX = /^(?<scope>@[^:]+)\s*:registry\s*=\s*['"]?(?<registry>.*?)['"]?$/
+        YARN_SCOPED_REGISTRY_REGEX = /['"](?<scope>@[^:]+):registry['"]\s((['"](?<registry>.*)['"])|(?<registry>.*))/
 
         def initialize(dependency:, credentials:, npmrc_file: nil,
-                       yarnrc_file: nil)
+                       yarnrc_file: nil, yarnrc_yml_file: nil)
           @dependency = dependency
           @credentials = credentials
           @npmrc_file = npmrc_file
           @yarnrc_file = yarnrc_file
+          @yarnrc_yml_file = yarnrc_yml_file
         end
 
         def registry
@@ -46,15 +46,24 @@ module Dependabot
           end
         end
 
+        def registry_from_rc(dependency_name)
+          return global_registry unless dependency_name.start_with?("@") && dependency_name.include?("/")
+
+          scope = dependency_name.split("/").first
+          scoped_registry(scope)
+        end
+
         private
 
-        attr_reader :dependency, :credentials, :npmrc_file, :yarnrc_file
+        attr_reader :dependency, :credentials, :npmrc_file, :yarnrc_file, :yarnrc_yml_file
 
         def first_registry_with_dependency_details
           @first_registry_with_dependency_details ||=
             known_registries.find do |details|
+              url = "#{details['registry'].gsub(%r{/+$}, '')}/#{escaped_dependency_name}"
+              url = "https://#{url}" unless url.start_with?("http")
               response = Dependabot::RegistryClient.get(
-                url: "https://#{details['registry'].gsub(%r{/+$}, '')}/#{escaped_dependency_name}",
+                url: url,
                 headers: auth_header_for(details["token"])
               )
               response.status < 400 && JSON.parse(response.body)
@@ -64,10 +73,12 @@ module Dependabot
               nil
             end&.fetch("registry")
 
-          @first_registry_with_dependency_details ||= global_registry
+          @first_registry_with_dependency_details ||= global_registry.sub(%r{/+$}, "").sub(%r{^.*?//}, "")
         end
 
         def registry_url
+          return registry if registry.start_with?("http")
+
           protocol =
             if registry_source_url
               registry_source_url.split("://").first
@@ -190,26 +201,56 @@ module Dependabot
           end
         end
 
+        # rubocop:disable Metrics/PerceivedComplexity
         def global_registry
+          return @global_registry if defined? @global_registry
+
           npmrc_file&.content.to_s.scan(NPM_GLOBAL_REGISTRY_REGEX) do
             next if Regexp.last_match[:registry].include?("${")
 
-            registry = Regexp.last_match[:registry].strip.
-                       sub(%r{/+$}, "").
-                       sub(%r{^.*?//}, "")
-            return registry
+            return @global_registry = Regexp.last_match[:registry].strip
           end
 
           yarnrc_file&.content.to_s.scan(YARN_GLOBAL_REGISTRY_REGEX) do
             next if Regexp.last_match[:registry].include?("${")
 
-            registry = Regexp.last_match[:registry].strip.
-                       sub(%r{/+$}, "").
-                       sub(%r{^.*?//}, "")
-            return registry
+            return @global_registry = Regexp.last_match[:registry].strip
+          end
+
+          if parsed_yarnrc_yml&.key?("npmRegistryServer")
+            return @global_registry = parsed_yarnrc_yml["npmRegistryServer"]
+          end
+
+          replaces_base = credentials.find { |cred| cred["type"] == "npm_registry" && cred["replaces-base"] == true }
+          if replaces_base
+            registry = replaces_base["registry"]
+            registry = "https://#{registry}" unless registry.start_with?("http")
+            return @global_registry = registry
+          end
+
+          "https://registry.npmjs.org"
+        end
+        # rubocop:enable Metrics/PerceivedComplexity
+
+        def scoped_registry(scope)
+          npmrc_file&.content.to_s.scan(NPM_SCOPED_REGISTRY_REGEX) do
+            next if Regexp.last_match[:registry].include?("${") || Regexp.last_match[:scope] != scope
+
+            return Regexp.last_match[:registry].strip
+          end
+
+          yarnrc_file&.content.to_s.scan(YARN_SCOPED_REGISTRY_REGEX) do
+            next if Regexp.last_match[:registry].include?("${") || Regexp.last_match[:scope] != scope
+
+            return Regexp.last_match[:registry].strip
+          end
+
+          if parsed_yarnrc_yml
+            yarn_berry_registry = parsed_yarnrc_yml.dig("npmScopes", scope.delete_prefix("@"), "npmRegistryServer")
+            return yarn_berry_registry if yarn_berry_registry
           end
 
-          "registry.npmjs.org"
+          global_registry
         end
 
         # npm registries expect slashes to be escaped
@@ -224,6 +265,13 @@ module Dependabot
 
           sources.find { |s| s[:type] == "registry" }&.fetch(:url)
         end
+
+        def parsed_yarnrc_yml
+          return unless yarnrc_yml_file
+          return @parsed_yarnrc_yml if defined? @parsed_yarnrc_yml
+
+          @parsed_yarnrc_yml = YAML.safe_load(yarnrc_yml_file.content)
+        end
       end
     end
   end

data/lib/dependabot/npm_and_yarn/update_checker/requirements_updater.rb

@@ -13,10 +13,9 @@ module Dependabot
   module NpmAndYarn
     class UpdateChecker
       class RequirementsUpdater
-        VERSION_REGEX = /[0-9]+(?:\.[A-Za-z0-9\-_]+)*/.freeze
-        SEPARATOR = /(?<=[a-zA-Z0-9*])[\s|]+(?![\s|-])/.freeze
-        ALLOWED_UPDATE_STRATEGIES =
-          %i(widen_ranges bump_versions bump_versions_if_necessary).freeze
+        VERSION_REGEX = /[0-9]+(?:\.[A-Za-z0-9\-_]+)*/
+        SEPARATOR = /(?<=[a-zA-Z0-9*])[\s|]+(?![\s|-])/
+        ALLOWED_UPDATE_STRATEGIES = %i(widen_ranges bump_versions bump_versions_if_necessary).freeze
 
         def initialize(requirements:, updated_source:, update_strategy:,
                        latest_resolvable_version:)

data/lib/dependabot/npm_and_yarn/update_checker/subdependency_version_resolver.rb

@@ -59,7 +59,9 @@ module Dependabot
           lockfile_name = Pathname.new(lockfile.name).basename.to_s
           path = Pathname.new(lockfile.name).dirname.to_s
 
-          updated_files = if lockfile.name.end_with?("yarn.lock")
+          updated_files = if lockfile.name.end_with?("yarn.lock") && yarn_berry?(lockfile)
+                            run_yarn_berry_updater(path, lockfile_name)
+                          elsif lockfile.name.end_with?("yarn.lock")
                             run_yarn_updater(path, lockfile_name)
                           else
                             run_npm_updater(path, lockfile_name, lockfile.content)
@@ -68,6 +70,15 @@ module Dependabot
           updated_files.fetch(lockfile_name)
         end
 
+        def yarn_berry?(yarn_lock)
+          return false unless Experiments.enabled?(:yarn_berry)
+
+          yaml = YAML.safe_load(yarn_lock.content)
+          yaml.key?("__metadata")
+        rescue StandardError
+          false
+        end
+
         def version_from_updated_lockfiles(updated_lockfiles)
           updated_files = dependency_files -
                           dependency_files_builder.yarn_locks -
@@ -109,6 +120,17 @@ module Dependabot
           sleep(rand(3.0..10.0)) && retry
         end
 
+        def run_yarn_berry_updater(path, lockfile_name)
+          SharedHelpers.with_git_configured(credentials: credentials) do
+            Dir.chdir(path) do
+              Helpers.run_yarn_commands(
+                "yarn up -R #{dependency.name}"
+              )
+              { lockfile_name => File.read(lockfile_name) }
+            end
+          end
+        end
+
         def run_npm_updater(path, lockfile_name, lockfile_content)
           SharedHelpers.with_git_configured(credentials: credentials) do
             Dir.chdir(path) do

data/lib/dependabot/npm_and_yarn/update_checker/version_resolver.rb

@@ -36,7 +36,7 @@ module Dependabot
             "\s>\s(?<requiring_dep>[^"]+)"\s
             has\s(incorrect|unmet)\speer\sdependency\s
             "(?<required_dep>[^"]+)"
-          /x.freeze
+          /x
 
         # Error message from npm install:
         # react-dom@15.2.0 requires a peer of react@^15.2.0 \
@@ -46,7 +46,7 @@ module Dependabot
             (?<requiring_dep>[^\s]+)\s
             requires\sa\speer\sof\s
             (?<required_dep>.+?)\sbut\snone\sis\sinstalled.
-          /x.freeze
+          /x
 
         # Error message from npm install:
         # npm ERR! Could not resolve dependency:
@@ -59,7 +59,7 @@ module Dependabot
           /
             npm\s(?:WARN|ERR!)\sCould\snot\sresolve\sdependency:\n
             npm\s(?:WARN|ERR!)\speer\s(?<required_dep>\S+@\S+(\s\S+)?)\sfrom\s(?<requiring_dep>\S+@\S+)
-          /x.freeze
+          /x
 
         def initialize(dependency:, credentials:, dependency_files:,
                        latest_allowable_version:, latest_version_finder:)

data/lib/dependabot/npm_and_yarn/update_checker/vulnerability_auditor.rb

@@ -21,6 +21,7 @@ module Dependabot
           @allow_removal = allow_removal
         end
 
+        # rubocop:disable Metrics/MethodLength
         # Finds any dependencies in the `package-lock.json` or `npm-shrinkwrap.json` that have
         # a subdependency on the given dependency that is locked to a vuln version range.
         #
@@ -41,7 +42,10 @@ module Dependabot
         #       dependency on the blocking dependency
         #   * :top_level_ancestors [Array<String>] the names of all top-level dependencies with a transitive
         #     dependency on the dependency
+        #   * :explanation [String] an explanation for why the project failed the vulnerability auditor run
         def audit(dependency:, security_advisories:)
+          Dependabot.logger.info("VulnerabilityAuditor: starting audit")
+
           fix_unavailable = {
             "dependency_name" => dependency.name,
             "fix_available" => false,
@@ -60,7 +64,10 @@ module Dependabot
             # `npm-shrinkwrap.js`, if present, takes precedence over `package-lock.js`.
             # Both files use the same format. See https://bit.ly/3lDIAJV for more.
             lockfile = (dependency_files_builder.shrinkwraps + dependency_files_builder.package_locks).first
-            return fix_unavailable unless lockfile
+            unless lockfile
+              Dependabot.logger.info("VulnerabilityAuditor: missing lockfile")
+              return fix_unavailable
+            end
 
             vuln_versions = security_advisories.map do |a|
               {
@@ -74,25 +81,37 @@ module Dependabot
               function: "npm:vulnerabilityAuditor",
               args: [Dir.pwd, vuln_versions]
             )
-            return fix_unavailable unless viable_audit_result?(audit_result, security_advisories)
 
+            validation_result = validate_audit_result(audit_result, security_advisories)
+            if validation_result != :viable
+              Dependabot.logger.info("VulnerabilityAuditor: audit result not viable: #{validation_result}")
+              fix_unavailable["explanation"] = explain_fix_unavailable(validation_result, dependency)
+              return fix_unavailable
+            end
+
+            Dependabot.logger.info("VulnerabilityAuditor: audit result viable")
             audit_result
           end
         rescue SharedHelpers::HelperSubprocessFailed => e
           log_helper_subprocess_failure(dependency, e)
           fix_unavailable
         end
+        # rubocop:enable Metrics/MethodLength
 
         private
 
         attr_reader :dependency_files, :credentials
 
-        def viable_audit_result?(audit_result, security_advisories)
-          validation_result = validate_audit_result(audit_result, security_advisories)
-          return true if validation_result == :viable
-
-          Dependabot.logger.info("VulnerabilityAuditor: audit result not viable: #{validation_result}")
-          false
+        def explain_fix_unavailable(validation_result, dependency)
+          case validation_result
+          when :fix_unavailable, :dependency_still_vulnerable, :downgrades_dependencies
+            "No patched version available for #{dependency.name}"
+          when :fix_incomplete
+            "The lockfile might be out of sync?"
+          when :vulnerable_dependency_removed
+            "#{dependency.name} was removed in the update. Dependabot is not able to " \
+            "deal with this yet, but you can still upgrade manually."
+          end
         end
 
         def validate_audit_result(audit_result, security_advisories)
@@ -100,6 +119,7 @@ module Dependabot
           return :vulnerable_dependency_removed if !@allow_removal && vulnerable_dependency_removed?(audit_result)
           return :dependency_still_vulnerable if dependency_still_vulnerable?(audit_result, security_advisories)
           return :downgrades_dependencies if downgrades_dependencies?(audit_result)
+          return :fix_incomplete if fix_incomplete?(audit_result)
 
           :viable
         end
@@ -132,6 +152,11 @@ module Dependabot
           current > target
         end
 
+        def fix_incomplete?(audit_result)
+          audit_result["fix_updates"].any? { |update| !update.key?("target_version") } ||
+            audit_result["fix_updates"].empty?
+        end
+
         def log_helper_subprocess_failure(dependency, error)
           # See `Dependabot::SharedHelpers.run_helper_subprocess` for details on error context
           context = error.error_context || {}