dependabot-npm_and_yarn 0.235.0 → 0.237.0

data/lib/dependabot/npm_and_yarn/file_fetcher.rb

@@ -2,6 +2,7 @@
 # frozen_string_literal: true
 
 require "json"
+require "sorbet-runtime"
 require "dependabot/experiments"
 require "dependabot/logger"
 require "dependabot/file_fetchers"
@@ -14,6 +15,9 @@ require "dependabot/npm_and_yarn/file_parser/lockfile_parser"
 module Dependabot
   module NpmAndYarn
     class FileFetcher < Dependabot::FileFetchers::Base # rubocop:disable Metrics/ClassLength
+      extend T::Sig
+      extend T::Helpers
+
       require_relative "file_fetcher/path_dependency_builder"
 
       # Npm always prefixes file paths in the lockfile "version" with "file:"
@@ -56,7 +60,7 @@ module Dependabot
       def ecosystem_versions
         package_managers = {}
 
-        package_managers["npm"] = Helpers.npm_version_numeric(package_lock.content) if package_lock
+        package_managers["npm"] = npm_version if package_lock
         package_managers["yarn"] = yarn_version if yarn_version
         package_managers["pnpm"] = pnpm_version if pnpm_version
         package_managers["shrinkwrap"] = 1 if shrinkwrap
@@ -67,8 +71,7 @@ module Dependabot
         }
       end
 
-      private
-
+      sig { override.returns(T::Array[DependencyFile]) }
       def fetch_files
         fetched_files = []
         fetched_files << package_json
@@ -82,6 +85,12 @@ module Dependabot
         fetched_files.uniq
       end
 
+      private
+
+      def recurse_submodules_when_cloning?
+        true
+      end
+
       def npm_files
         fetched_npm_files = []
         fetched_npm_files << package_lock if package_lock
@@ -161,10 +170,14 @@ module Dependabot
         @inferred_npmrc = nil
       end
 
+      def npm_version
+        Helpers.npm_version_numeric(package_lock.content)
+      end
+
       def yarn_version
         return @yarn_version if defined?(@yarn_version)
 
-        @yarn_version = package_manager.locked_version("yarn") || guess_yarn_version
+        @yarn_version = package_manager.requested_version("yarn") || guess_yarn_version
       end
 
       def guess_yarn_version
@@ -176,13 +189,19 @@ module Dependabot
       def pnpm_version
         return @pnpm_version if defined?(@pnpm_version)
 
-        @pnpm_version = package_manager.locked_version("pnpm") || guess_pnpm_version
+        version = package_manager.requested_version("pnpm") || guess_pnpm_version
+
+        if version && Version.new(version.to_s) < Version.new("7")
+          raise ToolVersionNotSupported.new("PNPM", version.to_s, "7.*, 8.*")
+        end
+
+        @pnpm_version = version
       end
 
       def guess_pnpm_version
         return unless pnpm_lock
 
-        Helpers.pnpm_major_version
+        Helpers.pnpm_version_numeric(pnpm_lock)
       end
 
       def package_manager

data/lib/dependabot/npm_and_yarn/file_parser/json_lock.rb

@@ -7,7 +7,7 @@ require "dependabot/npm_and_yarn/helpers"
 
 module Dependabot
   module NpmAndYarn
-    class FileParser
+    class FileParser < Dependabot::FileParsers::Base
       class JsonLock
         def initialize(dependency_file)
           @dependency_file = dependency_file

data/lib/dependabot/npm_and_yarn/file_parser/lockfile_parser.rb

@@ -7,7 +7,7 @@ require "dependabot/npm_and_yarn/helpers"
 
 module Dependabot
   module NpmAndYarn
-    class FileParser
+    class FileParser < Dependabot::FileParsers::Base
       class LockfileParser
         require "dependabot/npm_and_yarn/file_parser/yarn_lock"
         require "dependabot/npm_and_yarn/file_parser/pnpm_lock"

data/lib/dependabot/npm_and_yarn/file_parser/pnpm_lock.rb

@@ -5,7 +5,7 @@ require "dependabot/errors"
 
 module Dependabot
   module NpmAndYarn
-    class FileParser
+    class FileParser < Dependabot::FileParsers::Base
       class PnpmLock
         def initialize(dependency_file)
           @dependency_file = dependency_file

data/lib/dependabot/npm_and_yarn/file_parser/yarn_lock.rb

@@ -7,7 +7,7 @@ require "dependabot/npm_and_yarn/native_helpers"
 
 module Dependabot
   module NpmAndYarn
-    class FileParser
+    class FileParser < Dependabot::FileParsers::Base
       class YarnLock
         def initialize(dependency_file)
           @dependency_file = dependency_file

data/lib/dependabot/npm_and_yarn/file_parser.rb

@@ -11,6 +11,7 @@ require "dependabot/npm_and_yarn/helpers"
 require "dependabot/npm_and_yarn/native_helpers"
 require "dependabot/npm_and_yarn/version"
 require "dependabot/npm_and_yarn/requirement"
+require "dependabot/npm_and_yarn/registry_parser"
 require "dependabot/git_metadata_fetcher"
 require "dependabot/git_commit_checker"
 require "dependabot/errors"
@@ -34,6 +35,15 @@ module Dependabot
         )?$
       }ix
 
+      def self.each_dependency(json)
+        DEPENDENCY_TYPES.each do |type|
+          deps = json[type] || {}
+          deps.each do |name, requirement|
+            yield name, requirement, type
+          end
+        end
+      end
+
       def parse
         dependency_set = DependencySet.new
         dependency_set += manifest_dependencies
@@ -41,11 +51,16 @@ module Dependabot
 
         dependencies = Helpers.dependencies_with_all_versions_metadata(dependency_set)
 
-        # TODO: Currently, Dependabot can't handle dependencies that have both
-        # a git source *and* a non-git source. Fix that!
         dependencies.reject do |dep|
-          git_reqs =
-            dep.requirements.select { |r| r.dig(:source, :type) == "git" }
+          reqs = dep.requirements
+
+          # Ignore dependencies defined in support files, since we don't want PRs for those
+          support_reqs = reqs.select { |r| support_package_files.any? { |f| f.name == r[:file] } }
+          next true if support_reqs.any?
+
+          # TODO: Currently, Dependabot can't handle dependencies that have both
+          # a git source *and* a non-git source. Fix that!
+          git_reqs = reqs.select { |r| r.dig(:source, :type) == "git" }
           next false if git_reqs.none?
           next true if git_reqs.map { |r| r.fetch(:source) }.uniq.count > 1
 
@@ -59,22 +74,24 @@ module Dependabot
         dependency_set = DependencySet.new
 
         package_files.each do |file|
+          json = JSON.parse(file.content)
+
           # TODO: Currently, Dependabot can't handle flat dependency files
           # (and will error at the FileUpdater stage, because the
           # UpdateChecker doesn't take account of flat resolution).
-          next if JSON.parse(file.content)["flat"]
-
-          DEPENDENCY_TYPES.each do |type|
-            deps = JSON.parse(file.content)[type] || {}
-            deps.each do |name, requirement|
-              next unless requirement.is_a?(String)
-
-              requirement = "*" if requirement == ""
-              dep = build_dependency(
-                file: file, type: type, name: name, requirement: requirement
-              )
-              dependency_set << dep if dep
-            end
+          next if json["flat"]
+
+          self.class.each_dependency(json) do |name, requirement, type|
+            next unless requirement.is_a?(String)
+
+            # Skip dependencies using Yarn workspace cross-references as requirements
+            next if requirement.start_with?("workspace:")
+
+            requirement = "*" if requirement == ""
+            dep = build_dependency(
+              file: file, type: type, name: name, requirement: requirement
+            )
+            dependency_set << dep if dep
           end
         end
 
@@ -253,7 +270,10 @@ module Dependabot
         return unless resolved_url.start_with?("http")
         return if resolved_url.match?(/(?<!pkg\.)github/)
 
-        registry_source_for(resolved_url, name)
+        RegistryParser.new(
+          resolved_url: resolved_url,
+          credentials: credentials
+        ).registry_source_for(name)
       end
 
       def requirement_for(requirement)
@@ -286,69 +306,23 @@ module Dependabot
         }
       end
 
-      def registry_source_for(resolved_url, name)
-        url =
-          if resolved_url.include?("/~/")
-            # Gemfury format
-            resolved_url.split("/~/").first
-          elsif resolved_url.include?("/#{name}/-/#{name}")
-            # MyGet / Bintray format
-            resolved_url.split("/#{name}/-/#{name}").first
-                        .gsub("dl.bintray.com//", "api.bintray.com/npm/").
-              # GitLab format
-              gsub(%r{\/projects\/\d+}, "")
-          elsif resolved_url.include?("/#{name}/-/#{name.split('/').last}")
-            # Sonatype Nexus / Artifactory JFrog format
-            resolved_url.split("/#{name}/-/#{name.split('/').last}").first
-          elsif (cred_url = url_for_relevant_cred(resolved_url)) then cred_url
-          else
-            resolved_url.split("/")[0..2].join("/")
-          end
-
-        { type: "registry", url: url }
+      def support_package_files
+        @support_package_files ||= sub_package_files.select(&:support_file?)
       end
 
-      def url_for_relevant_cred(resolved_url)
-        resolved_url_host = URI(resolved_url).host
-
-        credential_matching_url =
-          credentials
-          .select { |cred| cred["type"] == "npm_registry" }
-          .sort_by { |cred| cred["registry"].length }
-          .find do |details|
-            next true if resolved_url_host == details["registry"]
-
-            uri = if details["registry"]&.include?("://")
-                    URI(details["registry"])
-                  else
-                    URI("https://#{details['registry']}")
-                  end
-            resolved_url_host == uri.host && resolved_url.include?(details["registry"])
-          end
-
-        return unless credential_matching_url
-
-        # Trim the resolved URL so that it ends at the same point as the
-        # credential registry
-        reg = credential_matching_url["registry"]
-        resolved_url.gsub(/#{Regexp.quote(reg)}.*/, "") + reg
+      def sub_package_files
+        @sub_package_files ||=
+          dependency_files.select { |f| f.name.end_with?("package.json") }
+                          .reject { |f| f.name == "package.json" }
+                          .reject { |f| f.name.include?("node_modules/") }
       end
 
       def package_files
         @package_files ||=
-          begin
-            sub_packages =
-              dependency_files
-              .select { |f| f.name.end_with?("package.json") }
-              .reject { |f| f.name == "package.json" }
-              .reject { |f| f.name.include?("node_modules/") }
-              .reject(&:support_file?)
-
-            [
-              dependency_files.find { |f| f.name == "package.json" },
-              *sub_packages
-            ].compact
-          end
+          [
+            dependency_files.find { |f| f.name == "package.json" },
+            *sub_package_files
+          ].compact
       end
 
       def version_class

data/lib/dependabot/npm_and_yarn/file_updater/npm_lockfile_updater.rb

@@ -13,7 +13,7 @@ require "dependabot/shared_helpers"
 # rubocop:disable Metrics/ClassLength
 module Dependabot
   module NpmAndYarn
-    class FileUpdater
+    class FileUpdater < Dependabot::FileUpdaters::Base
       class NpmLockfileUpdater
         require_relative "npmrc_builder"
         require_relative "package_json_updater"
@@ -562,13 +562,11 @@ module Dependabot
           return content if git_dependencies_to_lock.empty?
 
           json = JSON.parse(content)
-          NpmAndYarn::FileParser::DEPENDENCY_TYPES.each do |type|
-            json.fetch(type, {}).each do |nm, _|
-              updated_version = git_dependencies_to_lock.dig(nm, :version)
-              next unless updated_version
+          NpmAndYarn::FileParser.each_dependency(json) do |nm, _, type|
+            updated_version = git_dependencies_to_lock.dig(nm, :version)
+            next unless updated_version
 
-              json[type][nm] = git_dependencies_to_lock[nm][:version]
-            end
+            json[type][nm] = git_dependencies_to_lock[nm][:version]
           end
 
           indent = detect_indentation(content)
@@ -605,12 +603,10 @@ module Dependabot
         def lock_deps_with_latest_reqs(content)
           json = JSON.parse(content)
 
-          NpmAndYarn::FileParser::DEPENDENCY_TYPES.each do |type|
-            json.fetch(type, {}).each do |nm, requirement|
-              next unless requirement == "latest"
+          NpmAndYarn::FileParser.each_dependency(json) do |nm, requirement, type|
+            next unless requirement == "latest"
 
-              json[type][nm] = "*"
-            end
+            json[type][nm] = "*"
           end
 
           indent = detect_indentation(content)
@@ -721,17 +717,15 @@ module Dependabot
 
           dependency_names_to_restore = (dependencies.map(&:name) + git_dependencies_to_lock.keys).uniq
 
-          NpmAndYarn::FileParser::DEPENDENCY_TYPES.each do |type|
-            parsed_package_json.fetch(type, {}).each do |dependency_name, original_requirement|
-              next unless dependency_names_to_restore.include?(dependency_name)
+          NpmAndYarn::FileParser.each_dependency(parsed_package_json) do |dependency_name, original_requirement, type|
+            next unless dependency_names_to_restore.include?(dependency_name)
 
-              locked_requirement = parsed_updated_lockfile_content.dig("packages", "", type, dependency_name)
-              next unless locked_requirement
+            locked_requirement = parsed_updated_lockfile_content.dig("packages", "", type, dependency_name)
+            next unless locked_requirement
 
-              locked_req = %("#{dependency_name}": "#{locked_requirement}")
-              original_req = %("#{dependency_name}": "#{original_requirement}")
-              updated_lockfile_content = updated_lockfile_content.gsub(locked_req, original_req)
-            end
+            locked_req = %("#{dependency_name}": "#{locked_requirement}")
+            original_req = %("#{dependency_name}": "#{original_requirement}")
+            updated_lockfile_content = updated_lockfile_content.gsub(locked_req, original_req)
           end
 
           updated_lockfile_content

data/lib/dependabot/npm_and_yarn/file_updater/npmrc_builder.rb

@@ -5,7 +5,7 @@ require "dependabot/npm_and_yarn/file_updater"
 
 module Dependabot
   module NpmAndYarn
-    class FileUpdater
+    class FileUpdater < Dependabot::FileUpdaters::Base
       # Build a .npmrc file from the lockfile content, credentials, and any
       # committed .npmrc
       # We should refactor this to use UpdateChecker::RegistryFinder
@@ -140,7 +140,10 @@ module Dependabot
             @dependency_urls = dependencies.map do |dependency|
               UpdateChecker::RegistryFinder.new(
                 dependency: dependency,
-                credentials: credentials
+                credentials: credentials,
+                npmrc_file: npmrc_file,
+                yarnrc_file: yarnrc_file,
+                yarnrc_yml_file: yarnrc_yml_file
               ).dependency_url
             end
             return @dependency_urls
@@ -309,6 +312,11 @@ module Dependabot
                            .find { |f| f.name.end_with?(".yarnrc") }
         end
 
+        def yarnrc_yml_file
+          @yarnrc_yml_file ||= dependency_files
+                               .find { |f| f.name.end_with?(".yarnrc.yml") }
+        end
+
         def yarn_lock
           @yarn_lock ||= dependency_files.find { |f| f.name == "yarn.lock" }
         end

data/lib/dependabot/npm_and_yarn/file_updater/package_json_preparer.rb

@@ -6,7 +6,7 @@ require "dependabot/npm_and_yarn/file_parser"
 
 module Dependabot
   module NpmAndYarn
-    class FileUpdater
+    class FileUpdater < Dependabot::FileUpdaters::Base
       class PackageJsonPreparer
         def initialize(package_json_content:)
           @package_json_content = package_json_content
@@ -72,14 +72,12 @@ module Dependabot
 
           @git_ssh_requirements_to_swap = []
 
-          NpmAndYarn::FileParser::DEPENDENCY_TYPES.each do |t|
-            JSON.parse(package_json_content).fetch(t, {}).each do |_, req|
-              next unless req.is_a?(String)
-              next unless req.start_with?("git+ssh:")
+          NpmAndYarn::FileParser.each_dependency(JSON.parse(package_json_content)) do |_, req, _t|
+            next unless req.is_a?(String)
+            next unless req.start_with?("git+ssh:")
 
-              req = req.split("#").first
-              @git_ssh_requirements_to_swap << req
-            end
+            req = req.split("#").first
+            @git_ssh_requirements_to_swap << req
           end
 
           @git_ssh_requirements_to_swap

data/lib/dependabot/npm_and_yarn/file_updater/package_json_updater.rb

@@ -5,7 +5,7 @@ require "dependabot/npm_and_yarn/file_updater"
 
 module Dependabot
   module NpmAndYarn
-    class FileUpdater
+    class FileUpdater < Dependabot::FileUpdaters::Base
       class PackageJsonUpdater
         def initialize(package_json:, dependencies:)
           @package_json = package_json

data/lib/dependabot/npm_and_yarn/file_updater/pnpm_lockfile_updater.rb

@@ -3,11 +3,12 @@
 
 require "dependabot/npm_and_yarn/helpers"
 require "dependabot/npm_and_yarn/update_checker/registry_finder"
+require "dependabot/npm_and_yarn/registry_parser"
 require "dependabot/shared_helpers"
 
 module Dependabot
   module NpmAndYarn
-    class FileUpdater
+    class FileUpdater < Dependabot::FileUpdaters::Base
       class PnpmLockfileUpdater
         require_relative "npmrc_builder"
         require_relative "package_json_updater"
@@ -35,7 +36,8 @@ module Dependabot
 
         IRRESOLVABLE_PACKAGE = "ERR_PNPM_NO_MATCHING_VERSION"
         INVALID_REQUIREMENT = "ERR_PNPM_SPEC_NOT_SUPPORTED_BY_ANY_RESOLVER"
-        MISSING_PACKAGE = /(?<package_req>.*?) is not in the npm registry, or you have no permission to fetch it/
+        UNREACHABLE_GIT = %r{ERR_PNPM_FETCH_404[ [^:print:]]+GET (?<url>https://codeload\.github\.com/[^/]+/[^/]+)/}
+        MISSING_PACKAGE = /ERR_PNPM_FETCH_404[ [^:print:]]+GET (?<dependency_url>.*): Not Found - 404/
 
         def run_pnpm_update(pnpm_lock:)
           SharedHelpers.in_a_temporary_repo_directory(base_dir, repo_contents_path) do
@@ -87,11 +89,18 @@ module Dependabot
             raise_resolvability_error(error_message, pnpm_lock)
           end
 
+          if error_message.match?(UNREACHABLE_GIT)
+            dependency_url = error_message.match(UNREACHABLE_GIT).named_captures.fetch("url")
+
+            raise Dependabot::GitDependenciesNotReachable, dependency_url
+          end
+
           raise unless error_message.match?(MISSING_PACKAGE)
 
-          package_name = error_message.match(MISSING_PACKAGE)
-                                      .named_captures["package_req"]
-                                      .split(/(?<=\w)\@/).first
+          dependency_url = error_message.match(MISSING_PACKAGE)
+                                        .named_captures["dependency_url"]
+
+          package_name = RegistryParser.new(resolved_url: dependency_url, credentials: credentials).dependency_name
           raise_missing_package_error(package_name, error_message, pnpm_lock)
         end
 

data/lib/dependabot/npm_and_yarn/file_updater/yarn_lockfile_updater.rb

@@ -14,7 +14,7 @@ require "dependabot/errors"
 # rubocop:disable Metrics/ClassLength
 module Dependabot
   module NpmAndYarn
-    class FileUpdater
+    class FileUpdater < Dependabot::FileUpdaters::Base
       class YarnLockfileUpdater
         require_relative "npmrc_builder"
         require_relative "package_json_updater"

data/lib/dependabot/npm_and_yarn/helpers.rb

@@ -25,6 +25,16 @@ module Dependabot
         end
       end
 
+      # Mapping from lockfile versions to PNPM versions is at
+      # https://github.com/pnpm/spec/tree/274ff02de23376ad59773a9f25ecfedd03a41f64/lockfile, but simplify it for now.
+      def self.pnpm_version_numeric(pnpm_lock)
+        if pnpm_lockfile_version(pnpm_lock).to_f >= 5.4
+          8
+        else
+          6
+        end
+      end
+
       def self.fetch_yarnrc_yml_value(key, default_value)
         if File.exist?(".yarnrc.yml") && (yarnrc = YAML.load_file(".yarnrc.yml"))
           yarnrc.fetch(key, default_value)
@@ -44,20 +54,11 @@ module Dependabot
         @yarn_major_version ||= fetch_yarn_major_version
       end
 
-      def self.pnpm_major_version
-        @pnpm_major_version ||= fetch_pnpm_major_version
-      end
-
       def self.fetch_yarn_major_version
         output = SharedHelpers.run_shell_command("yarn --version")
         Version.new(output).major
       end
 
-      def self.fetch_pnpm_major_version
-        output = SharedHelpers.run_shell_command("pnpm --version")
-        Version.new(output).major
-      end
-
       def self.yarn_zero_install?
         File.exist?(".pnp.cjs")
       end
@@ -122,6 +123,10 @@ module Dependabot
         SharedHelpers.run_shell_command(command, fingerprint: fingerprint)
       end
 
+      def self.pnpm_lockfile_version(pnpm_lock)
+        pnpm_lock.content.match(/^lockfileVersion: ['"]?(?<version>[\d.]+)/)[:version]
+      end
+
       def self.dependencies_with_all_versions_metadata(dependency_set)
         dependency_set.dependencies.map do |dependency|
           dependency.metadata[:all_versions] = dependency_set.all_versions_for_name(dependency.name)

data/lib/dependabot/npm_and_yarn/package_manager.rb

@@ -8,11 +8,11 @@ module Dependabot
         @package_json = package_json
       end
 
-      def locked_version(name)
-        locked = @package_json.fetch("packageManager", nil)
-        return unless locked
+      def requested_version(name)
+        version = @package_json.fetch("packageManager", nil)
+        return unless version
 
-        version_match = locked.match(/#{name}@(?<version>\d+.\d+.\d+)/)
+        version_match = version.match(/#{name}@(?<version>\d+.\d+.\d+)/)
         version_match&.named_captures&.fetch("version", nil)
       end
     end

data/lib/dependabot/npm_and_yarn/registry_parser.rb

@@ -0,0 +1,75 @@
+# typed: false
+# frozen_string_literal: true
+
+module Dependabot
+  module NpmAndYarn
+    class RegistryParser
+      def initialize(resolved_url:, credentials:)
+        @resolved_url = resolved_url
+        @credentials = credentials
+      end
+
+      def registry_source_for(name)
+        url =
+          if resolved_url.include?("/~/")
+            # Gemfury format
+            resolved_url.split("/~/").first
+          elsif resolved_url.include?("/#{name}/-/#{name}")
+            # MyGet / Bintray format
+            resolved_url.split("/#{name}/-/#{name}").first
+                        .gsub("dl.bintray.com//", "api.bintray.com/npm/").
+              # GitLab format
+              gsub(%r{\/projects\/\d+}, "")
+          elsif resolved_url.include?("/#{name}/-/#{name.split('/').last}")
+            # Sonatype Nexus / Artifactory JFrog format
+            resolved_url.split("/#{name}/-/#{name.split('/').last}").first
+          elsif (cred_url = url_for_relevant_cred) then cred_url
+          else
+            resolved_url.split("/")[0..2].join("/")
+          end
+
+        { type: "registry", url: url }
+      end
+
+      def dependency_name
+        url_base = if resolved_url.include?("/-/")
+                     resolved_url.split("/-/").first
+                   else
+                     resolved_url
+                   end
+
+        url_base.split("/")[3..-1].join("/").gsub("%2F", "/")
+      end
+
+      private
+
+      attr_reader :resolved_url, :credentials
+
+      def url_for_relevant_cred
+        resolved_url_host = URI(resolved_url).host
+
+        credential_matching_url =
+          credentials
+          .select { |cred| cred["type"] == "npm_registry" }
+          .sort_by { |cred| cred["registry"].length }
+          .find do |details|
+            next true if resolved_url_host == details["registry"]
+
+            uri = if details["registry"]&.include?("://")
+                    URI(details["registry"])
+                  else
+                    URI("https://#{details['registry']}")
+                  end
+            resolved_url_host == uri.host && resolved_url.include?(details["registry"])
+          end
+
+        return unless credential_matching_url
+
+        # Trim the resolved URL so that it ends at the same point as the
+        # credential registry
+        reg = credential_matching_url["registry"]
+        resolved_url.gsub(/#{Regexp.quote(reg)}.*/, "") + reg
+      end
+    end
+  end
+end