dependabot-cargo 0.254.0 → 0.255.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 515ba13e5a17c85e0d1c425689323b57e96207d4fa690e5d7bb512d504c8ca1f
-  data.tar.gz: 92c17353748806f39f0d71aa662d4d9eca9a3446ab4b176e336ece75aa0d5515
+  metadata.gz: 1e8a251c7b5361b8d4aae2d7c1675a144c29c375574e23f2ecd531dbe8c9e6d5
+  data.tar.gz: b5c60f7a5ce3f50998e902524210dbb34790fcc72f098020d067b274948908b0
 SHA512:
-  metadata.gz: 8ff8cfbaeb15b891945b6494f37b6e1b1d022dcc2aabc76ece9c110b231eb528af18a3e08d9d55ee6f12e395ab1e47e46492eb54cf8e079fa139bd4d3eb182b3
-  data.tar.gz: ba1efb9eae5aebb5e49cdbdb4de29ef4d10e3e3e7659f2872faa2e4cf279b7a671db4295276dfeac81bebddec22a9454ca72cabba3fbc54cd5e3162add2c08ce
+  metadata.gz: 11d9b26fccaa6d1a1008c5550fd377fa70b0836f00d34db911eaa76102111663974b325b5dc9d603629b46787b3408340df81c2cba7ad8091d0a3bf970b4a5cd
+  data.tar.gz: 41f248cf390a8b9e864ad6ae77115f49af50bf0c1091de05bcead3f735556128a65ec7bcde9ce28bb669089ee45160b72757c007c5ace6d7258fcb20032dff80

data/lib/dependabot/cargo/file_fetcher.rb

@@ -46,7 +46,7 @@ module Dependabot
 
       sig { override.returns(T::Array[DependencyFile]) }
       def fetch_files
-        fetched_files = []
+        fetched_files = T.let([], T::Array[DependencyFile])
         fetched_files << cargo_toml
         fetched_files << cargo_lock if cargo_lock
         fetched_files << cargo_config if cargo_config
@@ -327,7 +327,10 @@ module Dependabot
       def cargo_config
         return @cargo_config if defined?(@cargo_config)
 
-        @cargo_config = fetch_file_if_present(".cargo/config.toml")
+        @cargo_config = fetch_support_file(".cargo/config.toml")
+
+        @cargo_config ||= fetch_support_file(".cargo/config")
+                          &.tap { |f| f.name = ".cargo/config.toml" }
       end
 
       def rust_toolchain

data/lib/dependabot/cargo/file_parser.rb

@@ -2,6 +2,7 @@
 # frozen_string_literal: true
 
 require "toml-rb"
+require "pathname"
 
 require "dependabot/dependency"
 require "dependabot/file_parsers"
@@ -9,6 +10,7 @@ require "dependabot/file_parsers/base"
 require "dependabot/cargo/requirement"
 require "dependabot/cargo/version"
 require "dependabot/errors"
+require "dependabot/cargo/registry_fetcher"
 
 # Relevant Cargo docs can be found at:
 # - https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -162,8 +164,77 @@ module Dependabot
         raise "Unexpected dependency declaration: #{declaration}" unless declaration.is_a?(Hash)
 
         return git_source_details(declaration) if declaration["git"]
+        return { type: "path" } if declaration["path"]
 
-        { type: "path" } if declaration["path"]
+        registry_source_details(declaration)
+      end
+
+      def registry_source_details(declaration)
+        registry_name = declaration["registry"]
+        return if registry_name.nil?
+
+        index_url = cargo_config_field("registries.#{registry_name}.index")
+        if index_url.nil?
+          raise "Registry index for #{registry_name} must be defined via " \
+                "cargo config"
+        end
+
+        if index_url.start_with?("sparse+")
+          sparse_registry_source_details(registry_name, index_url)
+        else
+          source = Source.from_url(index_url)
+          registry_fetcher = RegistryFetcher.new(
+            source: T.must(source),
+            credentials: credentials
+          )
+
+          {
+            type: "registry",
+            name: registry_name,
+            index: index_url,
+            dl: registry_fetcher.dl,
+            api: registry_fetcher.api
+          }
+        end
+      end
+
+      def sparse_registry_source_details(registry_name, index_url)
+        token = credentials.find do |cred|
+          cred["type"] == "cargo_registry" && cred["registry"] == registry_name
+        end&.fetch("token", nil)
+        # Fallback to configuration in the environment if available
+        token ||= cargo_config_from_env("registries.#{registry_name}.token")
+
+        headers = {}
+        headers["Authorization"] = "Token #{token}" if token
+
+        url = index_url.delete_prefix("sparse+")
+        url << "/" unless url.end_with?("/")
+        url << "config.json"
+        config_json = JSON.parse(RegistryClient.get(url: url, headers: headers).body)
+
+        {
+          type: "registry",
+          name: registry_name,
+          index: index_url,
+          dl: config_json["dl"],
+          api: config_json["api"]
+        }
+      end
+
+      # Looks up dotted key name in cargo config
+      # e.g. "registries.my_registry.index"
+      def cargo_config_field(key_name)
+        cargo_config_from_env(key_name) || cargo_config_from_file(key_name)
+      end
+
+      def cargo_config_from_env(key_name)
+        env_var = "CARGO_#{key_name.upcase.tr('-.', '_')}"
+        ENV.fetch(env_var, nil)
+      end
+
+      def cargo_config_from_file(key_name)
+        parsed_file(cargo_config).dig(*key_name.split("."))
       end
 
       def version_from_lockfile(name, declaration)
@@ -237,6 +308,10 @@ module Dependabot
         @lockfile ||= get_original_file("Cargo.lock")
       end
 
+      def cargo_config
+        @cargo_config ||= get_original_file(".cargo/config.toml")
+      end
+
       def version_class
         Cargo::Version
       end

data/lib/dependabot/cargo/file_updater/lockfile_updater.rb

@@ -7,6 +7,7 @@ require "dependabot/git_commit_checker"
 require "dependabot/cargo/file_updater"
 require "dependabot/cargo/file_updater/manifest_updater"
 require "dependabot/cargo/file_parser"
+require "dependabot/cargo/helpers"
 require "dependabot/shared_helpers"
 module Dependabot
   module Cargo
@@ -33,7 +34,7 @@ module Dependabot
             SharedHelpers.with_git_configured(credentials: credentials) do
               # Shell out to Cargo, which handles everything for us, and does
               # so without doing an install (so it's fast).
-              run_shell_command("cargo update -p #{dependency_spec}", fingerprint: "cargo update -p <dependency_spec>")
+              run_cargo_command("cargo update -p #{dependency_spec}", fingerprint: "cargo update -p <dependency_spec>")
             end
 
             updated_lockfile = File.read("Cargo.lock")
@@ -112,7 +113,6 @@ module Dependabot
             spec += ":#{git_previous_version}" if git_previous_version
           elsif dependency.previous_version
             spec += ":#{dependency.previous_version}"
-            spec = "https://github.com/rust-lang/crates.io-index#" + spec
           end
 
           spec
@@ -138,10 +138,14 @@ module Dependabot
           %(name = "#{dependency.name}"\nversion = "#{dependency.version}")
         end
 
-        def run_shell_command(command, fingerprint:)
+        def run_cargo_command(command, fingerprint:)
           start = Time.now
           command = SharedHelpers.escape_command(command)
-          stdout, process = Open3.capture2e(command)
+          Helpers.setup_credentials_in_environment(credentials)
+          # Pass through any registry tokens supplied via CARGO_REGISTRIES_...
+          # environment variables.
+          env = ENV.select { |key, _value| key.match(/^CARGO_REGISTRIES_/) }
+          stdout, process = Open3.capture2e(env, command)
           time_taken = Time.now - start
 
           # Raise an error with the output from the shell session if Cargo
@@ -187,6 +191,10 @@ module Dependabot
 
           File.write(lockfile.name, lockfile.content)
           File.write(toolchain.name, toolchain.content) if toolchain
+          return unless config
+
+          FileUtils.mkdir_p(File.dirname(config.name))
+          File.write(config.name, config.content)
         end
 
         def write_temporary_manifest_files
@@ -394,6 +402,10 @@ module Dependabot
             dependency_files.find { |f| f.name == "rust-toolchain" }
         end
 
+        def config
+          @config ||= dependency_files.find { |f| f.name == ".cargo/config.toml" }
+        end
+
         def virtual_manifest?(file)
           !file.content.include?("[package]")
         end

data/lib/dependabot/cargo/helpers.rb

@@ -0,0 +1,38 @@
+# typed: true
+# frozen_string_literal: true
+
+require "yaml"
+
+module Dependabot
+  module Cargo
+    module Helpers
+      def self.setup_credentials_in_environment(credentials)
+        credentials.each do |cred|
+          next if cred["type"] != "cargo_registry"
+
+          # If there is a 'token' property, then apply it.
+          # If there is not, it probably means we are running under dependabot-cli which stripped
+          # all tokens. So in that case, we assume that the dependabot proxy will re-inject the
+          # actual correct token, and we just use 'token' as a placeholder at this point.
+          # (We must add these environment variables here, or 'cargo update' will not think it is
+          # configured properly for the private registries.)
+
+          token_env_var = "CARGO_REGISTRIES_#{cred['cargo_registry'].upcase.tr('-', '_')}_TOKEN"
+
+          token = "placeholder_token"
+          if cred["token"].nil?
+            puts "Setting #{token_env_var} to 'placeholder_token' because dependabot-cli proxy will override it anyway"
+          else
+            token = cred["token"]
+            puts "Setting #{token_env_var} to provided token value"
+          end
+
+          ENV[token_env_var] ||= token
+        end
+
+        # And set CARGO_REGISTRY_GLOBAL_CREDENTIAL_PROVIDERS here as well, so Cargo will expect tokens
+        ENV["CARGO_REGISTRY_GLOBAL_CREDENTIAL_PROVIDERS"] ||= "cargo:token"
+      end
+    end
+  end
+end

data/lib/dependabot/cargo/metadata_finder.rb

@@ -10,12 +10,14 @@ module Dependabot
   module Cargo
     class MetadataFinder < Dependabot::MetadataFinders::Base
       SOURCE_KEYS = %w(repository homepage documentation).freeze
+      CRATES_IO_API = "https://crates.io/api/v1/crates"
 
       private
 
       def look_up_source
         case new_source_type
         when "default" then find_source_from_crates_listing
+        when "registry" then find_source_from_crates_listing
         when "git" then find_source_from_git_url
         else raise "Unexpected source type: #{new_source_type}"
         end
@@ -44,9 +46,47 @@ module Dependabot
       def crates_listing
         return @crates_listing unless @crates_listing.nil?
 
-        response = Dependabot::RegistryClient.get(url: "https://crates.io/api/v1/crates/#{dependency.name}")
+        info = dependency.requirements.filter_map { |r| r[:source] }.first
+        index = (info && info[:index]) || CRATES_IO_API
+
+        # Default request headers
+        hdrs = { "User-Agent" => "Dependabot (dependabot.com)" }
+
+        if index != CRATES_IO_API
+          # Add authentication headers if credentials are present for this registry
+          credentials.find { |cred| cred["type"] == "cargo_registry" && cred["registry"] == info[:name] }&.tap do |cred|
+            hdrs["Authorization"] = "Token #{cred['token']}"
+          end
+        end
+
+        url = metadata_fetch_url(dependency, index)
+
+        response = Excon.get(
+          url,
+          idempotent: true,
+          **SharedHelpers.excon_defaults(headers: hdrs)
+        )
+
         @crates_listing = JSON.parse(response.body)
       end
+
+      def metadata_fetch_url(dependency, index)
+        return "#{index}/#{dependency.name}" if index == CRATES_IO_API
+
+        # Determine cargo's index file path for the dependency
+        index = index.delete_prefix("sparse+")
+        name_length = dependency.name.length
+        dependency_path = case name_length
+                          when 1, 2
+                            "#{name_length}/#{dependency.name}"
+                          when 3
+                            "#{name_length}/#{dependency.name[0..1]}/#{dependency.name}"
+                          else
+                            "#{dependency.name[0..1]}/#{dependency.name[2..3]}/#{dependency.name}"
+                          end
+
+        "#{index}#{'/' unless index.end_with?('/')}#{dependency_path}"
+      end
     end
   end
 end

data/lib/dependabot/cargo/registry_fetcher.rb

@@ -0,0 +1,42 @@
+# typed: true
+# frozen_string_literal: true
+
+require "dependabot/file_fetchers"
+require "dependabot/file_fetchers/base"
+
+module Dependabot
+  module Cargo
+    class RegistryFetcher < Dependabot::FileFetchers::Base
+      def self.required_files_in?(filenames)
+        filenames.include?("config.json")
+      end
+
+      def self.required_files_message
+        "Repo must contain a config.json"
+      end
+
+      def dl
+        parsed_config_json["dl"].chomp("/")
+      end
+
+      def api
+        parsed_config_json["api"].chomp("/")
+      end
+
+      private
+
+      def fetch_files
+        fetched_files = []
+        fetched_files << config_json
+      end
+
+      def parsed_config_json
+        @parsed_config_json ||= JSON.parse(config_json.content)
+      end
+
+      def config_json
+        @config_json ||= fetch_file_from_host("config.json")
+      end
+    end
+  end
+end

data/lib/dependabot/cargo/update_checker/latest_version_finder.rb

@@ -13,6 +13,8 @@ module Dependabot
       class LatestVersionFinder
         extend T::Sig
 
+        CRATES_IO_API = "https://crates.io/api/v1/crates"
+
         def initialize(dependency:, dependency_files:, credentials:,
                        ignored_versions:, raise_on_ignored: false,
                        security_advisories:)
@@ -101,8 +103,64 @@ module Dependabot
         def crates_listing
           return @crates_listing unless @crates_listing.nil?
 
-          response = Dependabot::RegistryClient.get(url: "https://crates.io/api/v1/crates/#{dependency.name}")
+          info = dependency.requirements.filter_map { |r| r[:source] }.first
+          index = (info && info[:index]) || CRATES_IO_API
+
+          # Default request headers
+          hdrs = { "User-Agent" => "Dependabot (dependabot.com)" }
+
+          if index != CRATES_IO_API
+            # Add authentication headers if credentials are present for this registry
+            registry_creds = credentials.find do |cred|
+              cred["type"] == "cargo_registry" && cred["registry"] == info[:name]
+            end
+
+            unless registry_creds.nil?
+              # If there is a credential, but no actual token at this point, it means that dependabot-cli
+              # stripped the token from our credentials. In this case, the dependabot proxy will reintroduce
+              # the correct token, so we just use 'placeholder_token' as the token value.
+              token = registry_creds["token"] || "placeholder_token"
+
+              hdrs["Authorization"] = token
+            end
+          end
+
+          url = metadata_fetch_url(dependency, index)
+
+          # B4PR
+          puts "Calling #{url} to fetch metadata for #{dependency.name} from #{index}"
+
+          response = Excon.get(
+            url,
+            idempotent: true,
+            **SharedHelpers.excon_defaults(headers: hdrs)
+          )
+
           @crates_listing = JSON.parse(response.body)
+
+          # B4PR
+          puts "Fetched metadata for #{dependency.name} from #{index} successfully"
+          puts response.body
+
+          @crates_listing
+        end
+
+        def metadata_fetch_url(dependency, index)
+          return "#{index}/#{dependency.name}" if index == CRATES_IO_API
+
+          # Determine cargo's index file path for the dependency
+          index = index.delete_prefix("sparse+")
+          name_length = dependency.name.length
+          dependency_path = case name_length
+                            when 1, 2
+                              "#{name_length}/#{dependency.name}"
+                            when 3
+                              "#{name_length}/#{dependency.name[0..1]}/#{dependency.name}"
+                            else
+                              "#{dependency.name[0..1]}/#{dependency.name[2..3]}/#{dependency.name}"
+                            end
+
+          "#{index}#{'/' unless index.end_with?('/')}#{dependency_path}"
         end
 
         def wants_prerelease?

data/lib/dependabot/cargo/update_checker/version_resolver.rb

@@ -4,6 +4,7 @@
 require "toml-rb"
 require "open3"
 require "dependabot/shared_helpers"
+require "dependabot/cargo/helpers"
 require "dependabot/cargo/update_checker"
 require "dependabot/cargo/file_parser"
 require "dependabot/cargo/version"
@@ -31,6 +32,8 @@ module Dependabot
           return @latest_resolvable_version if defined?(@latest_resolvable_version)
 
           @latest_resolvable_version = fetch_latest_resolvable_version
+          rescue Dependabot::SharedHelpers::HelperSubprocessFailed => e
+            raise Dependabot::DependencyFileNotResolvable, e.message
         end
 
         private
@@ -127,7 +130,6 @@ module Dependabot
             spec += ":#{git_dependency_version}" if git_dependency_version
           elsif dependency.version
             spec += ":#{dependency.version}"
-            spec = "https://github.com/rust-lang/crates.io-index#" + spec
           end
 
           spec
@@ -137,15 +139,20 @@ module Dependabot
         # so without doing an install (so it's fast).
         def run_cargo_update_command
           run_cargo_command(
-            "cargo update -p #{dependency_spec} --verbose",
-            fingerprint: "cargo update -p <dependency_spec> --verbose"
+            "cargo update -p #{dependency_spec} -vv",
+            fingerprint: "cargo update -p <dependency_spec> -vv"
           )
         end
 
         def run_cargo_command(command, fingerprint: nil)
           start = Time.now
           command = SharedHelpers.escape_command(command)
-          stdout, process = Open3.capture2e(command)
+          Helpers.setup_credentials_in_environment(credentials)
+          # Pass through any registry tokens supplied via CARGO_REGISTRIES_...
+          # environment variables, and also any CARGO_REGISTRY_... configuration.
+          env = ENV.select { |key, _value| key.match(/^(CARGO_REGISTRY|CARGO_REGISTRIES)_/) }
+
+          stdout, process = Open3.capture2e(env, command)
           time_taken = Time.now - start
 
           # Raise an error with the output from the shell session if Cargo
@@ -168,6 +175,10 @@ module Dependabot
 
           File.write(lockfile.name, lockfile.content) if lockfile
           File.write(toolchain.name, toolchain.content) if toolchain
+          return unless config
+
+          FileUtils.mkdir_p(File.dirname(config.name))
+          File.write(config.name, config.content)
         end
 
         def check_rust_workspace_root
@@ -425,10 +436,14 @@ module Dependabot
         end
 
         def toolchain
-          @toolchain ||= prepared_dependency_files
+          @toolchain ||= original_dependency_files
                          .find { |f| f.name == "rust-toolchain" }
         end
 
+        def config
+          @config ||= original_dependency_files.find { |f| f.name == ".cargo/config.toml" }
+        end
+
         def git_dependency?
           GitCommitChecker.new(
             dependency: dependency,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dependabot-cargo
 version: !ruby/object:Gem::Version
-  version: 0.254.0
+  version: 0.255.0
 platform: ruby
 authors:
 - Dependabot
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-04-24 00:00:00.000000000 Z
+date: 2024-05-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dependabot-common
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.254.0
+        version: 0.255.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.254.0
+        version: 0.255.0
 - !ruby/object:Gem::Dependency
   name: debug
   requirement: !ruby/object:Gem::Requirement
@@ -164,6 +164,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.8.1
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.22.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.22.0
 - !ruby/object:Gem::Dependency
   name: turbo_tests
   requirement: !ruby/object:Gem::Requirement
@@ -234,7 +248,9 @@ files:
 - lib/dependabot/cargo/file_updater.rb
 - lib/dependabot/cargo/file_updater/lockfile_updater.rb
 - lib/dependabot/cargo/file_updater/manifest_updater.rb
+- lib/dependabot/cargo/helpers.rb
 - lib/dependabot/cargo/metadata_finder.rb
+- lib/dependabot/cargo/registry_fetcher.rb
 - lib/dependabot/cargo/requirement.rb
 - lib/dependabot/cargo/update_checker.rb
 - lib/dependabot/cargo/update_checker/file_preparer.rb
@@ -247,7 +263,7 @@ licenses:
 - Nonstandard
 metadata:
   bug_tracker_uri: https://github.com/dependabot/dependabot-core/issues
-  changelog_uri: https://github.com/dependabot/dependabot-core/releases/tag/v0.254.0
+  changelog_uri: https://github.com/dependabot/dependabot-core/releases/tag/v0.255.0
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -263,7 +279,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 3.1.0
 requirements: []
-rubygems_version: 3.3.26
+rubygems_version: 3.5.9
 signing_key: 
 specification_version: 4
 summary: Provides Dependabot support for Rust (Cargo)