dependabot-maven 0.317.0 → 0.318.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 21397e0916880a257f48425d1d22031d2f428e3c126be658b2b6081984da8404
-  data.tar.gz: 425286545ad1da908980dcfb20a7c8313c89862771e6af6c07d0b06ec1ad900c
+  metadata.gz: 7dddaacd9f55be6eed5e719f3e4fb005d0e09a1cbcaedaa434a74c3802e8ba4f
+  data.tar.gz: f7080365f342e6ecfdc094718c5bb186d2e228b014bb7e824db1648814320302
 SHA512:
-  metadata.gz: af0b726b16f926ac7052072253438bbed9cb899e20264dc4d19942c879dfefec73199eabaa3c13d7f89ea4f433365b2335bfb563c31d867072e37e4aaf26aa05
-  data.tar.gz: 4f2c3c8005a80ac3407939e5f8a94dd398da02b407bc35b41d43b075d2b275a1bff39e0c52732b35252d4342da4c3f86920061f97e3621bfb416725762c0e2b4
+  metadata.gz: a4f0f5efb938e2ae503c50e562be4169afaa47a5a058757938f69395dacc3c46693488daaf9db2e6151bdbf0d03c7dd879be15de5f8840aa4dd15abf04913e15
+  data.tar.gz: 6308ea5cb891f6f7bed9f9095381bdaceacd75ef3fef890b7d2fcdce0b5ac3b84f800924b0ff718f4fcc8cb35bcdc5c85592a99606c7af35bf77ec598bf5fb0c

data/lib/dependabot/maven/file_parser/maven_dependency_parser.rb

@@ -0,0 +1,126 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+require "dependabot/maven/file_parser"
+require "dependabot/maven/native_helpers"
+
+module Dependabot
+  module Maven
+    class FileParser
+      class MavenDependencyParser
+        extend T::Sig
+        require "dependabot/file_parsers/base/dependency_set"
+
+        DEPENDENCY_OUTPUT_FILE = "dependency-tree-output.json"
+
+        sig do
+          params(dependency_files: T::Array[Dependabot::DependencyFile])
+            .returns(Dependabot::FileParsers::Base::DependencySet)
+        end
+        def self.build_dependency_set(dependency_files)
+          dependency_set = Dependabot::FileParsers::Base::DependencySet.new
+
+          # Copy only pom.xml files to a temporary directory to
+          # output the dependency tree without building the project
+          SharedHelpers.in_a_temporary_directory do |temp_path|
+            # Create a directory structure that maintains relative relationships
+            project_directory = create_directory_structure(dependency_files, temp_path.to_s)
+
+            dependency_files.each do |pom|
+              pom_path = File.join(project_directory, pom.name)
+              pom_dir = File.dirname(pom_path)
+              FileUtils.mkdir_p(pom_dir)
+              File.write(pom_path, pom.content)
+            end
+
+            Dir.chdir(project_directory) do
+              NativeHelpers.run_mvn_dependency_tree_plugin(DEPENDENCY_OUTPUT_FILE)
+            end
+
+            # mvn CLI outputs dependency tree for each pom.xml file, collect them
+            # add into single dependency set
+            dependency_files.each do |pom|
+              pom_path = File.join(project_directory, pom.name)
+              pom_dir = File.dirname(pom_path)
+              output_file = File.join(pom_dir, DEPENDENCY_OUTPUT_FILE)
+
+              # If we run updater from sub-module, parent file might be included in dependency files,
+              # but mvn CLI will not generate dependency tree for it unless we start from the parent.
+              # In that case we can just skip it and focus only on current file and it's sub-modules.
+              unless File.exist?(output_file)
+                Dependabot.logger.warn("Dependency tree output file not found: #{output_file}")
+                next
+              end
+
+              dependency_tree = JSON.parse(File.read(output_file))
+              extract_dependencies_from_tree(pom, dependency_set, dependency_tree)
+            end
+          end
+
+          dependency_set
+        end
+
+        sig do
+          params(pom: Dependabot::DependencyFile,
+                 dependency_set: Dependabot::FileParsers::Base::DependencySet,
+                 dependency_tree: T::Hash[String, T.untyped]).void
+        end
+        def self.extract_dependencies_from_tree(pom, dependency_set, dependency_tree)
+          traverse_tree = T.let(-> {}, T.proc.params(node: T::Hash[String, T.untyped]).void)
+          traverse_tree = lambda do |node|
+            artifact_id = node["artifactId"]
+            group_id = node["groupId"]
+            version = node["version"]
+            type = node["type"]
+            classifier = node["classifier"].to_s.empty? ? nil : node["classifier"]
+            scope = node["scope"]
+
+            groups = scope == "test" ? ["test"] : []
+            dependency_set << Dependabot::Dependency.new(
+              name: "#{group_id}:#{artifact_id}",
+              version: version,
+              package_manager: "maven",
+              requirements: [{
+                requirement: version,
+                file: nil,
+                groups: groups,
+                source: nil,
+                metadata: {
+                  packaging_type: type,
+                  classifier: classifier,
+                  pom_file: pom.name
+                }
+              }]
+            )
+
+            node["children"]&.each(&traverse_tree)
+          end
+
+          traverse_tree.call(dependency_tree)
+        end
+
+        sig do
+          params(dependency_files: T::Array[Dependabot::DependencyFile], temp_path: String)
+            .returns(String)
+        end
+        def self.create_directory_structure(dependency_files, temp_path)
+          # Find the topmost directory level by finding the minimum number of "../" sequences
+          relative_top_depth = dependency_files.map do |pom|
+            Pathname.new(pom.name).cleanpath.to_s.scan("../").length
+          end.max || 0
+
+          # Create the base directory structure with the required depth
+          base_depth_path = (0...relative_top_depth).reduce(temp_path) do |path, i|
+            File.join(path, "l#{i}")
+          end
+
+          FileUtils.mkdir_p(base_depth_path)
+
+          base_depth_path
+        end
+      end
+    end
+  end
+end

data/lib/dependabot/maven/file_parser.rb

@@ -3,6 +3,7 @@
 
 require "nokogiri"
 require "sorbet-runtime"
+require "open3"
 
 require "dependabot/dependency"
 require "dependabot/file_parsers"
@@ -16,9 +17,11 @@ require "dependabot/errors"
 # - http://maven.apache.org/pom.html
 module Dependabot
   module Maven
+    # rubocop:disable Metrics/ClassLength
     class FileParser < Dependabot::FileParsers::Base
       extend T::Sig
       require "dependabot/file_parsers/base/dependency_set"
+      require_relative "file_parser/maven_dependency_parser"
       require_relative "file_parser/property_value_finder"
 
       # The following "dependencies" are candidates for updating:
@@ -41,9 +44,30 @@ module Dependabot
       sig { override.returns(T::Array[Dependabot::Dependency]) }
       def parse
         dependency_set = DependencySet.new
-        pomfiles.each { |pom| dependency_set += pomfile_dependencies(pom) }
-        extensionfiles.each { |extension| dependency_set += extensionfile_dependencies(extension) }
-        dependency_set.dependencies
+
+        dependencies = []
+        if Dependabot::Experiments.enabled?(:maven_transitive_dependencies)
+          dependency_set += MavenDependencyParser.build_dependency_set(pomfiles)
+
+          pomfiles.each { |pom| dependency_set += pomfile_dependencies(pom) }
+          extensionfiles.each { |extension| dependency_set += extensionfile_dependencies(extension) }
+
+          dependency_set.dependencies.each do |dep|
+            requirements = merge_requirements(dep.requirements)
+            dependencies << Dependabot::Dependency.new(
+              name: dep.name,
+              version: dep.version,
+              package_manager: "maven",
+              requirements: requirements
+            )
+          end
+        else
+          pomfiles.each { |pom| dependency_set += pomfile_dependencies(pom) }
+          extensionfiles.each { |extension| dependency_set += extensionfile_dependencies(extension) }
+          dependencies = dependency_set.dependencies
+        end
+
+        dependencies
       end
 
       sig { returns(Ecosystem) }
@@ -386,7 +410,75 @@ module Dependabot
       def check_required_files
         raise "No pom.xml!" unless get_original_file("pom.xml")
       end
+
+      # Merge dependency scan requirements with file parsing requirements.
+      # Since dependency scan evaluates properties, we need to combine results with XML parsing,
+      # so we know when certain requirement not a literal value and can differentiate transitive dependencies
+      # from direct dependencies.
+      sig do
+        params(requirements: T::Array[T::Hash[Symbol, T.untyped]]).returns(T::Array[T::Hash[Symbol, T.untyped]])
+      end
+      def merge_requirements(requirements)
+        return requirements if requirements.length <= 1
+
+        merged = []
+        used_indices = Set.new
+        requirements.each_with_index do |dep_scan_req, i|
+          next if used_indices.include?(i) || dep_scan_req.dig(:metadata, :pom_file).nil?
+
+          # Look for another requirement where pom_file matches property_source
+          match_index = requirements.find_index.with_index do |parsing_req, j|
+            j > i &&
+              !used_indices.include?(j) &&
+              dep_scan_req.dig(:metadata, :pom_file) == parsing_req.fetch(:file)
+          end
+
+          if match_index
+            parsing_req = T.must(requirements[match_index])
+
+            # Merge the two requirements
+            # We prefer file and requirement properties from parsed requirements,
+            # because they include correct file and not evaluated property value.
+            merged_req = {
+              requirement: parsing_req[:requirement],
+              file: parsing_req[:file],
+              groups: [*dep_scan_req[:groups], *parsing_req[:groups]].uniq.compact,
+              source: dep_scan_req[:source],
+              metadata: merge_metadata(dep_scan_req[:metadata], parsing_req[:metadata])
+            }
+
+            merged << merged_req
+            used_indices.add(i)
+            used_indices.add(match_index)
+          else
+            # No match found, keep the requirement as is
+            merged << dep_scan_req
+            used_indices.add(i)
+          end
+        end
+
+        merged
+      end
+
+      # Merge metadata from two requirements, combining all keys
+      sig do
+        params(metadata1: T::Hash[Symbol, T.untyped],
+               metadata2: T::Hash[Symbol, T.untyped]).returns(T::Hash[Symbol, T.untyped])
+      end
+      def merge_metadata(metadata1, metadata2)
+        metadata1.merge(metadata2) do |_key, old_value, new_value|
+          case [old_value, new_value]
+          in [nil, new_value] then new_value
+          in [old_value, nil] then old_value
+          in [old_value, new_value] if old_value == new_value then old_value
+          else
+            # If values differ, combine them
+            [*old_value, *new_value].uniq
+          end
+        end
+      end
     end
+    # rubocop:enable Metrics/ClassLength
   end
 end
 

data/lib/dependabot/maven/file_updater/declaration_finder.rb

@@ -61,7 +61,7 @@ module Dependabot
 
         sig { returns(Dependabot::DependencyFile) }
         def declaring_pom
-          filename = declaring_requirement.fetch(:file)
+          filename = declaring_requirement.fetch(:file) || declaring_requirement.dig(:metadata, :pom_file)
           declaring_pom = dependency_files.find { |f| f.name == filename }
           return declaring_pom if declaring_pom
 

data/lib/dependabot/maven/file_updater.rb

@@ -74,13 +74,14 @@ module Dependabot
           raise "Bad req match" unless new_req[:file] == T.must(old_req)[:file]
           next if new_req[:requirement] == T.must(old_req)[:requirement]
 
+          file_name = T.let(new_req.fetch(:file) || new_req.dig(:metadata, :pom_file), String)
           if new_req.dig(:metadata, :property_name)
             files = update_pomfiles_for_property_change(files, new_req)
-            pom = files.find { |f| f.name == new_req.fetch(:file) }
+            pom = files.find { |f| f.name == file_name }
             files[T.must(files.index(pom))] =
               remove_property_suffix_in_pom(dependency, T.must(pom), T.must(old_req))
           else
-            file = files.find { |f| f.name == new_req.fetch(:file) }
+            file = files.find { |f| f.name == file_name }
             files[T.must(files.index(file))] =
               update_version_in_file(dependency, T.must(file), T.must(old_req), new_req)
           end
@@ -119,11 +120,20 @@ module Dependabot
       end
       def update_version_in_file(dependency, file, previous_req, requirement)
         updated_content = T.must(file.content)
-
-        original_file_declarations(dependency, previous_req).each do |old_dec|
-          updated_content = updated_content.gsub(old_dec) do
-            updated_file_declaration(old_dec, previous_req, requirement)
+        original_file_declarations = original_file_declarations(dependency, previous_req)
+
+        if original_file_declarations.any?
+          # If the file already has a declaration for this dependency, we
+          # update the existing declaration with the new version.
+          original_file_declarations.each do |old_dec|
+            updated_content = updated_content.gsub(old_dec) do
+              updated_file_declaration(old_dec, previous_req, requirement)
+            end
           end
+        else
+          # If the file does not have a declaration for this dependency, we
+          # add a new declaration for it.
+          updated_content = add_new_declaration(updated_content, dependency, requirement)
         end
 
         raise "Expected content to change!" if updated_content == file.content
@@ -131,6 +141,53 @@ module Dependabot
         updated_file(file: file, content: updated_content)
       end
 
+      sig do
+        params(
+          content: String,
+          dependency: Dependabot::Dependency,
+          requirement: T::Hash[Symbol, T.untyped]
+        ).returns(String)
+      end
+      def add_new_declaration(content, dependency, requirement) # rubocop:disable Metrics/AbcSize
+        doc = Nokogiri::XML(content) { |config| config.default_xml.noblanks }
+        doc.remove_namespaces!
+
+        project = doc.at_xpath("//project")
+        raise "<project> element not found in the XML content" unless project
+
+        dependency_management = project.at_xpath("dependencyManagement")
+        unless dependency_management
+          dependency_management = Nokogiri::XML::Node.new("dependencyManagement", doc)
+          dependencies = Nokogiri::XML::Node.new("dependencies", doc)
+          dependency_management.add_child(dependencies)
+          project.add_child(dependency_management)
+        end
+
+        dependencies = dependency_management.at_xpath("dependencies")
+        unless dependencies
+          dependencies = Nokogiri::XML::Node.new("dependencies", doc)
+          dependency_management.add_child(dependencies)
+        end
+
+        dependency_node = Nokogiri::XML::Node.new("dependency", doc)
+
+        group_id = Nokogiri::XML::Node.new("groupId", doc)
+        group_id.content = dependency.name.split(":").first
+        dependency_node.add_child(group_id)
+
+        artifact_id = Nokogiri::XML::Node.new("artifactId", doc)
+        artifact_id.content = dependency.name.split(":").last
+        dependency_node.add_child(artifact_id)
+
+        version = Nokogiri::XML::Node.new("version", doc)
+        version.content = requirement.fetch(:requirement)
+        dependency_node.add_child(version)
+
+        dependencies.add_child(dependency_node)
+
+        doc.to_xml
+      end
+
       sig do
         params(
           dep: Dependabot::Dependency,

data/lib/dependabot/maven/native_helpers.rb

@@ -0,0 +1,41 @@
+# typed: strong
+# frozen_string_literal: true
+
+require "shellwords"
+require "sorbet-runtime"
+
+module Dependabot
+  module Maven
+    module NativeHelpers
+      extend T::Sig
+
+      sig do
+        params(file_name: String).void
+      end
+      def self.run_mvn_dependency_tree_plugin(file_name)
+        proxy_url = URI.parse(ENV.fetch("HTTPS_PROXY"))
+        stdout, _, status = Open3.capture3(
+          { "PROXY_HOST" => proxy_url.host },
+          "mvn",
+          "dependency:tree",
+          "-DoutputFile=#{file_name}",
+          "-DoutputType=json",
+          "-e"
+        )
+        Dependabot.logger.info("mvn dependency:tree output: STDOUT:#{stdout}")
+        handle_tool_error(stdout) unless status.success?
+      end
+
+      sig { params(output: String).void }
+      def self.handle_tool_error(output)
+        if (match = output.match(
+          %r{Could not transfer artifact (?<artifact>[^ ]+) from/to (?<repository_name>[^ ]+) \((?<repository_url>[^ ]+)\): status code: (?<status_code>[0-9]+)} # rubocop:disable Layout/LineLength
+        )) && (match[:status_code] == ("403") || match[:status_code] == ("401"))
+          raise Dependabot::PrivateSourceAuthenticationFailure, match[:repository_url]
+        end
+
+        raise DependabotError, "mvn CLI failed with an unhandled error"
+      end
+    end
+  end
+end

data/lib/dependabot/maven/package/package_details_fetcher.rb

@@ -356,7 +356,8 @@ module Dependabot
         # Returns the POM file for the dependency, if it exists.
         sig { returns(T.nilable(Dependabot::DependencyFile)) }
         def pom
-          filename = dependency.requirements.first&.fetch(:file)
+          filename = dependency.requirements.first&.fetch(:file) ||
+                     dependency.requirements.first&.dig(:metadata, :pom_file)
           dependency_files.find { |f| f.name == filename }
         end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dependabot-maven
 version: !ruby/object:Gem::Version
-  version: 0.317.0
+  version: 0.318.0
 platform: ruby
 authors:
 - Dependabot
@@ -15,14 +15,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.317.0
+        version: 0.318.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.317.0
+        version: 0.318.0
 - !ruby/object:Gem::Dependency
   name: debug
   requirement: !ruby/object:Gem::Requirement
@@ -244,6 +244,7 @@ files:
 - lib/dependabot/maven.rb
 - lib/dependabot/maven/file_fetcher.rb
 - lib/dependabot/maven/file_parser.rb
+- lib/dependabot/maven/file_parser/maven_dependency_parser.rb
 - lib/dependabot/maven/file_parser/pom_fetcher.rb
 - lib/dependabot/maven/file_parser/property_value_finder.rb
 - lib/dependabot/maven/file_parser/repositories_finder.rb
@@ -252,6 +253,7 @@ files:
 - lib/dependabot/maven/file_updater/property_value_updater.rb
 - lib/dependabot/maven/language.rb
 - lib/dependabot/maven/metadata_finder.rb
+- lib/dependabot/maven/native_helpers.rb
 - lib/dependabot/maven/package/package_details_fetcher.rb
 - lib/dependabot/maven/package_manager.rb
 - lib/dependabot/maven/requirement.rb
@@ -268,7 +270,7 @@ licenses:
 - MIT
 metadata:
   bug_tracker_uri: https://github.com/dependabot/dependabot-core/issues
-  changelog_uri: https://github.com/dependabot/dependabot-core/releases/tag/v0.317.0
+  changelog_uri: https://github.com/dependabot/dependabot-core/releases/tag/v0.318.0
 rdoc_options: []
 require_paths:
 - lib