bibliothecary 7.3.5 → 8.1.0

data/lib/bibliothecary/multi_parsers/cyclonedx.rb

@@ -0,0 +1,156 @@
+require 'json'
+require 'ox'
+
+# packageurl-ruby uses pattern-matching (https://docs.ruby-lang.org/en/2.7.0/NEWS.html#label-Pattern+matching)
+# which warns a whole bunch in Ruby 2.7 as being an experimental feature, but has
+# been accepted in Ruby 3.0 (https://rubyreferences.github.io/rubychanges/3.0.html#pattern-matching).
+Warning[:experimental] = false
+require 'package_url'
+Warning[:experimental] = true
+
+module Bibliothecary
+  module MultiParsers
+    module CycloneDX
+      include Bibliothecary::Analyser
+      include Bibliothecary::Analyser::TryCache
+
+      NoComponents = Class.new(StandardError)
+
+      class ManifestEntries
+        # If a purl type (key) exists, it will be used in a manifest for
+        # the key's value. If not, it's ignored.
+        #
+        # https://github.com/package-url/purl-spec/blob/master/PURL-TYPES.rst
+        PURL_TYPE_MAPPING = {
+          "golang" => :go,
+          "maven" => :maven,
+          "npm" => :npm,
+          "cargo" => :cargo,
+          "composer" => :packagist,
+          "conda" => :conda,
+          "cran" => :cran,
+          "gem" => :rubygems,
+          "hackage" => :hackage,
+          "hex" => :hex,
+          "nuget" => :nuget,
+          "pypi" => :pypi,
+          "swift" => :swift_pm
+        }
+
+        attr_reader :manifests
+
+        def initialize(parse_queue:)
+          @manifests = {}
+
+          # Instead of recursing, we'll work through a queue of components
+          # to process, letting the different parser add components to the
+          # queue however they need to  pull them from the source document.
+          @parse_queue = parse_queue
+        end
+
+        def <<(purl)
+          mapping = PURL_TYPE_MAPPING[purl.type]
+          return unless mapping
+
+          @manifests[mapping] ||= Set.new
+          @manifests[mapping] << {
+            name: self.class.full_name_for_purl(purl),
+            requirement: purl.version,
+            type: 'lockfile'
+          }
+        end
+
+        # Iterates over each manifest entry in the parse_queue, and accepts a block which will
+        # be called on each component. The block has two jobs: 1) add more sub-components
+        # to parse (if they exist), and 2) return the components purl.
+        def parse!(&block)
+          while @parse_queue.length > 0
+            component = @parse_queue.shift
+
+            purl_text = block.call(component, @parse_queue)
+
+            next unless purl_text
+
+            purl = PackageURL.parse(purl_text)
+
+            self << purl
+          end
+        end
+
+        def [](key)
+          @manifests[key]&.to_a
+        end
+
+        # @return [String] The properly namespaced package name
+        def self.full_name_for_purl(purl)
+          parts = [purl.namespace, purl.name].compact
+
+          case purl.type
+          when "maven"
+            parts.join(':')
+          else
+            parts.join('/')
+          end
+        end
+      end
+
+      def self.mapping
+        {
+          match_filename('cyclonedx.json') => {
+            kind: 'lockfile',
+            parser: :parse_cyclonedx_json
+          },
+          match_filename('cyclonedx.xml') => {
+            kind: 'lockfile',
+            parser: :parse_cyclonedx_xml
+          }
+        }
+      end
+
+      def parse_cyclonedx_json(file_contents, options: {})
+        manifest = nil
+
+        manifest = try_cache(options, options[:filename]) do
+          JSON.parse(file_contents)
+        end
+
+        raise NoComponents unless manifest["components"]
+
+        entries = ManifestEntries.new(parse_queue: manifest["components"])
+
+        entries.parse! do |component, parse_queue|
+          parse_queue.concat(component["components"]) if component["components"]
+
+          component["purl"]
+        end
+
+        entries[platform_name.to_sym]
+      end
+
+      def parse_cyclonedx_xml(file_contents, options: {})
+        manifest = try_cache(options, options[:filename]) do
+          Ox.parse(file_contents)
+        end
+
+        root = manifest
+        if root.respond_to?(:bom)
+          root = root.bom
+        end
+
+        raise NoComponents unless root.locate('components').first
+
+        entries = ManifestEntries.new(parse_queue: root.locate('components/*'))
+
+        entries.parse! do |component, parse_queue|
+          # #locate returns an empty array if nothing is found, so we can
+          # always safely concatenate it to the parse queue.
+          parse_queue.concat(component.locate('components/*'))
+
+          component.locate("purl").first&.text
+        end
+
+        entries[platform_name.to_sym]
+      end
+    end
+  end
+end

data/lib/bibliothecary/multi_parsers/json_runtime.rb

@@ -0,0 +1,16 @@
+module Bibliothecary
+  module MultiParsers
+    # Provide JSON Runtime Manifest parsing
+    module JSONRuntime
+      def parse_json_runtime_manifest(file_contents, options: {})
+        JSON.parse(file_contents).fetch('dependencies',[]).map do |name, requirement|
+          {
+            name: name,
+            requirement: requirement,
+            type: 'runtime'
+          }
+        end
+      end
+    end
+  end
+end

data/lib/bibliothecary/parsers/bower.rb

@@ -14,8 +14,8 @@ module Bibliothecary
         }
       end
 
-      def self.parse_manifest(manifest)
-        json = JSON.parse(manifest)
+      def self.parse_manifest(file_contents, options: {})
+        json = JSON.parse(file_contents)
         map_dependencies(json, 'dependencies', 'runtime') +
         map_dependencies(json, 'devDependencies', 'development')
       end

data/lib/bibliothecary/parsers/cargo.rb

@@ -16,7 +16,9 @@ module Bibliothecary
         }
       end
 
-      def self.parse_manifest(file_contents)
+      add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
+
+      def self.parse_manifest(file_contents, options: {})
         manifest = Tomlrb.parse(file_contents)
         manifest.fetch('dependencies', []).map do |name, requirement|
           if requirement.respond_to?(:fetch)
@@ -31,7 +33,7 @@ module Bibliothecary
           .compact
       end
 
-      def self.parse_lockfile(file_contents)
+      def self.parse_lockfile(file_contents, options: {})
         manifest = Tomlrb.parse(file_contents)
         manifest.fetch('package',[]).map do |dependency|
           next if not dependency['source'] or not dependency['source'].start_with?('registry+')

data/lib/bibliothecary/parsers/carthage.rb

@@ -20,16 +20,16 @@ module Bibliothecary
         }
       end
 
-      def self.parse_cartfile(manifest)
-        map_dependencies(manifest, 'cartfile')
+      def self.parse_cartfile(file_contents, options: {})
+        map_dependencies(file_contents, 'cartfile')
       end
 
-      def self.parse_cartfile_private(manifest)
-        map_dependencies(manifest, 'cartfile.private')
+      def self.parse_cartfile_private(file_contents, options: {})
+        map_dependencies(file_contents, 'cartfile.private')
       end
 
-      def self.parse_cartfile_resolved(manifest)
-        map_dependencies(manifest, 'cartfile.resolved')
+      def self.parse_cartfile_resolved(file_contents, options: {})
+        map_dependencies(file_contents, 'cartfile.resolved')
       end
 
       def self.map_dependencies(manifest, path)

data/lib/bibliothecary/parsers/clojars.rb

@@ -15,8 +15,8 @@ module Bibliothecary
         }
       end
 
-      def self.parse_manifest(manifest)
-        response = Typhoeus.post("#{Bibliothecary.configuration.clojars_parser_host}/project.clj", body: manifest)
+      def self.parse_manifest(file_contents, options: {})
+        response = Typhoeus.post("#{Bibliothecary.configuration.clojars_parser_host}/project.clj", body: file_contents)
         raise Bibliothecary::RemoteParsingError.new("Http Error #{response.response_code} when contacting: #{Bibliothecary.configuration.clojars_parser_host}/project.clj", response.response_code) unless response.success?
         json = JSON.parse response.body
         index = json.index("dependencies")

data/lib/bibliothecary/parsers/cocoapods.rb

@@ -5,6 +5,7 @@ module Bibliothecary
   module Parsers
     class CocoaPods
       include Bibliothecary::Analyser
+      extend Bibliothecary::MultiParsers::BundlerLikeManifest
 
       NAME_VERSION = '(?! )(.*?)(?: \(([^-]*)(?:-(.*))?\))?'.freeze
       NAME_VERSION_4 = /^ {4}#{NAME_VERSION}$/
@@ -32,7 +33,7 @@ module Bibliothecary
         }
       end
 
-      def self.parse_podfile_lock(file_contents)
+      def self.parse_podfile_lock(file_contents, options: {})
         manifest = YAML.load file_contents
         manifest['PODS'].map do |row|
           pod = row.is_a?(String) ? row : row.keys.first
@@ -45,17 +46,17 @@ module Bibliothecary
         end.compact
       end
 
-      def self.parse_podspec(file_contents)
+      def self.parse_podspec(file_contents, options: {})
         manifest = Gemnasium::Parser.send(:podspec, file_contents)
         parse_ruby_manifest(manifest)
       end
 
-      def self.parse_podfile(file_contents)
+      def self.parse_podfile(file_contents, options: {})
         manifest = Gemnasium::Parser.send(:podfile, file_contents)
         parse_ruby_manifest(manifest)
       end
 
-      def self.parse_json_manifest(file_contents)
+      def self.parse_json_manifest(file_contents, options: {})
         manifest = JSON.parse(file_contents)
         manifest['dependencies'].inject([]) do |deps, dep|
           deps.push({

data/lib/bibliothecary/parsers/conda.rb

@@ -26,13 +26,19 @@ module Bibliothecary
         }
       end
 
-      def self.parse_conda(info, kind = "manifest")
-        dependencies = call_conda_parser_web(info, kind)[kind.to_sym]
-        dependencies.map { |dep| dep.merge(type: "runtime") }
+      add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
+
+      def self.parse_conda(file_contents, options: {})
+        parse_conda_with_kind(file_contents, "manifest")
       end
 
-      def self.parse_conda_lockfile(info)
-        parse_conda(info, "lockfile")
+      def self.parse_conda_lockfile(file_contents, options: {})
+        parse_conda_with_kind(file_contents, "lockfile")
+      end
+
+      def self.parse_conda_with_kind(info, kind)
+        dependencies = call_conda_parser_web(info, kind)[kind.to_sym]
+        dependencies.map { |dep| dep.merge(type: "runtime") }
       end
 
       private_class_method def self.call_conda_parser_web(file_contents, kind)

data/lib/bibliothecary/parsers/cpan.rb

@@ -19,14 +19,14 @@ module Bibliothecary
         }
       end
 
-      def self.parse_json_manifest(file_contents)
+      def self.parse_json_manifest(file_contents, options: {})
         manifest = JSON.parse file_contents
         manifest['prereqs'].map do |_group, deps|
           map_dependencies(deps, 'requires', 'runtime')
         end.flatten
       end
 
-      def self.parse_yaml_manifest(file_contents)
+      def self.parse_yaml_manifest(file_contents, options: {})
         manifest = YAML.load file_contents
         map_dependencies(manifest, 'requires', 'runtime')
       end

data/lib/bibliothecary/parsers/cran.rb

@@ -16,7 +16,9 @@ module Bibliothecary
         }
       end
 
-      def self.parse_description(file_contents)
+      add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
+
+      def self.parse_description(file_contents, options: {})
         manifest = DebControl::ControlFileBase.parse(file_contents)
         parse_section(manifest, 'Depends') +
         parse_section(manifest, 'Imports') +

data/lib/bibliothecary/parsers/dub.rb

@@ -5,6 +5,7 @@ module Bibliothecary
   module Parsers
     class Dub
       include Bibliothecary::Analyser
+      extend Bibliothecary::MultiParsers::JSONRuntime
 
       def self.mapping
         {
@@ -19,8 +20,8 @@ module Bibliothecary
         }
       end
 
-      def self.parse_sdl_manifest(manifest)
-        SdlParser.new(:runtime, manifest).dependencies
+      def self.parse_sdl_manifest(file_contents, options: {})
+        SdlParser.new(:runtime, file_contents).dependencies
       end
     end
   end

data/lib/bibliothecary/parsers/elm.rb

@@ -4,6 +4,7 @@ module Bibliothecary
   module Parsers
     class Elm
       include Bibliothecary::Analyser
+      extend Bibliothecary::MultiParsers::JSONRuntime
 
       def self.mapping
         {
@@ -18,7 +19,7 @@ module Bibliothecary
         }
       end
 
-      def self.parse_json_lock(file_contents)
+      def self.parse_json_lock(file_contents, options: {})
         manifest = JSON.parse file_contents
         manifest.map do |name, requirement|
           {

data/lib/bibliothecary/parsers/generic.rb

@@ -14,7 +14,7 @@ module Bibliothecary
         }
       end
 
-      def self.parse_lockfile(file_contents)
+      def self.parse_lockfile(file_contents, options: {})
         table = CSV.parse(file_contents, headers: true)
 
         required_headers = ["platform", "name", "requirement"]
@@ -22,9 +22,9 @@ module Bibliothecary
         raise "Missing headers #{missing_headers} in CSV" unless missing_headers.empty?
 
         table.map.with_index do |row, idx|
-          line = idx + 1
+          line = idx + 2 # use 1-based index just like the 'csv' std lib, and count the headers as first row.
           required_headers.each do |h|
-            raise "missing field '#{h}' on line #{line}" if row[h].empty?
+            raise "missing field '#{h}' on line #{line}" if row[h].nil? || row[h].empty?
           end
           {
             platform: row['platform'],

data/lib/bibliothecary/parsers/go.rb

@@ -65,12 +65,14 @@ module Bibliothecary
         }
       end
 
-      def self.parse_godep_json(file_contents)
+      add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
+
+      def self.parse_godep_json(file_contents, options: {})
         manifest = JSON.parse file_contents
         map_dependencies(manifest, 'Deps', 'ImportPath', 'Rev', 'runtime')
       end
 
-      def self.parse_gpm(file_contents)
+      def self.parse_gpm(file_contents, options: {})
         deps = []
         file_contents.split("\n").each do |line|
           match = line.gsub(/(\#(.*))/, '').match(GPM_REGEXP)
@@ -84,38 +86,38 @@ module Bibliothecary
         deps
       end
 
-      def self.parse_govendor(file_contents)
+      def self.parse_govendor(file_contents, options: {})
         manifest = JSON.load file_contents
         map_dependencies(manifest, 'package', 'path', 'revision', 'runtime')
       end
 
-      def self.parse_glide_yaml(file_contents)
+      def self.parse_glide_yaml(file_contents, options: {})
         manifest = YAML.load file_contents
         map_dependencies(manifest, 'import', 'package', 'version', 'runtime') +
         map_dependencies(manifest, 'devImports', 'package', 'version', 'development')
       end
 
-      def self.parse_glide_lockfile(file_contents)
+      def self.parse_glide_lockfile(file_contents, options: {})
         manifest = YAML.load file_contents
         map_dependencies(manifest, 'imports', 'name', 'version', 'runtime')
       end
 
-      def self.parse_gb_manifest(file_contents)
+      def self.parse_gb_manifest(file_contents, options: {})
         manifest = JSON.parse file_contents
         map_dependencies(manifest, 'dependencies', 'importpath', 'revision', 'runtime')
       end
 
-      def self.parse_dep_toml(file_contents)
+      def self.parse_dep_toml(file_contents, options: {})
         manifest = Tomlrb.parse file_contents
         map_dependencies(manifest, 'constraint', 'name', 'version', 'runtime')
       end
 
-      def self.parse_dep_lockfile(file_contents)
+      def self.parse_dep_lockfile(file_contents, options: {})
         manifest = Tomlrb.parse file_contents
         map_dependencies(manifest, 'projects', 'name', 'revision', 'runtime')
       end
 
-      def self.parse_go_mod(file_contents)
+      def self.parse_go_mod(file_contents, options: {})
         deps = []
         file_contents.lines.map(&:strip).each do |line|
           next if line.match(GOMOD_IGNORABLE_REGEX)
@@ -130,7 +132,7 @@ module Bibliothecary
         deps
       end
 
-      def self.parse_go_sum(file_contents)
+      def self.parse_go_sum(file_contents, options: {})
         deps = []
         file_contents.lines.map(&:strip).each do |line|
           if match = line.match(GOSUM_REGEX)
@@ -144,7 +146,7 @@ module Bibliothecary
         deps.uniq
       end
 
-      def self.parse_go_resolved(file_contents)
+      def self.parse_go_resolved(file_contents, options: {})
         JSON.parse(file_contents)
           .select { |dep| dep["Main"] != "true" }
           .map { |dep| { name: dep["Path"], requirement: dep["Version"], type: 'runtime' } }

data/lib/bibliothecary/parsers/hackage.rb

@@ -19,7 +19,9 @@ module Bibliothecary
         }
       end
 
-      def self.parse_cabal(file_contents)
+      add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
+
+      def self.parse_cabal(file_contents, options: {})
         headers = {
           'Content-Type' => "text/plain;charset=utf-8"
         }
@@ -30,7 +32,7 @@ module Bibliothecary
         JSON.parse(response.body, symbolize_names: true)
       end
 
-      def self.parse_cabal_config(file_contents)
+      def self.parse_cabal_config(file_contents, options: {})
         manifest = DebControl::ControlFileBase.parse(file_contents)
         deps = manifest.first['constraints'].delete("\n").split(',').map(&:strip)
         deps.map do |dependency|

data/lib/bibliothecary/parsers/haxelib.rb

@@ -4,6 +4,7 @@ module Bibliothecary
   module Parsers
     class Haxelib
       include Bibliothecary::Analyser
+      extend Bibliothecary::MultiParsers::JSONRuntime
 
       def self.mapping
         {

data/lib/bibliothecary/parsers/hex.rb

@@ -18,8 +18,10 @@ module Bibliothecary
         }
       end
 
-      def self.parse_mix(manifest)
-        response = Typhoeus.post("#{Bibliothecary.configuration.mix_parser_host}/", body: manifest)
+      add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
+
+      def self.parse_mix(file_contents, options: {})
+        response = Typhoeus.post("#{Bibliothecary.configuration.mix_parser_host}/", body: file_contents)
         raise Bibliothecary::RemoteParsingError.new("Http Error #{response.response_code} when contacting: #{Bibliothecary.configuration.mix_parser_host}/", response.response_code) unless response.success?
         json = JSON.parse response.body
 
@@ -32,8 +34,8 @@ module Bibliothecary
         end
       end
 
-      def self.parse_mix_lock(manifest)
-        response = Typhoeus.post("#{Bibliothecary.configuration.mix_parser_host}/lock", body: manifest)
+      def self.parse_mix_lock(file_contents, options: {})
+        response = Typhoeus.post("#{Bibliothecary.configuration.mix_parser_host}/lock", body: file_contents)
         raise Bibliothecary::RemoteParsingError.new("Http Error #{response.response_code} when contacting: #{Bibliothecary.configuration.mix_parser_host}/", response.response_code) unless response.success?
         json = JSON.parse response.body
 

data/lib/bibliothecary/parsers/julia.rb

@@ -12,9 +12,9 @@ module Bibliothecary
         }
       end
 
-      def self.parse_require(manifest)
+      def self.parse_require(file_contents, options: {})
         deps = []
-        manifest.split("\n").each do |line|
+        file_contents.split("\n").each do |line|
           next if line.match(/^#/) || line.empty?
           split = line.split(/\s/)
           if line.match(/^@/)

data/lib/bibliothecary/parsers/maven.rb

@@ -38,7 +38,7 @@ module Bibliothecary
           },
           match_filename("pom.xml", case_insensitive: true) => {
             kind: 'manifest',
-            parser: :parse_pom_manifest
+            parser: :parse_standalone_pom_manifest
           },
           match_filename("build.gradle", case_insensitive: true) => {
             kind: 'manifest',
@@ -72,7 +72,9 @@ module Bibliothecary
         }
       end
 
-      def self.parse_ivy_manifest(file_contents)
+      add_multi_parser Bibliothecary::MultiParsers::CycloneDX
+
+      def self.parse_ivy_manifest(file_contents, options: {})
         manifest = Ox.parse file_contents
         manifest.dependencies.locate('dependency').map do |dependency|
           attrs = dependency.attributes
@@ -95,7 +97,7 @@ module Bibliothecary
         false
       end
 
-      def self.parse_ivy_report(file_contents)
+      def self.parse_ivy_report(file_contents, options: {})
         doc = Ox.parse file_contents
         root = doc.locate("ivy-report").first
         raise "ivy-report document does not have ivy-report at the root" if root.nil?
@@ -120,7 +122,7 @@ module Bibliothecary
         end.compact
       end
 
-      def self.parse_gradle_resolved(file_contents)
+      def self.parse_gradle_resolved(file_contents, options: {})
         type = nil
         file_contents.split("\n").map do |line|
           type_match = GRADLE_TYPE_REGEX.match(line)
@@ -151,7 +153,7 @@ module Bibliothecary
         end.compact.uniq {|item| [item[:name], item[:requirement], item[:type]]}
       end
 
-      def self.parse_maven_resolved(file_contents)
+      def self.parse_maven_resolved(file_contents, options: {})
         Strings::ANSI.sanitize(file_contents)
           .split("\n")
           .map(&method(:parse_resolved_dep_line))
@@ -159,7 +161,7 @@ module Bibliothecary
           .uniq
       end
 
-      def self.parse_maven_tree(file_contents)
+      def self.parse_maven_tree(file_contents, options: {})
         file_contents = file_contents.gsub(/\r\n?/, "\n")
         captures = file_contents.scan(/^\[INFO\](?:(?:\+-)|\||(?:\\-)|\s)+((?:[\w\.-]+:)+[\w\.\-${}]+)/).flatten.uniq
 
@@ -195,7 +197,13 @@ module Bibliothecary
         }
       end
 
-      def self.parse_pom_manifest(file_contents, parent_properties = {})
+      def self.parse_standalone_pom_manifest(file_contents, options: {})
+        parse_pom_manifest(file_contents, {}, options: options)
+      end
+
+      # parent_properties is used by Libraries:
+      # https://github.com/librariesio/libraries.io/blob/e970925aade2596a03268b6e1be785eba8502c62/app/models/package_manager/maven.rb#L129
+      def self.parse_pom_manifest(file_contents, parent_properties = {}, options: {})
         manifest = Ox.parse file_contents
         xml = manifest.respond_to?('project') ? manifest.project : manifest
         [].tap do |deps|
@@ -211,8 +219,8 @@ module Bibliothecary
         end
       end
 
-      def self.parse_gradle(manifest)
-        response = Typhoeus.post("#{Bibliothecary.configuration.gradle_parser_host}/parse", body: manifest)
+      def self.parse_gradle(file_contents, options: {})
+        response = Typhoeus.post("#{Bibliothecary.configuration.gradle_parser_host}/parse", body: file_contents)
         raise Bibliothecary::RemoteParsingError.new("Http Error #{response.response_code} when contacting: #{Bibliothecary.configuration.gradle_parser_host}/parse", response.response_code) unless response.success?
         json = JSON.parse(response.body)
         return [] unless json['dependencies']
@@ -227,7 +235,7 @@ module Bibliothecary
         end.compact
       end
 
-      def self.parse_gradle_kts(manifest)
+      def self.parse_gradle_kts(file_contents, options: {})
         # TODO: the gradle-parser side needs to be implemented for this, coming soon.
         []
       end
@@ -307,7 +315,7 @@ module Bibliothecary
         end
       end
 
-      def self.parse_sbt_update_full(file_contents)
+      def self.parse_sbt_update_full(file_contents, options: {})
         all_deps = []
         type = nil
         lines = file_contents.split("\n")

data/lib/bibliothecary/parsers/meteor.rb

@@ -4,6 +4,7 @@ module Bibliothecary
   module Parsers
     class Meteor
       include Bibliothecary::Analyser
+      extend Bibliothecary::MultiParsers::JSONRuntime
 
       def self.mapping
         {