bibliothecary 11.0.1 → 12.1.0

data/lib/bibliothecary/multi_parsers/json_runtime.rb

@@ -1,13 +1,16 @@
+# frozen_string_literal: true
+
 module Bibliothecary
   module MultiParsers
     # Provide JSON Runtime Manifest parsing
     module JSONRuntime
-      def parse_json_runtime_manifest(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
-        JSON.parse(file_contents).fetch("dependencies",[]).map do |name, requirement|
+      def parse_json_runtime_manifest(file_contents, options: {})
+        JSON.parse(file_contents).fetch("dependencies", []).map do |name, requirement|
           Dependency.new(
             name: name,
             requirement: requirement,
             type: "runtime",
+            source: options.fetch(:filename, nil)
           )
         end
       end

data/lib/bibliothecary/multi_parsers/spdx.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # packageurl-ruby uses pattern-matching (https://docs.ruby-lang.org/en/2.7.0/NEWS.html#label-Pattern+matching)
 # which warns a whole bunch in Ruby 2.7 as being an experimental feature, but has
 # been accepted in Ruby 3.0 (https://rubyreferences.github.io/rubychanges/3.0.html#pattern-matching).
@@ -43,7 +45,7 @@ module Bibliothecary
 
       def parse_spdx_tag_value(file_contents, options: {})
         entries = try_cache(options, options[:filename]) do
-          parse_spdx_tag_value_file_contents(file_contents)
+          parse_spdx_tag_value_file_contents(file_contents, options.fetch(:filename, nil))
         end
 
         raise NoEntries if entries.empty?
@@ -51,7 +53,7 @@ module Bibliothecary
         entries[platform_name.to_sym]
       end
 
-      def parse_spdx_tag_value_file_contents(file_contents)
+      def parse_spdx_tag_value_file_contents(file_contents, source = nil)
         entries = {}
         spdx_name = spdx_version = platform = purl_name = purl_version = nil
 
@@ -65,13 +67,14 @@ module Bibliothecary
             # Per the spec:
             # > A new package Information section is denoted by the package name (7.1) field.
             add_entry(entries: entries, platform: platform, purl_name: purl_name,
-                      spdx_name: spdx_name, purl_version: purl_version, spdx_version: spdx_version)
+                      spdx_name: spdx_name, purl_version: purl_version, spdx_version: spdx_version,
+                      source: source)
 
             # reset for this new package
             spdx_name = spdx_version = platform = purl_name = purl_version = nil
 
             # capture the new package's name
-            spdx_package_name = match[1]
+            spdx_name = match[1]
           elsif (match = stripped_line.match(PACKAGE_VERSION_REGEXP))
             spdx_version = match[1]
           elsif (match = stripped_line.match(PURL_REGEXP))
@@ -83,7 +86,8 @@ module Bibliothecary
         end
 
         add_entry(entries: entries, platform: platform, purl_name: purl_name,
-                  spdx_name: spdx_name, purl_version: purl_version, spdx_version: spdx_version)
+                  spdx_name: spdx_name, purl_version: purl_version, spdx_version: spdx_version,
+                  source: source)
 
         entries
       end
@@ -95,7 +99,7 @@ module Bibliothecary
 
       def parse_spdx_json(file_contents, options: {})
         entries = try_cache(options, options[:filename]) do
-          parse_spdx_json_file_contents(file_contents)
+          parse_spdx_json_file_contents(file_contents, options.fetch(:filename, nil))
         end
 
         raise NoEntries if entries.empty?
@@ -103,7 +107,7 @@ module Bibliothecary
         entries[platform_name.to_sym]
       end
 
-      def parse_spdx_json_file_contents(file_contents)
+      def parse_spdx_json_file_contents(file_contents, source = nil)
         entries = {}
         manifest = JSON.parse(file_contents)
 
@@ -111,33 +115,34 @@ module Bibliothecary
           spdx_name = package["name"]
           spdx_version = package["versionInfo"]
 
-          first_purl_string = package.dig("externalRefs")&.find { |ref| ref["referenceType"] == "purl" }&.dig("referenceLocator")
+          first_purl_string = package["externalRefs"]&.find { |ref| ref["referenceType"] == "purl" }&.dig("referenceLocator")
           purl = first_purl_string && PackageURL.parse(first_purl_string)
           platform = PurlUtil::PURL_TYPE_MAPPING[purl&.type]
           purl_name = PurlUtil.full_name(purl)
           purl_version = purl&.version
 
           add_entry(entries: entries, platform: platform, purl_name: purl_name,
-                    spdx_name: spdx_name, purl_version: purl_version, spdx_version: spdx_version)
+                    spdx_name: spdx_name, purl_version: purl_version, spdx_version: spdx_version,
+                    source: source)
         end
 
         entries
       end
 
-      def add_entry(entries:, platform:, purl_name:, spdx_name:, purl_version:, spdx_version:)
+      def add_entry(entries:, platform:, purl_name:, spdx_name:, purl_version:, spdx_version:, source: nil)
         package_name = purl_name || spdx_name
         package_version = purl_version || spdx_version
 
-        if platform && package_name && package_version
-          entries[platform.to_sym] ||= []
-          entries[platform.to_sym] << Dependency.new(
-            name: package_name,
-            requirement: package_version,
-            type: "lockfile"
-          )
-        end
-      end
+        return unless platform && package_name && package_version
 
+        entries[platform.to_sym] ||= []
+        entries[platform.to_sym] << Dependency.new(
+          name: package_name,
+          requirement: package_version,
+          type: "lockfile",
+          source: source
+        )
+      end
     end
   end
 end

data/lib/bibliothecary/parsers/bower.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "json"
 
 module Bibliothecary
@@ -16,10 +18,10 @@ module Bibliothecary
 
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
-      def self.parse_manifest(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
+      def self.parse_manifest(file_contents, options: {})
         json = JSON.parse(file_contents)
-        map_dependencies(json, "dependencies", "runtime") +
-        map_dependencies(json, "devDependencies", "development")
+        map_dependencies(json, "dependencies", "runtime", options.fetch(:filename, nil)) +
+          map_dependencies(json, "devDependencies", "development", options.fetch(:filename, nil))
       end
     end
   end

data/lib/bibliothecary/parsers/cargo.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Bibliothecary
   module Parsers
     class Cargo
@@ -20,7 +22,7 @@ module Bibliothecary
       add_multi_parser(Bibliothecary::MultiParsers::Spdx)
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
-      def self.parse_manifest(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
+      def self.parse_manifest(file_contents, options: {})
         manifest = Tomlrb.parse(file_contents)
 
         parsed_dependencies = []
@@ -30,10 +32,12 @@ module Bibliothecary
             if requirement.respond_to?(:fetch)
               requirement = requirement["version"] or next
             end
+
             Dependency.new(
               name: name,
               requirement: requirement,
               type: index.zero? ? "runtime" : "development",
+              source: options.fetch(:filename, nil)
             )
           end
         end
@@ -41,14 +45,16 @@ module Bibliothecary
         parsed_dependencies.flatten.compact
       end
 
-      def self.parse_lockfile(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
+      def self.parse_lockfile(file_contents, options: {})
         manifest = Tomlrb.parse(file_contents)
-        manifest.fetch("package",[]).map do |dependency|
-          next if not dependency["source"] or not dependency["source"].start_with?("registry+")
+        manifest.fetch("package", []).map do |dependency|
+          next if !(dependency["source"]) || !dependency["source"].start_with?("registry+")
+
           Dependency.new(
             name: dependency["name"],
             requirement: dependency["version"],
             type: "runtime",
+            source: options.fetch(:filename, nil)
           )
         end
           .compact

data/lib/bibliothecary/parsers/cocoapods.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "gemnasium/parser"
 require "yaml"
 
@@ -7,7 +9,7 @@ module Bibliothecary
       include Bibliothecary::Analyser
       extend Bibliothecary::MultiParsers::BundlerLikeManifest
 
-      NAME_VERSION = '(?! )(.*?)(?: \(([^-]*)(?:-(.*))?\))?'.freeze
+      NAME_VERSION = '(?! )(.*?)(?: \(([^-]*)(?:-(.*))?\))?'
       NAME_VERSION_4 = /^ {4}#{NAME_VERSION}$/
 
       def self.mapping
@@ -35,7 +37,7 @@ module Bibliothecary
 
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
-      def self.parse_podfile_lock(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
+      def self.parse_podfile_lock(file_contents, options: {})
         manifest = YAML.load file_contents
         manifest["PODS"].map do |row|
           pod = row.is_a?(String) ? row : row.keys.first
@@ -44,28 +46,30 @@ module Bibliothecary
             name: match[1].split("/").first,
             requirement: match[2],
             type: "runtime",
+            source: options.fetch(:filename, nil)
           )
         end.compact
       end
 
-      def self.parse_podspec(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
+      def self.parse_podspec(file_contents, options: {})
         manifest = Gemnasium::Parser.send(:podspec, file_contents)
-        parse_ruby_manifest(manifest)
+        parse_ruby_manifest(manifest, options.fetch(:filename, nil))
       end
 
-      def self.parse_podfile(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
+      def self.parse_podfile(file_contents, options: {})
         manifest = Gemnasium::Parser.send(:podfile, file_contents)
-        parse_ruby_manifest(manifest)
+        parse_ruby_manifest(manifest, options.fetch(:filename, nil))
       end
 
-      def self.parse_json_manifest(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
+      def self.parse_json_manifest(file_contents, options: {})
         manifest = JSON.parse(file_contents)
         manifest["dependencies"].inject([]) do |deps, dep|
           deps.push(Dependency.new(
-            name: dep[0],
-            requirement: dep[1],
-            type: "runtime",
-          ))
+                      name: dep[0],
+                      requirement: dep[1],
+                      type: "runtime",
+                      source: options.fetch(:filename, nil)
+                    ))
         end.uniq
       end
     end

data/lib/bibliothecary/parsers/conda.rb

@@ -1,4 +1,6 @@
-require "json"
+# frozen_string_literal: true
+
+require "yaml"
 
 module Bibliothecary
   module Parsers
@@ -15,14 +17,6 @@ module Bibliothecary
             parser: :parse_conda,
             kind: "manifest",
           },
-          match_filename("environment.yml.lock") => {
-            parser: :parse_conda_lockfile,
-            kind: "lockfile",
-          },
-          match_filename("environment.yaml.lock") => {
-            parser: :parse_conda_lockfile,
-            kind: "lockfile",
-          },
         }
       end
 
@@ -30,35 +24,64 @@ module Bibliothecary
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
       add_multi_parser(Bibliothecary::MultiParsers::Spdx)
 
-      def self.parse_conda(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
-        parse_conda_with_kind(file_contents, "manifest")
-      end
+      def self.parse_conda(file_contents, options: {})
+        manifest = YAML.load(file_contents)
+        deps = manifest["dependencies"]
+        deps.map do |dep|
+          next unless dep.is_a? String # only deal with strings to skip parsing pip stuff
 
-      def self.parse_conda_lockfile(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
-        parse_conda_with_kind(file_contents, "lockfile")
+          parsed = parse_name_requirement_from_matchspec(dep)
+          Dependency.new(**parsed, type: "runtime", source: options.fetch(:filename, nil))
+        end.compact
       end
 
-      def self.parse_conda_with_kind(info, kind)
-        dependencies = call_conda_parser_web(info, kind)[kind.to_sym]
-        dependencies.map { |dep_kv| Dependency.new(**dep_kv.merge(type: "runtime")) }
-      end
+      def self.parse_name_requirement_from_matchspec(matchspec)
+        # simplified version of the implementation in conda to handle what we care about
+        # https://github.com/conda/conda/blob/main/conda/models/match_spec.py#L598
+        # (channel(/subdir):(namespace):)name(version(build))[key1=value1,key2=value2]
+        return if matchspec.end_with?("@")
 
-      private_class_method def self.call_conda_parser_web(file_contents, kind)
-        host = Bibliothecary.configuration.conda_parser_host
-        response = Typhoeus.post(
-          "#{host}/parse",
-          headers: {
-            ContentType: "multipart/form-data",
-          },
-          body: {
-            file: file_contents,
-            # Unfortunately we do not get the filename in the mapping parsers, so hardcoding the file name depending on the kind
-            filename: kind == "manifest" ? "environment.yml" : "environment.yml.lock",
-          }
-        )
-        raise Bibliothecary::RemoteParsingError.new("Http Error #{response.response_code} when contacting: #{host}/parse", response.response_code) unless response.success?
+        # strip off comments and optional features
+        matchspec = matchspec.split("#", 2).first
+        matchspec = matchspec.split(" if ", 2).first
+
+        # strip off brackets
+        matchspec = matchspec.match(/^(.*)(?:\[(.*)\])?$/)[1]
 
-        JSON.parse(response.body, symbolize_names: true)
+        # strip off any parens
+        matchspec = matchspec.match(/^(.*)(?:(\(.*\)))?$/)[1]
+
+        # deal with channel and namespace, I wish there was rsplit in ruby
+        split = matchspec.reverse.split(":", 2)
+        matchspec = split.last.reverse
+
+        # split the name from the version/build combo
+        matches = matchspec.match(/([^ =<>!~]+)?([><!=~ ].+)?/)
+        name = matches[1]
+        version_build = matches[2]
+
+        version = nil
+        if matches && matches[2]
+          version_build = matches[2]
+          # and now deal with getting the version from version/build
+          matches = version_build.match(/((?:.+?)[^><!,|]?)(?:(?<![=!|,<>~])(?:[ =])([^-=,|<>~]+?))?$/)
+          version = if matches
+                      matches[1].strip
+                    else
+                      version_build.strip
+                    end
+        end
+        # if it's an exact requirement, lose the =
+        if version&.start_with?("==")
+          version = version[2..]
+        elsif version&.start_with?("=")
+          version = version[1..]
+        end
+
+        {
+          name: name,
+          requirement: version || "", # NOTE: this ignores build info
+        }
       end
     end
   end

data/lib/bibliothecary/parsers/cpan.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "yaml"
 require "json"
 
@@ -21,16 +23,16 @@ module Bibliothecary
 
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
-      def self.parse_json_manifest(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
+      def self.parse_json_manifest(file_contents, options: {})
         manifest = JSON.parse file_contents
         manifest["prereqs"].map do |_group, deps|
-          map_dependencies(deps, "requires", "runtime")
+          map_dependencies(deps, "requires", "runtime", options.fetch(:filename, nil))
         end.flatten
       end
 
-      def self.parse_yaml_manifest(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
+      def self.parse_yaml_manifest(file_contents, options: {})
         manifest = YAML.load file_contents
-        map_dependencies(manifest, "requires", "runtime")
+        map_dependencies(manifest, "requires", "runtime", options.fetch(:filename, nil))
       end
     end
   end

data/lib/bibliothecary/parsers/cran.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "deb_control"
 
 module Bibliothecary
@@ -20,16 +22,17 @@ module Bibliothecary
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
       add_multi_parser(Bibliothecary::MultiParsers::Spdx)
 
-      def self.parse_description(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
+      def self.parse_description(file_contents, options: {})
         manifest = DebControl::ControlFileBase.parse(file_contents)
-        parse_section(manifest, "Depends") +
-        parse_section(manifest, "Imports") +
-        parse_section(manifest, "Suggests") +
-        parse_section(manifest, "Enhances")
+        parse_section(manifest, "Depends", options.fetch(:filename, nil)) +
+          parse_section(manifest, "Imports", options.fetch(:filename, nil)) +
+          parse_section(manifest, "Suggests", options.fetch(:filename, nil)) +
+          parse_section(manifest, "Enhances", options.fetch(:filename, nil))
       end
 
-      def self.parse_section(manifest, name)
+      def self.parse_section(manifest, name, source = nil)
         return [] unless manifest.first[name]
+
         deps = manifest.first[name].delete("\n").split(",").map(&:strip)
         deps.map do |dependency|
           dep = dependency.match(REQUIRE_REGEXP)
@@ -37,6 +40,7 @@ module Bibliothecary
             name: dep[1],
             requirement: dep[2],
             type: name.downcase,
+            source: source
           )
         end
       end

data/lib/bibliothecary/parsers/dub.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "json"
 require "sdl_parser"
 
@@ -22,8 +24,8 @@ module Bibliothecary
 
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
-      def self.parse_sdl_manifest(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
-        SdlParser.new(:runtime, file_contents).dependencies
+      def self.parse_sdl_manifest(file_contents, options: {})
+        SdlParser.new(:runtime, file_contents, options.fetch(:filename, nil)).dependencies
       end
     end
   end

data/lib/bibliothecary/parsers/elm.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "json"
 
 module Bibliothecary
@@ -21,13 +23,14 @@ module Bibliothecary
 
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
-      def self.parse_json_lock(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
+      def self.parse_json_lock(file_contents, options: {})
         manifest = JSON.parse file_contents
         manifest.map do |name, requirement|
           Dependency.new(
             name: name,
             requirement: requirement,
             type: "runtime",
+            source: options.fetch(:filename, nil)
           )
         end
       end