bibliothecary 11.0.1 → 12.1.0

data/lib/bibliothecary/parsers/pypi.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Bibliothecary
   module Parsers
     class Pypi
@@ -24,15 +26,15 @@ module Bibliothecary
                           "requirements-docs.txt", "requirements/docs.txt",
                           "requirements-test.txt", "requirements/test.txt",
                           "requirements-tools.txt", "requirements/tools.txt") => {
-            kind: "manifest",
-            parser: :parse_requirements_txt,
-          },
-          lambda { |p| PIP_COMPILE_REGEXP.match(p) } => {
+                            kind: "manifest",
+                            parser: :parse_requirements_txt,
+                          },
+          ->(p) { PIP_COMPILE_REGEXP.match(p) } => {
             content_matcher: :pip_compile?,
             kind: "lockfile",
             parser: :parse_requirements_txt,
           },
-          lambda { |p| MANIFEST_REGEXP.match(p) } => {
+          ->(p) { MANIFEST_REGEXP.match(p) } => {
             kind: "manifest",
             parser: :parse_requirements_txt,
             can_have_lockfile: false,
@@ -79,14 +81,6 @@ module Bibliothecary
             parser: :parse_conda,
             kind: "manifest",
           },
-          match_filename("environment.yml.lock") => {
-            parser: :parse_conda,
-            kind: "lockfile",
-          },
-          match_filename("environment.yaml.lock") => {
-            parser: :parse_conda,
-            kind: "lockfile",
-          },
         }
       end
 
@@ -94,33 +88,34 @@ module Bibliothecary
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
       add_multi_parser(Bibliothecary::MultiParsers::Spdx)
 
-      def self.parse_pipfile(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
+      def self.parse_pipfile(file_contents, options: {})
         manifest = Tomlrb.parse(file_contents)
-        map_dependencies(manifest["packages"], "runtime") + map_dependencies(manifest["dev-packages"], "develop")
+        map_dependencies(manifest["packages"], "runtime", options.fetch(:filename, nil)) +
+          map_dependencies(manifest["dev-packages"], "develop", options.fetch(:filename, nil))
       end
 
-      def self.parse_pyproject(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
+      def self.parse_pyproject(file_contents, options: {})
         deps = []
 
         file_contents = Tomlrb.parse(file_contents)
 
         # Parse poetry [tool.poetry] deps
         poetry_manifest = file_contents.fetch("tool", {}).fetch("poetry", {})
-        deps += map_dependencies(poetry_manifest["dependencies"], "runtime")
+        deps += map_dependencies(poetry_manifest["dependencies"], "runtime", options.fetch(:filename, nil))
         # Poetry 1.0.0-1.2.0 way of defining dev deps
-        deps += map_dependencies(poetry_manifest["dev-dependencies"], "develop")
+        deps += map_dependencies(poetry_manifest["dev-dependencies"], "develop", options.fetch(:filename, nil))
         # Poetry's 1.2.0+ of defining dev deps
         poetry_manifest
           .fetch("group", {})
           .each_pair do |group_name, obj|
             group_name = "develop" if group_name == "dev"
-            deps += map_dependencies(obj.fetch("dependencies", {}), group_name)
+            deps += map_dependencies(obj.fetch("dependencies", {}), group_name, options.fetch(:filename, nil))
           end
 
         # Parse PEP621 [project] deps
         pep621_manifest = file_contents.fetch("project", {})
         pep621_deps = pep621_manifest.fetch("dependencies", []).map { |d| parse_pep_508_dep_spec(d) }
-        deps += map_dependencies(pep621_deps, "runtime")
+        deps += map_dependencies(pep621_deps, "runtime", options.fetch(:filename, nil))
 
         # We're combining both poetry+PEP621 deps instead of making them mutually exclusive, until we
         # find a reason not to ingest them both.
@@ -133,24 +128,26 @@ module Bibliothecary
         parse_pyproject(file_contents, options)
       end
 
-      def self.parse_conda(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
+      def self.parse_conda(file_contents, options: {})
         contents = YAML.safe_load(file_contents)
         return [] unless contents
 
         dependencies = contents["dependencies"]
-        pip = dependencies.find { |dep| dep.is_a?(Hash) && dep["pip"]}
+        pip = dependencies.find { |dep| dep.is_a?(Hash) && dep["pip"] }
         return [] unless pip
 
-        Pypi.parse_requirements_txt(pip["pip"].join("\n"))
+        Pypi.parse_requirements_txt(pip["pip"].join("\n"), options:)
       end
 
-      def self.map_dependencies(packages, type)
+      def self.map_dependencies(packages, type, source = nil)
         return [] unless packages
+
         packages.map do |name, info|
           Dependency.new(
             name: name,
             requirement: map_requirements(info),
             type: type,
+            source: source
           )
         end
       end
@@ -160,7 +157,7 @@ module Bibliothecary
           if info["version"]
             info["version"]
           elsif info["git"]
-            info["git"] + "#" + info["ref"]
+            "#{info['git']}##{info['ref']}"
           else
             "*"
           end
@@ -169,24 +166,26 @@ module Bibliothecary
         end
       end
 
-      def self.parse_pipfile_lock(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
+      def self.parse_pipfile_lock(file_contents, options: {})
         manifest = JSON.parse(file_contents)
         deps = []
         manifest.each do |group, dependencies|
           next if group == "_meta"
+
           group = "runtime" if group == "default"
           dependencies.each do |name, info|
             deps << Dependency.new(
               name: name,
               requirement: map_requirements(info),
               type: group,
+              source: options.fetch(:filename, nil)
             )
           end
         end
         deps
       end
 
-      def self.parse_poetry_lock(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
+      def self.parse_poetry_lock(file_contents, options: {})
         manifest = Tomlrb.parse(file_contents)
         deps = []
         manifest["package"].each do |package|
@@ -202,23 +201,28 @@ module Bibliothecary
             name: package["name"],
             requirement: map_requirements(package),
             type: group,
+            source: options.fetch(:filename, nil)
           )
         end
         deps
       end
 
-      def self.parse_setup_py(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
+      def self.parse_setup_py(file_contents, options: {})
         match = file_contents.match(INSTALL_REGEXP)
         return [] unless match
+
         deps = []
         match[1].gsub(/',(\s)?'/, "\n").split("\n").each do |line|
           next if line.match(/^#/)
+
           match = line.match(REQUIRE_REGEXP)
           next unless match
+
           deps << Dependency.new(
             name: match[1],
             requirement: match[-1],
             type: "runtime",
+            source: options.fetch(:filename, nil)
           )
         end
         deps
@@ -234,9 +238,10 @@ module Bibliothecary
         JSON.parse(file_contents)
           .map do |pkg|
             Dependency.new(
-                name: pkg.dig("package", "package_name"),
-                requirement: pkg.dig("package", "installed_version"),
-                type: "runtime",
+              name: pkg.dig("package", "package_name"),
+              requirement: pkg.dig("package", "installed_version"),
+              type: "runtime",
+              source: options.fetch(:filename, nil)
             )
           end
           .uniq
@@ -246,7 +251,7 @@ module Bibliothecary
       # https://pip.pypa.io/en/stable/cli/pip_install/#requirement-specifiers
       # and https://pip.pypa.io/en/stable/topics/vcs-support/#git.
       # Invalid lines in requirements.txt are skipped.
-      def self.parse_requirements_txt(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
+      def self.parse_requirements_txt(file_contents, options: {})
         deps = []
         type = case options[:filename]
                when /dev/ || /docs/ || /tools/
@@ -260,7 +265,7 @@ module Bibliothecary
         file_contents.split("\n").each do |line|
           if line["://"]
             begin
-              result = parse_requirements_txt_url(line, type)
+              result = parse_requirements_txt_url(line, type, options.fetch(:filename, nil))
             rescue URI::Error, NoEggSpecified
               next
             end
@@ -271,6 +276,7 @@ module Bibliothecary
               name: match[1],
               requirement: match[-1],
               type: type,
+              source: options.fetch(:filename, nil)
             )
           end
         end
@@ -278,7 +284,7 @@ module Bibliothecary
         deps.uniq
       end
 
-      def self.parse_requirements_txt_url(url, type=nil)
+      def self.parse_requirements_txt_url(url, type = nil, source = nil)
         uri = URI.parse(url)
         raise NoEggSpecified, "No egg specified in #{url}" unless uri.fragment
 
@@ -287,11 +293,16 @@ module Bibliothecary
 
         requirement = uri.path[/@(.+)$/, 1]
 
-        Dependency.new(name: name, requirement: requirement, type: type)
+        Dependency.new(
+          name: name,
+          requirement: requirement,
+          type: type,
+          source: source
+        )
       end
 
       def self.pip_compile?(file_contents)
-        return file_contents.include?("This file is autogenerated by pip-compile")
+        file_contents.include?("This file is autogenerated by pip-compile")
       rescue Exception # rubocop:disable Lint/RescueException
         # We rescue exception here since native libs can throw a non-StandardError
         # We don't want to throw errors during the matching phase, only during
@@ -305,7 +316,7 @@ module Bibliothecary
         name, requirement = dep.split(PEP_508_NAME_REGEXP, 2).last(2).map(&:strip)
         requirement = requirement.sub(/^[\s;]*/, "")
         requirement = "*" if requirement == ""
-        return name, requirement
+        [name, requirement]
       end
     end
   end

data/lib/bibliothecary/parsers/rubygems.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "gemnasium/parser"
 
 module Bibliothecary
@@ -6,7 +8,7 @@ module Bibliothecary
       include Bibliothecary::Analyser
       extend Bibliothecary::MultiParsers::BundlerLikeManifest
 
-      NAME_VERSION = '(?! )(.*?)(?: \(([^-]*)(?:-(.*))?\))?'.freeze
+      NAME_VERSION = '(?! )(.*?)(?: \(([^-]*)(?:-(.*))?\))?'
       NAME_VERSION_4 = /^ {4}#{NAME_VERSION}$/
       BUNDLED_WITH = /BUNDLED WITH/
 
@@ -15,17 +17,17 @@ module Bibliothecary
           match_filenames("Gemfile", "gems.rb") => {
             kind: "manifest",
             parser: :parse_gemfile,
-            related_to: [ "manifest", "lockfile" ],
+            related_to: %w[manifest lockfile],
           },
           match_extension(".gemspec") => {
             kind: "manifest",
             parser: :parse_gemspec,
-            related_to: [ "manifest", "lockfile" ],
+            related_to: %w[manifest lockfile],
           },
           match_filenames("Gemfile.lock", "gems.locked") => {
             kind: "lockfile",
             parser: :parse_gemfile_lock,
-            related_to: [ "manifest", "lockfile" ],
+            related_to: %w[manifest lockfile],
           },
         }
       end
@@ -34,7 +36,7 @@ module Bibliothecary
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
       add_multi_parser(Bibliothecary::MultiParsers::Spdx)
 
-      def self.parse_gemfile_lock(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
+      def self.parse_gemfile_lock(file_contents, options: {})
         file_contents.lines(chomp: true).map do |line|
           match = line.match(NAME_VERSION_4)
           bundler_match = line.match(BUNDLED_WITH)
@@ -42,29 +44,30 @@ module Bibliothecary
 
           if match
             name = match[1]
-            version = match[2].gsub(/\(|\)/,"")
+            version = match[2].gsub(/\(|\)/, "")
             Dependency.new(
               name: name,
               requirement: version,
               type: "runtime",
+              source: options.fetch(:filename, nil)
             )
           else
-            parse_bundler(file_contents)
+            parse_bundler(file_contents, options.fetch(:filename, nil))
           end
         end.compact
       end
 
-      def self.parse_gemfile(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
+      def self.parse_gemfile(file_contents, options: {})
         manifest = Gemnasium::Parser.send(:gemfile, file_contents)
-        parse_ruby_manifest(manifest)
+        parse_ruby_manifest(manifest, options.fetch(:filename, nil))
       end
 
-      def self.parse_gemspec(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
+      def self.parse_gemspec(file_contents, options: {})
         manifest = Gemnasium::Parser.send(:gemspec, file_contents)
-        parse_ruby_manifest(manifest)
+        parse_ruby_manifest(manifest, options.fetch(:filename, nil))
       end
 
-      def self.parse_bundler(file_contents)
+      def self.parse_bundler(file_contents, source = nil)
         bundled_with_index = file_contents.lines(chomp: true).find_index { |line| line.match(BUNDLED_WITH) }
         version = file_contents.lines(chomp: true).fetch(bundled_with_index + 1)&.strip
 
@@ -74,6 +77,7 @@ module Bibliothecary
           name: "bundler",
           requirement: version,
           type: "runtime",
+          source: source
         )
       end
     end

data/lib/bibliothecary/parsers/shard.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "yaml"
 
 module Bibliothecary
@@ -20,23 +22,24 @@ module Bibliothecary
 
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
-      def self.parse_yaml_lockfile(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
+      def self.parse_yaml_lockfile(file_contents, options: {})
         manifest = YAML.load file_contents
-        map_dependencies(manifest, "shards", "runtime")
+        map_dependencies(manifest, "shards", "runtime", options.fetch(:filename, nil))
       end
 
-      def self.parse_yaml_manifest(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
+      def self.parse_yaml_manifest(file_contents, options: {})
         manifest = YAML.load file_contents
-        map_dependencies(manifest, "dependencies", "runtime") +
-        map_dependencies(manifest, "development_dependencies", "runtime")
+        map_dependencies(manifest, "dependencies", "runtime", options.fetch(:filename, nil)) +
+          map_dependencies(manifest, "development_dependencies", "runtime", options.fetch(:filename, nil))
       end
 
-      def self.map_dependencies(hash, key, type)
-        hash.fetch(key,[]).map do |name, requirement|
+      def self.map_dependencies(hash, key, type, source = nil)
+        hash.fetch(key, []).map do |name, requirement|
           Dependency.new(
             name: name,
             requirement: requirement["version"],
             type: type,
+            source: source
           )
         end
       end

data/lib/bibliothecary/purl_util.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Bibliothecary
   class PurlUtil
     # If a purl type (key) exists, it will be used in a manifest for
@@ -13,14 +15,10 @@ module Bibliothecary
       "conda" => :conda,
       "cran" => :cran,
       "gem" => :rubygems,
-      "hackage" => :hackage,
-      "hex" => :hex,
       "nuget" => :nuget,
       "pypi" => :pypi,
-      "swift" => :swift_pm,
     }.freeze
 
-
     # @param purl [PackageURL]
     # @return [String] The properly namespaced package name
     def self.full_name(purl)

data/lib/bibliothecary/related_files_info.rb

@@ -1,17 +1,16 @@
+# frozen_string_literal: true
+
 module Bibliothecary
   class RelatedFilesInfo
-    attr_reader :path
-    attr_reader :platform
-    attr_reader :manifests
-    attr_reader :lockfiles
+    attr_reader :path, :platform, :manifests, :lockfiles
 
     # Create a set of RelatedFilesInfo for the provided file_infos,
-    # where each RelatedFilesInfo contains all the file_infos 
+    # where each RelatedFilesInfo contains all the file_infos
     def self.create_from_file_infos(file_infos)
       returns = []
 
       file_infos_by_directory = file_infos.group_by { |info| File.dirname(info.relative_path) }
-      file_infos_by_directory.values.each do |file_infos_for_path|
+      file_infos_by_directory.each_value do |file_infos_for_path|
         groupable, ungroupable = file_infos_for_path.partition(&:groupable?)
 
         # add ungroupable ones as separate RFIs
@@ -19,9 +18,9 @@ module Bibliothecary
           returns.append(RelatedFilesInfo.new([file_info]))
         end
 
-        file_infos_by_directory_by_package_manager = groupable.group_by { |info| info.package_manager}
+        file_infos_by_directory_by_package_manager = groupable.group_by(&:package_manager)
 
-        file_infos_by_directory_by_package_manager.values.each do |file_infos_in_directory_for_package_manager|
+        file_infos_by_directory_by_package_manager.each_value do |file_infos_in_directory_for_package_manager|
           returns.append(RelatedFilesInfo.new(file_infos_in_directory_for_package_manager))
         end
       end

data/lib/bibliothecary/runner/multi_manifest_filter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Bibliothecary
   class Runner
     class MultiManifestFilter
@@ -11,8 +13,8 @@ module Bibliothecary
         # @return [Boolean] True if we should skip processing
         def skip?
           !@file_analysis ||
-          !@file_analysis[:dependencies] ||
-          @file_analysis[:dependencies].empty?
+            !@file_analysis[:dependencies] ||
+            @file_analysis[:dependencies].empty?
         end
       end
 
@@ -80,7 +82,7 @@ module Bibliothecary
       end
 
       def partition_file_entries!
-        @single_file_entries, @multiple_file_entries = files_to_check.partition { |_file, count| count == 1  }
+        @single_file_entries, @multiple_file_entries = files_to_check.partition { |_file, count| count == 1 }
 
         @single_file_entries = @single_file_entries.map(&:first)
         @multiple_file_entries = @multiple_file_entries.map(&:first)
@@ -88,4 +90,3 @@ module Bibliothecary
     end
   end
 end
-

data/lib/bibliothecary/runner.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Bibliothecary
   # A class that allows bibliothecary to run with multiple configurations at once, rather than with one global.
   # A runner is created every time a file is targeted to be parsed. Don't call
@@ -24,8 +26,8 @@ module Bibliothecary
       analyses = analyses.flatten.compact
 
       info_list.select { |info| info.package_manager.nil? }.each do |info|
-        analyses.push(Bibliothecary::Analyser::create_error_analysis("unknown", info.relative_path, "unknown",
-                                                                     "No parser for this file type"))
+        analyses.push(Bibliothecary::Analyser.create_error_analysis("unknown", info.relative_path, "unknown",
+                                                                    "No parser for this file type"))
       end
 
       analyses
@@ -34,16 +36,16 @@ module Bibliothecary
 
     # deprecated; use load_file_info_list.
     def load_file_list(path)
-      load_file_info_list(path).map { |info| info.full_path }
+      load_file_info_list(path).map(&:full_path)
     end
 
     def applicable_package_managers(info)
       managers = package_managers.select { |pm| pm.match_info?(info) }
-      managers.length > 0 ? managers : [nil]
+      managers.empty? ? [nil] : managers
     end
 
     def package_managers
-      Bibliothecary::Parsers.constants.map{|c| Bibliothecary::Parsers.const_get(c) }.sort_by{|c| c.to_s.downcase }
+      Bibliothecary::Parsers.constants.map { |c| Bibliothecary::Parsers.const_get(c) }.sort_by { |c| c.to_s.downcase }
     end
 
     # Parses an array of format [{file_path: "", contents: ""},] to match
@@ -130,13 +132,13 @@ module Bibliothecary
     # because this API is used by libraries.io and we don't want to
     # download all .xml files from GitHub.
     def identify_manifests(file_list)
-      ignored_dirs_with_slash = ignored_dirs.map { |d| if d.end_with?("/") then d else d + "/" end }
+      ignored_dirs_with_slash = ignored_dirs.map { |d| d.end_with?("/") ? d : "#{d}/" }
       allowed_file_list = file_list.reject do |f|
         ignored_dirs.include?(f) || f.start_with?(*ignored_dirs_with_slash)
       end
-      allowed_file_list = allowed_file_list.reject{|f| ignored_files.include?(f)}
+      allowed_file_list = allowed_file_list.reject { |f| ignored_files.include?(f) }
       package_managers.map do |pm|
-        allowed_file_list.select do |file_path|
+        allowed_file_list.select do |file_path| # rubocop:disable  Style/SelectByRegexp (this is a rubocop false positive, match? is a custom method)
           # this is a call to match? without file contents, which will skip
           # ambiguous filenames that are only possibly a manifest
           pm.match?(file_path)
@@ -159,7 +161,7 @@ module Bibliothecary
     # This means we're likely analyzing these files twice in processing,
     # but we need that accurate package manager information.
     def filter_multi_manifest_entries(path, related_files_info_entries)
-      MultiManifestFilter.new(path: path, related_files_info_entries: related_files_info_entries , runner: self).results
+      MultiManifestFilter.new(path: path, related_files_info_entries: related_files_info_entries, runner: self).results
     end
 
     private
@@ -178,4 +180,4 @@ module Bibliothecary
   end
 end
 
-require_relative "./runner/multi_manifest_filter.rb"
+require_relative "runner/multi_manifest_filter"

data/lib/bibliothecary/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Bibliothecary
-  VERSION = "11.0.1"
+  VERSION = "12.1.0"
 end

data/lib/bibliothecary.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "bibliothecary/version"
 require "bibliothecary/dependency"
 require "bibliothecary/analyser"
@@ -10,16 +12,16 @@ require "bibliothecary/purl_util"
 require "find"
 require "tomlrb"
 
-Dir[File.expand_path("../bibliothecary/multi_parsers/*.rb", __FILE__)].each do |file|
+Dir[File.expand_path("bibliothecary/multi_parsers/*.rb", __dir__)].each do |file|
   require file
 end
-Dir[File.expand_path("../bibliothecary/parsers/*.rb", __FILE__)].each do |file|
+Dir[File.expand_path("bibliothecary/parsers/*.rb", __dir__)].each do |file|
   require file
 end
 
 module Bibliothecary
-  VERSION_OPERATORS = /[~^<>*"]/.freeze
-  INVALID_UTF8_ERROR_REGEXP = /invalid byte sequence/.freeze
+  VERSION_OPERATORS = /[~^<>*"]/
+  INVALID_UTF8_ERROR_REGEXP = /invalid byte sequence/
 
   def self.analyse(path, ignore_unparseable_files: true)
     runner.analyse(path, ignore_unparseable_files: ignore_unparseable_files)
@@ -86,6 +88,7 @@ module Bibliothecary
   rescue ArgumentError => e
     # Bibliothecary doesn't need to analyze non-UTF8 files like binary files, so just return blank.
     return "" if e.message.match?(INVALID_UTF8_ERROR_REGEXP)
+
     raise e
   end
 

data/lib/sdl_parser.rb

@@ -1,23 +1,28 @@
+# frozen_string_literal: true
+
 require "sdl4r"
 
 class SdlParser
   attr_reader :contents, :type
-  def initialize(type, contents)
+
+  def initialize(type, contents, source = nil)
     @contents = contents
     @type = type || "runtime"
+    @source = source
   end
 
   def dependencies
     parse.children("dependency").inject([]) do |deps, dep|
       deps.push(Bibliothecary::Dependency.new(
-        name: dep.value,
-        requirement: dep.attribute("version") || ">= 0",
-        type: type,
-      ))
+                  name: dep.value,
+                  requirement: dep.attribute("version") || ">= 0",
+                  type: type,
+                  source: @source
+                ))
     end.uniq
   end
 
   def parse
-    SDL4R::read(contents)
+    SDL4R.read(contents)
   end
 end