bibliothecary 8.7.4 → 8.7.6

data/lib/bibliothecary/parsers/cocoapods.rb

@@ -1,5 +1,5 @@
-require 'gemnasium/parser'
-require 'yaml'
+require "gemnasium/parser"
+require "yaml"
 
 module Bibliothecary
   module Parsers
@@ -13,58 +13,58 @@ module Bibliothecary
       def self.mapping
         {
           match_filename("Podfile") => {
-            kind: 'manifest',
-            parser: :parse_podfile
+            kind: "manifest",
+            parser: :parse_podfile,
           },
           match_extension(".podspec") => {
-            kind: 'manifest',
+            kind: "manifest",
             parser: :parse_podspec,
-            can_have_lockfile: false
+            can_have_lockfile: false,
           },
           match_filename("Podfile.lock") => {
-            kind: 'lockfile',
-            parser: :parse_podfile_lock
+            kind: "lockfile",
+            parser: :parse_podfile_lock,
           },
           match_extension(".podspec.json") => {
-            kind: 'manifest',
+            kind: "manifest",
             parser: :parse_json_manifest,
-            can_have_lockfile: false
-          }
+            can_have_lockfile: false,
+          },
         }
       end
 
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
-      def self.parse_podfile_lock(file_contents, options: {})
+      def self.parse_podfile_lock(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
         manifest = YAML.load file_contents
-        manifest['PODS'].map do |row|
+        manifest["PODS"].map do |row|
           pod = row.is_a?(String) ? row : row.keys.first
           match = pod.match(/(.+?)\s\((.+?)\)/i)
           {
-            name: match[1].split('/').first,
+            name: match[1].split("/").first,
             requirement: match[2],
-            type: 'runtime'
+            type: "runtime",
           }
         end.compact
       end
 
-      def self.parse_podspec(file_contents, options: {})
+      def self.parse_podspec(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
         manifest = Gemnasium::Parser.send(:podspec, file_contents)
         parse_ruby_manifest(manifest)
       end
 
-      def self.parse_podfile(file_contents, options: {})
+      def self.parse_podfile(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
         manifest = Gemnasium::Parser.send(:podfile, file_contents)
         parse_ruby_manifest(manifest)
       end
 
-      def self.parse_json_manifest(file_contents, options: {})
+      def self.parse_json_manifest(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
         manifest = JSON.parse(file_contents)
-        manifest['dependencies'].inject([]) do |deps, dep|
+        manifest["dependencies"].inject([]) do |deps, dep|
           deps.push({
             name: dep[0],
             requirement: dep[1],
-            type: 'runtime'
+            type: "runtime",
           })
         end.uniq
       end

data/lib/bibliothecary/parsers/conda.rb

@@ -9,20 +9,20 @@ module Bibliothecary
         {
           match_filename("environment.yml") => {
             parser: :parse_conda,
-            kind: "manifest"
+            kind: "manifest",
           },
           match_filename("environment.yaml") => {
             parser: :parse_conda,
-            kind: "manifest"
+            kind: "manifest",
           },
           match_filename("environment.yml.lock") => {
             parser: :parse_conda_lockfile,
-            kind: "lockfile"
+            kind: "lockfile",
           },
           match_filename("environment.yaml.lock") => {
             parser: :parse_conda_lockfile,
-            kind: "lockfile"
-          }
+            kind: "lockfile",
+          },
         }
       end
 
@@ -30,11 +30,11 @@ module Bibliothecary
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
       add_multi_parser(Bibliothecary::MultiParsers::Spdx)
 
-      def self.parse_conda(file_contents, options: {})
+      def self.parse_conda(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
         parse_conda_with_kind(file_contents, "manifest")
       end
 
-      def self.parse_conda_lockfile(file_contents, options: {})
+      def self.parse_conda_lockfile(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
         parse_conda_with_kind(file_contents, "lockfile")
       end
 
@@ -48,12 +48,12 @@ module Bibliothecary
         response = Typhoeus.post(
           "#{host}/parse",
           headers: {
-            ContentType: "multipart/form-data"
+            ContentType: "multipart/form-data",
           },
           body: {
             file: file_contents,
             # Unfortunately we do not get the filename in the mapping parsers, so hardcoding the file name depending on the kind
-            filename: kind == "manifest" ? "environment.yml" : "environment.yml.lock"
+            filename: kind == "manifest" ? "environment.yml" : "environment.yml.lock",
           }
         )
         raise Bibliothecary::RemoteParsingError.new("Http Error #{response.response_code} when contacting: #{host}/parse", response.response_code) unless response.success?

data/lib/bibliothecary/parsers/cpan.rb

@@ -1,5 +1,5 @@
-require 'yaml'
-require 'json'
+require "yaml"
+require "json"
 
 module Bibliothecary
   module Parsers
@@ -9,28 +9,28 @@ module Bibliothecary
       def self.mapping
         {
           match_filename("META.json", case_insensitive: true) => {
-            kind: 'manifest',
-            parser: :parse_json_manifest
+            kind: "manifest",
+            parser: :parse_json_manifest,
           },
           match_filename("META.yml", case_insensitive: true) => {
-            kind: 'manifest',
-            parser: :parse_yaml_manifest
-          }
+            kind: "manifest",
+            parser: :parse_yaml_manifest,
+          },
         }
       end
 
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
-      def self.parse_json_manifest(file_contents, options: {})
+      def self.parse_json_manifest(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
         manifest = JSON.parse file_contents
-        manifest['prereqs'].map do |_group, deps|
-          map_dependencies(deps, 'requires', 'runtime')
+        manifest["prereqs"].map do |_group, deps|
+          map_dependencies(deps, "requires", "runtime")
         end.flatten
       end
 
-      def self.parse_yaml_manifest(file_contents, options: {})
+      def self.parse_yaml_manifest(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
         manifest = YAML.load file_contents
-        map_dependencies(manifest, 'requires', 'runtime')
+        map_dependencies(manifest, "requires", "runtime")
       end
     end
   end

data/lib/bibliothecary/parsers/cran.rb

@@ -1,4 +1,4 @@
-require 'deb_control'
+require "deb_control"
 
 module Bibliothecary
   module Parsers
@@ -10,9 +10,9 @@ module Bibliothecary
       def self.mapping
         {
           match_filename("DESCRIPTION", case_insensitive: true) => {
-            kind: 'manifest',
-            parser: :parse_description
-          }
+            kind: "manifest",
+            parser: :parse_description,
+          },
         }
       end
 
@@ -20,23 +20,23 @@ module Bibliothecary
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
       add_multi_parser(Bibliothecary::MultiParsers::Spdx)
 
-      def self.parse_description(file_contents, options: {})
+      def self.parse_description(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
         manifest = DebControl::ControlFileBase.parse(file_contents)
-        parse_section(manifest, 'Depends') +
-        parse_section(manifest, 'Imports') +
-        parse_section(manifest, 'Suggests') +
-        parse_section(manifest, 'Enhances')
+        parse_section(manifest, "Depends") +
+        parse_section(manifest, "Imports") +
+        parse_section(manifest, "Suggests") +
+        parse_section(manifest, "Enhances")
       end
 
       def self.parse_section(manifest, name)
         return [] unless manifest.first[name]
-        deps = manifest.first[name].delete("\n").split(',').map(&:strip)
+        deps = manifest.first[name].delete("\n").split(",").map(&:strip)
         deps.map do |dependency|
           dep = dependency.match(REQUIRE_REGEXP)
           {
             name: dep[1],
-            requirement: dep[2] || '*',
-            type: name.downcase
+            requirement: dep[2] || "*",
+            type: name.downcase,
           }
         end
       end

data/lib/bibliothecary/parsers/dub.rb

@@ -1,5 +1,5 @@
-require 'json'
-require 'sdl_parser'
+require "json"
+require "sdl_parser"
 
 module Bibliothecary
   module Parsers
@@ -10,19 +10,19 @@ module Bibliothecary
       def self.mapping
         {
           match_filename("dub.json") => {
-            kind: 'manifest',
-            parser: :parse_json_runtime_manifest
+            kind: "manifest",
+            parser: :parse_json_runtime_manifest,
           },
           match_filename("dub.sdl") => {
-            kind: 'manifest',
-            parser: :parse_sdl_manifest
-          }
+            kind: "manifest",
+            parser: :parse_sdl_manifest,
+          },
         }
       end
 
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
-      def self.parse_sdl_manifest(file_contents, options: {})
+      def self.parse_sdl_manifest(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
         SdlParser.new(:runtime, file_contents).dependencies
       end
     end

data/lib/bibliothecary/parsers/elm.rb

@@ -1,4 +1,4 @@
-require 'json'
+require "json"
 
 module Bibliothecary
   module Parsers
@@ -9,25 +9,25 @@ module Bibliothecary
       def self.mapping
         {
           match_filenames("elm-package.json", "elm_dependencies.json") => {
-            kind: 'manifest',
-            parser: :parse_json_runtime_manifest
+            kind: "manifest",
+            parser: :parse_json_runtime_manifest,
           },
           match_filename("elm-stuff/exact-dependencies.json") => {
-            kind: 'lockfile',
-            parser: :parse_json_lock
-          }
+            kind: "lockfile",
+            parser: :parse_json_lock,
+          },
         }
       end
 
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
-      def self.parse_json_lock(file_contents, options: {})
+      def self.parse_json_lock(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
         manifest = JSON.parse file_contents
         manifest.map do |name, requirement|
           {
             name: name,
             requirement: requirement,
-            type: 'runtime'
+            type: "runtime",
           }
         end
       end

data/lib/bibliothecary/parsers/go.rb

@@ -1,5 +1,5 @@
-require 'yaml'
-require 'json'
+require "yaml"
+require "json"
 
 module Bibliothecary
   module Parsers
@@ -7,61 +7,66 @@ module Bibliothecary
       include Bibliothecary::Analyser
 
       GPM_REGEXP = /^(.+)\s+(.+)$/
-      GOMOD_REGEX = /^(require\s+)?(.+)\s+(.+)$/
-      GOMOD_IGNORABLE_REGEX = /^(\/\/|module\s|go\s|exclude\s|replace\s|require\s+\(|\))/m
-      GOSUM_REGEX = /^(.+)\s+(.+)\s+(.+)$/
+      GOMOD_REPLACEMENT_SEPARATOR_REGEXP = /\s=>\s/
+      GOMOD_DEP_REGEXP = /(?<name>\S+)\s?(?<requirement>[^\s=>]+)?/ # the " =>" negative character class is to make sure we don't capture the delimiter for "replace" deps
+      GOMOD_SINGLELINE_DEP_REGEXP = /^(?<category>require|exclude|replace|retract)\s+#{GOMOD_DEP_REGEXP}.*$/
+      GOMOD_MULTILINE_DEP_REGEXP = /^#{GOMOD_DEP_REGEXP}.*$/
+      GOMOD_MULTILINE_START_REGEXP = /^(?<category>require|exclude|replace|retract)\s+\(/
+      GOMOD_MULTILINE_END_REGEXP = /^\)/
+      GOMOD_COMMENT_REGEXP = /(\/\/(.*))/
+      GOSUM_REGEXP = /^(.+)\s+(.+)\s+(.+)$/
 
       def self.mapping
         {
           # Go Modules (recommended)
           match_filename("go.mod") => {
-            kind: 'manifest',
-            parser: :parse_go_mod
+            kind: "manifest",
+            parser: :parse_go_mod,
           },
           match_filename("go.sum") => {
-            kind: 'lockfile',
-            parser: :parse_go_sum
+            kind: "lockfile",
+            parser: :parse_go_sum,
           },
           # Glide (unmaintained: https://github.com/Masterminds/glide#go-modules)
           match_filename("glide.yaml") => {
-            kind: 'manifest',
-            parser: :parse_glide_yaml
+            kind: "manifest",
+            parser: :parse_glide_yaml,
           },
           match_filename("glide.lock") => {
-            kind: 'lockfile',
-            parser: :parse_glide_lockfile
+            kind: "lockfile",
+            parser: :parse_glide_lockfile,
           },
           # Godep (unmaintained: https://github.com/tools/godep)
           match_filename("Godeps/Godeps.json") => {
-            kind: 'manifest',
-            parser: :parse_godep_json
+            kind: "manifest",
+            parser: :parse_godep_json,
           },
           match_filename("Godeps", case_insensitive: true) => {
-            kind: 'manifest',
-            parser: :parse_gpm
+            kind: "manifest",
+            parser: :parse_gpm,
           },
           # Govendor (unmaintained: https://github.com/kardianos/govendor)
           match_filename("vendor/manifest") => {
-            kind: 'manifest',
-            parser: :parse_gb_manifest
+            kind: "manifest",
+            parser: :parse_gb_manifest,
           },
           match_filename("vendor/vendor.json") => {
-            kind: 'manifest',
-            parser: :parse_govendor
+            kind: "manifest",
+            parser: :parse_govendor,
           },
           # Go dep (deprecated: https://github.com/golang/dep#dep)
           match_filename("Gopkg.toml") => {
-            kind: 'manifest',
-            parser: :parse_dep_toml
+            kind: "manifest",
+            parser: :parse_dep_toml,
           },
           match_filename("Gopkg.lock") => {
-            kind: 'lockfile',
-            parser: :parse_dep_lockfile
+            kind: "lockfile",
+            parser: :parse_dep_lockfile,
           },
           match_filename("go-resolved-dependencies.json") => {
-            kind: 'lockfile',
-            parser: :parse_go_resolved
-          }
+            kind: "lockfile",
+            parser: :parse_go_resolved,
+          },
         }
       end
 
@@ -69,97 +74,172 @@ module Bibliothecary
       add_multi_parser(Bibliothecary::MultiParsers::Spdx)
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
-      def self.parse_godep_json(file_contents, options: {})
+      def self.parse_godep_json(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
         manifest = JSON.parse file_contents
-        map_dependencies(manifest, 'Deps', 'ImportPath', 'Rev', 'runtime')
+        map_dependencies(manifest, "Deps", "ImportPath", "Rev", "runtime")
       end
 
-      def self.parse_gpm(file_contents, options: {})
+      def self.parse_gpm(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
         deps = []
         file_contents.split("\n").each do |line|
-          match = line.gsub(/(\#(.*))/, '').match(GPM_REGEXP)
+          match = line.gsub(/(\#(.*))/, "").match(GPM_REGEXP)
           next unless match
           deps << {
             name: match[1].strip,
-            requirement: match[2].strip || '*',
-            type: 'runtime'
+            requirement: match[2].strip || "*",
+            type: "runtime",
           }
         end
         deps
       end
 
-      def self.parse_govendor(file_contents, options: {})
+      def self.parse_govendor(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
         manifest = JSON.load file_contents
-        map_dependencies(manifest, 'package', 'path', 'revision', 'runtime')
+        map_dependencies(manifest, "package", "path", "revision", "runtime")
       end
 
-      def self.parse_glide_yaml(file_contents, options: {})
+      def self.parse_glide_yaml(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
         manifest = YAML.load file_contents
-        map_dependencies(manifest, 'import', 'package', 'version', 'runtime') +
-        map_dependencies(manifest, 'devImports', 'package', 'version', 'development')
+        map_dependencies(manifest, "import", "package", "version", "runtime") +
+        map_dependencies(manifest, "devImports", "package", "version", "development")
       end
 
-      def self.parse_glide_lockfile(file_contents, options: {})
+      def self.parse_glide_lockfile(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
         manifest = YAML.load file_contents
-        map_dependencies(manifest, 'imports', 'name', 'version', 'runtime')
+        map_dependencies(manifest, "imports", "name", "version", "runtime")
       end
 
-      def self.parse_gb_manifest(file_contents, options: {})
+      def self.parse_gb_manifest(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
         manifest = JSON.parse file_contents
-        map_dependencies(manifest, 'dependencies', 'importpath', 'revision', 'runtime')
+        map_dependencies(manifest, "dependencies", "importpath", "revision", "runtime")
       end
 
-      def self.parse_dep_toml(file_contents, options: {})
+      def self.parse_dep_toml(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
         manifest = Tomlrb.parse file_contents
-        map_dependencies(manifest, 'constraint', 'name', 'version', 'runtime')
+        map_dependencies(manifest, "constraint", "name", "version", "runtime")
       end
 
-      def self.parse_dep_lockfile(file_contents, options: {})
+      def self.parse_dep_lockfile(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
         manifest = Tomlrb.parse file_contents
-        map_dependencies(manifest, 'projects', 'name', 'revision', 'runtime')
+        map_dependencies(manifest, "projects", "name", "revision", "runtime")
       end
 
-      def self.parse_go_mod(file_contents, options: {})
-        deps = []
-        file_contents.lines.map(&:strip).each do |line|
-          next if line.match(GOMOD_IGNORABLE_REGEX)
-          if match = line.gsub(/(\/\/(.*))/, '').match(GOMOD_REGEX)
-            deps << {
-              name: match[2].strip,
-              requirement: match[3].strip || '*',
-              type: 'runtime'
-            }
+      def self.parse_go_mod(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
+        categorized_deps = parse_go_mod_categorized_deps(file_contents)
+
+        deps = categorized_deps["require"]
+          .map do |dep|
+            # NOTE: A "replace" directive doesn't add the dep to the module graph unless the original dep is also in a "require" directive,
+            # so we need to track down replacements here and use those instead of the originals, if present.
+            #
+            # NOTE: The "replace" directive doesn't actually change the version reported from Go (e.g. "go mod graph"), it only changes
+            # the *source code*. So by replacing the deps here, we're giving more honest results than you'd get when asking go
+            # about the versions used.
+            replaced_dep = categorized_deps["replace"]
+              .find do |replacement_dep| 
+                replacement_dep[:original_name] == dep[:name] && 
+                  (replacement_dep[:original_requirement] == "*" || replacement_dep[:original_requirement] == dep[:requirement])
+              end
+              
+            replaced_dep || dep
           end
-        end
-        deps
+
+        return deps
       end
 
-      def self.parse_go_sum(file_contents, options: {})
+      def self.parse_go_mod_categorized_deps(file_contents)
+        current_multiline_category = nil
+        # docs: https://go.dev/ref/mod#go-mod-file-require
+        categorized_deps = {
+          "require" => [],
+          "exclude" => [], # these deps are not necessarily used by the module
+          "replace" => [], # these deps are not necessarily used by the module
+          "retract" => [], # TODO: these are not parsed correctly right now, but they shouldn't be returned in list of deps anyway.
+        }
+        file_contents
+          .lines
+          .reject { |line| line =~ /^#{GOMOD_COMMENT_REGEXP}/ } # ignore comment lines
+          .map { |line| line.strip.gsub(GOMOD_COMMENT_REGEXP, "") } # strip out trailing comments
+          .each do |line|
+            if line.match(GOMOD_MULTILINE_END_REGEXP) # detect the end of a multiline
+              current_multiline_category = nil
+            elsif (match = line.match(GOMOD_MULTILINE_START_REGEXP)) # or, detect the start of a multiline
+              current_multiline_category = match[1]
+            elsif (match = line.match(GOMOD_SINGLELINE_DEP_REGEXP)) # or, detect a singleline dep
+              categorized_deps[match[:category]] << go_mod_category_relative_dep(category: match[:category], line: line, match: match)
+            elsif (current_multiline_category && match = line.match(GOMOD_MULTILINE_DEP_REGEXP)) # otherwise, parse the multiline dep
+              categorized_deps[current_multiline_category] << go_mod_category_relative_dep(category: current_multiline_category, line: line, match: match) 
+            end
+          end
+        categorized_deps
+      end
+
+      def self.parse_go_sum(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
         deps = []
         file_contents.lines.map(&:strip).each do |line|
-          if match = line.match(GOSUM_REGEX)
+          if (match = line.match(GOSUM_REGEXP))
             deps << {
               name: match[1].strip,
-              requirement: match[2].strip.split('/').first || '*',
-              type: 'runtime'
+              requirement: match[2].strip.split("/").first || "*",
+              type: "runtime",
             }
           end
         end
         deps.uniq
       end
 
-      def self.parse_go_resolved(file_contents, options: {})
+      def self.parse_go_resolved(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
         JSON.parse(file_contents)
           .select { |dep| dep["Main"] != "true" }
-          .map { |dep| { name: dep["Path"], requirement: dep["Version"], type: dep.fetch("Scope") { "runtime" } } }
+          .map do |dep| 
+            if dep["Replace"].is_a?(String) && dep["Replace"] != "<nil>" && dep["Replace"] != ""
+              # NOTE: The "replace" directive doesn't actually change the version reported from Go (e.g. "go mod graph"), it only changes
+              # the *source code*. So by replacing the deps here, we're giving more honest results than you'd get when asking go
+              # about the versions used.
+              name, requirement = dep["Replace"].split(" ", 2)
+              requirement = "*" if requirement.to_s.strip == ""
+              { name: name, requirement: requirement, original_name: dep["Path"], original_requirement: dep["Version"], type: dep.fetch("Scope") { "runtime" } }
+            else
+              { name: dep["Path"], requirement: dep["Version"], type: dep.fetch("Scope") { "runtime" } }
+            end
+          end
       end
 
       def self.map_dependencies(manifest, attr_name, dep_attr_name, version_attr_name, type)
         manifest.fetch(attr_name,[]).map do |dependency|
           {
             name: dependency[dep_attr_name],
-            requirement: dependency[version_attr_name]  || '*',
-            type: type
+            requirement: dependency[version_attr_name]  || "*",
+            type: type,
+          }
+        end
+      end
+
+      # Returns our standard-ish dep Hash based on the category of dep matched ("require", "replace", etc.)
+      def self.go_mod_category_relative_dep(category:, line:, match:)
+        case category
+        when "replace" 
+          replacement_dep = line.split(GOMOD_REPLACEMENT_SEPARATOR_REGEXP, 2).last
+          replacement_match = replacement_dep.match(GOMOD_DEP_REGEXP)
+          {
+            original_name: match[:name],
+            original_requirement: match[:requirement] || "*",
+            name: replacement_match[:name],
+            requirement: replacement_match[:requirement] || "*",
+            type: "runtime",
+          }
+        when "retract"
+          {
+            name: match[:name],
+            requirement: match[:requirement] || "*",
+            type: "runtime",
+            deprecated: true,
+          }
+        else
+          {
+            name: match[:name],
+            requirement: match[:requirement] || "*",
+            type: "runtime",
           }
         end
       end