bibliothecary 8.3.4 → 8.3.7

data/Rakefile

@@ -3,4 +3,4 @@ require "rspec/core/rake_task"
 
 RSpec::Core::RakeTask.new(:spec)
 
-task :default => :spec
+task default: :spec

data/bibliothecary.gemspec

@@ -34,4 +34,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "webmock"
   spec.add_development_dependency "vcr"
+  spec.add_development_dependency "rubocop"
+  spec.add_development_dependency "rubocop-rails"
 end

data/lib/bibliothecary/multi_parsers/dependencies_csv.rb

@@ -41,8 +41,8 @@ module Bibliothecary
           "lockfile_requirement" => {
             match: [
               /^(lockfile |)requirement$/i,
-              /^version$/i,
-            ],
+              /^version$/i
+            ]
           },
           # Manifests have versions that can have operators.
           # However, since Bibliothecary only currently supports analyzing a
@@ -52,8 +52,8 @@ module Bibliothecary
           "requirement" => {
             match: [
               /^(lockfile |)requirement$/i,
-              /^version$/i,
-            ],
+              /^version$/i
+            ]
           },
           "type" => {
             default: "runtime",

data/lib/bibliothecary/parsers/cargo.rb

@@ -21,17 +21,23 @@ module Bibliothecary
 
       def self.parse_manifest(file_contents, options: {})
         manifest = Tomlrb.parse(file_contents)
-        manifest.fetch('dependencies', []).map do |name, requirement|
-          if requirement.respond_to?(:fetch)
-            requirement = requirement['version'] or next
+
+        parsed_dependencies = []
+
+        manifest.fetch_values('dependencies', 'dev-dependencies').each_with_index do |deps, index|
+          parsed_dependencies << deps.map do |name, requirement|
+            if requirement.respond_to?(:fetch)
+              requirement = requirement['version'] or next
+            end
+            {
+              name: name,
+              requirement: requirement,
+              type: index.zero? ? 'runtime' : 'development'
+            }
           end
-          {
-            name: name,
-            requirement: requirement,
-            type: 'runtime'
-          }
         end
-          .compact
+
+        parsed_dependencies.flatten.compact
       end
 
       def self.parse_lockfile(file_contents, options: {})

data/lib/bibliothecary/parsers/carthage.rb

@@ -35,7 +35,7 @@ module Bibliothecary
       end
 
       def self.map_dependencies(manifest, path)
-        response = Typhoeus.post("#{Bibliothecary.configuration.carthage_parser_host}/#{path}", params: {body: manifest})
+        response = Typhoeus.post("#{Bibliothecary.configuration.carthage_parser_host}/#{path}", params: { body: manifest })
         raise Bibliothecary::RemoteParsingError.new("Http Error #{response.response_code} when contacting: #{Bibliothecary.configuration.carthage_parser_host}/#{path}", response.response_code) unless response.success?
         json = JSON.parse(response.body)
 

data/lib/bibliothecary/parsers/conda.rb

@@ -9,20 +9,20 @@ module Bibliothecary
         {
           match_filename("environment.yml") => {
             parser: :parse_conda,
-            kind: "manifest",
+            kind: "manifest"
           },
           match_filename("environment.yaml") => {
             parser: :parse_conda,
-            kind: "manifest",
+            kind: "manifest"
           },
           match_filename("environment.yml.lock") => {
             parser: :parse_conda_lockfile,
-            kind: "lockfile",
+            kind: "lockfile"
           },
           match_filename("environment.yaml.lock") => {
             parser: :parse_conda_lockfile,
-            kind: "lockfile",
-          },
+            kind: "lockfile"
+          }
         }
       end
 
@@ -47,12 +47,12 @@ module Bibliothecary
         response = Typhoeus.post(
           "#{host}/parse",
           headers: {
-            ContentType: "multipart/form-data",
+            ContentType: "multipart/form-data"
           },
           body: {
             file: file_contents,
             # Unfortunately we do not get the filename in the mapping parsers, so hardcoding the file name depending on the kind
-            filename: kind == "manifest" ? "environment.yml" : "environment.yml.lock",
+            filename: kind == "manifest" ? "environment.yml" : "environment.yml.lock"
           }
         )
         raise Bibliothecary::RemoteParsingError.new("Http Error #{response.response_code} when contacting: #{host}/parse", response.response_code) unless response.success?

data/lib/bibliothecary/parsers/go.rb

@@ -61,7 +61,7 @@ module Bibliothecary
           match_filename("go-resolved-dependencies.json") => {
             kind: 'lockfile',
             parser: :parse_go_resolved
-          },
+          }
         }
       end
 

data/lib/bibliothecary/parsers/maven.rb

@@ -148,7 +148,6 @@ module Bibliothecary
 
           split = gradle_dep_match.captures[0]
 
-
           dep = line
             .split(split)[1].sub(/(\((c|n|\*)\))$/, "") # line ending legend: (c) means a dependency constraint, (n) means not resolved, or (*) means resolved previously, e.g. org.springframework.boot:spring-boot-starter-web:2.1.0.M3 (*)
             .sub(/ FAILED$/, "") # dependency could not be resolved (but still may have a version)
@@ -159,21 +158,28 @@ module Bibliothecary
           # \--- org.springframework.security:spring-security-test (n)
           next unless dep.length >= 3
 
-          dep_name = if dep.count == 6
-                       # get name from renamed package resolution "org:name:version -> renamed_org:name:version"
-                       dep[-3..-2]
-                     else
-                       # get name from version conflict resolution ("org:name:version -> version") and no-resolution ("org:name:version")
-                       dep[0..1]
-                     end
-
-          version = dep[-1]
-          {
-            name: dep_name.join(":"),
-            requirement: version,
-            type: type
-          }
-        end.compact.uniq {|item| [item[:name], item[:requirement], item[:type]]}
+          if dep.count == 6
+            # get name from renamed package resolution "org:name:version -> renamed_org:name:version"
+            {
+              original_name: dep[0,2].join(":"),
+              original_requirement: dep[2],
+              name: dep[-3..-2].join(":"),
+              requirement: dep[-1],
+              type: type
+            }
+          else
+            # get name from version conflict resolution ("org:name:version -> version") and no-resolution ("org:name:version")
+            {
+              name: dep[0..1].join(":"),
+              requirement: dep[-1],
+              type: type
+            }
+          end
+        end
+          .compact
+          # Prefer duplicate deps with the aliased ones first, so we don't lose the aliases in the next uniq step.
+          .sort_by { |dep| dep.key?(:original_name) || dep.key?(:original_requirement) ? 0 : 1 }
+          .uniq { |item| [item[:name], item[:requirement], item[:type]] }
       end
 
       def self.parse_maven_resolved(file_contents, options: {})

data/lib/bibliothecary/parsers/npm.rb

@@ -36,22 +36,16 @@ module Bibliothecary
       add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
-      def self.parse_shrinkwrap(file_contents, options: {})
-        manifest = JSON.parse(file_contents)
-        manifest.fetch('dependencies',[]).map do |name, requirement|
-          {
-            name: name,
-            requirement: requirement["version"],
-            type: 'runtime'
-          }
-        end
-      end
-
       def self.parse_package_lock(file_contents, options: {})
         manifest = JSON.parse(file_contents)
         parse_package_lock_deps_recursively(manifest.fetch('dependencies', []))
       end
 
+      class << self
+        # "package-lock.json" and "npm-shrinkwrap.json" have same format, so use same parsing logic
+        alias_method :parse_shrinkwrap, :parse_package_lock
+      end
+
       def self.parse_package_lock_deps_recursively(dependencies, depth=1)
         dependencies.flat_map do |name, requirement|
           type = requirement.fetch("dev", false) ? 'development' : 'runtime'
@@ -61,7 +55,7 @@ module Bibliothecary
             []
           else
             parse_package_lock_deps_recursively(requirement.fetch('dependencies', []), depth + 1)
-          end
+                               end
 
           [{
             name: name,
@@ -74,8 +68,12 @@ module Bibliothecary
       def self.parse_manifest(file_contents, options: {})
         manifest = JSON.parse(file_contents)
         raise "appears to be a lockfile rather than manifest format" if manifest.key?('lockfileVersion')
-        map_dependencies(manifest, 'dependencies', 'runtime') +
-        map_dependencies(manifest, 'devDependencies', 'development')
+        
+        (
+          map_dependencies(manifest, 'dependencies', 'runtime') +
+          map_dependencies(manifest, 'devDependencies', 'development')
+        )
+          .reject { |dep| dep[:name].start_with?("//") } # Omit comment keys. They are valid in package.json: https://groups.google.com/g/nodejs/c/NmL7jdeuw0M/m/yTqI05DRQrIJ
       end
 
       def self.parse_yarn_lock(file_contents, options: {})

data/lib/bibliothecary/parsers/nuget.rb

@@ -40,7 +40,7 @@ module Bibliothecary
           match_filename("project.assets.json") => {
             kind: 'lockfile',
             parser: :parse_project_assets_json
-          },
+          }
         }
       end
 
@@ -80,7 +80,7 @@ module Bibliothecary
           # do that yet so at least pick deterministically.
 
           # Note, frameworks can be empty, so remove empty ones and then return the last sorted item if any
-          frameworks = frameworks.delete_if { |k, v| v.empty? }
+          frameworks = frameworks.delete_if { |_k, v| v.empty? }
           return frameworks[frameworks.keys.sort.last] unless frameworks.empty?
         end
         []
@@ -92,7 +92,7 @@ module Bibliothecary
           {
             name: dependency.id,
             requirement: (dependency.version if dependency.respond_to? "version") || "*",
-            type: 'runtime'
+            type: dependency.respond_to?("developmentDependency") && dependency.developmentDependency == "true" ? 'development' : 'runtime'
           }
         end
       rescue
@@ -102,16 +102,22 @@ module Bibliothecary
       def self.parse_csproj(file_contents, options: {})
         manifest = Ox.parse file_contents
 
-        packages = manifest.locate('ItemGroup/PackageReference').map do |dependency|
+        packages = manifest.locate('ItemGroup/PackageReference').select{ |dep| dep.respond_to? "Include" }.map do |dependency|
           requirement = (dependency.Version if dependency.respond_to? "Version") || "*"
           if requirement.is_a?(Ox::Element)
             requirement = dependency.nodes.detect{ |n| n.value == "Version" }&.text
           end
 
+          type = if dependency.nodes.first && dependency.nodes.first.nodes.include?("all") && dependency.nodes.first.value.include?("PrivateAssets") || dependency.attributes[:PrivateAssets] == "All"
+                   "development"
+                 else
+                   "runtime"
+                 end
+
           {
             name: dependency.Include,
             requirement: requirement,
-            type: 'runtime'
+            type: type
           }
         end
         packages.uniq {|package| package[:name] }
@@ -125,7 +131,7 @@ module Bibliothecary
           {
             name: dependency.id,
             requirement: dependency.attributes[:version] || '*',
-            type: 'runtime'
+            type: dependency.respond_to?("developmentDependency") && dependency.developmentDependency == "true" ? 'development' : 'runtime'
           }
         end
       rescue
@@ -152,8 +158,8 @@ module Bibliothecary
         frameworks = {}
         manifest.fetch("targets",[]).each do |framework, deps|
           frameworks[framework] = deps
-                                    .select { |name, details| details["type"] == "package" }
-                                    .map do |name, details|
+                                    .select { |_name, details| details["type"] == "package" }
+                                    .map do |name, _details|
             name_split = name.split("/")
             {
               name: name_split[0],
@@ -168,7 +174,7 @@ module Bibliothecary
           # do that yet so at least pick deterministically.
 
           # Note, frameworks can be empty, so remove empty ones and then return the last sorted item if any
-          frameworks = frameworks.delete_if { |k, v| v.empty? }
+          frameworks = frameworks.delete_if { |_k, v| v.empty? }
           return frameworks[frameworks.keys.sort.last] unless frameworks.empty?
         end
         []

data/lib/bibliothecary/parsers/pypi.rb

@@ -16,6 +16,13 @@ module Bibliothecary
 
       def self.mapping
         {
+          match_filenames('requirements-dev.txt', 'requirements/dev.txt',
+                          'requirements-docs.txt', 'requirements/docs.txt',
+                          'requirements-test.txt', 'requirements/test.txt',
+                          'requirements-tools.txt', 'requirements/tools.txt') => {
+            kind: 'manifest',
+            parser: :parse_requirements_txt
+          },
           lambda { |p| PIP_COMPILE_REGEXP.match(p) } => {
             content_matcher: :pip_compile?,
             kind: 'lockfile',
@@ -28,11 +35,11 @@ module Bibliothecary
           },
           match_filename('requirements.frozen') => { # pattern exists to store frozen deps in requirements.frozen
             parser: :parse_requirements_txt,
-            kind: 'lockfile',
+            kind: 'lockfile'
           },
           match_filename('pip-resolved-dependencies.txt') => { # Inferred from pip
             kind: 'lockfile',
-            parser: :parse_requirements_txt,
+            parser: :parse_requirements_txt
           },
           match_filename("setup.py") => {
             kind: 'manifest',
@@ -58,20 +65,20 @@ module Bibliothecary
           # Pip dependencies can be embedded in conda environment files
           match_filename("environment.yml") => {
             parser: :parse_conda,
-            kind: "manifest",
+            kind: "manifest"
           },
           match_filename("environment.yaml") => {
             parser: :parse_conda,
-            kind: "manifest",
+            kind: "manifest"
           },
           match_filename("environment.yml.lock") => {
             parser: :parse_conda,
-            kind: "lockfile",
+            kind: "lockfile"
           },
           match_filename("environment.yaml.lock") => {
             parser: :parse_conda,
-            kind: "lockfile",
-          },
+            kind: "lockfile"
+          }
         }
       end
 
@@ -191,6 +198,15 @@ module Bibliothecary
       # Invalid lines in requirements.txt are skipped.
       def self.parse_requirements_txt(file_contents, options: {})
         deps = []
+        type = case options[:filename]
+               when /dev/ || /docs/ || /tools/
+                 'development'
+               when /test/
+                 'test'
+               else
+                 'runtime'
+               end
+
         file_contents.split("\n").each do |line|
           if line['://']
             begin
@@ -200,7 +216,7 @@ module Bibliothecary
             end
 
             deps << result.merge(
-              type: 'runtime'
+              type: type
             )
           else
             match = line.delete(' ').match(REQUIREMENTS_REGEXP)
@@ -209,7 +225,7 @@ module Bibliothecary
             deps << {
               name: match[1],
               requirement: match[-1] || '*',
-              type: 'runtime'
+              type: type
             }
           end
         end

data/lib/bibliothecary/runner/multi_manifest_filter.rb

@@ -1,6 +1,21 @@
 module Bibliothecary
   class Runner
     class MultiManifestFilter
+      # Wrap up a file analysis for easier validity testing
+      class FileAnalysis
+        def initialize(file_analysis)
+          @file_analysis = file_analysis
+        end
+
+        # Determine if we should skip this file analysis when processing
+        # @return [Boolean] True if we should skip processing
+        def skip?
+          !@file_analysis ||
+          !@file_analysis[:dependencies] ||
+          @file_analysis[:dependencies].empty?
+        end
+      end
+
       def initialize(path:, related_files_info_entries:, runner:)
         @path = path
         @related_files_info_entries = related_files_info_entries
@@ -38,15 +53,14 @@ module Bibliothecary
         return @multiple_file_results if @multiple_file_results
 
         @multiple_file_results = []
-        @multiple_file_entries.each do |file|
-          analysis = @runner.analyse_file(file, File.read(File.join(@path, file)))
 
-          rfis_for_file = @related_files_info_entries.find_all { |rfi| rfi.lockfiles.include?(file) }
+        each_analysis_and_rfis do |analysis, rfis_for_file|
           rfis_for_file.each do |rfi|
-            file_analysis = analysis.find { |a| a[:platform] == rfi.platform }
+            file_analysis = FileAnalysis.new(
+              analysis.find { |a| a[:platform] == rfi.platform }
+            )
 
-            next unless file_analysis
-            next if file_analysis[:dependencies].empty?
+            next if file_analysis.skip?
 
             @multiple_file_results << rfi
           end
@@ -55,8 +69,17 @@ module Bibliothecary
         @multiple_file_results
       end
 
+      def each_analysis_and_rfis
+        @multiple_file_entries.each do |file|
+          analysis = @runner.analyse_file(file, File.read(File.join(@path, file)))
+          rfis_for_file = @related_files_info_entries.find_all { |rfi| rfi.lockfiles.include?(file) }
+
+          yield analysis, rfis_for_file
+        end
+      end
+
       def partition_file_entries!
-        @single_file_entries, @multiple_file_entries = files_to_check.partition { |file, count| count == 1  }
+        @single_file_entries, @multiple_file_entries = files_to_check.partition { |_file, count| count == 1  }
 
         @single_file_entries = @single_file_entries.map(&:first)
         @multiple_file_entries = @multiple_file_entries.map(&:first)

data/lib/bibliothecary/version.rb

@@ -1,3 +1,3 @@
 module Bibliothecary
-  VERSION = "8.3.4"
+  VERSION = "8.3.7"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bibliothecary
 version: !ruby/object:Gem::Version
-  version: 8.3.4
+  version: 8.3.7
 platform: ruby
 authors:
 - Andrew Nesbitt
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-07-05 00:00:00.000000000 Z
+date: 2022-07-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: tomlrb
@@ -220,6 +220,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email:
 - andrewnez@gmail.com