bibliothecary 7.3.5 → 8.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c3e4510e233cc6052a908e31f4d60498481c2b940448a2727e8cf89f28fae9b6
-  data.tar.gz: fe1fe8895bfae3b445b8c36de74a0285e9121a6fa49fde52bc481bc78d3b8931
+  metadata.gz: 7ed029f460ea32073ccc2d06f6fb297926686064554b1447a7014ff8588382f1
+  data.tar.gz: 70482eb3f0cfe7afb13124904b1e713596d92e92d90cd6267153ff4319e9d356
 SHA512:
-  metadata.gz: 203d1b6888ea667b5f0cf661f7282239d1736f70547c734fb890e0f58ac43d429e4193bda6977a88ca880a2d7cfbd5c1989442f171e27aa6d583932d83898109
-  data.tar.gz: 2fbf1aaee7ba11983db23d163b57d32e2ba195fc2fa1dad71447becba40c3eafabb9a8834a2e5a12a24d42b46424cafc32837ad8c9157478734e1d17c8d9a98d
+  metadata.gz: b8a243cb926cf8e49835f11db86199f8fe8d45dc53b1f80a17c9b3c941bb5935e4c8b8c1a897b40704d8c6c3ddc271fd5d3139aa9373a8328eea3a538c383fac
+  data.tar.gz: d6dd8f873e0ce3cabf3f526e4e50393c36c1e3c6aacb7b1581bc7720bc022670ecd0cc96114df8d3dcc561341e3ee920d401a578eaf7fe5b066f64a33306d730

data/.circleci/config.yml

@@ -5,7 +5,7 @@ orbs:
 jobs:
   test:
     docker:
-      - image: circleci/ruby:2.6.6-stretch-node
+      - image: cimg/ruby:2.7.1
     executor: ruby/default
     steps:
     - checkout

data/README.md

@@ -7,6 +7,8 @@ Dependency manifest parsing library for https://libraries.io
 
 ## Installation
 
+Requires Ruby 2.7 or above.
+
 Add this line to your application's Gemfile:
 
 ```ruby
@@ -90,6 +92,11 @@ All available config options are in: https://github.com/librariesio/bibliothecar
   - paket.lock
   - *.csproj
   - project.assets.json
+- CycloneDX
+  - XML as cyclonedx.xml
+  - JSON as cyclonedx.json
+  - Note that CycloneDX manifests can contain information on multiple
+    package manager's packages!
 - Bower
   - bower.json
 - CPAN

data/bibliothecary.gemspec

@@ -27,6 +27,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "commander"
   spec.add_dependency "strings-ansi"
   spec.add_dependency "strings"
+  spec.add_dependency "packageurl-ruby"
 
   spec.add_development_dependency "pry"
   spec.add_development_dependency "rake", "~> 12.0"

data/lib/bibliothecary/analyser/analysis.rb

@@ -0,0 +1,110 @@
+module Bibliothecary
+  module Analyser
+    module Analysis
+      # Convenience method to create FileInfo objects from folder path and
+      # file list.
+      #
+      # @param folder_path [String]
+      # @param file_list [Array<String>]
+      # @param options [Hash]
+      def analyse(folder_path, file_list, options: {})
+        analyse_file_info(file_list.map { |full_path| FileInfo.new(folder_path, full_path) }, options: options)
+      end
+      alias analyze analyse
+
+      # Analyze a set of FileInfo objects and extract manifests from them all.
+      #
+      # @params file_info_list [Array<FileInfo>]
+      def analyse_file_info(file_info_list, options: {})
+        matching_info = file_info_list
+          .select(&method(:match_info?))
+
+        matching_info.flat_map do |info|
+          analyse_contents_from_info(info, options: options)
+            .merge(related_paths: related_paths(info, matching_info))
+        end
+      end
+      alias analyze_file_info analyse_file_info
+
+      def analyse_contents(filename, contents, options: {})
+        analyse_contents_from_info(FileInfo.new(nil, filename, contents), options: options)
+      end
+      alias analyze_contents analyse_contents
+
+      # This is the place a parsing operation will eventually end up.
+      #
+      # @param info [FileInfo]
+      def analyse_contents_from_info(info, options: {})
+        # If your Parser needs to return multiple responses for one file, please override this method
+        # For example see conda.rb
+        kind = determine_kind_from_info(info)
+        dependencies = parse_file(info.relative_path, info.contents, options: options)
+
+        dependencies_to_analysis(info, kind, dependencies)
+      rescue Bibliothecary::FileParsingError => e
+        Bibliothecary::Analyser::create_error_analysis(platform_name, info.relative_path, kind, e.message)
+      end
+      alias analyze_contents_from_info analyse_contents_from_info
+
+      def dependencies_to_analysis(info, kind, dependencies)
+        dependencies = dependencies || [] # work around any legacy parsers that return nil
+        if generic?
+          grouped = dependencies.group_by { |dep| dep[:platform] }
+          all_analyses = grouped.keys.map do |platform|
+            deplatformed_dependencies = grouped[platform].map { |d| d.delete(:platform); d }
+            Bibliothecary::Analyser::create_analysis(platform, info.relative_path, kind, deplatformed_dependencies)
+          end
+          # this is to avoid a larger refactor for the time being. The larger refactor
+          # needs to make analyse_contents return multiple analysis, or add another
+          # method that can return multiple and deprecate analyse_contents, perhaps.
+          raise "File contains zero or multiple platforms, currently must have exactly one" if all_analyses.length != 1
+          all_analyses.first
+        else
+          Bibliothecary::Analyser::create_analysis(platform_name, info.relative_path, kind, dependencies)
+        end
+      end
+
+      # Call the matching parse class method for this file with
+      # these contents
+      def parse_file(filename, contents, options: {})
+        details = first_matching_mapping_details(FileInfo.new(nil, filename, contents))
+
+        # this can be raised if we don't check match?/match_info?,
+        # OR don't have the file contents when we check them, so
+        # it turns out for example that a .xml file isn't a
+        # manifest after all.
+        raise Bibliothecary::FileParsingError.new("No parser for this file type", filename) unless details[:parser]
+
+        # The `parser` method should raise an exception if the file is malformed,
+        # should return empty [] if the file is fine but simply doesn't contain
+        # any dependencies, and should never return nil. At the time of writing
+        # this comment, some of the parsers return [] or nil to mean an error
+        # which is confusing to users.
+        send(details[:parser], contents, options: options.merge(filename: filename))
+
+      rescue Exception => e # default is StandardError but C bindings throw Exceptions
+        # the C xml parser also puts a newline at the end of the message
+        raise Bibliothecary::FileParsingError.new(e.message.strip, filename)
+      end
+
+      private
+
+      def related_paths(info, infos)
+        return [] unless determine_can_have_lockfile_from_info(info)
+
+        kind = determine_kind_from_info(info)
+        relate_to_kind = first_matching_mapping_details(info)
+          .fetch(:related_to, %w(manifest lockfile).reject { |k| k == kind })
+        dirname = File.dirname(info.relative_path)
+
+        infos
+          .reject { |i| i == info }
+          .select { |i| relate_to_kind.include?(determine_kind_from_info(i)) }
+          .select { |i| File.dirname(i.relative_path) == dirname }
+          .select(&method(:determine_can_have_lockfile_from_info))
+          .map(&:relative_path)
+          .sort
+      end
+    end
+  end
+end

data/lib/bibliothecary/analyser/determinations.rb

@@ -0,0 +1,27 @@
+module Bibliothecary
+  module Analyser
+    module Determinations
+      # calling this with contents=nil can produce less-informed
+      # results, but kept for back compat
+      def determine_kind(filename, contents = nil)
+        determine_kind_from_info(FileInfo.new(nil, filename, contents))
+      end
+
+      def determine_kind_from_info(info)
+        first_matching_mapping_details(info)
+          .fetch(:kind, nil)
+      end
+
+      # calling this with contents=nil can produce less-informed
+      # results, but kept for back compat
+      def determine_can_have_lockfile(filename, contents = nil)
+        determine_can_have_lockfile_from_info(FileInfo.new(nil, filename, contents))
+      end
+
+      def determine_can_have_lockfile_from_info(info)
+        first_matching_mapping_details(info)
+          .fetch(:can_have_lockfile, true)
+      end
+    end
+  end
+end

data/lib/bibliothecary/analyser/matchers.rb

@@ -0,0 +1,64 @@
+module Bibliothecary
+  module Analyser
+    module Matchers
+      def match_filename(filename, case_insensitive: false)
+        if case_insensitive
+          lambda { |path| path.downcase == filename.downcase || path.downcase.end_with?("/" + filename.downcase) }
+        else
+          lambda { |path| path == filename || path.end_with?("/" + filename) }
+        end
+      end
+
+      def match_filenames(*filenames)
+        lambda do |path|
+          filenames.any? { |f| path == f } ||
+            filenames.any? { |f| path.end_with?("/" + f) }
+        end
+      end
+
+      def match_extension(filename, case_insensitive: false)
+        if case_insensitive
+          lambda { |path| path.downcase.end_with?(filename.downcase) }
+        else
+          lambda { |path| path.end_with?(filename) }
+        end
+      end
+
+      def mapping_entry_match?(matcher, details, info)
+        if matcher.call(info.relative_path)
+          # we only want to load contents if we don't have them already
+          # and there's a content_matcher method to use
+          return true if details[:content_matcher].nil?
+          # this is the libraries.io case where we won't load all .xml
+          # files (for example) just to look at their contents, we'll
+          # assume they are not manifests.
+          return false if info.contents.nil?
+          return send(details[:content_matcher], info.contents)
+        else
+          return false
+        end
+      end
+
+      # this is broken with contents=nil because it can't look at file
+      # contents, so skips manifests that are ambiguously a
+      # manifest considering only the filename. However, those are
+      # the semantics that libraries.io uses since it doesn't have
+      # the files locally.
+      def match?(filename, contents = nil)
+        match_info?(FileInfo.new(nil, filename, contents))
+      end
+
+      def match_info?(info)
+        first_matching_mapping_details(info).any?
+      end
+
+      private
+
+      def first_matching_mapping_details(info)
+        mapping
+          .find { |matcher, details| mapping_entry_match?(matcher, details, info) }
+          &.last || {}
+      end
+    end
+  end
+end

data/lib/bibliothecary/analyser.rb

@@ -1,3 +1,7 @@
+require_relative './analyser/matchers.rb'
+require_relative './analyser/determinations.rb'
+require_relative './analyser/analysis.rb'
+
 module Bibliothecary
   module Analyser
     def self.create_error_analysis(platform_name, relative_path, kind, message)
@@ -23,75 +27,34 @@ module Bibliothecary
 
     def self.included(base)
       base.extend(ClassMethods)
+
+      # Group like-methods into separate modules for easier comprehension.
+      base.extend(Bibliothecary::Analyser::Matchers)
+      base.extend(Bibliothecary::Analyser::Determinations)
+      base.extend(Bibliothecary::Analyser::Analysis)
     end
-    module ClassMethods
-      def generic?
-        platform_name == "generic"
-      end
 
-      def mapping_entry_match?(matcher, details, info)
-        if matcher.call(info.relative_path)
-          # we only want to load contents if we don't have them already
-          # and there's a content_matcher method to use
-          return true if details[:content_matcher].nil?
-          # this is the libraries.io case where we won't load all .xml
-          # files (for example) just to look at their contents, we'll
-          # assume they are not manifests.
-          return false if info.contents.nil?
-          return send(details[:content_matcher], info.contents)
+    module TryCache
+      def try_cache(options, key)
+        if options[:cache]
+          options[:cache][key] ||= yield
+
+          options[:cache][key]
         else
-          return false
+          yield
         end
       end
+    end
 
-      def parse_file(filename, contents)
-        details = first_matching_mapping_details(FileInfo.new(nil, filename, contents))
-
-        # this can be raised if we don't check match?/match_info?,
-        # OR don't have the file contents when we check them, so
-        # it turns out for example that a .xml file isn't a
-        # manifest after all.
-        raise Bibliothecary::FileParsingError.new("No parser for this file type", filename) unless details[:parser]
-
-        # The `parser` method should raise an exception if the file is malformed,
-        # should return empty [] if the file is fine but simply doesn't contain
-        # any dependencies, and should never return nil. At the time of writing
-        # this comment, some of the parsers return [] or nil to mean an error
-        # which is confusing to users.
-        send(details[:parser], contents)
-
-      rescue Exception => e # default is StandardError but C bindings throw Exceptions
-        # the C xml parser also puts a newline at the end of the message
-        raise Bibliothecary::FileParsingError.new(e.message.strip, filename)
-      end
-
-      # this is broken with contents=nil because it can't look at file
-      # contents, so skips manifests that are ambiguously a
-      # manifest considering only the filename. However, those are
-      # the semantics that libraries.io uses since it doesn't have
-      # the files locally.
-      def match?(filename, contents = nil)
-        match_info?(FileInfo.new(nil, filename, contents))
-      end
-
-      def match_info?(info)
-        first_matching_mapping_details(info).any?
+    module ClassMethods
+      def generic?
+        platform_name == "generic"
       end
 
       def platform_name
         self.name.to_s.split('::').last.downcase
       end
 
-      def parse_json_runtime_manifest(file_contents)
-        JSON.parse(file_contents).fetch('dependencies',[]).map do |name, requirement|
-          {
-            name: name,
-            requirement: requirement,
-            type: 'runtime'
-          }
-        end
-      end
-
       def map_dependencies(hash, key, type)
         hash.fetch(key,[]).map do |name, requirement|
           {
@@ -102,141 +65,22 @@ module Bibliothecary
         end
       end
 
-      def analyse(folder_path, file_list)
-        analyse_file_info(file_list.map { |full_path| FileInfo.new(folder_path, full_path) })
-      end
-      alias analyze analyse
-
-      def analyse_file_info(file_info_list)
-        matching_info = file_info_list
-          .select(&method(:match_info?))
-
-        matching_info.flat_map do |info|
-          analyse_contents_from_info(info)
-            .merge(related_paths: related_paths(info, matching_info))
-        end
-      end
-      alias analyze_file_info analyse_file_info
-
-      def analyse_contents(filename, contents)
-        analyse_contents_from_info(FileInfo.new(nil, filename, contents))
-      end
-      alias analyze_contents analyse_contents
-
-      def dependencies_to_analysis(info, kind, dependencies)
-        dependencies = dependencies || [] # work around any legacy parsers that return nil
-        if generic?
-          analyses = []
-          grouped = dependencies.group_by { |dep| dep[:platform] }
-          all_analyses = grouped.keys.map do |platform|
-            deplatformed_dependencies = grouped[platform].map { |d| d.delete(:platform); d }
-            Bibliothecary::Analyser::create_analysis(platform, info.relative_path, kind, deplatformed_dependencies)
-          end
-          # this is to avoid a larger refactor for the time being. The larger refactor
-          # needs to make analyse_contents return multiple analysis, or add another
-          # method that can return multiple and deprecate analyse_contents, perhaps.
-          raise "File contains zero or multiple platforms, currently must have exactly one" if all_analyses.length != 1
-          all_analyses.first
-        else
-          Bibliothecary::Analyser::create_analysis(platform_name, info.relative_path, kind, dependencies)
-        end
-      end
-
-      def analyse_contents_from_info(info)
-        # If your Parser needs to return multiple responses for one file, please override this method
-        # For example see conda.rb
-        kind = determine_kind_from_info(info)
-        dependencies = parse_file(info.relative_path, info.contents)
-
-        dependencies_to_analysis(info, kind, dependencies)
-      rescue Bibliothecary::FileParsingError => e
-        Bibliothecary::Analyser::create_error_analysis(platform_name, info.relative_path, kind, e.message)
-      end
-      alias analyze_contents_from_info analyse_contents_from_info
+      # Add a MultiParser module to a Parser class. This extends the
+      # self.mapping method on the parser to include the multi parser's
+      # files to watch for, and it extends the Parser class with
+      # the multi parser for you.
+      #
+      # @param klass [Class] A Bibliothecary::MultiParsers class
+      def add_multi_parser(klass)
+        raise "No mapping found! You should place the add_multi_parser call below def self.mapping." unless respond_to?(:mapping)
 
-      # calling this with contents=nil can produce less-informed
-      # results, but kept for back compat
-      def determine_kind(filename, contents = nil)
-        determine_kind_from_info(FileInfo.new(nil, filename, contents))
-      end
-
-      def determine_kind_from_info(info)
-        first_matching_mapping_details(info)
-          .fetch(:kind, nil)
-      end
-
-      # calling this with contents=nil can produce less-informed
-      # results, but kept for back compat
-      def determine_can_have_lockfile(filename, contents = nil)
-        determine_can_have_lockfile_from_info(FileInfo.new(nil, filename, contents))
-      end
-
-      def determine_can_have_lockfile_from_info(info)
-        first_matching_mapping_details(info)
-          .fetch(:can_have_lockfile, true)
-      end
+        original_mapping = self.mapping
 
-      def parse_ruby_manifest(manifest)
-        manifest.dependencies.inject([]) do |deps, dep|
-          deps.push({
-            name: dep.name,
-            requirement: dep
-              .requirement
-              .requirements
-              .sort_by(&:last)
-              .map { |op, version| "#{op} #{version}" }
-              .join(", "),
-            type: dep.type
-          })
-        end.uniq
-      end
-
-      def match_filename(filename, case_insensitive: false)
-        if case_insensitive
-          lambda { |path| path.downcase == filename.downcase || path.downcase.end_with?("/" + filename.downcase) }
-        else
-          lambda { |path| path == filename || path.end_with?("/" + filename) }
-        end
-      end
-
-      def match_filenames(*filenames)
-        lambda do |path|
-          filenames.any? { |f| path == f } ||
-            filenames.any? { |f| path.end_with?("/" + f) }
+        define_singleton_method(:mapping) do
+          original_mapping.merge(klass.mapping)
         end
-      end
-
-      def match_extension(filename, case_insensitive: false)
-        if case_insensitive
-          lambda { |path| path.downcase.end_with?(filename.downcase) }
-        else
-          lambda { |path| path.end_with?(filename) }
-        end
-      end
-
-      private
-
-      def related_paths(info, infos)
-        return [] unless determine_can_have_lockfile_from_info(info)
-
-        kind = determine_kind_from_info(info)
-        relate_to_kind = first_matching_mapping_details(info)
-          .fetch(:related_to, %w(manifest lockfile).reject { |k| k == kind })
-        dirname = File.dirname(info.relative_path)
-
-        infos
-          .reject { |i| i == info }
-          .select { |i| relate_to_kind.include?(determine_kind_from_info(i)) }
-          .select { |i| File.dirname(i.relative_path) == dirname }
-          .select(&method(:determine_can_have_lockfile_from_info))
-          .map(&:relative_path)
-          .sort
-      end
 
-      def first_matching_mapping_details(info)
-        mapping
-          .find { |matcher, details| mapping_entry_match?(matcher, details, info) }
-          &.last || {}
+        send(:extend, klass)
       end
     end
   end

data/lib/bibliothecary/cli.rb

@@ -18,9 +18,9 @@ module Bibliothecary
         c.action do |_args, options|
           options.default path: './'
           output = Bibliothecary.analyse(options.path)
-          output.each do |manifest|
-            puts "#{manifest[:path]} (#{manifest[:platform]})"
-            manifest[:dependencies].group_by{|d| d[:type] }.each do |type, deps|
+          output.each do |file_contents|
+            puts "#{file_contents[:path]} (#{manifest[:platform]})"
+            file_contents[:dependencies].group_by{|d| d[:type] }.each do |type, deps|
               puts "  #{type}"
               deps.each do |dep|
                 puts "    #{dep[:name]} #{dep[:requirement]}"

data/lib/bibliothecary/file_info.rb

@@ -1,6 +1,8 @@
 require 'pathname'
 
 module Bibliothecary
+  # A representation of a file on the filesystem, with location information
+  # and package manager information if needed.
   class FileInfo
     attr_reader :folder_path
     attr_reader :relative_path

data/lib/bibliothecary/multi_parsers/bundler_like_manifest.rb

@@ -0,0 +1,22 @@
+module Bibliothecary
+  module MultiParsers
+    module BundlerLikeManifest
+      # this takes parsed Bundler and Bundler-like (CocoaPods)
+      # manifests and turns them into a list of dependencies.
+      def parse_ruby_manifest(manifest)
+        manifest.dependencies.inject([]) do |deps, dep|
+          deps.push({
+            name: dep.name,
+            requirement: dep
+              .requirement
+              .requirements
+              .sort_by(&:last)
+              .map { |op, version| "#{op} #{version}" }
+              .join(", "),
+            type: dep.type
+          })
+        end.uniq
+      end
+    end
+  end
+end