bibliothecary 8.2.0 → 8.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4dfafc16f9be53462f1eca5e0fa5b09dcc0a5d92531570630c087773ff87d18c
-  data.tar.gz: 3d0045a95cc5cff513474aae337e0a032543130cc7b87144adad8033138c6e17
+  metadata.gz: 6bf3193b6daf685565ca3fe2105ca8be8e03c5e83d1edcafe3ca6d3c7be12d8b
+  data.tar.gz: c2b96281c11a89bf49f830bbb0be00e35d18096dcc8a8e8b2d0038e1256a6eed
 SHA512:
-  metadata.gz: f984d7445345768463af5f2cb0906bf997e56ca03842ca538b65d95764ac427ed448a062593e293eeaa33779aaa2cb9b59b18f0715de813a6f018a815a03dc3f
-  data.tar.gz: 067f0a20d820635cd697b18c43254a818284a66b98a13367146a0f9063b63bfe9316427d32bf6b88d960f6283dd56adb4ad2f874c003c7e4c151d71181f3cd37
+  metadata.gz: d3d45d37ac4c8e3c982d708f454906ab41db86fc285ba800d90acb6d2e541ae7151e7d09c5ae10eccec2678760b24f2bbb3d5c2e77048b714aa71c953395ce70
+  data.tar.gz: b86748a552b1774895cad46925205a7c4915907d336436c945a7dc027d80e8feff7d12465c9524ea12a3495d4635e45c4cd32489456453b4c744a22fdf7c5c32

data/lib/bibliothecary/multi_parsers/dependencies_csv.rb

@@ -0,0 +1,151 @@
+require 'csv'
+
+module Bibliothecary
+  module MultiParsers
+    module DependenciesCSV
+      include Bibliothecary::Analyser
+      include Bibliothecary::Analyser::TryCache
+
+      def self.mapping
+        {
+          match_filename('dependencies.csv') => {
+            kind: 'lockfile',
+            parser: :parse_dependencies_csv
+          }
+        }
+      end
+
+      # Processing a CSV file isn't as exact as using a real manifest file,
+      # but you can get pretty close as long as the data you're importing
+      # is simple.
+      class CSVFile
+        # Header structures are:
+        #
+        # <field to fill in for dependency> => {
+        #   match: [<regexp of incoming column name to match in priority order, highest priority first>...],
+        #   [default]: <optional default value for this field>
+        # }
+        HEADERS = {
+          "platform" => {
+            match: [
+              /^platform$/i
+            ]
+          },
+          "name" => {
+            match: [
+              /^name$/i
+            ]
+          },
+          # Lockfiles have exact versions.
+          "lockfile_requirement" => {
+            match: [
+              /^(lockfile |)requirement$/i,
+              /^version$/i,
+            ],
+          },
+          # Manifests have versions that can have operators.
+          # However, since Bibliothecary only currently supports analyzing a
+          # single file as a single thing (either manifest or lockfile)
+          # we can't return manifest-y data. Only take the lockfile requirement
+          # when processing dependencies.csv for now.
+          "requirement" => {
+            match: [
+              /^(lockfile |)requirement$/i,
+              /^version$/i,
+            ],
+          },
+          "type" => {
+            default: "runtime",
+            match: [
+              /^(lockfile |)type$/i,
+              /^(manifest |)type$/i
+            ]
+          }
+        }
+
+        attr_reader :result
+
+        def initialize(file_contents)
+          @file_contents = file_contents
+
+          @result = nil
+
+          # A Hash of "our field name" => ["header in CSV file", "lower priority header in CSV file"]
+          @header_mappings = {}
+        end
+
+        def parse!
+          table = parse_and_validate_csv_file
+
+          @result = table.map.with_index do |row, idx|
+            HEADERS.each_with_object({}) do |(header, info), obj|
+              # find the first non-empty field in the row for this header, or nil if not found
+              row_data = row[@header_mappings[header]]
+
+              # some column have default data to fall back on
+              if row_data
+                obj[header.to_sym] = row_data
+              elsif info.has_key?(:default)
+                # if the default is nil, don't even add the key to the hash
+                obj[header.to_sym] = info[:default] if info[:default]
+              else
+                # use 1-based index just like the 'csv' std lib, and count the headers as first row.
+                raise "Missing required field '#{header}' on line #{idx + 2}."
+              end
+            end
+          end
+        end
+
+        private
+
+        def parse_and_validate_csv_file
+          table = CSV.parse(@file_contents, headers: true)
+
+          header_examination_results = map_table_headers_to_local_lookups(table, HEADERS)
+          unless header_examination_results[:missing].empty?
+            raise "Missing required headers #{header_examination_results[:missing].join(', ')} in CSV. Check to make sure header names are all lowercase."
+          end
+          @header_mappings = header_examination_results[:found]
+
+          table
+        end
+
+        def map_table_headers_to_local_lookups(table, local_lookups)
+          result = local_lookups.each_with_object({ found: {}, missing: [] }) do |(header, info), obj|
+            results = table.headers.each_with_object([]) do |table_header, matches|
+              info[:match].each_with_index do |match_regexp, index|
+                matches << [table_header, index] if table_header[match_regexp]
+              end
+            end
+
+            if results.empty?
+              # if a header has a default value it's optional
+              obj[:missing] << header unless info.has_key?(:default)
+            else
+              # select the highest priority header possible
+              obj[:found][header] ||= nil
+              obj[:found][header] = ([obj[:found][header]] + results).compact.min_by(&:last)
+            end
+          end
+
+          # strip off the priorities. only one mapping should remain.
+          result[:found].transform_values!(&:first)
+
+          result
+        end
+      end
+
+      def parse_dependencies_csv(file_contents, options: {})
+        csv_file = try_cache(options, options[:filename]) do
+          raw_csv_file = CSVFile.new(file_contents)
+          raw_csv_file.parse!
+          raw_csv_file
+        end
+
+        csv_file.result.find_all do |dependency|
+          dependency[:platform] == platform_name.to_s
+        end
+      end
+    end
+  end
+end

data/lib/bibliothecary/parsers/bower.rb

@@ -14,6 +14,8 @@ module Bibliothecary
         }
       end
 
+      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
+
       def self.parse_manifest(file_contents, options: {})
         json = JSON.parse(file_contents)
         map_dependencies(json, 'dependencies', 'runtime') +

data/lib/bibliothecary/parsers/cargo.rb

@@ -17,6 +17,7 @@ module Bibliothecary
       end
 
       add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
+      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
       def self.parse_manifest(file_contents, options: {})
         manifest = Tomlrb.parse(file_contents)

data/lib/bibliothecary/parsers/carthage.rb

@@ -20,6 +20,8 @@ module Bibliothecary
         }
       end
 
+      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
+
       def self.parse_cartfile(file_contents, options: {})
         map_dependencies(file_contents, 'cartfile')
       end

data/lib/bibliothecary/parsers/clojars.rb

@@ -15,6 +15,8 @@ module Bibliothecary
         }
       end
 
+      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
+
       def self.parse_manifest(file_contents, options: {})
         response = Typhoeus.post("#{Bibliothecary.configuration.clojars_parser_host}/project.clj", body: file_contents)
         raise Bibliothecary::RemoteParsingError.new("Http Error #{response.response_code} when contacting: #{Bibliothecary.configuration.clojars_parser_host}/project.clj", response.response_code) unless response.success?

data/lib/bibliothecary/parsers/cocoapods.rb

@@ -33,6 +33,8 @@ module Bibliothecary
         }
       end
 
+      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
+
       def self.parse_podfile_lock(file_contents, options: {})
         manifest = YAML.load file_contents
         manifest['PODS'].map do |row|

data/lib/bibliothecary/parsers/conda.rb

@@ -27,6 +27,7 @@ module Bibliothecary
       end
 
       add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
+      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
       def self.parse_conda(file_contents, options: {})
         parse_conda_with_kind(file_contents, "manifest")

data/lib/bibliothecary/parsers/cpan.rb

@@ -19,6 +19,8 @@ module Bibliothecary
         }
       end
 
+      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
+
       def self.parse_json_manifest(file_contents, options: {})
         manifest = JSON.parse file_contents
         manifest['prereqs'].map do |_group, deps|

data/lib/bibliothecary/parsers/cran.rb

@@ -17,6 +17,7 @@ module Bibliothecary
       end
 
       add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
+      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
       def self.parse_description(file_contents, options: {})
         manifest = DebControl::ControlFileBase.parse(file_contents)

data/lib/bibliothecary/parsers/dub.rb

@@ -20,6 +20,8 @@ module Bibliothecary
         }
       end
 
+      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
+
       def self.parse_sdl_manifest(file_contents, options: {})
         SdlParser.new(:runtime, file_contents).dependencies
       end

data/lib/bibliothecary/parsers/elm.rb

@@ -19,6 +19,8 @@ module Bibliothecary
         }
       end
 
+      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
+
       def self.parse_json_lock(file_contents, options: {})
         manifest = JSON.parse file_contents
         manifest.map do |name, requirement|

data/lib/bibliothecary/parsers/go.rb

@@ -66,6 +66,7 @@ module Bibliothecary
       end
 
       add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
+      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
       def self.parse_godep_json(file_contents, options: {})
         manifest = JSON.parse file_contents

data/lib/bibliothecary/parsers/hackage.rb

@@ -20,6 +20,7 @@ module Bibliothecary
       end
 
       add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
+      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
       def self.parse_cabal(file_contents, options: {})
         headers = {

data/lib/bibliothecary/parsers/haxelib.rb

@@ -14,6 +14,9 @@ module Bibliothecary
           }
         }
       end
+
+      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
     end
   end
 end
+

data/lib/bibliothecary/parsers/hex.rb

@@ -19,6 +19,7 @@ module Bibliothecary
       end
 
       add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
+      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
       def self.parse_mix(file_contents, options: {})
         response = Typhoeus.post("#{Bibliothecary.configuration.mix_parser_host}/", body: file_contents)

data/lib/bibliothecary/parsers/julia.rb

@@ -12,6 +12,8 @@ module Bibliothecary
         }
       end
 
+      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
+
       def self.parse_require(file_contents, options: {})
         deps = []
         file_contents.split("\n").each do |line|

data/lib/bibliothecary/parsers/maven.rb

@@ -18,10 +18,11 @@ module Bibliothecary
       # An intentionally overly-simplified regex to scrape deps from build.gradle.kts files. 
       # To be truly useful bibliothecary would need a full Kotlin parser that speaks Gradle, 
       # because the Kotlin DSL has many dynamic ways of declaring dependencies.
-      GRADLE_KTS_SIMPLE_REGEX = /(#{GRADLE_KTS_DEPENDENCY_METHODS.join('|')})\s*\(\s*"([^"]+)"\s*\)/m
       
-      # e.g. "group:artifactId:1.2.3"
-      GRADLE_KTS_GAV_REGEX = /([\w.-]+)\:([\w.-]+)(?:\:([\w.-]+))?/
+      GRADLE_KTS_VERSION_REGEX = /[\w.-]+/ # e.g. '1.2.3'
+      GRADLE_KTS_INTERPOLATED_VERSION_REGEX = /\$\{.*\}/ # e.g. '${my-project-settings["version"]}'
+      GRADLE_KTS_GAV_REGEX = /([\w.-]+)\:([\w.-]+)(?:\:(#{GRADLE_KTS_VERSION_REGEX}|#{GRADLE_KTS_INTERPOLATED_VERSION_REGEX}))?/
+      GRADLE_KTS_SIMPLE_REGEX = /(#{GRADLE_KTS_DEPENDENCY_METHODS.join('|')})\s*\(\s*"#{GRADLE_KTS_GAV_REGEX}"\s*\)\s*$/m # e.g. "group:artifactId:1.2.3"
 
       MAVEN_PROPERTY_REGEX = /\$\{(.+?)\}/
       MAX_DEPTH = 5
@@ -83,7 +84,8 @@ module Bibliothecary
         }
       end
 
-      add_multi_parser Bibliothecary::MultiParsers::CycloneDX
+      add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
+      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
       def self.parse_ivy_manifest(file_contents, options: {})
         manifest = Ox.parse file_contents
@@ -248,14 +250,13 @@ module Bibliothecary
 
       def self.parse_gradle_kts(file_contents, options: {})
         file_contents
-          .scan(GRADLE_KTS_SIMPLE_REGEX)                                                   # match 'implementation("group:artifactId:version")'
-          .map { |(_type, dep_match)| GRADLE_KTS_GAV_REGEX.match(dep_match) }              # extract ["group", "artifactId", ?"version"]
-          .reject { |gav_match| gav_match.nil? || gav_match[1].nil? || gav_match[2].nil? } # remove any with missing group/artifactId
-          .map { |gav_match|
+          .scan(GRADLE_KTS_SIMPLE_REGEX)                                                  # match 'implementation("group:artifactId:version")'
+          .reject { |(_type, group, artifactId, _version)| group.nil? || artifactId.nil? } # remove any matches with missing group/artifactId
+          .map { |(type, group, artifactId, version)|
             {
-              name: [gav_match[1], gav_match[2]].join(":"),
-              requirement: gav_match[3] || "*",
-              type: nil # TODO: we may be able to infer dep types using the _type var above.
+              name: [group, artifactId].join(":"),
+              requirement: version || "*",
+              type: type
             }
           }
       end

data/lib/bibliothecary/parsers/meteor.rb

@@ -14,6 +14,8 @@ module Bibliothecary
           }
         }
       end
+
+      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
     end
   end
 end

data/lib/bibliothecary/parsers/npm.rb

@@ -34,6 +34,7 @@ module Bibliothecary
       end
 
       add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
+      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
       def self.parse_shrinkwrap(file_contents, options: {})
         manifest = JSON.parse(file_contents)

data/lib/bibliothecary/parsers/nuget.rb

@@ -45,6 +45,7 @@ module Bibliothecary
       end
 
       add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
+      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
       def self.parse_project_lock_json(file_contents, options: {})
         manifest = JSON.parse file_contents

data/lib/bibliothecary/parsers/packagist.rb

@@ -19,6 +19,7 @@ module Bibliothecary
       end
 
       add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
+      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
       def self.parse_lockfile(file_contents, options: {})
         manifest = JSON.parse file_contents

data/lib/bibliothecary/parsers/pub.rb

@@ -18,6 +18,8 @@ module Bibliothecary
         }
       end
 
+      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
+
       def self.parse_yaml_manifest(file_contents, options: {})
         manifest = YAML.load file_contents
         map_dependencies(manifest, 'dependencies', 'runtime') +

data/lib/bibliothecary/parsers/pypi.rb

@@ -76,6 +76,7 @@ module Bibliothecary
       end
 
       add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
+      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
       def self.parse_pipfile(file_contents, options: {})
         manifest = Tomlrb.parse(file_contents)

data/lib/bibliothecary/parsers/rubygems.rb

@@ -30,6 +30,7 @@ module Bibliothecary
       end
 
       add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
+      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
       def self.parse_gemfile_lock(file_contents, options: {})
         file_contents.lines(chomp: true).map do |line|

data/lib/bibliothecary/parsers/shard.rb

@@ -18,6 +18,8 @@ module Bibliothecary
         }
       end
 
+      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
+
       def self.parse_yaml_lockfile(file_contents, options: {})
         manifest = YAML.load file_contents
         map_dependencies(manifest, 'shards', 'runtime')

data/lib/bibliothecary/parsers/swift_pm.rb

@@ -13,6 +13,7 @@ module Bibliothecary
       end
 
       add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
+      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
       def self.parse_package_swift(file_contents, options: {})
         response = Typhoeus.post("#{Bibliothecary.configuration.swift_parser_host}/to-json", body: file_contents)

data/lib/bibliothecary/related_files_info.rb

@@ -5,28 +5,52 @@ module Bibliothecary
     attr_reader :manifests
     attr_reader :lockfiles
 
+    # Create a set of RelatedFilesInfo for the provided file_infos,
+    # where each RelatedFilesInfo contains all the file_infos 
     def self.create_from_file_infos(file_infos)
       returns = []
-      paths = file_infos.group_by { |info| File.dirname(info.relative_path) }
-      paths.values.each do |path|
-        same_pm = path.group_by { |info| info.package_manager}
-        same_pm.values.each do |value|
-          returns.append(RelatedFilesInfo.new(value))
+
+      file_infos_by_directory = file_infos.group_by { |info| File.dirname(info.relative_path) }
+      file_infos_by_directory.values.each do |file_infos_for_path|
+        file_infos_by_directory_by_package_manager = file_infos_for_path.group_by { |info| info.package_manager}
+
+        file_infos_by_directory_by_package_manager.values.each do |file_infos_in_directory_for_package_manager|
+          returns.append(RelatedFilesInfo.new(file_infos_in_directory_for_package_manager))
         end
       end
+
       returns
     end
 
     def initialize(file_infos)
       package_manager = file_infos.first.package_manager
+      ordered_file_infos = file_infos
+
       if package_manager.respond_to?(:lockfile_preference_order)
-        file_infos = package_manager.lockfile_preference_order(file_infos)
+        ordered_file_infos = package_manager.lockfile_preference_order(file_infos)
       end
+
       @platform = package_manager.platform_name
       @path = Pathname.new(File.dirname(file_infos.first.relative_path)).cleanpath.to_path
+
+      @manifests = filter_file_infos_by_package_manager_type(
+        file_infos: ordered_file_infos,
+        package_manager: package_manager,
+        type: "manifest"
+      )
+
+      @lockfiles = filter_file_infos_by_package_manager_type(
+        file_infos: ordered_file_infos,
+        package_manager: package_manager,
+        type: "lockfile"
+      )
+    end
+
+    private
+
+    def filter_file_infos_by_package_manager_type(file_infos:, package_manager:, type:)
       # `package_manager.determine_kind_from_info(info)` can be an Array, so use include? which also works for string
-      @manifests = file_infos.select { |info| package_manager.determine_kind_from_info(info).include? "manifest" }.map(&:relative_path)
-      @lockfiles = file_infos.select { |info| package_manager.determine_kind_from_info(info).include? "lockfile" }.map(&:relative_path)
+      file_infos.select { |info| package_manager.determine_kind_from_info(info).include?(type) }.map(&:relative_path)
     end
   end
 end

data/lib/bibliothecary/runner/multi_manifest_filter.rb

@@ -0,0 +1,67 @@
+module Bibliothecary
+  class Runner
+    class MultiManifestFilter
+      def initialize(path:, related_files_info_entries:, runner:)
+        @path = path
+        @related_files_info_entries = related_files_info_entries
+        @runner = runner
+      end
+
+      # Standalone multi manifest files should *always* be treated as lockfiles,
+      # since there's no human-written manifest file to go with them.
+      def files_to_check
+        @files_to_check ||= @related_files_info_entries.each_with_object({}) do |files_info, all|
+          files_info.lockfiles.each do |file|
+            all[file] ||= 0
+            all[file] += 1
+          end
+        end
+      end
+
+      def results
+        partition_file_entries!
+
+        no_lockfile_results + single_file_results + multiple_file_results
+      end
+
+      def no_lockfile_results
+        @no_lockfile_results ||= @related_files_info_entries.find_all { |rfi| rfi.lockfiles.empty? }
+      end
+
+      def single_file_results
+        @single_file_results ||= @single_file_entries.map do |file|
+          @related_files_info_entries.find { |rfi| rfi.lockfiles.include?(file) }
+        end
+      end
+
+      def multiple_file_results
+        return @multiple_file_results if @multiple_file_results
+
+        @multiple_file_results = []
+        @multiple_file_entries.each do |file|
+          analysis = @runner.analyse_file(file, File.read(File.join(@path, file)))
+
+          rfis_for_file = @related_files_info_entries.find_all { |rfi| rfi.lockfiles.include?(file) }
+          rfis_for_file.each do |rfi|
+            file_analysis = analysis.find { |a| a[:platform] == rfi.platform }
+
+            next unless file_analysis
+            next if file_analysis[:dependencies].empty?
+
+            @multiple_file_results << rfi
+          end
+        end
+
+        @multiple_file_results
+      end
+
+      def partition_file_entries!
+        @single_file_entries, @multiple_file_entries = files_to_check.partition { |file, count| count == 1  }
+
+        @single_file_entries = @single_file_entries.map(&:first)
+        @multiple_file_entries = @multiple_file_entries.map(&:first)
+      end
+    end
+  end
+end
+

data/lib/bibliothecary/runner.rb

@@ -3,7 +3,6 @@ module Bibliothecary
   # A runner is created every time a file is targeted to be parsed. Don't call
   # parse methods directory! Use a Runner.
   class Runner
-
     def initialize(configuration)
       @configuration = configuration
       @options = {
@@ -47,9 +46,11 @@ module Bibliothecary
       Bibliothecary::Parsers.constants.map{|c| Bibliothecary::Parsers.const_get(c) }.sort_by{|c| c.to_s.downcase }
     end
 
+    # Parses an array of format [{file_path: "", contents: ""},] to match
+    # on both filename matches and on content_match patterns.
+    #
+    # @return [Array<Bibliothecary::FileInfo>] A list of FileInfo, one for each package manager match for each file
     def load_file_info_list_from_contents(file_path_contents_hash)
-      # Parses an array of format [{file_path: "", contents: ""},] to match
-      #  on both filename matches, and one content_match patterns.
       file_list = []
 
       file_path_contents_hash.each do |file|
@@ -57,7 +58,7 @@ module Bibliothecary
 
         next if ignored_files.include?(info.relative_path)
 
-        add_files_to_list(file_list, info)
+        add_matching_package_managers_for_file_to_list(file_list, info)
       end
 
       file_list
@@ -71,7 +72,7 @@ module Bibliothecary
 
         next if ignored_files.include?(info.relative_path)
 
-        add_files_to_list(file_list, info)
+        add_matching_package_managers_for_file_to_list(file_list, info)
       end
 
       file_list
@@ -87,12 +88,15 @@ module Bibliothecary
         next unless FileTest.file?(subpath)
         next if ignored_files.include?(info.relative_path)
 
-        add_files_to_list(file_list, info)
+        add_matching_package_managers_for_file_to_list(file_list, info)
       end
 
       file_list
     end
 
+    # Get a list of files in this path grouped by filename and repeated by package manager.
+    #
+    # @return [Array<Bibliothecary::RelatedFilesInfo>]
     def find_manifests(path)
       RelatedFilesInfo.create_from_file_infos(load_file_info_list(path).reject { |info| info.package_manager.nil? })
     end
@@ -101,10 +105,16 @@ module Bibliothecary
       RelatedFilesInfo.create_from_file_infos(load_file_info_list_from_paths(paths).reject { |info| info.package_manager.nil? })
     end
 
+    # file_path_contents_hash contains an Array of { file_path, contents }
     def find_manifests_from_contents(file_path_contents_hash)
-      RelatedFilesInfo.create_from_file_infos(load_file_info_list_from_contents(file_path_contents_hash).reject { |info| info.package_manager.nil? })
+      RelatedFilesInfo.create_from_file_infos(
+        load_file_info_list_from_contents(
+          file_path_contents_hash
+        ).reject { |info| info.package_manager.nil? }
+      )
     end
 
+    # Read a manifest file and extract the list of dependencies from that file.
     def analyse_file(file_path, contents)
       package_managers.select { |pm| pm.match?(file_path, contents) }.map do |pm|
         pm.analyse_contents(file_path, contents, options: @options)
@@ -140,15 +150,30 @@ module Bibliothecary
       @configuration.ignored_files
     end
 
+    # We don't know what file groups are in multi file manifests until
+    # we process them. In those cases, process those, then reject the
+    # RelatedFilesInfo objects that aren't in the manifest.
+    #
+    # This means we're likely analyzing these files twice in processing,
+    # but we need that accurate package manager information.
+    def filter_multi_manifest_entries(path, related_files_info_entries)
+      MultiManifestFilter.new(path: path, related_files_info_entries: related_files_info_entries , runner: self).results
+    end
+
     private
 
-    def add_files_to_list(file_list, info)
-      applicable_package_managers(info).each do |package_manager|
-        file = info.dup
-        file.package_manager = package_manager
+    # Get the list of all package managers that apply to the file provided
+    # as file_info, and, for each one, duplicate file_info and fill in
+    # the appropriate package manager.
+    def add_matching_package_managers_for_file_to_list(file_list, file_info)
+      applicable_package_managers(file_info).each do |package_manager|
+        new_file_info = file_info.dup
+        new_file_info.package_manager = package_manager
 
-        file_list.push(file)
+        file_list.push(new_file_info)
       end
     end
   end
 end
+
+require_relative './runner/multi_manifest_filter.rb'

data/lib/bibliothecary/version.rb

@@ -1,3 +1,3 @@
 module Bibliothecary
-  VERSION = "8.2.0"
+  VERSION = "8.2.3"
 end

data/lib/bibliothecary.rb

@@ -16,6 +16,8 @@ Dir[File.expand_path('../bibliothecary/parsers/*.rb', __FILE__)].each do |file|
 end
 
 module Bibliothecary
+  VERSION_OPERATORS = /[~^<>*"]/
+
   def self.analyse(path, ignore_unparseable_files: true)
     runner.analyse(path, ignore_unparseable_files: ignore_unparseable_files)
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bibliothecary
 version: !ruby/object:Gem::Version
-  version: 8.2.0
+  version: 8.2.3
 platform: ruby
 authors:
 - Andrew Nesbitt
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-04-29 00:00:00.000000000 Z
+date: 2022-05-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: tomlrb
@@ -260,6 +260,7 @@ files:
 - lib/bibliothecary/file_info.rb
 - lib/bibliothecary/multi_parsers/bundler_like_manifest.rb
 - lib/bibliothecary/multi_parsers/cyclonedx.rb
+- lib/bibliothecary/multi_parsers/dependencies_csv.rb
 - lib/bibliothecary/multi_parsers/json_runtime.rb
 - lib/bibliothecary/parsers/bower.rb
 - lib/bibliothecary/parsers/cargo.rb
@@ -271,7 +272,6 @@ files:
 - lib/bibliothecary/parsers/cran.rb
 - lib/bibliothecary/parsers/dub.rb
 - lib/bibliothecary/parsers/elm.rb
-- lib/bibliothecary/parsers/generic.rb
 - lib/bibliothecary/parsers/go.rb
 - lib/bibliothecary/parsers/hackage.rb
 - lib/bibliothecary/parsers/haxelib.rb
@@ -289,6 +289,7 @@ files:
 - lib/bibliothecary/parsers/swift_pm.rb
 - lib/bibliothecary/related_files_info.rb
 - lib/bibliothecary/runner.rb
+- lib/bibliothecary/runner/multi_manifest_filter.rb
 - lib/bibliothecary/version.rb
 - lib/sdl_parser.rb
 homepage: https://github.com/librariesio/bibliothecary

data/lib/bibliothecary/parsers/generic.rb

@@ -1,39 +0,0 @@
-require 'csv'
-
-module Bibliothecary
-  module Parsers
-    class Generic
-      include Bibliothecary::Analyser
-
-      def self.mapping
-        {
-          match_filename("dependencies.csv") => {
-            kind: 'lockfile',
-            parser: :parse_lockfile
-          }
-        }
-      end
-
-      def self.parse_lockfile(file_contents, options: {})
-        table = CSV.parse(file_contents, headers: true)
-
-        required_headers = ["platform", "name", "requirement"]
-        missing_headers = required_headers - table.headers
-        raise "Missing headers #{missing_headers} in CSV" unless missing_headers.empty?
-
-        table.map.with_index do |row, idx|
-          line = idx + 2 # use 1-based index just like the 'csv' std lib, and count the headers as first row.
-          required_headers.each do |h|
-            raise "missing field '#{h}' on line #{line}" if row[h].nil? || row[h].empty?
-          end
-          {
-            platform: row['platform'],
-            name: row['name'],
-            requirement: row['requirement'],
-            type: row.fetch('type', 'runtime'),
-          }
-        end
-      end
-    end
-  end
-end