bibliothecary 6.3.1 → 6.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ba88fac4e7fbc938bf270504f1b41787f314105f237bb1e4e080e5ad82379aef
-  data.tar.gz: 28de14e318ae6684074ac38e8a7b2f1fb57fa83d21a97e7908d6afe110bb5d22
+  metadata.gz: e4a0567c8ec55a4efdce0ef3582ffb085587a4efcc50b91e3f3596444629d60c
+  data.tar.gz: 63c5203b34b171ef99ce8d9beac0e05aabc2fcb202c1125a3184a2968ba6faba
 SHA512:
-  metadata.gz: a07ec3aeb787fe83e7a4d415f83b5aeefe1649eb590a320a48a90053b26b5bf215e6649799bc5792e9e20e5e6fc920bddd0f43c3e8f08111d34baf2695f64ed9
-  data.tar.gz: 3da7fe10d6c6e036091a82e84a1cbfa80ef9ee58f37289629fe8076b1c49da38a7dc40d03dfe7b56b3e1a12327dc714bf220576a2ca6b072ab66a7934bf06c0d
+  metadata.gz: 4e68baf27de703a993167504612de962db9ae01b8aa8ab84e2de8556074521076fe7ea27dc0e5596c1188dfd747bb541a31898da5c6ca6f2e40d0af2e7adbd09
+  data.tar.gz: 25f3d2f9f38bcce1c339851e5b68b1f92f354ecd0b4ea0608f258c69c7a6a0a0376dd4f2e160208c537aa1370aa0298680af6d10c22c4e82c89a14936ed9aa69

data/.tidelift.yml

@@ -0,0 +1,2 @@
+ci:
+  extra_ignore_directories: [ spec/fixtures ]

data/.travis.yml

@@ -1,6 +1,5 @@
 language: ruby
 rvm:
-- 2.4.2
 - 2.5.1
 cache: bundler
 before_install:

data/bibliothecary.gemspec

@@ -27,6 +27,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "commander"
 
   spec.add_development_dependency "bundler", "~> 1.11"
+  spec.add_development_dependency "pry"
   spec.add_development_dependency "rake", "~> 12.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "webmock"

data/lib/bibliothecary.rb

@@ -1,55 +1,71 @@
 require "bibliothecary/version"
 require "bibliothecary/analyser"
 require "bibliothecary/configuration"
+require "bibliothecary/runner"
+require "bibliothecary/exceptions"
+require "bibliothecary/file_info"
+require "find"
 
 Dir[File.expand_path('../bibliothecary/parsers/*.rb', __FILE__)].each do |file|
   require file
 end
 
 module Bibliothecary
-  def self.analyse(path)
-    cmd = `find #{path} -type f | grep -vE "#{ignored_files_regex}"`
-    file_list = cmd.split("\n").sort
-    package_managers.map{|pm| pm.analyse(path, file_list) }.flatten.compact
+  def self.analyse(path, ignore_unparseable_files: true)
+    runner.analyse(path, ignore_unparseable_files: ignore_unparseable_files)
+  end
+
+  # deprecated; use load_file_info_list.
+  def self.load_file_list(path)
+    runner.load_file_list(path)
+  end
+
+  def self.init_package_manager(info)
+    runner.init_package_manager(info)
+  end
+
+  def self.load_file_info_list(path)
+    runner.load_file_info_list(path)
   end
 
   def self.analyse_file(file_path, contents)
-    package_managers.map do |pm|
-      pm.analyse_contents(file_path, contents)
-    end.flatten.uniq.compact
+    runner.analyse_file(file_path, contents)
   end
 
   def self.identify_manifests(file_list)
-    allowed_file_list = file_list.reject{|f| f.start_with?(*ignored_files) }
-    package_managers.map do |pm|
-      allowed_file_list.select do |file_path|
-        pm.match?(file_path)
-      end
-    end.flatten.uniq.compact
+    runner.identify_manifests(file_list)
   end
 
   def self.package_managers
-    Bibliothecary::Parsers.constants.map{|c| Bibliothecary::Parsers.const_get(c) }.sort_by{|c| c.to_s.downcase }
+    runner.package_managers
   end
 
-  def self.ignored_files
-    configuration.ignored_files
+  def self.ignored_dirs
+    configuration.ignored_dirs
   end
 
-  def self.ignored_files_regex
-    ignored_files.join('|')
+  def self.ignored_files
+    configuration.ignored_files
   end
 
   class << self
     attr_writer :configuration
   end
 
+  def self.runner
+    configuration
+    @runner
+  end
+
   def self.configuration
     @configuration ||= Configuration.new
+    @runner = Runner.new(@configuration)
+    @configuration
   end
 
   def self.reset
     @configuration = Configuration.new
+    @runner = Runner.new(@configuration)
   end
 
   def self.configure

data/lib/bibliothecary/analyser.rb

@@ -1,20 +1,81 @@
 module Bibliothecary
   module Analyser
+    def self.create_error_analysis(platform_name, relative_path, kind, message)
+      {
+        platform: platform_name,
+        path: relative_path,
+        dependencies: nil,
+        kind: kind,
+        success: false,
+        error_message: message
+      }
+    end
+
+    def self.create_analysis(platform_name, relative_path, kind, dependencies)
+      {
+        platform: platform_name,
+        path: relative_path,
+        dependencies: dependencies,
+        kind: kind,
+        success: true
+      }
+    end
+
     def self.included(base)
       base.extend(ClassMethods)
     end
     module ClassMethods
+      def mapping_entry_match?(regex, details, info)
+        if info.relative_path.match(regex)
+          # we only want to load contents if we don't have them already
+          # and there's a content_matcher method to use
+          return true if details[:content_matcher].nil?
+          # this is the libraries.io case where we won't load all .xml
+          # files (for example) just to look at their contents, we'll
+          # assume they are not manifests.
+          return false if info.contents.nil?
+          return send(details[:content_matcher], info.contents)
+        else
+          return false
+        end
+      end
+
       def parse_file(filename, contents)
         mapping.each do |regex, details|
-          if filename.match(regex)
-            return send(details[:parser], contents)
+          if mapping_entry_match?(regex, details, FileInfo.new(nil, filename, contents))
+            begin
+              # The `parser` method should raise an exception if the file is malformed,
+              # should return empty [] if the file is fine but simply doesn't contain
+              # any dependencies, and should never return nil. At the time of writing
+              # this comment, some of the parsers return [] or nil to mean an error
+              # which is confusing to users.
+              return send(details[:parser], contents)
+            rescue Exception => e # default is StandardError but C bindings throw Exceptions
+              # the C xml parser also puts a newline at the end of the message
+              raise Bibliothecary::FileParsingError.new(e.message.strip, filename)
+            end
           end
         end
-        return []
+        # this can be raised if we don't check match?/match_info?,
+        # OR don't have the file contents when we check them, so
+        # it turns out for example that a .xml file isn't a
+        # manifest after all.
+        raise Bibliothecary::FileParsingError.new("No parser for this file type", filename)
+      end
+
+      # this is broken with contents=nil because it can't look at file
+      # contents, so skips manifests that are ambiguously a
+      # manifest considering only the filename. However, those are
+      # the semantics that libraries.io uses since it doesn't have
+      # the files locally.
+      def match?(filename, contents = nil)
+        match_info?(FileInfo.new(nil, filename, contents))
       end
 
-      def match?(filename)
-        mapping.keys.any?{|regex| filename.match(regex) }
+      def match_info?(info)
+        mapping.any? do |regex, details|
+          mapping_entry_match?(regex, details, info)
+        end
       end
 
       def platform_name
@@ -43,15 +104,15 @@ module Bibliothecary
 
       def set_related_paths_field(by_dirname_dest, by_dirname_source)
         by_dirname_dest.each do |dirname, analyses|
-          analyses.each do |analysis|
-            source_analyses = by_dirname_source[dirname].map { |source_analysis| source_analysis[:path] }
+          analyses.each do |(info, analysis)|
+            source_analyses = by_dirname_source[dirname].map { |(info, source_analysis)| info.relative_path }
             analysis[:related_paths] = source_analyses.sort
           end
         end
       end
 
       def add_related_paths(analyses)
-        analyses.each do |analysis|
+        analyses.each do |(info, analysis)|
           analysis[:related_paths] = []
         end
 
@@ -67,13 +128,16 @@ module Bibliothecary
           "lockfile" => Hash.new { |h, k| h[k] = [] }
         }
 
-        analyses.each do |analysis|
-          dirname = File.dirname(analysis[:path])
-          by_dirname[analysis[:kind]][dirname].push(analysis)
+        analyses.each do |(info, analysis)|
+          dirname = File.dirname(info.relative_path)
+          by_dirname[analysis[:kind]][dirname].push([info, analysis])
         end
 
         by_dirname["manifest"].each do |_, manifests|
-          manifests.delete_if { |manifest| !determine_can_have_lockfile(manifest[:path]) }
+          # This determine_can_have_lockfile in theory needs the file contents but
+          # in practice doesn't right now since the only mapping that needs
+          # file contents is a lockfile and not a manifest so won't reach here.
+          manifests.delete_if { |(info, manifest)| !determine_can_have_lockfile_from_info(info) }
         end
 
         set_related_paths_field(by_dirname["manifest"], by_dirname["lockfile"])
@@ -81,44 +145,60 @@ module Bibliothecary
       end
 
       def analyse(folder_path, file_list)
-        analyses = file_list.map do |path|
-          filename = path.gsub(folder_path, '').gsub(/^\//, '')
-          next unless match?(filename)
-          contents = File.open(path).read
-          analyse_contents(filename, contents)
-        end.compact
+        analyse_file_info(file_list.map { |full_path| FileInfo.new(folder_path, full_path) })
+      end
+
+      def analyse_file_info(file_info_list)
+        analyses = file_info_list.map do |info|
+          next unless match_info?(info)
+          [info, analyse_contents_from_info(info)]
+        end
+
+        # strip the ones we failed to analyse
+        analyses = analyses.reject { |(info, analysis)| analysis.nil? }
 
         add_related_paths(analyses)
 
-        analyses
+        analyses.map { |(info, analysis)| analysis }
       end
 
       def analyse_contents(filename, contents)
-        dependencies = parse_file(filename, contents)
-        if dependencies && dependencies.any?
-          {
-            platform: platform_name,
-            path: filename,
-            dependencies: dependencies,
-            kind: determine_kind(filename)
-          }
-        end
-      # rescue
-      #   nil
+        analyse_contents_from_info(FileInfo.new(nil, filename, contents))
+      end
+
+      def analyse_contents_from_info(info)
+        kind = determine_kind_from_info(info)
+        dependencies = parse_file(info.relative_path, info.contents)
+
+        Bibliothecary::Analyser::create_analysis(platform_name, info.relative_path, kind, dependencies || [])
+      rescue Bibliothecary::FileParsingError => e
+        Bibliothecary::Analyser::create_error_analysis(platform_name, info.relative_path, kind, e.message)
+      end
+
+      # calling this with contents=nil can produce less-informed
+      # results, but kept for back compat
+      def determine_kind(filename, contents = nil)
+        determine_kind_from_info(FileInfo.new(nil, filename, contents))
       end
 
-      def determine_kind(filename)
+      def determine_kind_from_info(info)
         mapping.each do |regex, details|
-          if filename.match(regex)
+          if mapping_entry_match?(regex, details, info)
             return details[:kind]
           end
         end
-        return []
+        return nil
+      end
+
+      # calling this with contents=nil can produce less-informed
+      # results, but kept for back compat
+      def determine_can_have_lockfile(filename, contents = nil)
+        determine_can_have_lockfile_from_info(FileInfo.new(nil, filename, contents))
       end
 
-      def determine_can_have_lockfile(filename)
+      def determine_can_have_lockfile_from_info(info)
         mapping.each do |regex, details|
-          if filename.match(regex)
+          if mapping_entry_match?(regex, details, info)
             return details.fetch(:can_have_lockfile, true)
           end
         end

data/lib/bibliothecary/configuration.rb

@@ -1,5 +1,6 @@
 module Bibliothecary
   class Configuration
+    attr_accessor :ignored_dirs
     attr_accessor :ignored_files
     attr_accessor :carthage_parser_host
     attr_accessor :clojars_parser_host
@@ -10,7 +11,8 @@ module Bibliothecary
     attr_accessor :cabal_parser_host
 
     def initialize
-      @ignored_files = ['.git', 'node_modules', 'bower_components', 'spec/fixtures', 'vendor', 'dist']
+      @ignored_dirs = ['.git', 'node_modules', 'bower_components', 'vendor', 'dist']
+      @ignored_files = []
       @carthage_parser_host = 'https://carthage.libraries.io'
       @clojars_parser_host  = 'https://clojars.libraries.io'
       @mix_parser_host      = 'https://mix.libraries.io'

data/lib/bibliothecary/exceptions.rb

@@ -0,0 +1,18 @@
+module Bibliothecary
+  class RemoteParsingError < StandardError
+    attr_accessor :code
+    def initialize(msg, code)
+      @code = code
+      super(msg)
+    end
+  end
+
+  class FileParsingError < StandardError
+    attr_accessor :filename
+    def initialize(msg, filename)
+      @filename = filename
+      msg = "#{filename}: #{msg}" unless msg.include?(filename)
+      super("#{msg}")
+    end
+  end
+end

data/lib/bibliothecary/file_info.rb

@@ -0,0 +1,51 @@
+require 'pathname'
+
+module Bibliothecary
+  class FileInfo
+    attr_reader :folder_path
+    attr_reader :relative_path
+    attr_reader :full_path
+    attr_accessor :package_manager
+
+    def contents
+      @contents ||=
+        begin
+          if @folder_path.nil?
+            # if we have no folder_path then we aren't dealing with a
+            # file that's actually on the filesystem
+            nil
+          else
+            File.open(@full_path).read
+          end
+        end
+    end
+
+    # If the FileInfo represents an actual file on disk,
+    # the contents can be nil and lazy-loaded; we allow
+    # contents to be passed in here to allow pulling them
+    # from somewhere other than the disk.
+    def initialize(folder_path, full_path, contents = nil)
+      # Note that cleanpath does NOT touch the filesystem,
+      # leaving the lazy-load of file contents as the only
+      # time we touch the filesystem.
+
+      full_pathname = Pathname.new(full_path)
+      @full_path = full_pathname.cleanpath.to_path
+
+      if folder_path.nil?
+        # this is the case where we e.g. have filenames from the GitHub API
+        # and don't have a local folder
+        @folder_path = nil
+        @relative_path = @full_path
+      else
+        folder_pathname = Pathname.new(folder_path)
+        @folder_path = folder_pathname.cleanpath.to_path
+        @relative_path = full_pathname.relative_path_from(folder_pathname).cleanpath.to_path
+      end
+
+      @contents = contents
+
+      @package_manager = nil
+    end
+  end
+end

data/lib/bibliothecary/parsers/carthage.rb

@@ -34,6 +34,7 @@ module Bibliothecary
 
       def self.map_dependencies(manifest, path)
         response = Typhoeus.post("#{Bibliothecary.configuration.carthage_parser_host}/#{path}", params: {body: manifest})
+        raise Bibliothecary::RemoteParsingError.new("Http Error #{response.response_code} when contacting: #{Bibliothecary.configuration.carthage_parser_host}/#{path}", response.response_code) unless response.success?
         json = JSON.parse(response.body)
 
         json.map do |dependency|

data/lib/bibliothecary/parsers/clojars.rb

@@ -17,6 +17,7 @@ module Bibliothecary
 
       def self.parse_manifest(manifest)
         response = Typhoeus.post("#{Bibliothecary.configuration.clojars_parser_host}/project.clj", body: manifest)
+        raise Bibliothecary::RemoteParsingError.new("Http Error #{response.response_code} when contacting: #{Bibliothecary.configuration.clojars_parser_host}/project.clj", response.response_code) unless response.success?
         json = JSON.parse response.body
         index = json.index("dependencies")
 

data/lib/bibliothecary/parsers/hackage.rb

@@ -26,11 +26,8 @@ module Bibliothecary
 
         response = Typhoeus.post("#{Bibliothecary.configuration.cabal_parser_host}/parse", headers: headers, body: file_contents)
 
-        if response.response_code == 200 then
-          JSON.parse(response.body, symbolize_names: true)
-        else
-          []
-        end
+        raise Bibliothecary::RemoteParsingError.new("Http Error #{response.response_code} when contacting: #{Bibliothecary.configuration.cabal_parser_host}/parse", response.response_code) unless response.success?
+        JSON.parse(response.body, symbolize_names: true)
       end
 
       def self.parse_cabal_config(file_contents)

data/lib/bibliothecary/parsers/hex.rb

@@ -20,6 +20,7 @@ module Bibliothecary
 
       def self.parse_mix(manifest)
         response = Typhoeus.post("#{Bibliothecary.configuration.mix_parser_host}/", body: manifest)
+        raise Bibliothecary::RemoteParsingError.new("Http Error #{response.response_code} when contacting: #{Bibliothecary.configuration.mix_parser_host}/", response.response_code) unless response.success?
         json = JSON.parse response.body
 
         json.map do |name, version|
@@ -33,6 +34,7 @@ module Bibliothecary
 
       def self.parse_mix_lock(manifest)
         response = Typhoeus.post("#{Bibliothecary.configuration.mix_parser_host}/lock", body: manifest)
+        raise Bibliothecary::RemoteParsingError.new("Http Error #{response.response_code} when contacting: #{Bibliothecary.configuration.mix_parser_host}/", response.response_code) unless response.success?
         json = JSON.parse response.body
 
         json.map do |name, info|

data/lib/bibliothecary/parsers/maven.rb

@@ -5,6 +5,12 @@ module Bibliothecary
     class Maven
       include Bibliothecary::Analyser
 
+      # e.g. "annotationProcessor - Annotation processors and their dependencies for source set 'main'."
+      GRADLE_TYPE_REGEX = /^(\w+)/
+
+      # "|    \\--- com.google.guava:guava:23.5-jre (*)"
+      GRADLE_DEP_REGEX = /(\+---|\\---){1}/
+
       def self.mapping
         {
           /^ivy\.xml$|.*\/ivy\.xml$/i => {
@@ -18,6 +24,15 @@ module Bibliothecary
           /^build.gradle$|.*\/build.gradle$/i => {
             kind: 'manifest',
             parser: :parse_gradle
+          },
+          /^.+.xml$/i => {
+            content_matcher: :ivy_report?,
+            kind: 'lockfile',
+            parser: :parse_ivy_report
+          },
+          /^gradle-dependencies-q\.txt|.*\/gradle-dependencies-q\.txt$/i => {
+            kind: 'lockfile',
+            parser: :parse_gradle_resolved
           }
         }
       end
@@ -32,8 +47,66 @@ module Bibliothecary
             type: 'runtime'
           }
         end
-      rescue
-        []
+      end
+
+      def self.ivy_report?(file_contents)
+        doc = Ox.parse file_contents
+        root = doc&.locate("ivy-report")&.first
+        return !root.nil?
+      rescue Exception => e
+        # We rescue exception here since native libs can throw a non-StandardError
+        # We don't want to throw errors during the matching phase, only during
+        # parsing after we match.
+        false
+      end
+
+      def self.parse_ivy_report(file_contents)
+        doc = Ox.parse file_contents
+        root = doc.locate("ivy-report").first
+        raise "ivy-report document does not have ivy-report at the root" if root.nil?
+        info = doc.locate("ivy-report/info").first
+        raise "ivy-report document lacks <info> element" if info.nil?
+        type = info.attributes[:conf]
+        type = "unknown" if type.nil?
+        modules = doc.locate("ivy-report/dependencies/module")
+        modules.map do |mod|
+          attrs = mod.attributes
+          org = attrs[:organisation]
+          name = attrs[:name]
+          version = mod.locate('revision').first&.attributes[:name]
+
+          next nil if org.nil? or name.nil? or version.nil?
+
+          {
+            name: "#{org}:#{name}",
+            requirement: version,
+            type: type
+          }
+        end.compact
+      end
+
+      def self.parse_gradle_resolved(file_contents)
+        type = nil
+        file_contents.split("\n").map do |line|
+          type_match = GRADLE_TYPE_REGEX.match(line)
+          type = type_match.captures[0] if type_match
+
+          gradle_dep_match = GRADLE_DEP_REGEX.match(line)
+          next unless gradle_dep_match
+
+          split = gradle_dep_match.captures[0]
+
+          # org.springframework.boot:spring-boot-starter-web:2.1.0.M3 (*)
+          # Lines can end with (n) or (*) to indicate that something was not resolved (n) or resolved previously (*).
+          dep = line.split(split)[1].sub(/\(n\)$/, "").sub(/\(\*\)$/,"").strip.split(":")
+          version = dep[-1]
+          version = version.split("->")[-1].strip if line.include?("->")
+          {
+            name: dep[0, dep.length - 1].join(":"),
+            requirement: version,
+            type: type
+          }
+        end.compact.uniq {|item| [item[:name], item[:requirement], item[:type]]}
       end
 
       def self.parse_pom_manifest(file_contents)
@@ -51,12 +124,11 @@ module Bibliothecary
             type: extract_pom_dep_info(xml, dependency, 'scope') || 'runtime'
           }
         end
-      rescue
-        []
       end
 
       def self.parse_gradle(manifest)
         response = Typhoeus.post("#{Bibliothecary.configuration.gradle_parser_host}/parse", body: manifest)
+        raise Bibliothecary::RemoteParsingError.new("Http Error #{response.response_code} when contacting: #{Bibliothecary.configuration.gradle_parser_host}/parse", response.response_code) unless response.success?
         json = JSON.parse(response.body)
         return [] unless json['dependencies']
         json['dependencies'].map do |dependency|

data/lib/bibliothecary/parsers/npm.rb

@@ -40,23 +40,31 @@ module Bibliothecary
       def self.parse_package_lock(file_contents)
         manifest = JSON.parse(file_contents)
         manifest.fetch('dependencies',[]).map do |name, requirement|
+          if requirement.fetch("dev", false)
+            type = 'development'
+          else
+            type = 'runtime'
+          end
           {
             name: name,
             requirement: requirement["version"],
-            type: 'runtime'
+            type: type
           }
         end
       end
 
       def self.parse_manifest(file_contents)
         manifest = JSON.parse(file_contents)
+        raise "appears to be a lockfile rather than manifest format" if manifest.key?('lockfileVersion')
         map_dependencies(manifest, 'dependencies', 'runtime') +
         map_dependencies(manifest, 'devDependencies', 'development')
       end
 
       def self.parse_yarn_lock(file_contents)
         response = Typhoeus.post("#{Bibliothecary.configuration.yarn_parser_host}/parse", body: file_contents)
-        return [] unless response.response_code == 200
+
+        raise Bibliothecary::RemoteParsingError.new("Http Error #{response.response_code} when contacting: #{Bibliothecary.configuration.yarn_parser_host}/parse", response.response_code) unless response.success?
+
         json = JSON.parse(response.body, symbolize_names: true)
         json.uniq.map do |dep|
           {

data/lib/bibliothecary/parsers/nuget.rb

@@ -52,7 +52,7 @@ module Bibliothecary
         manifest.packages.locate('package').map do |dependency|
           {
             name: dependency.id,
-            requirement: dependency.version,
+            requirement: (dependency.version if dependency.respond_to? "version") || "*",
             type: 'runtime'
           }
         end
@@ -65,7 +65,7 @@ module Bibliothecary
         packages = manifest.locate('ItemGroup/PackageReference').map do |dependency|
           {
             name: dependency.Include,
-            requirement: dependency.Version,
+            requirement: (dependency.Version if dependency.respond_to? "Version") || "*",
             type: 'runtime'
           }
         end

data/lib/bibliothecary/parsers/swift_pm.rb

@@ -14,6 +14,7 @@ module Bibliothecary
 
       def self.parse_package_swift(manifest)
         response = Typhoeus.post("#{Bibliothecary.configuration.swift_parser_host}/to-json", body: manifest)
+        raise Bibliothecary::RemoteParsingError.new("Http Error #{response.response_code} when contacting: #{Bibliothecary.configuration.swift_parser_host}/to-json", response.response_code) unless response.success?
         json = JSON.parse(response.body)
         json["dependencies"].map do |dependency|
           name = dependency['url'].gsub(/^https?:\/\//, '').gsub(/\.git$/,'')

data/lib/bibliothecary/runner.rb

@@ -0,0 +1,98 @@
+module Bibliothecary
+  # A class that allows bibliothecary to run with multiple configurations at once, rather than with one global
+  class Runner
+
+    def initialize(configuration)
+      @configuration = configuration
+    end
+
+    def analyse(path, ignore_unparseable_files: true)
+      info_list = load_file_info_list(path)
+
+      info_list = info_list.reject { |info| info.package_manager.nil? } if ignore_unparseable_files
+
+      # Each package manager needs to see its entire list so it can
+      # associate related manifests and lockfiles for example.
+      analyses = package_managers.map do |pm|
+        matching_infos = info_list.select { |info| info.package_manager == pm }
+        pm.analyse_file_info(matching_infos)
+      end
+      analyses = analyses.flatten.compact
+
+      info_list.select { |info| info.package_manager.nil? }.each do |info|
+        analyses.push(Bibliothecary::Analyser::create_error_analysis('unknown', info.relative_path, 'unknown',
+                                                                     'No parser for this file type'))
+      end
+
+      analyses
+    end
+
+    # deprecated; use load_file_info_list.
+    def load_file_list(path)
+      load_file_info_list(path).map { |info| info.full_path }
+    end
+
+    def init_package_manager(info)
+      # set the package manager on each info
+      matches = package_managers.select { |pm| pm.match_info?(info) }
+
+      info.package_manager = matches[0] if matches.length == 1
+
+      # this is a bug at the moment if it's raised (we don't handle it sensibly)
+      raise "Multiple package managers fighting over #{info.relative_path}: #{matches.map(&:to_s)}" if matches.length > 1
+    end
+
+    def package_managers
+      Bibliothecary::Parsers.constants.map{|c| Bibliothecary::Parsers.const_get(c) }.sort_by{|c| c.to_s.downcase }
+    end
+
+    def load_file_info_list(path)
+      file_list = []
+      Find.find(path) do |subpath|
+        info = FileInfo.new(path, subpath)
+        Find.prune if FileTest.directory?(subpath) && ignored_dirs.include?(info.relative_path)
+        next unless FileTest.file?(subpath)
+        next if ignored_files.include?(info.relative_path)
+
+        init_package_manager(info)
+
+        file_list.push(info)
+      end
+      file_list
+    end
+
+    def analyse_file(file_path, contents)
+      package_managers.select { |pm| pm.match?(file_path, contents) }.map do |pm|
+        pm.analyse_contents(file_path, contents)
+      end.flatten.uniq.compact
+    end
+
+    # this skips manifests sometimes because it doesn't look at file
+    # contents and can't establish from only regexes that the thing
+    # is a manifest. We exclude rather than include ambiguous filenames
+    # because this API is used by libraries.io and we don't want to
+    # download all .xml files from GitHub.
+    def identify_manifests(file_list)
+      ignored_dirs_with_slash = ignored_dirs.map { |d| if d.end_with?("/") then d else d + "/" end }
+      allowed_file_list = file_list.reject do |f|
+        ignored_dirs.include?(f) || f.start_with?(*ignored_dirs_with_slash)
+      end
+      allowed_file_list = allowed_file_list.reject{|f| ignored_files.include?(f)}
+      package_managers.map do |pm|
+        allowed_file_list.select do |file_path|
+          # this is a call to match? without file contents, which will skip
+          # ambiguous filenames that are only possibly a manifest
+          pm.match?(file_path)
+        end
+      end.flatten.uniq.compact
+    end
+
+    def ignored_dirs
+      @configuration.ignored_dirs
+    end
+
+    def ignored_files
+      @configuration.ignored_files
+    end
+  end
+end

data/lib/bibliothecary/version.rb

@@ -1,3 +1,3 @@
 module Bibliothecary
-  VERSION = "6.3.1"
+  VERSION = "6.3.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bibliothecary
 version: !ruby/object:Gem::Version
-  version: 6.3.1
+  version: 6.3.2
 platform: ruby
 authors:
 - Andrew Nesbitt
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-07-12 00:00:00.000000000 Z
+date: 2019-03-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: toml-rb
@@ -122,6 +122,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.11'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -190,13 +204,11 @@ extra_rdoc_files: []
 files:
 - ".codeclimate.yml"
 - ".github/CONTRIBUTING.md"
-- ".github/ISSUE_TEMPLATE.md"
-- ".github/PULL_REQUEST_TEMPLATE.md"
-- ".github/SUPPORT.md"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
 - ".ruby-version"
+- ".tidelift.yml"
 - ".travis.yml"
 - CODE_OF_CONDUCT.md
 - Gemfile
@@ -212,6 +224,8 @@ files:
 - lib/bibliothecary/analyser.rb
 - lib/bibliothecary/cli.rb
 - lib/bibliothecary/configuration.rb
+- lib/bibliothecary/exceptions.rb
+- lib/bibliothecary/file_info.rb
 - lib/bibliothecary/parsers/bower.rb
 - lib/bibliothecary/parsers/cargo.rb
 - lib/bibliothecary/parsers/carthage.rb
@@ -236,6 +250,7 @@ files:
 - lib/bibliothecary/parsers/rubygems.rb
 - lib/bibliothecary/parsers/shard.rb
 - lib/bibliothecary/parsers/swift_pm.rb
+- lib/bibliothecary/runner.rb
 - lib/bibliothecary/version.rb
 - lib/sdl_parser.rb
 homepage: https://github.com/librariesio/bibliothecary

data/.github/ISSUE_TEMPLATE.md

@@ -1,18 +0,0 @@
-Thanks for taking the time to raise an issue. This template should guide you through the process of submitting a bug, enhancement or feature request. Please erase any part of this template that is not relevant to your issue. 
-
-## Bugs
-Before submitting a bug report:
-
-- [ ] Double-check that the bug is persistent,
-- [ ] Double-check the bug hasn't already been reported [on our issue tracker](https://github.com/issues?utf8=%E2%9C%93&q=is%3Aopen+is%3Aissue+org%3Alibrariesio), they *should* be labelled `bug` or `bugsnag`.
-
-If you have completed those steps then please replace this section with a description of the steps taken to recreate the bug, the expected behavior and the observed behavior.
-
-## Enhancements and Features
-
-Before submitting an enhancement or feature request:
-
-- [ ] Check that the enhancement is not already [in our issue tracker](https://github.com/issues?utf8=%E2%9C%93&q=is%3Aopen+is%3Aissue+org%3Alibrariesio), they should be labelled 'enhancement'., 
-- [ ] For large feature requests, check that your request aligns with our strategy http://docs.libraries.io/strategy.
- 
-If you have complete the above step then please replace this section with a description of your proposed enhancement or feature, the motivation for it, an approach and any alternative approaches considered, and whether you are willing and able to create a pull request for it. Note that we may close this issue if it's not something we're planning on working on.

data/.github/PULL_REQUEST_TEMPLATE.md

@@ -1,10 +0,0 @@
-Thanks taking the time to contribute. This template should help guide you through the process of creating a pull request for review. Please erase any part of this template that is not relevant to your pull request:
-
-
-- [ ] Have you followed the guidelines for [contributors](http://docs.libraries.io/contributorshandbook)?
-- [ ] Have you checked to ensure there aren't other open pull requests on the repository for a similar change?
-- [ ] Is there a corresponding ticket for your pull request?
-- [ ] Have you written new tests for your changes?
-- [ ] Have you successfully run the project with your changes locally?
-
-If so then please replace this section with a link to the ticket(s) it addressed, an explanation of your change and why you think we should include it. Thanks again! 

data/.github/SUPPORT.md

@@ -1,10 +0,0 @@
-# Libraries.io Support
-
-If you're looking for support for Libraries.io there are a lot of options, check out:
-
-* Documentation — https://docs.libraries.io
-* Email — support@libraries.io
-* Twitter — https://twitter.com/librariesio
-* Chat — https://slack.libraries.io
-
-On Discuss and in the Libraries.io Slack team, there are a bunch of helpful community members that should be willing to point you in the right direction.