bibliothecary 6.10.3 → 6.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: af1a007a1d21ef7ecfeffcbe97319185c31da96d252f27a58619e60afcc3faf8
-  data.tar.gz: 64fb93438b9f3af750b2d69206ec7601e52b77bb5236a2783ad724a064125f33
+  metadata.gz: caaa965267d6272e22ec385e0ad56505f645cdc3437163ec2a9d40471c1e620b
+  data.tar.gz: a2ad1df3e6481e0faed3d27039557b5b248f73fe5fa9864cd7d9462e215af72a
 SHA512:
-  metadata.gz: f4816e56d6cea72b20deaa2f2832a27ce1c2d99ffe5ac224e7259bb3595b699083ec32dbd2b2b756ec0ea8c82aa411908f4ab83eed366e41726f5d634e241630
-  data.tar.gz: 1f5c52f3c4ad451a2c32e93bc129694580bb7f1c5af8769e859cfcc74ee928ea8a3ed0595c84c7a5ff13bad118ce4ff02bd519ee12bdd7c5f688dc2e98076e17
+  metadata.gz: 2a333bfa32d1c31abf164afd8a09b7060ffe2809b43199273e706e83dd01d9638e57f85787390deba7e923a9b26494e47074718df319bac159d9cfbb4bb32a26
+  data.tar.gz: 7a68137ae7f02721a2ae8ae94dad59783a07ba291c440cbc6469af0164b0f57a83b055dd16ac5ca58ba2a00de76e217a8fd778b30c98b1585760aa47f60c3ff1

data/.circleci/config.yml

@@ -0,0 +1,28 @@
+version: 2.1
+orbs:
+  ruby: circleci/ruby@0.1.2
+
+jobs:
+  test:
+    docker:
+      - image: circleci/ruby:2.6.6-stretch-node
+    executor: ruby/default
+    steps:
+    - checkout
+    - run:
+        name: Which bundler?
+        command: bundle -v
+    - ruby/bundle-install
+    - run:
+        name: Run specs
+        command: bundle exec rake spec
+    - run:
+        name: CodeClimate
+        command: bundle exec codeclimate-test-reporter
+
+workflows:
+  version: 2.1
+  test:
+    jobs:
+      - test
+

data/lib/bibliothecary.rb

@@ -21,8 +21,8 @@ module Bibliothecary
     runner.load_file_list(path)
   end
 
-  def self.init_package_manager(info)
-    runner.init_package_manager(info)
+  def self.applicable_package_managers(info)
+    runner.applicable_package_managers(info)
   end
 
   def self.load_file_info_list(path)

data/lib/bibliothecary/parsers/conda.rb

@@ -26,14 +26,13 @@ module Bibliothecary
         }
       end
 
-      def self.parse_conda(info)
-        dependencies = call_conda_parser_web(info, "manifest")[:manifest]
+      def self.parse_conda(info, kind = "manifest")
+        dependencies = call_conda_parser_web(info, kind)[kind.to_sym]
         dependencies.map { |dep| dep.merge(type: "runtime") }
       end
 
       def self.parse_conda_lockfile(info)
-        dependencies = call_conda_parser_web(info, "lockfile")[:lockfile]
-        dependencies.map { |dep| dep.merge(type: "runtime") }
+        parse_conda(info, "lockfile")
       end
 
       private_class_method def self.call_conda_parser_web(file_contents, kind)

data/lib/bibliothecary/parsers/maven.rb

@@ -154,7 +154,7 @@ module Bibliothecary
       end
       def self.parse_maven_tree(file_contents)
         file_contents = file_contents.gsub(/\r\n?/, "\n")
-        captures = file_contents.scan(/^\[INFO\][\s|+-\\]+([[\w.-]+:]+)(?=$|\s\(optional\)$)/).flatten.uniq
+        captures = file_contents.scan(/^\[INFO\](?:(?:\+-)|\||(?:\\-)|\s)+((?:[\w\.-]+:)+[\w\.\-${}]+)/).flatten.uniq
         captures.map do |item|
           parts = item.split(":")
           case parts.count
@@ -205,8 +205,8 @@ module Bibliothecary
         json = JSON.parse(response.body)
         return [] unless json['dependencies']
         json['dependencies'].map do |dependency|
-          name = [dependency["group"], dependency["name"]].join(':')
-          next unless name =~ (/[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+(\.[A-Za-z0-9_-])?\:[A-Za-z0-9_-]/)
+          name = gradle_dependency_name(dependency["group"], dependency["name"])
+          next unless name =~ /[\w-]+\.[\w_-]+(\.[\w-])?\:[\w-]/
           {
             name: name,
             requirement: dependency["version"],
@@ -215,6 +215,20 @@ module Bibliothecary
         end.compact
       end
 
+      def self.gradle_dependency_name(group, name)
+        if group.empty? && name.include?(":")
+          group, name = name.split(":", 2)
+        end
+
+        # Strip comments, and single/doublequotes
+        [group, name].map do |part|
+          part
+            .gsub(/\s*\/\/.*$/, "") # Comments
+            .gsub(/^["']/, "") # Beginning single/doublequotes
+            .gsub(/["']$/, "") # Ending single/doublequotes
+        end.join(":")
+      end
+
       def self.extract_pom_info(xml, location, parent_properties = {})
         extract_pom_dep_info(xml, xml, location, parent_properties)
       end

data/lib/bibliothecary/parsers/npm.rb

@@ -22,6 +22,10 @@ module Bibliothecary
           match_filename("package-lock.json") => {
             kind: 'lockfile',
             parser: :parse_package_lock
+          },
+          match_filename("npm-ls.json") => {
+            kind: 'lockfile',
+            parser: :parse_ls
           }
         }
       end
@@ -75,6 +79,25 @@ module Bibliothecary
           }
         end
       end
+
+      def self.parse_ls(file_contents)
+        manifest = JSON.parse(file_contents)
+
+        transform_tree_to_array(manifest.fetch('dependencies', {}))
+      end
+
+      private_class_method def self.transform_tree_to_array(deps_by_name)
+        deps_by_name.map do |name, metadata|
+          [
+            {
+              name: name,
+              requirement: metadata["version"],
+              lockfile_requirement: metadata.fetch("from", "").split('@').last,
+              type: "runtime"
+            }
+          ] + transform_tree_to_array(metadata.fetch("dependencies", {}))
+        end.flatten(1)
+      end
     end
   end
 end

data/lib/bibliothecary/parsers/pypi.rb

@@ -39,7 +39,24 @@ module Bibliothecary
           match_filename("poetry.lock") => {
             kind: 'lockfile',
             parser: :parse_poetry_lock
-          }
+          },
+          # Pip dependencies can be embedded in conda environment files
+          match_filename("environment.yml") => {
+            parser: :parse_conda,
+            kind: "manifest",
+          },
+          match_filename("environment.yaml") => {
+            parser: :parse_conda,
+            kind: "manifest",
+          },
+          match_filename("environment.yml.lock") => {
+            parser: :parse_conda,
+            kind: "lockfile",
+          },
+          match_filename("environment.yaml.lock") => {
+            parser: :parse_conda,
+            kind: "lockfile",
+          },
         }
       end
 
@@ -53,6 +70,17 @@ module Bibliothecary
         map_dependencies(manifest['dependencies'], 'runtime') + map_dependencies(manifest['dev-dependencies'], 'develop')
       end
 
+      def self.parse_conda(file_contents)
+        contents = YAML.safe_load(file_contents)
+        return [] unless contents
+
+        dependencies = contents["dependencies"]
+        pip = dependencies.find { |dep| dep.is_a?(Hash) && dep["pip"]}
+        return [] unless pip
+
+        Pypi.parse_requirements_txt(pip["pip"].join("\n"))
+      end
+
       def self.map_dependencies(packages, type)
         return [] unless packages
         packages.map do |name, info|

data/lib/bibliothecary/runner.rb

@@ -32,14 +32,9 @@ module Bibliothecary
       load_file_info_list(path).map { |info| info.full_path }
     end
 
-    def init_package_manager(info)
-      # set the package manager on each info
-      matches = package_managers.select { |pm| pm.match_info?(info) }
-
-      info.package_manager = matches[0] if matches.length == 1
-
-      # this is a bug at the moment if it's raised (we don't handle it sensibly)
-      raise "Multiple package managers fighting over #{info.relative_path}: #{matches.map(&:to_s)}" if matches.length > 1
+    def applicable_package_managers(info)
+      managers = package_managers.select { |pm| pm.match_info?(info) }
+      managers.length > 0 ? managers : [nil]
     end
 
     def package_managers
@@ -48,29 +43,41 @@ module Bibliothecary
 
     def load_file_info_list_from_paths(paths)
       file_list = []
+
       paths.each do |path|
         info = FileInfo.new(nil, path)
 
         next if ignored_files.include?(info.relative_path)
 
-        init_package_manager(info)
-        file_list.push(info)
+        applicable_package_managers(info).each do |package_manager|
+          file = info.dup
+          file.package_manager = package_manager
+
+          file_list.push(file)
+        end
       end
+
       file_list
     end
 
     def load_file_info_list(path)
       file_list = []
+
       Find.find(path) do |subpath|
         info = FileInfo.new(path, subpath)
+
         Find.prune if FileTest.directory?(subpath) && ignored_dirs.include?(info.relative_path)
         next unless FileTest.file?(subpath)
         next if ignored_files.include?(info.relative_path)
 
-        init_package_manager(info)
+        applicable_package_managers(info).each do |package_manager|
+          file = info.dup
+          file.package_manager = package_manager
 
-        file_list.push(info)
+          file_list.push(file)
+        end
       end
+
       file_list
     end
 

data/lib/bibliothecary/version.rb

@@ -1,3 +1,3 @@
 module Bibliothecary
-  VERSION = "6.10.3"
+  VERSION = "6.11.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bibliothecary
 version: !ruby/object:Gem::Version
-  version: 6.10.3
+  version: 6.11.0
 platform: ruby
 authors:
 - Andrew Nesbitt
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-03-18 00:00:00.000000000 Z
+date: 2021-05-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: toml-rb
@@ -216,6 +216,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".circleci/config.yml"
 - ".codeclimate.yml"
 - ".github/CONTRIBUTING.md"
 - ".gitignore"
@@ -288,7 +289,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Find and parse manifests