bibliothecary 8.6.3 → 8.6.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bbb80d11b51f48774a10ad1617525f5a01f580f10571a90b418bcc974ad07da4
-  data.tar.gz: 9a3c8bf41e7a7e461e28ea966acdd98ee5b4fb49d3fb94c59294734a6bd04b6a
+  metadata.gz: 58c0282aec64e81ed313f123670287697f3718e64276e1260030833d70baf892
+  data.tar.gz: be3e9107d5fb2a0968acca81878d4c0bab062bf5b9d994121af2d682a4e8d278
 SHA512:
-  metadata.gz: 0aa9c475c57d5a5fb902cfbd049f12d113ed0ab0472fcd0ee788c07ff3cece579a92fb6d65e0b91d2ecc4415b1cb10c3fcdf2487b539990a11c785ab4078046e
-  data.tar.gz: 6d35f0e547438be4e1cb6fe981ed9a96ffb86a3cdfd3dc8b633b7502412505f7884e0c12792987659c7e547dd48c99333dbe45add7e39bc0b5d4d727ef1cdf0a
+  metadata.gz: 211d953af085e4495325cdd450816f6747ce42b1d8e9c55852e2a50786190466a885cd70b8c454ec1c5d6872ce55f6d8a8ecddd03be7ca2faa02450216f84242
+  data.tar.gz: 74ba743b92e5858c8cc6d7341bf2dda8ff9ce7fdabdff58f44e544554fbc538ae9909726cfcb588d222fa9e1b490d0db9e4ff45896f6f13c375fd1e4ea978f95

data/Gemfile

@@ -1,5 +1,8 @@
 source 'https://rubygems.org'
 
+# Temporarily pegging to HEAD until 0.2.1 is released: https://github.com/piotrmurach/strings-ansi/pull/2
+gem "strings-ansi", ref: "35d0c9430cf0a8022dc12bdab005bce296cb9f00", git: "git@github.com:piotrmurach/strings-ansi.git"
+
 # Specify your gem's dependencies in bibliothecary.gemspec
 gemspec
 

data/bibliothecary.gemspec

@@ -25,7 +25,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "deb_control"
   spec.add_dependency "sdl4r"
   spec.add_dependency "commander"
-  spec.add_dependency "strings-ansi"
+  spec.add_dependency "strings-ansi" # NB this is also pegged to a git sha in Gemfile temporarily.  
   spec.add_dependency "strings"
   spec.add_dependency "packageurl-ruby"
 

data/lib/bibliothecary/cli.rb

@@ -19,7 +19,7 @@ module Bibliothecary
           options.default path: './'
           output = Bibliothecary.analyse(options.path)
           output.each do |file_contents|
-            puts "#{file_contents[:path]} (#{manifest[:platform]})"
+            puts "#{file_contents[:path]} (#{file_contents[:platform]})"
             file_contents[:dependencies].group_by{|d| d[:type] }.each do |type, deps|
               puts "  #{type}"
               deps.each do |dep|

data/lib/bibliothecary/multi_parsers/cyclonedx.rb

@@ -17,26 +17,6 @@ module Bibliothecary
       NoComponents = Class.new(StandardError)
 
       class ManifestEntries
-        # If a purl type (key) exists, it will be used in a manifest for
-        # the key's value. If not, it's ignored.
-        #
-        # https://github.com/package-url/purl-spec/blob/master/PURL-TYPES.rst
-        PURL_TYPE_MAPPING = {
-          "golang" => :go,
-          "maven" => :maven,
-          "npm" => :npm,
-          "cargo" => :cargo,
-          "composer" => :packagist,
-          "conda" => :conda,
-          "cran" => :cran,
-          "gem" => :rubygems,
-          "hackage" => :hackage,
-          "hex" => :hex,
-          "nuget" => :nuget,
-          "pypi" => :pypi,
-          "swift" => :swift_pm
-        }
-
         attr_reader :manifests
 
         def initialize(parse_queue:)
@@ -49,7 +29,7 @@ module Bibliothecary
         end
 
         def <<(purl)
-          mapping = PURL_TYPE_MAPPING[purl.type]
+          mapping = Bibliothecary::PURL_TYPE_MAPPING[purl.type]
           return unless mapping
 
           @manifests[mapping] ||= Set.new

data/lib/bibliothecary/multi_parsers/spdx.rb

@@ -0,0 +1,98 @@
+# packageurl-ruby uses pattern-matching (https://docs.ruby-lang.org/en/2.7.0/NEWS.html#label-Pattern+matching)
+# which warns a whole bunch in Ruby 2.7 as being an experimental feature, but has
+# been accepted in Ruby 3.0 (https://rubyreferences.github.io/rubychanges/3.0.html#pattern-matching).
+Warning[:experimental] = false
+require 'package_url'
+Warning[:experimental] = true
+
+module Bibliothecary
+  module MultiParsers
+    module Spdx
+      include Bibliothecary::Analyser
+      include Bibliothecary::Analyser::TryCache
+
+      # e.g. 'SomeText:' (allowing for leading whitespace)
+      WELLFORMED_LINE_REGEX = /^\s*[a-zA-Z]+:/
+
+      # e.g. 'PackageName: (allowing for excessive whitespace)
+      PACKAGE_NAME_REGEX = /^\s*PackageName:\s*(.*)/
+
+      # e.g. 'PackageVersion:' (allowing for excessive whitespace)
+      PACKAGE_VERSION_REGEX =/^\s*PackageVersion:\s*(.*)/
+
+      # e.g. "ExternalRef: PACKAGE-MANAGER purl (allowing for excessive whitespace)
+      PURL_REGEX = /^\s*ExternalRef:\s*PACKAGE[-|_]MANAGER\s*purl\s*(.*)/
+
+      NoEntries = Class.new(StandardError)
+      MalformedFile = Class.new(StandardError)
+
+      def self.mapping
+        {
+          match_extension('.spdx') => {
+            kind: 'lockfile',
+            parser: :parse_spdx_tag_value,
+            ungroupable: true
+          }
+        }
+      end
+
+      def parse_spdx_tag_value(file_contents, options: {})
+        entries = try_cache(options, options[:filename]) do
+          parse_spdx_tag_value_file_contents(file_contents)
+        end
+
+        raise NoEntries if entries.empty?
+
+        entries[platform_name.to_sym]
+      end
+
+      def get_platform(purl_string)
+        platform = PackageURL.parse(purl_string).type
+
+        Bibliothecary::PURL_TYPE_MAPPING[platform]
+      end
+
+      def parse_spdx_tag_value_file_contents(file_contents)
+        entries = {}
+
+        package_name = nil
+        package_version = nil
+        platform = nil
+
+        file_contents.split("\n").each do |line|
+          stripped_line = line.strip
+
+          next if skip_line?(stripped_line)
+
+          raise MalformedFile unless stripped_line.match(WELLFORMED_LINE_REGEX)
+
+          if (match = stripped_line.match(PACKAGE_NAME_REGEX))
+            package_name = match[1]
+          elsif (match = stripped_line.match(PACKAGE_VERSION_REGEX))
+            package_version = match[1]
+          elsif (match = stripped_line.match(PURL_REGEX))
+            platform ||= get_platform(match[1])
+          end
+
+          unless package_name.nil? || package_version.nil? || platform.nil?
+            entries[platform.to_sym] ||= []
+            entries[platform.to_sym] << {
+              name: package_name,
+              requirement: package_version,
+              type: 'lockfile'
+            }
+
+            package_name = package_version = platform = nil
+          end
+        end
+
+        entries
+      end
+
+      def skip_line?(stripped_line)
+        # Ignore blank lines and comments
+        stripped_line == "" || stripped_line[0] == "#"
+      end
+    end
+  end
+end

data/lib/bibliothecary/parsers/cargo.rb

@@ -17,6 +17,7 @@ module Bibliothecary
       end
 
       add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
+      add_multi_parser(Bibliothecary::MultiParsers::Spdx)
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
       def self.parse_manifest(file_contents, options: {})

data/lib/bibliothecary/parsers/conda.rb

@@ -28,6 +28,7 @@ module Bibliothecary
 
       add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
+      add_multi_parser(Bibliothecary::MultiParsers::Spdx)
 
       def self.parse_conda(file_contents, options: {})
         parse_conda_with_kind(file_contents, "manifest")

data/lib/bibliothecary/parsers/cran.rb

@@ -18,6 +18,7 @@ module Bibliothecary
 
       add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
+      add_multi_parser(Bibliothecary::MultiParsers::Spdx)
 
       def self.parse_description(file_contents, options: {})
         manifest = DebControl::ControlFileBase.parse(file_contents)

data/lib/bibliothecary/parsers/go.rb

@@ -66,6 +66,7 @@ module Bibliothecary
       end
 
       add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
+      add_multi_parser(Bibliothecary::MultiParsers::Spdx)
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
       def self.parse_godep_json(file_contents, options: {})

data/lib/bibliothecary/parsers/hackage.rb

@@ -21,6 +21,7 @@ module Bibliothecary
 
       add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
+      add_multi_parser(Bibliothecary::MultiParsers::Spdx)
 
       def self.parse_cabal(file_contents, options: {})
         headers = {

data/lib/bibliothecary/parsers/hex.rb

@@ -20,6 +20,7 @@ module Bibliothecary
 
       add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
+      add_multi_parser(Bibliothecary::MultiParsers::Spdx)
 
       def self.parse_mix(file_contents, options: {})
         response = Typhoeus.post("#{Bibliothecary.configuration.mix_parser_host}/", body: file_contents)

data/lib/bibliothecary/parsers/maven.rb

@@ -25,15 +25,15 @@ module Bibliothecary
       # Deprecated methods: https://docs.gradle.org/current/userguide/upgrading_version_6.html#sec:configuration_removal
       GRADLE_DEPENDENCY_METHODS = %w(api compile compileClasspath compileOnly compileOnlyApi implementation runtime runtimeClasspath runtimeOnly testCompile testCompileOnly testImplementation testRuntime testRuntimeOnly)
 
-      # Intentionally overly-simplified regexes to scrape deps from build.gradle (Groovy) and build.gradle.kts (Kotlin) files. 
-      # To be truly useful bibliothecary would need full Groovy / Kotlin parsers that speaks Gradle, 
+      # Intentionally overly-simplified regexes to scrape deps from build.gradle (Groovy) and build.gradle.kts (Kotlin) files.
+      # To be truly useful bibliothecary would need full Groovy / Kotlin parsers that speaks Gradle,
       # because the Groovy and Kotlin DSLs have many dynamic ways of declaring dependencies.
       GRADLE_VERSION_REGEX = /[\w.-]+/ # e.g. '1.2.3'
       GRADLE_VAR_INTERPOLATION_REGEX = /\$\w+/ # e.g. '$myVersion'
       GRADLE_CODE_INTERPOLATION_REGEX = /\$\{.*\}/ # e.g. '${my-project-settings["version"]}'
       GRADLE_GAV_REGEX = /([\w.-]+)\:([\w.-]+)(?:\:(#{GRADLE_VERSION_REGEX}|#{GRADLE_VAR_INTERPOLATION_REGEX}|#{GRADLE_CODE_INTERPOLATION_REGEX}))?/ # e.g. "group:artifactId:1.2.3"
-      GRADLE_GROOVY_SIMPLE_REGEX = /(#{GRADLE_DEPENDENCY_METHODS.join('|')})\s*\(?\s*['"]#{GRADLE_GAV_REGEX}['"]/m 
-      GRADLE_KOTLIN_SIMPLE_REGEX = /(#{GRADLE_DEPENDENCY_METHODS.join('|')})\s*\(\s*"#{GRADLE_GAV_REGEX}"/m 
+      GRADLE_GROOVY_SIMPLE_REGEX = /(#{GRADLE_DEPENDENCY_METHODS.join('|')})\s*\(?\s*['"]#{GRADLE_GAV_REGEX}['"]/m
+      GRADLE_KOTLIN_SIMPLE_REGEX = /(#{GRADLE_DEPENDENCY_METHODS.join('|')})\s*\(\s*"#{GRADLE_GAV_REGEX}"/m
 
       MAVEN_PROPERTY_REGEX = /\$\{(.+?)\}/
       MAX_DEPTH = 5
@@ -96,6 +96,7 @@ module Bibliothecary
       end
 
       add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
+      add_multi_parser(Bibliothecary::MultiParsers::Spdx)
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
       def self.parse_ivy_manifest(file_contents, options: {})
@@ -162,7 +163,7 @@ module Bibliothecary
           # so we treat these projects as "internal" deps with requirement of "1.0.0"
           if (project_match = line.match(GRADLE_PROJECT_REGEX))
             # an empty project name is self-referential (i.e. a cycle), and we don't need to track the manifest's project itself, e.g. "+--- project :"
-            next if project_match[1].nil? 
+            next if project_match[1].nil?
 
             # project names can have colons (e.g. for gradle projects in subfolders), which breaks maven artifact naming assumptions, so just replace them with hyphens.
             project_name = project_match[1].gsub(/:/, "-")
@@ -221,8 +222,11 @@ module Bibliothecary
       end
 
       def self.parse_maven_tree(file_contents, options: {})
-        file_contents = file_contents.gsub(/\r\n?/, "\n")
-        captures = file_contents.scan(/^\[INFO\](?:(?:\+-)|\||(?:\\-)|\s)+((?:[\w\.-]+:)+[\w\.\-${}]+)/).flatten.uniq
+        captures = Strings::ANSI.sanitize(file_contents)
+          .gsub(/\r\n?/, "\n")
+          .scan(/^\[INFO\](?:(?:\+-)|\||(?:\\-)|\s)+((?:[\w\.-]+:)+[\w\.\-${}]+)/)
+          .flatten
+          .uniq
 
         deps = captures.map do |item|
           parts = item.split(":")

data/lib/bibliothecary/parsers/npm.rb

@@ -34,6 +34,7 @@ module Bibliothecary
       end
 
       add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
+      add_multi_parser(Bibliothecary::MultiParsers::Spdx)
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
       def self.parse_package_lock(file_contents, options: {})
@@ -57,9 +58,9 @@ module Bibliothecary
       def self.parse_package_lock_v1(manifest)
         parse_package_lock_deps_recursively(manifest.fetch('dependencies', []))
       end
-      
+
       def self.parse_package_lock_v2(manifest)
-        # "packages" is a flat object where each key is the installed location of the dep, e.g. node_modules/foo/node_modules/bar. 
+        # "packages" is a flat object where each key is the installed location of the dep, e.g. node_modules/foo/node_modules/bar.
         manifest
           .fetch("packages")
           .reject { |name, dep| name == "" } # this is the lockfile's package itself
@@ -68,7 +69,7 @@ module Bibliothecary
               name: name.split("node_modules/").last,
               requirement: dep["version"],
               type: dep.fetch("dev", false) || dep.fetch("devOptional", false)  ? "development" : "runtime"
-            }  
+            }
           end
       end
 
@@ -94,7 +95,7 @@ module Bibliothecary
       def self.parse_manifest(file_contents, options: {})
         manifest = JSON.parse(file_contents)
         raise "appears to be a lockfile rather than manifest format" if manifest.key?('lockfileVersion')
-        
+
         (
           map_dependencies(manifest, 'dependencies', 'runtime') +
           map_dependencies(manifest, 'devDependencies', 'development')

data/lib/bibliothecary/parsers/nuget.rb

@@ -46,6 +46,7 @@ module Bibliothecary
 
       add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
+      add_multi_parser(Bibliothecary::MultiParsers::Spdx)
 
       def self.parse_project_lock_json(file_contents, options: {})
         manifest = JSON.parse file_contents

data/lib/bibliothecary/parsers/packagist.rb

@@ -20,6 +20,7 @@ module Bibliothecary
 
       add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
+      add_multi_parser(Bibliothecary::MultiParsers::Spdx)
 
       def self.parse_lockfile(file_contents, options: {})
         manifest = JSON.parse file_contents

data/lib/bibliothecary/parsers/pypi.rb

@@ -87,6 +87,7 @@ module Bibliothecary
 
       add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
+      add_multi_parser(Bibliothecary::MultiParsers::Spdx)
 
       def self.parse_pipfile(file_contents, options: {})
         manifest = Tomlrb.parse(file_contents)
@@ -94,10 +95,10 @@ module Bibliothecary
       end
 
       def self.parse_pyproject(file_contents, options: {})
-        deps = []  
+        deps = []
 
         file_contents = Tomlrb.parse(file_contents)
-        
+
         # Parse poetry [tool.poetry] deps
         poetry_manifest = file_contents.fetch('tool', {}).fetch('poetry', {})
         deps += map_dependencies(poetry_manifest['dependencies'], 'runtime')

data/lib/bibliothecary/parsers/rubygems.rb

@@ -32,6 +32,7 @@ module Bibliothecary
 
       add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
+      add_multi_parser(Bibliothecary::MultiParsers::Spdx)
 
       def self.parse_gemfile_lock(file_contents, options: {})
         file_contents.lines(chomp: true).map do |line|

data/lib/bibliothecary/parsers/swift_pm.rb

@@ -14,6 +14,7 @@ module Bibliothecary
 
       add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
+      add_multi_parser(Bibliothecary::MultiParsers::Spdx)
 
       def self.parse_package_swift(file_contents, options: {})
         response = Typhoeus.post("#{Bibliothecary.configuration.swift_parser_host}/to-json", body: file_contents)

data/lib/bibliothecary/purl_util.rb

@@ -0,0 +1,21 @@
+module Bibliothecary
+  # If a purl type (key) exists, it will be used in a manifest for
+  # the key's value. If not, it's ignored.
+  #
+  # https://github.com/package-url/purl-spec/blob/master/PURL-TYPES.rst
+  PURL_TYPE_MAPPING = {
+    "golang" => :go,
+    "maven" => :maven,
+    "npm" => :npm,
+    "cargo" => :cargo,
+    "composer" => :packagist,
+    "conda" => :conda,
+    "cran" => :cran,
+    "gem" => :rubygems,
+    "hackage" => :hackage,
+    "hex" => :hex,
+    "nuget" => :nuget,
+    "pypi" => :pypi,
+    "swift" => :swift_pm
+  }.freeze
+end

data/lib/bibliothecary/version.rb

@@ -1,3 +1,3 @@
 module Bibliothecary
-  VERSION = "8.6.3"
+  VERSION = "8.6.5"
 end

data/lib/bibliothecary.rb

@@ -5,6 +5,7 @@ require "bibliothecary/runner"
 require "bibliothecary/exceptions"
 require "bibliothecary/file_info"
 require "bibliothecary/related_files_info"
+require "bibliothecary/purl_util"
 require "find"
 require "tomlrb"
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bibliothecary
 version: !ruby/object:Gem::Version
-  version: 8.6.3
+  version: 8.6.5
 platform: ruby
 authors:
 - Andrew Nesbitt
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-05-30 00:00:00.000000000 Z
+date: 2023-08-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: tomlrb
@@ -248,7 +248,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: 
+description:
 email:
 - andrewnez@gmail.com
 executables:
@@ -290,6 +290,7 @@ files:
 - lib/bibliothecary/multi_parsers/cyclonedx.rb
 - lib/bibliothecary/multi_parsers/dependencies_csv.rb
 - lib/bibliothecary/multi_parsers/json_runtime.rb
+- lib/bibliothecary/multi_parsers/spdx.rb
 - lib/bibliothecary/parsers/bower.rb
 - lib/bibliothecary/parsers/cargo.rb
 - lib/bibliothecary/parsers/carthage.rb
@@ -315,6 +316,7 @@ files:
 - lib/bibliothecary/parsers/rubygems.rb
 - lib/bibliothecary/parsers/shard.rb
 - lib/bibliothecary/parsers/swift_pm.rb
+- lib/bibliothecary/purl_util.rb
 - lib/bibliothecary/related_files_info.rb
 - lib/bibliothecary/runner.rb
 - lib/bibliothecary/runner/multi_manifest_filter.rb
@@ -324,7 +326,7 @@ homepage: https://github.com/librariesio/bibliothecary
 licenses:
 - AGPL-3.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -340,7 +342,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.1.6
-signing_key: 
+signing_key:
 specification_version: 4
 summary: Find and parse manifests
 test_files: []