bibliothecary 10.2.0 → 10.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 593043fd2afa0c57546f1947d788d7d4882cc398b36ebcc5774cb89446ba07d1
-  data.tar.gz: fcf4b7b67a9b0d6de622d6bf4390bf20b64f6c015e13253ffad21257bd142f81
+  metadata.gz: 8b6fe7d18802ded7298cac3401086635c59397905fceb05aecb3bc777a3e9457
+  data.tar.gz: 532ff9bb7bcb77a56648165ade6dc1110f43cc06cd1c79945c3ff8ecdde02e6d
 SHA512:
-  metadata.gz: 9b6922980540f62a05e27a46c612937dc165c4815dab3ca8462ad137e1cb4905ac9a015a7a47777126336eefa5a50794027696d1be9340e88206b71885c6893a
-  data.tar.gz: b283d69564bbd69f8684cd4ac530551c42c5ab1c174da3edaa84f9ca4ee9750abc26cd1d6a977ccf69709fe52cb8d821d7875f6821c456ac5727b1a196dbd060
+  metadata.gz: 61ec2054d0905c6f3cb448603bbecc44679e3b07425b8c18c0d5985cc1135788c36525f7567577f7f9c767b5744cf2db1e58c8cd445f35f0894354aacdad2049
+  data.tar.gz: 19e8740cefc0cb5098579f933813bdd5912ba5ff716097c69fac603572b9484b04cd4d60710f4048002ed85b5b40a586be10b114ca3a652eb34bbf3fc1354e39

data/.ruby-version

@@ -1 +1 @@
-3.0.7
+3.0.7

data/CHANGELOG.md

@@ -13,11 +13,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Removed
 
+## [10.2.2] - 2024-09-25
+
+### Added
+
+- Support parsing *.spdx.json files
+
 ## [10.2.0] - 2024-08-27
 
 ### Changed
 
-- `Bibliothecary::Dependency#requirement` now defaults to all versions (`"*"`) instead of `nil` if no version range is specified for the dependency,
+- `Bibliothecary::Dependency#requirement` now defaults to all versions (`"*"`) instead of `nil` if no version range is specified for the dependency.
 
 ## [10.1.0] - 2024-07-23
 

data/Gemfile

@@ -1,8 +1,5 @@
 source "https://rubygems.org"
 
-# Temporarily pegging to HEAD until 0.2.1 is released: https://github.com/piotrmurach/strings-ansi/pull/2
-gem "strings-ansi", ref: "35d0c9430cf0a8022dc12bdab005bce296cb9f00", git: "git@github.com:piotrmurach/strings-ansi.git"
-
 # Specify your gem's dependencies in bibliothecary.gemspec
 gemspec
 

data/README.md

@@ -97,6 +97,11 @@ All available config options are in: https://github.com/librariesio/bibliothecar
   - JSON as cyclonedx.json
   - Note that CycloneDX manifests can contain information on multiple
     package manager's packages!
+- SPDX
+  - tag:value as *.spdx
+  - JSON as *.spdx.json
+  - Note that SPDX manifests can contain information on multiple
+    package manager's packages!
 - Bower
   - bower.json
 - CPAN

data/bibliothecary.gemspec

@@ -25,8 +25,6 @@ Gem::Specification.new do |spec|
   spec.add_dependency "deb_control"
   spec.add_dependency "sdl4r"
   spec.add_dependency "commander"
-  spec.add_dependency "strings-ansi" # NB this is also pegged to a git sha in Gemfile temporarily.  
-  spec.add_dependency "strings"
   spec.add_dependency "packageurl-ruby"
 
   spec.add_development_dependency "pry"

data/lib/bibliothecary/multi_parsers/cyclonedx.rb

@@ -29,12 +29,12 @@ module Bibliothecary
         end
 
         def <<(purl)
-          mapping = Bibliothecary::PURL_TYPE_MAPPING[purl.type]
+          mapping = PurlUtil::PURL_TYPE_MAPPING[purl.type]
           return unless mapping
 
           @manifests[mapping] ||= Set.new
           @manifests[mapping] <<  Dependency.new(
-            name: self.class.full_name_for_purl(purl),
+            name: PurlUtil.full_name(purl),
             requirement: purl.version,
             type: "lockfile",
           )
@@ -60,18 +60,6 @@ module Bibliothecary
         def [](key)
           @manifests[key]&.to_a
         end
-
-        # @return [String] The properly namespaced package name
-        def self.full_name_for_purl(purl)
-          parts = [purl.namespace, purl.name].compact
-
-          case purl.type
-          when "maven"
-            parts.join(":")
-          else
-            parts.join("/")
-          end
-        end
       end
 
       def self.mapping

data/lib/bibliothecary/multi_parsers/spdx.rb

@@ -18,7 +18,7 @@ module Bibliothecary
       PACKAGE_NAME_REGEXP = /^\s*PackageName:\s*(.*)/
 
       # e.g. 'PackageVersion:' (allowing for excessive whitespace)
-      PACKAGE_VERSION_REGEXP =/^\s*PackageVersion:\s*(.*)/
+      PACKAGE_VERSION_REGEXP = /^\s*PackageVersion:\s*(.*)/
 
       # e.g. "ExternalRef: PACKAGE-MANAGER purl (allowing for excessive whitespace)
       PURL_REGEXP = /^\s*ExternalRef:\s*PACKAGE[-|_]MANAGER\s*purl\s*(.*)/
@@ -33,6 +33,11 @@ module Bibliothecary
             parser: :parse_spdx_tag_value,
             ungroupable: true,
           },
+          match_extension(".spdx.json") => {
+            kind: "lockfile",
+            parser: :parse_spdx_json,
+            ungroupable: true,
+          },
         }
       end
 
@@ -46,53 +51,93 @@ module Bibliothecary
         entries[platform_name.to_sym]
       end
 
-      def get_platform(purl_string)
-        platform = PackageURL.parse(purl_string).type
-
-        Bibliothecary::PURL_TYPE_MAPPING[platform]
-      end
-
       def parse_spdx_tag_value_file_contents(file_contents)
         entries = {}
+        spdx_name = spdx_version = platform = purl_name = purl_version = nil
 
-        package_name = nil
-        package_version = nil
-        platform = nil
-
-        file_contents.split("\n").each do |line|
+        file_contents.each_line do |line|
           stripped_line = line.strip
+          next if skip_tag_value_line?(stripped_line)
 
-          next if skip_line?(stripped_line)
-
-          raise MalformedFile unless stripped_line.match(WELLFORMED_LINE_REGEXP)
+          raise MalformedFile unless stripped_line.match?(WELLFORMED_LINE_REGEXP)
 
           if (match = stripped_line.match(PACKAGE_NAME_REGEXP))
-            package_name = match[1]
+            # Per the spec:
+            # > A new package Information section is denoted by the package name (7.1) field.
+            add_entry(entries: entries, platform: platform, purl_name: purl_name,
+                      spdx_name: spdx_name, purl_version: purl_version, spdx_version: spdx_version)
+
+            # reset for this new package
+            spdx_name = spdx_version = platform = purl_name = purl_version = nil
+
+            # capture the new package's name
+            spdx_package_name = match[1]
           elsif (match = stripped_line.match(PACKAGE_VERSION_REGEXP))
-            package_version = match[1]
+            spdx_version = match[1]
           elsif (match = stripped_line.match(PURL_REGEXP))
-            platform ||= get_platform(match[1])
+            purl = PackageURL.parse(match[1])
+            platform ||= purl.type
+            purl_name ||= PurlUtil.full_name(purl)
+            purl_version ||= purl.version
           end
+        end
 
-          unless package_name.nil? || package_version.nil? || platform.nil?
-            entries[platform.to_sym] ||= []
-            entries[platform.to_sym] << Dependency.new(
-              name: package_name,
-              requirement: package_version,
-              type: "lockfile",
-            )
+        add_entry(entries: entries, platform: platform, purl_name: purl_name,
+                  spdx_name: spdx_name, purl_version: purl_version, spdx_version: spdx_version)
 
-            package_name = package_version = platform = nil
-          end
+        entries
+      end
+
+      def skip_tag_value_line?(stripped_line)
+        # Ignore blank lines and comments
+        stripped_line.empty? || stripped_line.start_with?("#")
+      end
+
+      def parse_spdx_json(file_contents, options: {})
+        entries = try_cache(options, options[:filename]) do
+          parse_spdx_json_file_contents(file_contents)
+        end
+
+        raise NoEntries if entries.empty?
+
+        entries[platform_name.to_sym]
+      end
+
+      def parse_spdx_json_file_contents(file_contents)
+        entries = {}
+        manifest = JSON.parse(file_contents)
+
+        manifest["packages"]&.each do |package|
+          spdx_name = package["name"]
+          spdx_version = package["versionInfo"]
+
+          first_purl_string = package.dig("externalRefs")&.find { |ref| ref["referenceType"] == "purl" }&.dig("referenceLocator")
+          purl = first_purl_string && PackageURL.parse(first_purl_string)
+          platform = purl&.type
+          purl_name = PurlUtil.full_name(purl)
+          purl_version = purl&.version
+
+          add_entry(entries: entries, platform: platform, purl_name: purl_name,
+                    spdx_name: spdx_name, purl_version: purl_version, spdx_version: spdx_version)
         end
 
         entries
       end
 
-      def skip_line?(stripped_line)
-        # Ignore blank lines and comments
-        stripped_line == "" || stripped_line[0] == "#"
+      def add_entry(entries:, platform:, purl_name:, spdx_name:, purl_version:, spdx_version:)
+        package_name = purl_name || spdx_name
+        package_version = purl_version || spdx_version
+
+        if platform && package_name && package_version
+          entries[platform.to_sym] ||= []
+          entries[platform.to_sym] << Dependency.new(
+            name: package_name,
+            requirement: package_version,
+            type: "lockfile"
+          )
+        end
       end
+
     end
   end
 end

data/lib/bibliothecary/parsers/maven.rb

@@ -1,5 +1,4 @@
 require "ox"
-require "strings-ansi"
 
 # Known shortcomings and unimplemented Maven features:
 #   pom.xml
@@ -57,6 +56,27 @@ module Bibliothecary
       # e.g. "[info]  "
       SBT_IGNORE_REGEXP = /^\[info\]\s*$/
 
+
+      # Copied from the "strings-ansi" gem, because it seems abandoned: https://github.com/piotrmurach/strings-ansi/pull/2
+      # From: https://github.com/piotrmurach/strings-ansi/blob/35d0c9430cf0a8022dc12bdab005bce296cb9f00/lib/strings/ansi.rb#L14-L29
+      # License: MIT
+      # The regex to match ANSI codes
+      ANSI_MATCHER = %r{
+        (?>\033(
+          \[[\[?>!]?\d*(;\d+)*[ ]?[a-zA-Z~@$^\]_\{\\] # graphics
+          |
+          \#?\d # cursor modes
+          |
+          [)(%+\-*/. ](\d|[a-zA-Z@=%]|) # character sets
+          |
+          O[p-xA-Z] # special keys
+          |
+          [a-zA-Z=><~\}|] # cursor movement
+          |
+          \]8;[^;]*;.*?(\033\\|\07) # hyperlink
+        ))
+      }x.freeze
+      
       def self.mapping
         {
           match_filename("ivy.xml", case_insensitive: true) => {
@@ -218,7 +238,8 @@ module Bibliothecary
       end
 
       def self.parse_maven_resolved(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
-        Strings::ANSI.sanitize(file_contents)
+        file_contents
+          .gsub(ANSI_MATCHER, "")
           .split("\n")
           .map(&method(:parse_resolved_dep_line))
           .compact
@@ -226,7 +247,8 @@ module Bibliothecary
       end
 
       def self.parse_maven_tree(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
-        captures = Strings::ANSI.sanitize(file_contents)
+        captures = file_contents
+          .gsub(ANSI_MATCHER, "")
           .gsub(/\r\n?/, "\n")
           .scan(/^\[INFO\](?:(?:\+-)|\||(?:\\-)|\s)+((?:[\w\.-]+:)+[\w\.\-${}]+)/)
           .flatten

data/lib/bibliothecary/purl_util.rb

@@ -1,21 +1,39 @@
 module Bibliothecary
-  # If a purl type (key) exists, it will be used in a manifest for
-  # the key's value. If not, it's ignored.
-  #
-  # https://github.com/package-url/purl-spec/blob/master/PURL-TYPES.rst
-  PURL_TYPE_MAPPING = {
-    "golang" => :go,
-    "maven" => :maven,
-    "npm" => :npm,
-    "cargo" => :cargo,
-    "composer" => :packagist,
-    "conda" => :conda,
-    "cran" => :cran,
-    "gem" => :rubygems,
-    "hackage" => :hackage,
-    "hex" => :hex,
-    "nuget" => :nuget,
-    "pypi" => :pypi,
-    "swift" => :swift_pm,
-  }.freeze
+  class PurlUtil
+    # If a purl type (key) exists, it will be used in a manifest for
+    # the key's value. If not, it's ignored.
+    #
+    # https://github.com/package-url/purl-spec/blob/master/PURL-TYPES.rst
+    PURL_TYPE_MAPPING = {
+      "golang" => :go,
+      "maven" => :maven,
+      "npm" => :npm,
+      "cargo" => :cargo,
+      "composer" => :packagist,
+      "conda" => :conda,
+      "cran" => :cran,
+      "gem" => :rubygems,
+      "hackage" => :hackage,
+      "hex" => :hex,
+      "nuget" => :nuget,
+      "pypi" => :pypi,
+      "swift" => :swift_pm,
+    }.freeze
+
+
+    # @param purl [PackageURL]
+    # @return [String] The properly namespaced package name
+    def self.full_name(purl)
+      return nil if purl.nil?
+
+      parts = [purl.namespace, purl.name].compact
+
+      case purl.type
+      when "maven"
+        parts.join(":")
+      else
+        parts.join("/")
+      end
+    end
+  end
 end

data/lib/bibliothecary/version.rb

@@ -1,3 +1,3 @@
 module Bibliothecary
-  VERSION = "10.2.0"
+  VERSION = "10.2.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bibliothecary
 version: !ruby/object:Gem::Version
-  version: 10.2.0
+  version: 10.2.2
 platform: ruby
 authors:
 - Andrew Nesbitt
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-08-27 00:00:00.000000000 Z
+date: 2024-09-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: tomlrb
@@ -108,34 +108,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: strings-ansi
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: strings
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: packageurl-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -266,7 +238,6 @@ files:
 - ".rubocop.yml"
 - ".ruby-version"
 - ".tidelift.yml"
-- ".travis.yml"
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
@@ -277,7 +248,6 @@ files:
 - bin/bibliothecary
 - bin/console
 - bin/setup
-- dependencyci.yml
 - lib/bibliothecary.rb
 - lib/bibliothecary/analyser.rb
 - lib/bibliothecary/analyser/analysis.rb

data/.travis.yml

@@ -1,12 +0,0 @@
-language: ruby
-rvm:
-- 2.6.6
-cache: bundler
-before_install:
-- gem update --system
-- gem install bundler
-script:
-- bundle exec rake spec && bundle exec codeclimate-test-reporter
-notifications:
-  slack:
-    secure: MkuDOE57uFrypxJGB7fy6Mq7uT77SzC3ApXVbdpx8k+4ihwLK+7Gyn0IzJ0f24B9PprH2RzmFoHk6XPjSjVCAEoCHvuv2ntHPX3FqxkBXdkb3NjKjOvV1+rRt9D+3sG6IEY5M5ak7j34W0FGgczNQs3+IOGCs3NIJP/h2mAL02w8oZFU2LMGYY6gX7LL+z4q65Ag5mDMwN9kslmWELO7k8xLPunlCh9jRWbZpxzFKwsHC4HtygejWwijNlSAt6Rk2XaPJ4jR8PO8uvWR1+iXxOVFErZ/nriybYMdThpLUdLWWHn6n+a3lu9vpo0KtnMrVq/E5KSaM1bilKxTuFR35AU9kbMv2L9cs65sRz1rSa2CnfN+788icesDyePh6ZjDOb+5zvxOViDvoEc9cIogf6JJT8hDpvF0zDqEuU+CTvh+3AqRGS7yjlsviQZofB8Fr/VMjZzuHL45T1+4O46eryO1PZpNYlq9svLRgpkdmu5A+SFwlM2K7Zc4ND5GSlCnWdGx1ThBp1u1lFYvw2IMeU5bylS+ak4xt+S2YH5Hmc0y7F0nvB54mmap8T7GGc7dFhLaLTaTTijHF70HJgsDKCnPsRqk5Bt2h7grBMT6t9AG+6Hg0QVFJ4ZSfnM2IBb57naVhUpMFjYT9fnLrjIAg9rY1GW6kkowFdoj6mvFhAM=

data/dependencyci.yml

@@ -1,9 +0,0 @@
-platforms:
-  NPM:
-    jade:
-      tests:
-        deprecated: skip
-  NuGet:
-    tests:
-      unlicensed: skip
-      removed: skip