bibliothecary 8.6.4 → 8.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 89c5d8cb9fca421230fa967e5fbc3ce68d9178b0c065cb0c20737464739fab4e
-  data.tar.gz: 7ae9d50b5a82ae46f2c473eee0b483f2376674c97dd94ff8b1c70690973dd3d5
+  metadata.gz: 39af4653f19e0376f945656791b9dc2eb6e60f5d519a9a0e947e4623827376d2
+  data.tar.gz: c2a6b01a1d14abffde071d07a95a28e851e85c61a89fe3ebb5239c82f2cef3fd
 SHA512:
-  metadata.gz: 29b2d8750cb9611c3a7cef9946fbaf12e8eb6d73d95ecbd604b4ac41c4428a07361f8d67db8ccece341d1f5c59c2b70510695f947bc060ae7a44f3e395126589
-  data.tar.gz: 418b9879a3028dc396e996972c49be58fb1a63a3ec707e035d7b9c9cccdbdcefe919ff741a37d71a719cffc15145b4f362534ed83e72e8debbff2e73c4ffdee8
+  metadata.gz: 5e923a36a18760a6f3acdc05f7330de7704c42f235fd1bbee880dc7d7ea3bf1604ea8ecd060f6984db7a32ac6b552d10cd19d8b6257153fc7c19f55430234661
+  data.tar.gz: a65aed02e5de4338a4c48ce44f77b2119fee0a26be51c7bac1bd18cdf8a51ceebed0305e16b965483cf4d2452ba63f3557de93acd29b474428245d62bceac707

data/lib/bibliothecary/multi_parsers/cyclonedx.rb

@@ -17,26 +17,6 @@ module Bibliothecary
       NoComponents = Class.new(StandardError)
 
       class ManifestEntries
-        # If a purl type (key) exists, it will be used in a manifest for
-        # the key's value. If not, it's ignored.
-        #
-        # https://github.com/package-url/purl-spec/blob/master/PURL-TYPES.rst
-        PURL_TYPE_MAPPING = {
-          "golang" => :go,
-          "maven" => :maven,
-          "npm" => :npm,
-          "cargo" => :cargo,
-          "composer" => :packagist,
-          "conda" => :conda,
-          "cran" => :cran,
-          "gem" => :rubygems,
-          "hackage" => :hackage,
-          "hex" => :hex,
-          "nuget" => :nuget,
-          "pypi" => :pypi,
-          "swift" => :swift_pm
-        }
-
         attr_reader :manifests
 
         def initialize(parse_queue:)
@@ -49,7 +29,7 @@ module Bibliothecary
         end
 
         def <<(purl)
-          mapping = PURL_TYPE_MAPPING[purl.type]
+          mapping = Bibliothecary::PURL_TYPE_MAPPING[purl.type]
           return unless mapping
 
           @manifests[mapping] ||= Set.new

data/lib/bibliothecary/multi_parsers/spdx.rb

@@ -0,0 +1,98 @@
+# packageurl-ruby uses pattern-matching (https://docs.ruby-lang.org/en/2.7.0/NEWS.html#label-Pattern+matching)
+# which warns a whole bunch in Ruby 2.7 as being an experimental feature, but has
+# been accepted in Ruby 3.0 (https://rubyreferences.github.io/rubychanges/3.0.html#pattern-matching).
+Warning[:experimental] = false
+require 'package_url'
+Warning[:experimental] = true
+
+module Bibliothecary
+  module MultiParsers
+    module Spdx
+      include Bibliothecary::Analyser
+      include Bibliothecary::Analyser::TryCache
+
+      # e.g. 'SomeText:' (allowing for leading whitespace)
+      WELLFORMED_LINE_REGEX = /^\s*[a-zA-Z]+:/
+
+      # e.g. 'PackageName: (allowing for excessive whitespace)
+      PACKAGE_NAME_REGEX = /^\s*PackageName:\s*(.*)/
+
+      # e.g. 'PackageVersion:' (allowing for excessive whitespace)
+      PACKAGE_VERSION_REGEX =/^\s*PackageVersion:\s*(.*)/
+
+      # e.g. "ExternalRef: PACKAGE-MANAGER purl (allowing for excessive whitespace)
+      PURL_REGEX = /^\s*ExternalRef:\s*PACKAGE[-|_]MANAGER\s*purl\s*(.*)/
+
+      NoEntries = Class.new(StandardError)
+      MalformedFile = Class.new(StandardError)
+
+      def self.mapping
+        {
+          match_extension('.spdx') => {
+            kind: 'lockfile',
+            parser: :parse_spdx_tag_value,
+            ungroupable: true
+          }
+        }
+      end
+
+      def parse_spdx_tag_value(file_contents, options: {})
+        entries = try_cache(options, options[:filename]) do
+          parse_spdx_tag_value_file_contents(file_contents)
+        end
+
+        raise NoEntries if entries.empty?
+
+        entries[platform_name.to_sym]
+      end
+
+      def get_platform(purl_string)
+        platform = PackageURL.parse(purl_string).type
+
+        Bibliothecary::PURL_TYPE_MAPPING[platform]
+      end
+
+      def parse_spdx_tag_value_file_contents(file_contents)
+        entries = {}
+
+        package_name = nil
+        package_version = nil
+        platform = nil
+
+        file_contents.split("\n").each do |line|
+          stripped_line = line.strip
+
+          next if skip_line?(stripped_line)
+
+          raise MalformedFile unless stripped_line.match(WELLFORMED_LINE_REGEX)
+
+          if (match = stripped_line.match(PACKAGE_NAME_REGEX))
+            package_name = match[1]
+          elsif (match = stripped_line.match(PACKAGE_VERSION_REGEX))
+            package_version = match[1]
+          elsif (match = stripped_line.match(PURL_REGEX))
+            platform ||= get_platform(match[1])
+          end
+
+          unless package_name.nil? || package_version.nil? || platform.nil?
+            entries[platform.to_sym] ||= []
+            entries[platform.to_sym] << {
+              name: package_name,
+              requirement: package_version,
+              type: 'lockfile'
+            }
+
+            package_name = package_version = platform = nil
+          end
+        end
+
+        entries
+      end
+
+      def skip_line?(stripped_line)
+        # Ignore blank lines and comments
+        stripped_line == "" || stripped_line[0] == "#"
+      end
+    end
+  end
+end

data/lib/bibliothecary/parsers/cargo.rb

@@ -17,6 +17,7 @@ module Bibliothecary
       end
 
       add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
+      add_multi_parser(Bibliothecary::MultiParsers::Spdx)
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
       def self.parse_manifest(file_contents, options: {})

data/lib/bibliothecary/parsers/conda.rb

@@ -28,6 +28,7 @@ module Bibliothecary
 
       add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
+      add_multi_parser(Bibliothecary::MultiParsers::Spdx)
 
       def self.parse_conda(file_contents, options: {})
         parse_conda_with_kind(file_contents, "manifest")

data/lib/bibliothecary/parsers/cran.rb

@@ -18,6 +18,7 @@ module Bibliothecary
 
       add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
+      add_multi_parser(Bibliothecary::MultiParsers::Spdx)
 
       def self.parse_description(file_contents, options: {})
         manifest = DebControl::ControlFileBase.parse(file_contents)

data/lib/bibliothecary/parsers/go.rb

@@ -66,6 +66,7 @@ module Bibliothecary
       end
 
       add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
+      add_multi_parser(Bibliothecary::MultiParsers::Spdx)
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
       def self.parse_godep_json(file_contents, options: {})

data/lib/bibliothecary/parsers/hackage.rb

@@ -21,6 +21,7 @@ module Bibliothecary
 
       add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
+      add_multi_parser(Bibliothecary::MultiParsers::Spdx)
 
       def self.parse_cabal(file_contents, options: {})
         headers = {

data/lib/bibliothecary/parsers/hex.rb

@@ -20,6 +20,7 @@ module Bibliothecary
 
       add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
+      add_multi_parser(Bibliothecary::MultiParsers::Spdx)
 
       def self.parse_mix(file_contents, options: {})
         response = Typhoeus.post("#{Bibliothecary.configuration.mix_parser_host}/", body: file_contents)

data/lib/bibliothecary/parsers/maven.rb

@@ -1,6 +1,10 @@
 require 'ox'
 require 'strings-ansi'
 
+# Known shortcomings and unimplemented Maven features:
+#   pom.xml
+#     <exclusions> cannot be taken into account (because it requires knowledge of transitive deps)
+#     <properties> are the only thing inherited from parent poms currenly
 module Bibliothecary
   module Parsers
     class Maven
@@ -25,15 +29,15 @@ module Bibliothecary
       # Deprecated methods: https://docs.gradle.org/current/userguide/upgrading_version_6.html#sec:configuration_removal
       GRADLE_DEPENDENCY_METHODS = %w(api compile compileClasspath compileOnly compileOnlyApi implementation runtime runtimeClasspath runtimeOnly testCompile testCompileOnly testImplementation testRuntime testRuntimeOnly)
 
-      # Intentionally overly-simplified regexes to scrape deps from build.gradle (Groovy) and build.gradle.kts (Kotlin) files. 
-      # To be truly useful bibliothecary would need full Groovy / Kotlin parsers that speaks Gradle, 
+      # Intentionally overly-simplified regexes to scrape deps from build.gradle (Groovy) and build.gradle.kts (Kotlin) files.
+      # To be truly useful bibliothecary would need full Groovy / Kotlin parsers that speaks Gradle,
       # because the Groovy and Kotlin DSLs have many dynamic ways of declaring dependencies.
       GRADLE_VERSION_REGEX = /[\w.-]+/ # e.g. '1.2.3'
       GRADLE_VAR_INTERPOLATION_REGEX = /\$\w+/ # e.g. '$myVersion'
       GRADLE_CODE_INTERPOLATION_REGEX = /\$\{.*\}/ # e.g. '${my-project-settings["version"]}'
       GRADLE_GAV_REGEX = /([\w.-]+)\:([\w.-]+)(?:\:(#{GRADLE_VERSION_REGEX}|#{GRADLE_VAR_INTERPOLATION_REGEX}|#{GRADLE_CODE_INTERPOLATION_REGEX}))?/ # e.g. "group:artifactId:1.2.3"
-      GRADLE_GROOVY_SIMPLE_REGEX = /(#{GRADLE_DEPENDENCY_METHODS.join('|')})\s*\(?\s*['"]#{GRADLE_GAV_REGEX}['"]/m 
-      GRADLE_KOTLIN_SIMPLE_REGEX = /(#{GRADLE_DEPENDENCY_METHODS.join('|')})\s*\(\s*"#{GRADLE_GAV_REGEX}"/m 
+      GRADLE_GROOVY_SIMPLE_REGEX = /(#{GRADLE_DEPENDENCY_METHODS.join('|')})\s*\(?\s*['"]#{GRADLE_GAV_REGEX}['"]/m
+      GRADLE_KOTLIN_SIMPLE_REGEX = /(#{GRADLE_DEPENDENCY_METHODS.join('|')})\s*\(\s*"#{GRADLE_GAV_REGEX}"/m
 
       MAVEN_PROPERTY_REGEX = /\$\{(.+?)\}/
       MAX_DEPTH = 5
@@ -96,6 +100,7 @@ module Bibliothecary
       end
 
       add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
+      add_multi_parser(Bibliothecary::MultiParsers::Spdx)
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
       def self.parse_ivy_manifest(file_contents, options: {})
@@ -162,7 +167,7 @@ module Bibliothecary
           # so we treat these projects as "internal" deps with requirement of "1.0.0"
           if (project_match = line.match(GRADLE_PROJECT_REGEX))
             # an empty project name is self-referential (i.e. a cycle), and we don't need to track the manifest's project itself, e.g. "+--- project :"
-            next if project_match[1].nil? 
+            next if project_match[1].nil?
 
             # project names can have colons (e.g. for gradle projects in subfolders), which breaks maven artifact naming assumptions, so just replace them with hyphens.
             project_name = project_match[1].gsub(/:/, "-")
@@ -269,18 +274,40 @@ module Bibliothecary
         manifest = Ox.parse file_contents
         xml = manifest.respond_to?('project') ? manifest.project : manifest
         [].tap do |deps|
-          ['dependencies/dependency', 'dependencyManagement/dependencies/dependency'].each do |deps_xpath|
-            xml.locate(deps_xpath).each do |dep|
-              dep_hash = {
-                name: "#{extract_pom_dep_info(xml, dep, 'groupId', parent_properties)}:#{extract_pom_dep_info(xml, dep, 'artifactId', parent_properties)}",
-                requirement: extract_pom_dep_info(xml, dep, 'version', parent_properties),
-                type: extract_pom_dep_info(xml, dep, 'scope', parent_properties) || 'runtime',
-              }
-              # optional field is, itself, optional, and will be either "true" or "false"
-              optional = extract_pom_dep_info(xml, dep, 'optional', parent_properties)
-              dep_hash[:optional] = optional == "true" unless optional.nil?
-              deps.push(dep_hash)
+          # <dependencyManagement> is a namespace to specify artifact configuration (e.g. version), but it doesn't
+          # actually add dependencies to your project. Grab these and keep them for reference while parsing <dependencies>
+          # Ref: https://maven.apache.org/pom.html#Dependency_Management
+          # Ref: https://maven.apache.org/guides/introduction/introduction-to-dependency-mechanism.html#transitive-dependencies
+          dependencyManagement = xml.locate("dependencyManagement/dependencies/dependency").map do |dep|
+            {
+              groupId: extract_pom_dep_info(xml, dep, "groupId", parent_properties),
+              artifactId: extract_pom_dep_info(xml, dep, "artifactId", parent_properties),
+              version: extract_pom_dep_info(xml, dep, "version", parent_properties),
+              scope: extract_pom_dep_info(xml, dep, "scope", parent_properties),
+            }
+          end
+          # <dependencies> is the namespace that will add dependencies to your project.
+          xml.locate("dependencies/dependency").each do |dep|
+            groupId = extract_pom_dep_info(xml, dep, 'groupId', parent_properties)
+            artifactId = extract_pom_dep_info(xml, dep, 'artifactId', parent_properties)
+            version = extract_pom_dep_info(xml, dep, 'version', parent_properties)
+            scope = extract_pom_dep_info(xml, dep, 'scope', parent_properties)
+
+            # Use any dep configurations from <dependencyManagement> as fallbacks
+            if (depConfig = dependencyManagement.find { |d| d[:groupId] == groupId && d[:artifactId] == artifactId })
+              version ||= depConfig[:version]
+              scope ||= depConfig[:scope]
             end
+
+            dep_hash = {
+              name: "#{groupId}:#{artifactId}",
+              requirement: version,
+              type: scope || 'runtime',
+            }
+            # optional field is, itself, optional, and will be either "true" or "false"
+            optional = extract_pom_dep_info(xml, dep, 'optional', parent_properties)
+            dep_hash[:optional] = optional == "true" unless optional.nil?
+            deps.push(dep_hash)
           end
         end
       end

data/lib/bibliothecary/parsers/npm.rb

@@ -34,6 +34,7 @@ module Bibliothecary
       end
 
       add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
+      add_multi_parser(Bibliothecary::MultiParsers::Spdx)
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
       def self.parse_package_lock(file_contents, options: {})
@@ -57,9 +58,9 @@ module Bibliothecary
       def self.parse_package_lock_v1(manifest)
         parse_package_lock_deps_recursively(manifest.fetch('dependencies', []))
       end
-      
+
       def self.parse_package_lock_v2(manifest)
-        # "packages" is a flat object where each key is the installed location of the dep, e.g. node_modules/foo/node_modules/bar. 
+        # "packages" is a flat object where each key is the installed location of the dep, e.g. node_modules/foo/node_modules/bar.
         manifest
           .fetch("packages")
           .reject { |name, dep| name == "" } # this is the lockfile's package itself
@@ -68,7 +69,7 @@ module Bibliothecary
               name: name.split("node_modules/").last,
               requirement: dep["version"],
               type: dep.fetch("dev", false) || dep.fetch("devOptional", false)  ? "development" : "runtime"
-            }  
+            }
           end
       end
 
@@ -94,7 +95,7 @@ module Bibliothecary
       def self.parse_manifest(file_contents, options: {})
         manifest = JSON.parse(file_contents)
         raise "appears to be a lockfile rather than manifest format" if manifest.key?('lockfileVersion')
-        
+
         (
           map_dependencies(manifest, 'dependencies', 'runtime') +
           map_dependencies(manifest, 'devDependencies', 'development')

data/lib/bibliothecary/parsers/nuget.rb

@@ -46,6 +46,7 @@ module Bibliothecary
 
       add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
+      add_multi_parser(Bibliothecary::MultiParsers::Spdx)
 
       def self.parse_project_lock_json(file_contents, options: {})
         manifest = JSON.parse file_contents

data/lib/bibliothecary/parsers/packagist.rb

@@ -20,6 +20,7 @@ module Bibliothecary
 
       add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
+      add_multi_parser(Bibliothecary::MultiParsers::Spdx)
 
       def self.parse_lockfile(file_contents, options: {})
         manifest = JSON.parse file_contents

data/lib/bibliothecary/parsers/pypi.rb

@@ -87,6 +87,7 @@ module Bibliothecary
 
       add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
+      add_multi_parser(Bibliothecary::MultiParsers::Spdx)
 
       def self.parse_pipfile(file_contents, options: {})
         manifest = Tomlrb.parse(file_contents)
@@ -94,10 +95,10 @@ module Bibliothecary
       end
 
       def self.parse_pyproject(file_contents, options: {})
-        deps = []  
+        deps = []
 
         file_contents = Tomlrb.parse(file_contents)
-        
+
         # Parse poetry [tool.poetry] deps
         poetry_manifest = file_contents.fetch('tool', {}).fetch('poetry', {})
         deps += map_dependencies(poetry_manifest['dependencies'], 'runtime')

data/lib/bibliothecary/parsers/rubygems.rb

@@ -32,6 +32,7 @@ module Bibliothecary
 
       add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
+      add_multi_parser(Bibliothecary::MultiParsers::Spdx)
 
       def self.parse_gemfile_lock(file_contents, options: {})
         file_contents.lines(chomp: true).map do |line|

data/lib/bibliothecary/parsers/swift_pm.rb

@@ -14,6 +14,7 @@ module Bibliothecary
 
       add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
+      add_multi_parser(Bibliothecary::MultiParsers::Spdx)
 
       def self.parse_package_swift(file_contents, options: {})
         response = Typhoeus.post("#{Bibliothecary.configuration.swift_parser_host}/to-json", body: file_contents)

data/lib/bibliothecary/purl_util.rb

@@ -0,0 +1,21 @@
+module Bibliothecary
+  # If a purl type (key) exists, it will be used in a manifest for
+  # the key's value. If not, it's ignored.
+  #
+  # https://github.com/package-url/purl-spec/blob/master/PURL-TYPES.rst
+  PURL_TYPE_MAPPING = {
+    "golang" => :go,
+    "maven" => :maven,
+    "npm" => :npm,
+    "cargo" => :cargo,
+    "composer" => :packagist,
+    "conda" => :conda,
+    "cran" => :cran,
+    "gem" => :rubygems,
+    "hackage" => :hackage,
+    "hex" => :hex,
+    "nuget" => :nuget,
+    "pypi" => :pypi,
+    "swift" => :swift_pm
+  }.freeze
+end

data/lib/bibliothecary/version.rb

@@ -1,3 +1,3 @@
 module Bibliothecary
-  VERSION = "8.6.4"
+  VERSION = "8.7.0"
 end

data/lib/bibliothecary.rb

@@ -5,6 +5,7 @@ require "bibliothecary/runner"
 require "bibliothecary/exceptions"
 require "bibliothecary/file_info"
 require "bibliothecary/related_files_info"
+require "bibliothecary/purl_util"
 require "find"
 require "tomlrb"
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bibliothecary
 version: !ruby/object:Gem::Version
-  version: 8.6.4
+  version: 8.7.0
 platform: ruby
 authors:
 - Andrew Nesbitt
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-07-02 00:00:00.000000000 Z
+date: 2023-09-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: tomlrb
@@ -290,6 +290,7 @@ files:
 - lib/bibliothecary/multi_parsers/cyclonedx.rb
 - lib/bibliothecary/multi_parsers/dependencies_csv.rb
 - lib/bibliothecary/multi_parsers/json_runtime.rb
+- lib/bibliothecary/multi_parsers/spdx.rb
 - lib/bibliothecary/parsers/bower.rb
 - lib/bibliothecary/parsers/cargo.rb
 - lib/bibliothecary/parsers/carthage.rb
@@ -315,6 +316,7 @@ files:
 - lib/bibliothecary/parsers/rubygems.rb
 - lib/bibliothecary/parsers/shard.rb
 - lib/bibliothecary/parsers/swift_pm.rb
+- lib/bibliothecary/purl_util.rb
 - lib/bibliothecary/related_files_info.rb
 - lib/bibliothecary/runner.rb
 - lib/bibliothecary/runner/multi_manifest_filter.rb