spandx 0.12.3 → 0.13.4

data/lib/spandx/dotnet/project_file.rb

@@ -6,9 +6,9 @@ module Spandx
       attr_reader :catalogue, :document, :nuget
 
       def initialize(path)
-        @path = path
-        @dir = File.dirname(path)
-        @document = Nokogiri::XML(IO.read(path)).tap(&:remove_namespaces!)
+        @path = Pathname(path)
+        @dir = @path.dirname
+        @document = Nokogiri::XML(@path.read).tap(&:remove_namespaces!)
       end
 
       def package_references

data/lib/spandx/java/index.rb

@@ -13,13 +13,16 @@ module Spandx
         @directory = directory
         @source = source
         @name = 'maven'
+        @cache = ::Spandx::Core::Cache.new(@name, root: directory)
       end
 
       def update!(catalogue:, output:)
         each do |metadata|
-          name = "#{metadata.group_id}:#{metadata.artifact_id}:#{metadata.version}"
-          output.puts [name, metadata.licenses_from(catalogue)].inspect
+          name = "#{metadata.group_id}:#{metadata.artifact_id}"
+          output.puts [name, metadata.version, metadata.licenses_from(catalogue)].inspect
+          @cache.insert(name, metadata.version, metadata.licenses_from(catalogue))
         end
+        @cache.rebuild_index
       end
 
       def each

data/lib/spandx/java/parsers/maven.rb

@@ -4,26 +4,26 @@ module Spandx
   module Java
     module Parsers
       class Maven < ::Spandx::Core::Parser
-        def matches?(filename)
-          File.basename(filename) == 'pom.xml'
+        def match?(path)
+          path.basename.fnmatch?('pom.xml')
         end
 
-        def parse(filename)
-          document = Nokogiri.XML(IO.read(filename)).tap(&:remove_namespaces!)
+        def parse(path)
+          document = Nokogiri.XML(path.read).tap(&:remove_namespaces!)
           document.search('//project/dependencies/dependency').map do |node|
-            map_from(node)
+            map_from(path, node)
           end
         end
 
         private
 
-        def map_from(node)
+        def map_from(path, node)
           artifact_id = node.at_xpath('./artifactId').text
           group_id = node.at_xpath('./groupId').text
           version = node.at_xpath('./version').text
 
           ::Spandx::Core::Dependency.new(
-            package_manager: :maven,
+            path: path,
             name: "#{group_id}:#{artifact_id}",
             version: version
           )

data/lib/spandx/js/parsers/npm.rb

@@ -4,14 +4,14 @@ module Spandx
   module Js
     module Parsers
       class Npm < ::Spandx::Core::Parser
-        def matches?(filename)
+        def match?(filename)
           File.basename(filename) == 'package-lock.json'
         end
 
-        def parse(file_path)
+        def parse(path)
           items = Set.new
-          each_metadata(file_path) do |metadata|
-            items.add(map_from(metadata))
+          each_metadata(path) do |metadata|
+            items.add(map_from(path, metadata))
           end
           items
         end
@@ -25,9 +25,9 @@ module Spandx
           end
         end
 
-        def map_from(metadata)
+        def map_from(path, metadata)
           Spandx::Core::Dependency.new(
-            package_manager: :npm,
+            path: path,
             name: metadata['name'],
             version: metadata['version'],
             meta: metadata

data/lib/spandx/js/parsers/yarn.rb

@@ -4,21 +4,21 @@ module Spandx
   module Js
     module Parsers
       class Yarn < ::Spandx::Core::Parser
-        def matches?(filename)
-          File.basename(filename) == 'yarn.lock'
+        def match?(filename)
+          filename.basename.fnmatch?('yarn.lock')
         end
 
-        def parse(file_path)
-          YarnLock.new(file_path).each_with_object(Set.new) do |metadata, memo|
-            memo << map_from(metadata)
+        def parse(path)
+          YarnLock.new(path).each_with_object(Set.new) do |metadata, memo|
+            memo << map_from(path, metadata)
           end
         end
 
         private
 
-        def map_from(metadata)
+        def map_from(path, metadata)
           ::Spandx::Core::Dependency.new(
-            package_manager: :yarn,
+            path: path,
             name: metadata['name'],
             version: metadata['version'],
             meta: metadata

data/lib/spandx/php/parsers/composer.rb

@@ -4,24 +4,24 @@ module Spandx
   module Php
     module Parsers
       class Composer < ::Spandx::Core::Parser
-        def matches?(filename)
-          File.basename(filename) == 'composer.lock'
+        def match?(path)
+          path.basename.fnmatch? 'composer.lock'
         end
 
-        def parse(file_path)
+        def parse(path)
           items = Set.new
-          composer_lock = JSON.parse(IO.read(file_path))
+          composer_lock = JSON.parse(path.read)
           composer_lock['packages'].concat(composer_lock['packages-dev']).each do |dependency|
-            items.add(map_from(dependency))
+            items.add(map_from(path, dependency))
           end
           items
         end
 
         private
 
-        def map_from(dependency)
+        def map_from(path, dependency)
           Spandx::Core::Dependency.new(
-            package_manager: :composer,
+            path: path,
             name: dependency['name'],
             version: dependency['version'],
             meta: dependency

data/lib/spandx/python/index.rb

@@ -12,28 +12,18 @@ module Spandx
         @name = 'pypi'
         @source = 'https://pypi.org'
         @pypi = Pypi.new
-        Thread.abort_on_exception = true
+        @cache = ::Spandx::Core::Cache.new(@name, root: directory)
       end
 
       def update!(*)
         queue = Queue.new
         [fetch(queue), save(queue)].each(&:join)
+        cache.rebuild_index
       end
 
       private
 
-      def files(pattern)
-        Dir.glob(pattern, base: directory).sort.each do |file|
-          fullpath = File.join(directory, file)
-          yield fullpath unless File.directory?(fullpath)
-        end
-      end
-
-      def sort_index!
-        files('**/pypi') do |path|
-          IO.write(path, IO.readlines(path).sort.join)
-        end
-      end
+      attr_reader :cache
 
       def fetch(queue)
         Thread.new do
@@ -50,29 +40,10 @@ module Spandx
             item = queue.deq
             break if item == :stop
 
-            insert!(item[:name], item[:version], item[:license])
+            cache.insert(item[:name], item[:version], [item[:license]])
           end
         end
       end
-
-      def digest_for(components)
-        Digest::SHA1.hexdigest(Array(components).join('/'))
-      end
-
-      def data_dir_for(name)
-        File.join(directory, digest_for(name)[0...2].downcase)
-      end
-
-      def data_file_for(name)
-        File.join(data_dir_for(name), 'pypi')
-      end
-
-      def insert!(name, version, license)
-        return if license.nil? || license.empty?
-
-        csv = CSV.generate_line([name, version, license], force_quotes: true)
-        IO.write(data_file_for(name), csv, mode: 'a')
-      end
     end
   end
 end

data/lib/spandx/python/parsers/pipfile_lock.rb

@@ -4,8 +4,8 @@ module Spandx
   module Python
     module Parsers
       class PipfileLock < ::Spandx::Core::Parser
-        def matches?(filename)
-          filename.match?(/Pipfile.*\.lock/)
+        def match?(path)
+          path.basename.fnmatch?('Pipfile*.lock')
         end
 
         def parse(lockfile)
@@ -19,10 +19,10 @@ module Spandx
         private
 
         def dependencies_from(lockfile)
-          json = JSON.parse(IO.read(lockfile))
+          json = JSON.parse(lockfile.read)
           each_dependency(json) do |name, version|
             yield ::Spandx::Core::Dependency.new(
-              package_manager: :pypi,
+              path: lockfile,
               name: name,
               version: version,
               meta: json

data/lib/spandx/python/pypi.rb

@@ -96,7 +96,5 @@ module Spandx
         Nokogiri::HTML(http.get(url).body)
       end
     end
-
-    PyPI = Pypi
   end
 end

data/lib/spandx/python/source.rb

@@ -28,11 +28,23 @@ module Spandx
         end
       end
 
+      def ==(other)
+        name == other.name &&
+          uri.to_s == other.uri.to_s &&
+          verify_ssl == other.verify_ssl
+      end
+
+      def eql(other)
+        self == other
+      end
+
       class << self
         def sources_from(json)
           meta = json['_meta']
           meta['sources'].map do |hash|
             new(hash)
+          rescue URI::InvalidURIError
+            default
           end
         end
 

data/lib/spandx/ruby/parsers/gemfile_lock.rb

@@ -6,31 +6,32 @@ module Spandx
       class GemfileLock < ::Spandx::Core::Parser
         STRIP_BUNDLED_WITH = /^BUNDLED WITH$(\r?\n)   (?<major>\d+)\.\d+\.\d+/m.freeze
 
-        def matches?(filename)
-          filename.match?(/Gemfile.*\.lock/) ||
-            filename.match?(/gems.*\.lock/)
+        def match?(pathname)
+          basename = pathname.basename
+          basename.fnmatch?('Gemfile*.lock') ||
+            basename.fnmatch?('gems*.lock')
         end
 
         def parse(lockfile)
           dependencies_from(lockfile).map do |specification|
-            map_from(specification)
+            map_from(lockfile, specification)
           end
         end
 
         private
 
         def dependencies_from(filepath)
-          content = IO.read(filepath)
-          Dir.chdir(File.dirname(filepath)) do
+          content = filepath.read.sub(STRIP_BUNDLED_WITH, '')
+          Dir.chdir(filepath.dirname) do
             ::Bundler::LockfileParser
-              .new(content.sub(STRIP_BUNDLED_WITH, ''))
+              .new(content)
               .specs
           end
         end
 
-        def map_from(specification)
+        def map_from(lockfile, specification)
           ::Spandx::Core::Dependency.new(
-            package_manager: :rubygems,
+            path: lockfile,
             name: specification.name,
             version: specification.version.to_s,
             meta: {

data/lib/spandx/spdx/catalogue.rb

@@ -33,13 +33,17 @@ module Spandx
         end
 
         def from_file(path)
-          from_json(IO.read(path))
+          from_json(Pathname.new(path).read)
         end
 
         def from_git
           from_json(Spandx.git[:spdx].read('json/licenses.json'))
         end
 
+        def default
+          from_git
+        end
+
         def empty
           @empty ||= new(licenses: [])
         end

data/lib/spandx/spdx/composite_license.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Spdx
+    class CompositeLicense < License
+      def self.from_expression(expression, catalogue)
+        tree = Spdx::Expression.new.parse(expression)
+        new(tree[0], catalogue)
+      rescue Parslet::ParseFailed
+        nil
+      end
+
+      def initialize(tree, catalogue)
+        @catalogue = catalogue
+        @tree = tree
+        super({})
+      end
+
+      def id
+        if right
+          [left.id, operator, right.id].compact.join(' ').squeeze(' ').strip
+        else
+          left.id.to_s
+        end
+      end
+
+      def name
+        if right
+          [left.name, operator, right.name].compact.join(' ').squeeze(' ').strip
+        else
+          left.name
+        end
+      end
+
+      private
+
+      def left
+        node_for(@tree[:left])
+      end
+
+      def operator
+        @tree[:op].to_s.upcase
+      end
+
+      def right
+        node_for(@tree[:right])
+      end
+
+      def node_for(item)
+        return if item.nil?
+
+        if item.is_a?(Hash)
+          self.class.new(item, @catalogue)
+        else
+          @catalogue[item.to_s] || License.unknown(item.to_s)
+        end
+      end
+    end
+  end
+end

data/lib/spandx/spdx/expression.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Spdx
+    class Expression < Parslet::Parser
+      # https://spdx.org/spdx-specification-21-web-version
+      #
+      # idstring              = 1*(ALPHA / DIGIT / "-" / "." )
+      # license-id            = <short form license identifier in Appendix I.1>
+      # license-exception-id  = <short form license exception identifier in Appendix I.2>
+      # license-ref           = ["DocumentRef-"1*(idstring)":"]"LicenseRef-"1*(idstring)
+      # simple-expression = license-id / license-id"+" / license-ref
+      # compound-expression =  1*1(simple-expression /
+      #                             simple-expression "WITH" license-exception-id /
+      #                             compound-expression "AND" compound-expression /
+      #                             compound-expression "OR" compound-expression ) /
+      #                           "(" compound-expression ")" )
+      #
+      # license-expression =  1*1(simple-expression / compound-expression)
+      rule(:lparen) { str('(') }
+      rule(:rparen) { str(')') }
+      rule(:digit) { match('\d') }
+      rule(:space) { match('\s') }
+      rule(:space?) { space.maybe }
+      rule(:alpha) { match['a-zA-Z'] }
+      rule(:colon) { str(':') }
+      rule(:dot) { str('.') }
+      rule(:plus) { str('+') }
+      rule(:plus?) { plus.maybe }
+      rule(:hyphen) { str('-') }
+      rule(:hyphen?) { hyphen.maybe }
+      rule(:with_op) { str('with') | str('WITH') }
+      rule(:and_op) { str('AND') | str('and') }
+      rule(:or_op) { str('OR') | str('or') }
+
+      # idstring              = 1*(ALPHA / DIGIT / "-" / "." )
+      rule(:id_character) { alpha | digit | hyphen | dot }
+      rule(:id_string) { id_character.repeat(1) }
+
+      # license-id = <short form license identifier in Appendix I.1>
+      rule(:license_id) do
+        id_string
+      end
+
+      # license-ref = ["DocumentRef-"1*(idstring)":"]"LicenseRef-"1*(idstring)
+      rule(:license_ref) do
+        (str('DocumentRef-') >> id_string >> colon).repeat(0, 1) >> str('LicenseRef-') >> id_string
+      end
+
+      # simple-expression = license-id / license-id"+" / license-ref
+      rule(:simple_expression) do
+        license_id >> plus? | license_ref
+      end
+
+      rule(:exception) do
+        match['eE'] >> str('xception')
+      end
+
+      rule(:version) do
+        digit >> dot >> digit
+      end
+
+      # license-exception-id = <short form license exception identifier in Appendix I.2>
+      rule(:license_exception_id) do
+        # alpha.repeat(1) >> hyphen >> exception >> (hyphen? >> version)
+        id_string
+      end
+
+      # simple-expression "WITH" license-exception-id
+      rule(:with_expression) do
+        simple_expression.as(:left) >> space >> with_op.as(:op) >> space >> license_exception_id.as(:right)
+      end
+
+      rule(:binary_operator) do
+        (or_op | and_op).as(:op)
+      end
+
+      rule(:binary_right) do
+        space >> binary_operator >> space >> (binary_expression | simple_expression).as(:right)
+      end
+
+      # compound-expression "AND" compound-expression
+      # compound-expression "OR" compound-expression
+      rule(:binary_expression) do
+        simple_expression.as(:left) >> binary_right
+      end
+
+      # (BSD-2-Clause OR MIT OR Apache-2.0)
+      #
+      #
+      # compound-expression =  1*1(
+      #     simple-expression /
+      #     simple-expression "WITH" license-exception-id /
+      #     compound-expression "AND" compound-expression /
+      #     compound-expression "OR" compound-expression
+      #   ) / "(" compound-expression ")")
+      rule(:compound_expression) do
+        lparen >> compound_expression >> space? >> rparen |
+          (
+            binary_expression |
+            with_expression |
+            simple_expression.as(:left)
+          ).repeat(1, 1)
+      end
+
+      # license-expression =  1*1(simple-expression / compound-expression)
+      rule(:license_expression) do
+        (compound_expression | simple_expression).repeat(1, 1)
+      end
+
+      root(:license_expression)
+    end
+  end
+end