spandx 0.12.3 → 0.13.0

data/lib/spandx/core/license_plugin.rb

@@ -3,7 +3,7 @@
 module Spandx
   module Core
     class LicensePlugin < Spandx::Core::Plugin
-      def initialize(catalogue: Spdx::Catalogue.from_git)
+      def initialize(catalogue: Spdx::Catalogue.default)
         @guess = Guess.new(catalogue)
       end
 
@@ -25,8 +25,8 @@ module Spandx
       end
 
       def cache_for(dependency, git: Spandx.git)
-        db = git[dependency.package_manager.to_sym] || git[:cache]
-        Spandx::Core::Cache.new(dependency.package_manager, db: db)
+        git = git[dependency.package_manager.to_sym] || git[:cache]
+        Spandx::Core::Cache.new(dependency.package_manager, root: "#{git.root}/.index")
       end
 
       def known?(package_manager)

data/lib/spandx/core/line_io.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Core
+    class LineIo
+      def initialize(absolute_path)
+        file_descriptor = IO.sysopen(absolute_path)
+        @io = IO.new(file_descriptor)
+        @buffer = ''
+      end
+
+      def each(&block)
+        @buffer << @io.sysread(512) until @buffer.include?($INPUT_RECORD_SEPARATOR)
+
+        line, @buffer = @buffer.split($INPUT_RECORD_SEPARATOR, 2)
+        block.call(line)
+        each(&block)
+      rescue EOFError
+        @io.close
+      end
+    end
+  end
+end

data/lib/spandx/core/path_traversal.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Core
+    class PathTraversal
+      attr_reader :root
+
+      def initialize(root, recursive: true)
+        @root = root
+        @recursive = recursive
+      end
+
+      def each(&block)
+        each_file_in(root, &block)
+      end
+
+      def to_enum
+        Enumerator.new do |yielder|
+          each do |item|
+            yielder.yield item
+          end
+        end
+      end
+
+      private
+
+      def recursive?
+        @recursive
+      end
+
+      def each_file_in(dir, &block)
+        files = File.directory?(dir) ? Dir.glob(File.join(dir, '*')) : [dir]
+        files.each do |file|
+          if File.directory?(file)
+            each_file_in(file, &block) if recursive?
+          else
+            Spandx.logger.debug(file)
+            block.call(file)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/spandx/core/relation.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Core
+    class Relation
+      attr_reader :io, :index
+
+      def initialize(io, index)
+        @io = io
+        @index = index
+      end
+
+      def each
+        size.times do |n|
+          yield row(n)
+        end
+      end
+
+      def size
+        index.size
+      end
+
+      def row(number)
+        offset = number.zero? ? 0 : index.position_for(number)
+        return unless offset
+
+        io.seek(offset)
+        parse_row(io.gets)
+      end
+
+      private
+
+      def parse_row(line)
+        CsvParser.parse(line)
+      end
+    end
+  end
+end

data/lib/spandx/core/thread_pool.rb

@@ -5,12 +5,16 @@ module Spandx
     class ThreadPool
       def initialize(size: Etc.nprocessors)
         @size = size
-        @jobs = Queue.new
-        @pool = size.times { start_worker_thread }
+        @queue = Queue.new
+        @pool = size.times.map { start_worker_thread }
       end
 
       def schedule(*args, &block)
-        @jobs << [block, args]
+        @queue.enq([block, args])
+      end
+
+      def done?
+        @queue.empty?
       end
 
       def shutdown
@@ -21,13 +25,20 @@ module Spandx
         @pool.map(&:join)
       end
 
+      def self.open
+        pool = new
+        yield pool
+      ensure
+        pool.shutdown
+      end
+
       private
 
       def start_worker_thread
         Thread.new do
           catch(:exit) do
             loop do
-              job, args = @jobs.deq
+              job, args = @queue.deq
               job.call(*args)
             end
           end

data/lib/spandx/dotnet/index.rb

@@ -4,98 +4,40 @@ module Spandx
   module Dotnet
     class Index
       DEFAULT_DIR = File.expand_path(File.join(Dir.home, '.local', 'share', 'spandx'))
-      attr_reader :directory, :name
+      attr_reader :cache, :directory, :name, :gateway
 
-      def initialize(directory: DEFAULT_DIR)
+      def initialize(directory: DEFAULT_DIR, gateway: Spandx::Dotnet::NugetGateway.new)
         @directory = directory ? File.expand_path(directory) : DEFAULT_DIR
         @name = 'nuget'
+        @gateway = gateway
+        @cache = Spandx::Core::Cache.new(@name, root: directory)
       end
 
-      def licenses_for(name:, version:)
-        search_key = [name, version].join
-        CSV.open(data_file_for(name), 'r') do |io|
-          found = io.readlines.bsearch { |x| search_key <=> [x[0], x[1]].join }
-          found ? found[2].split('-|-') : []
-        end
-      end
-
-      def update!(catalogue:, output: StringIO.new)
-        catalogue.version
-        insert_latest(Spandx::Dotnet::NugetGateway.new) do |page|
-          output.puts "Checkpoint #{page}"
-          checkpoint!(page)
-        end
-        sort_index!
+      def update!(*)
+        queue = Queue.new
+        [fetch(queue), save(queue)].each(&:join)
+        cache.rebuild_index
       end
 
       private
 
-      def files(pattern)
-        Dir.glob(File.join(directory, pattern)).sort.each do |file|
-          fullpath = File.join(directory, file)
-          next if File.directory?(fullpath)
-          next unless File.exist?(fullpath)
-
-          yield fullpath
+      def fetch(queue)
+        Thread.new do
+          gateway.each do |item|
+            queue.enq(item)
+          end
+          queue.enq(:stop)
         end
       end
 
-      def sort_index!
-        files('**/nuget') do |path|
-          next if File.extname(path) == '.checkpoints'
-
-          IO.write(path, IO.readlines(path).sort.join)
-        end
-      end
-
-      def digest_for(components)
-        Digest::SHA1.hexdigest(Array(components).join('/'))
-      end
-
-      def data_dir_for(name)
-        digest = digest_for(name)
-        File.join(directory, digest[0...2].downcase)
-      end
-
-      def data_file_for(name)
-        File.join(data_dir_for(name), 'nuget')
-      end
-
-      def checkpoints_filepath
-        @checkpoints_filepath ||= File.join(directory, 'nuget.checkpoints')
-      end
-
-      def checkpoints
-        @checkpoints ||= File.exist?(checkpoints_filepath) ? JSON.parse(IO.read(checkpoints_filepath)) : {}
-      end
-
-      def checkpoint!(page)
-        checkpoints[page.to_s] = Time.now.utc
-        IO.write(checkpoints_filepath, JSON.pretty_generate(checkpoints))
-      end
-
-      def insert(name, version, license)
-        path = data_file_for(name)
-        FileUtils.mkdir_p(File.dirname(path))
-        IO.write(
-          path,
-          CSV.generate_line([name, version, license], force_quotes: true),
-          mode: 'a'
-        )
-      end
-
-      def completed_pages
-        checkpoints.keys.map(&:to_i)
-      end
-
-      def insert_latest(gateway)
-        current_page = completed_pages.max || 0
-        gateway.each(start_page: current_page) do |spec, page|
-          break if checkpoints[page.to_s]
+      def save(queue)
+        Thread.new do
+          loop do
+            item = queue.deq
+            break if item == :stop
 
-          yield current_page if current_page && page != current_page
-          current_page = page
-          insert(spec['id'], spec['version'], spec['licenseExpression'])
+            cache.insert(item['id'], item['version'], [item['licenseExpression']])
+          end
         end
       end
     end

data/lib/spandx/java/index.rb

@@ -13,13 +13,16 @@ module Spandx
         @directory = directory
         @source = source
         @name = 'maven'
+        @cache = ::Spandx::Core::Cache.new(@name, root: directory)
       end
 
       def update!(catalogue:, output:)
         each do |metadata|
-          name = "#{metadata.group_id}:#{metadata.artifact_id}:#{metadata.version}"
-          output.puts [name, metadata.licenses_from(catalogue)].inspect
+          name = "#{metadata.group_id}:#{metadata.artifact_id}"
+          output.puts [name, metadata.version, metadata.licenses_from(catalogue)].inspect
+          @cache.insert(name, metadata.version, metadata.licenses_from(catalogue))
         end
+        @cache.rebuild_index
       end
 
       def each

data/lib/spandx/python/index.rb

@@ -12,28 +12,18 @@ module Spandx
         @name = 'pypi'
         @source = 'https://pypi.org'
         @pypi = Pypi.new
-        Thread.abort_on_exception = true
+        @cache = ::Spandx::Core::Cache.new(@name, root: directory)
       end
 
       def update!(*)
         queue = Queue.new
         [fetch(queue), save(queue)].each(&:join)
+        cache.rebuild_index
       end
 
       private
 
-      def files(pattern)
-        Dir.glob(pattern, base: directory).sort.each do |file|
-          fullpath = File.join(directory, file)
-          yield fullpath unless File.directory?(fullpath)
-        end
-      end
-
-      def sort_index!
-        files('**/pypi') do |path|
-          IO.write(path, IO.readlines(path).sort.join)
-        end
-      end
+      attr_reader :cache
 
       def fetch(queue)
         Thread.new do
@@ -50,29 +40,10 @@ module Spandx
             item = queue.deq
             break if item == :stop
 
-            insert!(item[:name], item[:version], item[:license])
+            cache.insert(item[:name], item[:version], [item[:license]])
           end
         end
       end
-
-      def digest_for(components)
-        Digest::SHA1.hexdigest(Array(components).join('/'))
-      end
-
-      def data_dir_for(name)
-        File.join(directory, digest_for(name)[0...2].downcase)
-      end
-
-      def data_file_for(name)
-        File.join(data_dir_for(name), 'pypi')
-      end
-
-      def insert!(name, version, license)
-        return if license.nil? || license.empty?
-
-        csv = CSV.generate_line([name, version, license], force_quotes: true)
-        IO.write(data_file_for(name), csv, mode: 'a')
-      end
     end
   end
 end

data/lib/spandx/spdx/catalogue.rb

@@ -40,6 +40,10 @@ module Spandx
           from_json(Spandx.git[:spdx].read('json/licenses.json'))
         end
 
+        def default
+          from_git
+        end
+
         def empty
           @empty ||= new(licenses: [])
         end

data/lib/spandx/spdx/composite_license.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Spdx
+    class CompositeLicense < License
+      def self.from_expression(expression, catalogue)
+        tree = Spdx::Expression.new.parse(expression)
+        new(tree[0], catalogue)
+      rescue Parslet::ParseFailed
+        nil
+      end
+
+      def initialize(tree, catalogue)
+        @catalogue = catalogue
+        @tree = tree
+        super({})
+      end
+
+      def id
+        if right
+          [left.id, operator, right.id].compact.join(' ').squeeze(' ').strip
+        else
+          left.id.to_s
+        end
+      end
+
+      def name
+        if right
+          [left.name, operator, right.name].compact.join(' ').squeeze(' ').strip
+        else
+          left.name
+        end
+      end
+
+      private
+
+      def left
+        node_for(@tree[:left])
+      end
+
+      def operator
+        @tree[:op].to_s.upcase
+      end
+
+      def right
+        node_for(@tree[:right])
+      end
+
+      def node_for(item)
+        return if item.nil?
+
+        if item.is_a?(Hash)
+          self.class.new(item, @catalogue)
+        else
+          @catalogue[item.to_s] || License.unknown(item.to_s)
+        end
+      end
+    end
+  end
+end