spandx 0.12.3 → 0.13.0

data/lib/spandx/core/concurrent.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Core
+    class Concurrent
+      include Enumerable
+
+      def self.map(items, pool:, &block)
+        queue = Queue.new
+
+        items.each do |item|
+          pool.schedule([item, block]) do |marshalled_item, callable|
+            queue.enq(callable.call(marshalled_item))
+          end
+        end
+
+        new(queue, items.size)
+      end
+
+      attr_reader :queue, :size
+
+      def initialize(queue, size)
+        @queue = queue
+        @size = size
+      end
+
+      def each
+        size.times { yield queue.deq }
+      end
+
+      def to_enum
+        Enumerator.new do |yielder|
+          each do |item|
+            yielder.yield item
+          end
+        end
+      end
+    end
+  end
+end

data/lib/spandx/core/content.rb

@@ -13,30 +13,12 @@ module Spandx
         @tokens ||= tokenize(canonicalize(raw)).to_set
       end
 
-      def similar?(other, algorithm: :dice_coefficient)
-        case algorithm
-        when :dice_coefficient
-          similarity_score(other, algorithm: algorithm) > 89.0
-        when :levenshtein
-          similarity_score(other, algorithm: algorithm) < 3
-        when :jaro_winkler
-          similarity_score(other, algorithm: algorithm) > 89.0
-        end
+      def similar?(other, threshold: 89.0)
+        similarity_score(other) > threshold
       end
 
-      def similarity_score(other, algorithm: :dice_coefficient)
-        case algorithm
-        when :dice_coefficient
-          dice_coefficient(other)
-        when :levenshtein
-          require 'text'
-
-          Text::Levenshtein.distance(raw, other.raw, 100)
-        when :jaro_winkler
-          require 'jaro_winkler'
-
-          JaroWinkler.distance(raw, other.raw) * 100.0
-        end
+      def similarity_score(other)
+        dice_coefficient(other)
       end
 
       private
@@ -46,7 +28,7 @@ module Spandx
       end
 
       def tokenize(content)
-        content.to_s.scan(/[a-zA-Z]+/)
+        content.to_s.scan(/[a-zA-Z\d.]+/)
       end
 
       def blank?(content)

data/lib/spandx/core/data_file.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Core
+    class DataFile
+      include Enumerable
+
+      attr_reader :absolute_path
+
+      def initialize(absolute_path)
+        @absolute_path = Pathname.new(absolute_path)
+        FileUtils.mkdir_p(@absolute_path.dirname)
+      end
+
+      def each
+        return unless exist?
+
+        open_file(mode: 'rb') do |io|
+          while (line = io.gets)
+            yield CsvParser.parse(line)
+          end
+        end
+      end
+
+      def search(name:, version:)
+        return if name.nil? || name.empty?
+        return if version.nil? || name.empty?
+        return unless absolute_path.exist?
+
+        term = "#{name}-#{version}"
+        index.search do |row|
+          term <=> "#{row[0]}-#{row[1]}"
+        end
+      end
+
+      def insert(name, version, licenses)
+        return if [name, version].any? { |x| x.nil? || x.empty? }
+
+        open_file(mode: 'a') do |io|
+          io.write(to_csv([name, version, licenses.join('-|-')]))
+        end
+      end
+
+      def exist?
+        absolute_path.exist?
+      end
+
+      def open_file(mode: 'rb')
+        absolute_path.open(mode) { |io| yield io }
+      rescue Errno::ENOENT => error
+        Spandx.logger.error(error)
+        nil
+      end
+
+      def index
+        @index ||= IndexFile.new(self)
+      end
+
+      private
+
+      def to_csv(array)
+        array.to_csv(force_quotes: true)
+      end
+    end
+  end
+end

data/lib/spandx/core/git.rb

@@ -3,37 +3,21 @@
 module Spandx
   module Core
     class Git
-      attr_reader :path, :url
+      attr_reader :root, :url
 
       def initialize(url:)
         @url = url
-        @path = path_for(url)
-      end
-
-      def update!
-        dotgit? ? pull! : clone!
-      end
-
-      def expand_path(relative_path)
-        File.join(path, relative_path)
+        @root = path_for(url)
       end
 
       def read(path)
-        update! unless dotgit?
+        full_path = File.join(root, path)
 
-        full_path = expand_path(path)
         IO.read(full_path) if File.exist?(full_path)
       end
 
-      def open(path, mode: 'r')
-        update! unless dotgit?
-
-        full_path = expand_path(path)
-        return unless File.exist?(full_path)
-
-        File.open(full_path, mode) do |io|
-          yield io
-        end
+      def update!
+        dotgit? ? pull! : clone!
       end
 
       private
@@ -45,24 +29,18 @@ module Spandx
       end
 
       def dotgit?
-        File.directory?(File.join(path, '.git'))
+        File.directory?(File.join(root, '.git'))
       end
 
       def clone!
-        system('git', 'clone', '--quiet', url, path)
+        system('git', 'clone', '--quiet', '--depth=1', '--single-branch', '--branch', 'master', url, root)
       end
 
       def pull!
-        within do
+        Dir.chdir(root) do
           system('git', 'pull', '--no-rebase', '--quiet', 'origin', 'master')
         end
       end
-
-      def within
-        Dir.chdir(path) do
-          yield
-        end
-      end
     end
 
     Database = Git

data/lib/spandx/core/guess.rb

@@ -9,80 +9,77 @@ module Spandx
         @catalogue = catalogue
       end
 
-      def license_for(raw, algorithm: :dice_coefficient)
-        raw.is_a?(Hash) ? from_hash(raw, algorithm) : from_string(raw, algorithm)
+      def license_for(raw)
+        raw.is_a?(Hash) ? from_hash(raw) : from_string(raw)
       end
 
       private
 
-      def from_hash(hash, algorithm)
-        from_string(hash[:name], algorithm) ||
-          from_url(hash[:url], algorithm) ||
+      def from_hash(hash)
+        from_string(hash[:name]) ||
+          from_url(hash[:url]) ||
           unknown(hash[:name] || hash[:url])
       end
 
-      def from_string(raw, algorithm)
+      def from_string(raw)
+        return if raw.nil?
+
         content = Content.new(raw)
 
         catalogue[raw] ||
-          match_name(content, algorithm) ||
-          match_body(content, algorithm) ||
+          catalogue[raw.split(' ').join('-')] ||
+          match_name(content) ||
+          match_body(content) ||
           unknown(raw)
       end
 
-      def from_url(url, algorithm)
+      def from_url(url)
         return if url.nil? || url.empty?
 
         response = Spandx.http.get(url)
         return unless Spandx.http.ok?(response)
 
-        license_for(response.body, algorithm: algorithm)
+        license_for(response.body)
       end
 
-      def match_name(content, _algorithm)
+      def match_name(content)
+        return if content.tokens.size < 2 || content.tokens.size > 10
+
+        result = from_expression(content)
+        return result if result
+
+        threshold = 85.0
         catalogue.find do |license|
-          score = content.similarity_score(::Spandx::Core::Content.new(license.name))
-          score > 85
+          content.similar?(Content.new(license.name), threshold: threshold)
         end
       end
 
-      def match_body(content, algorithm)
+      def match_body(content)
         score = Score.new(nil, nil)
-        threshold = threshold_for(algorithm)
-        direction = algorithm == :levenshtein ? method(:min) : method(:max)
-
+        threshold = 89.0
         catalogue.each do |license|
-          direction.call(content, license, score, threshold, algorithm) unless license.deprecated_license_id?
+          next if license.deprecated_license_id?
+
+          percentage = content.similarity_score(content_for(license))
+          next if percentage < threshold
+          next if score.score >= percentage
+
+          score.update(percentage, license)
         end
         score&.item
       end
 
-      def unknown(text)
-        ::Spandx::Spdx::License.unknown(text)
-      end
-
-      def threshold_for(algorithm)
-        {
-          dice_coefficient: 89.0,
-          jaro_winkler: 80.0,
-          levenshtein: 80.0,
-        }[algorithm.to_sym]
+      def content_for(license)
+        ::Spandx::Core::Content.new(Spandx.git[:spdx].read("text/#{license.id}.txt") || '')
       end
 
-      def min(target, other, score, threshold, algorithm)
-        percentage = target.similarity_score(other.content, algorithm: algorithm)
-        return if percentage > threshold
-        return if score.score > 0.0 && score.score < percentage
-
-        score.update(percentage, other)
+      def unknown(text)
+        ::Spandx::Spdx::License.unknown(text)
       end
 
-      def max(target, other, score, threshold, algorithm)
-        percentage = target.similarity_score(other.content, algorithm: algorithm)
-        return if percentage < threshold
-        return if score.score >= percentage
-
-        score.update(percentage, other)
+      def from_expression(content)
+        Spandx::Spdx::CompositeLicense
+          .from_expression(content.raw, catalogue)
       end
     end
   end

data/lib/spandx/core/http.rb

@@ -8,7 +8,12 @@ module Spandx
       def initialize(driver: Http.default_driver, retries: 3)
         @driver = driver
         @retries = retries
-        @circuits = Hash.new { |hash, key| hash[key] = Circuit.new(key) }
+        semaphore = Mutex.new
+        @circuits = Hash.new do |hash, key|
+          semaphore.synchronize do
+            hash[key] = Circuit.new(key)
+          end
+        end
       end
 
       def get(uri, default: nil, escape: true)

data/lib/spandx/core/index_file.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Core
+    class IndexFile
+      UINT_32_DIRECTIVE = 'V'
+      UINT_32_SIZE = 4
+
+      attr_reader :data_file, :path
+
+      def initialize(data_file)
+        @data_file = data_file
+        @path = Pathname.new("#{data_file.absolute_path}.idx")
+        @entries = size.positive? ? Array.new(size) : []
+      end
+
+      def each
+        total = path.size / UINT_32_SIZE
+        total.times do |n|
+          yield position_for(n)
+        end
+      end
+
+      def search(min: 0, max: size)
+        scan do |reader|
+          until min >= max
+            mid = mid_for(min, max)
+            row = reader.row(mid)
+            comparison = yield row
+            return row if comparison.zero?
+
+            comparison.positive? ? (min = mid + 1) : (max = mid)
+          end
+        end
+      end
+
+      def size
+        path.exist? ? path.size / UINT_32_SIZE : 0
+      end
+
+      def position_for(row_number)
+        return if row_number > size
+
+        entry = entries[row_number]
+        return entry if entry
+
+        bytes = IO.binread(path, UINT_32_SIZE, offset_for(row_number))
+        entry = bytes.unpack1(UINT_32_DIRECTIVE)
+        entries[row_number] = entry
+        entry
+      end
+
+      def update!
+        return unless data_file.exist?
+
+        sort(data_file)
+        rebuild_index!
+      end
+
+      private
+
+      attr_reader :entries
+
+      def scan
+        data_file.open_file(mode: 'rb') do |io|
+          yield Relation.new(io, self)
+        end
+      end
+
+      def offset_for(row_number)
+        row_number * UINT_32_SIZE
+      end
+
+      def sort(data_file)
+        data_file.absolute_path.write(data_file.absolute_path.readlines.sort.uniq.join)
+      end
+
+      def rebuild_index!
+        data_file.open_file do |data_io|
+          File.open(path, mode: 'wb') do |index_io|
+            lines_in(data_io).each do |pos|
+              index_io.write([pos].pack(UINT_32_DIRECTIVE))
+            end
+          end
+        end
+      end
+
+      def lines_in(io)
+        lines = [0]
+        io.seek(0)
+        lines << io.pos while io.gets
+        lines.pop if lines.size > 1
+        lines
+      end
+
+      def mid_for(min, max)
+        (max - min) == 1 ? min : (((max - min) / 2) + min)
+      end
+    end
+  end
+end