spandx 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 440933f7b4b8706a0a4da8ed5c84c5db80f93e40e7db258715bb73b11c5cbb31
-  data.tar.gz: 182e055dea23b17d3bb67fc16ce84ce245aa2dbfd247d9eead1b33162bf05472
+  metadata.gz: 6b3a0275ae468012967376aa3bd813b3dc1efef7783eb11d0151d7babf4ac6a8
+  data.tar.gz: b86bd82707ad5afa0b83c59674e4f0d08c12733d5f1788038845f30079bb8ba6
 SHA512:
-  metadata.gz: 26f4bc7516adf9f96c3725b15acc6ead6a2bdad6deaaf718a145106dcbec9f7c95c0f2053163f3c913a7466d2a8c3af860d4903a78bcb6eec8329f05a0cdb17f
-  data.tar.gz: 441abbd1f3cd090514627386e6f175ff5e53ba2909a2722f7e34e0f9a5e514532c431ffdd54e23d501dd20c6626c382b0fba215939989b7e0d32ba477bf86aa9
+  metadata.gz: f71dd4bf8438ec8857f622076ea345f3c48c8dce3354949459b564ace2c0c002bcc95d09d9d906ba9063bdeb092ade90160f51bdcfb9900a0225114d1aa5aa80
+  data.tar.gz: e0bbbdd0156924ac21d9730aa322b332f4d3c1ef43b530f58c56f411c57bfb95c154574bda77c61dda3fbd80b9c8fc9e8a2fea4c09053fa6bfda2c5a6ac86dc5

data/CHANGELOG.md

@@ -1,4 +1,4 @@
-Version 0.5.0
+Version 0.6.0
 
 # Changelog
 
@@ -9,6 +9,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.6.0] - 2020-03-03
+### Added
+- Add `spandx index update` command to fetch the latest `spandx-rubygems` index.
+
+### Removed
+- Drop `spandx-rubygems` dependency.
+
+### Changed
+- Pull latest `spandx-rubygems` index via git.
+- Perform binary search on CSV index.
+
 ## [0.5.0] - 2020-02-13
 ### Added
 - Add jaro winkler string similarity support.
@@ -73,7 +84,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Provide ruby API to the latest SPDX catalogue.
 
-[Unreleased]: https://github.com/mokhan/spandx/compare/v0.5.0...HEAD
+[Unreleased]: https://github.com/mokhan/spandx/compare/v0.6.0...HEAD
+[0.6.0]: https://github.com/mokhan/spandx/compare/v0.5.0...v0.6.0
 [0.5.0]: https://github.com/mokhan/spandx/compare/v0.4.1...v0.5.0
 [0.4.1]: https://github.com/mokhan/spandx/compare/v0.4.0...v0.4.1
 [0.4.0]: https://github.com/mokhan/spandx/compare/v0.3.0...v0.4.0

data/lib/spandx/cli/command.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Cli
+    class Command
+      extend Forwardable
+
+      def_delegators :command, :run
+
+      def execute(*)
+        raise(NotImplementedError, "#{self.class}##{__method__} must be implemented")
+      end
+
+      def command(**options)
+        require 'tty-command'
+        TTY::Command.new(options)
+      end
+
+      def cursor
+        require 'tty-cursor'
+        TTY::Cursor
+      end
+
+      def editor
+        require 'tty-editor'
+        TTY::Editor
+      end
+
+      def generator
+        require 'tty-file'
+        TTY::File
+      end
+
+      def pager(**options)
+        require 'tty-pager'
+        TTY::Pager.new(options)
+      end
+
+      def platform
+        require 'tty-platform'
+        TTY::Platform.new
+      end
+
+      def prompt(**options)
+        require 'tty-prompt'
+        TTY::Prompt.new(options)
+      end
+
+      def screen
+        require 'tty-screen'
+        TTY::Screen
+      end
+
+      def which(*args)
+        require 'tty-which'
+        TTY::Which.which(*args)
+      end
+
+      def exec_exist?(*args)
+        require 'tty-which'
+        TTY::Which.exist?(*args)
+      end
+    end
+  end
+end

data/lib/spandx/cli/commands/index/build.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Cli
+    module Commands
+      class Index
+        class Build < Spandx::Cli::Command
+          def initialize(options)
+            @options = options
+          end
+
+          def execute(output: $stdout)
+            catalogue = Spandx::Spdx::Catalogue.from_git
+            indexes.each do |index|
+              index.update!(catalogue: catalogue)
+            end
+            output.puts 'OK'
+          end
+
+          private
+
+          def indexes
+            [
+              Spandx::Dotnet::Index.new(directory: @options[:directory])
+            ]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/spandx/cli/commands/index/update.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Cli
+    module Commands
+      class Index
+        class Update < Spandx::Cli::Command
+          def initialize(options)
+            @options = options
+          end
+
+          def execute(output: $stdout)
+            [
+              'https://github.com/mokhan/spandx-rubygems.git',
+              'https://github.com/spdx/license-list-data.git',
+            ].each do |url|
+              output.puts "Updating #{url}..."
+              Spandx::Core::Database.new(url: url).update!
+            end
+            output.puts 'OK'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/spandx/cli/commands/index.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Cli
+    module Commands
+      class Index < Thor
+        require 'spandx/cli/commands/index/build'
+        require 'spandx/cli/commands/index/update'
+
+        namespace :index
+
+        desc 'build', 'Build a package index'
+        method_option :help, aliases: '-h', type: :boolean, desc: 'Display usage information'
+        method_option :directory, aliases: '-d', type: :string, desc: 'Directory to build index in', default: '.index'
+        def build(*)
+          if options[:help]
+            invoke :help, ['build']
+          else
+            Spandx::Cli::Commands::Index::Build.new(options).execute
+          end
+        end
+
+        desc 'update', 'Update the offline indexes'
+        method_option :help, aliases: '-h', type: :boolean,
+                             desc: 'Display usage information'
+        def update(*)
+          if options[:help]
+            invoke :help, ['update']
+          else
+            Spandx::Cli::Commands::Index::Update.new(options).execute
+          end
+        end
+      end
+    end
+  end
+end

data/lib/spandx/cli/commands/scan.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Cli
+    module Commands
+      class Scan < Spandx::Cli::Command
+        attr_reader :lockfile
+
+        def initialize(lockfile, options)
+          @lockfile = lockfile ? ::Pathname.new(File.expand_path(lockfile)) : nil
+          @options = options
+        end
+
+        def execute(output: $stdout)
+          if lockfile.nil?
+            output.puts 'OK'
+          else
+            report = ::Spandx::Core::Report.new
+            ::Spandx::Core::Parser.for(lockfile).parse(lockfile).each do |dependency|
+              report.add(dependency)
+            end
+            output.puts report.to_json
+          end
+        end
+      end
+    end
+  end
+end

data/lib/spandx/cli.rb

@@ -1,11 +1,10 @@
 # frozen_string_literal: true
 
 require 'thor'
-
 require 'spandx'
-require 'spandx/command'
-require 'spandx/commands/build'
-require 'spandx/commands/scan'
+require 'spandx/cli/command'
+require 'spandx/cli/commands/index'
+require 'spandx/cli/commands/scan'
 
 module Spandx
   class CLI < Thor
@@ -17,19 +16,7 @@ module Spandx
     end
     map %w[--version -v] => :version
 
-    desc 'build', 'Build a package index'
-    method_option :help, aliases: '-h', type: :boolean,
-                         desc: 'Display usage information'
-    method_option :directory, aliases: '-d', type: :string,
-                              desc: 'Directory to build index in'
-    def build(*)
-      if options[:help]
-        invoke :help, ['build']
-      else
-        require_relative 'commands/build'
-        Spandx::Commands::Build.new(options).execute
-      end
-    end
+    register Spandx::Cli::Commands::Index, 'index', 'index [SUBCOMMAND]', 'Command description...'
 
     desc 'scan LOCKFILE', 'Scan a lockfile and list dependencies/licenses'
     method_option :help, aliases: '-h', type: :boolean,
@@ -38,7 +25,7 @@ module Spandx
       if options[:help]
         invoke :help, ['scan']
       else
-        Spandx::Commands::Scan.new(lockfile, options).execute
+        Spandx::Cli::Commands::Scan.new(lockfile, options).execute
       end
     end
   end

data/lib/spandx/core/content.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Core
+    class Content
+      attr_reader :raw
+
+      def initialize(raw)
+        @raw = raw
+      end
+
+      def tokens
+        @tokens ||= tokenize(canonicalize(raw)).to_set
+      end
+
+      def similar?(other, algorithm: :dice_coefficient)
+        case algorithm
+        when :dice_coefficient
+          similarity_score(other, algorithm: algorithm) > 89.0
+        when :levenshtein
+          similarity_score(other, algorithm: algorithm) < 3
+        when :jaro_winkler
+          similarity_score(other, algorithm: algorithm) > 89.0
+        end
+      end
+
+      def similarity_score(other, algorithm: :dice_coefficient)
+        case algorithm
+        when :dice_coefficient
+          dice_coefficient(other)
+        when :levenshtein
+          require 'text'
+
+          Text::Levenshtein.distance(raw, other.raw, 100)
+        when :jaro_winkler
+          require 'jaro_winkler'
+
+          JaroWinkler.distance(raw, other.raw) * 100.0
+        end
+      end
+
+      private
+
+      def canonicalize(content)
+        content&.downcase
+      end
+
+      def tokenize(content)
+        content.to_s.scan(/[a-zA-Z]+/)
+      end
+
+      def blank?(content)
+        content.nil? || content.chomp.strip.empty?
+      end
+
+      # https://en.wikibooks.org/wiki/Algorithm_Implementation/Strings/Dice%27s_coefficient#Ruby
+      def dice_coefficient(other)
+        overlap = (tokens & other.tokens).size
+        total = tokens.size + other.tokens.size
+        100.0 * (overlap * 2.0 / total)
+      end
+    end
+  end
+end

data/lib/spandx/core/database.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Core
+    class Database
+      attr_reader :path, :url
+
+      def initialize(url:)
+        @url = url
+        @path = path_for(url)
+      end
+
+      def update!
+        dotgit? ? pull! : clone!
+      end
+
+      def expand_path(relative_path)
+        File.join(path, relative_path)
+      end
+
+      def read(path)
+        update! unless dotgit?
+
+        full_path = expand_path(path)
+        IO.read(full_path) if File.exist?(full_path)
+      end
+
+      def open(path, mode: 'r')
+        update! unless dotgit?
+
+        File.open(expand_path(path), mode) do |io|
+          yield io
+        end
+      end
+
+      private
+
+      def path_for(url)
+        uri = URI.parse(url)
+        name = uri.path.gsub(/\.git$/, '')
+        File.expand_path(File.join(Dir.home, '.local', 'share', name))
+      end
+
+      def dotgit?
+        File.directory?(File.join(path, '.git'))
+      end
+
+      def clone!
+        system('git', 'clone', '--quiet', url, path)
+      end
+
+      def pull!
+        within do
+          system('git', 'pull', '--no-rebase', '--quiet', 'origin', 'master')
+        end
+      end
+
+      def within
+        Dir.chdir(path) do
+          yield
+        end
+      end
+    end
+  end
+end

data/lib/spandx/core/dependency.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Core
+    class Dependency
+      attr_reader :name, :version, :licenses
+
+      def initialize(name:, version:, licenses: [])
+        @name = name
+        @version = version
+        @licenses = licenses
+      end
+
+      def to_h
+        {
+          name: name,
+          version: version,
+          licenses: licenses.compact.map(&:id)
+        }
+      end
+    end
+  end
+end

data/lib/spandx/core/guess.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Core
+    class Guess
+      attr_reader :catalogue
+
+      def initialize(catalogue)
+        @catalogue = catalogue
+      end
+
+      def license_for(raw_content, algorithm: :dice_coefficient)
+        content = Content.new(raw_content)
+        score = Score.new(nil, nil)
+        threshold = threshold_for(algorithm)
+        direction = algorithm == :levenshtein ? method(:min) : method(:max)
+
+        catalogue.each do |license|
+          direction.call(content, license, score, threshold, algorithm) unless license.deprecated_license_id?
+        end
+        score&.item&.id
+      end
+
+      private
+
+      def threshold_for(algorithm)
+        {
+          dice_coefficient: 89.0,
+          jaro_winkler: 80.0,
+          levenshtein: 80.0,
+        }[algorithm.to_sym]
+      end
+
+      def min(target, other, score, threshold, algorithm)
+        percentage = target.similarity_score(other.content, algorithm: algorithm)
+        return if percentage > threshold
+        return if score.score > 0.0 && score.score < percentage
+
+        score.update(percentage, other)
+      end
+
+      def max(target, other, score, threshold, algorithm)
+        percentage = target.similarity_score(other.content, algorithm: algorithm)
+        return if percentage < threshold
+        return if score.score >= percentage
+
+        score.update(percentage, other)
+      end
+    end
+  end
+end

data/lib/spandx/{parsers/base.rb → core/parser.rb}

@@ -1,8 +1,14 @@
 # frozen_string_literal: true
 
 module Spandx
-  module Parsers
-    class Base
+  module Core
+    class Parser
+      UNKNOWN = Class.new do
+        def self.parse(*_args)
+          []
+        end
+      end
+
       attr_reader :catalogue
 
       def initialize(catalogue:)
@@ -25,6 +31,14 @@ module Spandx
         def registry
           @registry ||= []
         end
+
+        def for(path, catalogue: Spandx::Spdx::Catalogue.from_git)
+          result = ::Spandx::Core::Parser.find do |x|
+            x.matches?(File.basename(path))
+          end
+
+          result&.new(catalogue: catalogue) || UNKNOWN
+        end
       end
     end
   end

data/lib/spandx/core/report.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Core
+    class Report
+      def initialize(report: { version: '1.0', packages: [] })
+        @report = report
+      end
+
+      def add(dependency)
+        @report[:packages].push(dependency.to_h)
+      end
+
+      def to_h
+        @report
+      end
+
+      def to_json(*_args)
+        JSON.pretty_generate(to_h)
+      end
+    end
+  end
+end

data/lib/spandx/core/score.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Core
+    class Score
+      include Comparable
+
+      attr_reader :score, :item
+
+      def initialize(score, item)
+        update(score || 0.0, item)
+      end
+
+      def update(score, item)
+        @score = score
+        @item = item
+      end
+
+      def empty?
+        score.nil? || item.nil?
+      end
+
+      def <=>(other)
+        score <=> other.score
+      end
+
+      def to_s
+        "#{score}: #{item}"
+      end
+    end
+  end
+end

data/lib/spandx/dotnet/index.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Dotnet
+    class Index
+      DEFAULT_DIR = File.expand_path(File.join(Dir.home, '.local', 'share', 'spandx'))
+      attr_reader :directory
+
+      def initialize(directory: DEFAULT_DIR)
+        @directory = directory ? File.expand_path(directory) : DEFAULT_DIR
+      end
+
+      def update!(catalogue:, limit: nil)
+        counter = 0
+        gateway = Spandx::Dotnet::NugetGateway.new(catalogue: catalogue)
+        gateway.each do |spec|
+          next unless spec['licenseExpression']
+
+          write([gateway.host, spec['id'], spec['version']], spec['licenseExpression'])
+
+          if limit
+            counter += 1
+            break if counter > limit
+          end
+        end
+      end
+
+      def indexed?(key)
+        File.exist?(data_file_for(digest_for(key)))
+      end
+
+      def read(key)
+        open_data(digest_for(key), mode: 'r', &:read)
+      end
+
+      def write(key, data)
+        return if data.nil? || data.empty?
+
+        open_data(digest_for(key)) do |x|
+          x.write(data)
+        end
+      end
+
+      private
+
+      def digest_for(components)
+        Digest::SHA1.hexdigest(Array(components).join('/'))
+      end
+
+      def open_data(key, mode: 'w')
+        FileUtils.mkdir_p(data_dir_for(key))
+        File.open(data_file_for(key), mode) do |file|
+          yield file
+        end
+      end
+
+      def data_dir_for(index_key)
+        File.join(directory, *index_key.scan(/../)).downcase
+      end
+
+      def data_file_for(key)
+        File.join(data_dir_for(key), 'data')
+      end
+
+      def upsert!(spec)
+        return unless spec['licenseExpression']
+
+        write([host, spec['id'], spec['version']], spec['licenseExpression'])
+      end
+    end
+  end
+end