spandx 0.11.0 → 0.12.0

data/lib/spandx/core/table.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Core
+    class Table
+      def initialize
+        @rows = []
+        @max_justification = 0
+        yield self
+      end
+
+      def <<(item)
+        row = item.to_a
+        new_max = row[0].size
+        @max_justification = new_max + 1 if new_max > @max_justification
+        @rows << row
+      end
+
+      def to_s
+        @rows.map do |row|
+          row.each.with_index.map do |cell, index|
+            justification = index.zero? ? @max_justification : 15
+            Array(cell).join(', ').ljust(justification, ' ')
+          end.join
+        end
+      end
+    end
+  end
+end

data/lib/spandx/dotnet/index.rb

@@ -4,10 +4,11 @@ module Spandx
   module Dotnet
     class Index
       DEFAULT_DIR = File.expand_path(File.join(Dir.home, '.local', 'share', 'spandx'))
-      attr_reader :directory
+      attr_reader :directory, :name
 
       def initialize(directory: DEFAULT_DIR)
         @directory = directory ? File.expand_path(directory) : DEFAULT_DIR
+        @name = 'nuget'
       end
 
       def licenses_for(name:, version:)
@@ -19,7 +20,8 @@ module Spandx
       end
 
       def update!(catalogue:, output: StringIO.new)
-        insert_latest(Spandx::Dotnet::NugetGateway.new(catalogue: catalogue)) do |page|
+        catalogue.version
+        insert_latest(Spandx::Dotnet::NugetGateway.new) do |page|
           output.puts "Checkpoint #{page}"
           checkpoint!(page)
         end
@@ -29,14 +31,17 @@ module Spandx
       private
 
       def files(pattern)
-        Dir.glob(pattern, base: directory).sort.each do |file|
+        Dir.glob(File.join(directory, pattern)).sort.each do |file|
           fullpath = File.join(directory, file)
-          yield fullpath unless File.directory?(fullpath)
+          next if File.directory?(fullpath)
+          next unless File.exist?(fullpath)
+
+          yield fullpath
         end
       end
 
       def sort_index!
-        files('**/*') do |path|
+        files('**/nuget') do |path|
           next if File.extname(path) == '.checkpoints'
 
           IO.write(path, IO.readlines(path).sort.join)

data/lib/spandx/dotnet/nuget_gateway.rb

@@ -5,23 +5,17 @@ module Spandx
     # https://api.nuget.org/v3-flatcontainer/#{name}/#{version}/#{name}.nuspec
     # https://api.nuget.org/v3-flatcontainer/#{package.name}/index.json
     # https://docs.microsoft.com/en-us/nuget/api/package-base-address-resource
-    class NugetGateway
-      attr_reader :host
-
-      def initialize(http: Spandx.http, catalogue:)
+    class NugetGateway < ::Spandx::Core::Gateway
+      def initialize(http: Spandx.http)
         @http = http
-        @guess = Core::Guess.new(catalogue)
-        @host = 'api.nuget.org'
       end
 
-      def licenses_for(name, version)
-        found = cache.licenses_for(name: name, version: version)
-        return found if found.any?
-
-        document = nuspec_for(name, version)
+      def licenses_for(dependency)
+        extract_licenses_from(nuspec_for(dependency.name, dependency.version))
+      end
 
-        extract_licenses_from(document) ||
-          guess_licenses_from(document)
+      def matches?(dependency)
+        dependency.package_manager == :nuget
       end
 
       def each(start_page: 0)
@@ -34,21 +28,17 @@ module Spandx
 
       private
 
-      attr_reader :http, :guess
-
-      def cache
-        @cache ||= ::Spandx::Core::Cache.new(:nuget, url: 'https://github.com/mokhan/spandx-index.git')
-      end
+      attr_reader :http
 
       def each_page(start_page:)
-        url = "https://#{host}/v3/catalog0/index.json"
+        url = 'https://api.nuget.org/v3/catalog0/index.json'
         items_from(fetch_json(url))
           .find_all { |page| page_number_from(page['@id']) >= start_page }
           .each { |page| yield fetch_json(page['@id']) }
       end
 
       def nuspec_url_for(name, version)
-        "https://#{host}/v3-flatcontainer/#{name}/#{version}/#{name}.nuspec"
+        "https://api.nuget.org/v3-flatcontainer/#{name}/#{version}/#{name}.nuspec"
       end
 
       def nuspec_for(name, version)
@@ -62,20 +52,19 @@ module Spandx
       # TODO: Fix parsing https://github.com/NuGet/Home/wiki/Packaging-License-within-the-nupkg#license
       def extract_licenses_from(document)
         licenses = document.search('//package/metadata/license')
-        licenses.map(&:text) if licenses.any?
-      end
-
-      def guess_licenses_from(document)
-        document
-          .search('//package/metadata/licenseUrl')
-          .map { |node| guess_license_for(node.text) }
-          .compact
+        if licenses.any?
+          licenses.map(&:text)
+        else
+          document
+            .search('//package/metadata/licenseUrl')
+            .map { |node| download_license(node.text) }
+            .compact
+        end
       end
 
-      def guess_license_for(url)
+      def download_license(url)
         response = http.get(url)
-
-        guess.license_for(response.body) if http.ok?(response)
+        http.ok?(response) ? response.body : url
       end
 
       def fetch_json(url)

data/lib/spandx/dotnet/parsers/csproj.rb

@@ -4,7 +4,7 @@ module Spandx
   module Dotnet
     module Parsers
       class Csproj < ::Spandx::Core::Parser
-        def self.matches?(filename)
+        def matches?(filename)
           ['.csproj', '.props'].include?(File.extname(filename))
         end
 
@@ -19,21 +19,12 @@ module Spandx
 
         def map_from(package_reference)
           ::Spandx::Core::Dependency.new(
+            package_manager: :nuget,
             name: package_reference.name,
             version: package_reference.version,
-            licenses: licenses_for(package_reference)
+            meta: package_reference
           )
         end
-
-        def licenses_for(package_reference)
-          nuget
-            .licenses_for(package_reference.name, package_reference.version)
-            .map { |x| catalogue[x] }
-        end
-
-        def nuget
-          @nuget ||= NugetGateway.new(catalogue: catalogue)
-        end
       end
     end
   end

data/lib/spandx/dotnet/parsers/packages_config.rb

@@ -4,7 +4,7 @@ module Spandx
   module Dotnet
     module Parsers
       class PackagesConfig < ::Spandx::Core::Parser
-        def self.matches?(filename)
+        def matches?(filename)
           filename.match?(/packages\.config/)
         end
 
@@ -19,20 +19,12 @@ module Spandx
         def map_from(node)
           name = attribute_for('id', node)
           version = attribute_for('version', node)
-          ::Spandx::Core::Dependency.new(
-            name: name,
-            version: version,
-            licenses: nuget.licenses_for(name, version).map { |x| catalogue[x] }
-          )
+          ::Spandx::Core::Dependency.new(package_manager: :nuget, name: name, version: version)
         end
 
         def attribute_for(key, node)
           node.attribute(key)&.value&.strip || node.at_xpath("./#{key}")&.content&.strip
         end
-
-        def nuget
-          @nuget ||= NugetGateway.new(catalogue: catalogue)
-        end
       end
     end
   end

data/lib/spandx/dotnet/parsers/sln.rb

@@ -4,14 +4,14 @@ module Spandx
   module Dotnet
     module Parsers
       class Sln < ::Spandx::Core::Parser
-        def self.matches?(filename)
+        def matches?(filename)
           filename.match?(/.*\.sln/)
         end
 
         def parse(file_path)
           project_paths_from(file_path).map do |path|
             ::Spandx::Core::Parser
-              .for(path, catalogue: catalogue)
+              .for(path)
               .parse(path)
           end.flatten
         end

data/lib/spandx/java/gateway.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Java
+    class Gateway < ::Spandx::Core::Gateway
+      DEFAULT_SOURCE = 'https://repo.maven.apache.org/maven2'
+
+      attr_reader :http
+
+      def initialize(http: Spandx.http)
+        @http = http
+      end
+
+      def matches?(dependency)
+        dependency.package_manager == :maven
+      end
+
+      def licenses_for(dependency)
+        group_id, artifact_id = dependency.name.split(':')
+        metadata_for(
+          group_id: group_id,
+          artifact_id: artifact_id,
+          version: dependency.version
+        ).licenses
+      end
+
+      def metadata_for(group_id:, artifact_id:, version:)
+        ::Spandx::Java::Metadata.new(
+          artifact_id: artifact_id,
+          group_id: group_id,
+          version: version,
+          source: DEFAULT_SOURCE
+        )
+      end
+    end
+  end
+end

data/lib/spandx/java/index.rb

@@ -1,13 +1,95 @@
 # frozen_string_literal: true
 
+require 'tempfile'
+
 module Spandx
   module Java
     class Index
-      def initialize(directory:)
+      include Enumerable
+
+      attr_reader :name, :source
+
+      def initialize(directory:, source: 'https://repo.maven.apache.org/maven2')
         @directory = directory
+        @source = source
+        @name = 'maven'
+      end
+
+      def update!(catalogue:, output:)
+        each do |metadata|
+          name = "#{metadata.group_id}:#{metadata.artifact_id}:#{metadata.version}"
+          output.puts [name, metadata.licenses_from(catalogue)].inspect
+        end
+      end
+
+      def each
+        each_index_url do |url|
+          each_record_from("#{source}/.index/#{url}") do |record|
+            group_id, artifact_id, version = record['u'].split('|')
+            yield Metadata.new(artifact_id: artifact_id, group_id: group_id, version: version)
+          end
+        end
+      end
+
+      private
+
+      def each_record(io, record = {})
+        until io.eof?
+          field_count = io.read(4).unpack1('N').to_i # read 4 bytes for field count
+          field_count.times do |_n|
+            io.read(1) # flags
+            key = read_key(io)
+            value = read_value(io)
+            record[key] = value
+          end
+          yield record
+        end
+      end
+
+      def read_key(io)
+        length = io.read(2).unpack1('n').to_i # unsigned 16 bit int
+        io.read(length)
+      end
+
+      def read_value(io)
+        length = io.read(4).unpack1('N').to_i
+        io.read(length)
       end
 
-      def update!(catalogue:, output:); end
+      def each_record_from(url)
+        stream_from(url) do |io|
+          # read version
+          io.read(1)
+          # read timestamp
+          io.read(8)
+          # read records
+          each_record(io) do |x|
+            yield x
+          end
+        end
+      end
+
+      def each_index_url
+        html = Nokogiri::HTML(http.get("#{source}/.index/").body)
+        html.css('a[href*="nexus-maven-repository-index"]').each do |anchor|
+          url = anchor['href']
+          yield url if url.match(/\d+\.gz$/)
+        end
+      end
+
+      def stream_from(url, path: Tempfile.new.path)
+        return unless system("curl --progress-bar \"#{url}\" > #{path}", exception: true)
+
+        Zlib::GzipReader.open(path) do |gz|
+          yield gz
+        end
+      ensure
+        File.unlink(path) if File.exist?(path)
+      end
+
+      def http
+        Spandx.http
+      end
     end
   end
 end

data/lib/spandx/java/metadata.rb

@@ -3,15 +3,18 @@
 module Spandx
   module Java
     class Metadata
-      attr_reader :artifact_id, :group_id, :version
+      attr_reader :artifact_id, :group_id, :version, :source
 
-      def initialize(artifact_id:, group_id:, version:)
+      def initialize(artifact_id:, group_id:, version:, source: 'https://repo.maven.apache.org/maven2')
         @artifact_id = artifact_id
         @group_id = group_id.tr('.', '/')
         @version = version
+        @source = source
       end
 
       def licenses
+        return [] unless pom
+
         pom.search('//licenses/license').map do |node|
           {
             name: node.at_xpath('./name').text,
@@ -28,7 +31,7 @@ module Spandx
 
       def spec_url
         [
-          'https://repo.maven.apache.org/maven2',
+          source,
           group_id,
           artifact_id,
           version,

data/lib/spandx/java/parsers/maven.rb

@@ -4,39 +4,29 @@ module Spandx
   module Java
     module Parsers
       class Maven < ::Spandx::Core::Parser
-        def self.matches?(filename)
+        def matches?(filename)
           File.basename(filename) == 'pom.xml'
         end
 
         def parse(filename)
           document = Nokogiri.XML(IO.read(filename)).tap(&:remove_namespaces!)
           document.search('//project/dependencies/dependency').map do |node|
-            metadata = metadata_for(node)
-            ::Spandx::Core::Dependency.new(
-              name: metadata.artifact_id,
-              version: metadata.version,
-              licenses: metadata.licenses.map { |x| search_catalogue_for(x) }.compact
-            )
+            map_from(node)
           end
         end
 
         private
 
-        def metadata_for(node)
-          ::Spandx::Java::Metadata.new(
-            artifact_id: node.at_xpath('./artifactId').text,
-            group_id: node.at_xpath('./groupId').text,
-            version: node.at_xpath('./version').text
-          )
-        end
-
-        def search_catalogue_for(license_hash)
-          name = ::Spandx::Core::Content.new(license_hash[:name])
+        def map_from(node)
+          artifact_id = node.at_xpath('./artifactId').text
+          group_id = node.at_xpath('./groupId').text
+          version = node.at_xpath('./version').text
 
-          catalogue.find do |license|
-            score = name.similarity_score(::Spandx::Core::Content.new(license.name))
-            score > 85
-          end
+          ::Spandx::Core::Dependency.new(
+            package_manager: :maven,
+            name: "#{group_id}:#{artifact_id}",
+            version: version
+          )
         end
       end
     end