spandx 0.11.0 → 0.12.0

data/lib/spandx/js/parsers/npm.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Js
+    module Parsers
+      class Npm < ::Spandx::Core::Parser
+        def matches?(filename)
+          File.basename(filename) == 'package-lock.json'
+        end
+
+        def parse(file_path)
+          items = Set.new
+          each_metadata(file_path) do |metadata|
+            items.add(map_from(metadata))
+          end
+          items
+        end
+
+        private
+
+        def each_metadata(file_path)
+          package_lock = JSON.parse(IO.read(file_path))
+          package_lock['dependencies'].each do |name, metadata|
+            yield metadata.merge('name' => name)
+          end
+        end
+
+        def map_from(metadata)
+          Spandx::Core::Dependency.new(
+            package_manager: :npm,
+            name: metadata['name'],
+            version: metadata['version'],
+            meta: metadata
+          )
+        end
+      end
+    end
+  end
+end

data/lib/spandx/js/parsers/yarn.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Js
+    module Parsers
+      class Yarn < ::Spandx::Core::Parser
+        def matches?(filename)
+          File.basename(filename) == 'yarn.lock'
+        end
+
+        def parse(file_path)
+          YarnLock.new(file_path).each_with_object(Set.new) do |metadata, memo|
+            memo << map_from(metadata)
+          end
+        end
+
+        private
+
+        def map_from(metadata)
+          ::Spandx::Core::Dependency.new(
+            package_manager: :yarn,
+            name: metadata['name'],
+            version: metadata['version'],
+            meta: metadata
+          )
+        end
+      end
+    end
+  end
+end

data/lib/spandx/js/yarn_lock.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Js
+    class YarnLock
+      include Enumerable
+
+      START_OF_DEPENDENCY_REGEX = %r{^"?(?<name>(@|\w|-|\.|/)+)@}i.freeze
+      INJECT_COLON = /(?<=\w|")\s(?=\w|")/.freeze
+
+      attr_reader :file_path
+
+      def initialize(file_path)
+        @file_path = file_path
+        @metadatum = collect_metadatum
+      end
+
+      def each
+        @metadatum.each do |metadata|
+          yield metadata
+        end
+      end
+
+      private
+
+      def collect_metadatum
+        items = Set.new
+        File.open(file_path, 'r') do |io|
+          until io.eof?
+            metadata = map_from(io)
+            next if metadata.nil? || metadata.empty?
+
+            items << metadata
+          end
+        end
+        items
+      end
+
+      def map_from(io)
+        header = io.readline
+        return unless (matches = header.match(START_OF_DEPENDENCY_REGEX))
+
+        metadata_from(matches[:name].gsub(/"/, ''), io)
+      end
+
+      def metadata_from(name, io)
+        YAML.safe_load(to_yaml(name, read_lines(io)))
+      end
+
+      def to_yaml(name, lines)
+        (["name: \"#{name}\""] + lines)
+          .map { |x| x.sub(INJECT_COLON, ': ') }
+          .join("\n")
+      end
+
+      def read_lines(io)
+        [].tap do |lines|
+          line = io.readline.strip
+          until line.empty? || io.eof?
+            lines << line
+            line = io.readline.strip
+          end
+        end
+      end
+    end
+  end
+end

data/lib/spandx/js/yarn_pkg.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Js
+    class YarnPkg < ::Spandx::Core::Gateway
+      DEFAULT_SOURCE = 'https://registry.yarnpkg.com'
+      attr_reader :http
+
+      def initialize(http: Spandx.http)
+        @http = http
+      end
+
+      def matches?(dependency)
+        %i[npm yarn].include?(dependency.package_manager)
+      end
+
+      def licenses_for(dependency)
+        metadata = metadata_for(dependency)
+
+        return [] if metadata.empty?
+
+        [metadata['license']].compact
+      end
+
+      def metadata_for(dependency)
+        uri = uri_for(dependency)
+        response = http.get(uri, escape: false)
+
+        if http.ok?(response)
+          json = JSON.parse(response.body)
+          json['versions'] ? json['versions'][dependency.version] : json
+        else
+          {}
+        end
+      end
+
+      private
+
+      def uri_for(dependency)
+        URI.parse(source_for(dependency)).tap do |uri|
+          uri.path = if dependency.name.include?('/')
+                       '/' + dependency.name.sub('/', '%2f')
+                     else
+                       '/' + dependency.name + '/' + dependency.version
+                     end
+        end
+      end
+
+      def source_for(dependency)
+        if dependency.meta['resolved']
+          uri = URI.parse(dependency.meta['resolved'])
+          "#{uri.scheme}://#{uri.host}:#{uri.port}"
+        else
+          DEFAULT_SOURCE
+        end
+      end
+    end
+  end
+end

data/lib/spandx/php/packagist_gateway.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Php
+    class PackagistGateway < ::Spandx::Core::Gateway
+      attr_reader :http
+
+      def initialize(http: Spandx.http)
+        @http = http
+      end
+
+      def matches?(dependency)
+        dependency.package_manager == :composer
+      end
+
+      def licenses_for(dependency)
+        response = http.get("https://repo.packagist.org/p/#{dependency.name}.json")
+        return [] unless http.ok?(response)
+
+        json = JSON.parse(response.body)
+        json['packages'][dependency.name][dependency.version]['license']
+      end
+    end
+  end
+end

data/lib/spandx/php/parsers/composer.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Php
+    module Parsers
+      class Composer < ::Spandx::Core::Parser
+        def matches?(filename)
+          File.basename(filename) == 'composer.lock'
+        end
+
+        def parse(file_path)
+          items = Set.new
+          composer_lock = JSON.parse(IO.read(file_path))
+          composer_lock['packages'].concat(composer_lock['packages-dev']).each do |dependency|
+            items.add(map_from(dependency))
+          end
+          items
+        end
+
+        private
+
+        def map_from(dependency)
+          Spandx::Core::Dependency.new(
+            package_manager: :composer,
+            name: dependency['name'],
+            version: dependency['version'],
+            meta: dependency
+          )
+        end
+      end
+    end
+  end
+end

data/lib/spandx/python/index.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Python
+    class Index
+      include Enumerable
+
+      attr_reader :directory, :name, :pypi, :source
+
+      def initialize(directory:)
+        @directory = directory
+        @name = 'pypi'
+        @source = 'https://pypi.org'
+        @pypi = Pypi.new
+        Thread.abort_on_exception = true
+      end
+
+      def update!(*)
+        queue = Queue.new
+        [fetch(queue), save(queue)].each(&:join)
+      end
+
+      private
+
+      def files(pattern)
+        Dir.glob(pattern, base: directory).sort.each do |file|
+          fullpath = File.join(directory, file)
+          yield fullpath unless File.directory?(fullpath)
+        end
+      end
+
+      def sort_index!
+        files('**/pypi') do |path|
+          IO.write(path, IO.readlines(path).sort.join)
+        end
+      end
+
+      def fetch(queue)
+        Thread.new do
+          pypi.each do |dependency|
+            queue.enq(dependency)
+          end
+          queue.enq(:stop)
+        end
+      end
+
+      def save(queue)
+        Thread.new do
+          loop do
+            item = queue.deq
+            break if item == :stop
+
+            insert!(item[:name], item[:version], item[:license])
+          end
+        end
+      end
+
+      def digest_for(components)
+        Digest::SHA1.hexdigest(Array(components).join('/'))
+      end
+
+      def data_dir_for(name)
+        File.join(directory, digest_for(name)[0...2].downcase)
+      end
+
+      def data_file_for(name)
+        File.join(data_dir_for(name), 'pypi')
+      end
+
+      def insert!(name, version, license)
+        return if license.nil? || license.empty?
+
+        csv = CSV.generate_line([name, version, license], force_quotes: true)
+        IO.write(data_file_for(name), csv, mode: 'a')
+      end
+    end
+  end
+end

data/lib/spandx/python/parsers/pipfile_lock.rb

@@ -4,18 +4,14 @@ module Spandx
   module Python
     module Parsers
       class PipfileLock < ::Spandx::Core::Parser
-        def self.matches?(filename)
+        def matches?(filename)
           filename.match?(/Pipfile.*\.lock/)
         end
 
         def parse(lockfile)
           results = []
-          dependencies_from(lockfile) do |x|
-            results << ::Spandx::Core::Dependency.new(
-              name: x[:name],
-              version: x[:version],
-              licenses: x[:licenses]
-            )
+          dependencies_from(lockfile) do |dependency|
+            results << dependency
           end
           results
         end
@@ -24,16 +20,20 @@ module Spandx
 
         def dependencies_from(lockfile)
           json = JSON.parse(IO.read(lockfile))
-          each_dependency(pypi_for(json), json) do |name, version, definition|
-            yield({ name: name, version: version, licenses: [catalogue[definition['license']]] })
+          each_dependency(json) do |name, version|
+            yield ::Spandx::Core::Dependency.new(
+              package_manager: :pypi,
+              name: name,
+              version: version,
+              meta: json
+            )
           end
         end
 
-        def each_dependency(pypi, json, groups: %w[default develop])
+        def each_dependency(json, groups: %w[default develop])
           groups.each do |group|
             json[group].each do |name, value|
-              version = canonicalize(value['version'])
-              yield name, version, pypi.definition_for(name, version)
+              yield name, canonicalize(value['version'])
             end
           end
         end
@@ -41,10 +41,6 @@ module Spandx
         def canonicalize(version)
           version.gsub(/==/, '')
         end
-
-        def pypi_for(json)
-          PyPI.new(sources: Source.sources_from(json))
-        end
       end
     end
   end

data/lib/spandx/python/pypi.rb

@@ -2,18 +2,101 @@
 
 module Spandx
   module Python
-    class PyPI
-      def initialize(sources: [Source.default])
-        @sources = sources
+    class Pypi < ::Spandx::Core::Gateway
+      SUBSTITUTIONS = [
+        '-py2.py3',
+        '-py2',
+        '-py3',
+        '-none-any.whl',
+        '.tar.gz',
+        '.zip',
+      ].freeze
+
+      def initialize(http: Spandx.http)
+        @http = http
+        @definitions = {}
+      end
+
+      def matches?(dependency)
+        dependency.package_manager == :pypi
+      end
+
+      def each(sources: default_sources)
+        each_package(sources) { |x| yield x }
+      end
+
+      def licenses_for(dependency)
+        definition = definition_for(
+          dependency.name,
+          dependency.version,
+          sources: sources_for(dependency)
+        )
+        [definition['license']]
       end
 
-      def definition_for(name, version)
-        @sources.each do |source|
-          response = source.lookup(name, version)
-          return JSON.parse(response.body).fetch('info', {}) if response
+      def definition_for(name, version, sources: default_sources)
+        @definitions.fetch([name, version]) do |key|
+          sources.each do |source|
+            response = source.lookup(name, version)
+            next if response.empty?
+
+            match = response.fetch('info', {})
+            @definitions[key] = match
+            return match
+          end
+          {}
         end
-        {}
+      end
+
+      def version_from(url)
+        path = SUBSTITUTIONS.inject(URI.parse(url).path.split('/')[-1]) do |memo, item|
+          memo.gsub(item, '')
+        end
+
+        return if path.rindex('-').nil?
+
+        path.scan(/-\d+\..*/)[-1][1..-1]
+      end
+
+      private
+
+      attr_reader :http
+
+      def sources_for(dependency)
+        return default_sources if dependency.meta.empty?
+
+        ::Spandx::Python::Source.sources_from(dependency.meta)
+      end
+
+      def default_sources
+        [Source.default]
+      end
+
+      def each_package(sources)
+        sources.each do |source|
+          html_from(source, '/simple/').css('a[href*="/simple"]').each do |node|
+            each_version(source, node[:href]) do |dependency|
+              definition = source.lookup(dependency[:name], dependency[:version])
+              yield dependency.merge(license: definition['license'])
+            end
+          end
+        end
+      end
+
+      def each_version(source, path)
+        html = html_from(source, path)
+        name = html.css('h1')[0].content.gsub('Links for ', '')
+        html.css('a').each do |node|
+          yield({ name: name, version: version_from(node[:href]) })
+        end
+      end
+
+      def html_from(source, path)
+        url = URI.join(source.uri.to_s, path).to_s
+        Nokogiri::HTML(http.get(url).body)
       end
     end
+
+    PyPI = Pypi
   end
 end