rudisco 1.0.0

data/sources/rudisco/helpers.rb

@@ -0,0 +1,74 @@
+# encoding: utf-8
+# frozen_string_literal: true
+module Rudisco
+  module Helpers
+    ##
+    # Opens +url+ in a browser.
+    #
+    # @param [String] url
+
+    def self.open_in_browser(url)
+      return if url.to_s.empty?
+
+      if defined?(Launchy) && Launchy.respond_to?(:open)
+        Launchy.open url
+      else
+        if RbConfig::CONFIG['host_os'] =~ /mswin|mingw|cygwin/
+          system "start #{url}"
+        elsif RbConfig::CONFIG['host_os'] =~ /darwin/
+          system "open #{url}"
+        elsif RbConfig::CONFIG['host_os'] =~ /linux|bsd/
+          system "xdg-open #{url}"
+        end
+      end
+    end
+
+    ##
+    # Downloads file from +url+ to +path+.
+    #
+    # @param [String] url
+    #
+    # @param [String, Pathname] path
+    #
+    # @exception NotAUrl
+    #   Raised when +url+ not a string or empty.
+    # @exception DirNotExists
+    #   Raised when +path+ is not a path to directory.
+
+    def self.download(url, path)
+      raise NotAUrl, url unless url.is_a?(String) && !url.empty?
+      raise DirNotExists, path unless Dir.exists? path
+
+      system "wget -P #{path} #{url} > /dev/null 2>&1"
+    end
+
+    ##
+    # Clones git project from +url+ to +path+.
+    #
+    # @param [String] url
+    #
+    # @param [String, Pathname] path
+    #
+    # @exception NotAUrl
+    #   Raised when +url+ not a string or empty.
+    # @exception DirShouldNotExists
+    #   Raised when git can not clone project, since directory
+    #   where it should be saved already exists.
+
+    def self.git_clone(url, path)
+      raise NotAUrl, url unless url.is_a?(String) && !url.empty?
+      raise DirShouldNotExists, path if Dir.exists? path
+
+      system "git clone #{url} #{path} > /dev/null 2>&1"
+    end
+
+    class NotAUrl < Error # no-doc
+      def initialize(url); super "Url '#{url}' is not a string or empty!" end; end
+
+    class DirNotExists < Error # no-doc
+      def initialize(dir); super "Directory '#{dir}' not exists!" end; end
+
+    class DirShouldNotExists < Error # no-doc
+      def initialize(dir); super "Directory '#{dir}' should not exist!" end; end
+  end # module Helpers
+end # module Rudisco

data/sources/rudisco/models/gem/actions.rb

@@ -0,0 +1,116 @@
+# encoding: utf-8
+# frozen_string_literal: true
+module Rudisco
+  class GemActions
+    # @param [Rudisco::Gem] gem
+    #
+    # @return [Rudisco::GemActions]
+
+    def initialize(gem)
+      @gem = gem
+    end
+
+    ##
+    # Creates an instance of GemActions class.
+    #
+    # @param [Symbol] action
+    #   Described in Gem#action
+    #
+    # @param [Hash] params
+    #   Additional parameters. Optional.
+    #
+    # @exception Rudisco::GemActions::ActionUnknown
+    #   Raised when action is unknown.
+
+    def complete(action, params = {})
+      if respond_to? action
+        send action, params
+      else
+        raise Unknown, action
+      end
+    end
+
+    # ----------------------------------------------------
+    # ------------- External actions methods -------------
+    # ----------------------------------------------------
+
+    ##
+    # Updates gems.
+    #
+    # Used to get latest information about gem. This need because some
+    # db.attributes getting update even when gem version not changed.
+
+    def update(params = {})
+      Gem.corteges_scanning gem
+    end
+
+    ##
+    # Opens in browser documentation page
+
+    def open_documentation(params = {})
+      Helpers.open_in_browser gem.documentation_url
+    end
+
+    ##
+    # Opens in browser bug tracker page
+
+    def open_bug_tracker(params = {})
+      Helpers.open_in_browser gem.bug_tracker_url
+    end
+
+    ##
+    # Opens in browser code sources page.
+
+    def open_sources(params = {})
+      Helpers.open_in_browser gem.source_code_url
+    end
+
+    ##
+    # Opens in browser wiki page.
+
+    def open_wiki(params = {})
+      Helpers.open_in_browser gem.wiki_url
+    end
+
+    ##
+    # Opens in browser gem page on rubygems.org
+
+    def open_rubygems(params = {})
+      Helpers.open_in_browser gem.project_url
+    end
+
+    ##
+    # Downloads gem.
+    #
+    # @param [Hash] params
+    # @option params [String] :path (ENV['HOME'])
+
+    def download(params = {})
+      path = params[:path] || ENV['HOME']
+
+      Helpers::download gem.gem_url, path
+    end
+
+    ##
+    # Clones gem from git.
+    #
+    # @param [Hash] params
+    # @option params [String] :path (ENV['HOME'])
+
+    def git_clone(params = {})
+      path = params[:path] || ENV['HOME']
+      path = File.join(path, gem.name)
+
+      Helpers::git_clone gem.source_code_url, path
+    end
+
+    private
+
+    # @return [Rudisco::Gem]
+
+    attr_reader :gem
+
+    class Unknown < Error # no-doc
+      def initialize(action); super "Action '#{action}' is unknown!" end; end
+  end # class GemActions
+end # module Rudisco

data/sources/rudisco/models/gem/dataset_methods.rb

@@ -0,0 +1,28 @@
+# encoding: utf-8
+# frozen_string_literal: true
+module Sequel::Plugins::GemExtendedDataset
+  module DatasetMethods
+    # @see Rudisco::Gem#action
+    #
+    # @return [Gem::Dataset]
+
+    def action(command, params = {})
+      each { |cortege| cortege.action command, params }
+
+      return self
+    end
+
+    # @see Rudisco::Gem#find_phrase
+    #
+    # @return [Gem::Dataset]
+
+    def find_phrase(word)
+      search_filter =
+        Sequel.ilike(:name, "%#{word}%") || Sequel.ilike(:description, "%#{word}%")
+
+      dataset = where search_filter
+
+      return dataset
+    end
+  end # module DatasetMethods
+end # Sequel::Plugins::GemExtendedDataset

data/sources/rudisco/models/gem/rubygems_scanner.rb

@@ -0,0 +1,195 @@
+# encoding: utf-8
+# frozen_string_literal: true
+module Rudisco
+  module RubyGemsScanner # no-doc
+    ##
+    # Scans rubygems.org for new gems. Also marks gems as outdated when
+    # new gem version is out.
+    #
+    # To load an actual information for newest gems use #deep_scanning.
+
+    def surface_scanning
+      _surface_scanning
+    end
+
+    ##
+    # Long-term task. Updates database with an actual information. Multithread.
+    #
+    # Note: speed limited by rubygems.org.
+    #
+    #   Can be increased with advanced concurrent processes setup for
+    #   sqlite database.
+    #
+    #   However, this is slow since rubygems.org allowing certain requests
+    #   per second count.
+    #
+    # @param [Proc<Integer>] callback
+    #   Returns count of updated gems, optional.
+
+    def deep_scanning(&callback)
+      _deep_scanning &callback
+    end
+
+    ##
+    # Updates cortege(s).
+    #
+    # Used to get latest information about gem(s) total downloads
+    # count and so on.
+    #
+    # @param [Rudisco::Gem, Array<Rudisco::Gem>] corteges
+
+    def corteges_scanning(corteges)
+      rubygems_manage_corteges Array(corteges)
+    end
+
+    private
+
+    def _surface_scanning # no-doc
+      rubygems =
+        begin
+          tmp_file = Tempfile.new 'gems.tmp'
+
+          system "gem list --remote > #{tmp_file.path}"
+          rubygems = CSV.open tmp_file.path
+          rubygems = rubygems.map { |r| r.join.delete('()').split }
+          tmp_file.unlink
+
+          rubygems
+        end
+      sqlite_gems = select_hash :name, :version
+
+      ## ADD NEW GEMS TO DATABASE
+      db.transaction do
+        new_gems = rubygems.select { |gem| sqlite_gems[gem[0]].nil? }
+
+        new_gems.each do |gem|
+          insert name: gem[0], version: gem[1]
+        end
+      end
+
+      ## UPDATE RECORDS IF NEED
+      db.transaction do
+        update_gems = rubygems.reject { |gem| sqlite_gems[gem[0]].nil? }
+                              .select { |gem| sqlite_gems[gem[0]] != gem[1] }
+
+        update_gems.each do |gem|
+          cortege = where(:name => gem[0]).first
+          next if cortege[:version] > gem[1] # skip downgraded gems
+
+          cortege[:version] = gem[1]
+          cortege[:need_update] = true
+          cortege.save
+        end
+      end
+    end
+
+    def _deep_scanning(&callback) # no-doc
+      _surface_scanning # todo this call should be optional
+
+      bunch =
+        begin
+          gem_update_count   = where(need_update: true).count
+          threads_count      = gem_update_count > 105 ? 35 : 1
+          bunch_sub_arr_size = gem_update_count / threads_count + 1
+
+          bunch_tmp =
+            Array.new(threads_count) do |i|
+              self.where(need_update: true)
+                  .order(:id)
+                  .limit(bunch_sub_arr_size).offset(i * bunch_sub_arr_size)
+            end
+
+          bunch_tmp
+        end # bunch = [ [], [], [], [], ... [] ]
+
+      threads = []
+      bunch.count.times do |i|
+        threads << Thread.new(bunch[i], callback) do |sub_array, callback_proc|
+          sub_array.each_slice(20) do |gems|
+            db.transaction { rubygems_manage_corteges gems }
+            callback_proc.call gems.count unless callback_proc.nil?
+          end
+        end
+       end
+      threads.each &:join
+    end
+
+    # @param [String] name
+    #   Gem name.
+    #
+    # @return [Nil, String]
+    #   Returns +nil+ when rubygems.org not responded, otherwise it
+    #   returns encoded json as a string.
+
+    def send_request_to_rubygems(name)
+      for try_count in 1...25 do
+        begin
+          sleep try_count * 0.35
+          url = URI.parse "https://rubygems.org/api/v1/gems/#{name}.json"
+          response = Net::HTTP.get_response(url)
+
+          return response.body unless response.body.empty?
+        rescue Errno::ECONNRESET
+          next
+        end
+      end
+
+      return nil
+    end
+
+    ##
+    # Updates +cortege+ with a data from +hsh+.
+    #
+    # @param [Rudisco::Gem] cortege
+    #
+    # @param [Hash] hsh
+
+    def update_cortege(cortege, hsh)
+      cortege[:description] = hsh["info"].to_s
+      cortege[:authors]     = hsh["authors"].to_s
+      cortege[:license]     = Array(hsh["licenses"]).join
+      cortege[:sha]         = hsh["sha"].to_s
+
+      # "$ gem -list" and rubygems.org/api can send different information about
+      # last version for a gem. To prevent collision this code not downgrades
+      # gem version.
+      #
+      # In other words "$ gem list" have higher priority under rubygems.org/api
+      if hsh["version"].to_s > cortege[:version]
+        cortege[:version] = hsh["version"].to_s
+      end
+
+      cortege[:source_code_url]   = hsh["source_code_uri"].to_s
+      cortege[:project_url]       = hsh["project_uri"].to_s
+      cortege[:gem_url]           = hsh["gem_uri"].to_s
+      cortege[:wiki_url]          = hsh["wiki_uri"].to_s
+      cortege[:documentation_url] = hsh["documentation_uri"].to_s
+      cortege[:mailing_list_url]  = hsh["mailing_list_uri"].to_s
+      cortege[:bug_tracker_url]   = hsh["bug_tracker_uri"].to_s
+
+      cortege[:total_downloads]   = hsh["downloads"].to_i
+      cortege[:version_downloads] = hsh["version_downloads"].to_i
+
+      cortege[:need_update] = false
+    end
+
+    # @param [Array<Rudisco::Gem>] corteges
+
+    def rubygems_manage_corteges(corteges)
+      corteges.each do |cortege|
+        response = send_request_to_rubygems cortege[:name]
+
+        if response.nil?
+          next
+        elsif response =~ /could not be found/
+          cortege.destroy # gem was deleted from rubygems.org
+        else
+          data = JSON.parse response
+          update_cortege cortege, data
+
+          cortege.save
+        end
+      end
+    end
+  end # module RubyGemsScanner
+end # module Rudisco

data/sources/rudisco/models/gem.rb

@@ -0,0 +1,95 @@
+# encoding: utf-8
+# frozen_string_literal: true
+module Rudisco
+
+  ##
+  # == Table +gems+ structure:
+  #
+  # CREATE TABLE `gems` (
+  # 	`id`	integer NOT NULL PRIMARY KEY AUTOINCREMENT,
+  #
+  # 	`name`	      Text,
+  # 	`description`	Text,
+  # 	`authors`	    Text,
+  # 	`version`	    Text,
+  # 	`license`	    Text,
+  # 	`sha`	        Text,
+  #
+  # 	`source_code_url`	  Text,
+  # 	`project_url`	      Text,
+  # 	`gem_url`	          Text,
+  # 	`wiki_url`	        Text,
+  # 	`documentation_url`	Text,
+  # 	`mailing_list_url`	Text,
+  # 	`bug_tracker_url`	  Text,
+  #
+  # 	`total_downloads`	  Integer,
+  # 	`version_downloads`	Integer,
+  #
+  # 	`need_update`	Boolean DEFAULT (1)
+  # );
+
+  class Gem < Sequel::Model
+    extend RubyGemsScanner
+    plugin :gem_extended_dataset
+
+    ##
+    # Case insensitive phrase search in :description, :name columns.
+    #
+    # @example
+    #   phrase = 'rails'
+    #   gems = Rudisco::Gem.find_phrase(phrase)
+    #                      .select(:name, :description)
+    #
+    #   gems.each do |record|
+    #     puts record[:name]
+    #     puts record[:description]
+    #     puts
+    #   end
+    #
+    # @param [String] phrase
+    #
+    # @return [Gem::Dataset]
+
+    def self.find_phrase(phrase)
+      result = from(:gems).find_phrase phrase
+
+      return result
+    end
+
+    ##
+    # Provides external actions for specified cortege.
+    #
+    # @example
+    #   path_to_load = File.join(__dir__, '..', 'tmp')
+    #
+    #   sample = Rudisco::Gem.exclude(source_code_url: '').first
+    #   sample.action(:open_sources)
+    #         .action(:git_clone, path: path_to_load)
+    #
+    #   sample2 = Rudisco::Gem.limit(2)
+    #   sample2.action(:download, path: path_to_load)
+    #
+    # @param [Symbol] command
+    #   Expecting +command+ values:
+    #     :update             @see GemActions#update
+    #     :open_documentation @see GemActions#open_documentation
+    #     :open_bug_tracker   @see GemActions#open_bug_tracker
+    #     :open_sources       @see GemActions#open_sources
+    #     :open_wiki          @see GemActions#open_wiki
+    #     :open_rubygems      @see GemActions#open_rubygmes
+    #     :download           @see GemActions#download
+    #     :git_clone          @see GemActions#git_clone
+    #
+    # @param [Hash] params
+    #   Additional parameters. Optional.
+    #
+    # @return [Rudisco::Gem]
+
+    def action(command, params = {})
+      GemActions.new(self).complete command, params
+
+      return self
+    end
+  end # class Gem
+end # module Rudisco

data/sources/rudisco/project_structure.rb

@@ -0,0 +1,55 @@
+# encoding: utf-8
+# frozen_string_literal: true
+module Rudisco
+  # === Dependencies from core lib
+
+  require 'csv'
+  require 'net/http'
+  require 'uri'
+  require 'json'
+  require 'tempfile'
+
+  # === Dependencies from rubygems.org
+
+  require 'sqlite3'
+  require 'sequel'
+  require 'thor'
+  require 'command_line_reporter'
+
+  # === Project structure
+
+  ##
+  # Ancestor class for Rudisco's exceptions.
+
+  class Error < StandardError; end
+
+  ##
+  # Loads *.rb files in requested order.
+
+  def self.load(**params)
+    params[:files].each do |f|
+      require File.join(__dir__, params[:folder].to_s, f)
+    end
+  end
+  private_class_method :load
+
+  # === Core
+
+  load files: %w(helpers sqlite)
+
+  load folder: 'models/gem',
+       files: %w(rubygems_scanner actions dataset_methods)
+
+  load folder: 'models',
+       files: %w(gem)
+
+  # === Command-Line-Interface
+
+  load folder: 'cli',
+       files: %w(presentation)
+
+  load folder: 'cli/presentation',
+       files: %w(find show update download git_clone open)
+
+  load files: %w(cli/cli) # cli/routes
+end # module Rudisco

data/sources/rudisco/sqlite.rb

@@ -0,0 +1,12 @@
+# encoding: utf-8
+# frozen_string_literal: true
+module Rudisco
+  path_to_database =
+    File.join(__dir__, '../../sources/database/rudisco.db')
+
+  Sequel.connect "sqlite://#{path_to_database}",
+                 max_connections: 40,
+                 pool_sleep_time: 0.015,
+                 single_threaded: true,
+                 pool_timeout: 60000
+end # module Rudisco

data/sources/rudisco.rb

@@ -0,0 +1,13 @@
+# encoding: utf-8
+# frozen_string_literal: true
+#
+# Rubygems database (https://rubygems.org)
+#
+# @see https://github.com/Medvedu/rudisco
+# @see readme.md
+#
+# @author Kuzichev Michael
+# @license MIT
+module Rudisco
+  require_relative 'rudisco/project_structure'
+end # module Rudisco

data/spec/rudisco/models/gem_spec.rb

@@ -0,0 +1,40 @@
+# encoding: utf-8
+require 'spec_helper'
+
+module Rudisco
+  describe Gem do
+    it 'includes certain columns' do
+      expect(described_class.columns)
+        .to include(:id, :name, :description, :source_code_url)
+    end
+
+    it 'works as Sequel ORM' do
+      expect(described_class.where {total_downloads > 10000000 }.count)
+        .to be > 0
+    end
+
+    # ----------------------------------------------------
+
+    describe "#find_phrase" do
+      it 'case insensitive searches in :description, :name columns' do
+        gems = described_class.find_phrase 'monkey'
+
+        gems.each do |gem|
+          expect((gem.name =~ /monkey/i) || (gem.description =~ /monkey/i))
+            .to_not be_nil
+        end
+      end
+    end # describe "#find_phrase"
+
+    # ----------------------------------------------------
+
+    describe "#action" do
+      it 'raises an exception when action is unknown' do
+        sample = described_class.first
+
+        expect{ sample.action :misspelled_action }
+          .to raise_exception GemActions::Unknown
+      end
+    end # describe "#action"
+  end # describe Gem
+end # module Rudisco

data/spec/spec_helper.rb

@@ -0,0 +1,5 @@
+# encoding: utf-8
+require_relative '../sources/rudisco'
+
+RSpec.configure do |config|
+end