kimurai 1.0.0

data/lib/kimurai/cli/ansible_command_builder.rb

@@ -0,0 +1,71 @@
+require 'cliver'
+
+module Kimurai
+  class CLI
+    class AnsibleCommandBuilder
+      def initialize(user_host, options, playbook:, vars: {})
+        @user_host = user_host
+        @options = options
+        @playbook = playbook
+        @vars = vars
+      end
+
+      def get
+        unless Cliver.detect("ansible-playbook")
+          raise "Can't find `ansible-playbook` executable, to install: " \
+            "Mac OS X: `$ brew install ansible`, Ubuntu: `$ sudo apt install ansible`"
+        end
+
+        user = @user_host[/(.*?)\@/, 1]
+        host = @user_host[/\@(.+)/, 1] || @user_host
+        inventory = @options["port"] ? "#{host}:#{@options['port']}," : "#{host},"
+
+        gem_dir = Gem::Specification.find_by_name("kimurai").gem_dir
+        playbook_path = gem_dir + "/lib/kimurai/automation/" + "#{@playbook}.yml"
+
+        command = [
+          "ansible-playbook", playbook_path,
+          "--inventory", inventory,
+          "--ssh-extra-args", "-oForwardAgent=yes",
+          "--connection", @options["local"] ? "local" : "smart",
+          "--extra-vars", "ansible_python_interpreter=/usr/bin/python3"
+        ]
+
+        if File.exists? "config/automation.yml"
+          require 'yaml'
+          if config = YAML.load_file("config/automation.yml").dig(@playbook)
+            config.each { |key, value| @vars[key] = value unless @vars[key] }
+          end
+        end
+
+        @vars.each do |key, value|
+          next unless value.present?
+          command.push "--extra-vars", "#{key}=#{value}"
+        end
+
+        if user
+          command.push "--user", user
+        end
+
+        if @options["ask-sudo"]
+          command.push "--ask-become-pass"
+        end
+
+        if @options["ask-auth-pass"]
+          unless Cliver.detect("sshpass")
+            raise "Can't find `sshpass` executable for password authentication, to install: " \
+              "Mac OS X: `$ brew install http://git.io/sshpass.rb`, Ubuntu: `$ sudo apt install sshpass`"
+          end
+
+          command.push "--ask-pass"
+        end
+
+        if ssh_key_path = @options["ssh-key-path"]
+          command.push "--private-key", ssh_key_path
+        end
+
+        command
+      end
+    end
+  end
+end

data/lib/kimurai/cli/generator.rb

@@ -0,0 +1,57 @@
+module Kimurai
+  class CLI
+    class Generator < Thor::Group
+      include Thor::Actions
+
+      def self.source_root
+        File.dirname(File.expand_path('..', __FILE__))
+      end
+
+      def generate_project(project_name)
+        directory "template", project_name
+        inside(project_name) do
+          run "bundle install"
+          run "git init"
+        end
+      end
+
+      def generate_spider(spider_name, in_project:)
+        spider_path = in_project ? "spiders/#{spider_name}.rb" : "./#{spider_name}.rb"
+        raise "Spider #{spider_path} already exists" if File.exists? spider_path
+
+        spider_class = to_spider_class(spider_name)
+        create_file spider_path do
+          <<~RUBY
+            class #{spider_class} < #{in_project ? 'ApplicationSpider' : 'Kimurai::Base'}
+              @name = "#{spider_name}"
+              @start_urls = []
+              @config = {}
+
+              def parse(response, url:, data: {})
+              end
+            end
+          RUBY
+        end
+
+        unless in_project
+          insert_into_file spider_path, "  @engine = :mechanize\n", after: "@name = \"#{spider_name}\"\n"
+          prepend_to_file spider_path, "require 'kimurai'\n\n"
+          append_to_file spider_path, "\n#{spider_class}.crawl!"
+        end
+      end
+
+      def generate_schedule
+        copy_file "template/config/schedule.rb", "./schedule.rb"
+      end
+
+      private
+
+      def to_spider_class(string)
+        string.sub(/^./) { $&.capitalize }
+          .gsub(/(?:_|(\/))([a-z\d]*)/) { "#{$1}#{$2.capitalize}" }
+          .gsub(/(?:-|(\/))([a-z\d]*)/) { "Dash#{$2.capitalize}" }
+          .gsub(/(?:\.|(\/))([a-z\d]*)/) { "#{$1}#{$2.capitalize}" }
+      end
+    end
+  end
+end

data/lib/kimurai/core_ext/array.rb

@@ -0,0 +1,14 @@
+class Array
+  def in_sorted_groups(number, fill_width = nil)
+    sorted_groups = Array.new(number) { |a| a = [] }
+
+    self.in_groups_of(number, fill_width).each do |group|
+      number.times do |i|
+        group.fetch(i) rescue next
+        sorted_groups[i] << group[i]
+      end
+    end
+
+    sorted_groups
+  end
+end

data/lib/kimurai/core_ext/numeric.rb

@@ -0,0 +1,19 @@
+class Numeric
+  # https://stackoverflow.com/a/1679963
+  def duration
+    secs  = self.to_int
+    mins  = secs / 60
+    hours = mins / 60
+    days  = hours / 24
+
+    if days > 0
+      "#{days}d, #{hours % 24}h"
+    elsif hours > 0
+      "#{hours}h, #{mins % 60}m"
+    elsif mins > 0
+      "#{mins}m, #{secs % 60}s"
+    elsif secs >= 0
+      "#{secs}s"
+    end
+  end
+end

data/lib/kimurai/core_ext/string.rb

@@ -0,0 +1,7 @@
+require 'murmurhash3'
+
+class String
+  def to_id
+    MurmurHash3::V32.str_hash(self)
+  end
+end

data/lib/kimurai/pipeline.rb

@@ -0,0 +1,25 @@
+module Kimurai
+  class Pipeline
+    class DropItemError < StandardError; end
+    def self.name
+      self.to_s.sub(/.*?::/, "").underscore.to_sym
+    end
+
+    include BaseHelper
+    attr_accessor :spider
+
+    def name
+      self.class.name
+    end
+
+    ###
+
+    def unique?(scope, value)
+      spider.unique?(scope, value)
+    end
+
+    def save_to(path, item, format:, position: true)
+      spider.save_to(path, item, format: format, position: position)
+    end
+  end
+end

data/lib/kimurai/runner.rb

@@ -0,0 +1,72 @@
+require 'pmap'
+
+module Kimurai
+  class Runner
+    attr_reader :jobs, :spiders
+
+    def initialize(parallel_jobs:)
+      @jobs = parallel_jobs
+      @spiders = Kimurai.list
+
+      if time_zone = Kimurai.configuration.time_zone
+        Kimurai.time_zone = time_zone
+      end
+    end
+
+    def run!
+      start_time = Time.now
+      run_id = start_time.to_i
+      running_pids = []
+
+      ENV.store("RBCAT_COLORIZER", "false")
+
+      run_info = {
+        id: run_id,
+        status: :processing,
+        start_time: start_time,
+        stop_time: nil,
+        environment: Kimurai.env,
+        concurrent_jobs: jobs,
+        spiders: spiders.keys
+      }
+
+      at_exit do
+        # Prevent queue to process new intems while executing at_exit body
+        Thread.list.each { |t| t.kill if t != Thread.main }
+        # Kill currently running spiders
+        running_pids.each { |pid| Process.kill("INT", pid) }
+
+        error = $!
+        stop_time = Time.now
+
+        if error.nil?
+          run_info.merge!(status: :completed, stop_time: stop_time)
+        else
+          run_info.merge!(status: :failed, error: error.inspect, stop_time: stop_time)
+        end
+
+        if at_stop_callback = Kimurai.configuration.runner_at_stop_callback
+          at_stop_callback.call(run_info)
+        end
+        puts "<<< Runner: stopped: #{run_info}"
+      end
+
+      puts ">>> Runner: started: #{run_info}"
+      if at_start_callback = Kimurai.configuration.runner_at_start_callback
+        at_start_callback.call(run_info)
+      end
+
+      spiders.peach_with_index(jobs) do |spider, i|
+        spider_name = spider[0]
+        puts "> Runner: started spider: #{spider_name}, index: #{i}"
+
+        pid = spawn("bundle", "exec", "kimurai", "crawl", spider_name, [:out, :err] => "log/#{spider_name}.log")
+        running_pids << pid
+        Process.wait pid
+
+        running_pids.delete(pid)
+        puts "< Runner: stopped spider: #{spider_name}, index: #{i}"
+      end
+    end
+  end
+end

data/lib/kimurai/template/.gitignore

@@ -0,0 +1,18 @@
+/.bundle
+/cache
+/node_modules
+
+/log/*
+!/log/.keep
+
+/tmp/*
+!/tmp/.keep
+
+/db/*
+!/db/.keep
+
+.byebug_history
+*.swp
+.env
+
+capybara-*.png

data/lib/kimurai/template/.ruby-version

@@ -0,0 +1 @@
+2.5.1

data/lib/kimurai/template/Gemfile

@@ -0,0 +1,20 @@
+source 'https://rubygems.org'
+git_source(:github) { |repo| "https://github.com/#{repo}.git" }
+
+ruby '2.5.1'
+
+# Framework
+gem 'kimurai', '~> 1.0'
+
+# Require files in directory and child directories recursively
+gem 'require_all'
+
+# Dotenv
+gem 'dotenv'
+
+# To debug spiders:
+group :development do
+  gem 'byebug', platforms: :mri
+  gem 'pry'
+end
+

data/lib/kimurai/template/README.md

@@ -0,0 +1,3 @@
+# README
+
+New Kimurai project readme

data/lib/kimurai/template/config/application.rb

@@ -0,0 +1,32 @@
+Kimurai.configure do |config|
+  # Default logger has colored mode in development.
+  # If you would like to disable it, set `colorize_logger` to false.
+  # config.colorize_logger = false
+
+  # Logger level for default logger:
+  # config.log_level = :info
+
+  # Custom logger:
+  # config.logger = Logger.new(STDOUT)
+
+  # Custom time zone (for logs):
+  # config.time_zone = "UTC"
+  # config.time_zone = "Europe/Moscow"
+
+  # At start callback for a runner. Accepts argument with info as hash with
+  # keys: id, status, start_time, environment, concurrent_jobs, spiders list.
+  # For example, you can use this callback to send notification when runner was started:
+  # config.runner_at_start_callback = lambda do |info|
+  #   json = JSON.pretty_generate(info)
+  #   Sender.send_notification("Started session: #{json}")
+  # end
+
+  # At stop callback for a runner. Accepts argument with info as hash with
+  # all `runner_at_start_callback` keys plus additional `stop_time` key. Also `status` contains
+  # stop status of a runner (completed or failed).
+  # You can use this callback to send notification when runner has been stopped:
+  # config.runner_at_stop_callback = lambda do |info|
+  #   json = JSON.pretty_generate(info)
+  #   Sender.send_notification("Stopped session: #{json}")
+  # end
+end

data/lib/kimurai/template/config/automation.yml

@@ -0,0 +1,13 @@
+# software versions to install for `setup` command
+setup:
+  ruby: 2.5.1
+  # check latest here http://phantomjs.org/download.html
+  phantomjs: 2.1.1
+  # check latest here https://github.com/mozilla/geckodriver/releases/
+  geckodriver: 0.21.0
+  # check latest here https://sites.google.com/a/chromium.org/chromedriver/downloads
+  chromedriver: 2.39
+# settings for deploy command, you can use cli options as well (--repo-url, --git-key-path)
+deploy:
+  # repo_url: git@bitbucket.org:username/repo_name.git
+  # repo_key_path: ~/.ssh/id_rsa

data/lib/kimurai/template/config/boot.rb

@@ -0,0 +1,22 @@
+# require project gems
+require 'bundler/setup'
+Bundler.require(:default, Kimurai.env)
+
+# require custom ENV variables located in .env file
+require 'dotenv/load'
+
+# require initializers
+Dir.glob(File.join("./config/initializers", "*.rb"), &method(:require))
+
+# require helpers
+Dir.glob(File.join("./helpers", "*.rb"), &method(:require))
+
+# require pipelines
+Dir.glob(File.join("./pipelines", "*.rb"), &method(:require))
+
+# require spiders recursively in the `spiders/` folder
+require_relative '../spiders/application_spider'
+require_all "spiders"
+
+# require Kimurai configuration
+require_relative 'application'

data/lib/kimurai/template/config/schedule.rb

@@ -0,0 +1,57 @@
+### Settings ###
+require 'tzinfo'
+
+# Export current PATH to the cron
+env :PATH, ENV["PATH"]
+
+# Use 24 hour format when using `at:` option
+set :chronic_options, hours24: true
+
+# Use local_to_utc helper to setup execution time using your local timezone instead
+# of server's timezone (which is probably and should be UTC, to check run `$ timedatectl`).
+# Also maybe you'll want to set same timezone in kimurai as well (use `Kimurai.configuration.time_zone =` for that),
+# to have spiders logs in a specific time zone format.
+# Example usage of helper:
+# every 1.day, at: local_to_utc("7:00", zone: "Europe/Moscow") do
+#   crawl "google_spider.com", output: "log/google_spider.com.log"
+# end
+def local_to_utc(time_string, zone:)
+  TZInfo::Timezone.get(zone).local_to_utc(Time.parse(time))
+end
+
+# Note: by default Whenever exports cron commands with :environment == "production".
+# Note: Whenever can only append log data to a log file (>>). If you want
+# to overwrite (>) log file before each run, pass lambda:
+# crawl "google_spider.com", output: -> { "> log/google_spider.com.log 2>&1" }
+
+# Project job types
+job_type :crawl,  "cd :path && KIMURAI_ENV=:environment bundle exec kimurai crawl :task :output"
+job_type :runner, "cd :path && KIMURAI_ENV=:environment bundle exec kimurai runner --jobs :task :output"
+
+# Single file job type
+job_type :single, "cd :path && KIMURAI_ENV=:environment ruby :task :output"
+# Single with bundle exec
+job_type :single_bundle, "cd :path && KIMURAI_ENV=:environment bundle exec ruby :task :output"
+
+### Schedule ###
+# Usage (check examples here https://github.com/javan/whenever#example-schedulerb-file):
+# every 1.day do
+  # Example to schedule a single spider in the project:
+  # crawl "google_spider.com", output: "log/google_spider.com.log"
+
+  # Example to schedule all spiders in the project using runner. Each spider will write
+  # it's own output to the `log/spider_name.log` file (handled by a runner itself).
+  # Runner output will be written to log/runner.log file.
+  # Argument number it's a count of concurrent jobs:
+  # runner 3, output:"log/runner.log"
+
+  # Example to schedule single spider (without project):
+  # single "single_spider.rb", output: "single_spider.log"
+# end
+
+### How to set a cron schedule ###
+# Run: `$ whenever --update-crontab --load-file config/schedule.rb`.
+# If you don't have whenever command, install the gem: `$ gem install whenever`.
+
+### How to cancel a schedule ###
+# Run: `$ whenever --clear-crontab --load-file config/schedule.rb`.