tanakai 1.5.0

data/lib/tanakai/cli.rb

@@ -0,0 +1,183 @@
+require 'thor'
+
+module Tanakai
+  class CLI < Thor
+    map %w[--version -v] => :__print_version
+
+    desc "generate", "Generator, available types: project, spider, schedule"
+    def generate(generator_type, *args)
+      case generator_type
+      when "project"
+        project_name = args.shift
+        raise "Provide project name to generate a new project" unless project_name.present?
+        Generator.new.generate_project(project_name)
+      when "spider"
+        spider_name = args.shift
+        raise "Provide spider name to generate a spider" unless spider_name.present?
+        Generator.new.generate_spider(spider_name, in_project: inside_project?)
+      when "schedule"
+        Generator.new.generate_schedule
+      else
+        raise "Don't know this generator type: #{generator_type}"
+      end
+    end
+
+    ###
+
+    desc "setup", "Setup server"
+    option :port, aliases: :p, type: :string, banner: "Port for ssh connection"
+    option "ask-sudo", type: :boolean, banner: "Provide sudo password for a user to install system-wide packages"
+    option "ask-auth-pass", type: :boolean, banner: "Auth using password"
+    option "ssh-key-path", type: :string, banner: "Auth using ssh key"
+    option :local, type: :boolean, banner: "Run setup on a local machine (Ubuntu only)"
+    def setup(user_host)
+      command = AnsibleCommandBuilder.new(user_host, options, playbook: "setup").get
+
+      pid = spawn *command
+      Process.wait pid
+    end
+
+    desc "deploy", "Deploy project to the server and update cron schedule"
+    option :port, aliases: :p, type: :string, banner: "Port for ssh connection"
+    option "ask-auth-pass", type: :boolean, banner: "Auth using password"
+    option "ssh-key-path", type: :string, banner: "Auth using ssh key"
+    option "repo-url", type: :string, banner: "Repo url"
+    option "repo-key-path", type: :string, banner: "SSH key for a git repo"
+    option "skip-check", type: :boolean, default: false, banner: "Skip git repository checks"
+    def deploy(user_host)
+      unless options["skip-check"]
+        if !`git status --short`.empty?
+          raise "Deploy: Please commit your changes first"
+        elsif `git remote`.empty?
+          raise "Deploy: Please add remote origin repository to your repo first"
+        elsif !`git rev-list master...origin/master`.empty?
+          raise "Deploy: Please push your commits to the remote origin repo first"
+        end
+      end
+
+      repo_url = options["repo-url"] ? options["repo-url"] : `git remote get-url origin`.strip
+      repo_name = repo_url[/\/([^\/]*)\.git/i, 1]
+
+      command = AnsibleCommandBuilder.new(user_host, options, playbook: "deploy",
+        vars: { repo_url: repo_url, repo_name: repo_name, repo_key_path: options["repo-key-path"] }
+      ).get
+
+      pid = spawn *command
+      Process.wait pid
+    end
+
+    ###
+
+    desc "crawl", "Run a particular spider by it's name"
+    def crawl(spider_name)
+      raise "Can't find Tanakai project" unless inside_project?
+      require './config/boot'
+
+      unless klass = Tanakai.find_by_name(spider_name)
+        raise "Can't find spider with name `#{spider_name}` in the project. " \
+          "To list all available spiders, run: `$ bundle exec tanakai list`"
+      end
+
+      # Set time_zone if exists
+      if time_zone = Tanakai.configuration.time_zone
+        Tanakai.time_zone = time_zone
+      end
+
+      klass.crawl!
+    end
+
+    desc "parse", "Parse url in the particular spider method"
+    option :url, type: :string, required: true, banner: "Url to pass to the method"
+    def parse(spider_name, method_name)
+      raise "Can't find Tanakai project" unless inside_project?
+      require './config/boot'
+
+      unless klass = Tanakai.find_by_name(spider_name)
+        raise "Can't find spider with name `#{spider_name}` in the project. " \
+          "To list all available spiders, run: `$ bundle exec tanakai list`"
+      end
+
+      klass.parse!(method_name, url: options["url"])
+    end
+
+    desc "console", "Start Tanakai console"
+    option :engine, type: :string, banner: "Engine to use"
+    option :url, type: :string, banner: "Url to process"
+    def console(spider_name = nil)
+      require 'pry'
+      require './config/boot' if inside_project?
+
+      if spider_name
+        raise "Can't find Tanakai project" unless inside_project?
+
+        unless klass = Tanakai.find_by_name(spider_name)
+          raise "Can't find spider with name `#{spider_name}` in the project. " \
+            "To list all available spiders, run: `$ bundle exec tanakai list`"
+        end
+      else
+        klass = inside_project? ? ApplicationSpider : ::Tanakai::Base
+      end
+
+      engine = options["engine"]&.delete(":")&.to_sym
+      if url = options["url"]
+        klass.new(engine).request_to(:console, url: options["url"])
+      else
+        klass.new(engine).public_send(:console)
+      end
+    end
+
+    desc "list", "List all available spiders in the current project"
+    def list
+      raise "Can't find Tanakai project" unless inside_project?
+      require './config/boot'
+
+      Tanakai.list.keys.sort.each { |name| puts name }
+    end
+
+    desc "runner", "Run all spiders in the project in queue"
+    option :include, type: :array, default: [], banner: "List of spiders to run"
+    option :exclude, type: :array, default: [], banner: "List of spiders to exclude from run"
+    option :jobs, aliases: :j, type: :numeric, default: 1, banner: "The number of concurrent jobs"
+    def runner
+      raise "Can't find Tanakai project" unless inside_project?
+
+      jobs = options["jobs"]
+      raise "Jobs count can't be 0" if jobs == 0
+
+      require './config/boot'
+      require 'tanakai/runner'
+
+      spiders = options["include"].presence || Tanakai.list.keys
+      spiders -= options["exclude"]
+
+      Runner.new(spiders, jobs).run!
+    end
+
+    desc "--version, -v", "Print the version"
+    def __print_version
+      puts VERSION
+    end
+
+    desc "dashboard", "Run dashboard"
+    def dashboard
+      raise "Can't find Tanakai project" unless inside_project?
+
+      require './config/boot'
+      if Object.const_defined?("Tanakai::Dashboard")
+        require 'tanakai/dashboard/app'
+        Tanakai::Dashboard::App.run!
+      else
+        raise "Tanakai::Dashboard is not defined"
+      end
+    end
+
+    private
+
+    def inside_project?
+      Dir.exists?("spiders") && File.exists?("./config/boot.rb")
+    end
+  end
+end
+
+require_relative 'cli/generator'
+require_relative 'cli/ansible_command_builder'

data/lib/tanakai/core_ext/array.rb

@@ -0,0 +1,14 @@
+class Array
+  def in_sorted_groups(number, fill_width = nil)
+    sorted_groups = Array.new(number) { |a| a = [] }
+
+    self.in_groups_of(number, fill_width).each do |group|
+      number.times do |i|
+        group.fetch(i) rescue next
+        sorted_groups[i] << group[i]
+      end
+    end
+
+    sorted_groups
+  end
+end

data/lib/tanakai/core_ext/hash.rb

@@ -0,0 +1,5 @@
+class Hash
+  def deep_merge_excl(second, exclude)
+    self.merge(second.slice(*exclude)).deep_merge(second.except(*exclude))
+  end
+end

data/lib/tanakai/core_ext/numeric.rb

@@ -0,0 +1,19 @@
+class Numeric
+  # https://stackoverflow.com/a/1679963
+  def duration
+    secs  = self.to_int
+    mins  = secs / 60
+    hours = mins / 60
+    days  = hours / 24
+
+    if days > 0
+      "#{days}d, #{hours % 24}h"
+    elsif hours > 0
+      "#{hours}h, #{mins % 60}m"
+    elsif mins > 0
+      "#{mins}m, #{secs % 60}s"
+    elsif secs >= 0
+      "#{secs}s"
+    end
+  end
+end

data/lib/tanakai/core_ext/string.rb

@@ -0,0 +1,7 @@
+require 'murmurhash3'
+
+class String
+  def to_id
+    MurmurHash3::V32.str_hash(self)
+  end
+end

data/lib/tanakai/pipeline.rb

@@ -0,0 +1,33 @@
+module Tanakai
+  class Pipeline
+    class DropItemError < StandardError; end
+    def self.name
+      self.to_s.sub(/.*?::/, "").underscore.to_sym
+    end
+
+    include BaseHelper
+    attr_accessor :spider
+
+    def name
+      self.class.name
+    end
+
+    ###
+
+    def storage
+      spider.storage
+    end
+
+    def unique?(scope, value)
+      spider.unique?(scope, value)
+    end
+
+    def save_to(path, item, format:, position: true, append: false)
+      spider.save_to(path, item, format: format, position: position, append: append)
+    end
+
+    def logger
+      spider.logger
+    end
+  end
+end

data/lib/tanakai/runner.rb

@@ -0,0 +1,60 @@
+require 'pmap'
+
+module Tanakai
+  class Runner
+    attr_reader :jobs, :spiders, :session_info
+
+    def initialize(spiders, parallel_jobs)
+      @jobs = parallel_jobs
+      @spiders = spiders
+      @start_time = Time.now
+
+      @session_info = {
+        id: @start_time.to_i,
+        status: :processing,
+        start_time: @start_time,
+        stop_time: nil,
+        environment: Tanakai.env,
+        concurrent_jobs: @jobs,
+        spiders: @spiders
+      }
+
+      if time_zone = Tanakai.configuration.time_zone
+        Tanakai.time_zone = time_zone
+      end
+
+      ENV.store("SESSION_ID", @start_time.to_i.to_s)
+      ENV.store("RBCAT_COLORIZER", "false")
+    end
+
+    def run!(exception_on_fail: true)
+      puts ">>> Runner: started: #{session_info}"
+      if at_start_callback = Tanakai.configuration.runner_at_start_callback
+        at_start_callback.call(session_info)
+      end
+
+      running = true
+      spiders.peach_with_index(jobs) do |spider, i|
+        next unless running
+
+        puts "> Runner: started spider: #{spider}, index: #{i}"
+        pid = spawn("bundle", "exec", "tanakai", "crawl", spider, [:out, :err] => "log/#{spider}.log")
+        Process.wait pid
+
+        puts "< Runner: stopped spider: #{spider}, index: #{i}"
+      end
+    rescue StandardError, SignalException, SystemExit => e
+      running = false
+
+      session_info.merge!(status: :failed, error: e.inspect, stop_time: Time.now)
+      exception_on_fail ? raise(e) : [session_info, e]
+    else
+      session_info.merge!(status: :completed, stop_time: Time.now)
+    ensure
+      if at_stop_callback = Tanakai.configuration.runner_at_stop_callback
+        at_stop_callback.call(session_info)
+      end
+      puts "<<< Runner: stopped: #{session_info}"
+    end
+  end
+end

data/lib/tanakai/template/.gitignore

@@ -0,0 +1,18 @@
+/.bundle
+/cache
+/node_modules
+
+/log/*
+!/log/.keep
+
+/tmp/*
+!/tmp/.keep
+
+/db/*
+!/db/.keep
+
+.byebug_history
+*.swp
+.env
+
+capybara-*.png

data/lib/tanakai/template/Gemfile

@@ -0,0 +1,28 @@
+source 'https://rubygems.org'
+git_source(:github) { |repo| "https://github.com/#{repo}.git" }
+
+ruby '>= 2.5'
+
+# Framework
+gem 'tanakai'
+
+# Require files in directory and child directories recursively
+gem 'require_all'
+
+# Dotenv
+gem 'dotenv'
+
+# To debug spiders:
+group :development do
+  gem 'byebug', platforms: :mri
+  gem 'pry'
+end
+
+# If you want to save items to the database, require one of these gems:
+# gem 'sqlite3'
+# gem 'pg'
+# gem 'mysql2'
+
+# And use your preferred ORM/database connector:
+# gem 'activerecord', require: 'active_record'
+# gem 'sequel'

data/lib/tanakai/template/README.md

@@ -0,0 +1,3 @@
+# README
+
+New Tanakai project readme

data/lib/tanakai/template/config/application.rb

@@ -0,0 +1,37 @@
+Tanakai.configure do |config|
+  # Default logger has colored mode in development.
+  # If you would like to disable it, set `colorize_logger` to false.
+  # config.colorize_logger = false
+
+  # Logger level for default logger:
+  # config.log_level = :info
+
+  # Custom logger:
+  # config.logger = Logger.new(STDOUT)
+
+  # Custom time zone (for logs):
+  # config.time_zone = "UTC"
+  # config.time_zone = "Europe/Moscow"
+
+  # At start callback for a runner. Accepts argument with info as hash with
+  # keys: id, status, start_time, environment, concurrent_jobs, spiders list.
+  # For example, you can use this callback to send notification when runner was started:
+  # config.runner_at_start_callback = lambda do |info|
+  #   json = JSON.pretty_generate(info)
+  #   Sender.send_notification("Started session: #{json}")
+  # end
+
+  # At stop callback for a runner. Accepts argument with info as hash with
+  # all `runner_at_start_callback` keys plus additional `stop_time` key. Also `status` contains
+  # stop status of a runner (completed or failed).
+  # You can use this callback to send notification when runner has been stopped:
+  # config.runner_at_stop_callback = lambda do |info|
+  #   json = JSON.pretty_generate(info)
+  #   Sender.send_notification("Stopped session: #{json}")
+  # end
+
+  # Provide custom chrome binary path (default is any available chrome/chromium in the PATH):
+  # config.selenium_chrome_path = "/usr/bin/chromium-browser"
+  # Provide custom selenium chromedriver path (default is "/usr/local/bin/chromedriver"):
+  # config.chromedriver_path = "/usr/local/bin/chromedriver"
+end

data/lib/tanakai/template/config/automation.yml

@@ -0,0 +1,13 @@
+# software versions to install for `setup` command
+setup:
+  ruby: 2.5.1
+  # check latest here http://phantomjs.org/download.html
+  phantomjs: 2.1.1
+  # check latest here https://github.com/mozilla/geckodriver/releases/
+  geckodriver: 0.21.0
+  # check latest here https://sites.google.com/a/chromium.org/chromedriver/downloads
+  chromedriver: 2.39
+# settings for deploy command, you can use cli options as well (--repo-url, --git-key-path)
+deploy:
+  # repo_url: git@bitbucket.org:username/repo_name.git
+  # repo_key_path: ~/.ssh/id_rsa

data/lib/tanakai/template/config/boot.rb

@@ -0,0 +1,22 @@
+# require project gems
+require 'bundler/setup'
+Bundler.require(:default, Tanakai.env)
+
+# require custom ENV variables located in .env file
+require 'dotenv/load'
+
+# require initializers
+Dir.glob(File.join("./config/initializers", "*.rb"), &method(:require))
+
+# require helpers
+Dir.glob(File.join("./helpers", "*.rb"), &method(:require))
+
+# require pipelines
+Dir.glob(File.join("./pipelines", "*.rb"), &method(:require))
+
+# require spiders recursively in the `spiders/` folder
+require_relative '../spiders/application_spider'
+require_all "spiders"
+
+# require Tanakai configuration
+require_relative 'application'

data/lib/tanakai/template/config/schedule.rb

@@ -0,0 +1,57 @@
+### Settings ###
+require 'tzinfo'
+
+# Export current PATH to the cron
+env :PATH, ENV["PATH"]
+
+# Use 24 hour format when using `at:` option
+set :chronic_options, hours24: true
+
+# Use local_to_utc helper to setup execution time using your local timezone instead
+# of server's timezone (which is probably and should be UTC, to check run `$ timedatectl`).
+# Also maybe you'll want to set same timezone in tanakai as well (use `Tanakai.configuration.time_zone =` for that),
+# to have spiders logs in a specific time zone format.
+# Example usage of helper:
+# every 1.day, at: local_to_utc("7:00", zone: "Europe/Moscow") do
+#   crawl "google_spider.com", output: "log/google_spider.com.log"
+# end
+def local_to_utc(time_string, zone:)
+  TZInfo::Timezone.get(zone).local_to_utc(Time.parse(time_string))
+end
+
+# Note: by default Whenever exports cron commands with :environment == "production".
+# Note: Whenever can only append log data to a log file (>>). If you want
+# to overwrite (>) log file before each run, pass lambda:
+# crawl "google_spider.com", output: -> { "> log/google_spider.com.log 2>&1" }
+
+# Project job types
+job_type :crawl,  "cd :path && TANAKAI_ENV=:environment bundle exec tanakai crawl :task :output"
+job_type :runner, "cd :path && TANAKAI_ENV=:environment bundle exec tanakai runner --jobs :task :output"
+
+# Single file job type
+job_type :single, "cd :path && TANAKAI_ENV=:environment ruby :task :output"
+# Single with bundle exec
+job_type :single_bundle, "cd :path && TANAKAI_ENV=:environment bundle exec ruby :task :output"
+
+### Schedule ###
+# Usage (check examples here https://github.com/javan/whenever#example-schedulerb-file):
+# every 1.day do
+  # Example to schedule a single spider in the project:
+  # crawl "google_spider.com", output: "log/google_spider.com.log"
+
+  # Example to schedule all spiders in the project using runner. Each spider will write
+  # it's own output to the `log/spider_name.log` file (handled by a runner itself).
+  # Runner output will be written to log/runner.log file.
+  # Argument number it's a count of concurrent jobs:
+  # runner 3, output:"log/runner.log"
+
+  # Example to schedule single spider (without project):
+  # single "single_spider.rb", output: "single_spider.log"
+# end
+
+### How to set a cron schedule ###
+# Run: `$ whenever --update-crontab --load-file config/schedule.rb`.
+# If you don't have whenever command, install the gem: `$ gem install whenever`.
+
+### How to cancel a schedule ###
+# Run: `$ whenever --clear-crontab --load-file config/schedule.rb`.

data/lib/tanakai/template/helpers/application_helper.rb

@@ -0,0 +1,3 @@
+module ApplicationHelper
+  # Put here custom methods which are will be available for any spider
+end

data/lib/tanakai/template/pipelines/saver.rb

@@ -0,0 +1,11 @@
+class Saver < Tanakai::Pipeline
+  def process_item(item, options: {})
+    # Here you can save item to the database, send it to a remote API or
+    # simply save item to a file format using `save_to` helper:
+
+    # To get the name of a current spider: `spider.class.name`
+    # save_to "db/#{spider.class.name}.json", item, format: :pretty_json
+
+    item
+  end
+end

data/lib/tanakai/template/pipelines/validator.rb

@@ -0,0 +1,24 @@
+class Validator < Tanakai::Pipeline
+  def process_item(item, options: {})
+    # Here you can validate item and raise `DropItemError`
+    # if one of the validations failed. Examples:
+
+    # Check item sku for uniqueness using buit-in `unique?` helper:
+    # unless unique?(:sku, item[:sku])
+    #   raise DropItemError, "Item sku is not unique"
+    # end
+
+    # Drop item if title length shorter than 5 symbols:
+    # if item[:title].size < 5
+    #   raise DropItemError, "Item title is short"
+    # end
+
+    # Drop item if it doesn't contains any images:
+    # unless item[:images].present?
+    #   raise DropItemError, "Item images are not present"
+    # end
+
+    # Pass item to the next pipeline (if it wasn't dropped)
+    item
+  end
+end