vessel 0.2.0 → 0.3.0

data/lib/vessel/driver/page.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Vessel
+  class Driver
+    class Page
+      def close
+        raise NotImplementedError
+      end
+
+      def go_to(_)
+        raise NotImplementedError
+      end
+
+      def headers=(_)
+        raise NotImplementedError
+      end
+
+      def headers
+        raise NotImplementedError
+      end
+
+      def cookies=(_)
+        raise NotImplementedError
+      end
+
+      def cookies
+        raise NotImplementedError
+      end
+
+      def status
+        raise NotImplementedError
+      end
+
+      def body
+        raise NotImplementedError
+      end
+
+      def size
+        raise NotImplementedError
+      end
+
+      def current_url
+        raise NotImplementedError
+      end
+
+      def blacklist=(_)
+        raise NotImplementedError
+      end
+
+      def whitelist=(_)
+        raise NotImplementedError
+      end
+
+      def full_response
+        [status, headers, body]
+      end
+    end
+  end
+end

data/lib/vessel/driver/registry.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require "singleton"
+
+module Vessel
+  class Driver
+    class Registry
+      include Singleton
+
+      attr_reader :drivers
+
+      def initialize
+        @drivers = {}
+      end
+
+      def register(name, klass)
+        @drivers[name.to_sym] = klass
+      end
+
+      def build(settings = nil)
+        name = settings[:driver_name].to_sym
+        @drivers.fetch(name.to_sym).new(settings)
+      end
+    end
+  end
+end

data/lib/vessel/driver.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+require "vessel/driver/page"
+require "vessel/driver/registry"
+
+module Vessel
+  class Driver
+    def self.driver_name(name)
+      Registry.instance.register(name, self)
+    end
+
+    def self.direct_network_errors
+      [
+        ::Net::HTTP::Persistent::Error,
+        ::Net::HTTPFatalError,
+        ::Net::ReadTimeout,
+        ::Net::OpenTimeout,
+        ::SocketError,
+        ::Errno::ECONNRESET
+      ].freeze
+    end
+
+    def self.indirect_network_errors
+      [].freeze
+    end
+
+    attr_reader :browser, :settings, :proxy
+
+    def initialize(settings)
+      @last_request = nil
+      @settings = settings
+      @visited_urls = Concurrent::Map.new { 0 }
+      @proxy = settings[:proxy]&.new
+      start
+      at_exit { stop }
+    end
+
+    def start
+      raise NotImplementedError
+    end
+
+    def stop
+      raise NotImplementedError
+    end
+
+    def create_page(proxy: nil)
+      raise NotImplementedError
+    end
+
+    def restart
+      stop
+      start
+    end
+
+    def go_to(request)
+      url = request.url.to_s
+      response = Response.new(**request.to_handler)
+      return [response, request] if request.stub?
+
+      # The url is registered before the page is created, otherwise threads
+      # racing for the same url all pass the check and visit it in parallel.
+      attempt = visit(url)
+      if request.once && attempt > 1
+        Logger.info("Driver: rejecting #{url} visited #{attempt - 1} time(s)")
+        response.rejected = true
+        return [response, request]
+      end
+
+      begin
+        page = prepare_page(request)
+        delay(request.delay)
+        Logger.info("Driver: visiting #{url}, last_request = #{@last_request.to_i}")
+        page.go_to(url)
+        @last_request = Time.now.to_i
+        response = Response.new(page: page, attempt: attempt, **request.to_handler)
+        [response, request]
+      rescue StandardError => e
+        if network_error?(e)
+          Logger.error("Driver: network issue for #{url}, attempt ##{attempt}")
+          Logger.error("Driver: #{e.class}: #{e.message}")
+          if attempt < settings[:network_error_attempts]
+            attempt = visit(url)
+            restart
+            retry
+          end
+        end
+
+        [response, request, e]
+      end
+    end
+
+    private
+
+    def visit(url)
+      @visited_urls.compute(url) { |v| v.to_i + 1 }
+    end
+
+    def proxy_options
+      options = proxy&.next
+      Logger.debug("Driver: setting proxy #{options[:host]}:#{options[:port]}") if options
+      options
+    end
+
+    def delay(delay)
+      # Delay is set between requests when we don't want to bombard server with them.
+      # So it requires a crawler to be single threaded. Otherwise it doesn't make sense.
+      return if settings[:max_threads] > 1
+      return unless @last_request
+
+      begin
+        interval = @last_request + delay - Time.now.to_i
+        raise if interval.positive?
+      rescue RuntimeError
+        sleep(interval)
+        retry
+      end
+    end
+
+    def prepare_page(request)
+      page = create_page(proxy: proxy_options)
+      page.blacklist = settings[:blacklist] if settings[:blacklist]
+      page.whitelist = settings[:whitelist] if settings[:whitelist]
+      page.headers = request.headers if request.headers
+      page.cookies = request.cookies if request.cookies
+      page
+    end
+
+    def network_error?(error)
+      self.class.direct_network_errors.include?(error.class) ||
+        self.class.indirect_network_errors.include?(error.cause.class)
+    end
+  end
+end

data/lib/vessel/engine/stats.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Vessel
+  class Engine
+    module Stats
+      private
+
+      def stats
+        {
+          req_enqueued: req_enqueued.value, res_dequeued: res_dequeued.value,
+          res_handled: res_handled.value, item_pipelined: item_pipelined.value,
+          item_processed: item_processed.value, item_sent: item_sent.value,
+          item_rejected: item_rejected.value, idling: idle?
+        }
+      end
+
+      def increase(name)
+        send(name).increment
+        @crawler_class.new.after_change(name, stats)
+      end
+
+      def req_enqueued
+        @req_enqueued ||= Concurrent::AtomicFixnum.new
+      end
+
+      def res_dequeued
+        @res_dequeued ||= Concurrent::AtomicFixnum.new
+      end
+
+      def res_handled
+        @res_handled ||= Concurrent::AtomicFixnum.new
+      end
+
+      def item_pipelined
+        @item_pipelined ||= Concurrent::AtomicFixnum.new
+      end
+
+      def item_processed
+        @item_processed ||= Concurrent::AtomicFixnum.new
+      end
+
+      def item_sent
+        @item_sent ||= Concurrent::AtomicFixnum.new
+      end
+
+      def item_rejected
+        @item_rejected ||= Concurrent::AtomicFixnum.new
+      end
+    end
+  end
+end

data/lib/vessel/engine.rb

@@ -1,63 +1,142 @@
 # frozen_string_literal: true
 
+require "vessel/engine/stats"
+
 module Vessel
   class Engine
-    def self.run(*args, &block)
-      new(*args, &block).tap(&:run)
+    include Stats
+
+    def self.run(...)
+      new(...).tap(&:run)
     end
 
-    attr_reader :crawler_class, :settings, :scheduler, :middleware
+    attr_reader :crawler_class, :settings, :scheduler, :middleware_scheduler
 
-    def initialize(klass, &block)
+    def initialize(klass, &)
       @crawler_class = klass
       @settings = klass.settings
-      @middleware = block || Middleware.build(*settings[:middleware])
       @queue = SizedQueue.new(settings[:max_threads])
       @scheduler = Scheduler.new(@queue, settings)
+      @middleware_scheduler = MiddlewareScheduler.new(settings, &)
     end
 
     def run
-      scheduler.post(*start_requests)
-
-      until @queue.closed?
-        message = @queue.pop
-
-        raise(message) if message.is_a?(Exception)
-
-        page, request = message
-        args = [request.method, request.data].compact
-        handle(page, args)
+      at_exit do
+        crawler_class.new.info(stats)
+        crawler_class.new.before_stop
+      end
 
-        @queue.close if idle?
+      trap_sigint
+      crawler_class.new.before_start
+      crawler_class.new.before(stats)
+      @info_task = Concurrent::TimerTask.execute(execution_interval: crawler_class::INFO_INTERVAL) do
+        crawler_class.new.info(stats)
       end
+      schedule(crawler_class.start_requests)
+      event_loop
+    ensure
+      @info_task&.shutdown
+      scheduler.stop
+      middleware_scheduler.stop
+      crawler_class.new.after(stats)
+    end
 
+    def parse(url, handler, data = nil)
+      trap_sigint
+      request = crawler_class.build_request(url: url, handler: handler, data: data)
+      schedule(request)
+      event_loop
     ensure
       scheduler.stop
+      middleware_scheduler.stop
     end
 
-    def handle(page, args)
-      crawler = @crawler_class.new(page)
+    def handle(response, request, error)
+      increase(:res_dequeued)
+      return if response.rejected
+
+      crawler = crawler_class.new(response)
+      crawler_class.store_cookies(response.cookies) if settings[:allow_cookies]
+      args = error ? [:on_error, request, error] : [request.handler].compact
+      Logger.debug("Engine: crawler starts processing #{response.url} with :#{response.handler}")
       crawler.send(*args) do |*result|
-        if result.flatten.all? { |i| i.is_a?(Request) }
-          scheduler.post(*result.flatten)
+        result = result.flatten
+        if result.all?(Request)
+          Logger.debug("Engine: :#{response.handler} enqueued requests #{result.map { |r| r.url.to_s }}")
+          schedule(result)
         else
-          @middleware&.call(*result)
+          result.each { |fields| process(fields) }
         end
       end
+      Logger.debug("Engine: crawler stopped processing #{response.url}")
+    rescue StandardError => e
+      Logger.error("Engine: there was an issue while processing #{response.url}")
+      Logger.error("Engine: `#{e.class}: #{e.message}`\n#{e.backtrace.join("\n")}")
     ensure
-      page.close if page
+      increase(:res_handled)
+      response.close
     end
 
     private
 
-    def start_requests
-      Request.build(*settings[:start_urls])
+    def engine_idle?
+      @queue.empty? &&
+        req_enqueued.value == res_dequeued.value &&
+        res_dequeued.value == res_handled.value &&
+        item_pipelined.value == item_processed.value &&
+        item_processed.value == (item_sent.value + item_rejected.value)
+    end
+
+    def engine_and_scheduler_idle?
+      engine_idle? && @scheduler.idle?
     end
 
     def idle?
-      @queue.empty? &&
-      @scheduler.queue_length.zero? &&
-      @scheduler.scheduled_task_count == @scheduler.completed_task_count
+      engine_and_scheduler_idle? && middleware_scheduler.idle?
+    end
+
+    def schedule(*requests)
+      requests.flatten.size.times { increase(:req_enqueued) }
+      scheduler.post(*requests)
+    end
+
+    def process(fields)
+      increase(:item_pipelined)
+      middleware_scheduler&.call(fields)&.add_observer(self, :on_item_processed)
+    end
+
+    def on_item_processed(_time, value, error)
+      increase(:item_processed)
+      increase(:item_sent) if value && !error
+      increase(:item_rejected) if error
+
+      if error && !error.is_a?(Middleware::InvalidItemError)
+        Logger.error("Engine: issue `#{error.class}: #{error.message}` raised in the middleware")
+        Logger.error("Engine: \n#{error.backtrace.join("\n")}")
+      end
+
+      @queue.close if engine_and_scheduler_idle? && middleware_scheduler.idle?(after: true)
+    end
+
+    def trap_sigint
+      trap "SIGINT" do
+        puts "\n\nEngine: terminating...\n\n"
+        @queue.clear
+        @queue.close
+        scheduler.force_kill
+        middleware_scheduler.force_kill
+      end
+    end
+
+    def event_loop
+      until @queue.closed?
+        message = @queue.pop
+        break unless message
+
+        response, request, error = message
+        handle(response, request, error)
+        @queue.close if idle?
+      end
     end
   end
 end

data/lib/vessel/loader.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Vessel
+  class Loader
+    attr_reader :crawlers
+
+    def initialize
+      @crawlers = {}
+    end
+
+    def templator
+      Templator
+    end
+
+    def load_crawlers
+      Dir["crawlers/**/*.rb"].each { |file| require(file) }
+    end
+
+    def load_crawler(name)
+      path = File.expand_path("crawlers/#{name}.rb")
+      require(path) if File.exist?(path)
+    end
+
+    def register(name, klass)
+      @crawlers.merge!(name => klass)
+    end
+
+    def find_crawler(name)
+      @crawlers[name]
+    end
+
+    def list_crawlers
+      @crawlers.keys.sort
+    end
+  end
+end

data/lib/vessel/logger.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require "logger"
+require "forwardable"
+
+module Vessel
+  class Logger
+    class << self
+      extend Forwardable
+
+      delegate %i[debug info warn error] => :instance
+
+      attr_writer :instance
+
+      def instance
+        @instance ||= begin
+          $stdout.sync = true
+          instance = ::Logger.new($stdout)
+          instance.level = ::Logger::DEBUG
+          instance
+        end
+      end
+    end
+  end
+end

data/lib/vessel/middleware.rb

@@ -2,22 +2,20 @@
 
 module Vessel
   class Middleware
-    attr_reader :middleware
+    class InvalidItemError < RuntimeError; end
 
-    def self.build(*classes)
-      classes.inject { |base, klass| base.new(klass.new) }
+    attr_reader :settings
+
+    def initialize(settings)
+      @settings = settings
     end
 
-    def initialize(middleware = nil)
-      @middleware = middleware
+    def call
+      raise NotImplementedError
     end
 
     def ==(other)
       self.class == other.class
     end
-
-    def call
-      raise NotImplementedError
-    end
   end
 end

data/lib/vessel/middleware_scheduler.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require "concurrent"
+require "forwardable"
+
+module Vessel
+  class MiddlewareScheduler
+    extend Forwardable
+
+    delegate %i[scheduled_task_count completed_task_count queue_length] => :pool
+
+    attr_reader :settings, :middlewares
+
+    def initialize(settings, &block)
+      @settings = settings
+      @middlewares = if block_given?
+                       [block]
+                     else
+                       settings[:middleware].map { |m| Object.const_get(m).new(settings) }
+                     end
+    end
+
+    def call(fields)
+      Concurrent::Future.execute(executor: pool) do
+        middlewares.inject(fields.to_h) { |hash, middleware| middleware.call(hash, fields) }
+      end
+    end
+
+    def pool
+      @pool ||= Concurrent::ThreadPoolExecutor.new(
+        min_threads: settings[:min_threads],
+        max_threads: settings[:max_threads],
+        max_queue: 0
+      )
+    end
+
+    def idle?(after: false)
+      completed_tasks = completed_task_count + (after ? 1 : 0)
+      queue_length.zero? && scheduled_task_count == completed_tasks
+    end
+
+    def stop
+      pool.shutdown
+      pool.kill unless pool.wait_for_termination(30)
+    end
+
+    def force_kill
+      pool.send(:ns_kill_execution)
+    end
+  end
+end

data/lib/vessel/proxy.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Vessel
+  class Proxy
+    attr_reader :proxies
+
+    def next
+      @index ||= 0
+      @prepared ||= prepare
+      return if proxies.nil? || proxies.empty?
+
+      settings = proxies[@index % proxies.size]
+      @index += 1
+      settings
+    end
+
+    def prepare
+      raise NotImplementedError
+    end
+  end
+
+  class RoundRobinProxy < Proxy
+    PROXIES = [].freeze
+
+    def initialize
+      @proxies = self.class::PROXIES
+      super
+    end
+
+    def prepare
+      true
+    end
+  end
+
+  class ShuffledProxy < RoundRobinProxy
+    def prepare
+      @proxies = @proxies.shuffle
+      true
+    end
+  end
+end

data/lib/vessel/request.rb

@@ -1,27 +1,44 @@
 # frozen_string_literal: true
 
 require "addressable/uri"
+require "vessel/util"
 
 module Vessel
   class Request
-    attr_reader :url, :uri, :method, :data
+    include Vessel::Util
 
-    def self.build(*urls)
-      urls.empty? ? [new] : urls.map { |url| new(url: url) }
-    end
+    DEFAULT_HANDLER = :parse
 
-    def initialize(url: nil, method: :parse, data: nil)
-      if url
-        @url = url.to_s
-        @uri = Addressable::URI.parse(@url)
-      end
+    attr_reader :url, :handler, :data, :delay, :cookies, :headers, :once
 
-      @method = method
-      @data = data.freeze if data
+    def initialize(url: nil, callback: nil, handler: nil, delay: 0, cookies: nil, headers: nil, data: nil, once: true)
+      @url = Addressable::URI.parse(url) if url
+      @delay = delay.is_a?(Range) ? rand(delay) : delay
+      @cookies = cookies
+      @headers = headers
+      @handler = handler || callback || DEFAULT_HANDLER
+      @data = deep_clone(data) || {}
+      @once = once
     end
 
+    alias callback handler
+
     def stub?
       !url
     end
+
+    def to_handler
+      { url: url, data: data, handler: handler }
+    end
+
+    def ==(other)
+      url == other.url &&
+        data == other.data &&
+        handler == other.handler &&
+        delay == other.delay &&
+        cookies == other.cookies &&
+        headers == other.headers &&
+        once == other.once
+    end
   end
 end