toro 0.0.1 → 0.0.2

data/lib/toro/manager.rb

@@ -0,0 +1,143 @@
+module Toro
+  class Shutdown < Interrupt; end
+
+  class Manager
+    include Actor
+    include ActorManager
+
+    attr_reader :busy, :ready
+
+    def initialize(options={})
+      defaults = {
+        concurrency: 1,
+        queues: [Toro.options[:default_queue]],
+      }
+      options = defaults.merge(options)
+      @queues = options[:queues]
+      @threads = {}
+      @ready = options[:concurrency].times.map do
+        processor = Processor.new_link(current_actor)
+        processor.proxy_id = processor.object_id
+        processor
+      end
+      @busy = []
+      @is_done = false
+      @fetcher = Fetcher.new({ manager: current_actor, queues: options[:queues] })
+      @listener = Listener.new({ queues: @queues, fetcher: @fetcher, manager: current_actor })
+    end
+
+    def start
+      @is_done = false
+      @listener.async.start
+      @ready.each { dispatch }
+      heartbeat
+    end
+
+    def stop
+      @is_done = true
+
+      Toro.logger.debug "Shutting down #{@ready.size} quiet workers"
+      @ready.each { |processor| processor.terminate if processor.alive? }
+      @ready.clear
+      @fetcher.terminate if @fetcher.alive?
+      if @listener.alive?
+        actors[:listener].stop if actors[:listener]
+        @listener.terminate
+      end
+      return if clean_up_for_graceful_shutdown
+
+      hard_shutdown_in(Toro.options[:hard_shutdown_time])
+    end
+
+    def assign(job)
+      raise 'No processors ready' if !is_ready?
+      processor = @ready.pop
+      @busy << processor
+      processor.async.process(job)
+    end
+
+    def is_ready?
+      !@ready.empty?
+    end
+
+    def dispatch
+      raise "No processors, cannot continue!" if @ready.empty? && @busy.empty?
+      raise "No ready processor!?" if @ready.empty?
+      @fetcher.async.fetch
+    end
+
+    def clean_up_for_graceful_shutdown
+      if @busy.empty?
+        shutdown
+        return true
+      end
+
+      after(Toro.options[:graceful_shutdown_time]) { clean_up_for_graceful_shutdown }
+      false
+    end
+
+    def hard_shutdown_in(delay)
+      Toro.logger.info "Pausing up to #{delay} seconds to allow workers to finish..."
+
+      after(delay) do
+        # We've reached the timeout and we still have busy processors.
+        # They must die but their messages shall live on.
+        Toro.logger.warn "Terminating #{@busy.size} busy worker threads"
+
+        requeue
+
+        @busy.each do |processor|
+          if processor.alive? && thread = @threads.delete(processor.object_id)
+            thread.raise Shutdown
+          end
+        end
+
+        signal_shutdown
+      end
+    end
+
+    def shutdown
+      requeue
+      signal_shutdown
+    end
+
+    def requeue
+      Toro::Database.with_connection do
+        Job.where(status: 'running', started_by: Toro.process_identity).
+          update_all(status: 'queued', started_by: nil, started_at: nil)
+      end
+    end
+
+    def signal_shutdown
+      after(0) { signal(:shutdown) }
+    end
+
+    def set_thread(proxy_id, thread)
+      @threads[proxy_id] = thread
+    end
+
+    def heartbeat
+      return if stopped?
+
+      after(5) do
+        heartbeat
+      end
+    end
+
+    def processor_complete(processor)
+      @threads.delete(processor.object_id)
+      @busy.delete(processor)
+      if stopped?
+        processor.terminate if processor.alive?
+        shutdown if @busy.empty?
+      else
+        @ready << processor if processor.alive?
+        dispatch
+      end
+    end
+
+    def stopped?
+      @is_done
+    end
+  end
+end

data/lib/toro/middleware/chain.rb

@@ -0,0 +1,81 @@
+module Toro
+  module Middleware
+    class Chain
+      include Enumerable
+      attr_reader :entries
+
+      def initialize_copy(copy)
+        copy.instance_variable_set(:@entries, entries.dup)
+      end
+
+      def each(&block)
+        entries.each(&block)
+      end
+
+      def initialize
+        @entries = []
+        yield self if block_given?
+      end
+
+      def remove(klass)
+        entries.delete_if { |entry| entry.klass == klass }
+      end
+
+      def add(klass, *args)
+        remove(klass) if exists?(klass)
+        entries << Entry.new(klass, *args)
+      end
+
+      def insert_before(oldklass, newklass, *args)
+        i = entries.index { |entry| entry.klass == newklass }
+        new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
+        i = entries.index { |entry| entry.klass == oldklass } || 0
+        entries.insert(i, new_entry)
+      end
+
+      def insert_after(oldklass, newklass, *args)
+        i = entries.index { |entry| entry.klass == newklass }
+        new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
+        i = entries.index { |entry| entry.klass == oldklass } || entries.count - 1
+        entries.insert(i+1, new_entry)
+      end
+
+      def exists?(klass)
+        any? { |entry| entry.klass == klass }
+      end
+
+      def retrieve
+        map(&:make_new)
+      end
+
+      def clear
+        entries.clear
+      end
+
+      def invoke(*args, &final_action)
+        chain = retrieve.dup
+        traverse_chain = lambda do
+          if chain.empty?
+            final_action.call
+          else
+            chain.shift.call(*args, &traverse_chain)
+          end
+        end
+        traverse_chain.call
+      end
+    end
+
+    class Entry
+      attr_reader :klass
+
+      def initialize(klass, *args)
+        @klass = klass
+        @args  = args
+      end
+
+      def make_new
+        @klass.new(*@args)
+      end
+    end
+  end
+end

data/lib/toro/middleware/server/error.rb

@@ -0,0 +1,19 @@
+module Toro
+  module Middleware
+    module Server
+      class Error
+        def call(job, worker)
+          begin
+            yield
+          rescue Exception => exception
+            job.update_attributes(
+              status: 'failed',
+              finished_at: Time.now
+            )
+            raise exception
+          end
+        end
+      end
+    end
+  end
+end

data/lib/toro/middleware/server/error_storage.rb

@@ -0,0 +1,22 @@
+module Toro
+  module Middleware
+    module Server
+      class ErrorStorage
+        def call(job, worker)
+          begin
+            yield
+          rescue Exception => exception
+            job.reload
+            job.set_properties(
+              'error:class' => exception.class.name,
+              'error:message' => exception.message,
+              'error:backtrace' => exception.backtrace
+            )
+            job.save
+            raise exception
+          end
+        end
+      end
+    end
+  end
+end

data/lib/toro/middleware/server/properties.rb

@@ -0,0 +1,15 @@
+module Toro
+  module Middleware
+    module Server
+      class Properties
+        def call(job, worker)
+          result = yield
+          if result.is_a?(Hash) && result[:job_properties].is_a?(Hash)
+            job.set_properties(result[:job_properties])
+            job.save
+          end
+        end
+      end
+    end
+  end
+end

data/lib/toro/middleware/server/retry.rb

@@ -0,0 +1,25 @@
+module Toro
+  module Middleware
+    module Server
+      class Retry
+        def call(job, worker)
+          begin
+            yield
+          rescue Exception => exception
+            if worker.toro_options[:retry_interval]
+              interval = worker.toro_options[:retry_interval]
+              job.reload
+              job.properties ||= {}
+              job.properties['retry:errors'] ||= []
+              job.properties['retry:errors'] << "#{exception.class.name} -- #{exception.message} -- #{Time.now}"
+              job.status = 'scheduled'
+              job.scheduled_at = Time.now + interval
+              job.save
+            end
+            raise exception
+          end
+        end
+      end
+    end
+  end
+end

data/lib/toro/monitor.rb

@@ -0,0 +1,34 @@
+require 'slim'
+require 'jquery-datatables-rails'
+require 'rails-datatables'
+require 'action_view'
+
+directory = File.dirname(File.absolute_path(__FILE__))
+require "#{directory}/monitor/custom_views.rb"
+require "#{directory}/monitor/time_formatter.rb"
+require "#{directory}/monitor/engine.rb" if defined?(Rails)
+
+module Toro
+  module Monitor
+    DEFAULTS = {
+      :charts => nil,
+      :javascripts => [],
+      :poll_interval => 3000
+    }
+
+    class << self
+      def options
+        @options ||= DEFAULTS.dup
+      end
+
+      def options=(options)
+        @options = options
+      end
+
+      def root_path
+        toro_monitor_path = Toro::Monitor::Engine.routes.url_helpers.toro_monitor_path
+        "#{::Rails.application.config.relative_url_root}#{toro_monitor_path}"
+      end
+    end
+  end
+end

data/lib/toro/monitor/custom_views.rb

@@ -0,0 +1,26 @@
+module Toro
+  module Monitor
+    class CustomViews
+      @views = []
+
+      class << self
+        def add(name, path, &block)
+          @views << {
+            name: name,
+            path: path,
+            filter: block
+          }
+        end
+
+        def for_job(job)
+          views = []
+          @views.each do |view|
+            is_valid = view[:filter].call(job)
+            views << view.dup if is_valid
+          end
+          views
+        end
+      end
+    end
+  end
+end

data/lib/toro/monitor/engine.rb

@@ -0,0 +1,11 @@
+module Toro
+  module Monitor
+    class Engine < ::Rails::Engine
+      isolate_namespace Monitor
+
+      initializer "toro.asset_pipeline" do |app|
+        app.config.assets.precompile << 'toro/monitor/application.js'
+      end
+    end
+  end
+end

data/lib/toro/monitor/time_formatter.rb

@@ -0,0 +1,27 @@
+module Toro
+  module Monitor
+    class TimeFormatter
+      class << self
+        include ActionView::Helpers::DateHelper
+
+        def distance_of_time(from_time, to_time)
+          replacements = {
+            'less than ' => '',
+            'about ' => '',
+            ' days' => 'd',
+            ' day' => 'd',
+            ' hours' => 'h',
+            ' hour' => 'h',
+            ' minutes' => 'm',
+            ' minute' => 'm',
+            ' seconds' => 's',
+            ' second' => 's'
+          }
+          phrase = distance_of_time_in_words(from_time, to_time, include_seconds: true)
+          replacements.each { |from, to| phrase.gsub!(from, to) }
+          phrase
+        end
+      end
+    end
+  end
+end

data/lib/toro/processor.rb

@@ -0,0 +1,48 @@
+module Toro
+  class Processor
+    include Actor
+
+    attr_accessor :proxy_id
+
+    class << self
+      def default_middleware
+        Middleware::Chain.new do |middleware|
+          middleware.add Middleware::Server::Properties
+          middleware.add Middleware::Server::Retry
+          middleware.add Middleware::Server::ErrorStorage
+          middleware.add Middleware::Server::Error
+        end
+      end
+    end
+
+    def initialize(manager)
+      @manager = manager
+    end
+
+    def process(job)
+      @manager.set_thread(proxy_id, Thread.current)
+
+      Toro.logger.info "Processing #{job}"
+      worker = job.class_name.constantize
+
+      Toro::Database.with_connection do
+        begin
+          Toro.server_middleware.invoke(job, worker) do
+            worker.new.perform(*job.args)
+          end
+        rescue Exception => exception
+          Toro.logger.error "#{exception.class}: #{exception.message}"
+          Toro.logger.error exception.backtrace.join("\n")
+        else
+          Toro.logger.info "Processed #{job}"
+          job.update_attributes(
+            status: 'complete',
+            finished_at: Time.now
+          )
+        end
+      end
+      
+      @manager.processor_complete(current_actor) if @manager.alive?
+    end
+  end
+end