active_worker 0.50.0

data/lib/active_worker/controller.rb

@@ -0,0 +1,112 @@
+module ActiveWorker
+  class Controller
+    extend Behavior::ExecuteConcurrently
+    extend JobQueue::RunRemotely
+
+    attr_reader :configuration
+
+    def self.execute_expanded(configuration_id)
+      config = Configuration.find(configuration_id)
+      configurations = config.expand_for_threads
+
+      execute_concurrently(configurations)
+
+      after_thread_launch_methods.each { |method| send(method, config, configurations) }
+
+      wait_for_children
+    ensure
+      worker_cleanup_methods.each { |method| send(method, configurations) }
+    end
+
+    def self.execute(configuration)
+      worker = new(configuration)
+      worker.started
+      worker.execute
+      worker.finished
+    end
+
+    def self.handle_error(error, method, params)
+      configuration_id = params.shift
+      configuration = Configuration.find(configuration_id)
+
+      if threaded?
+        FailureEvent.from_error(configuration, error)
+      end
+
+      if forking? && parent?
+        ParentEvent.create_error_from_configuration(configuration, error)
+        kill_children
+        wait_for_children
+      end
+
+      if forked?
+        FailureEvent.create_error_from_configuration(configuration, error)
+      end
+
+    rescue Exception => e
+      puts "ERROR: While Handling #{error} Had : #{e}\n#{e.backtrace.join("\n")}"
+    end
+
+    def self.handle_termination(params)
+      if forking? && parent?
+        kill_children
+        wait_for_children
+      end
+
+      configuration_id = params.shift
+      configuration = Configuration.find(configuration_id)
+
+      if threaded?
+        TerminationEvent.from_termination(configuration)
+      end
+      if forked?
+        TerminationEvent.create_termination_from_configuration(configuration)
+      end
+
+    end
+
+    def self.after_thread_launch(method)
+      after_thread_launch_methods << method
+    end
+
+    def self.after_thread_launch_methods
+      @after_thread_launch_methods ||= []
+    end
+
+    def self.worker_cleanup(method)
+      worker_cleanup_methods << method
+    end
+
+    def self.worker_cleanup_methods
+      @worker_cleanup_methods ||= []
+    end
+
+    def initialize(configuration)
+      @configuration = configuration
+      setup
+    end
+
+    def setup
+
+    end
+
+    def started
+      configuration.started
+    end
+
+    def execute
+      raise "Can't call execute on base controller #{configuration.inspect}'"
+    end
+
+    def finished
+      configuration.finished
+    end
+
+    private
+
+    def finished?(configurations)
+      FinishedEvent.exists_for_configurations?(configurations)
+    end
+
+  end
+end

data/lib/active_worker/event.rb

@@ -0,0 +1,68 @@
+require_relative "behavior/has_root_object"
+module ActiveWorker
+  class Event
+    include Mongoid::Document
+    include Mongoid::Timestamps
+    include Behavior::HasRootObject
+
+    belongs_to :configuration, :class_name => "ActiveWorker::Configuration"
+    alias_method :root_owner, :configuration
+    root_object_relation :events
+
+    index({:configuration_id => -1, :_type => 1}, {:background => true})
+
+    before_save :set_message, :set_process_information
+
+    validates_presence_of :configuration, :message => "Events must be owned by a Configuration"
+
+    field :message, :type => String
+    field :host, :type => String
+    field :process_id, :type => Integer
+    field :worker_pid, :type => Integer
+
+    scope :for_root_object_id, lambda {|root_object_id| where(:root_object_id => root_object_id).descending(:created_at)}
+
+
+    def self.exists_for_configurations?(configurations)
+      where(:configuration_id.in => configurations.map(&:id)).count == configurations.size
+    end
+
+    def fields_for_view
+      view_fields = {}
+      fields.keys.each do |field|
+        view_fields[field] = self.send(field)
+      end
+      view_fields
+    end
+
+    def event_type
+      self.class.name.split('::').last.underscore
+    end
+
+    def set_process_information
+      self.host = HostInformation.hostname
+      self.process_id = get_pid
+      #self.worker_pid = get_worker_pid
+    end
+
+    def get_pid
+      Process.pid.to_i
+    end
+
+    def get_worker_pid
+      worker = JobQueue::QueueManager.new.active_jobs_for_configurations([configuration.to_param]).first
+      return worker["pid"] if worker
+      nil
+    end
+
+    def set_message
+      return if message
+      self.message = generate_message
+    end
+
+    def generate_message
+      "#{configuration.event_name} base message"
+    end
+
+  end
+end

data/lib/active_worker/expandable.rb

@@ -0,0 +1,77 @@
+module ActiveWorker
+  module Expandable
+
+    def self.included(base)
+      base.template_field :number_of_threads, type: Integer, default: 1
+      base.template_field :number_of_workers, type: Integer, default: 1
+      base.belongs_to :thread_root_configuration, :polymorphic => true
+      base.has_many :thread_expanded_configurations,
+                    :as => :thread_root_configuration,
+                    :class_name => 'ActiveWorker::Configuration',
+                    :autosave => true
+    end
+
+    def expand_for_threads
+      maps = expansion_maps_for_threads
+      maps[1..-1].each do |map_hash|
+        map_hash[:thread_root_configuration_id] = id
+        map_hash[:thread_root_configuration_type] = _type
+      end
+      expand_from_maps(maps)
+    end
+
+    def expand_for_workers
+      maps = expansion_maps_for_workers
+      expand_from_maps(maps)
+    end
+
+    def configurations_for_events
+      [self] + thread_expanded_configurations
+    end
+
+    private
+
+    def expand_from_maps(maps)
+      first_config = true
+      maps.map do |map_hash|
+        if first_config
+          first_config = false
+          modify_for_expansion(map_hash)
+        else
+          create_for_expansion(map_hash)
+        end
+      end
+    end
+
+    def create_for_expansion(options = {})
+      self.class.create(defined_fields.merge(parent_configuration: parent_configuration,
+                                             root_object_id: root_object_id,
+                                             root_object_type: root_object_type,
+                                             renderable: false).merge(options))
+    end
+
+    def modify_for_expansion(options = {})
+      self.update_attributes!(options)
+      self
+    end
+
+
+    def expansion_maps_for_workers
+      expansion_maps_for(number_of_workers)
+    end
+
+    def expansion_maps_for_threads
+      expansion_maps_for(number_of_threads)
+
+    end
+
+    def expansion_maps_for(number_of_configurations)
+      maps = []
+      number_of_configurations.times do
+        maps << {}
+      end
+      maps
+    end
+
+  end
+end

data/lib/active_worker/failure_event.rb

@@ -0,0 +1,16 @@
+module ActiveWorker
+  class FailureEvent < ActiveWorker::FinishedEvent
+    extend Behavior::CreateFromError
+
+    def self.from_error(thread_root_configuration, error)
+      events = []
+      thread_root_configuration.configurations_for_events.each do |configuration|
+        events << create_error_from_configuration(configuration,error) unless configuration.completed?
+      end
+      events
+    end
+
+
+
+  end
+end

data/lib/active_worker/finished_event.rb

@@ -0,0 +1,9 @@
+module ActiveWorker
+  class FinishedEvent < ActiveWorker::Event
+    def generate_message
+      "#{configuration.event_name} finished"
+    end
+
+    after_create :notify_root_of_finished
+  end
+end

data/lib/active_worker/host_information.rb

@@ -0,0 +1,13 @@
+module ActiveWorker
+  module HostInformation
+
+    def self.hostname
+      `hostname -f`.chomp
+    end
+
+    def self.short_hostname
+      `hostname`.chomp
+    end
+
+  end
+end

data/lib/active_worker/job_queue/job_executer.rb

@@ -0,0 +1,52 @@
+require 'resque/errors'
+
+module ActiveWorker
+  module JobQueue
+    class JobExecuter
+
+      Thread.abort_on_exception = false
+
+      @queue = :execute
+
+      def self.perform(args)
+        execute_task_from_args(args)
+      end
+
+      def self.execute_task_from_args(args)
+        class_name = args["class_name"]
+        method     = args["method"]
+        params     = args["params"]
+
+        klass = class_name.constantize
+        klass.send(method,*params)
+
+      rescue Resque::TermException => e
+        handle_termination(klass, params, e)
+      rescue SignalException => e
+        handle_exception(e, method, params, klass)
+        raise e
+      rescue Exception => e
+        handle_exception(e, method, params, klass)
+      end
+
+      def self.handle_exception(e, method, params, klass)
+        log_error "Handling exception for #{klass} because #{e.message}"
+        klass.handle_error e, method, params
+      rescue Exception => handle_error_error
+        log_error "Handle error exception: #{handle_error_error.message}"
+        log_error handle_error_error.backtrace.join("\n")
+      end
+
+      def self.handle_termination(klass, params, exception = nil)
+        log_error "Handling #{exception || "termination" } for #{klass}"
+        klass.handle_termination params
+        exit
+      end
+
+      def self.log_error(message)
+        puts "JOB EXECUTOR: #{message}"
+      end
+
+    end
+  end
+end

data/lib/active_worker/job_queue/queue_manager.rb

@@ -0,0 +1,46 @@
+module ActiveWorker
+  module JobQueue
+    class QueueManager
+
+      def active_jobs
+        Resque.working.map {|w| create_job_hash_from_worker(w)}.compact
+      end
+
+      def active_jobs_for_configurations(configuration_ids)
+        workers = Resque.working.select do|w|
+          configuration_ids.include? configuration_id_from_worker(w)
+        end
+        workers.map do |w|
+          create_job_hash_from_worker(w)
+        end
+      end
+
+      def configuration_id_from_worker(worker)
+        params = params_from_worker(worker)
+        params.first if params
+      end
+
+      def create_job_hash_from_worker(worker)
+        worker_id = worker.to_s.split(":")
+        host = worker_id[0]
+        pid = worker_id[1]
+        queues = worker_id[2].split(",")
+        args = args_from_worker(worker)
+
+        if worker_id && host && pid && queues && args
+          return {"host" => host, "queues" => queues, "pid" => pid, "args" => args }
+        end
+      end
+
+      def params_from_worker(worker)
+        args = args_from_worker(worker)
+        args["params"] if args
+      end
+
+      def args_from_worker(worker)
+        payload = worker.job["payload"]
+        payload["args"].first if payload
+      end
+    end
+  end
+end

data/lib/active_worker/job_queue/run_remotely.rb

@@ -0,0 +1,52 @@
+module ActiveWorker
+  module JobQueue
+    module RunRemotely
+
+      THREADED = "threaded"
+      RESQUE   = "resque"
+
+
+      def self.worker_mode=(mode)
+        @@worker_mode = mode
+      end
+
+      def self.worker_mode
+        @@worker_mode
+      end
+
+      self.worker_mode = RESQUE
+
+      class RemoteRunner
+        def initialize(klass)
+          @klass = klass
+        end
+
+        def method_missing(method,*params)
+          args = construct_args(method,params)
+          thread = nil
+          case RunRemotely.worker_mode
+            when THREADED
+              thread = Thread.new do
+                ActiveWorker::JobQueue::JobExecuter.execute_task_from_args(args)
+              end
+            when RESQUE
+              Resque.enqueue(JobExecuter,args)
+          end
+          thread
+        end
+
+        def construct_args(method, params)
+          {
+              "class_name" => @klass.to_s,
+              "method"     => method.to_s,
+              "params"     => params
+          }
+        end
+      end
+
+      def run_remotely
+        RemoteRunner.new(self.to_s)
+      end
+    end
+  end
+end