qs 0.5.0 → 0.6.0

data/lib/qs/job_handler.rb

@@ -1,14 +1,14 @@
+require 'much-plugin'
 require 'qs/message_handler'
 
 module Qs
 
   module JobHandler
+    include MuchPlugin
 
-    def self.included(klass)
-      klass.class_eval do
-        include Qs::MessageHandler
-        include InstanceMethods
-      end
+    plugin_included do
+      include Qs::MessageHandler
+      include InstanceMethods
     end
 
     module InstanceMethods
@@ -28,6 +28,22 @@ module Qs
 
     end
 
+    module TestHelpers
+
+      def self.included(klass)
+        require 'qs/test_runner'
+      end
+
+      def test_runner(handler_class, args = nil)
+        Qs::JobTestRunner.new(handler_class, args)
+      end
+
+      def test_handler(handler_class, args = nil)
+        test_runner(handler_class, args).handler
+      end
+
+    end
+
   end
 
 end

data/lib/qs/message_handler.rb

@@ -1,12 +1,13 @@
+require 'much-plugin'
+
 module Qs
 
   module MessageHandler
+    include MuchPlugin
 
-    def self.included(klass)
-      klass.class_eval do
-        extend ClassMethods
-        include InstanceMethods
-      end
+    plugin_included do
+      extend ClassMethods
+      include InstanceMethods
     end
 
     module InstanceMethods
@@ -15,37 +16,40 @@ module Qs
         @qs_runner = runner
       end
 
-      def init
-        run_callback 'before_init'
+      def qs_init
+        self.qs_run_callback 'before_init'
         self.init!
-        run_callback 'after_init'
+        self.qs_run_callback 'after_init'
       end
 
       def init!
       end
 
-      def run
-        run_callback 'before_run'
+      def qs_run
+        self.qs_run_callback 'before_run'
         self.run!
-        run_callback 'after_run'
+        self.qs_run_callback 'after_run'
       end
 
       def run!
-        raise NotImplementedError
+      end
+
+      def qs_run_callback(callback)
+        (self.class.send("#{callback}_callbacks") || []).each do |callback|
+          self.instance_eval(&callback)
+        end
+      end
+
+      def ==(other_handler)
+        self.class == other_handler.class
       end
 
       private
 
       # Helpers
 
-      def params; @qs_runner.params; end
       def logger; @qs_runner.logger; end
-
-      def run_callback(callback)
-        (self.class.send("#{callback}_callbacks") || []).each do |callback|
-          self.instance_eval(&callback)
-        end
-      end
+      def params; @qs_runner.params; end
 
     end
 

data/lib/qs/payload_handler.rb

@@ -26,7 +26,6 @@ module Qs
       benchmark = Benchmark.measure{ run!(@daemon_data, @queue_item) }
       @queue_item.time_taken = RoundedTime.new(benchmark.real)
       log_complete(@queue_item)
-      raise_if_debugging!(@queue_item.exception)
     end
 
     private
@@ -67,10 +66,6 @@ module Qs
       log_exception(queue_item.exception)
     end
 
-    def raise_if_debugging!(exception)
-      raise exception if exception && ENV['QS_DEBUG']
-    end
-
     def log_received
       log_verbose "===== Received message ====="
     end
@@ -99,9 +94,8 @@ module Qs
     end
 
     def log_exception(exception)
-      backtrace = exception.backtrace.join("\n")
-      message = "#{exception.class}: #{exception.message}\n#{backtrace}"
-      log_verbose(message, :error)
+      log_verbose("#{exception.class}: #{exception.message}", :error)
+      (exception.backtrace || []).each{ |l| log_verbose(l, :error) }
     end
 
     def log_verbose(message, level = :info)

data/lib/qs/qs_runner.rb

@@ -15,10 +15,10 @@ module Qs
 
     def run
       OptionalTimeout.new(self.timeout) do
-        run_callbacks self.handler_class.before_callbacks
-        self.handler.init
-        self.handler.run
-        run_callbacks self.handler_class.after_callbacks
+        self.handler.qs_run_callback 'before'
+        self.handler.qs_init
+        self.handler.qs_run
+        self.handler.qs_run_callback 'after'
       end
     rescue TimeoutError => exception
       error = TimeoutError.new "#{handler_class} timed out (#{timeout}s)"
@@ -28,10 +28,6 @@ module Qs
 
     private
 
-    def run_callbacks(callbacks)
-      callbacks.each{ |proc| self.handler.instance_eval(&proc) }
-    end
-
     module OptionalTimeout
       def self.new(timeout, &block)
         if !timeout.nil?

data/lib/qs/runner.rb

@@ -5,16 +5,15 @@ module Qs
   class Runner
 
     attr_reader :handler_class, :handler
-    attr_reader :message, :params, :logger
+    attr_reader :logger, :message, :params
 
     def initialize(handler_class, args = nil)
-      @handler_class = handler_class
-
-      a = args || {}
-      @message = a[:message]
-      @params  = a[:params] || {}
-      @logger  = a[:logger] || Qs::NullLogger.new
+      args ||= {}
+      @logger  = args[:logger] || Qs::NullLogger.new
+      @message = args[:message]
+      @params  = args[:params] || {}
 
+      @handler_class = handler_class
       @handler = @handler_class.new(self)
     end
 

data/lib/qs/test_runner.rb

@@ -9,25 +9,25 @@ module Qs
   class TestRunner < Runner
 
     def initialize(handler_class, args = nil)
-      args = (args || {}).dup
+      a = (args || {}).dup
       super(handler_class, {
-        :message => args.delete(:message),
-        :params  => normalize_params(args.delete(:params) || {}),
-        :logger  => args.delete(:logger)
+        :logger  => a.delete(:logger),
+        :message => a.delete(:message),
+        :params  => normalize_params(a.delete(:params) || {})
       })
-      args.each{ |key, value| self.handler.send("#{key}=", value) }
+      a.each{ |key, value| self.handler.send("#{key}=", value) }
 
-      self.handler.init
+      self.handler.qs_init
     end
 
     def run
-      self.handler.run
+      self.handler.qs_run
     end
 
     private
 
-    # Stringify and encode/decode to ensure params are valid and are
-    # in the format they would normally be when a handler is built and run.
+    # stringify and encode/decode to ensure params are valid and are
+    # in the format they would normally be when a live handler is built and run.
     def normalize_params(params)
       params = Qs::Payload::StringifyParams.new(params)
       Qs.decode(Qs.encode(params))
@@ -39,13 +39,13 @@ module Qs
 
     def initialize(handler_class, args = nil)
       if !handler_class.include?(Qs::JobHandler)
-        raise InvalidJobHandlerError, "#{handler_class.inspect} is not a"\
-                                      " Qs::JobHandler"
+        raise InvalidJobHandlerError, "#{handler_class.inspect} is not a " \
+                                      "Qs::JobHandler"
       end
 
-      args = (args || {}).dup
-      args[:message] = args.delete(:job) if args.key?(:job)
-      super(handler_class, args)
+      a = (args || {}).dup
+      a[:message] = a.delete(:job) if a.key?(:job)
+      super(handler_class, a)
     end
 
   end
@@ -54,13 +54,13 @@ module Qs
 
     def initialize(handler_class, args = nil)
       if !handler_class.include?(Qs::EventHandler)
-        raise InvalidEventHandlerError, "#{handler_class.inspect} is not a"\
-                                      " Qs::EventHandler"
+        raise InvalidEventHandlerError, "#{handler_class.inspect} is not a " \
+                                        "Qs::EventHandler"
       end
 
-      args = (args || {}).dup
-      args[:message] = args.delete(:event) if args.key?(:event)
-      super(handler_class, args)
+      a = (args || {}).dup
+      a[:message] = a.delete(:event) if a.key?(:event)
+      super(handler_class, a)
     end
 
   end

data/lib/qs/version.rb

@@ -1,3 +1,3 @@
 module Qs
-  VERSION = "0.5.0"
+  VERSION = "0.6.0"
 end

data/lib/qs/worker.rb

@@ -0,0 +1,88 @@
+require 'dat-worker-pool/worker'
+require 'much-plugin'
+require 'qs/client'
+require 'qs/daemon'
+require 'qs/daemon_data'
+require 'qs/payload_handler'
+
+module Qs
+
+  module Worker
+    include MuchPlugin
+
+    plugin_included do
+      include DatWorkerPool::Worker
+      include InstanceMethods
+
+      on_available{ params[:qs_worker_available].signal }
+
+      on_error{ |e, wi| qs_handle_exception(e, wi) }
+
+    end
+
+    module InstanceMethods
+
+      def work!(queue_item)
+        Qs::PayloadHandler.new(params[:qs_daemon_data], queue_item).run
+      end
+
+      private
+
+      # this only catches errors that happen outside of running the payload
+      # handler, the only known use-case for this is dwps shutdown errors; if
+      # there isn't a queue item (this can happen when an idle worker is being
+      # forced to exit) then we don't need to do anything; if we never started
+      # processing the queue item, its safe to requeue it, otherwise it happened
+      # while it was being processed (by the payload handler) or after it was
+      # processed, for these cases, either the payload handler caught the error
+      # (while it was being processed) or we don't care because its been
+      # processed and the worker is just finishing up
+      def qs_handle_exception(exception, queue_item)
+        return if queue_item.nil?
+        if !queue_item.started
+          qs_log "Worker error, requeueing message because it hasn't started", :error
+          params[:qs_client].prepend(
+            queue_item.queue_redis_key,
+            queue_item.encoded_payload
+          )
+        else
+          qs_log "Worker error after message was processed, ignoring", :error
+        end
+        qs_log "#{exception.class}: #{exception.message}", :error
+        qs_log (exception.backtrace || []).join("\n"), :error
+      end
+
+      def qs_log(message, level = :info)
+        params[:qs_logger].send(level, "[Qs-#{number}] #{message}")
+      end
+
+    end
+
+    module TestHelpers
+      include MuchPlugin
+
+      plugin_included do
+        include DatWorkerPool::Worker::TestHelpers
+        include InstanceMethods
+      end
+
+      module InstanceMethods
+
+        def test_runner(worker_class, options = nil)
+          options ||= {}
+          options[:params] = {
+            :qs_daemon_data      => Qs::DaemonData.new,
+            :qs_client           => Qs::TestClient.new({}),
+            :qs_worker_available => Qs::Daemon::WorkerAvailable.new,
+            :qs_logger           => Qs::NullLogger.new
+          }.merge(options[:params] || {})
+          super(worker_class, options)
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/qs.gemspec

@@ -18,11 +18,13 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
-  gem.add_dependency("dat-worker-pool", ["~> 0.5"])
+  gem.add_dependency("dat-worker-pool", ["~> 0.6"])
   gem.add_dependency("hella-redis",     ["~> 0.3"])
+  gem.add_dependency("much-plugin",     ["~> 0.1"])
   gem.add_dependency("ns-options",      ["~> 1.1"])
   gem.add_dependency("SystemTimer",     ["~> 1.2"])
 
   gem.add_development_dependency("assert", ["~> 2.15"])
-  gem.add_development_dependency("scmd",   ["~> 2.3"])
+  gem.add_development_dependency("scmd",   ["~> 3.0"])
+
 end

data/test/support/client_spy.rb

@@ -0,0 +1,43 @@
+require 'qs/client'
+
+class ClientSpy < Qs::TestClient
+  attr_reader :calls
+
+  def initialize(redis_config = nil)
+    super(redis_config || {})
+    @calls = []
+    @list  = []
+    @mutex = Mutex.new
+    @cv    = ConditionVariable.new
+  end
+
+  def block_dequeue(*args)
+    @calls << Call.new(:block_dequeue, args)
+    if @list.empty?
+      @mutex.synchronize{ @cv.wait(@mutex) }
+    end
+    @list.shift
+  end
+
+  def append(*args)
+    @calls << Call.new(:append, args)
+    @list  << args
+    @cv.signal
+  end
+
+  def prepend(*args)
+    @calls << Call.new(:prepend, args)
+    @list  << args
+    @cv.signal
+  end
+
+  def clear(*args)
+    @calls << Call.new(:clear, args)
+  end
+
+  def ping
+    @calls << Call.new(:ping)
+  end
+
+  Call = Struct.new(:command, :args)
+end

data/test/support/factory.rb

@@ -10,7 +10,8 @@ module Factory
     klass ||= StandardError
     message ||= Factory.text
     exception = nil
-    begin; raise(klass, message); rescue StandardError => exception; end
+    begin; raise(klass, message); rescue klass => exception; end
+    exception.set_backtrace(nil) if Factory.boolean
     exception
   end
 

data/test/support/message_handler.rb

@@ -0,0 +1,17 @@
+require 'qs/message_handler'
+require 'qs/test_runner'
+
+# manually define some test helpers since we don't provide helpers for
+# general message handlers as the user facing helpers are the job/event ones
+
+module Qs::MessageHandler
+
+  module TestHelpers
+
+    def test_runner(handler_class, args = nil)
+      Qs::TestRunner.new(handler_class, args)
+    end
+
+  end
+
+end

data/test/system/daemon_tests.rb

@@ -144,27 +144,6 @@ module Qs::Daemon
 
   end
 
-  class NoWorkersAvailableTests < SystemTests
-    desc "when no workers are available"
-    setup do
-      AppDaemon.workers 0 # no workers available, don't do this
-      setup_app_and_dispatcher_daemon
-    end
-
-    should "shutdown when stopped" do
-      @app_daemon.stop
-      @app_thread.join 2 # give it time to shutdown, should be faster
-      assert_false @app_thread.alive?
-    end
-
-    should "shutdown when halted" do
-      @app_daemon.halt
-      @app_thread.join 2 # give it time to shutdown, should be faster
-      assert_false @app_thread.alive?
-    end
-
-  end
-
   class ShutdownWithoutTimeoutTests < SystemTests
     desc "without a shutdown timeout"
     setup do
@@ -270,6 +249,24 @@ module Qs::Daemon
 
   end
 
+  class WithEnvProcessLabelTests < SystemTests
+    desc "with a process label env var"
+    setup do
+      ENV['QS_PROCESS_LABEL'] = Factory.string
+
+      @daemon = AppDaemon.new
+    end
+    teardown do
+      ENV.delete('QS_PROCESS_LABEL')
+    end
+    subject{ @daemon }
+
+    should "set the daemons process label to the env var" do
+      assert_equal ENV['QS_PROCESS_LABEL'], subject.process_label
+    end
+
+  end
+
   class DaemonRunner
     def initialize(app_daemon, dispatcher_daemon = nil)
       @app_daemon = app_daemon

data/test/unit/daemon_data_tests.rb

@@ -9,71 +9,87 @@ class Qs::DaemonData
   class UnitTests < Assert::Context
     desc "Qs::DaemonData"
     setup do
-      @name             = Factory.string
-      @process_label    = Factory.string
-      @pid_file         = Factory.file_path
-      @min_workers      = Factory.integer
-      @max_workers      = Factory.integer
-      @start_procs      = Factory.integer(3).times.map{ proc{} }
-      @shutdown_procs   = Factory.integer(3).times.map{ proc{} }
-      @sleep_procs      = Factory.integer(3).times.map{ proc{} }
-      @wakeup_procs     = Factory.integer(3).times.map{ proc{} }
-      @logger           = Factory.string
-      @verbose_logging  = Factory.boolean
-      @shutdown_timeout = Factory.integer
-      @error_procs      = [ proc{ Factory.string } ]
-      @queue_redis_keys = Factory.integer(3).times.map{ Factory.string }
+      @current_env_process_label = ENV['QS_PROCESS_LABEL']
+      ENV['QS_PROCESS_LABEL']    = Factory.string
+
+      @current_env_debug = ENV['QS_DEBUG']
+      ENV['QS_DEBUG']    = Factory.string
 
       @routes = (0..Factory.integer(3)).map do
         Qs::Route.new(Factory.string, TestHandler.to_s).tap(&:validate!)
       end
 
-      @daemon_data = Qs::DaemonData.new({
-        :name                  => @name,
-        :process_label         => @process_label,
-        :pid_file              => @pid_file,
-        :min_workers           => @min_workers,
-        :max_workers           => @max_workers,
-        :worker_start_procs    => @start_procs,
-        :worker_shutdown_procs => @shutdown_procs,
-        :worker_sleep_procs    => @sleep_procs,
-        :worker_wakeup_procs   => @wakeup_procs,
-        :logger                => @logger,
-        :verbose_logging       => @verbose_logging,
-        :shutdown_timeout      => @shutdown_timeout,
-        :error_procs           => @error_procs,
-        :queue_redis_keys      => @queue_redis_keys,
-        :routes                => @routes
-      })
+      @config_hash = {
+        :name             => Factory.string,
+        :pid_file         => Factory.file_path,
+        :worker_class     => Class.new,
+        :worker_params    => { Factory.string => Factory.string },
+        :num_workers      => Factory.integer,
+        :logger           => Factory.string,
+        :verbose_logging  => Factory.boolean,
+        :shutdown_timeout => Factory.integer,
+        :error_procs      => [ proc{ Factory.string } ],
+        :queue_redis_keys => Factory.integer(3).times.map{ Factory.string },
+        :routes           => @routes
+      }
+      @daemon_data = Qs::DaemonData.new(@config_hash)
+    end
+    teardown do
+      ENV['QS_DEBUG']         = @current_env_debug
+      ENV['QS_PROCESS_LABEL'] = @current_env_process_label
     end
     subject{ @daemon_data }
 
-    should have_readers :name, :process_label
-    should have_readers :pid_file
-    should have_readers :min_workers, :max_workers
-    should have_readers :worker_start_procs, :worker_shutdown_procs
-    should have_readers :worker_sleep_procs, :worker_wakeup_procs
-    should have_readers :logger, :verbose_logging
+    should have_readers :name, :process_label, :pid_file
+    should have_readers :worker_class, :worker_params, :num_workers
+    should have_readers :debug, :logger, :dwp_logger, :verbose_logging
     should have_readers :shutdown_timeout
-    should have_readers :error_procs
-    should have_readers :queue_redis_keys, :routes
+    should have_readers :error_procs, :queue_redis_keys, :routes
     should have_imeths :route_for
 
     should "know its attributes" do
-      assert_equal @name,             subject.name
-      assert_equal @process_label,    subject.process_label
-      assert_equal @pid_file,         subject.pid_file
-      assert_equal @min_workers,      subject.min_workers
-      assert_equal @max_workers,      subject.max_workers
-      assert_equal @start_procs,      subject.worker_start_procs
-      assert_equal @shutdown_procs,   subject.worker_shutdown_procs
-      assert_equal @sleep_procs,      subject.worker_sleep_procs
-      assert_equal @wakeup_procs,     subject.worker_wakeup_procs
-      assert_equal @logger,           subject.logger
-      assert_equal @verbose_logging,  subject.verbose_logging
-      assert_equal @shutdown_timeout, subject.shutdown_timeout
-      assert_equal @error_procs,      subject.error_procs
-      assert_equal @queue_redis_keys, subject.queue_redis_keys
+      h = @config_hash
+      assert_equal h[:name],             subject.name
+      assert_equal h[:pid_file],         subject.pid_file
+      assert_equal h[:worker_class],     subject.worker_class
+      assert_equal h[:worker_params],    subject.worker_params
+      assert_equal h[:num_workers],      subject.num_workers
+      assert_equal h[:logger],           subject.logger
+      assert_equal h[:verbose_logging],  subject.verbose_logging
+      assert_equal h[:shutdown_timeout], subject.shutdown_timeout
+      assert_equal h[:error_procs],      subject.error_procs
+      assert_equal h[:queue_redis_keys], subject.queue_redis_keys
+    end
+
+    should "use process label env var if set" do
+      ENV['QS_PROCESS_LABEL'] = Factory.string
+      daemon_data = Qs::DaemonData.new(@config_hash)
+      assert_equal ENV['QS_PROCESS_LABEL'], daemon_data.process_label
+
+      ENV['QS_PROCESS_LABEL'] = ""
+      daemon_data = Qs::DaemonData.new(@config_hash)
+      assert_equal @config_hash[:name], daemon_data.process_label
+
+      ENV.delete('QS_PROCESS_LABEL')
+      daemon_data = Qs::DaemonData.new(@config_hash)
+      assert_equal @config_hash[:name], daemon_data.process_label
+    end
+
+    should "use debug env var if set" do
+      ENV['QS_DEBUG'] = Factory.string
+      daemon_data = Qs::DaemonData.new(@config_hash)
+      assert_true daemon_data.debug
+      assert_equal @config_hash[:logger], daemon_data.dwp_logger
+
+      ENV['QS_DEBUG'] = ""
+      daemon_data = Qs::DaemonData.new(@config_hash)
+      assert_false daemon_data.debug
+      assert_nil daemon_data.dwp_logger
+
+      ENV.delete('QS_DEBUG')
+      daemon_data = Qs::DaemonData.new(@config_hash)
+      assert_false daemon_data.debug
+      assert_nil daemon_data.dwp_logger
     end
 
     should "build a routes lookup hash" do
@@ -97,8 +113,9 @@ class Qs::DaemonData
       daemon_data = Qs::DaemonData.new
       assert_nil daemon_data.name
       assert_nil daemon_data.pid_file
-      assert_nil daemon_data.min_workers
-      assert_nil daemon_data.max_workers
+      assert_nil daemon_data.worker_class
+      assert_equal({}, daemon_data.worker_params)
+      assert_nil daemon_data.num_workers
       assert_nil daemon_data.logger
       assert_false daemon_data.verbose_logging
       assert_nil daemon_data.shutdown_timeout