qs 0.5.0 → 0.6.0

data/Gemfile

@@ -2,8 +2,8 @@ source 'https://rubygems.org'
 
 gemspec
 
-gem 'rake'
-gem 'pry', "~> 0.9.0"
+gem 'rake', "~> 10.4.0"
+gem 'pry',  "~> 0.9.0"
 
 platform :ruby_18 do
   gem 'json', '~> 1.8'

data/bench/report.rb

@@ -4,7 +4,10 @@ require 'bench/setup'
 
 class BenchRunner
 
-  TIME_MODIFIER = 10 ** 4 # 4 decimal places
+  BUNDLE_EXEC        = "bundle exec --keep-file-descriptors".freeze
+  RUN_QS_BENCH_QUEUE = "#{BUNDLE_EXEC} ./bin/qs bench/config.qs".freeze
+  RUN_QS_DISPATCHER  = "#{BUNDLE_EXEC} ./bin/qs bench/dispatcher.qs".freeze
+  TIME_MODIFIER      = 10 ** 4 # 4 decimal places
 
   def initialize
     output_file_path = if ENV['OUTPUT_FILE']
@@ -24,6 +27,13 @@ class BenchRunner
     @event_params     = { 'size' => 100_000 }
 
     @progress_reader, @progress_writer = IO.pipe
+    @run_qs_scmd_opts = {
+      :env => {
+        'BENCH_REPORT'      => 'yes',
+        'BENCH_PROGRESS_IO' => @progress_writer.fileno.to_s
+      },
+      :options => { @progress_writer => @progress_writer }
+    }
 
     @results = {}
   end
@@ -43,12 +53,12 @@ class BenchRunner
 
     size = @results.values.map(&:size).max
     output "\n", false
-    output "Enqueueing #{@number_of_jobs} Jobs Time: #{@results[:enqueueing_jobs].rjust(size)}"
-    output "Running #{@number_of_jobs} Jobs Time:    #{@results[:running_jobs].rjust(size)}"
+    output "Enqueueing #{@number_of_jobs} Jobs Time: #{@results[:enqueueing_jobs].rjust(size)}s"
+    output "Running #{@number_of_jobs} Jobs Time:    #{@results[:running_jobs].rjust(size)}s"
 
     output "\n", false
-    output "Publishing #{@number_of_events} Events Time: #{@results[:publishing_events].rjust(size)}"
-    output "Running #{@number_of_events} Events Time:    #{@results[:running_events].rjust(size)}"
+    output "Publishing #{@number_of_events} Events Time: #{@results[:publishing_events].rjust(size)}s"
+    output "Running #{@number_of_events} Events Time:    #{@results[:running_events].rjust(size)}s"
 
     output "\n"
     output "Done running benchmark report"
@@ -69,11 +79,7 @@ class BenchRunner
   end
 
   def benchmark_running_jobs
-    cmd_str = "bundle exec ./bin/qs bench/config.qs"
-    cmd = Scmd.new(cmd_str, {
-      'BENCH_REPORT'      => 'yes',
-      'BENCH_PROGRESS_IO' => @progress_writer.fileno
-    })
+    cmd = Scmd.new(RUN_QS_BENCH_QUEUE, @run_qs_scmd_opts)
 
     output "Running jobs"
     begin
@@ -113,17 +119,8 @@ class BenchRunner
   end
 
   def benchmark_running_events
-    bench_queue_cmd_str = "bundle exec ./bin/qs bench/config.qs"
-    bench_queue_cmd = Scmd.new(bench_queue_cmd_str, {
-      'BENCH_REPORT'      => 'yes',
-      'BENCH_PROGRESS_IO' => @progress_writer.fileno
-    })
-
-    dispatcher_queue_cmd_str = "bundle exec ./bin/qs bench/dispatcher.qs"
-    dispatcher_queue_cmd = Scmd.new(dispatcher_queue_cmd_str, {
-      'BENCH_REPORT'      => 'yes',
-      'BENCH_PROGRESS_IO' => @progress_writer.fileno
-    })
+    bench_queue_cmd      = Scmd.new(RUN_QS_BENCH_QUEUE, @run_qs_scmd_opts)
+    dispatcher_queue_cmd = Scmd.new(RUN_QS_DISPATCHER, @run_qs_scmd_opts)
 
     output "Running events"
     begin

data/bench/report.txt

@@ -9,10 +9,10 @@ Publishing events
 Running events
 ....................................................................................................
 
-Enqueueing 10000 Jobs Time:  1.6858
-Running 10000 Jobs Time:    11.3011
+Enqueueing 10000 Jobs Time:  1.7498s
+Running 10000 Jobs Time:    13.0530s
 
-Publishing 10000 Events Time:  2.4596
-Running 10000 Events Time:    16.0522
+Publishing 10000 Events Time:  2.6319s
+Running 10000 Events Time:    17.8165s
 
 Done running benchmark report

data/lib/qs/daemon.rb

@@ -1,4 +1,5 @@
 require 'dat-worker-pool'
+require 'much-plugin'
 require 'ns-options'
 require 'pathname'
 require 'system_timer'
@@ -6,24 +7,22 @@ require 'thread'
 require 'qs'
 require 'qs/client'
 require 'qs/daemon_data'
-require 'qs/io_pipe'
 require 'qs/logger'
-require 'qs/payload_handler'
 require 'qs/queue_item'
+require 'qs/worker'
 
 module Qs
 
   module Daemon
+    include MuchPlugin
 
     InvalidError = Class.new(ArgumentError)
 
     SIGNAL = '.'.freeze
 
-    def self.included(klass)
-      klass.class_eval do
-        extend ClassMethods
-        include InstanceMethods
-      end
+    plugin_included do
+      extend ClassMethods
+      include InstanceMethods
     end
 
     module InstanceMethods
@@ -31,29 +30,39 @@ module Qs
       attr_reader :daemon_data, :logger
       attr_reader :signals_redis_key, :queue_redis_keys
 
-      # * Set the size of the client to the max workers + 1. This ensures we
-      #   have 1 connection for fetching work from redis and at least 1
-      #   connection for each worker to requeue its message when hard-shutdown.
+      # set the size of the client to the num workers + 1, this ensures we have
+      # 1 connection for fetching work from redis and at least 1 connection for
+      # each worker to requeue its message when hard-shutdown
       def initialize
         self.class.configuration.validate!
         Qs.init
         @daemon_data = DaemonData.new(self.class.configuration.to_hash)
-        @logger = @daemon_data.logger
+        @logger      = @daemon_data.logger
 
         @client = QsClient.new(Qs.redis_config.merge({
           :timeout => 1,
-          :size    => self.daemon_data.max_workers + 1
+          :size    => self.daemon_data.num_workers + 1
         }))
         @queue_redis_keys = self.daemon_data.queue_redis_keys
 
-        @work_loop_thread = nil
-        @worker_pool      = nil
-
         @signals_redis_key = "signals:#{@daemon_data.name}-" \
                              "#{Socket.gethostname}-#{::Process.pid}"
 
-        @worker_available_io = IOPipe.new
-        @signal = Signal.new(:stop)
+        @worker_available = WorkerAvailable.new
+
+        @worker_pool = DatWorkerPool.new(self.daemon_data.worker_class, {
+          :num_workers   => self.daemon_data.num_workers,
+          :logger        => self.daemon_data.dwp_logger,
+          :worker_params => self.daemon_data.worker_params.merge({
+            :qs_daemon_data      => self.daemon_data,
+            :qs_client           => @client,
+            :qs_worker_available => @worker_available,
+            :qs_logger           => @logger
+          })
+        })
+
+        @thread = nil
+        @state = State.new(:stop)
       rescue InvalidError => exception
         exception.set_backtrace(caller)
         raise exception
@@ -72,158 +81,98 @@ module Qs
       end
 
       def running?
-        !!(@work_loop_thread && @work_loop_thread.alive?)
+        !!(@thread && @thread.alive?)
       end
 
-      # * Ping redis to check that it can communicate with redis before running,
-      #   this is friendlier than starting and continously erroring because it
-      #   can't dequeue.
+      # ping to check that it can communicate with redis before running, this is
+      # friendlier than starting and continously erroring because it can't
+      # dequeue
       def start
         @client.ping
-        @signal.set :start
-        @work_loop_thread ||= Thread.new{ work_loop }
+        @state.set :run
+        @thread ||= Thread.new{ work_loop }
       end
 
       def stop(wait = false)
         return unless self.running?
-        @signal.set :stop
-        wakeup_work_loop_thread
+        @state.set :stop
+        wakeup_thread
         wait_for_shutdown if wait
       end
 
       def halt(wait = false)
         return unless self.running?
-        @signal.set :halt
-        wakeup_work_loop_thread
+        @state.set :halt
+        wakeup_thread
         wait_for_shutdown if wait
       end
 
       private
 
-      def process(queue_item)
-        Qs::PayloadHandler.new(self.daemon_data, queue_item).run
-      end
-
       def work_loop
-        log "Starting work loop", :debug
-        setup_redis_and_ios
-        @worker_pool = build_worker_pool
-        process_inputs while @signal.start?
-        log "Stopping work loop", :debug
+        setup
+        fetch_messages while @state.run?
       rescue StandardError => exception
-        @signal.set :stop
+        @state.set :stop
         log "Error occurred while running the daemon, exiting", :error
         log "#{exception.class}: #{exception.message}", :error
-        log exception.backtrace.join("\n"), :error
+        (exception.backtrace || []).each{ |l| log(l, :error) }
       ensure
-        shutdown_worker_pool
-        @worker_available_io.teardown
-        @work_loop_thread = nil
-        log "Stopped work loop", :debug
+        teardown
       end
 
-      def setup_redis_and_ios
-        # clear any signals that are already on the signals redis list
+      # clear any signals that are already on the signals list in redis
+      def setup
         @client.clear(self.signals_redis_key)
-        @worker_available_io.setup
-      end
-
-      def build_worker_pool
-        wp = DatWorkerPool.new(
-          self.daemon_data.min_workers,
-          self.daemon_data.max_workers
-        ){ |queue_item| process(queue_item) }
-
-        # add internal callbacks
-        wp.on_worker_error do |worker, exception, queue_item|
-          handle_worker_exception(exception, queue_item)
+        @worker_pool.start
+      end
+
+      # shuffle the queue redis keys to avoid queue starvation, redis will pull
+      # messages off queues in the order they are passed to the command, by
+      # shuffling we ensure they are randomly ordered so every queue should get
+      # a chance; use 0 for the brpop timeout which means block indefinitely;
+      # rescue runtime errors so the daemon thread doesn't fail if redis is
+      # temporarily down, sleep for a second to keep the thread from thrashing
+      # by repeatedly erroring if redis is down
+      def fetch_messages
+        if !@worker_pool.worker_available? && @state.run?
+          @worker_available.wait
         end
-        wp.on_worker_sleep{ @worker_available_io.write(SIGNAL) }
-
-        # add any configured callbacks
-        self.daemon_data.worker_start_procs.each{ |cb| wp.on_worker_start(&cb) }
-        self.daemon_data.worker_shutdown_procs.each{ |cb|  wp.on_worker_shutdown(&cb) }
-        self.daemon_data.worker_sleep_procs.each{ |cb| wp.on_worker_sleep(&cb) }
-        self.daemon_data.worker_wakeup_procs.each{ |cb| wp.on_worker_wakeup(&cb) }
-
-        wp.start
-        wp
-      end
-
-      # * Shuffle the queue redis keys to avoid queue starvation. Redis will
-      #   pull messages off queues in the order they are passed to the command,
-      #   by shuffling we ensure they are randomly ordered so every queue should
-      #   get a chance.
-      # * Use 0 for the brpop timeout which means block indefinitely.
-      # * Rescue runtime errors so the daemon thread doesn't fail if redis is
-      #   temporarily down. Sleep for a second to keep the thread from thrashing
-      #   by repeatedly erroring if redis is down.
-      def process_inputs
-        wait_for_available_worker
-        return unless @worker_pool.worker_available? && @signal.start?
+        return unless @worker_pool.worker_available? && @state.run?
 
         begin
           args = [self.signals_redis_key, self.queue_redis_keys.shuffle, 0].flatten
           redis_key, encoded_payload = @client.block_dequeue(*args)
           if redis_key != @signals_redis_key
-            @worker_pool.add_work(QueueItem.new(redis_key, encoded_payload))
+            @worker_pool.push(QueueItem.new(redis_key, encoded_payload))
           end
         rescue RuntimeError => exception
-          log "Error dequeueing #{exception.message.inspect}", :error
-          log exception.backtrace.join("\n"), :error
+          log "Error occurred while dequeueing", :error
+          log "#{exception.class}: #{exception.message}", :error
+          (exception.backtrace || []).each{ |l| log(l, :error) }
           sleep 1
         end
       end
 
-      def wait_for_available_worker
-        if !@worker_pool.worker_available? && @signal.start?
-          @worker_available_io.wait
-          @worker_available_io.read
-        end
-      end
-
-      def shutdown_worker_pool
-        return unless @worker_pool
-        timeout = @signal.stop? ? self.daemon_data.shutdown_timeout : 0
-        if timeout
-          log "Shutting down, waiting up to #{timeout} seconds for work to finish"
-        else
-          log "Shutting down, waiting for work to finish"
-        end
+      def teardown
+        timeout = @state.halt? ? 0 : self.daemon_data.shutdown_timeout
         @worker_pool.shutdown(timeout)
+
         log "Requeueing #{@worker_pool.work_items.size} message(s)"
-        @worker_pool.work_items.each do |ri|
-          @client.prepend(ri.queue_redis_key, ri.encoded_payload)
+        @worker_pool.work_items.each do |qi|
+          @client.prepend(qi.queue_redis_key, qi.encoded_payload)
         end
+      ensure
+        @thread = nil
       end
 
-      def wait_for_shutdown
-        @work_loop_thread.join if @work_loop_thread
+      def wakeup_thread
+        @client.append(self.signals_redis_key, SIGNAL)
+        @worker_available.signal
       end
 
-      def wakeup_work_loop_thread
-        @client.append(self.signals_redis_key, SIGNAL)
-        @worker_available_io.write(SIGNAL)
-      end
-
-      # * This only catches errors that happen outside of running the payload
-      #   handler. The only known use-case for this is dat worker pools
-      #   hard-shutdown errors.
-      # * If there isn't a queue item (this can happen when an idle worker is
-      #   being forced to exit) then we don't need to do anything.
-      # * If we never started processing the queue item, its safe to requeue it.
-      #   Otherwise it happened while processing so the payload handler caught
-      #   it or it happened after the payload handler which we don't care about.
-      def handle_worker_exception(exception, queue_item)
-        return if queue_item.nil?
-        if !queue_item.started
-          log "Worker error, requeueing message because it hasn't started", :error
-          @client.prepend(queue_item.queue_redis_key, queue_item.encoded_payload)
-        else
-          log "Worker error after message was processed, ignoring", :error
-        end
-        log "#{exception.class}: #{exception.message}", :error
-        log exception.backtrace.join("\n"), :error
+      def wait_for_shutdown
+        @thread.join if @thread
       end
 
       def log(message, level = :info)
@@ -246,34 +195,20 @@ module Qs
         self.configuration.pid_file(*args)
       end
 
-      def min_workers(*args)
-        self.configuration.min_workers(*args)
-      end
-
-      def max_workers(*args)
-        self.configuration.max_workers(*args)
-      end
-
-      def workers(*args)
-        self.min_workers(*args)
-        self.max_workers(*args)
-      end
-
-      def on_worker_start(&block)
-        self.configuration.worker_start_procs << block
-      end
-
-      def on_worker_shutdown(&block)
-        self.configuration.worker_shutdown_procs << block
+      def worker_class(new_worker_class = nil)
+        self.configuration.worker_class = new_worker_class if new_worker_class
+        self.configuration.worker_class
       end
 
-      def on_worker_sleep(&block)
-        self.configuration.worker_sleep_procs << block
+      def worker_params(new_worker_params = nil )
+        self.configuration.worker_params = new_worker_params if new_worker_params
+        self.configuration.worker_params
       end
 
-      def on_worker_wakeup(&block)
-        self.configuration.worker_wakeup_procs << block
+      def num_workers(*args)
+        self.configuration.num_workers(*args)
       end
+      alias :workers :num_workers
 
       def verbose_logging(*args)
         self.configuration.verbose_logging(*args)
@@ -307,28 +242,24 @@ module Qs
       option :name,     String,  :required => true
       option :pid_file, Pathname
 
-      option :min_workers, Integer, :default => 1
-      option :max_workers, Integer, :default => 4
+      option :num_workers, Integer, :default => 4
 
       option :verbose_logging, :default => true
       option :logger,          :default => proc{ Qs::NullLogger.new }
 
       option :shutdown_timeout
 
-      attr_accessor :process_label
       attr_accessor :init_procs, :error_procs
+      attr_accessor :worker_class, :worker_params
       attr_accessor :queues
-      attr_reader :worker_start_procs, :worker_shutdown_procs
-      attr_reader :worker_sleep_procs, :worker_wakeup_procs
 
       def initialize(values = nil)
         super(values)
-        @process_label = !(v = ENV['QS_PROCESS_LABEL'].to_s).empty? ? v : self.name
         @init_procs, @error_procs = [], []
-        @worker_start_procs, @worker_shutdown_procs = [], []
-        @worker_sleep_procs, @worker_wakeup_procs   = [], []
+        @worker_class  = DefaultWorker
+        @worker_params = nil
         @queues = []
-        @valid  = nil
+        @valid = nil
       end
 
       def routes
@@ -337,14 +268,11 @@ module Qs
 
       def to_hash
         super.merge({
-          :process_label         => self.process_label,
-          :error_procs           => self.error_procs,
-          :worker_start_procs    => self.worker_start_procs,
-          :worker_shutdown_procs => self.worker_shutdown_procs,
-          :worker_sleep_procs    => self.worker_sleep_procs,
-          :worker_wakeup_procs   => self.worker_wakeup_procs,
-          :routes                => self.routes,
-          :queue_redis_keys      => self.queues.map(&:redis_key)
+          :error_procs      => self.error_procs,
+          :worker_class     => self.worker_class,
+          :worker_params    => self.worker_params,
+          :routes           => self.routes,
+          :queue_redis_keys => self.queues.map(&:redis_key)
         })
       end
 
@@ -358,32 +286,30 @@ module Qs
         if self.queues.empty? || !self.required_set?
           raise InvalidError, "a name and queue must be configured"
         end
+        if !self.worker_class.kind_of?(Class) || !self.worker_class.include?(Qs::Worker)
+          raise InvalidError, "worker class must include `#{Qs::Worker}`"
+        end
         self.routes.each(&:validate!)
         @valid = true
       end
     end
 
-    class Signal
-      def initialize(value)
-        @value = value
-        @mutex = Mutex.new
-      end
-
-      def set(value)
-        @mutex.synchronize{ @value = value }
-      end
+    DefaultWorker = Class.new{ include Qs::Worker }
 
-      def start?
-        @mutex.synchronize{ @value == :start }
+    class WorkerAvailable
+      def initialize
+        @mutex    = Mutex.new
+        @cond_var = ConditionVariable.new
       end
 
-      def stop?
-        @mutex.synchronize{ @value == :stop }
-      end
+      def wait;   @mutex.synchronize{ @cond_var.wait(@mutex) }; end
+      def signal; @mutex.synchronize{ @cond_var.signal };       end
+    end
 
-      def halt?
-        @mutex.synchronize{ @value == :halt }
-      end
+    class State < DatWorkerPool::LockedObject
+      def run?;  self.value == :run;  end
+      def stop?; self.value == :stop; end
+      def halt?; self.value == :halt; end
     end
 
   end

data/lib/qs/daemon_data.rb

@@ -7,33 +7,32 @@ module Qs
     # options one time here and memoize their values. This way, we don't pay the
     # NsOptions overhead when reading them while handling a message.
 
-    attr_reader :name, :process_label
-    attr_reader :pid_file
-    attr_reader :min_workers, :max_workers
-    attr_reader :worker_start_procs, :worker_shutdown_procs
-    attr_reader :worker_sleep_procs, :worker_wakeup_procs
-    attr_reader :logger, :verbose_logging
+    attr_reader :name, :process_label, :pid_file
+    attr_reader :worker_class, :worker_params, :num_workers
+    attr_reader :debug, :logger, :dwp_logger, :verbose_logging
     attr_reader :shutdown_timeout
-    attr_reader :error_procs
-    attr_reader :queue_redis_keys, :routes
+    attr_reader :error_procs, :queue_redis_keys, :routes
 
     def initialize(args = nil)
       args ||= {}
-      @name                  = args[:name]
-      @process_label         = args[:process_label]
-      @pid_file              = args[:pid_file]
-      @min_workers           = args[:min_workers]
-      @max_workers           = args[:max_workers]
-      @worker_start_procs    = args[:worker_start_procs]
-      @worker_shutdown_procs = args[:worker_shutdown_procs]
-      @worker_sleep_procs    = args[:worker_sleep_procs]
-      @worker_wakeup_procs   = args[:worker_wakeup_procs]
-      @logger                = args[:logger]
-      @verbose_logging       = !!args[:verbose_logging]
-      @shutdown_timeout      = args[:shutdown_timeout]
-      @error_procs           = args[:error_procs] || []
-      @queue_redis_keys      = args[:queue_redis_keys] || []
-      @routes                = build_routes(args[:routes] || [])
+      @name          = args[:name]
+      @process_label = !(v = ENV['QS_PROCESS_LABEL'].to_s).empty? ? v : args[:name]
+      @pid_file      = args[:pid_file]
+
+      @worker_class  = args[:worker_class]
+      @worker_params = args[:worker_params] || {}
+      @num_workers   = args[:num_workers]
+
+      @debug           = !ENV['QS_DEBUG'].to_s.empty?
+      @logger          = args[:logger]
+      @dwp_logger      = @logger if @debug
+      @verbose_logging = !!args[:verbose_logging]
+
+      @shutdown_timeout = args[:shutdown_timeout]
+
+      @error_procs      = args[:error_procs] || []
+      @queue_redis_keys = args[:queue_redis_keys] || []
+      @routes           = build_routes(args[:routes] || [])
     end
 
     def route_for(route_id)

data/lib/qs/event_handler.rb

@@ -1,14 +1,14 @@
+require 'much-plugin'
 require 'qs/message_handler'
 
 module Qs
 
   module EventHandler
+    include MuchPlugin
 
-    def self.included(klass)
-      klass.class_eval do
-        include Qs::MessageHandler
-        include InstanceMethods
-      end
+    plugin_included do
+      include Qs::MessageHandler
+      include InstanceMethods
     end
 
     module InstanceMethods
@@ -29,6 +29,22 @@ module Qs
 
     end
 
+    module TestHelpers
+
+      def self.included(klass)
+        require 'qs/test_runner'
+      end
+
+      def test_runner(handler_class, args = nil)
+        Qs::EventTestRunner.new(handler_class, args)
+      end
+
+      def test_handler(handler_class, args = nil)
+        test_runner(handler_class, args).handler
+      end
+
+    end
+
   end
 
 end