qs 0.3.0 → 0.4.0

data/lib/qs/daemon.rb

@@ -9,7 +9,7 @@ require 'qs/daemon_data'
 require 'qs/io_pipe'
 require 'qs/logger'
 require 'qs/payload_handler'
-require 'qs/redis_item'
+require 'qs/queue_item'
 
 module Qs
 
@@ -33,7 +33,7 @@ module Qs
 
       # * Set the size of the client to the max workers + 1. This ensures we
       #   have 1 connection for fetching work from redis and at least 1
-      #   connection for each worker to requeue its job when hard-shutdown.
+      #   connection for each worker to requeue its message when hard-shutdown.
       def initialize
         self.class.configuration.validate!
         Qs.init
@@ -96,8 +96,8 @@ module Qs
 
       private
 
-      def process(redis_item)
-        Qs::PayloadHandler.new(self.daemon_data, redis_item).run
+      def process(queue_item)
+        Qs::PayloadHandler.new(self.daemon_data, queue_item).run
       end
 
       def work_loop
@@ -128,9 +128,9 @@ module Qs
         wp = DatWorkerPool.new(
           self.daemon_data.min_workers,
           self.daemon_data.max_workers
-        ){ |redis_item| process(redis_item) }
-        wp.on_worker_error do |worker, exception, redis_item|
-          handle_worker_exception(exception, redis_item)
+        ){ |queue_item| process(queue_item) }
+        wp.on_worker_error do |worker, exception, queue_item|
+          handle_worker_exception(exception, queue_item)
         end
         wp.on_worker_sleep{ @worker_available_io.write(SIGNAL) }
         wp.start
@@ -138,8 +138,8 @@ module Qs
       end
 
       # * Shuffle the queue redis keys to avoid queue starvation. Redis will
-      #   pull jobs off queues in the order they are passed to the command, by
-      #   shuffling we ensure they are randomly ordered so every queue should
+      #   pull messages off queues in the order they are passed to the command,
+      #   by shuffling we ensure they are randomly ordered so every queue should
       #   get a chance.
       # * Use 0 for the brpop timeout which means block indefinitely.
       # * Rescue runtime errors so the daemon thread doesn't fail if redis is
@@ -151,9 +151,9 @@ module Qs
 
         begin
           args = [self.signals_redis_key, self.queue_redis_keys.shuffle, 0].flatten
-          redis_key, serialized_payload = @client.block_dequeue(*args)
+          redis_key, encoded_payload = @client.block_dequeue(*args)
           if redis_key != @signals_redis_key
-            @worker_pool.add_work(RedisItem.new(redis_key, serialized_payload))
+            @worker_pool.add_work(QueueItem.new(redis_key, encoded_payload))
           end
         rescue RuntimeError => exception
           log "Error dequeueing #{exception.message.inspect}", :error
@@ -178,9 +178,9 @@ module Qs
           log "Shutting down, waiting for work to finish"
         end
         @worker_pool.shutdown(timeout)
-        log "Requeueing #{@worker_pool.work_items.size} job(s)"
+        log "Requeueing #{@worker_pool.work_items.size} message(s)"
         @worker_pool.work_items.each do |ri|
-          @client.prepend(ri.queue_redis_key, ri.serialized_payload)
+          @client.prepend(ri.queue_redis_key, ri.encoded_payload)
         end
       end
 
@@ -196,18 +196,18 @@ module Qs
       # * This only catches errors that happen outside of running the payload
       #   handler. The only known use-case for this is dat worker pools
       #   hard-shutdown errors.
-      # * If there isn't a redis item (this can happen when an idle worker is
+      # * If there isn't a queue item (this can happen when an idle worker is
       #   being forced to exit) then we don't need to do anything.
-      # * If we never started processing the redis item, its safe to requeue it.
+      # * If we never started processing the queue item, its safe to requeue it.
       #   Otherwise it happened while processing so the payload handler caught
       #   it or it happened after the payload handler which we don't care about.
-      def handle_worker_exception(exception, redis_item)
-        return if redis_item.nil?
-        if !redis_item.started
-          log "Worker error, requeueing job because it hasn't started", :error
-          @client.prepend(redis_item.queue_redis_key, redis_item.serialized_payload)
+      def handle_worker_exception(exception, queue_item)
+        return if queue_item.nil?
+        if !queue_item.started
+          log "Worker error, requeueing message because it hasn't started", :error
+          @client.prepend(queue_item.queue_redis_key, queue_item.encoded_payload)
         else
-          log "Worker error after job was processed, ignoring", :error
+          log "Worker error after message was processed, ignoring", :error
         end
         log "#{exception.class}: #{exception.message}", :error
         log exception.backtrace.join("\n"), :error

data/lib/qs/daemon_data.rb

@@ -5,7 +5,7 @@ module Qs
     # The daemon uses this to "compile" its configuration for speed. NsOptions
     # is relatively slow everytime an option is read. To avoid this, we read the
     # options one time here and memoize their values. This way, we don't pay the
-    # NsOptions overhead when reading them while handling a job.
+    # NsOptions overhead when reading them while handling a message.
 
     attr_reader :name
     attr_reader :pid_file
@@ -29,14 +29,14 @@ module Qs
       @routes           = build_routes(args[:routes] || [])
     end
 
-    def route_for(name)
-      @routes[name] || raise(NotFoundError, "no service named '#{name}'")
+    def route_for(route_id)
+      @routes[route_id] || raise(NotFoundError, "unknown message '#{route_id}'")
     end
 
     private
 
     def build_routes(routes)
-      routes.inject({}){ |h, route| h.merge(route.name => route) }
+      routes.inject({}){ |h, route| h.merge(route.id => route) }
     end
 
   end

data/lib/qs/dispatch_job.rb

@@ -0,0 +1,36 @@
+require 'qs'
+require 'qs/event'
+require 'qs/job'
+
+module Qs
+
+  class DispatchJob < Qs::Job
+
+    def self.event(job)
+      Qs::Event.new(job.params['event_channel'], job.params['event_name'], {
+        :params       => job.params['event_params'],
+        :publisher    => job.params['event_publisher'],
+        :published_at => job.created_at
+      })
+    end
+
+    def initialize(event_channel, event_name, options = nil)
+      options ||= {}
+      event_params    = options.delete(:event_params)    || {}
+      event_publisher = options.delete(:event_publisher) || Qs.event_publisher
+      options[:params] = {
+        'event_channel'   => event_channel,
+        'event_name'      => event_name,
+        'event_params'    => event_params,
+        'event_publisher' => event_publisher
+      }
+      super(Qs.dispatcher_job_name, options)
+    end
+
+    def event
+      @event ||= self.class.event(self)
+    end
+
+  end
+
+end

data/lib/qs/dispatch_job_handler.rb

@@ -0,0 +1,79 @@
+require 'qs'
+require 'qs/dispatch_job'
+require 'qs/job_handler'
+
+module Qs
+
+  module DispatchJobHandler
+
+    def self.included(klass)
+      klass.class_eval do
+        include Qs::JobHandler
+        include InstanceMethods
+      end
+    end
+
+    module InstanceMethods
+
+      attr_reader :event, :subscribed_queue_names
+
+      def init!
+        @event = Qs::DispatchJob.event(job)
+        @subscribed_queue_names = Qs.event_subscribers(@event)
+        @qs_failed_dispatches = []
+      end
+
+      def run!
+        logger.info "Dispatching #{self.event.route_name}"
+        logger.info "  params:       #{self.event.params.inspect}"
+        logger.info "  publisher:    #{self.event.publisher}"
+        logger.info "  published at: #{self.event.published_at}"
+        logger.info "Found #{self.subscribed_queue_names.size} subscribed queue(s):"
+        self.subscribed_queue_names.each do |queue_name|
+          qs_dispatch(queue_name, self.event)
+        end
+        qs_handle_errors(self.event, @qs_failed_dispatches)
+      end
+
+      private
+
+      def qs_dispatch(queue_name, event)
+        Qs.push(queue_name, Qs::Payload.event_hash(event))
+        logger.info "  => #{queue_name}"
+      rescue StandardError => exception
+        logger.info "  => #{queue_name} (failed)"
+        @qs_failed_dispatches << FailedDispatch.new(queue_name, exception)
+      end
+
+      def qs_handle_errors(event, failed_dispatches)
+        return if failed_dispatches.empty?
+        logger.info "Failed to dispatch the event to " \
+                    "#{failed_dispatches.size} subscribed queues"
+        descriptions = failed_dispatches.map do |fail|
+          exception_desc = "#{fail.exception.class}: #{fail.exception.message}"
+          logger.info "#{fail.queue_name}"
+          logger.info "  #{exception_desc}"
+          logger.info "  #{fail.exception.backtrace.first}"
+          "#{fail.queue_name} - #{exception_desc}"
+        end
+        message = "#{event.route_name} event wasn't dispatched to:\n" \
+                  "  #{descriptions.join("\n  ")}"
+        raise DispatchError.new(message, failed_dispatches)
+      end
+
+    end
+
+    FailedDispatch = Struct.new(:queue_name, :exception)
+
+    class DispatchError < RuntimeError
+      attr_reader :failed_dispatches
+
+      def initialize(message, failed_dispatches)
+        super message
+        @failed_dispatches = failed_dispatches
+      end
+    end
+
+  end
+
+end

data/lib/qs/dispatcher_queue.rb

@@ -0,0 +1,19 @@
+require 'qs/queue'
+require 'qs/dispatch_job_handler'
+
+module Qs
+
+  module DispatcherQueue
+
+    def self.new(options)
+      options[:queue_class].new do
+        name options[:queue_name]
+        job options[:job_name], options[:job_handler_class_name]
+      end
+    end
+
+    RunDispatchJob = Class.new{ include Qs::DispatchJobHandler }
+
+  end
+
+end

data/lib/qs/error_handler.rb

@@ -31,24 +31,24 @@ module Qs
 
   class ErrorContext
     attr_reader :daemon_data
-    attr_reader :queue_name, :serialized_payload
-    attr_reader :job, :handler_class
+    attr_reader :queue_name, :encoded_payload
+    attr_reader :message, :handler_class
 
     def initialize(args)
-      @daemon_data        = args[:daemon_data]
-      @queue_name         = Queue::RedisKey.parse_name(args[:queue_redis_key].to_s)
-      @serialized_payload = args[:serialized_payload]
-      @job                = args[:job]
-      @handler_class      = args[:handler_class]
+      @daemon_data     = args[:daemon_data]
+      @queue_name      = Queue::RedisKey.parse_name(args[:queue_redis_key].to_s)
+      @encoded_payload = args[:encoded_payload]
+      @message         = args[:message]
+      @handler_class   = args[:handler_class]
     end
 
     def ==(other)
       if other.kind_of?(self.class)
-        self.daemon_data        == other.daemon_data &&
-        self.queue_name         == other.queue_name &&
-        self.serialized_payload == other.serialized_payload &&
-        self.job                == other.job &&
-        self.handler_class      == other.handler_class
+        self.daemon_data     == other.daemon_data &&
+        self.queue_name      == other.queue_name &&
+        self.encoded_payload == other.encoded_payload &&
+        self.message         == other.message &&
+        self.handler_class   == other.handler_class
       else
         super
       end

data/lib/qs/event.rb

@@ -0,0 +1,82 @@
+require 'qs/message'
+
+module Qs
+
+  class Event < Message
+
+    PAYLOAD_TYPE = 'event'
+
+    attr_reader :channel, :name, :publisher, :published_at
+
+    def initialize(channel, name, options = nil)
+      options ||= {}
+      options[:params] ||= {}
+      validate!(channel, name, options[:params])
+      @channel      = channel
+      @name         = name
+      @publisher    = options[:publisher]
+      @published_at = options[:published_at] || Time.now
+      super(PAYLOAD_TYPE, options)
+    end
+
+    def route_name
+      @route_name ||= Event::RouteName.new(self.channel, self.name)
+    end
+
+    def subscribers_redis_key
+      @subscribers_redis_key ||= SubscribersRedisKey.new(self.route_name)
+    end
+
+    def inspect
+      reference = '0x0%x' % (self.object_id << 1)
+      "#<#{self.class}:#{reference} " \
+      "@channel=#{self.channel.inspect} " \
+      "@name=#{self.name.inspect} " \
+      "@params=#{self.params.inspect} " \
+      "@publisher=#{self.publisher.inspect} " \
+      "@published_at=#{self.published_at.inspect}>"
+    end
+
+    def ==(other)
+      if other.kind_of?(self.class)
+        self.payload_type == other.payload_type &&
+        self.channel      == other.channel      &&
+        self.name         == other.name         &&
+        self.params       == other.params       &&
+        self.publisher    == other.publisher    &&
+        self.published_at == other.published_at
+      else
+        super
+      end
+    end
+
+    private
+
+    def validate!(channel, name, params)
+      problem = if channel.to_s.empty?
+        "The event doesn't have a channel."
+      elsif name.to_s.empty?
+        "The event doesn't have a name."
+      elsif !params.kind_of?(::Hash)
+        "The event's params are not valid."
+      end
+      raise(InvalidError, problem) if problem
+    end
+
+    module RouteName
+      def self.new(event_channel, event_name)
+        "#{event_channel}:#{event_name}"
+      end
+    end
+
+    module SubscribersRedisKey
+      def self.new(route_name)
+        "events:#{route_name}:subscribers"
+      end
+    end
+
+    InvalidError = Class.new(ArgumentError)
+
+  end
+
+end

data/lib/qs/event_handler.rb

@@ -0,0 +1,34 @@
+require 'qs/message_handler'
+
+module Qs
+
+  module EventHandler
+
+    def self.included(klass)
+      klass.class_eval do
+        include Qs::MessageHandler
+        include InstanceMethods
+      end
+    end
+
+    module InstanceMethods
+
+      def inspect
+        reference = '0x0%x' % (self.object_id << 1)
+        "#<#{self.class}:#{reference} @event=#{event.inspect}>"
+      end
+
+      private
+
+      # Helpers
+
+      def event;              @qs_runner.message; end
+      def event_channel;      event.channel;      end
+      def event_name;         event.name;         end
+      def event_published_at; event.published_at; end
+
+    end
+
+  end
+
+end

data/lib/qs/event_handler_test_helpers.rb

@@ -0,0 +1,17 @@
+require 'qs/test_runner'
+
+module Qs::EventHandler
+
+  module TestHelpers
+
+    def test_runner(handler_class, args = nil)
+      Qs::EventTestRunner.new(handler_class, args)
+    end
+
+    def test_handler(handler_class, args = nil)
+      test_runner(handler_class, args).handler
+    end
+
+  end
+
+end

data/lib/qs/job.rb

@@ -1,26 +1,24 @@
+require 'qs/message'
+
 module Qs
 
-  class Job
+  class Job < Message
 
-    def self.parse(payload)
-      created_at = Time.at(payload['created_at'].to_i)
-      self.new(payload['name'], payload['params'], created_at)
-    end
+    PAYLOAD_TYPE = 'job'
 
-    attr_reader :name, :params, :created_at
+    attr_reader :name, :created_at
 
-    def initialize(name, params, created_at = nil)
-      validate!(name, params)
+    def initialize(name, options = nil)
+      options ||= {}
+      options[:params] ||= {}
+      validate!(name, options[:params])
       @name       = name
-      @params     = params
-      @created_at = created_at || Time.now
+      @created_at = options[:created_at] || Time.now
+      super(PAYLOAD_TYPE, options)
     end
 
-    def to_payload
-      { 'name'       => self.name.to_s,
-        'params'     => StringifyParams.new(self.params),
-        'created_at' => self.created_at.to_i
-      }
+    def route_name
+      self.name
     end
 
     def inspect
@@ -33,7 +31,10 @@ module Qs
 
     def ==(other)
       if other.kind_of?(self.class)
-        self.to_payload == other.to_payload
+        self.payload_type == other.payload_type &&
+        self.name         == other.name         &&
+        self.params       == other.params       &&
+        self.created_at   == other.created_at
       else
         super
       end
@@ -47,24 +48,11 @@ module Qs
       elsif !params.kind_of?(::Hash)
         "The job's params are not valid."
       end
-      raise(BadJobError, problem) if problem
+      raise(InvalidError, problem) if problem
     end
 
-    module StringifyParams
-      def self.new(object)
-        case(object)
-        when Hash
-          object.inject({}){ |h, (k, v)| h.merge(k.to_s => self.new(v)) }
-        when Array
-          object.map{ |item| self.new(item) }
-        else
-          object
-        end
-      end
-    end
+    InvalidError = Class.new(ArgumentError)
 
   end
 
-  BadJobError = Class.new(ArgumentError)
-
 end