qs 0.6.1 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 --- 
 SHA1: 
-  data.tar.gz: 8f69f568a5b9a346be6a45f7f439fe835263240b
-  metadata.gz: b11ca0b892b3fc054d3c350c529c6933d415ac01
+  data.tar.gz: 110c7cbf5c19120501d4dfae9f332e428363aff3
+  metadata.gz: 053fda8d651bb4509a99537ef8b8edb906a80acd
 SHA512: 
-  data.tar.gz: 18e0f1388fc35d85e8e324433be89bdccba9ee658b10797ed85c756646f51552109cf5c8fca4276e300cb53dda81cf78be772939d9fdc100e0146b0be64d37e3
-  metadata.gz: e83ff088d753074d81c84d686d25b977e8a875ef6b0174e8455785d23fc25057fc5e1af4259801c97d0234c7ddc848f170e6009ee00cdac7c06cdb94edb9f7e8
+  data.tar.gz: 6c145c40e72aecad5c87b33709a3a3aa097a07a4bd08a17819d23965efd36fdbe46312a01fd638f346e4e34a2c7cc4c2ab85af7a9951083e97aae68f93527624
+  metadata.gz: 56bff595f01f50dba796e2b436b842376188d3c031ab6cb88395bb7cc41e04250217c04ef5eedc78ba221586795045ba8e6b739816cbe65d89cb01dacd9607ab

data/bench/config.qs

@@ -1,3 +1,4 @@
+$LOAD_PATH.push(File.expand_path('../..', __FILE__))
 require 'qs'
 
 ENV['LOG_NAME'] = 'log/bench_daemon.log'

data/bench/dispatcher.qs

@@ -1,3 +1,4 @@
+$LOAD_PATH.push(File.expand_path('../..', __FILE__))
 require 'qs'
 
 ENV['LOG_NAME'] = 'log/bench_dispatcher_daemon.log'

data/bench/report.rb

@@ -1,3 +1,5 @@
+$LOAD_PATH.push(File.expand_path('../..', __FILE__))
+
 require 'benchmark'
 require 'scmd'
 require 'bench/setup'

data/bench/setup.rb

@@ -16,7 +16,7 @@ else
   File.open('/dev/null', 'w')
 end
 
-Qs.config.dispatcher.queue_name = 'bench-dispatcher'
+Qs.config.dispatcher_queue_name = 'bench-dispatcher'
 Qs.config.event_publisher = 'Bench Script'
 Qs.init
 

data/lib/qs.rb

@@ -1,4 +1,3 @@
-require 'ns-options'
 require 'qs/version'
 require 'qs/client'
 require 'qs/daemon'
@@ -15,28 +14,24 @@ module Qs
   end
 
   def self.init
-    self.config.redis.url ||= RedisUrl.new(
-      self.config.redis.ip,
-      self.config.redis.port,
-      self.config.redis.db
-    )
+    self.config.validate!
 
     @dispatcher_queue ||= DispatcherQueue.new({
       :queue_class            => self.config.dispatcher_queue_class,
-      :queue_name             => self.config.dispatcher.queue_name,
-      :job_name               => self.config.dispatcher.job_name,
-      :job_handler_class_name => self.config.dispatcher.job_handler_class_name
+      :queue_name             => self.config.dispatcher_queue_name,
+      :job_name               => self.config.dispatcher_job_name,
+      :job_handler_class_name => self.config.dispatcher_job_handler_class_name
     })
 
     @encoder ||= self.config.encoder
     @decoder ||= self.config.decoder
-    @client  ||= Client.new(self.redis_config)
+    @client  ||= Client.new(self.redis_connect_hash)
     @redis   ||= @client.redis
     true
   end
 
   def self.reset!
-    self.config.reset
+    @config           = nil
     @dispatcher_queue = nil
     @encoder          = nil
     @decoder          = nil
@@ -46,19 +41,19 @@ module Qs
   end
 
   def self.enqueue(queue, job_name, params = nil)
-    @client.enqueue(queue, job_name, params)
+    self.client.enqueue(queue, job_name, params)
   end
 
   def self.publish(channel, name, params = nil)
-    @client.publish(channel, name, params)
+    self.client.publish(channel, name, params)
   end
 
   def self.publish_as(publisher, channel, name, params = nil)
-    @client.publish_as(publisher, channel, name, params)
+    self.client.publish_as(publisher, channel, name, params)
   end
 
   def self.push(queue_name, payload)
-    @client.push(queue_name, payload)
+    self.client.push(queue_name, payload)
   end
 
   def self.encode(payload)
@@ -82,15 +77,15 @@ module Qs
   end
 
   def self.client
-    @client
+    @client || NullClient.new
   end
 
   def self.redis
     @redis
   end
 
-  def self.redis_config
-    self.config.redis.to_hash
+  def self.redis_connect_hash
+    self.config.redis_connect_hash
   end
 
   def self.dispatcher_queue
@@ -98,7 +93,7 @@ module Qs
   end
 
   def self.dispatcher_job_name
-    self.config.dispatcher.job_name
+    self.config.dispatcher_job_name
   end
 
   def self.event_publisher
@@ -110,46 +105,105 @@ module Qs
   end
 
   class Config
-    include NsOptions::Proxy
 
-    option :encoder, Proc, :default => proc{ |p| ::JSON.dump(p) }
-    option :decoder, Proc, :default => proc{ |p| ::JSON.load(p) }
+    DEFAULT_ENCODER = proc{ |payload|         ::JSON.dump(payload)         }
+    DEFAULT_DECODER = proc{ |encoded_payload| ::JSON.load(encoded_payload) }
 
-    option :timeout, Float
+    DEFAULT_DISPATCHER_QUEUE_CLASS            = Queue
+    DEFAULT_DISPATCHER_QUEUE_NAME             = 'dispatcher'.freeze
+    DEFAULT_DISPATCHER_JOB_NAME               = 'run_dispatch_job'.freeze
+    DEFAULT_DISPATCHER_JOB_HANDLER_CLASS_NAME = DispatcherQueue::RunDispatchJob.to_s.freeze
 
-    option :event_publisher, String
+    DEFAULT_REDIS_IP      = '127.0.0.1'.freeze
+    DEFAULT_REDIS_PORT    = 6379.freeze
+    DEFAULT_REDIS_DB      = 0.freeze
+    DEFAULT_REDIS_NS      = 'qs'.freeze
+    DEFAULT_REDIS_DRIVER  = 'ruby'.freeze
+    DEFAULT_REDIS_TIMEOUT = 1.freeze
+    DEFAULT_REDIS_SIZE    = 4.freeze
 
-    namespace :dispatcher do
-      option :queue_name,             String, :default => 'dispatcher'
-      option :job_name,               String, :default => 'run_dispatch_job'
-      option :job_handler_class_name, String, :default => DispatcherQueue::RunDispatchJob.to_s
-    end
+    attr_accessor :encoder, :decoder, :timeout, :event_publisher
+    attr_accessor :dispatcher_queue_class, :dispatcher_queue_name
+    attr_accessor :dispatcher_job_name, :dispatcher_job_handler_class_name
+    attr_accessor :redis_ip, :redis_port, :redis_db, :redis_ns
+    attr_accessor :redis_driver, :redis_timeout, :redis_size, :redis_url
 
-    namespace :redis do
-      option :ip,   :default => '127.0.0.1'
-      option :port, :default => 6379
-      option :db,   :default => 0
+    def initialize
+      @encoder         = DEFAULT_ENCODER
+      @decoder         = DEFAULT_DECODER
+      @timeout         = nil
+      @event_publisher = nil
+
+      @dispatcher_queue_class            = DEFAULT_DISPATCHER_QUEUE_CLASS
+      @dispatcher_queue_name             = DEFAULT_DISPATCHER_QUEUE_NAME
+      @dispatcher_job_name               = DEFAULT_DISPATCHER_JOB_NAME
+      @dispatcher_job_handler_class_name = DEFAULT_DISPATCHER_JOB_HANDLER_CLASS_NAME
+
+      @redis_ip      = DEFAULT_REDIS_IP
+      @redis_port    = DEFAULT_REDIS_PORT
+      @redis_db      = DEFAULT_REDIS_DB
+      @redis_ns      = DEFAULT_REDIS_NS
+      @redis_driver  = DEFAULT_REDIS_DRIVER
+      @redis_timeout = DEFAULT_REDIS_TIMEOUT
+      @redis_size    = DEFAULT_REDIS_SIZE
+      @redis_url     = nil
+
+      @valid = nil
+    end
 
-      option :url
+    # the keys here should be compatible with HellaRedis connection configs
+    # https://github.com/redding/hella-redis#connection
+    def redis_connect_hash
+      { :ip       => self.redis_ip,
+        :port     => self.redis_port,
+        :db       => self.redis_db,
+        :redis_ns => self.redis_ns,
+        :driver   => self.redis_driver,
+        :timeout  => self.redis_timeout,
+        :size     => self.redis_size,
+        :url      => self.redis_url
+      }
+    end
 
-      option :redis_ns, String,  :default => 'qs'
-      option :driver,   String,  :default => 'ruby'
-      option :timeout,  Integer, :default => 1
-      option :size,     Integer, :default => 4
+    def valid?
+      !!@valid
     end
 
-    attr_accessor :dispatcher_queue_class
+    def validate!
+      return @valid if !@valid.nil? # only need to run this once per config
 
-    def initialize
-      self.dispatcher_queue_class = Queue
+      # set the `redis_url`
+      self.redis_url ||= RedisUrl.new(self.redis_ip, self.redis_port, self.redis_db)
+
+      @valid = true
     end
+
   end
 
   module RedisUrl
+
     def self.new(ip, port, db)
       return if ip.to_s.empty? || port.to_s.empty? || db.to_s.empty?
       "redis://#{ip}:#{port}/#{db}"
     end
+
   end
 
+  class NullClient
+    def initialize
+      @error = UninitializedError.new("Qs hasn't been initialized")
+    end
+
+    def enqueue(*args);             raise @error; end
+    def publish(*args);             raise @error; end
+    def publish_as(*args);          raise @error; end
+    def push(*args);                raise @error; end
+    def sync_subscriptions(*args);  raise @error; end
+    def clear_subscriptions(*args); raise @error; end
+    def event_subscribers(*args);   raise @error; end
+  end
+
+  UninitializedError = Class.new(RuntimeError)
+  TimeoutError       = Class.new(RuntimeError)
+
 end

data/lib/qs/client.rb

@@ -25,10 +25,10 @@ module Qs
 
     module InstanceMethods
 
-      attr_reader :redis_config, :redis
+      attr_reader :redis_connect_hash, :redis
 
-      def initialize(redis_config)
-        @redis_config = redis_config
+      def initialize(redis_connect_hash)
+        @redis_connect_hash = redis_connect_hash
       end
 
       def enqueue(queue, job_name, job_params = nil)
@@ -119,7 +119,7 @@ module Qs
 
     def initialize(*args)
       super
-      @redis = HellaRedis::Connection.new(self.redis_config)
+      @redis = HellaRedis::Connection.new(self.redis_connect_hash)
     end
 
     def push(queue_name, payload_hash)
@@ -145,7 +145,7 @@ module Qs
     def initialize(*args)
       super
       require 'hella-redis/connection_spy'
-      @redis = HellaRedis::ConnectionSpy.new(self.redis_config)
+      @redis = HellaRedis::ConnectionSpy.new(self.redis_connect_hash)
       @pushed_items = []
     end
 

data/lib/qs/config_file.rb

@@ -35,7 +35,7 @@ module Qs
       full_path_with_sanford
     end
 
-    # This evaluates the file and creates a proc using it's contents. This is
+    # This evaluates the file and creates a proc using its contents. This is
     # a trick borrowed from Rack. It is essentially converting a file into a
     # proc and then instance eval'ing it. This has a couple benefits:
     # * The obvious benefit is the file is evaluated in the context of this

data/lib/qs/daemon.rb

@@ -1,8 +1,5 @@
 require 'dat-worker-pool'
 require 'much-plugin'
-require 'ns-options'
-require 'pathname'
-require 'system_timer'
 require 'thread'
 require 'qs'
 require 'qs/client'
@@ -16,9 +13,8 @@ module Qs
   module Daemon
     include MuchPlugin
 
-    InvalidError = Class.new(ArgumentError)
-
-    SIGNAL = '.'.freeze
+    SIGNAL               = '.'.freeze
+    FETCH_ERR_SLEEP_TIME = 1.0.freeze
 
     plugin_included do
       extend ClassMethods
@@ -27,28 +23,46 @@ module Qs
 
     module InstanceMethods
 
-      attr_reader :daemon_data, :logger
-      attr_reader :signals_redis_key, :queue_redis_keys
+      attr_reader :daemon_data, :signals_redis_key
 
-      # set the size of the client to the num workers + 1, this ensures we have
-      # 1 connection for fetching work from redis and at least 1 connection for
-      # each worker to requeue its message when hard-shutdown
       def initialize
-        self.class.configuration.validate!
+        config = self.class.config
+        begin
+          config.validate!
+        rescue InvalidError => exception
+          exception.set_backtrace(caller)
+          raise exception
+        end
         Qs.init
-        @daemon_data = DaemonData.new(self.class.configuration.to_hash)
-        @logger      = @daemon_data.logger
 
-        @client = QsClient.new(Qs.redis_config.merge({
-          :timeout => 1,
-          :size    => self.daemon_data.num_workers + 1
-        }))
-        @queue_redis_keys = self.daemon_data.queue_redis_keys
+        @daemon_data = DaemonData.new({
+          :name             => config.name,
+          :pid_file         => config.pid_file,
+          :shutdown_timeout => config.shutdown_timeout,
+          :worker_class     => config.worker_class,
+          :worker_params    => config.worker_params,
+          :num_workers      => config.num_workers,
+          :error_procs      => config.error_procs,
+          :logger           => config.logger,
+          :queues           => config.queues,
+          :verbose_logging  => config.verbose_logging,
+          :routes           => config.routes
+        })
 
-        @signals_redis_key = "signals:#{@daemon_data.name}-" \
+        @signals_redis_key = "signals:#{self.daemon_data.name}-" \
                              "#{Socket.gethostname}-#{::Process.pid}"
 
+        @thread           = nil
         @worker_available = WorkerAvailable.new
+        @state            = State.new(:stop)
+
+        # set the size of the client to the num workers + 1, this ensures we
+        # have 1 connection for fetching work from redis and at least 1
+        # connection for each worker to requeue its message when hard-shutdown
+        @client = QsClient.new(Qs.redis_connect_hash.merge({
+          :timeout => 1,
+          :size    => self.daemon_data.num_workers + 1
+        }))
 
         @worker_pool = DatWorkerPool.new(self.daemon_data.worker_class, {
           :num_workers   => self.daemon_data.num_workers,
@@ -57,15 +71,9 @@ module Qs
             :qs_daemon_data      => self.daemon_data,
             :qs_client           => @client,
             :qs_worker_available => @worker_available,
-            :qs_logger           => @logger
+            :qs_logger           => self.logger
           })
         })
-
-        @thread = nil
-        @state = State.new(:stop)
-      rescue InvalidError => exception
-        exception.set_backtrace(caller)
-        raise exception
       end
 
       def name
@@ -80,14 +88,22 @@ module Qs
         @daemon_data.pid_file
       end
 
+      def logger
+        @daemon_data.logger
+      end
+
+      def queue_redis_keys
+        @daemon_data.queue_redis_keys
+      end
+
       def running?
         !!(@thread && @thread.alive?)
       end
 
-      # ping to check that it can communicate with redis before running, this is
-      # friendlier than starting and continously erroring because it can't
-      # dequeue
       def start
+        # ping to check that it can communicate with redis before running,
+        # this is friendlier than starting and continously erroring because
+        # it can't dequeue
         @client.ping
         @state.set :run
         @thread ||= Thread.new{ work_loop }
@@ -121,25 +137,25 @@ module Qs
         teardown
       end
 
-      # clear any signals that are already on the signals list in redis
       def setup
-        @client.clear(self.signals_redis_key)
+       # clear any signals that are already on the signals list in redis
+       @client.clear(self.signals_redis_key)
         @worker_pool.start
       end
 
-      # shuffle the queue redis keys to avoid queue starvation, redis will pull
-      # messages off queues in the order they are passed to the command, by
-      # shuffling we ensure they are randomly ordered so every queue should get
-      # a chance; use 0 for the brpop timeout which means block indefinitely;
-      # rescue runtime errors so the daemon thread doesn't fail if redis is
-      # temporarily down, sleep for a second to keep the thread from thrashing
-      # by repeatedly erroring if redis is down
       def fetch_messages
         if !@worker_pool.worker_available? && @state.run?
           @worker_available.wait
         end
         return unless @worker_pool.worker_available? && @state.run?
 
+        # shuffle the queue redis keys to avoid queue starvation, redis will
+        # pull messages off queues in the order they are passed to the command,
+        # by shuffling we ensure they are randomly ordered so every queue
+        # should  get a chance; use 0 for the brpop timeout which means block
+        # indefinitely; rescue runtime errors so the daemon thread doesn't fail
+        # if redis is temporarily down, sleep for a second to keep the thread
+        # from thrashing by repeatedly erroring if redis is down
         begin
           args = [self.signals_redis_key, self.queue_redis_keys.shuffle, 0].flatten
           redis_key, encoded_payload = @client.block_dequeue(*args)
@@ -150,7 +166,7 @@ module Qs
           log "Error occurred while dequeueing", :error
           log "#{exception.class}: #{exception.message}", :error
           (exception.backtrace || []).each{ |l| log(l, :error) }
-          sleep 1
+          sleep FETCH_ERR_SLEEP_TIME
         end
       end
 
@@ -183,86 +199,102 @@ module Qs
 
     module ClassMethods
 
-      def configuration
-        @configuration ||= Configuration.new
+      def config
+        @config ||= Config.new
+      end
+
+      def name(value = nil)
+        self.config.name = value if !value.nil?
+        self.config.name
       end
 
-      def name(*args)
-        self.configuration.name(*args)
+      def pid_file(value = nil)
+        self.config.pid_file = value if !value.nil?
+        self.config.pid_file
       end
 
-      def pid_file(*args)
-        self.configuration.pid_file(*args)
+      def shutdown_timeout(value = nil)
+        self.config.shutdown_timeout = value if !value.nil?
+        self.config.shutdown_timeout
       end
 
-      def worker_class(new_worker_class = nil)
-        self.configuration.worker_class = new_worker_class if !new_worker_class.nil?
-        self.configuration.worker_class
+      def worker_class(value = nil)
+        self.config.worker_class = value if !value.nil?
+        self.config.worker_class
       end
 
-      def worker_params(new_worker_params = nil)
-        self.configuration.worker_params = new_worker_params if !new_worker_params.nil?
-        self.configuration.worker_params
+      def worker_params(value = nil)
+        self.config.worker_params = value if !value.nil?
+        self.config.worker_params
       end
 
-      def num_workers(*args)
-        self.configuration.num_workers(*args)
+      def num_workers(new_num_workers = nil)
+        self.config.num_workers = new_num_workers if new_num_workers
+        self.config.num_workers
       end
       alias :workers :num_workers
 
-      def verbose_logging(*args)
-        self.configuration.verbose_logging(*args)
+      def init(&block)
+        self.config.init_procs << block
       end
 
-      def logger(*args)
-        self.configuration.logger(*args)
+      def init_procs
+        self.config.init_procs
       end
 
-      def shutdown_timeout(*args)
-        self.configuration.shutdown_timeout(*args)
+      def error(&block)
+        self.config.error_procs << block
       end
 
-      def init(&block)
-        self.configuration.init_procs << block
+      def error_procs
+        self.config.error_procs
       end
 
-      def error(&block)
-        self.configuration.error_procs << block
+      def logger(value = nil)
+        self.config.logger = value if !value.nil?
+        self.config.logger
       end
 
-      def queue(queue)
-        self.configuration.queues << queue
+      def queue(value)
+        self.config.queues << value
       end
 
       def queues
-        self.configuration.queues
+        self.config.queues
       end
 
-    end
+      # flags
 
-    class Configuration
-      include NsOptions::Proxy
+      def verbose_logging(value = nil)
+        self.config.verbose_logging = value if !value.nil?
+        self.config.verbose_logging
+      end
 
-      option :name,     String,  :required => true
-      option :pid_file, Pathname
+    end
 
-      option :num_workers, Integer, :default => 4
+    class Config
 
-      option :verbose_logging, :default => true
-      option :logger,          :default => proc{ Qs::NullLogger.new }
+      DEFAULT_NUM_WORKERS = 4.freeze
 
-      option :shutdown_timeout
+      attr_accessor :name, :pid_file, :shutdown_timeout
+      attr_accessor :worker_class, :worker_params, :num_workers
+      attr_accessor :init_procs, :error_procs, :logger, :queues
+      attr_accessor :verbose_logging
 
-      attr_accessor :init_procs, :error_procs
-      attr_accessor :worker_class, :worker_params
-      attr_accessor :queues
+      def initialize
+        @name             = nil
+        @pid_file         = nil
+        @shutdown_timeout = nil
+        @worker_class     = DefaultWorker
+        @worker_params    = nil
+        @num_workers      = DEFAULT_NUM_WORKERS
+        @init_procs       = []
+        @error_procs      = []
+        @logger           = Qs::NullLogger.new
+        @queues           = []
+
+        @verbose_logging = true
 
-      def initialize(values = nil)
-        super(values)
-        @init_procs, @error_procs = [], []
-        @worker_class  = DefaultWorker
-        @worker_params = nil
-        @queues = []
         @valid = nil
       end
 
@@ -270,35 +302,37 @@ module Qs
         @queues.map(&:routes).flatten
       end
 
-      def to_hash
-        super.merge({
-          :error_procs      => self.error_procs,
-          :worker_class     => self.worker_class,
-          :worker_params    => self.worker_params,
-          :routes           => self.routes,
-          :queue_redis_keys => self.queues.map(&:redis_key)
-        })
-      end
-
       def valid?
         !!@valid
       end
 
+      # for the config to be considered "valid", a few things need to happen.
+      # The key here is that this only needs to be done _once_ for each config.
+
       def validate!
-        return @valid if !@valid.nil?
+        return @valid if !@valid.nil? # only need to run this once per config
+
+        # ensure all user and plugin configs/settings are applied
         self.init_procs.each(&:call)
-        if self.queues.empty? || !self.required_set?
-          raise InvalidError, "a name and queue must be configured"
+        if self.queues.empty? || self.name.nil?
+          raise InvalidError, "a name and at least 1 queue must be configured"
         end
+
+        # validate the worker class
         if !self.worker_class.kind_of?(Class) || !self.worker_class.include?(Qs::Worker)
           raise InvalidError, "worker class must include `#{Qs::Worker}`"
         end
+
+        # validate the routes
         self.routes.each(&:validate!)
-        @valid = true
+
+        @valid = true # if it made it this far, it's valid!
       end
+
     end
 
     DefaultWorker = Class.new{ include Qs::Worker }
+    InvalidError  = Class.new(ArgumentError)
 
     class WorkerAvailable
       def initialize