qs 0.3.0 → 0.4.0

data/bench/config.qs

@@ -1,20 +1,7 @@
 require 'qs'
-require 'bench/queue'
 
-ROOT_PATH = Pathname.new(File.expand_path('../..', __FILE__))
-
-LOGGER = if ENV['BENCH_REPORT']
-  Logger.new(ROOT_PATH.join('log/bench_daemon.log').to_s)
-else
-  Logger.new(STDOUT)
-end
-LOGGER.datetime_format = "" # turn off the datetime in the logs
-
-PROGRESS_IO = if ENV['BENCH_PROGRESS_IO']
-  ::IO.for_fd(ENV['BENCH_PROGRESS_IO'].to_i)
-else
-  File.open('/dev/null', 'w')
-end
+ENV['LOG_NAME'] = 'log/bench_daemon.log'
+require 'bench/setup'
 
 class BenchDaemon
   include Qs::Daemon
@@ -27,21 +14,11 @@ class BenchDaemon
 
   queue BenchQueue
 
-  # if jobs fail notify the bench report so it doesn't hang forever on IO.select
-  error do |exception, daemon_data, job|
+  # if fails notify the bench report so it doesn't hang forever on IO.select
+  error do |exception, context|
     PROGRESS_IO.write_nonblock('F')
   end
 
-  class Multiply
-    include Qs::JobHandler
-
-    after{ PROGRESS_IO.write_nonblock('.') }
-
-    def run!
-      'a' * params['size']
-    end
-  end
-
 end
 
 run BenchDaemon.new

data/bench/dispatcher.qs

@@ -0,0 +1,24 @@
+require 'qs'
+
+ENV['LOG_NAME'] = 'log/bench_dispatcher_daemon.log'
+require 'bench/setup'
+
+class DispatcherDaemon
+  include Qs::Daemon
+
+  name     'bench-dispatcher'
+  pid_file ROOT_PATH.join('tmp/bench_dispatcher_daemon.pid').to_s
+
+  logger LOGGER
+  verbose_logging false
+
+  queue Qs.dispatcher_queue
+
+  # if fails notify the bench report so it doesn't hang forever on IO.select
+  error do |exception, context|
+    PROGRESS_IO.write_nonblock('F')
+  end
+
+end
+
+run DispatcherDaemon.new

data/bench/report.rb

@@ -1,6 +1,6 @@
 require 'benchmark'
 require 'scmd'
-require 'bench/queue'
+require 'bench/setup'
 
 class BenchRunner
 
@@ -18,23 +18,37 @@ class BenchRunner
     @job_name       = 'multiply'
     @job_params     = { 'size' => 100_000 }
 
+    @number_of_events = ENV['NUM_EVENTS'] || 10_000
+    @event_channel    = 'something'
+    @event_name       = 'happened'
+    @event_params     = { 'size' => 100_000 }
+
     @progress_reader, @progress_writer = IO.pipe
 
     @results = {}
-
-    Qs.init
   end
 
   def run
     output "Running benchmark report..."
     output("\n", false)
 
-    benchmark_adding_jobs
+    Qs.client.clear(BenchQueue.redis_key)
+    Qs.client.clear(Qs.dispatcher_queue.redis_key)
+
+    benchmark_enqueueing_jobs
     benchmark_running_jobs
 
+    benchmark_publishing_events
+    benchmark_running_events
+
     size = @results.values.map(&:size).max
-    output "Adding #{@number_of_jobs} Jobs Time:  #{@results[:adding_jobs].rjust(size)}s"
-    output "Running #{@number_of_jobs} Jobs Time: #{@results[:running_jobs].rjust(size)}s"
+    output "\n", false
+    output "Enqueueing #{@number_of_jobs} Jobs Time: #{@results[:enqueueing_jobs].rjust(size)}"
+    output "Running #{@number_of_jobs} Jobs Time:    #{@results[:running_jobs].rjust(size)}"
+
+    output "\n", false
+    output "Publishing #{@number_of_events} Events Time: #{@results[:publishing_events].rjust(size)}"
+    output "Running #{@number_of_events} Events Time:    #{@results[:running_events].rjust(size)}"
 
     output "\n"
     output "Done running benchmark report"
@@ -42,15 +56,15 @@ class BenchRunner
 
   private
 
-  def benchmark_adding_jobs
-    output "Adding jobs"
+  def benchmark_enqueueing_jobs
+    output "Enqueuing jobs"
     benchmark = Benchmark.measure do
       (1..@number_of_jobs).each do |n|
         BenchQueue.add(@job_name, @job_params)
         output('.', false) if ((n - 1) % 100 == 0)
       end
     end
-    @results[:adding_jobs] = round_and_display(benchmark.real)
+    @results[:enqueueing_jobs] = round_and_display(benchmark.real)
     output("\n", false)
   end
 
@@ -78,7 +92,6 @@ class BenchRunner
         end
       end
       @results[:running_jobs] = round_and_display(benchmark.real)
-      output("\n", false)
     ensure
       cmd.kill('TERM')
       cmd.wait(5)
@@ -87,6 +100,63 @@ class BenchRunner
     output("\n", false)
   end
 
+  def benchmark_publishing_events
+    output "Publishing events"
+    benchmark = Benchmark.measure do
+      (1..@number_of_events).each do |n|
+        Qs.publish(@event_channel, @event_name, @event_params)
+        output('.', false) if ((n - 1) % 100 == 0)
+      end
+    end
+    @results[:publishing_events] = round_and_display(benchmark.real)
+    output("\n", false)
+  end
+
+  def benchmark_running_events
+    bench_queue_cmd_str = "bundle exec ./bin/qs bench/config.qs"
+    bench_queue_cmd = Scmd.new(bench_queue_cmd_str, {
+      'BENCH_REPORT'      => 'yes',
+      'BENCH_PROGRESS_IO' => @progress_writer.fileno
+    })
+
+    dispatcher_queue_cmd_str = "bundle exec ./bin/qs bench/dispatcher.qs"
+    dispatcher_queue_cmd = Scmd.new(dispatcher_queue_cmd_str, {
+      'BENCH_REPORT'      => 'yes',
+      'BENCH_PROGRESS_IO' => @progress_writer.fileno
+    })
+
+    output "Running events"
+    begin
+      benchmark = Benchmark.measure do
+        bench_queue_cmd.start
+        if !bench_queue_cmd.running?
+          raise "failed to start qs process: #{bench_queue_cmd_str.inspect}"
+        end
+
+        dispatcher_queue_cmd.start
+        if !dispatcher_queue_cmd.running?
+          raise "failed to start qs process: #{dispatcher_queue_cmd_str.inspect}"
+        end
+
+        progress = 0
+        while progress < @number_of_jobs
+          ::IO.select([@progress_reader])
+          result = @progress_reader.read_nonblock(1)
+          progress += 1
+          output(result, false) if ((progress - 1) % 100 == 0)
+        end
+      end
+      @results[:running_events] = round_and_display(benchmark.real)
+    ensure
+      dispatcher_queue_cmd.kill('TERM')
+      bench_queue_cmd.kill('TERM')
+      dispatcher_queue_cmd.wait(5)
+      bench_queue_cmd.wait(5)
+    end
+
+    output("\n", false)
+  end
+
   private
 
   def output(message, puts = true)

data/bench/report.txt

@@ -1,11 +1,18 @@
 Running benchmark report...
 
-Adding jobs
+Enqueuing jobs
 ....................................................................................................
 Running jobs
 ....................................................................................................
+Publishing events
+....................................................................................................
+Running events
+....................................................................................................
+
+Enqueueing 10000 Jobs Time:  1.6858
+Running 10000 Jobs Time:    11.3011
 
-Adding 10000 Jobs Time:   1.6133s
-Running 10000 Jobs Time: 12.4769s
+Publishing 10000 Events Time:  2.4596
+Running 10000 Events Time:    16.0522
 
 Done running benchmark report

data/bench/setup.rb

@@ -0,0 +1,55 @@
+require 'qs'
+require 'json'
+
+ROOT_PATH = Pathname.new(File.expand_path('../..', __FILE__))
+
+LOGGER = if ENV['BENCH_REPORT']
+  Logger.new(ROOT_PATH.join(ENV['LOG_NAME']).to_s)
+else
+  Logger.new(STDOUT)
+end
+LOGGER.datetime_format = "" # turn off the datetime in the logs
+
+PROGRESS_IO = if ENV['BENCH_PROGRESS_IO']
+  ::IO.for_fd(ENV['BENCH_PROGRESS_IO'].to_i)
+else
+  File.open('/dev/null', 'w')
+end
+
+Qs.config.dispatcher.queue_name = 'bench-dispatcher'
+Qs.config.event_publisher = 'Bench Script'
+Qs.init
+
+BenchQueue = Qs::Queue.new do
+  name 'bench'
+
+  job 'multiply', 'BenchHandlers::Multiply'
+
+  event 'something', 'happened', 'BenchHandlers::SomethingHappened'
+
+end
+BenchQueue.sync_subscriptions
+
+module BenchHandlers
+
+  class Multiply
+    include Qs::JobHandler
+
+    after{ PROGRESS_IO.write_nonblock('.') }
+
+    def run!
+      'a' * params['size']
+    end
+  end
+
+  class SomethingHappened
+    include Qs::EventHandler
+
+    after{ PROGRESS_IO.write_nonblock('.') }
+
+    def run!
+      'a' * params['size']
+    end
+  end
+
+end

data/lib/qs.rb

@@ -2,6 +2,8 @@ require 'ns-options'
 require 'qs/version'
 require 'qs/client'
 require 'qs/daemon'
+require 'qs/dispatcher_queue'
+require 'qs/event_handler'
 require 'qs/job_handler'
 require 'qs/queue'
 
@@ -19,19 +21,27 @@ module Qs
       self.config.redis.db
     )
 
-    @serializer   ||= self.config.serializer
-    @deserializer ||= self.config.deserializer
-    @client       ||= Client.new(self.redis_config)
-    @redis        ||= @client.redis
+    @dispatcher_queue ||= DispatcherQueue.new({
+      :queue_class            => self.config.dispatcher_queue_class,
+      :queue_name             => self.config.dispatcher.queue_name,
+      :job_name               => self.config.dispatcher.job_name,
+      :job_handler_class_name => self.config.dispatcher.job_handler_class_name
+    })
+
+    @encoder ||= self.config.encoder
+    @decoder ||= self.config.decoder
+    @client  ||= Client.new(self.redis_config)
+    @redis   ||= @client.redis
     true
   end
 
   def self.reset!
     self.config.reset
-    @serializer   = nil
-    @deserializer = nil
-    @client       = nil
-    @redis        = nil
+    @dispatcher_queue = nil
+    @encoder          = nil
+    @decoder          = nil
+    @client           = nil
+    @redis            = nil
     true
   end
 
@@ -39,16 +49,36 @@ module Qs
     @client.enqueue(queue, job_name, params)
   end
 
+  def self.publish(channel, name, params = nil)
+    @client.publish(channel, name, params)
+  end
+
+  def self.publish_as(publisher, channel, name, params = nil)
+    @client.publish_as(publisher, channel, name, params)
+  end
+
   def self.push(queue_name, payload)
     @client.push(queue_name, payload)
   end
 
-  def self.serialize(payload)
-    @serializer.call(payload)
+  def self.encode(payload)
+    @encoder.call(payload)
+  end
+
+  def self.decode(encoded_payload)
+    @decoder.call(encoded_payload)
   end
 
-  def self.deserialize(serialized_payload)
-    @deserializer.call(serialized_payload)
+  def self.sync_subscriptions(queue)
+    self.client.sync_subscriptions(queue)
+  end
+
+  def self.clear_subscriptions(queue)
+    self.client.clear_subscriptions(queue)
+  end
+
+  def self.event_subscribers(event)
+    self.client.event_subscribers(event)
   end
 
   def self.client
@@ -63,16 +93,40 @@ module Qs
     self.config.redis.to_hash
   end
 
+  def self.dispatcher_queue
+    @dispatcher_queue
+  end
+
+  def self.dispatcher_job_name
+    self.config.dispatcher.job_name
+  end
+
+  def self.event_publisher
+    self.config.event_publisher
+  end
+
+  def self.published_events
+    self.dispatcher_queue.published_events
+  end
+
   class Config
     include NsOptions::Proxy
 
-    option :serializer,   Proc, :default => proc{ |p| ::JSON.dump(p) }
-    option :deserializer, Proc, :default => proc{ |p| ::JSON.load(p) }
+    option :encoder, Proc, :default => proc{ |p| ::JSON.dump(p) }
+    option :decoder, Proc, :default => proc{ |p| ::JSON.load(p) }
 
     option :timeout, Float
 
+    option :event_publisher, String
+
+    namespace :dispatcher do
+      option :queue_name,             String, :default => 'dispatcher'
+      option :job_name,               String, :default => 'run_dispatch_job'
+      option :job_handler_class_name, String, :default => DispatcherQueue::RunDispatchJob.to_s
+    end
+
     namespace :redis do
-      option :ip,   :default => 'localhost'
+      option :ip,   :default => '127.0.0.1'
       option :port, :default => 6379
       option :db,   :default => 0
 
@@ -83,6 +137,12 @@ module Qs
       option :timeout,  Integer, :default => 1
       option :size,     Integer, :default => 4
     end
+
+    attr_accessor :dispatcher_queue_class
+
+    def initialize
+      self.dispatcher_queue_class = Queue
+    end
   end
 
   module RedisUrl

data/lib/qs/client.rb

@@ -1,6 +1,8 @@
 require 'hella-redis'
 require 'qs'
+require 'qs/dispatch_job'
 require 'qs/job'
+require 'qs/payload'
 require 'qs/queue'
 
 module Qs
@@ -29,13 +31,24 @@ module Qs
         @redis_config = redis_config
       end
 
-      def enqueue(queue, job_name, params = nil)
-        job = Qs::Job.new(job_name, params || {})
+      def enqueue(queue, job_name, job_params = nil)
+        job = Qs::Job.new(job_name, :params => job_params)
         enqueue!(queue, job)
         job
       end
 
-      def push(queue_name, payload)
+      def publish(channel, name, params = nil)
+        publish!(channel, name, :event_params => params)
+      end
+
+      def publish_as(publisher, channel, name, params = nil)
+        publish!(channel, name, {
+          :event_params    => params,
+          :event_publisher => publisher,
+        })
+      end
+
+      def push(queue_name, payload_hash)
         raise NotImplementedError
       end
 
@@ -43,12 +56,12 @@ module Qs
         self.redis.with{ |c| c.brpop(*args) }
       end
 
-      def append(queue_redis_key, serialized_payload)
-        self.redis.with{ |c| c.lpush(queue_redis_key, serialized_payload) }
+      def append(queue_redis_key, encoded_payload)
+        self.redis.with{ |c| c.lpush(queue_redis_key, encoded_payload) }
       end
 
-      def prepend(queue_redis_key, serialized_payload)
-        self.redis.with{ |c| c.rpush(queue_redis_key, serialized_payload) }
+      def prepend(queue_redis_key, encoded_payload)
+        self.redis.with{ |c| c.rpush(queue_redis_key, encoded_payload) }
       end
 
       def clear(redis_key)
@@ -59,6 +72,44 @@ module Qs
         self.redis.with{ |c| c.ping }
       end
 
+      def sync_subscriptions(queue)
+        pattern = Qs::Event::SubscribersRedisKey.new('*')
+        all_event_subs_keys = self.redis.with{ |c| c.keys(pattern) }
+
+        event_subs_keys = queue.event_route_names.map do |route_name|
+          Qs::Event::SubscribersRedisKey.new(route_name)
+        end
+        redis_transaction do |c|
+          all_event_subs_keys.each{ |key| c.srem(key, queue.name) }
+          event_subs_keys.each{ |key| c.sadd(key, queue.name) }
+        end
+      end
+
+      def clear_subscriptions(queue)
+        pattern = Qs::Event::SubscribersRedisKey.new('*')
+        event_subs_keys = self.redis.with{ |c| c.keys(pattern) }
+
+        redis_transaction do |c|
+          event_subs_keys.each{ |key| c.srem(key, queue.name) }
+        end
+      end
+
+      def event_subscribers(event)
+        self.redis.with{ |c| c.smembers(event.subscribers_redis_key) }
+      end
+
+      private
+
+      def publish!(channel, name, options = nil)
+        dispatch_job = DispatchJob.new(channel, name, options)
+        enqueue!(Qs.dispatcher_queue, dispatch_job)
+        dispatch_job.event
+      end
+
+      def redis_transaction
+        self.redis.with{ |c| c.pipelined{ c.multi{ yield c } } }
+      end
+
     end
 
   end
@@ -71,17 +122,17 @@ module Qs
       @redis = HellaRedis::Connection.new(self.redis_config)
     end
 
-    def push(queue_name, payload)
-      queue_redis_key    = Queue::RedisKey.new(queue_name)
-      serialized_payload = Qs.serialize(payload)
-      self.append(queue_redis_key, serialized_payload)
+    def push(queue_name, payload_hash)
+      queue_redis_key = Queue::RedisKey.new(queue_name)
+      encoded_payload = Qs.encode(payload_hash)
+      self.append(queue_redis_key, encoded_payload)
     end
 
     private
 
     def enqueue!(queue, job)
-      serialized_payload = Qs.serialize(job.to_payload)
-      self.append(queue.redis_key, serialized_payload)
+      encoded_payload = Qs::Payload.serialize(job)
+      self.append(queue.redis_key, encoded_payload)
     end
 
   end
@@ -98,11 +149,11 @@ module Qs
       @pushed_items = []
     end
 
-    def push(queue_name, payload)
-      # attempt to serialize (and then throw away) the payload, this will error
-      # on the developer if it can't be serialized
-      Qs.serialize(payload)
-      @pushed_items << PushedItem.new(queue_name, payload)
+    def push(queue_name, payload_hash)
+      # attempt to encode (and then throw away) the payload hash, this will
+      # error on the developer if it can't be encoded
+      Qs.encode(payload_hash)
+      @pushed_items << PushedItem.new(queue_name, payload_hash)
     end
 
     def reset!
@@ -112,13 +163,13 @@ module Qs
     private
 
     def enqueue!(queue, job)
-      # attempt to serialize (and then throw away) the job payload, this will
-      # error on the developer if it can't serialize the job
-      Qs.serialize(job.to_payload)
+      # attempt to serialize (and then throw away) the job, this will error on
+      # the developer if it can't serialize the job
+      Qs::Payload.serialize(job)
       queue.enqueued_jobs << job
     end
 
-    PushedItem = Struct.new(:queue_name, :payload)
+    PushedItem = Struct.new(:queue_name, :payload_hash)
 
   end