qs 0.0.1 → 0.1.0

data/lib/qs.rb

@@ -1,5 +1,95 @@
-require "qs/version"
+require 'ns-options'
+require 'qs/version'
+require 'qs/client'
+require 'qs/daemon'
+require 'qs/job_handler'
+require 'qs/queue'
 
 module Qs
-  # TODO: your code goes here...
+
+  def self.config; @config ||= Config.new; end
+  def self.configure(&block)
+    block.call(self.config)
+  end
+
+  def self.init
+    self.config.redis.url ||= RedisUrl.new(
+      self.config.redis.ip,
+      self.config.redis.port,
+      self.config.redis.db
+    )
+
+    @serializer   ||= self.config.serializer
+    @deserializer ||= self.config.deserializer
+    @client       ||= Client.new(self.redis_config)
+    @redis        ||= @client.redis
+    true
+  end
+
+  def self.reset!
+    self.config.reset
+    @serializer   = nil
+    @deserializer = nil
+    @client       = nil
+    @redis        = nil
+    true
+  end
+
+  def self.enqueue(queue, job_name, params = nil)
+    @client.enqueue(queue, job_name, params)
+  end
+
+  def self.push(queue_name, payload)
+    @client.push(queue_name, payload)
+  end
+
+  def self.serialize(payload)
+    @serializer.call(payload)
+  end
+
+  def self.deserialize(serialized_payload)
+    @deserializer.call(serialized_payload)
+  end
+
+  def self.client
+    @client
+  end
+
+  def self.redis
+    @redis
+  end
+
+  def self.redis_config
+    self.config.redis.to_hash
+  end
+
+  class Config
+    include NsOptions::Proxy
+
+    option :serializer,   Proc, :default => proc{ |p| ::JSON.dump(p) }
+    option :deserializer, Proc, :default => proc{ |p| ::JSON.load(p) }
+
+    option :timeout, Float
+
+    namespace :redis do
+      option :ip,   :default => 'localhost'
+      option :port, :default => 6379
+      option :db,   :default => 0
+
+      option :url
+
+      option :redis_ns, String,  :default => 'qs'
+      option :driver,   String,  :default => 'ruby'
+      option :timeout,  Integer, :default => 1
+      option :size,     Integer, :default => 4
+    end
+  end
+
+  module RedisUrl
+    def self.new(ip, port, db)
+      return if ip.to_s.empty? || port.to_s.empty? || db.to_s.empty?
+      "redis://#{ip}:#{port}/#{db}"
+    end
+  end
+
 end

data/qs.gemspec

@@ -11,13 +11,18 @@ Gem::Specification.new do |gem|
   gem.description = %q{Define message queues. Process jobs and events. Profit.}
   gem.summary     = %q{Define message queues. Process jobs and events. Profit.}
   gem.homepage    = "http://github.com/redding/qs"
+  gem.license     = 'MIT'
 
   gem.files         = `git ls-files`.split($/)
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
-  gem.add_development_dependency("assert")
-  # TODO: gem.add_dependency("gem-name", ["~> 0.0"])
+  gem.add_dependency("dat-worker-pool", ["~> 0.5"])
+  gem.add_dependency("hella-redis",     ["~> 0.2"])
+  gem.add_dependency("ns-options",      ["~> 1.1"])
+  gem.add_dependency("SystemTimer",     ["~> 1.2"])
 
+  gem.add_development_dependency("assert", ["~> 2.14"])
+  gem.add_development_dependency("scmd",   ["~> 2.3"])
 end

data/test/helper.rb

@@ -4,4 +4,11 @@
 # add the root dir to the load path
 $LOAD_PATH.unshift(File.expand_path("../..", __FILE__))
 
-# TODO: put test helpers here...
+require 'pry' # require pry for debugging (`binding.pry`)
+
+require 'pathname'
+ROOT_PATH = Pathname.new(File.expand_path('../..', __FILE__))
+
+require 'test/support/factory'
+
+require 'json' # so the default serializer/deserializer procs will work

data/test/support/app_daemon.rb

@@ -0,0 +1,74 @@
+require 'qs'
+
+AppQueue = Qs::Queue.new do
+  name 'app_main'
+
+  job_handler_ns 'AppHandlers'
+
+  job 'basic',   'Basic'
+  job 'error',   'Error'
+  job 'timeout', 'Timeout'
+  job 'slow',    'Slow'
+end
+
+class AppDaemon
+  include Qs::Daemon
+
+  name 'app'
+
+  logger Logger.new(ROOT_PATH.join('log/app_daemon.log').to_s)
+  verbose_logging true
+
+  queue AppQueue
+
+  error do |exception, context|
+    job_name = context.job.name if context.job
+    case(job_name)
+    when 'error', 'timeout'
+      message = "#{exception.class}: #{exception.message}"
+      Qs.redis.with{ |c| c.set('last_error', message) }
+    when 'slow'
+      Qs.redis.with{ |c| c.set('last_error', exception.class.to_s) }
+    end
+  end
+
+end
+
+module AppHandlers
+
+  class Basic
+    include Qs::JobHandler
+
+    def run!
+      Qs.redis.with{ |c| c.set(params['key'], params['value']) }
+    end
+  end
+
+  class Error
+    include Qs::JobHandler
+
+    def run!
+      raise params['error_message']
+    end
+  end
+
+  class Timeout
+    include Qs::JobHandler
+
+    timeout 0.2
+
+    def run!
+      sleep 2
+    end
+  end
+
+  class Slow
+    include Qs::JobHandler
+
+    def run!
+      sleep 5
+      Qs.redis.with{ |c| c.set('slow', 'finished') }
+    end
+  end
+
+end

data/test/support/config.qs

@@ -0,0 +1,7 @@
+require 'test/support/app_daemon'
+
+if !defined?(TestConstant)
+  TestConstant = Class.new
+end
+
+run AppDaemon.new

data/test/support/config_files/invalid.qs

@@ -0,0 +1 @@
+run 'test'

data/test/support/config_files/valid.qs

@@ -0,0 +1,7 @@
+require 'qs'
+
+class MyDaemon
+  include Qs::Daemon
+end
+
+run MyDaemon.new

data/test/support/config_invalid_run.qs

@@ -0,0 +1,3 @@
+MyDaemon = Class.new
+
+run MyDaemon

data/test/support/factory.rb

@@ -0,0 +1,14 @@
+require 'assert/factory'
+
+module Factory
+  extend Assert::Factory
+
+  def self.exception(klass = nil, message = nil)
+    klass ||= StandardError
+    message ||= Factory.text
+    exception = nil
+    begin; raise(klass, message); rescue StandardError => exception; end
+    exception
+  end
+
+end

data/test/support/pid_file_spy.rb

@@ -0,0 +1,19 @@
+class PIDFileSpy
+
+  attr_reader :pid, :write_called, :remove_called
+
+  def initialize(pid)
+    @pid = pid
+    @write_called = false
+    @remove_called = false
+  end
+
+  def write
+    @write_called = true
+  end
+
+  def remove
+    @remove_called = true
+  end
+
+end

data/test/support/runner_spy.rb

@@ -0,0 +1,17 @@
+class RunnerSpy
+
+  attr_reader :handler_class, :args, :handler
+  attr_reader :run_called
+
+  def initialize(handler_class, args = nil)
+    @handler_class = handler_class
+    @args          = args
+    @handler       = Factory.string
+    @run_called    = false
+  end
+
+  def run
+    @run_called = true
+  end
+
+end

data/test/system/daemon_tests.rb

@@ -0,0 +1,226 @@
+require 'assert'
+require 'qs/daemon'
+
+require 'test/support/app_daemon'
+
+module Qs::Daemon
+
+  class SystemTests < Assert::Context
+    desc "Qs::Daemon"
+    setup do
+      Qs.reset!
+      @qs_test_mode = ENV['QS_TEST_MODE']
+      ENV['QS_TEST_MODE'] = nil
+      Qs.init
+      @orig_config = AppDaemon.configuration.to_hash
+    end
+    teardown do
+      @daemon_runner.stop if @daemon_runner
+      AppDaemon.configuration.apply(@orig_config) # reset daemon config
+      Qs.redis.with{ |c| c.del('slow') }
+      Qs.redis.with{ |c| c.del('last_error') }
+      Qs.client.clear(AppQueue.redis_key)
+      Qs.reset!
+      ENV['QS_TEST_MODE'] = @qs_test_mode
+    end
+
+  end
+
+  class RunningDaemonSetupTests < SystemTests
+    setup do
+      @daemon = AppDaemon.new
+      @daemon_runner = DaemonRunner.new(@daemon)
+      @thread = @daemon_runner.start
+    end
+
+  end
+
+  class BasicJobTests < RunningDaemonSetupTests
+    desc "with a basic job added"
+    setup do
+      @key, @value = [Factory.string, Factory.string]
+      AppQueue.add('basic', {
+        'key'   => @key,
+        'value' => @value
+      })
+      @thread.join 0.5
+    end
+
+    should "run the job" do
+      assert_equal @value, Qs.redis.with{ |c| c.get(@key) }
+    end
+
+  end
+
+  class JobThatErrorsTests < RunningDaemonSetupTests
+    desc "with a job that errors"
+    setup do
+      @error_message = Factory.text
+      AppQueue.add('error', 'error_message' => @error_message)
+      @thread.join 0.5
+    end
+
+    should "run the configured error handler procs" do
+      exp = "RuntimeError: #{@error_message}"
+      assert_equal exp, Qs.redis.with{ |c| c.get('last_error') }
+    end
+
+  end
+
+  class TimeoutJobTests < RunningDaemonSetupTests
+    desc "with a job that times out"
+    setup do
+      AppQueue.add('timeout')
+      @thread.join 1 # let the daemon have time to process the job
+    end
+
+    should "run the configured error handler procs" do
+      handler_class = AppHandlers::Timeout
+      exp = "Qs::TimeoutError: #{handler_class} timed out " \
+            "(#{handler_class.timeout}s)"
+      assert_equal exp, Qs.redis.with{ |c| c.get('last_error') }
+    end
+
+  end
+
+  class NoWorkersAvailableTests < SystemTests
+    desc "when no workers are available"
+    setup do
+      AppDaemon.workers 0 # no workers available, don't do this
+      @daemon = AppDaemon.new
+      @daemon_runner = DaemonRunner.new(@daemon)
+      @thread = @daemon_runner.start
+    end
+
+    should "shutdown when stopped" do
+      @daemon.stop
+      @thread.join 2 # give it time to shutdown, should be faster
+      assert_false @thread.alive?
+    end
+
+    should "shutdown when halted" do
+      @daemon.halt
+      @thread.join 2 # give it time to shutdown, should be faster
+      assert_false @thread.alive?
+    end
+
+  end
+
+  class ShutdownWithoutTimeoutTests < SystemTests
+    desc "without a shutdown timeout"
+    setup do
+      AppDaemon.shutdown_timeout nil # disable shutdown timeout
+      @daemon = AppDaemon.new
+      @daemon_runner = DaemonRunner.new(@daemon)
+      @thread = @daemon_runner.start
+
+      AppQueue.add('slow')
+      @thread.join 1 # let the daemon have time to process the job
+    end
+
+    should "shutdown and let the job finished" do
+      @daemon.stop
+      @thread.join 10 # give it time to shutdown, should be faster
+      assert_false @thread.alive?
+      assert_equal 'finished', Qs.redis.with{ |c| c.get('slow') }
+    end
+
+    should "shutdown and not let the job finished" do
+      @daemon.halt
+      @thread.join 2 # give it time to shutdown, should be faster
+      assert_false @thread.alive?
+      assert_nil Qs.redis.with{ |c| c.get('slow') }
+      exp = "Qs::ShutdownError"
+      assert_equal exp, Qs.redis.with{ |c| c.get('last_error') }
+    end
+
+  end
+
+  class ShutdownWithTimeoutTests < SystemTests
+    desc "with a shutdown timeout"
+    setup do
+      AppDaemon.shutdown_timeout 1
+      @daemon = AppDaemon.new
+      @daemon_runner = DaemonRunner.new(@daemon)
+      @thread = @daemon_runner.start
+
+      AppQueue.add('slow')
+      @thread.join 1 # let the daemon have time to process the job
+    end
+
+    should "shutdown and not let the job finished" do
+      @daemon.stop
+      @thread.join 2 # give it time to shutdown, should be faster
+      assert_false @thread.alive?
+      assert_nil Qs.redis.with{ |c| c.get('slow') }
+      exp = "Qs::ShutdownError"
+      assert_equal exp, Qs.redis.with{ |c| c.get('last_error') }
+    end
+
+    should "shutdown and not let the job finished" do
+      @daemon.halt
+      @thread.join 2 # give it time to shutdown, should be faster
+      assert_false @thread.alive?
+      assert_nil Qs.redis.with{ |c| c.get('slow') }
+      exp = "Qs::ShutdownError"
+      assert_equal exp, Qs.redis.with{ |c| c.get('last_error') }
+    end
+
+  end
+
+  class ShutdownWithUnprocessedRedisItemTests < SystemTests
+    desc "with a redis item that gets picked up but doesn't get processed"
+    setup do
+      Assert.stub(Qs::PayloadHandler, :new){ sleep 5 }
+
+      AppDaemon.shutdown_timeout 1
+      AppDaemon.workers 2
+      @daemon = AppDaemon.new
+      @daemon_runner = DaemonRunner.new(@daemon)
+      @thread = @daemon_runner.start
+
+      AppQueue.add('slow')
+      AppQueue.add('slow')
+      AppQueue.add('basic')
+      @thread.join 1 # let the daemon have time to process jobs
+    end
+
+    should "shutdown and requeue the redis item" do
+      @daemon.stop
+      @thread.join 2 # give it time to shutdown, should be faster
+      assert_false @thread.alive?
+      # TODO - better way to read whats on a queue
+      serialized_payloads = Qs.redis.with{ |c| c.lrange(AppQueue.redis_key, 0, 3) }
+      names = serialized_payloads.map{ |sp| Qs::Job.parse(Qs.deserialize(sp)).name }
+      assert_equal ['basic', 'slow', 'slow'], names
+    end
+
+    should "shutdown and requeue the redis item" do
+      @daemon.halt
+      @thread.join 2 # give it time to shutdown, should be faster
+      assert_false @thread.alive?
+      # TODO - better way to read whats on a queue
+      serialized_payloads = Qs.redis.with{ |c| c.lrange(AppQueue.redis_key, 0, 3) }
+      names = serialized_payloads.map{ |sp| Qs::Job.parse(Qs.deserialize(sp)).name }
+      assert_equal ['basic', 'slow', 'slow'], names
+    end
+
+  end
+
+  class DaemonRunner
+    def initialize(daemon)
+      @daemon = daemon
+      @thread = nil
+    end
+
+    def start
+      @thread = @daemon.start
+    end
+
+    def stop
+      @daemon.halt
+      @thread.join if @thread
+    end
+  end
+
+end

data/test/unit/cli_tests.rb

@@ -0,0 +1,188 @@
+require 'assert'
+require 'qs/cli'
+
+require 'qs/daemon'
+
+class Qs::CLI
+
+  class UnitTests < Assert::Context
+    desc "Qs::CLI"
+    setup do
+      @kernel_spy = KernelSpy.new
+      @file_path = Factory.file_path
+
+      @daemon = TestDaemon.new
+
+      @config_file = FakeConfigFile.new(@daemon)
+      Assert.stub(Qs::ConfigFile, :new).with(@file_path){ @config_file }
+
+      @cli = Qs::CLI.new(@kernel_spy)
+    end
+    subject{ @cli }
+
+    should have_cmeths :run
+    should have_imeths :run
+
+  end
+
+  class CommandTests < UnitTests
+    setup do
+      @process_spy = ProcessSpy.new
+      @process_signal_spy = ProcessSignalSpy.new
+    end
+
+  end
+
+  class DefaultsTests < CommandTests
+    desc "with no command or file path"
+    setup do
+      file_path = 'config.qs'
+      Assert.stub(Qs::ConfigFile, :new).with(file_path){ @config_file }
+      Assert.stub(Qs::Process, :new).with(@daemon, :daemonize => false) do
+        @process_spy
+      end
+
+      @cli.run
+    end
+
+    should "have defaulted the command and file path" do
+      assert_true @process_spy.run_called
+    end
+
+  end
+
+  class RunTests < CommandTests
+    desc "with the run command"
+    setup do
+      Assert.stub(Qs::Process, :new).with(@daemon, :daemonize => false) do
+        @process_spy
+      end
+
+      @cli.run(@file_path, 'run')
+    end
+
+    should "have built and run a non-daemonized process" do
+      assert_true @process_spy.run_called
+    end
+
+  end
+
+  class StartTests < CommandTests
+    desc "with the start command"
+    setup do
+      Assert.stub(Qs::Process, :new).with(@daemon, :daemonize => true) do
+        @process_spy
+      end
+
+      @cli.run(@file_path, 'start')
+    end
+
+    should "have built and run a daemonized process" do
+      assert_true @process_spy.run_called
+    end
+
+  end
+
+  class StopTests < CommandTests
+    desc "with the stop command"
+    setup do
+      Assert.stub(Qs::ProcessSignal, :new).with(@daemon, 'TERM') do
+        @process_signal_spy
+      end
+
+      @cli.run(@file_path, 'stop')
+    end
+
+    should "have built and sent a TERM signal" do
+      assert_true @process_signal_spy.send_called
+    end
+
+  end
+
+  class RestartTests < CommandTests
+    desc "with the restart command"
+    setup do
+      Assert.stub(Qs::ProcessSignal, :new).with(@daemon, 'USR2') do
+        @process_signal_spy
+      end
+
+      @cli.run(@file_path, 'restart')
+    end
+
+    should "have built and sent a USR2 signal" do
+      assert_true @process_signal_spy.send_called
+    end
+
+  end
+
+  class InvalidCommandTests < UnitTests
+    desc "with an invalid command"
+    setup do
+      @command = Factory.string
+      @cli.run(@file_path, @command)
+    end
+
+    should "output the error with the help" do
+      expected = "#{@command.inspect} is not a valid command"
+      assert_includes expected, @kernel_spy.output
+      assert_includes "Usage: qs", @kernel_spy.output
+    end
+
+  end
+
+  class KernelSpy
+    attr_reader :exit_status
+
+    def initialize
+      @output = StringIO.new
+      @exit_status = nil
+    end
+
+    def output
+      @output.rewind
+      @output.read
+    end
+
+    def puts(message)
+      @output.puts(message)
+    end
+
+    def exit(code)
+      @exit_status = code
+    end
+  end
+
+  class TestDaemon
+    include Qs::Daemon
+
+    name Factory.string
+    queue Qs::Queue.new{ name Factory.string }
+  end
+
+  FakeConfigFile = Struct.new(:daemon)
+
+  class ProcessSpy
+    attr_reader :run_called
+
+    def initialize
+      @run_called = false
+    end
+
+    def run
+      @run_called = true
+    end
+  end
+
+  class ProcessSignalSpy
+    attr_reader :send_called
+
+    def initialize
+      @send_called = false
+    end
+
+    def send
+      @send_called = true
+    end
+  end
+
+end