disque_jockey 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ec24b527a949eebcfd10f84df9ea4c15224442de
-  data.tar.gz: bf38d59c5f6b95b619171f8fd64d74129d253470
+  metadata.gz: c49083bcd86ac5171c257bd78eeded4abbd5c99f
+  data.tar.gz: 868ababcc280ffa34de1b8270c07b00c04bc299c
 SHA512:
-  metadata.gz: 45bcdbf55a10a3420285c16d8a9007c4b7b7d95a956780e1ba6db9d98524a2877cee3a1a2b82aad7aec0056258315659f35b4f5a3aa55fffb3d60139bfa3805f
-  data.tar.gz: 74e2838237a2a8793d7dffb0f09839ce72a10feffd618bce9f797edefc040e94184f28112bc457c87071647380a6ca09690ca2f9451d4dfd999ebd72cd976658
+  metadata.gz: 8fa502414a8a708a693cff041773bb2ae9201d7d0be70dd5fd5eafe2bedd3b6dc31852f7e9f1f516f2dffe52e7998e70923f49eb2a07817d3b768bc683dd632e
+  data.tar.gz: 8dec61c2d6e36a1347147114f00e6290aa2cd2c2d4ae83b76124fdc4f68581e3727208c1f5554b2aa376fd76c212aa2e8e87ed0681bd0e2d1dc00a6f5058f43c

data/README.md

@@ -1,6 +1,7 @@
 # DisqueJockey
 DisqueJockey is a fast, concurrent background job processing framework for the Disque message queue.
-# Getting Started
+
+## Installation
 First, you should run a Disque server if you aren't already doing so.
 Disque source and build instructions can be found at: https://github.com/antirez/disque
 
@@ -48,10 +49,17 @@ Your worker class must do two things:
 
 Lastly, you must place your worker in a directory named 'workers'
 
-Once your worker is written and placed in a workers directory, you can call disque_jockey from the command line and it will start up your workers and begin delivering jobs to them.
+## Starting Disque Jockey
+Once your worker is written and placed in a workers directory, you can call `disque_jockey start` from the command line and it will start up your workers and begin delivering jobs to them.
+
+To see all the command line options, use the help command:
+```
+disque_jockey help start
+```
 
+To start disque_jockey with the desired options:
 ````
-disque_jockey
+disque_jockey start --env=production --daemonize=true --worker-groups=10  --nodes=127.0.0.1:7111,34.45.231.124:4242
 ````
 
 Messages successfully handled by a worker (ie no exceptions raised from the handle method) will be acknowledged and removed from the queue.
@@ -61,6 +69,5 @@ DisqueJockey is not a currently a production-ready system, and there are a numbe
 Here is a list of functionality I'd like to add to DisqueJockey in the near future:
 - Allow workers to set auto-acknowledge or fast-acknowledge of messages.
 - Better test coverage around worker groups
-- Command line options (e.g. environment)
 - Rails integration (ActiveJob Adapter)
 - More use cases in the README (e.g. how to use alongside Rails)

data/bin/disque_jockey

@@ -1,4 +1,6 @@
 #!/usr/bin/env ruby
 
+$:.unshift(File.expand_path("../../lib", __FILE__))
+
 require 'disque_jockey'
-DisqueJockey.run!
+DisqueJockey::CLI.start(ARGV)

data/disque_jockey.gemspec

@@ -19,6 +19,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ['lib']
   gem.add_runtime_dependency 'disque'
   gem.add_runtime_dependency 'logging'
+  gem.add_runtime_dependency 'thor'
   gem.add_development_dependency('rspec', '~> 3.1', '>= 3.0')
   gem.add_development_dependency('rake')
   gem.add_development_dependency('pry')

data/lib/disque_jockey/cli/help.rb

@@ -0,0 +1,24 @@
+require "thor"
+
+module DisqueJockey
+  class CLI < Thor
+    class Help
+      class << self
+        def start
+          <<-EOL
+            Starts disque_jockey processes to start processing Disque jobs.
+
+            Examples:
+
+            $ disque_jockey start --env=production
+
+            $ disque_jockey start --env=development --daemonize=true --work-groups=5
+
+            $ disque_jockey start --nodes=127.0.0.1:6534,54.634.23.43:3452,546.23.124.34:4353
+          EOL
+        end
+      end
+    end
+  end
+end
+

data/lib/disque_jockey/cli.rb

@@ -0,0 +1,18 @@
+require "thor"
+
+module DisqueJockey
+  class CLI < Thor
+    desc "start", "Start disque_jockey"
+    option :env, :desc => "set environment"
+    option :worker_groups, :desc => "set number of worker groups"
+    option :log_path, :desc => "set path to logs"
+    option :nodes, :desc => "set nodes"
+    option :daemonize, :type => :boolean, :desc => "run disque_jockey as daemon"
+    long_desc DisqueJockey::CLI::Help.start
+
+    def start
+      DisqueJockey.run!(options)
+    end
+  end
+end
+

data/lib/disque_jockey/configuration.rb

@@ -1,29 +1,34 @@
 module DisqueJockey
   class Configuration
-    attr_accessor :logger, :worker_groups, :log_path, :env
-    def initialize
-      # set defaults
-      @worker_groups = worker_groups_for_environment
-      @log_path = log_path_for_environment
-    end
 
-    def env
-      @env ||= ENV['DISQUE_JOCKEY_ENV'] || 'development'
+    attr_accessor :logger, :worker_groups, :log_path, :env, :nodes, :daemonize
+
+    def initialize(options={})
+      @env = options["env"] || ENV["DISQUE_JOCKEY_ENV"] || "development"
+      @worker_groups = (options["worker_groups"] || 2).to_i
+      @log_path = options["log_path"] || log_path_default
+      @nodes = parse_nodes(options["nodes"]) || ["127.0.0.1:7711"]
+      @daemonize = options["daemonize"] || daemonize_default
     end
 
     def daemonize?
-      env != 'development'
+      @daemonize
     end
 
-    def log_path_for_environment
-      env == 'test' ? 'spec/log' : 'log'
+    private
+
+    def parse_nodes(nodes)
+      return unless nodes
+      nodes.split(",")
     end
 
-    # TODO: just read this from a config file
-    def worker_groups_for_environment
-      env == 'development' ? 2 : 4
+    def log_path_default
+      env == "test" ? "spec/log" : "log"
     end
 
+    def daemonize_default
+      env != 'development'
+    end
   end
 end
 

data/lib/disque_jockey/exceptions.rb

@@ -0,0 +1 @@
+class NoWorkersFoundError < StandardError; end

data/lib/disque_jockey/supervisor.rb

@@ -32,6 +32,7 @@ module DisqueJockey
     end
 
     def self.spawn_worker_groups
+      raise NoWorkersFoundError, "No workers found! See README for instructions on loading workrers" if worker_classes.empty?
       DisqueJockey.configuration.worker_groups.times { spawn_worker_group }
     end
 
@@ -46,7 +47,7 @@ module DisqueJockey
       # DisqueJockey only exits if it receives a 
       # kill signal
       loop do
-        @dead_disque_jockeys.each do
+        @dead_worker_groups.each do
           child_pid = @dead_disque_jockeys.shift
           logger.error "Child worker group exited: #{child_pid}"
           child_pids.delete(child_pid)
@@ -57,7 +58,7 @@ module DisqueJockey
     end
 
     def self.trap_signals_in_parent
-      @dead_disque_jockeys = []
+      @dead_worker_groups = []
       %w(QUIT TERM INT ABRT CLD).each do |sig|
         trap(sig) do
           if sig == 'CLD'
@@ -66,7 +67,7 @@ module DisqueJockey
             # This needs to be reentrant, so we queue up dead child
             # processes to be handled in the run loop, rather than 
             # acting here
-            @dead_disque_jockeys << Process.wait
+            @dead_worker_groups << Process.wait
           else 
             begin
               child_pids.each { |pid| Process.kill(sig, pid) }
@@ -81,4 +82,4 @@ module DisqueJockey
 
 
   end
-end
+end

data/lib/disque_jockey/version.rb

@@ -1,3 +1,3 @@
 module DisqueJockey
-  VERSION = '0.0.1'
+  VERSION = '0.0.2'
 end

data/lib/disque_jockey/worker_group.rb

@@ -6,7 +6,6 @@ module DisqueJockey
 
     def initialize(worker_classes = [])
       @worker_classes = worker_classes # array of classes to instantiate in our group
-      @worker_pool = {} # initialize a hash for storing workers
     end
 
     def work!
@@ -36,15 +35,11 @@ module DisqueJockey
       end
     end 
 
-    # instantiate all the workers we want and start giving
-    # them jobs to do
+    # for each worker class, create a worker pool and have
+    # them start fetching jobs and working
     def start_workers
       @worker_classes.each do |worker_class|
-        build_worker_pool(worker_class)
-        # Each worker_class (and hence, queue), get its own
-        # thread because the Disque client library blocks
-        # when waiting for a job from a queue.
-        Thread.new { fetch_job_and_work(worker_class) }
+        Thread.new { WorkerPool.new(worker_class).work! }
       end
     end
 
@@ -58,51 +53,5 @@ module DisqueJockey
       end
     end
 
-    # The worker pool gives us a fixed number of worker instances of each class
-    # to do the work.  This could be improved by dynamically instantiating
-    # and removing workers from the pool based on workload.  For now, we use
-    # a fixed number.
-    def build_worker_pool(worker_class)
-      Supervisor.logger.info("Launching #{worker_class.thread_count} #{worker_class}s")
-      worker_class.thread_count.times do
-        # Use the Queue class so we access our worker pools
-        # from different threads without issues.
-        @worker_pool[worker_class] ||= Queue.new
-        @worker_pool[worker_class].push worker_class.new(Logger)
-      end
-    end
-
-    # Here we actually get jobs to work on and hand them off to worker
-    # instances.
-    def fetch_job_and_work(worker_class)
-      broker = Broker.new
-      loop do
-        # this method blocks until a job is returned
-        _, job_id, job = broker.fetch_message_from(worker_class.queue_name)
-        # Queue#pop will block until a worker becomes available
-        worker = @worker_pool[worker_class].pop
-        # now that we have a worker, give it a thread to do its work in
-        # so we can fetch the next job without waiting.
-        Thread.new do
-          begin
-            # Raise a timeout error if the worker takes too long
-            Timeout::timeout(worker_class.timeout_seconds) { worker.handle(job) }
-            # acknowlege the job once we've handled it
-            broker.acknowledge(job_id)
-          rescue StandardError => exception
-            worker.log_exception(exception)
-            # TODO: Need to implement retry logic
-            #       Also should do more helpful logging around worker timeouts
-            #       (explain the error, log the job and maybe metadata)
-          end
-          # We're done working, so put the worker back in the pool
-          @worker_pool[worker_class].push(worker)
-
-        end
-
-
-      end
-    end
-
   end
 end

data/lib/disque_jockey/worker_pool.rb

@@ -0,0 +1,54 @@
+module DisqueJockey
+  class WorkerPool
+
+    def initialize(worker_class)
+      @worker_class = worker_class
+      @pool = Queue.new
+      @broker = Broker.new(DisqueJockey.configuration.nodes)
+      build_worker_pool
+    end
+
+    def work!
+      endless_loop do
+        # fetching from broker blocks until a job is returned
+        _, job_id, job = @broker.fetch_message_from(@worker_class.queue_name)
+
+        with_worker do |worker|
+          Thread.new { handle_job(worker, job, job_id) }
+        end
+
+      end
+    end
+
+    private
+
+    # this method exists so we can stub the endless loop in tests
+    def endless_loop
+      loop { yield }
+    end
+
+    def with_worker
+      # @pool.pop will block until a worker becomes available
+      worker = @pool.pop
+      yield worker
+      @pool.push(worker)
+    end
+
+    def handle_job(worker, job, job_id)
+      begin
+        Timeout::timeout(@worker_class.timeout_seconds) { worker.handle(job) }
+        @broker.acknowledge(job_id)
+      rescue StandardError => exception
+        worker.log_exception(exception)
+        # TODO: Need to implement retry logic
+        #       Also should do more helpful logging around worker timeouts
+        #       (explain the error, log the job and maybe metadata)
+      end
+    end
+
+    def build_worker_pool
+      @worker_class.thread_count.times { @pool << @worker_class.new(Logger) }      
+    end
+
+  end
+end

data/lib/disque_jockey.rb

@@ -1,13 +1,16 @@
 require 'disque_jockey/version'
+require 'disque_jockey/cli/help'
+require 'disque_jockey/cli'
+require 'disque_jockey/exceptions'
 require 'disque_jockey/broker'
 require 'disque_jockey/logger'
 require 'disque_jockey/supervisor'
 require 'disque_jockey/worker'
 require 'disque_jockey/configuration'
+require 'disque_jockey/worker_pool'
 require 'disque_jockey/worker_group'
 require 'timeout'
 
-
 module DisqueJockey
   # raise exceptions in all threads so we don't fail silently
   Thread.abort_on_exception = true
@@ -17,10 +20,13 @@ module DisqueJockey
   end
 
   def self.configure
-    yield(self.configuration)
+    yield(configuration)
   end
 
-  def self.run!
+  def self.run!(options={})
+    @configuration = DisqueJockey::Configuration.new(options)
+
     DisqueJockey::Supervisor.work!
   end
-end
+end
+

data/spec/disque_jockey/configuration_spec.rb

@@ -9,11 +9,46 @@ describe DisqueJockey::Configuration do
     it { expect(subject.daemonize?).to eq false }
   end
 
-  it "defines a log path method" do
-    expect(DisqueJockey::Configuration.new).to respond_to(:log_path)
-    expect(DisqueJockey::Configuration.new).to respond_to(:log_path=)
-    config = DisqueJockey::Configuration.new
-    config.log_path = 'spec-path'
-    expect(config.log_path).to eq 'spec-path'
+  describe "provides configurable attributes" do
+    it { expect(subject).to respond_to(:log_path) }
+    it { expect(subject).to respond_to(:log_path=) }
+    it { expect(subject).to respond_to(:nodes) }
+    it { expect(subject).to respond_to(:nodes=) }
+    it { expect(subject).to respond_to(:worker_groups) }
+    it { expect(subject).to respond_to(:worker_groups=) }
+    it { expect(subject).to respond_to(:log_path) }
+    it { expect(subject).to respond_to(:log_path=) }
+    it { expect(subject).to respond_to(:env) }
+    it { expect(subject).to respond_to(:env=) }
+  end
+
+  context "when there are no command line options" do
+    it "uses configuration defaults" do
+      config = DisqueJockey::Configuration.new({})
+      
+      expect(config.daemonize).to be true
+      expect(config.env).to eq("test")
+      expect(config.log_path).to eq("spec/log")
+      expect(config.nodes).to eq(["127.0.0.1:7711"])
+      expect(config.worker_groups).to eq(2)
+    end
+  end
+
+  context "when there are command line options" do
+    it "sets them in the configuration" do
+      cli_opts = {
+        "env" => "production",
+        "daemonize" => true,
+        "worker_groups" => "10",
+        "nodes" => "324.545.23.12:5453,98.437.437.23:43534"
+      }
+      config = DisqueJockey::Configuration.new(cli_opts)
+      expect(config.daemonize).to be true
+      expect(config.env).to eq("production")
+      expect(config.log_path).to eq("log")
+      expect(config.nodes).to eq(["324.545.23.12:5453","98.437.437.23:43534"])
+      expect(config.worker_groups).to eq(10)
+    end
   end
 end
+

data/spec/disque_jockey/supervisor_spec.rb

@@ -18,14 +18,19 @@ describe DisqueJockey::Supervisor do
   end
 
   it "::spawn_worker_groups spawns as many worker groups as the config says" do
-    group = double("WorkerGroup", work!: true)
-    expect(DisqueJockey::WorkerGroup).to receive(:new).twice.and_return(group)
+    subject.instance_variable_set(:@worker_classes, ['something'])
     allow(Process).to receive(:fork).and_yield
     allow_any_instance_of(DisqueJockey::Configuration).to receive(:worker_groups).and_return 2
+    group = double("WorkerGroup", work!: true)
+    expect(DisqueJockey::WorkerGroup).to receive(:new).twice.and_return(group)
     expect(group).to receive(:work!).twice
     subject.send(:spawn_worker_groups)
   end
 
+  it "::spawn_worker_groups raises an error if there are no worker classes" do
+    expect{subject.send(:spawn_worker_groups)}.to raise_error(NoWorkersFoundError)
+  end
+
   it "::load_workers loads classes in a workers directory" do
     expect(Object.const_defined?('FixtureWorker')).to eq false
     subject.send(:load_workers)

data/spec/disque_jockey/worker_group_spec.rb

@@ -28,30 +28,6 @@ module DisqueJockey
         end
         subject.new(@worker_classes).work!
       end
-
-
-      it "gives workers jobs to perform" do
-        allow_any_instance_of(Broker).to receive(:fetch_message_from).and_return(['dummy', 'test_id', 'test job'])
-        expect_any_instance_of(SecondSpecWorker).to receive(:handle).at_least(:once)
-        subject.new(@worker_classes).work!
-      end
-
-      describe "handling logic around jobs" do
-
-        it "times out workers that take too long" do
-          allow_any_instance_of(Broker).to receive(:fetch_message_from).and_return(['dummy', 'test_id', 'test job'])
-          allow_any_instance_of(subject).to receive(:work_until_signal) { sleep(0.2) }
-          expect_any_instance_of(SlowWorker).to receive(:log_exception).at_least(:once)
-          subject.new([SlowWorker]).work!
-        end
-
-        xit "acknowledges jobs if they are processed without errors" do
-          allow_any_instance_of(subject).to receive(:work_until_signal) { sleep(0.1) }
-          allow_any_instance_of(Broker).to receive(:fetch_message_from).and_return(['dummy', 'test_id', 'test job'])
-          expect_any_instance_of(Broker).to receive(:acknowledge).with('test_id').at_least(:once)
-          subject.new([SecondSpecWorker]).work!
-        end
-      end
     end
 
   end

data/spec/disque_jockey/worker_pool_spec.rb

@@ -0,0 +1,44 @@
+require 'spec_helper'
+require 'disque_jockey/worker_shared_setup'
+
+module DisqueJockey
+  describe WorkerPool do
+    include_context "worker setup"
+
+    before(:each) do
+      allow_any_instance_of(Broker).to receive(:fetch_message_from).and_return(['dummy', 'test_id', 'test job'])
+      allow_any_instance_of(Broker).to receive(:acknowledge)
+      allow_any_instance_of(WorkerPool).to receive(:endless_loop).and_yield
+      allow(Thread).to receive(:new).and_yield
+    end
+
+    it "instantiates the number of workers specified" do
+      expect(SpecWorker).to receive(:new).exactly(SpecWorker.thread_count).times
+      described_class.new(SpecWorker)
+    end
+
+    it "instantiates brokers with nodes from configuration" do
+      expect(Broker).to receive(:new).with(DisqueJockey.configuration.nodes)
+      described_class.new(SpecWorker)
+    end
+
+    it "gives workers jobs to perform" do
+      @mock_worker = double("Worker", handle: true)
+      @mock_worker_class = double("WorkerClass", thread_count: 1, new: @mock_worker, timeout_seconds: 1, queue_name: 'q')
+      worker_pool = WorkerPool.new(@mock_worker_class)
+      expect(@mock_worker).to receive(:handle)
+      worker_pool.work!
+    end
+
+    it "times out workers that take too long" do
+      expect_any_instance_of(SlowWorker).to receive(:log_exception).at_least(:once)
+      WorkerPool.new(SlowWorker).work!
+    end
+
+    it "acknowledges jobs if they are processed without errors" do
+      expect_any_instance_of(Broker).to receive(:acknowledge).with('test_id').at_least(:once)
+      WorkerPool.new(SecondSpecWorker).work!
+    end
+
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: disque_jockey
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Devin Riley
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-14 00:00:00.000000000 Z
+date: 2015-06-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: disque
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -103,18 +117,23 @@ files:
 - disque_jockey.gemspec
 - lib/disque_jockey.rb
 - lib/disque_jockey/broker.rb
+- lib/disque_jockey/cli.rb
+- lib/disque_jockey/cli/help.rb
 - lib/disque_jockey/configuration.rb
+- lib/disque_jockey/exceptions.rb
 - lib/disque_jockey/logger.rb
 - lib/disque_jockey/supervisor.rb
 - lib/disque_jockey/version.rb
 - lib/disque_jockey/worker.rb
 - lib/disque_jockey/worker_group.rb
+- lib/disque_jockey/worker_pool.rb
 - spec/disque_jockey/broker_spec.rb
 - spec/disque_jockey/configuration_spec.rb
 - spec/disque_jockey/disque_jockey_spec.rb
 - spec/disque_jockey/logger_spec.rb
 - spec/disque_jockey/supervisor_spec.rb
 - spec/disque_jockey/worker_group_spec.rb
+- spec/disque_jockey/worker_pool_spec.rb
 - spec/disque_jockey/worker_shared_setup.rb
 - spec/disque_jockey/worker_spec.rb
 - spec/fixtures/workers/fixture_worker.rb
@@ -151,6 +170,7 @@ test_files:
 - spec/disque_jockey/logger_spec.rb
 - spec/disque_jockey/supervisor_spec.rb
 - spec/disque_jockey/worker_group_spec.rb
+- spec/disque_jockey/worker_pool_spec.rb
 - spec/disque_jockey/worker_shared_setup.rb
 - spec/disque_jockey/worker_spec.rb
 - spec/fixtures/workers/fixture_worker.rb