disque_jockey 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c49083bcd86ac5171c257bd78eeded4abbd5c99f
-  data.tar.gz: 868ababcc280ffa34de1b8270c07b00c04bc299c
+  metadata.gz: e595757036b4f0fa828fcb9dda551db3f211ed7e
+  data.tar.gz: 81687ba961860e23ffd2b877b81166b8c8f3ea31
 SHA512:
-  metadata.gz: 8fa502414a8a708a693cff041773bb2ae9201d7d0be70dd5fd5eafe2bedd3b6dc31852f7e9f1f516f2dffe52e7998e70923f49eb2a07817d3b768bc683dd632e
-  data.tar.gz: 8dec61c2d6e36a1347147114f00e6290aa2cd2c2d4ae83b76124fdc4f68581e3727208c1f5554b2aa376fd76c212aa2e8e87ed0681bd0e2d1dc00a6f5058f43c
+  metadata.gz: 9bbd8ab91aba802d6f1c7364fe833eef0f20ee0813593cdad4fe3bb7c63c1508cf34db86f73530461283d5946c3520c472a5cfe48e0a1474c775c7b618972bd7
+  data.tar.gz: 4f294a19102ce434dd3c55d97425cde430680903e1241a2ef44d02c53dae2a7eecd9f8a796570d9cbba161ba0974889c2c732349ac8e5e807926f1c5d331c345

data/README.md

@@ -63,6 +63,26 @@ disque_jockey start --env=production --daemonize=true --worker-groups=10  --node
 ````
 
 Messages successfully handled by a worker (ie no exceptions raised from the handle method) will be acknowledged and removed from the queue.
+## Worker Configuration
+ DisqueJockey::Worker implements some class methods that help you configure your worker.  You call them the same way you call the `subscribe_to` method, at the top of your class.
+
+```ruby
+require 'disque_jockey'
+class HighlyConfiguredWorker < DisqueJockey::Worker
+  subscribe_to 'example-queue'
+  threads 7
+  fast_ack true
+  timeout 5
+  
+  def handle(job)
+    logger.info("Peforming job: #{job}")
+  end
+end
+```
+- *Fast Acknowledgements*: call ````fast_ack true```` to use FASTACKs (https://github.com/antirez/disque#fast-acknowledges) in disque to acknowledge your messages.  Please note that fast_ack will make it more likely you will process a job more than once in the event of a network partition.  fast_ack is false by default.
+- *Threads*: To devote more threads to your worker class use ````threads 5```` .  Threads are set to two by default and have a maximum value of 10.
+- *Timeout*: To set the number of seconds your worker will process a job until raising a TimeoutError, use ````timeout 45````.  Timeout is set to 30 seconds by default and has a maximum value of 3600 seconds (one hour).
+
 
 ##Roadmap:
 DisqueJockey is not a currently a production-ready system, and there are a number of goals for it that have not been met yet.

data/lib/disque_jockey/broker.rb

@@ -13,10 +13,26 @@ module DisqueJockey
 
     def acknowledge(job_id)
       response = @client.call('ACKJOB', job_id)
-      # If there is an error acking the job the Disque client 
-      # *returns* an error object but doesn't raise it, 
-      # so we raise it here ourselves.
+      raise_error_or_return_true(response)
+    end
+
+    def fast_acknowledge(job_id)
+      response = @client.call('FASTACK', job_id)
+      raise_error_or_return_true(response)
+    end
+
+    def publish(*args)
+      @client.push(*args)
+    end
+
+    private
+
+    # If there is an error acking the job the Disque client
+    # *returns* an error object but doesn't raise it,
+    # so we raise it here ourselves.
+    def raise_error_or_return_true(response)
       response.is_a?(RuntimeError) ? raise(response) : true
     end
+
   end
-end
+end

data/lib/disque_jockey/version.rb

@@ -1,3 +1,3 @@
 module DisqueJockey
-  VERSION = '0.0.2'
+  VERSION = '0.0.3'
 end

data/lib/disque_jockey/worker.rb

@@ -11,13 +11,18 @@ module DisqueJockey
     end
 
     class << self
-      attr_reader :queue_name, :thread_count, :timeout_seconds
+      attr_reader :queue_name, :thread_count, :timeout_seconds, :use_fast_ack
 
       # This worker class will subscribe to queue
       def subscribe_to(queue)
         @queue_name = queue
       end
 
+      # whehter to use Disque fast acknowledgements
+      def fast_ack(value)
+        @use_fast_ack = !!value
+      end
+
       # minimum number of worker instances of a given worker class.
       def threads(size)
         @thread_count = [[size, 1].max, 10].min
@@ -36,10 +41,11 @@ module DisqueJockey
         # these are the defaults
         type.threads 2
         type.timeout 30
+        type.fast_ack false
         # register the new worker type so we can start giving it jobs
         Supervisor.register_worker(type)
       end
     end
 
   end
-end
+end

data/lib/disque_jockey/worker_pool.rb

@@ -37,7 +37,7 @@ module DisqueJockey
     def handle_job(worker, job, job_id)
       begin
         Timeout::timeout(@worker_class.timeout_seconds) { worker.handle(job) }
-        @broker.acknowledge(job_id)
+        @worker_class.use_fast_ack ? @broker.fast_acknowledge(job_id) : @broker.acknowledge(job_id)
       rescue StandardError => exception
         worker.log_exception(exception)
         # TODO: Need to implement retry logic
@@ -47,8 +47,8 @@ module DisqueJockey
     end
 
     def build_worker_pool
-      @worker_class.thread_count.times { @pool << @worker_class.new(Logger) }      
+      @worker_class.thread_count.times { @pool << @worker_class.new(Logger) }
     end
 
   end
-end
+end

data/spec/disque_jockey/broker_spec.rb

@@ -1,7 +1,7 @@
 require 'spec_helper'
 module DisqueJockey
   describe Broker do
-    # Note: You actually have to run a Disque server 
+    # Note: You actually have to run a Disque server
     # locally for these tests to pass
     before(:all) do
       begin
@@ -34,7 +34,7 @@ module DisqueJockey
     end
 
     describe '#acknowledge' do
-      it "returns removes job from queue and returns true if it succeeds" do
+      it "removes job from queue and returns true if it succeeds" do
         @client.call('DEBUG', 'FLUSHALL')
         job_id = @client.push('test_queue', 'test job', 1000)
         expect(@client.call('QLEN', 'test_queue')).to eq 1
@@ -43,9 +43,34 @@ module DisqueJockey
       end
 
       it "raises an error for a bad job id" do
-        expect{@broker.acknowledge('bad_id')}.to raise_error(RuntimeError)
+        expect{ @broker.acknowledge('bad_id') }.to raise_error(RuntimeError)
+      end
+    end
+
+    describe "#publish" do
+      it "publishes a job to Disque" do
+        test_queue = 'publish_test_queue'
+        test_job = 'job'
+        @broker.publish(test_queue, test_job, 1000)
+        fetched = @client.fetch(from: ['publish_test_queue']).first
+        expect(fetched.first).to eq test_queue
+        expect(fetched.last).to eq test_job
+      end
+    end
+
+    describe "#fast_acknowledge" do
+      it "raises an error for a bad job id" do
+        expect{@broker.fast_acknowledge('bad_id')}.to raise_error(RuntimeError)
+      end
+
+      it "acknowledges jobs" do
+        @client.call('DEBUG', 'FLUSHALL')
+        job_id = @client.push('test_queue', 'test job', 1000)
+        expect(@client.call('QLEN', 'test_queue')).to eq 1
+        expect(@broker.fast_acknowledge(job_id)).to eq true
+        expect(@client.call('QLEN', 'test_queue')).to eq 0
       end
     end
 
   end
-end
+end

data/spec/disque_jockey/worker_pool_spec.rb

@@ -24,7 +24,13 @@ module DisqueJockey
 
     it "gives workers jobs to perform" do
       @mock_worker = double("Worker", handle: true)
-      @mock_worker_class = double("WorkerClass", thread_count: 1, new: @mock_worker, timeout_seconds: 1, queue_name: 'q')
+      @mock_worker_class = double(
+        "WorkerClass",
+        thread_count: 1,
+        new: @mock_worker,
+        timeout_seconds: 1,
+        queue_name: 'q',
+        use_fast_ack: false)
       worker_pool = WorkerPool.new(@mock_worker_class)
       expect(@mock_worker).to receive(:handle)
       worker_pool.work!
@@ -36,9 +42,16 @@ module DisqueJockey
     end
 
     it "acknowledges jobs if they are processed without errors" do
+      SecondSpecWorker.fast_ack(false)
       expect_any_instance_of(Broker).to receive(:acknowledge).with('test_id').at_least(:once)
       WorkerPool.new(SecondSpecWorker).work!
     end
 
+    it "fast_acknowledges jobs if the worker uses fast_ack" do
+      SecondSpecWorker.fast_ack(true)
+      expect_any_instance_of(Broker).to receive(:fast_acknowledge).with('test_id').at_least(:once)
+      WorkerPool.new(SecondSpecWorker).work!
+    end
+
   end
-end
+end

data/spec/disque_jockey/worker_shared_setup.rb

@@ -6,9 +6,10 @@ shared_context "worker setup" do
       subscribe_to "test"
       def handle(job); end
     end
-    
+
     class SecondSpecWorker < DisqueJockey::Worker
       subscribe_to "other-test"
+      fast_ack true
       threads 1
       timeout 1
       def handle(job); end

data/spec/disque_jockey/worker_spec.rb

@@ -6,8 +6,8 @@ describe DisqueJockey::Worker do
   include_context "worker setup"
 
   it "defines class methods" do
-    [ :queue_name, :thread_count, :timeout_seconds, 
-      :subscribe_to, :timeout, :threads
+    [ :queue_name, :thread_count, :timeout_seconds,
+      :subscribe_to, :timeout, :threads, :fast_ack, :use_fast_ack
     ].each do |method|
       expect(DisqueJockey::Worker).to respond_to(method)
     end
@@ -24,26 +24,34 @@ describe DisqueJockey::Worker do
   end
 
   context "defaults" do
-    it "sets a default for timeout" do
+    it "timeout is 30 seconds" do
       expect(SpecWorker.timeout_seconds).to be(30)
     end
 
-    it "sets a default for thread_count" do
+    it "thread_count is 2" do
       expect(SpecWorker.thread_count).to be(2)
     end
+
+    it "use_fast_ack is false" do
+      expect(SpecWorker.use_fast_ack).to eq false
+    end
+
   end
 
   context "class methods"
   context "overrides" do
-    it "allows overrides for timeout, and thread_count" do
+
+    it "allows overrides for timeout, threads, and fast_ack" do
+      # These traits are set in worker_shared_setup.rb
       expect(SecondSpecWorker.timeout_seconds).to be(1)
       expect(SecondSpecWorker.thread_count).to be(1)
+      expect(SecondSpecWorker.use_fast_ack).to be(true)
     end
   end
-  
+
   context "instance methods"
   describe "#initialize" do
-    before do 
+    before do
       @logger = double(:logger, info: true, error: true, warn: true)
       allow(@logger).to receive(:new).and_return(@logger)
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: disque_jockey
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Devin Riley