job_dispatch 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b05fed659c186b001b926e08650b3a281160668f
-  data.tar.gz: 6ed34896dfbe1a5dcc1382a36b39e7c3e73f2f2f
+  metadata.gz: e00ac3ea419f26f9e3d8ae4a52a73c267f306ce7
+  data.tar.gz: 2f2941010e8775f2e6ec0759952438fd899f17a5
 SHA512:
-  metadata.gz: 4a211880bafb99581f6a4d8bd884ddcc37d68170f5ba28fde9943d0cdf2eb9735a6cd3c78f683ef4c923f02bc4c19d5479461904d8b25d060088c088048e2747
-  data.tar.gz: 30656712d978383424f23945daa8ae77e3f61dd6c4314d324b8063203e6ecf604347588074b9db0f9ea39fed4b0289869d55d20dc5e3dfaed901caea7d5ad674
+  metadata.gz: 03c7b5606808c97b37c0c3903ec9058db29f8f96741a5b4b42a2220e459017249b9b48cb6b701c8fbd75392e0ba212bee8dc38ff29257512fd119e39b3739671
+  data.tar.gz: 414171d3052dcfdc29f5c8c30f3216aeb755d4bdf96c205452016241c1ac493bd859c374d29c24d1287709092bc16e0d04cda87b020a13b9f4228bf9f8bf13a7

data/CHANGELOG.md

@@ -0,0 +1,12 @@
+# job_dispatch change log
+
+## Version 0.0.2
+
+* Broker sends an idle command to a worker immediately upon connect. This helps recover from a case where a worker
+  has been running for some time before the dispatcher starts. (Particularly with a Windows worker using NetMQ where
+  a closed socket may still send messages.)
+* Improve Ruby worker serialisation of exceptions into job result.
+
+## Version 0.0.1
+
+* First release

data/lib/job_dispatch/broker.rb

@@ -14,7 +14,7 @@ module JobDispatch
     POLL_TIME = 5.123
     STOP_SIGNALS = %w[INT TERM KILL]
 
-    IdleWorker = Struct.new :worker_id, :idle_since, :queue, :worker_name
+    IdleWorker = Struct.new :worker_id, :idle_since, :queue, :worker_name, :idle_count
 
 
     # any object that will respond to `next_job_for_queue`, which should return a job, or nil if there
@@ -67,6 +67,7 @@ module JobDispatch
         puts "JobDispatch::Broker running in process #{Process.pid}"
         JobDispatch.logger.info("JobDispatch::Broker running in process #{Process.pid}")
         @running = true
+        @running_thread = Thread.current
         poller = ZMQ::Poller.new
 
         @socket = JobDispatch::Broker::Socket.new(@worker_bind_address)
@@ -99,10 +100,12 @@ module JobDispatch
               puts "JobDispatch::Broker shutting down, due to #{signal_name} signal"
               @running = false
               @status = "SHUTDOWN"
-              sleep 1
+              # sleep 1
               process_quit
-              sleep 1
+              sleep 1 # let ZMQ send the messages before we close the socket.
             end
+          rescue StandardError => e
+            JobDispatch.logger.error "Unexpected exception: #{e}"
           end
         end
       ensure
@@ -112,6 +115,14 @@ module JobDispatch
     end
 
 
+    def stop
+      if running?
+        @running = false
+        @running_thread.raise SignalException.new("TERM") unless Thread.current == @running_thread
+      end
+    end
+
+
     def process_messages(poller)
       # TODO: calculate the amount of time to sleep to wake up such that a scheduled event happens as close
       # as possible to the time it was supposed to happen. This could additionally mean that the POLL_TIME
@@ -158,7 +169,7 @@ module JobDispatch
         case command.command
           when "ready"
             # add to list of workers who are ready for work
-            add_available_worker(command)
+            add_available_worker(command, 0)
 
             # don't reply, leaves worker blocked waiting for a job to do.
             reply = nil
@@ -172,7 +183,7 @@ module JobDispatch
 
             if command.worker_ready?
               # a completed job also means the worker is available for more work.
-              add_available_worker(command)
+              add_available_worker(command, 1)
               reply = nil
             else
               reply.parameters = {:status => 'thanks'}
@@ -235,9 +246,10 @@ module JobDispatch
     def send_idle_commands(idle_time=nil)
       idle_time ||= Time.now
       idle_time -= WORKER_IDLE_TIME
-      idle_workers = @workers_waiting_for_jobs.select { |worker_id, worker| worker.idle_since < idle_time }
+      idle_workers = @workers_waiting_for_jobs.select { |worker_id, worker| worker.idle_since < idle_time || worker.idle_count == 0 }
       idle_workers.each do |worker_id, worker|
         send_job_to_worker(InternalJob.new('idle', worker.queue), worker_id)
+        worker.idle_count += 1
       end
     end
 
@@ -268,10 +280,18 @@ module JobDispatch
 
 
     # add a worker to the list of workers available for jobs.
-    def add_available_worker(command)
+    def add_available_worker(command, idle_count=0)
       JobDispatch.logger.info("JobDispatch::Broker Worker '#{command.worker_id.to_json}' available for work on queue '#{command.queue}'")
+
+      # immediately remove any existing workers with the given name. If a worker has closed its connection and opened
+      # a new one (perhaps it started a long time before the broker did)
+
+      if command.worker_name # this is only sent on initial requests.
+        remove_worker_named(command.worker_name)
+      end
+
       queue = command.queue
-      idle_worker = IdleWorker.new(command.worker_id, Time.now, queue, command.worker_name)
+      idle_worker = IdleWorker.new(command.worker_id, Time.now, queue, command.worker_name, idle_count)
       workers_waiting_for_jobs[command.worker_id] = idle_worker
       queues[queue] << command.worker_id
       if command.worker_name # this is only sent on initial requests.
@@ -282,20 +302,23 @@ module JobDispatch
     # remove a worker from available list. Worker is shutting down or indicating that it will no longer
     # be available for doing work.
     def remove_available_worker(command)
-      JobDispatch.logger.info("JobDispatch::Broker Worker '#{command.worker_id.to_json}' available for work on queue '#{command.queue}'")
+      JobDispatch.logger.info("JobDispatch::Broker Removing Worker '#{command.worker_id.to_json}' available for work on queue '#{command.queue}'")
 
       # the goodbye command is sent by another socket connection, so the worker_id (socket identity) will
-      # not match the socket actually waiting for work.
+      # not match the socket actually waiting for work. Remove the worker by its name, not socket identity
+
+      remove_worker_named(command.worker_name)
+      {status: "see ya later"}
+    end
 
-      keys = worker_names.select { |id, name| name == command.worker_name }.keys
+    def remove_worker_named(worker_name)
+      keys = worker_names.select { |id, name| name == worker_name }.keys
       keys.each do |worker_id|
         workers_waiting_for_reply.delete(worker_id) # socket will be closing, no need to send it anything.
         worker = workers_waiting_for_jobs.delete(worker_id)
         queues[worker.queue].delete(worker_id) if worker
         worker_names.delete(worker_id)
       end
-
-      {status: "see ya later"}
     end
 
     def dispatch_jobs_to_workers

data/lib/job_dispatch/identity.rb

@@ -1,5 +1,5 @@
 require 'active_support/core_ext'
-require 'active_support/core_ext/object/json'
+require 'active_support/core_ext/object/to_json'
 
 module JobDispatch
   # Identity encapsulates a ZeroMQ socket identity, which is a string of binary characters, typically

data/lib/job_dispatch/version.rb

@@ -1,3 +1,3 @@
 module JobDispatch
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/lib/job_dispatch/worker.rb

@@ -8,6 +8,9 @@ module JobDispatch
   #
   class Worker
 
+    class StopError < StandardError
+    end
+
     IDLE_TIME = 3
     IDLE_COUNT = 10
 
@@ -38,11 +41,12 @@ module JobDispatch
 
     def run
       @running = true
+      @running_thread = Thread.current
       while running?
-        puts "connecting"
+        # puts "connecting"
         connect
-        puts "asking for work"
-        ask_for_work
+        # puts "asking for work"
+        ask_for_work rescue StopError
 
         # if we are idle for too many times, the broker has restarted or gone away, and we will be stuck in receive
         # state, so we need to close the socket and make a new one to ask for work again.
@@ -60,13 +64,14 @@ module JobDispatch
               idle
               idle_count += 1
             end
-          rescue Interrupt
-            puts "Worker stopping."
-            JobDispatch.logger.info("Worker #{}")
+          rescue Interrupt, StopError
+            JobDispatch.logger.info("Worker stopping.")
             stop
             disconnect
+            # Tell the broker goodbye so that we are removed from the idle worker list and no more jobs will come here.
             connect
             send_goodbye
+            sleep(0.1) # let the socket send the message before we disconnect...
           end
         end
         disconnect
@@ -86,7 +91,10 @@ module JobDispatch
     end
 
     def stop
-      @running = false
+      if running?
+        @running_thread.raise StopError unless @running_thread == Thread.current
+        @running = false
+      end
     end
 
     def self.touch(timeout=nil)
@@ -111,7 +119,7 @@ module JobDispatch
     end
 
     def idle
-      puts "waiting for job to do…"
+      # puts "waiting for job to do…"
     end
   end
 end

data/lib/job_dispatch/worker/item.rb

@@ -30,7 +30,11 @@ module JobDispatch
           @result = @klass.__send__(method.to_sym, *params)
           @status = :success
         rescue StandardError => ex
-          @result = ex
+          @result = {
+              class: ex.class.to_s,
+              message: ex.to_s,
+              backtrace: ex.backtrace,
+          }
           @status = :error
         ensure
           Thread.current["JobDispatch::Worker.job_id"] = nil

data/spec/job_dispatch/broker_spec.rb

@@ -81,7 +81,6 @@ describe JobDispatch::Broker do
         subject.workers_waiting_for_reply << worker_id3
         subject.process_command(Command.new(worker_id3, {command: 'ready', worker_name: 'test worker 2'}))
 
-
         @job = FactoryGirl.build :job
         @socket = double('Broker::Socket', :send_command => nil)
         subject.stub(:socket => @socket)
@@ -166,6 +165,14 @@ describe JobDispatch::Broker do
         @result = subject.process_command(command)
         expect(subject.worker_names[worker_id]).to eq('ruby worker')
       end
+
+      let(:command2) { Command.new(worker_id2, {command: 'ready', queue: 'example', worker_name: 'ruby worker'}) }
+
+      it "duplicate ready has only 1 worker" do
+        @result = subject.process_command(command)
+        @result = subject.process_command(command2)
+        expect(subject.queues[:example].count).to eq(1)
+      end
     end
 
 
@@ -405,34 +412,72 @@ describe JobDispatch::Broker do
         command = Command.new(worker_id2, {command: 'ready', queue: 'example'})
         @result = subject.process_command(command)
       end
-
     end
 
-    it "that have waited long enough receive idle commands" do
-      @socket.should_receive(:send_command) do |cmd|
-        expect(cmd.worker_id).to eq(worker_id)
-        expect(cmd.parameters[:command]).to eq('idle')
+    context "already done an idle" do
+      before do
+        JobDispatch::Broker::IdleWorker.any_instance.stub(:idle_count => 1)
       end
 
-      Timecop.freeze(@time + JobDispatch::Broker::WORKER_IDLE_TIME + 1) do
-        subject.send_idle_commands
+      it "that have waited long enough receive idle commands" do
+        @socket.should_receive(:send_command) do |cmd|
+          expect(cmd.worker_id).to eq(worker_id)
+          expect(cmd.parameters[:command]).to eq('idle')
+        end
+
+        Timecop.freeze(@time + JobDispatch::Broker::WORKER_IDLE_TIME + 1) do
+          subject.send_idle_commands
+        end
+
+        expect(subject.workers_waiting_for_reply).not_to include(worker_id)
+        expect(subject.queues[:example]).not_to include(worker_id)
       end
 
-      expect(subject.workers_waiting_for_reply).not_to include(worker_id)
-      expect(subject.queues[:example]).not_to include(worker_id)
+      it "that have not waited long enough are still waiting" do
+        @socket.should_receive(:send_command) do |cmd|
+          expect(cmd.worker_id).not_to eq(worker_id2)
+        end
+
+        Timecop.freeze(@time + JobDispatch::Broker::WORKER_IDLE_TIME + 1) do
+          subject.send_idle_commands
+        end
+
+        expect(subject.workers_waiting_for_reply).to include(worker_id2)
+        expect(subject.queues[:example]).to include(worker_id2)
+      end
     end
 
-    it "that have not waited long enough are still waiting" do
-      @socket.should_receive(:send_command) do |cmd|
-        expect(cmd.worker_id).not_to eq(worker_id2)
+    context "have not done an idle" do
+      before do
+        JobDispatch::Broker::IdleWorker.any_instance.stub(:idle_count => 0)
       end
 
-      Timecop.freeze(@time + JobDispatch::Broker::WORKER_IDLE_TIME + 1) do
-        subject.send_idle_commands
+      it "that have waited long enough receive idle commands" do
+        @socket.should_receive(:send_command) do |cmd|
+          expect(cmd.worker_id).to eq(worker_id)
+          expect(cmd.parameters[:command]).to eq('idle')
+        end
+
+        Timecop.freeze(@time + JobDispatch::Broker::WORKER_IDLE_TIME + 1) do
+          subject.send_idle_commands
+        end
+
+        expect(subject.workers_waiting_for_reply).not_to include(worker_id)
+        expect(subject.queues[:example]).not_to include(worker_id)
       end
 
-      expect(subject.workers_waiting_for_reply).to include(worker_id2)
-      expect(subject.queues[:example]).to include(worker_id2)
+      it "that have not waited long enough are still waiting" do
+        @socket.should_receive(:send_command) do |cmd|
+          expect(cmd.worker_id).not_to eq(worker_id2)
+        end
+
+        Timecop.freeze(@time + JobDispatch::Broker::WORKER_IDLE_TIME + 1) do
+          subject.send_idle_commands
+        end
+
+        expect(subject.workers_waiting_for_reply).not_to include(worker_id2)
+        expect(subject.queues[:example]).not_to include(worker_id2)
+      end
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: job_dispatch
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Matt Connolly
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-20 00:00:00.000000000 Z
+date: 2014-05-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rbczmq
@@ -177,6 +177,7 @@ files:
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
+- CHANGELOG.md
 - Gemfile
 - Guardfile
 - LICENSE.txt