job_dispatch 0.0.2 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e00ac3ea419f26f9e3d8ae4a52a73c267f306ce7
-  data.tar.gz: 2f2941010e8775f2e6ec0759952438fd899f17a5
+  metadata.gz: 401fd02367fe4acc36558989fe00883365afcd87
+  data.tar.gz: f45f4e6595a88462a394deb27a103d9a9d274ae5
 SHA512:
-  metadata.gz: 03c7b5606808c97b37c0c3903ec9058db29f8f96741a5b4b42a2220e459017249b9b48cb6b701c8fbd75392e0ba212bee8dc38ff29257512fd119e39b3739671
-  data.tar.gz: 414171d3052dcfdc29f5c8c30f3216aeb755d4bdf96c205452016241c1ac493bd859c374d29c24d1287709092bc16e0d04cda87b020a13b9f4228bf9f8bf13a7
+  metadata.gz: d07c1dc6f62db463ec888a8e3130893bfb37541854756477200ecc77d79ae8e0e615d4773925b18f71798fccb59d3bb308c1e8ec7e1696958b9b694dd5e7c161
+  data.tar.gz: f3a9e4646d3f4e0e1aea6b5b7691758f3b29057215bbc104875239375a6efb0a99f1ea035d86e7e6ed6dc913d87dffcf8731b42c8e8874a52400aadcdccb1348

data/.gitignore

@@ -16,4 +16,6 @@ test/tmp
 test/version_tmp
 tmp
 .idea
+*.orig
+*.swp
 

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 # job_dispatch change log
 
+## Version 0.1.0
+
+* Added 'num_tcp_connections' to status command response
+* Fixed leaking sockets from job-worker command.
+
 ## Version 0.0.2
 
 * Broker sends an idle command to a worker immediately upon connect. This helps recover from a case where a worker

data/bin/job-worker

@@ -28,8 +28,6 @@ JobDispatch.load_config_from_yml('config/job_dispatch.yml', ENV["RAILS_ENV"])
 
 require File.expand_path('config/environment.rb', ROOT_DIR)
 
-JobDispatch.logger = Rails.logger
-
 endpoint = JobDispatch.config.broker[:connect]
 if endpoint.nil? || endpoint.empty?
   $stderr.puts "No Job Dispatch broker connect address has been specified."

data/lib/job_dispatch/broker.rb

@@ -11,7 +11,7 @@ module JobDispatch
   class Broker
 
     WORKER_IDLE_TIME = 10.123
-    POLL_TIME = 5.123
+    POLL_TIME = 31
     STOP_SIGNALS = %w[INT TERM KILL]
 
     IdleWorker = Struct.new :worker_id, :idle_since, :queue, :worker_name, :idle_count
@@ -32,6 +32,7 @@ module JobDispatch
     attr :job_subscribers # Key: job_id, value: list of Socket Identities waiting for job completion notifications.
     attr :pub_socket
     attr_accessor :reply_exceptions
+    attr :queues_ready
 
     def initialize(worker_bind_address, wakeup_bind_address, publish_bind_address=nil)
       @worker_bind_address = worker_bind_address
@@ -48,6 +49,7 @@ module JobDispatch
       @jobs_in_progress_workers = {} #key: job_id, value: worker_id
       @worker_names = {} # Key: Symbol socket identity, value: String claimed name of worker
       @job_subscribers = {} # Key: job_id, value: list of Socket Identities waiting for job completion notifications.
+      @queues_ready = {} # Key: Symbol queue name, value: bool ready?
       @status = "OK"
       @reply_exceptions = true
 
@@ -127,7 +129,7 @@ module JobDispatch
       # TODO: calculate the amount of time to sleep to wake up such that a scheduled event happens as close
       # as possible to the time it was supposed to happen. This could additionally mean that the POLL_TIME
       # could be arbitrarily large. As any communication with the broker will wake it immediately.
-      poll_time = POLL_TIME
+      poll_time = JobDispatch.config.broker_options.try(:[], :poll_time) || POLL_TIME
       poller.poll(poll_time)
 
       if @wake_socket && poller.readables.include?(@wake_socket)
@@ -136,6 +138,7 @@ module JobDispatch
 
       if poller.readables.include?(socket.socket)
         command = read_command
+        JobDispatch.logger.debug("JobDispatch::Broker received command: #{command.command}(#{command.parameters.inspect})")
         reply = process_command(command)
         send_command(reply) if reply
       end
@@ -217,6 +220,12 @@ module JobDispatch
           when "enqueue"
             reply.parameters = create_job(command)
 
+          when "last"
+            reply.parameters = last_job(command)
+
+          when "fetch"
+            reply.parameters = fetch_job(command)
+
           when "quit"
             process_quit
             reply.parameters = {:status => 'bye'}
@@ -233,6 +242,7 @@ module JobDispatch
         if reply_exceptions
           # all others reply over socket.
           JobDispatch.logger.error("JobDispatch::Broker #{e}")
+          JobDispatch.logger.error e.backtrace.join("\n")
           reply.parameters = {:status => 'error', :message => e.to_s}
         else
           # used during testing to raise errors so that Rspec can catch them as a test failure.
@@ -294,6 +304,7 @@ module JobDispatch
       idle_worker = IdleWorker.new(command.worker_id, Time.now, queue, command.worker_name, idle_count)
       workers_waiting_for_jobs[command.worker_id] = idle_worker
       queues[queue] << command.worker_id
+      queues_ready[queue] = true
       if command.worker_name # this is only sent on initial requests.
         worker_names[command.worker_id] = command.worker_name
       end
@@ -323,9 +334,9 @@ module JobDispatch
 
     def dispatch_jobs_to_workers
       # dequeue jobs from database for each queue
-      @queues.each_pair do |queue, worker_ids|
+      queues.each_pair do |queue, worker_ids|
         # we only need to check the database if there are available workers in that queue
-        if worker_ids.count > 0
+        if worker_ids.count > 0 && queues_ready[queue]
           worker_id = worker_ids.first
 
           job = begin
@@ -344,8 +355,10 @@ module JobDispatch
             job.expire_execution_at = Time.now + (job.timeout || Job::DEFAULT_EXECUTION_TIMEOUT)
             job.status = JobDispatch::Job::IN_PROGRESS
             job.save
-
             publish_job_status(job)
+          else
+            # no job. mark the queue as not ready so we don't repeatedly check for jobs in an empty queue.
+            queues_ready[queue] = false
           end
         end
       end
@@ -425,19 +438,24 @@ module JobDispatch
 
 
     def json_for_job(job)
-      hash = if job.respond_to? :as_job_queue_item
-               job.as_job_queue_item
-             else
-               job.as_json
-             end.with_indifferent_access
-      hash[:id] = hash[:id].to_s
-      hash
+      if job
+        hash = if job.respond_to? :as_job_queue_item
+                 job.as_job_queue_item
+               else
+                 job.as_json
+               end.with_indifferent_access
+        hash[:id] = hash[:id].to_s
+        hash
+      end
     end
 
 
     def status_response
+      num_tcp_connections = `lsof -p #{Process.pid}`.split.select { |l| l=~ /TCP/ }.count
+
       response = {
           :status => status,
+          :num_tcp_connections => num_tcp_connections,
           :queues => {}
       }
 
@@ -477,14 +495,15 @@ module JobDispatch
     # @return [Hash] result to be sent to client.
     def touch_job(command)
       job_id = command.parameters[:job_id]
-      timeout = command.parameters[:timeout] || Job::DEFAULT_EXECUTION_TIMEOUT
       job = @jobs_in_progress[job_id]
       if job
+        timeout = command.parameters[:timeout] || job.timeout || Job::DEFAULT_EXECUTION_TIMEOUT
         job.expire_execution_at = Time.now + timeout
         JobDispatch.logger.info("JobDispatch::Broker#touch timeout on job #{job_id} to #{job.expire_execution_at}")
         job.save
         {status: "success"}
       else
+        JobDispatch.logger.info("JobDispatch::Broker#touch job #{job_id} not in progress.")
         {status: "error", message: "the specified job does not appear to be in progress"}
       end
     end
@@ -494,10 +513,42 @@ module JobDispatch
         raise MissingParameterError, "Missing 'job' from command" unless command.parameters[:job].present?
 
         job_attrs = command.parameters[:job]
+        job_attrs[:queue] ||= :default
         job = job_source.create!(job_attrs)
+        queues_ready[job_attrs[:queue].to_sym] = true
         {status: 'success', job_id: job.id.to_s}
       rescue StandardError => e
         JobDispatch.logger.error "JobDispatch::Broker#create_job error: #{e}"
+        JobDispatch.logger.error e.backtrace.join("\n")
+        {status: 'error', message: e.to_s}
+      end
+    end
+
+    def last_job(command)
+      begin
+        queue = command.parameters[:queue] || 'default'
+        job = job_source.where(:queue => queue).last
+        if job
+          {status: 'success', job: json_for_job(job)}
+        else
+          {status: 'error', message: 'no last job'}
+        end
+      rescue StandardError => e
+        JobDispatch.logger.error e.to_s
+        JobDispatch.logger.error e.backtrace.join("\n")
+        {status: 'error', message: e.to_s}
+      end
+    end
+
+    def fetch_job(command)
+      begin
+        raise "Missing parameter 'job_id'" unless command.parameters[:job_id]
+        job = job_source.find(command.parameters[:job_id])
+        raise "Job not found" unless job
+        {status: 'success', job: json_for_job(job)}
+      rescue StandardError => e
+        JobDispatch.logger.error e.to_s
+        JobDispatch.logger.error e.backtrace.join("\n")
         {status: 'error', message: e.to_s}
       end
     end

data/lib/job_dispatch/client.rb

@@ -34,14 +34,46 @@ module JobDispatch
       SynchronousProxy.new(self, target, options)
     end
 
-    def enqueue(job_attrs)
+    # Enqueue a job to be processed describe by the passed job attributes.
+    #
+    # Required attributes:
+    #   target: The target object that will execute the job. typically a class.
+    #   method: the message to be sent to the target.
+    # Optional:
+    #   parameters: an array of parameters to be passed to the method.
+    #   timeout: number of seconds after which the job is considered timed out and failed.
+    def enqueue(job_attrs={})
       send_request('enqueue', {job: job_attrs})
     end
 
+    # send a message to the dispatcher requesting to be notified when the job completes (or fails).
     def notify(job_id)
       send_request('notify', {job_id: job_id})
     end
 
+    # as the dispatcher what was the last job enqueued on the given queue (or default)
+    def last(queue=nil)
+      job_or_raise send_request('last', {queue: queue||'default'})
+    end
+
+    # fetch the complete details for hte last job
+    def fetch(job_id)
+      job_or_raise send_request('fetch', {job_id: job_id})
+    end
+
+    private
+
+    def job_or_raise(response)
+      if response.is_a?(Hash) && response[:status] == 'success'
+        response[:job]
+      else
+        p response
+        raise ClientError, response[:message]
+      end
+    end
+  end
+
+  class ClientError < StandardError
   end
 end
 

data/lib/job_dispatch/version.rb

@@ -1,3 +1,3 @@
 module JobDispatch
-  VERSION = "0.0.2"
+  VERSION = "0.1.0"
 end

data/lib/job_dispatch/worker.rb

@@ -102,6 +102,7 @@ module JobDispatch
       job_id = Thread.current["JobDispatch::Worker.job_id"]
       if sock && job_id
         sock.send_touch(job_id, timeout)
+        JobDispatch.logger.debug { "touching job #{job_id}"}
       end
     end
 

data/lib/job_dispatch/worker/item.rb

@@ -36,6 +36,7 @@ module JobDispatch
               backtrace: ex.backtrace,
           }
           @status = :error
+          JobDispatch.logger.debug ex
         ensure
           Thread.current["JobDispatch::Worker.job_id"] = nil
           JobDispatch.logger.info "Worker completed job #{job_id}: #{target}.#{method}, status: #{@status}"

data/lib/job_dispatch/worker/socket.rb

@@ -8,11 +8,14 @@ module JobDispatch
     class Socket
 
       attr :socket
+      attr :touch_socket
       attr :item_class
 
       def initialize(connect_address, item_klass)
         @socket = JobDispatch.context.socket(ZMQ::REQ)
         @socket.connect(connect_address)
+        @touch_socket = JobDispatch.context.socket(ZMQ::DEALER)
+        @touch_socket.connect(connect_address)
         @item_class = item_klass
       end
 
@@ -29,7 +32,14 @@ module JobDispatch
       end
 
       def close
-        @socket.close
+        if @socket
+          @socket.close rescue nil
+          @socket = nil
+        end
+        if @touch_socket
+          @touch_socket.close rescue nil
+          @touch_socket = nil
+        end
       end
 
       def identity
@@ -45,8 +55,9 @@ module JobDispatch
       #
       # @return [JobDispatch::Item] the item to be processed (or nil if there isn't a valid job)
       def read_item
-        json = @socket.recv
         begin
+          drain_touch_socket
+          json = @socket.recv
           params = JSON.parse(json)
           case params["command"]
             when "job"
@@ -67,6 +78,14 @@ module JobDispatch
         item
       end
 
+      # drain any messages that may have been received on the touch socket.
+      def drain_touch_socket
+        loop do
+          message = @touch_socket.recv_nonblock
+          break if message.nil?
+        end
+      end
+
       # after execution, send the response.
       def send_response(job_id, status, result)
         JobDispatch.logger.info "Worker #{Process.pid} completed job_id: #{job_id}: #{status}, result: #{result}"
@@ -86,11 +105,8 @@ module JobDispatch
             job_id: job_id
         }
         hash[:timeout] = timeout if timeout
-        @socket.send(JSON.dump(hash))
-        json = @socket.recv # wait for acknowledgement... this could be done via pub/sub to be asynchronous.
-        JSON.parse(json) rescue {:error => "Failed to decode JSON from dispatcher: #{json}"}
+        @touch_socket.send(JSON.dump(hash))
       end
-
     end
   end
 end

data/spec/job_dispatch/broker_spec.rb

@@ -488,6 +488,8 @@ describe JobDispatch::Broker do
         @job = FactoryGirl.build :job
         @socket = double('Broker::Socket', :send_command => nil)
         subject.stub(:socket => @socket)
+        @job_class = double('JobClass')
+        JobDispatch.config.job_class = @job_class
       end
 
       it "the job is sent to an idle worker" do
@@ -497,10 +499,8 @@ describe JobDispatch::Broker do
           expect(cmd.parameters[:target]).to eq(@job.target)
         end
 
-        job_class = double('JobClass')
-        job_class.stub(:dequeue_job_for_queue).and_return(@job)
-        job_class.should_receive(:dequeue_job_for_queue).with('example')
-        JobDispatch.config.job_class = job_class
+        @job_class.stub(:dequeue_job_for_queue).and_return(@job)
+        @job_class.should_receive(:dequeue_job_for_queue).with('example')
 
         # send ready command => adds idle worker state
         subject.workers_waiting_for_reply << worker_id # simulating read_command
@@ -511,9 +511,29 @@ describe JobDispatch::Broker do
         }))
         expect(@result).to be_nil # no immediate response
         expect(subject.workers_waiting_for_jobs[worker_id]).not_to be_nil
+        subject.queues_ready[:example] = true
 
         subject.dispatch_jobs_to_workers
       end
+
+      it "when no job is found, the queue is marked inactive" do
+        # send ready command => adds idle worker state
+        subject.workers_waiting_for_reply << worker_id # simulating read_command
+        @result = subject.process_command(Command.new(worker_id, {
+            command: 'ready',
+            queue: 'example',
+            worker_name: 'ruby worker',
+        }))
+
+        @job_class.stub(:dequeue_job_for_queue).and_return(nil)
+
+        expect(@result).to be_nil # no immediate response
+        expect(subject.workers_waiting_for_jobs[worker_id]).not_to be_nil
+        subject.queues_ready[:example] = true
+
+        subject.dispatch_jobs_to_workers
+        expect(subject.queues_ready[:example]).to be_false
+      end
     end
 
     context "when an error occurs dequeuing jobs" do
@@ -578,7 +598,7 @@ describe JobDispatch::Broker do
 
   context "touching a job" do
     before :each do
-      @time = Time.now
+      @time = Time.now.change(:usec => 0)
       # this worker will be IDLE
       @job = FactoryGirl.build :job, :expire_execution_at => @time + 5.seconds
       @job_id = @job.id.to_s
@@ -594,7 +614,7 @@ describe JobDispatch::Broker do
       Timecop.freeze(@time) do
         subject.touch_job(Command.new(worker_id, {command: "touch", job_id: @job_id}))
       end
-      expect(@job.expire_execution_at).to eq(@time + JobDispatch::Job::DEFAULT_EXECUTION_TIMEOUT)
+      expect(@job.expire_execution_at).to eq(@time + @job.timeout)
     end
 
     it "updates the expire_execution_at time with a custom timeout" do
@@ -608,10 +628,10 @@ describe JobDispatch::Broker do
   context "enqueue a job" do
     before :each do
       @job_attrs = FactoryGirl.attributes_for :job
+      JobDispatch.config.job_class = double('JobClass')
     end
 
     it "Creates a job" do
-      JobDispatch.config.job_class = double('JobClass')
       JobDispatch.config.job_class.should_receive(:create!).with(@job_attrs)
       command = Command.new(:some_client, {command: "enqueue", job: @job_attrs})
       subject.process_command(command)
@@ -619,7 +639,6 @@ describe JobDispatch::Broker do
 
     it "returns the job id" do
       job_id = 12345
-      JobDispatch.config.job_class = double('JobClass')
       JobDispatch.config.job_class.stub(:create! => double('Job', :id => job_id))
       command = Command.new(:some_client, {command: "enqueue", job: @job_attrs})
       result = subject.process_command(command)
@@ -628,7 +647,6 @@ describe JobDispatch::Broker do
     end
 
     it "returns an error if the arguments are no good" do
-      JobDispatch.config.job_class = double('JobClass')
       JobDispatch.config.job_class.stub(:create!).and_raise("no good") # simulate some database save error
       command = Command.new(:some_client, {command: "enqueue", job: @job_attrs})
       result = subject.process_command(command)
@@ -637,13 +655,18 @@ describe JobDispatch::Broker do
     end
 
     it "returns an error if the 'job' parameter is missing" do
-      JobDispatch.config.job_class = double('JobClass')
       command = Command.new(:some_client, {command: "enqueue"})
       result = subject.process_command(command)
       expect(result.parameters[:status]).to eq('error')
     end
-  end
 
+    it "marks the queue as ready" do
+      JobDispatch.config.job_class.stub(:create! => double('Job', :id => 1))
+      command = Command.new(:some_client, {command: "enqueue", job: @job_attrs})
+      result = subject.process_command(command)
+      expect(subject.queues_ready[:default]).to be_true
+    end
+  end
 
   context "'notify'" do
 
@@ -779,4 +802,61 @@ describe JobDispatch::Broker do
     end
   end
 
+  context "last" do
+    let(:json){ {'id' => '12341234', 'target' => 'Example', 'method' => 'method'}}
+    before do
+      @job_class = double('JobClass')
+      JobDispatch.config.job_class = @job_class
+    end
+    it "returns last job in specified queue" do
+      command = Command.new(:client, {command: 'last', queue: 'ruby'})
+      relation = double('relation')
+      @job_class.should_receive(:where).with(queue: 'ruby').and_return(relation)
+      relation.should_receive(:last).and_return(double("job", id: "12341234", as_json: json))
+      result = subject.process_command(command)
+      expect(result.parameters).to eq({status: 'success', job: json})
+    end
+    it "returns last job in default queue" do
+      command = Command.new(:client, {command: 'last'})
+      relation = double('relation')
+      @job_class.should_receive(:where).with(queue: 'default').and_return(relation)
+      relation.should_receive(:last).and_return(double("job", id: "12341234", as_json: json))
+      result = subject.process_command(command)
+      expect(result.parameters).to eq({status: 'success', job: json})
+    end
+    it "handles no last job" do
+      command = Command.new(:client, {command: 'last'})
+      relation = double('relation')
+      @job_class.should_receive(:where).with(queue: 'default').and_return(relation)
+      relation.should_receive(:last).and_return(nil)
+      result = subject.process_command(command)
+      expect(result.parameters).to eq({status: 'error', message: 'no last job'})
+    end
+  end
+
+  context "fetch" do
+    before do
+      @job_class = double('JobClass')
+      JobDispatch.config.job_class = @job_class
+    end
+    it "returns the job" do
+      command = Command.new(:client, {command: 'fetch', job_id: '12341234'})
+      json = {'id' => '12341234', 'queue' => 'ruby', 'target' => 'String', 'method' => 'new'}
+      job = double("Job", as_json: json)
+      @job_class.should_receive(:find).with('12341234').and_return(job)
+      result = subject.process_command(command)
+      expect(result.parameters).to eq({status: 'success', job: json})
+    end
+    it "returns error when job_id is not present" do
+      command = Command.new(:client, {command: 'fetch'})
+      result = subject.process_command(command)
+      expect(result.parameters[:status]).to eq('error')
+    end
+    it "returns error when job_id is not found" do
+      command = Command.new(:client, {command: 'fetch', job_id: '12341234'})
+      @job_class.should_receive(:find).with('12341234').and_raise(StandardError, "not found")
+      result = subject.process_command(command)
+      expect(result.parameters[:status]).to eq('error')
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: job_dispatch
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.1.0
 platform: ruby
 authors:
 - Matt Connolly
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-13 00:00:00.000000000 Z
+date: 2014-07-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rbczmq