postjob 0.5.11 → 0.5.12

data/lib/postjob.rb

@@ -38,6 +38,8 @@ module Postjob
   attr_accessor :fast_mode # :nodoc:
   self.fast_mode = false
 
+  CRON_INTERVAL_MIN = 60
+
   # Enqueues a workflow.
   #
   # Options include
@@ -91,8 +93,8 @@ module Postjob
       greedy          = spec.options.greedy
     end
 
-    if cron_interval && cron_interval < 300
-      raise "cron interval must be at least 5 minutes"
+    if cron_interval && cron_interval < CRON_INTERVAL_MIN
+      raise "cron interval must be at least #{CRON_INTERVAL_MIN} seconds"
     end
 
     # -- disable existing cron jobs -------------------------------------------
@@ -105,15 +107,18 @@ module Postjob
 
     tags = stringify_hash(tags) if tags
 
-    job = Queue.enqueue_job current_worker_session.id, workflow, *args, queue: queue,
-                                                                        parent_id: parent_id,
-                                                                        max_attempts: max_attempts,
-                                                                        timeout: timeout,
-                                                                        tags: tags,
-                                                                        version: version,
-                                                                        cron_interval: cron_interval,
-                                                                        sticky: sticky,
-                                                                        greedy: greedy
+    session_id = current_session_id if current_session?
+    session_id ||= "00000000-0000-0000-0000-000000000000"
+
+    job = Queue.enqueue_job session_id, workflow, *args, queue: queue,
+                                                         parent_id: parent_id,
+                                                         max_attempts: max_attempts,
+                                                         timeout: timeout,
+                                                         tags: tags,
+                                                         version: version,
+                                                         cron_interval: cron_interval,
+                                                         sticky: sticky,
+                                                         greedy: greedy
 
     logger.info "Generated process #{job}"
     job.id
@@ -160,16 +165,20 @@ module Postjob
   # b) the block yielded into returns false.
   #
   # This method returns the number of processed jobs.
-  def run(count: nil, queue: nil, &block)
+  def run(count: nil, queues: nil, heartbeat: true, &block)
+    queues ||= [ "ruby" ]
+
     # to run 10^12 jobs that would take 1 msecs each we would need, at least,
     # 760 years - so this default should be fine. Also, someone should update
     # the machine in the meantime :)
     count ||= 1_000_000_000_000
 
+    start_worker_session!(heartbeat: heartbeat, queues: queues)
+
     processed_jobs_count = 0
 
     loop do
-      processed_job_id, shutdown = Postjob.step(queue: queue)
+      processed_job_id, shutdown = Postjob.step(queues: queues)
       processed_jobs_count += 1 if processed_job_id
 
       break if processed_jobs_count >= count
@@ -177,12 +186,31 @@ module Postjob
       break if shutdown == :shutdown
 
       next if processed_job_id
-      Queue::Notifications.wait_for_new_job(current_worker_session.id, queue: queue)
+      shutdown = Queue::Notifications.wait_for_new_job(current_session_id, queues: queues)
+      break if shutdown == :shutdown
     end
 
     processed_jobs_count
   end
 
+  def start_worker_session!(heartbeat: true, queues: nil) # :nodoc:
+    queues ||= [ "ruby" ]
+
+    # We can't restart a new worker_session (this is currently not supported)
+    # and probably also unnecessary. Instead we ignore this call, as long as
+    # the queue setting is identical.
+    #
+    # A call to start_worker_session! is valid only during tests.
+    if @worker_session
+      raise ArgumentError, "You cannot restart a worker session" if ENV["RACK_ENV"] != "test"
+      raise ArgumentError, "You cannot restart a worker_session with different queues" if @worker_session.queues != queues
+
+      return @worker_session
+    end
+
+    @worker_session = WorkerSession.start!(Registry.runnable_workflows_with_versions, heartbeat: heartbeat, queues: queues)
+  end
+
   # Runs a single job
   #
   # This method tries to check out a runnable job. If it finds one the
@@ -195,18 +223,25 @@ module Postjob
   #   self.run to terminate the run loop.
   #
   # or nil, when no job could be checked out.
-  def step(queue: nil)
-    job = Postjob::Queue.checkout(current_worker_session.id, queue: queue)
+  def step(queues: nil)
+    expect! queues => [Array, nil]
+    queues ||= [ "ruby" ]
+
+    job = Postjob::Queue.checkout(current_session_id, queues: queues)
 
     [ job.id, process_job(job) ] if job
   end
 
+  def current_session?
+    @worker_session != @nil
+  end
+
   # This method connects to the queue. This means it registers as a new worker_session,
   # if there was no worker_session yet.
-  def current_worker_session
-    @worker_session ||= begin
-      WorkerSession.start!(Registry.runnable_workflows_with_versions)
-    end
+  def current_session_id
+    raise("worker_session hasn't been started yet.") unless @worker_session
+
+    @worker_session.id
   end
 
   # Explicitely resolve a workflow.
@@ -214,7 +249,7 @@ module Postjob
     job = Queue.find_job_by_token(token)
     raise "No job with token #{token}" unless job
 
-    Queue.set_job_result current_worker_session.id, job, result, version: nil
+    Queue.set_job_result current_session_id, job, result, version: nil
   end
 
   # Registers a workflow.
@@ -264,13 +299,14 @@ module Postjob
       raise "Integrity check failed: job's workflow version changed (from #{job.workflow_version} to #{version})"
     end
 
-    worker_session_id = current_worker_session.id
+    session_id = current_session_id if current_session?
+    session_id ||= "00000000-0000-0000-0000-000000000000"
 
     case status
-    when :failed    then Queue.set_job_error worker_session_id, job, *value, status: :failed, version: version
-    when :err       then Queue.set_job_error worker_session_id, job, *value, status: :err, version: version
-    when :pending   then Queue.set_job_pending worker_session_id, job, version: version
-    when :ok        then Queue.set_job_result worker_session_id, job, value, version: version
+    when :failed    then Queue.set_job_error session_id, job, *value, status: :failed, version: version
+    when :err       then Queue.set_job_error session_id, job, *value, status: :err, version: version
+    when :pending   then Queue.set_job_pending session_id, job, version: version
+    when :ok        then Queue.set_job_result session_id, job, value, version: version
     else            raise ArgumentError, "Invalid status #{status.inspect}"
     end
 

data/lib/tools/heartbeat.rb

@@ -56,7 +56,8 @@ module Heartbeat # :nodoc:
     monitor = Monitor.new(cycle_length_seconds)
 
     loop do
-      do_continue = yield monitor.measure
+      measurement = monitor.measure
+      do_continue = yield measurement
       break if do_continue == false
       monitor.sleep
     end

data/spec/postjob/events/job_event_spec.rb

@@ -69,12 +69,12 @@ describe "Job Events" do
 
     it "sets the job's last_worker_session_id" do
       last_worker_session_ids = jobs.map(&:last_worker_session_id)
-      expect(last_worker_session_ids.uniq).to eq([Postjob.current_worker_session.id])
+      expect(last_worker_session_ids.uniq).to eq([Postjob.current_session_id])
     end
 
     it "records the worker session id" do
       worker_session_ids = events.map(&:worker_session_id).uniq
-      expect(worker_session_ids).to eq([Postjob.current_worker_session.id])
+      expect(worker_session_ids).to eq([Postjob.current_session_id])
     end
   end
 end

data/spec/postjob/worker_session_spec.rb

@@ -11,7 +11,7 @@ describe "Postjob::WorkerSession.start!" do
 
   it "creates a new session" do
     expect do
-      session = Postjob::WorkerSession.start!(workflows_with_versions)
+      session = Postjob::WorkerSession.start!(workflows_with_versions, queues: [ "ruby" ])
       expect(session.id).to match(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i)
     end.to change { session_count }.by(1)
   end

data/spec/postjob/zombie_spec.rb

@@ -0,0 +1,54 @@
+require "spec_helper"
+
+module ZombieSpecWorkflow
+  Postjob.register_workflow self
+end
+
+describe "Zombie Detection" do
+  include TestHelper
+
+  before do
+    # announce the ZombieSpecWorkflow
+    # Postjob.enqueue! "ZombieSpecWorkflow"
+
+    # # Start a worker session.
+    session = Postjob.start_worker_session!(heartbeat: false)
+    @host_id = session.host_id
+  end
+
+  let(:host_id) { @host_id }
+
+  def send_heartbeat!
+    # pass in a artificial heartbeat
+    ::Postjob::Queue.host_heartbeat(host_id, {})
+  end
+
+  context "with a processing job which has the latest heartbeat from its host for > 5 minute" do
+    before do
+      @job_id = Postjob.enqueue! "ZombieSpecWorkflow"
+      SQL.ask "UPDATE postjob.postjobs SET status='processing'"
+      send_heartbeat!
+      SQL.ask "UPDATE postjob.events SET created_at = (now() at time zone 'utc' - interval '6 minutes')"
+    end
+
+    context "when sending a heartbeat" do
+      before do
+        send_heartbeat!
+      end
+
+      it "changes the job to zombie" do
+        job = SQL.ask "SELECT * FROM postjob.postjobs WHERE id= #{@job_id}", into: Postjob::Job
+
+        expect(job.error).to eq("Zombie")
+        expect(job.error_message).to match("host .* disappeared")
+        expect(job.status).to eq("err")
+      end
+
+      it "creates a zombie event with a zombie_count" do
+        event = SQL.ask "SELECT * FROM postjob.events WHERE name='zombie' ORDER BY created_at DESC LIMIT 1", into: OpenStruct
+        expect(event.name).to eq("zombie")
+        expect(event.attributes).to eq("zombie_count" => 1)
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -29,6 +29,8 @@ if ENV["VERBOSE"]
   Simple::SQL.logger = Postjob.logger = logger
 end
 
+SQL = Simple::SQL
+
 # Ensure there is a tmp directory for testing host id file ...
 Dir.mkdir("tmp") unless File.exist?("tmp")
 

data/spec/support/test_helper.rb

@@ -34,15 +34,8 @@ module TestHelper
     end
   end
 
-  def print_sql(sql, *args)
-    require "table_print"
-
-    records = Simple::SQL.all(sql, *args, into: Hash)
-    tp records
-  end
-
   def print_jobs
-    print_sql <<~SQL
+    sql = <<~SQL
       SELECT
         id,
         workflow AS job,
@@ -59,5 +52,7 @@ module TestHelper
         FROM postjobs
       ORDER BY id
     SQL
+
+    Simple::SQL.print(sql)
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: postjob
 version: !ruby/object:Gem::Version
-  version: 0.5.11
+  version: 0.5.12
 platform: ruby
 authors:
 - radiospiel
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-08-17 00:00:00.000000000 Z
+date: 2018-08-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -123,7 +123,7 @@ dependencies:
         version: '0.4'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.4.8
+        version: 0.4.15
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -133,27 +133,27 @@ dependencies:
         version: '0.4'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.4.8
+        version: 0.4.15
 - !ruby/object:Gem::Dependency
   name: simple-cli
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '0'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.2.4
+        version: 0.2.8
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '0'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.2.4
+        version: 0.2.8
 - !ruby/object:Gem::Dependency
   name: table_print
   requirement: !ruby/object:Gem::Requirement
@@ -231,9 +231,11 @@ files:
 - lib/postjob/cli/enqueue.rb
 - lib/postjob/cli/events.rb
 - lib/postjob/cli/heartbeat.rb
+- lib/postjob/cli/helpers.rb
 - lib/postjob/cli/hosts.rb
 - lib/postjob/cli/job.rb
 - lib/postjob/cli/ps.rb
+- lib/postjob/cli/queues.rb
 - lib/postjob/cli/run.rb
 - lib/postjob/cli/sessions.rb
 - lib/postjob/cli/version.rb
@@ -241,6 +243,7 @@ files:
 - lib/postjob/host.rb
 - lib/postjob/job.rb
 - lib/postjob/migrations.rb
+- lib/postjob/migrations/001_helpers.sql
 - lib/postjob/migrations/002_statuses.rb
 - lib/postjob/migrations/003_postjobs.sql
 - lib/postjob/migrations/003a_processing.sql
@@ -283,7 +286,6 @@ files:
 - spec/postjob/enqueue_spec.rb
 - spec/postjob/events/heartbeat_event_spec.rb
 - spec/postjob/events/job_event_spec.rb
-- spec/postjob/events/zombie_event_spec.rb
 - spec/postjob/full_workflow_spec.rb
 - spec/postjob/host_spec.rb
 - spec/postjob/job_control/error_status_spec.rb
@@ -297,6 +299,7 @@ files:
 - spec/postjob/step_spec.rb
 - spec/postjob/worker_session_spec.rb
 - spec/postjob/workflows/child_workflow_spec.rb
+- spec/postjob/zombie_spec.rb
 - spec/spec_helper.rb
 - spec/support/configure_active_record.rb
 - spec/support/configure_database.rb

data/spec/postjob/events/zombie_event_spec.rb

@@ -1,61 +0,0 @@
-require "spec_helper"
-
-module HeartbeatSpecWorkflow
-  Postjob.register_workflow self
-end
-
-describe "Heartbeat Events" do
-  include TestHelper
-
-  before do
-    Postjob.enqueue! "HeartbeatSpecWorkflow"
-
-    @host_id = Simple::SQL.ask "SELECT id FROM postjob.hosts WHERE id != $1", null_host_id
-
-    send_heartbeat!
-
-    Simple::SQL.ask "UPDATE postjob.postjobs SET status='processing'"
-  end
-
-  let(:host_id) { @host_id || raise("host_id must be set") }
-
-  def send_heartbeat!
-    ::Postjob::Queue.host_heartbeat(host_id, {})
-  end
-
-  describe "::Postjob::Queue.host_heartbeat" do
-    context "zombies in the database" do
-      before do
-        Simple::SQL.ask "UPDATE postjob.events SET created_at = (now() at time zone 'utc' - interval '2 days')"
-      end
-
-      it "does creates a zombie events" do
-        send_heartbeat!
-
-        event = Simple::SQL.ask <<~SQL, into: Hash
-          SELECT * FROM postjob.events
-          WHERE name='zombie' ORDER BY created_at DESC
-        SQL
-
-        expect(event).to include(
-          attributes: { "zombie_count" => 1 },
-          name: "zombie",
-          postjob_id: nil
-        )
-      end
-    end
-  end
-
-  context "no zombies in the database, 2" do
-    it "does creates a zombie events" do
-      send_heartbeat!
-
-      event = Simple::SQL.ask <<~SQL, into: Hash
-        SELECT * FROM postjob.events
-        WHERE name='zombie' ORDER BY created_at DESC
-      SQL
-
-      expect(event).to be_nil
-    end
-  end
-end