postjob 0.1.7 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 041cef7de8f64274ba69bf907217d1268031c9cc
-  data.tar.gz: 8afbfab8f0320026c7bd89fdffdbdf88d497f7e3
+  metadata.gz: c750b67a0ab97557768d02f7a3cf2644646a8e1e
+  data.tar.gz: 4b1b93d7c1a261329d92081459c58492cf5a7923
 SHA512:
-  metadata.gz: 3d30fc8fa4578c3031d543aef08422d6680a8f6379032ce0665b320b8bf39b071a93d332c7da212590c782ae53b6c52e7b592423b2892a83c5ff3f3ea91d23a7
-  data.tar.gz: 4ce5f6935aa6cc807ba5d9592d2762eecd932afa449578100b5f5967a94197d35995cc81bd2efe4f3b1fcbf7ed89268d31a9d165e444c5d0850e1e57bc0a5e04
+  metadata.gz: 1310163dededb197a2c53deb03bfc6f0b0e7ec399ab33e870edab619b085fc86633c566b9a0152a07df3c2ef8cae580ebd9f4dccb95435c203794e266946c1d1
+  data.tar.gz: 3e6a73a8f11f8dd828a099ce8b8380e0872a6e23379109d85681d3107937f25d9a992745c48214d0d309b4b37acc681cab5b01620bf30a96f3e3945ae238e769

data/lib/postjob/cli/db.rb

@@ -1,28 +1,48 @@
 require "postjob/cli"
+require "postjob/migrations"
 
 module Postjob::CLI
   def db_migrate
-    require "postjob/migrations"
-
     connect_to_database!
     Postjob::Migrations.migrate!
   end
 
   def db_unmigrate
-    require "postjob/migrations"
+    if !force
+      confirm! <<~TXT
+        Really unmigrating database? This will destroy all data about postjobs!
+        (To prevent the need to confirm run with '--force'.)
+      TXT
+    end
 
     connect_to_database!
     Postjob::Migrations.unmigrate!
   end
 
-  def db_remigrate
-    require "postjob/migrations"
+  def db_remigrate(force: false)
+    if !force
+      confirm! <<~TXT
+        Really remigrating database? This will destroy all data about postjobs!
+        (To prevent the need to confirm run with '--force'.)
+      TXT
+    end
 
     connect_to_database!
     Postjob::Migrations.unmigrate!
     Postjob::Migrations.migrate!
   end
 
+  def confirm!(msg)
+    STDERR.puts <<~TXT
+      #{msg.chomp}
+
+      Press return to continue, ^C to cancel...
+    TXT
+    STDIN.gets
+  rescue Interrupt
+    raise "Cancelled by user"
+  end
+
   private
 
   USE_ACTIVE_RECORD = false

data/lib/postjob/cli/job.rb

@@ -4,7 +4,7 @@ module Postjob::CLI
   # Adds a workflow to the job table, with name <workflow> and the given
   # arguments.
   #
-  # Note that the workflow will receive the arguments as strings and must be 
+  # Note that the workflow will receive the arguments as strings and must be
   # prepared to handle these.
   def job_enqueue(workflow, *args, queue: "ruby", tags: nil)
     connect_to_database!
@@ -22,7 +22,7 @@ module Postjob::CLI
     full_job_id || logger.error("No such job: #{job_id}")
 
     job_ids = Simple::SQL.all <<~SQL
-      SELECT id FROM postjob.postjobs 
+      SELECT id FROM postjob.postjobs
       WHERE (full_id LIKE '#{full_job_id}.%' OR full_id='#{full_job_id}')
         AND status IN ('failed', 'err', 'timeout')
     SQL
@@ -42,7 +42,7 @@ module Postjob::CLI
       NOTIFY postjob_notifications
     SQL
 
-    logger.warn "The following jobs have been reset: #{job_ids.join(", ")}"
+    logger.warn "The following jobs have been reset: #{job_ids.join(', ')}"
   end
 
   private

data/lib/postjob/cli/ps.rb

@@ -57,7 +57,7 @@ module Postjob::CLI
     limit = Integer(limit)
 
     unless ids.empty?
-      ps_full *ids, limit: limit, tags: tags
+      ps_full(*ids, limit: limit, tags: tags)
       return
     end
 
@@ -72,10 +72,10 @@ module Postjob::CLI
 
   # Show all information about this job
   def ps_show(id, *ids)
-    ids = ([ id ] + ids).map { |s| Integer(s) }
+    ids = ([id] + ids).map { |s| Integer(s) }
 
     jobs = Simple::SQL.records <<~SQL, ids, into: Postjob::Job
-    SELECT * FROM postjob.postjobs WHERE id = ANY($1)
+      SELECT * FROM postjob.postjobs WHERE id = ANY($1)
     SQL
 
     jobs.each do |job|

data/lib/postjob/queue/encoder.rb

@@ -18,6 +18,8 @@ module Postjob::Queue::Encoder
   extend self
 
   def encode(data)
+    return if data.nil?
+
     verify_encodable!(data)
     JSON.generate(data)
   end

data/lib/postjob/queue/notifications.rb

@@ -47,26 +47,16 @@ module Postjob::Queue::Notifications
   # returns the maximum number of seconds to wait until the
   # next runnable or timeoutable job comes up.
   def time_to_next_job
-    queries = []
-
-    escaped_workflows_and_versions = Postjob::Registry.sql_escaped_workflows_and_versions
-    if escaped_workflows_and_versions != ""
-      queries.push <<~SQL
-        SELECT
-          EXTRACT(EPOCH FROM MIN(next_run_at) - (now() at time zone 'utc'))
-        FROM #{TABLE_NAME}
-        WHERE status = 'ready' AND ((workflow, workflow_version) IN (#{escaped_workflows_and_versions}))
-      SQL
-    end
-
-    queries.push <<~SQL
-      SELECT
-        EXTRACT(EPOCH FROM MIN(timing_out_at) - (now() at time zone 'utc'))
-      FROM #{TABLE_NAME}
-      WHERE status IN ('ready', 'sleep')
+    Simple::SQL.ask <<~SQL, Postjob::Registry.workflows_with_versions
+      SELECT EXTRACT(EPOCH FROM (MIN(next_event_at) - (now() at time zone 'utc'))) FROM (
+          SELECT MIN(timing_out_at) AS next_event_at
+          FROM #{TABLE_NAME}
+          WHERE status IN ('ready', 'sleep')
+        UNION
+          SELECT MIN(next_run_at) AS next_event_at
+          FROM #{TABLE_NAME}
+          WHERE status = 'ready' AND (workflow || workflow_version = ANY ($1))
+      ) sq
     SQL
-
-    timestamps = Simple::SQL.all(queries.join(" UNION "))
-    timestamps.compact.min
   end
 end

data/lib/postjob/queue.rb

@@ -99,7 +99,7 @@ module Postjob::Queue
 
     SQL.ask <<~SQL, job.id, results
       UPDATE #{TABLE_NAME}
-      SET results=$2, status='ok', next_run_at=NULL
+      SET results=$2, status='ok', next_run_at=NULL, error=NULL, error_message=NULL, error_backtrace=NULL
       WHERE id=$1
     SQL
 
@@ -136,7 +136,7 @@ module Postjob::Queue
     # subtract 1, since this check runs *after* the current run was done,
     # but before it was written to the database.
     if status == :err && remaining_attempts(job) > 1
-      [ "ready", next_run_at_fragment ]
+      [ "err", next_run_at_fragment ]
     elsif status == :timeout
       [ "timeout", "NULL" ]
     else
@@ -272,24 +272,6 @@ module Postjob::Queue
     [workflow, workflow_method]
   end
 
-  def runnable_sql_fragment
-    escaped_workflows_and_versions = Postjob::Registry.sql_escaped_workflows_and_versions
-    return "FALSE" if escaped_workflows_and_versions == ""
-
-    <<~SQL
-      next_run_at <= (now() at time zone 'utc')
-      AND status = 'ready'
-      AND ((workflow, workflow_version) IN (#{escaped_workflows_and_versions}))
-    SQL
-  end
-
-  def timing_out_sql_fragment
-    <<~SQL
-      timing_out_at <= (now() at time zone 'utc')
-      AND status IN ('ready', 'sleep')
-    SQL
-  end
-
   public
 
   def checkout_runnable
@@ -299,16 +281,16 @@ module Postjob::Queue
         timing_out_at <= (now() at time zone 'utc') AS timed_out
       FROM #{TABLE_NAME}
       WHERE
-        (#{runnable_sql_fragment})
+        (next_run_at <= (now() at time zone 'utc') AND status IN ('ready', 'err') AND workflow || workflow_version = ANY ($1))
         OR
-        (#{timing_out_sql_fragment})
+        (timing_out_at <= (now() at time zone 'utc') AND status IN ('ready', 'err', 'sleep'))
       ORDER BY (LEAST(next_run_at, timing_out_at))
       FOR UPDATE SKIP LOCKED
       LIMIT 1
     SQL
 
     SQL.transaction do
-      job = SQL.record sql, into: Job
+      job = SQL.record sql, Postjob::Registry.workflows_with_versions, into: Job
       yield job if job
       job
     end

data/lib/postjob/registry.rb

@@ -6,35 +6,20 @@ module Postjob::Registry
     instance.values.uniq
   end
 
-  # [TODO] - it would be nicer if Simple::SQL would properly build a Postgres version
-  # of the workflow names and versions.
-  #
-  def sql_escaped_workflows_and_versions
-    @sql_escaped_workflows_and_versions ||= begin
-      instance.keys.map do |name, workflow_version|
-        escaped_name    = sql_escape(name)
-        escaped_version = sql_escape(workflow_version)
-        "(#{escaped_name}, #{escaped_version})"
-      end.join(", ")
-    end
+  def workflows_with_versions
+    @workflows_with_versions ||= []
   end
 
-  private
-
-  def sql_escape(s)
-    "'" + PG::Connection.escape_string(s) + "'"
-  end
-
-  public
-
   # Used for tests
   def reset! # :nodoc:
-    @instance = @sql_escaped_workflows_and_versions = nil
+    @instance = @workflows_with_versions = nil
   end
 
   def register(workflow, _options = {})
     instance[[workflow.name, ""]] = workflow
     instance[[workflow.name, workflow.workflow_version]] = workflow
+
+    workflows_with_versions << workflow.name << "#{workflow.name}#{workflow.workflow_version}"
   end
 
   def lookup!(name:, version:)

data/lib/postjob.rb

@@ -1,6 +1,7 @@
 require "expectation"
 require "simple/sql"
 require "timeout"
+require "time"
 
 module Postjob
 end

data/spec/postjob/enqueue_spec.rb

@@ -1,5 +1,3 @@
-# rubocop:disable Metrics/BlockLength
-
 require "spec_helper"
 
 describe "Postjob.enqueue!" do

data/spec/postjob/full_workflow_spec.rb

@@ -1,3 +1,5 @@
+# rubocop:disable Style/MethodMissing
+
 require "spec_helper"
 
 #

data/spec/postjob/job_control/error_status_spec.rb

@@ -0,0 +1,73 @@
+require "spec_helper"
+
+# DO NOT TRY THIS IN THE REAL WORLD!
+#
+# This workflow uses a global value, FailureWorkflow.attempts, to count the
+# number of attempts to run a workflow. This turns the workflow into a stateful
+# affair WHICH BREAKS THE GENERAL CONTRACT OF POSTJOB.
+module FailureWorkflow
+  extend self
+
+  attr_accessor :attempts
+
+  def run(number_of_failing_attempts)
+    self.attempts += 1
+    if self.attempts <= number_of_failing_attempts
+      raise "FailureWorkflow failed"
+    end
+
+    "result"
+  end
+
+  Postjob.register_workflow self
+end
+
+describe "error states" do
+  before do
+    FailureWorkflow.attempts = 0
+  end
+
+  def load_job
+    TestHelper.load_job(job_id)
+  end
+
+  let!(:job_id) { Postjob.enqueue!("FailureWorkflow", 1) }
+
+  context "when running a job that runs into an exception" do
+    before do
+      Postjob.process_all
+    end
+
+    it "sets the job's status to err" do
+      expect(load_job.status).to eq("err")
+    end
+
+    it "sets the job's error information" do
+      expect(load_job.error_message).to eq("FailureWorkflow failed")
+      expect(load_job.error).to eq("RuntimeError")
+      expect(load_job.error_backtrace).to be_a(Array)
+    end
+  end
+
+  context "when rerunning the job so that it succeeds" do
+    before do
+      while Postjob.process_all > 0
+        sleep 0.1
+      end
+    end
+
+    it "sets the job's status to ok" do
+      expect(load_job.status).to eq("ok")
+    end
+
+    it "sets the job's result" do
+      expect(load_job.result).to eq("result")
+    end
+
+    it "cleared out the error entries" do
+      expect(load_job.error_message).to be_nil
+      expect(load_job.error).to be_nil
+      expect(load_job.error_backtrace).to be_nil
+    end
+  end
+end

data/spec/postjob/job_control/max_attempts_spec.rb

@@ -1,5 +1,3 @@
-# rubocop:disable Metrics/BlockLength
-
 require "spec_helper"
 
 module MaxAttemptWorkflow

data/spec/postjob/queue/search_spec.rb

@@ -1,7 +1,5 @@
 require "spec_helper"
 
-# rubocop:disable Metrics/BlockLength
-
 module SearchWorkflow
   def self.run
     set_workflow_status "starting-up"

data/spec/postjob/run_spec.rb

@@ -1,5 +1,3 @@
-# rubocop:disable Metrics/BlockLength
-
 require "spec_helper"
 
 module TwoLevelWorkflow

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: postjob
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.1.8
 platform: ruby
 authors:
 - radiospiel
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-03-02 00:00:00.000000000 Z
+date: 2018-03-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -236,6 +236,7 @@ files:
 - lib/postjob/workflow.rb
 - spec/postjob/enqueue_spec.rb
 - spec/postjob/full_workflow_spec.rb
+- spec/postjob/job_control/error_status_spec.rb
 - spec/postjob/job_control/manual_spec.rb
 - spec/postjob/job_control/max_attempts_spec.rb
 - spec/postjob/job_control/timeout_spec.rb