postjob 0.2.2 → 0.3.0

data/lib/postjob/runner.rb

@@ -41,9 +41,6 @@ module Postjob::Runner
     when :manual  then workflow = "__manual__"
     when Symbol   then workflow = "#{current_job.workflow}.#{workflow}"
     when Module   then workflow = workflow.name
-    when String   then :nop
-    else
-      raise ArgumentError, "Unsupported workflow spec #{workflow.inspect}. Did you run await(fun(a, b)) instead of await(:fun, a, b)"
     end
 
     ::Postjob::Queue.find_or_create_childjob(self.current_job, workflow, args,
@@ -81,12 +78,14 @@ module Postjob::Runner
   #
   # returns a tuple [status, value], which follows the following pattern:
   #
-  # - <tt>[ <runner-version>, :ok, value ]</tt>: job completed successfully
-  # - <tt>[ <runner-version>, :sleep, nil ]</tt>: job has to wait on a child job
-  # - <tt>[ <runner-version>, :err, <err> ]</tt>: job errored with a recoverable error
-  # - <tt>[ <runner-version>, :failed, <err> ]</tt>: job failed with a non-recoverable error
+  # - <tt>[ <runner-version>, :ok, value, nil ]</tt>: job completed successfully
+  # - <tt>[ <runner-version>, :sleep, nil, nil ]</tt>: job has to wait on a child job
+  # - <tt>[ <runner-version>, :err, <err>, nil ]</tt>: job errored with a recoverable error
+  # - <tt>[ <runner-version>, :failed, <err>, <shutdown> ]</tt>: job failed
+  #       with a non-recoverable error
   #
   # <err> is a tuple [ error-class-name, error-message, stacktrace ].
+  # <shutdown> is either nil or :shutdown
   #
   def process_job(job)
     expect! job => Job
@@ -94,15 +93,15 @@ module Postjob::Runner
     workflow = Postjob::Registry.lookup!(name: job.workflow, version: job.workflow_version)
 
     with_current_job(job) do
-      status, value = invoke_workflow workflow, job
+      status, value, shutdown = invoke_workflow workflow, job
       log_result! job, status, value
-      [ workflow.workflow_version, status, value ]
+      [ workflow.workflow_version, status, value, shutdown ]
     end
   end
 
   private
 
-  # runs a job. Returns a [ runner, status, value ] tuple.
+  # runs a job. Returns a [ status, value, shutdown ] tuple.
   def invoke_workflow(workflow, job)
     value = catch(:pending) {
       expect! job.args => [Array, nil]
@@ -117,8 +116,8 @@ module Postjob::Runner
     }
 
     case value
-    when :pending then [ :pending, nil ]
-    else               [ :ok, value ]
+    when :pending then [ :pending, nil, nil ]
+    else               [ :ok, value, nil ]
     end
   rescue ArgumentError, LocalJumpError, NameError, RegexpError, ScriptError, TypeError
     Postjob.logger.error "#{$!}, from\n\t#{$!.backtrace[0, 10].join("\n\t")}"
@@ -127,9 +126,19 @@ module Postjob::Runner
     return_exception :err, $!
   end
 
+  def should_shutdown?(exception)
+    exception.is_a?(Interrupt)
+  end
+
   def return_exception(state, exception)
+    # get and shorten backtrace.
     error_backtrace = exception.backtrace[0, 10]
-    [ state, [exception.class.name, exception.to_s, error_backtrace] ]
+
+    curdir = "#{Dir.getwd}/"
+    error_backtrace = error_backtrace.map { |path| path.start_with?(curdir) ? path[curdir.length..-1] : path }
+
+    shutdown = should_shutdown?(exception) ? :shutdown : nil
+    [ state, [exception.class.name, exception.to_s, error_backtrace], shutdown ]
   end
 
   def log_result!(job, status, value)

data/lib/postjob.rb

@@ -101,14 +101,14 @@ module Postjob
     processed_jobs_count = 0
 
     loop do
-      processed_job = Postjob.step
-      processed_jobs_count += 1 if processed_job
+      processed_job_id, shutdown = Postjob.step
+      processed_jobs_count += 1 if processed_job_id
 
       break if processed_jobs_count >= count
-      break if block && (yield(processed_job) == false)
-
-      next if processed_job
+      break if block && yield(processed_job_id) == false
+      break if shutdown == :shutdown
 
+      next if processed_job_id
       Queue::Notifications.wait_for_new_job
     end
 
@@ -118,42 +118,44 @@ module Postjob
   # Runs a single job
   #
   # This method tries to check out a runnable job. If it finds one the
-  # job is processed (via Postjob.process_job) and returned. If not,
-  # this method just returns nil.
+  # job is processed (via Postjob.process_job).
+  #
+  # This method returns a tuple [ <job>, <shutdown> ], where
+  #
+  # - <job-id> is the id of the job which has been processed;
+  # - <shutdown> is a flag, either :shutdown or nil. :shutdown notifies
+  #   self.run to terminate the run loop.
+  #
+  # or nil, when no job could be checked out.
   def step
-    Queue.checkout_runnable do |job|
-      process_job job
-    end
+    job = Queue.checkout(Registry.workflows_with_versions)
+    [ job.id, process_job(job) ] if job
   end
 
   private
 
   # This method is called from tests. Otherwise it is supposed to be private.
+  #
+  # It returns either nil or :shutdown
   def process_job(job) # :nodoc:
     expect! job => Job
 
-    version, status, value = if job.timed_out
-                               [job.workflow_version, :timeout, nil]
-                             else
-                               Runner.process_job(job)
-                             end
+    version, status, value, shutdown = Runner.process_job(job)
 
+    expect! version => String
     if job.workflow_version != "" && version != job.workflow_version
       raise "Integrity check failed: job's workflow version changed (from #{job.workflow_version} to #{version})"
     end
 
-    expect! version => String
-
     case status
     when :failed    then Queue.set_job_error job, *value, status: :failed, version: version
     when :err       then Queue.set_job_error job, *value, status: :err, version: version
-    when :timeout   then Queue.set_job_error job, "Timeout", "Timeout", status: :timeout, version: version
     when :pending   then Queue.set_job_pending job, version: version
     when :ok        then Queue.set_job_result job, value, version: version
     else            raise ArgumentError, "Invalid status #{status.inspect}"
     end
 
-    [status, value]
+    shutdown
   end
 
   public

data/spec/postjob/enqueue_spec.rb

@@ -1,24 +1,36 @@
 require "spec_helper"
 
 describe "Postjob.enqueue!" do
+  let(:workflow_name) { "WorkflowName" }
+
   include TestHelper
 
   context "with valid arguments" do
     it "creates a job" do
-      Postjob.enqueue! "FooishWorkflow"
+      Postjob.enqueue! workflow_name
       expect(TestHelper.jobs_count).to eq(1)
     end
 
     it "returns the created job" do
-      id = Postjob.enqueue! "FooishWorkflow"
+      id = Postjob.enqueue! workflow_name
       newest_job = TestHelper.newest_job
       expect(newest_job.id).to eq(id)
     end
 
+    it "uses the parent id and sets a full id with two jobs" do
+      id1 = Postjob.enqueue! workflow_name
+      id2 = Postjob.enqueue! workflow_name, parent_id: id1
+
+      job1 = load_job id1
+      job2 = load_job id2
+
+      expect(job2.full_id).to eq("#{job1.id}.#{job2.id}")
+    end
+
     it "uses the parent id and sets a full id" do
-      id1 = Postjob.enqueue! "FooishWorkflow"
-      id2 = Postjob.enqueue! "FooishWorkflow", parent_id: id1
-      id3 = Postjob.enqueue! "FooishWorkflow", parent_id: id2
+      id1 = Postjob.enqueue! workflow_name
+      id2 = Postjob.enqueue! workflow_name, parent_id: id1
+      id3 = Postjob.enqueue! workflow_name, parent_id: id2
 
       job1 = load_job id1
       job2 = load_job id2
@@ -29,37 +41,37 @@ describe "Postjob.enqueue!" do
     end
 
     it "sets arguments" do
-      id1 = Postjob.enqueue! "FooishWorkflow", 1, "two", "three"
+      id1 = Postjob.enqueue! workflow_name, 1, "two", "three"
       job1 = load_job id1
       expect(job1.args).to eq([1, "two", "three"])
 
-      id2 = Postjob.enqueue! "FooishWorkflow"
+      id2 = Postjob.enqueue! workflow_name
       job2 = load_job id2
       expect(job2.args).to eq([])
     end
 
     it "sets queue" do
-      id1 = Postjob.enqueue! "FooishWorkflow"
+      id1 = Postjob.enqueue! workflow_name
       job1 = load_job id1
       expect(job1.queue).to eq("q")
 
-      id1 = Postjob.enqueue! "FooishWorkflow", queue: "bla"
+      id1 = Postjob.enqueue! workflow_name, queue: "bla"
       job1 = load_job id1
       expect(job1.queue).to eq("bla")
     end
 
     it "sets max_attempts" do
-      id1 = Postjob.enqueue! "FooishWorkflow"
+      id1 = Postjob.enqueue! workflow_name
       job1 = load_job id1
       expect(job1.max_attempts).to eq(5)
 
-      id1 = Postjob.enqueue! "FooishWorkflow", max_attempts: 2
+      id1 = Postjob.enqueue! workflow_name, max_attempts: 2
       job1 = load_job id1
       expect(job1.max_attempts).to eq(2)
     end
 
     it "sets the status to ready" do
-      id1 = Postjob.enqueue! "FooishWorkflow", 1, "two", "three"
+      id1 = Postjob.enqueue! workflow_name, 1, "two", "three"
       job1 = load_job id1
       expect(job1.status).to eq("ready")
     end
@@ -76,9 +88,9 @@ describe "Postjob.enqueue!" do
     end
 
     it "raises an error with invalid parent_id" do
-      id1 = Postjob.enqueue! "FooishWorkflow"
+      id1 = Postjob.enqueue! workflow_name
 
-      expect { Postjob.enqueue! "FooishWorkflow", parent_id: id1 - 1 }.to raise_error(PG::ForeignKeyViolation)
+      expect { Postjob.enqueue! workflow_name, parent_id: id1 - 1 }.to raise_error(PG::ForeignKeyViolation)
     end
   end
 end

data/spec/postjob/job_control/error_status_spec.rb

@@ -13,7 +13,7 @@ module FailureWorkflow
   def run(number_of_failing_attempts)
     self.attempts += 1
     if self.attempts <= number_of_failing_attempts
-      raise "FailureWorkflow failed"
+      raise "Failure/Workflow failed"
     end
 
     "result"
@@ -43,7 +43,7 @@ describe "error states" do
     end
 
     it "sets the job's error information" do
-      expect(load_job.error_message).to eq("FailureWorkflow failed")
+      expect(load_job.error_message).to eq("Failure/Workflow failed")
       expect(load_job.error).to eq("RuntimeError")
       expect(load_job.error_backtrace).to be_a(Array)
     end

data/spec/postjob/job_control/manual_spec.rb

@@ -24,15 +24,13 @@ describe "manual processing" do
     TestHelper.load_job "SELECT * FROM postjobs WHERE parent_id=$1", id
   end
 
-  def token
-    load_token(load_child_job)
-  end
-
-  it "creates a token" do
+  it "creates a UUID token" do
+    token = load_token(load_child_job)
     expect! token => /[A-F0-9]{8}-[A-F0-9]{4}-[A-F0-9]{4}-[a-fA-F0-9]{4}-[A-F0-9]{12}/i
   end
 
-  it "the token can be used to resolve the load_child_job" do
+  it "the token can be used to resolve the child job" do
+    token = load_token(load_child_job)
     Postjob.resolve token: token, result: "foobar"
     expect(load_child_job.status).to eq("ok")
     expect(load_child_job.result).to eq("foobar")

data/spec/postjob/job_control/max_attempts_spec.rb

@@ -52,7 +52,9 @@ describe "max_attempts" do
     let!(:id) { Postjob.enqueue!("MaxAttemptWorkflow", 5, max_attempts: 1) }
 
     it "fails the childjob with a timeout" do
-      sleep 0.03 while Postjob.process_all > 0
+      while Postjob.process_all > 0
+        sleep 0.1
+      end
 
       expect(load_child_job.failed_attempts).to eq(5)
       expect(load_child_job.status).to eq("failed")

data/spec/postjob/process_job_spec.rb

@@ -13,8 +13,9 @@ describe "Postjob.process_job" do
 
   let!(:job) { Postjob.enqueue! "BarWorkflow" }
 
-  it "runs the job returning the result" do
-    expect(Postjob.process_job(newest_job)).to eq([:ok, "Foo"])
+  it "runs the job returning either nil or :shutdown" do
+    # Note: the :shutdown case is really hard to test.
+    expect(Postjob.process_job(newest_job)).to eq(nil)
   end
 
   it "updates the job version" do

data/spec/postjob/queue/encoder_spec.rb

@@ -16,6 +16,10 @@ describe "Postjob::Queue::Encoder" do
     job.result
   end
 
+  it "encodes nil into nil" do
+    expect(run_workflow(input: nil, output: nil)).to eq(nil)
+  end
+
   it "allows encoding of numbers" do
     expect(run_workflow(input: 10, output: 12)).to eq(12)
   end

data/spec/postjob/run_spec.rb

@@ -17,7 +17,7 @@ module TwoLevelWorkflow
   Postjob.register_workflow self
 end
 
-describe "Postjob.Postjob.process_all" do
+describe "Postjob.process_all" do
   include TestHelper
 
   let!(:id) { Postjob.enqueue! "TwoLevelWorkflow" }

data/spec/postjob/step_spec.rb

@@ -14,8 +14,8 @@ describe "Postjob.step" do
   let!(:job_id) { Postjob.enqueue! "StepWorkflow" }
 
   it "runs the job is there is one" do
-    processed_job = Postjob.step
-    expect(processed_job.id).to eq(job_id)
+    rv = Postjob.step
+    expect(rv).to eq([job_id, nil])
   end
 
   it "returns nil if there is none" do

data/spec/postjob/{sub_workflow_spec.rb → workflows/child_workflow_spec.rb}

@@ -1,6 +1,6 @@
 require "spec_helper"
 
-module SubWorkflow
+module WorkflowWithChild
   def self.my_string
     "my string"
   end
@@ -16,7 +16,7 @@ end
 describe "Sub Workflows" do
   include TestHelper
 
-  let!(:id) { Postjob.enqueue! "SubWorkflow" }
+  let!(:id) { Postjob.enqueue! "WorkflowWithChild" }
 
   it "runs the job returning the result" do
     Postjob.process_all

data/spec/spec_helper.rb

@@ -7,7 +7,7 @@ require "awesome_print"
 
 unless ENV["SKIP_SIMPLE_COV"]
   SimpleCov.start do
-    minimum_coverage 88
+    minimum_coverage 90
     add_filter "/spec/"
   end
 end

data/spec/support/configure_database.rb

@@ -7,6 +7,7 @@ end
 RSpec.configure do |config|
   config.around(:each) do |example|
     Simple::SQL.ask "DELETE FROM postjob.postjobs"
+    Simple::SQL.ask "DELETE FROM postjob.tokens"
     example.run
   end
 end

data/spec/support/test_helper.rb

@@ -8,15 +8,15 @@ module TestHelper
   end
 
   def newest_job
-    Simple::SQL.ask "SELECT * FROM postjobs ORDER BY id DESC", into: Postjob::Job
+    Simple::SQL.record "SELECT * FROM postjobs ORDER BY id DESC", into: Postjob::Job
   end
 
   def load_job(id_or_sql, *args)
     case id_or_sql
     when Integer
-      Simple::SQL.ask "SELECT * FROM postjobs WHERE id=$1", id_or_sql, into: Postjob::Job
+      Simple::SQL.record "SELECT * FROM postjobs WHERE id=$1", id_or_sql, into: Postjob::Job
     else
-      Simple::SQL.ask id_or_sql, *args, into: Postjob::Job
+      Simple::SQL.record id_or_sql, *args, into: Postjob::Job
     end
   end
 
@@ -27,7 +27,7 @@ module TestHelper
   def print_sql(sql, *args)
     require "table_print"
 
-    records = Simple::SQL.all(sql, *args)
+    records = Simple::SQL.records(sql, *args)
     tp records
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: postjob
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.3.0
 platform: ruby
 authors:
 - radiospiel
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-04-04 00:00:00.000000000 Z
+date: 2018-03-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -120,20 +120,20 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: '0.2'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.3.7
+        version: 0.2.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: '0.2'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.3.7
+        version: 0.2.5
 - !ruby/object:Gem::Dependency
   name: simple-cli
   requirement: !ruby/object:Gem::Requirement
@@ -213,6 +213,18 @@ files:
 - lib/postjob/error.rb
 - lib/postjob/job.rb
 - lib/postjob/migrations.rb
+- lib/postjob/migrations/002_statuses.rb
+- lib/postjob/migrations/003_postjobs.sql
+- lib/postjob/migrations/003a_processing.sql
+- lib/postjob/migrations/003b_processing_columns.sql
+- lib/postjob/migrations/004_tokens.sql
+- lib/postjob/migrations/005_helpers.sql
+- lib/postjob/migrations/006_enqueue.sql
+- lib/postjob/migrations/006a_processing.sql
+- lib/postjob/migrations/007_job_results.sql
+- lib/postjob/migrations/008_checkout_runnable.sql
+- lib/postjob/migrations/008a_childjobs.sql
+- lib/postjob/migrations/009_tokens.sql
 - lib/postjob/queue.rb
 - lib/postjob/queue/encoder.rb
 - lib/postjob/queue/notifications.rb
@@ -232,7 +244,7 @@ files:
 - spec/postjob/queue/search_spec.rb
 - spec/postjob/run_spec.rb
 - spec/postjob/step_spec.rb
-- spec/postjob/sub_workflow_spec.rb
+- spec/postjob/workflows/child_workflow_spec.rb
 - spec/spec_helper.rb
 - spec/support/configure_active_record.rb
 - spec/support/configure_database.rb