postjob 0.5.9 → 0.5.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e96734197915c9658e20b670186347fa9112a95a
-  data.tar.gz: a607352a5f6675c9bb6ae80fd0f289f7142692d6
+  metadata.gz: 3a27daa3a8a98739c5b78fe5b74cab7fb9f8d16b
+  data.tar.gz: 41eab0fe1314594cbeaaa06b364e476855f7839c
 SHA512:
-  metadata.gz: f1b8e20e7893ef1fcd3a1d53466714bcf5c106e1dec67a0745f25f29db04185c92bb7ac6e9cdf83dce299bea3f6f20cd3f6aac27c1c36551862157d902d4b1ba
-  data.tar.gz: 0c0fe39302fa99ff9fd8fe03f70a2c81a950276cc0e6b4d86c95b96657e917da3289f176d71273d71736e2c47932aa265e9616a5b37e1df2a87d03d6fffe699d
+  metadata.gz: f89717293ad0920aef170e08a9fbca783e95d509c47153c2e217f9d13f445eb49725b84d349aa1e868858f4a53584f1247e3b0934ceec88ec63ff1e0e37c5f22
+  data.tar.gz: c600c26793d24e26f51626b0de1e7c741fe0cb11db3f85562f6308ae2184738cd82bdef7d9b8a0bdcb9841987f1b1186c37fddac89a7cc343de0adf5239f1688

data/lib/postjob/cli/enqueue.rb

@@ -6,7 +6,7 @@ module Postjob::CLI
   #
   # Note that the workflow will receive the arguments as strings and must be
   # prepared to handle these.
-  def enqueue(workflow, *args, queue: "ruby", tags: nil)
+  def enqueue(workflow, *args, queue: nil, tags: nil)
     connect_to_database!
 
     Postjob.enqueue! workflow, *args, queue: queue, tags: parse_tags(tags)

data/lib/postjob/cli/ps.rb

@@ -6,13 +6,14 @@
 module Postjob::CLI
   private
 
-  def ps_query(tags:, limit:)
+  def ps_query(tags:, limit:, queue:)
     limit = Integer(limit)
 
     sql = <<-SQL
       SELECT
         postjobs.id,
         full_id,
+        postjobs.queue,
         workflow
           || (CASE WHEN workflow_version != '' THEN '@' ELSE '' END)
           || workflow_version
@@ -27,7 +28,7 @@ module Postjob::CLI
         max_attempts,
         cron_interval AS cron,
         CASE
-          WHEN is_sticky THEN COALESCE(sticky_host_id::varchar, 'yes')
+          WHEN is_sticky THEN COALESCE(substring(sticky_host_id::varchar for 6) || '...', 'yes')
           ELSE 'no'
           END AS sticky,
         next_run_at,
@@ -40,6 +41,7 @@ module Postjob::CLI
 
     scope = Simple::SQL::Scope.new(sql)
     scope = scope.where("tags @> ?", Postjob::Queue::Encoder.encode(parse_tags(tags))) if tags
+    scope = scope.where(queue: queue) if queue
 
     scope
       .paginate(per: limit, page: 1)
@@ -59,9 +61,10 @@ module Postjob::CLI
   #
   # For a listing of all jobs in the system use ps:full, see 'postjob help ps:full'
   # for details.
-  def ps(*ids, limit: "100", tags: nil)
+  def ps(*ids, limit: "100", tags: nil, queue: nil)
     expect! limit => /\A\d+\z/
     limit = Integer(limit)
+    queue = queue.split(",") if queue
 
     connect_to_database!
 
@@ -73,16 +76,18 @@ module Postjob::CLI
       return
     end
 
-    query = ps_query(tags: tags, limit: limit)
-    query = query.where("root_id=id OR status NOT IN ('ready', 'sleep', 'ok') OR failed_attempts > 0")
+    query = ps_query(tags: tags, limit: limit, queue: queue)
+    query = query.where("root_id=id OR status NOT IN ('sleep', 'ok') OR failed_attempts > 0")
 
     print_results query: query
   end
 
-  def ps_full(*ids, limit: "100", tags: nil)
+  def ps_full(*ids, limit: "100", tags: nil, queue: nil)
+    queue = queue.split(",") if queue
+
     connect_to_database!
 
-    query = ps_query(tags: tags, limit: limit)
+    query = ps_query(tags: tags, limit: limit, queue: queue)
 
     unless ids.empty?
       parsed_ids = parse_ids(*ids)
@@ -93,13 +98,13 @@ module Postjob::CLI
   end
 
   # Show up-to-date information once per second
-  def ps_top(*ids, limit: "100", tags: nil, full: false)
+  def ps_top(*ids, limit: "100", tags: nil, full: false, queue: nil)
     loop do
       system "clear"
       if full
-        ps_full(*ids, limit: limit, tags: tags)
+        ps_full(*ids, limit: limit, tags: tags, queue: queue)
       else
-        ps(*ids, limit: limit, tags: tags)
+        ps(*ids, limit: limit, tags: tags, queue: queue)
       end
       sleep 1
     end

data/lib/postjob/cli/run.rb

@@ -11,16 +11,18 @@ module Postjob::CLI
   # Parameters:
   #
   # - count=<count> maximum number of jobs to process. Default: unlimited.
+  # - queue run only the specified queue.
   # - quiet don't show progress.
   #
-  def run(count: nil, quiet: false)
+  def run(count: nil, queue: nil, quiet: false)
     count = Integer(count) if count
+    queue = queue.split(",") if queue
 
     connect_to_database!
 
     logger.success "Starting runner with pid #{$$}"
 
-    processed = Postjob.run(count: count) do |job_id|
+    processed = Postjob.run(count: count, queue: queue) do |job_id|
       logger.info "Processed job w/id #{job_id}" if job_id
       STDERR.print "." unless quiet
     end

data/lib/postjob/cli.rb

@@ -5,6 +5,8 @@ Dir.glob("#{File.dirname(__FILE__)}/cli/**/*.rb").sort.each do |path|
   load(path)
 end
 
+load "config/postjob_cli.rb" if File.exist?("config/postjob_cli.rb")
+
 module Postjob::CLI
   include ::Simple::CLI
 

data/lib/postjob/job.rb

@@ -32,6 +32,8 @@ class Postjob::Job < Postjob::Record
   attr_reader :timed_out
   attr_reader :tags
   attr_reader :last_worker_session_id
+  attr_reader :is_sticky
+  attr_reader :sticky_host_id
 
   STATUSES = %w(ok ready processing sleep err failed timeout)
 

data/lib/postjob/migrations/013a_checkout_runnable.sql

@@ -1,5 +1,6 @@
 DROP FUNCTION IF EXISTS {SCHEMA_NAME}.time_to_next_job(workflows_with_versions varchar[]);  -- removed in 0.5.0
-CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.time_to_next_job(p_worker_session_id UUID)
+DROP FUNCTION IF EXISTS {SCHEMA_NAME}.time_to_next_job(p_worker_session_id UUID);           -- removed in 0.5.7
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.time_to_next_job(p_worker_session_id UUID, p_queue varchar[])
   RETURNS float
 AS $$
 DECLARE
@@ -17,6 +18,7 @@ BEGIN
       SELECT MIN(next_run_at) AS processable_at
       FROM {SCHEMA_NAME}.postjobs
       WHERE status IN ('ready', 'err')
+        AND (p_queue IS NULL OR queue = ANY (p_queue))
         AND (workflow || workflow_version) = ANY (session.workflows)
         AND COALESCE(sticky_host_id, {SCHEMA_NAME}._null_uuid()) IN (session.host_id, {SCHEMA_NAME}._null_uuid())
   ) sq;
@@ -26,7 +28,12 @@ END;
 $$ LANGUAGE plpgsql;
 
 DROP FUNCTION IF EXISTS {SCHEMA_NAME}.checkout(workflows_with_versions varchar[], p_fast_mode BOOLEAN);  -- removed in 0.5.0
-CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.checkout(p_worker_session_id UUID, p_fast_mode BOOLEAN)
+DROP FUNCTION IF EXISTS {SCHEMA_NAME}.checkout(p_worker_session_id UUID, p_fast_mode BOOLEAN);           -- removed in 0.5.8
+
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.checkout(
+    p_worker_session_id UUID,
+    p_fast_mode BOOLEAN,
+    p_queue varchar[])
   RETURNS SETOF {SCHEMA_NAME}.postjobs
 AS $$
 DECLARE
@@ -52,8 +59,9 @@ BEGIN
         AND s.next_run_at <= (now() at time zone 'utc')
         AND (s.workflow || s.workflow_version) = ANY (session.workflows)
         AND COALESCE(s.sticky_host_id, {SCHEMA_NAME}._null_uuid()) IN (session.host_id, {SCHEMA_NAME}._null_uuid())
+        AND (p_queue IS NULL OR queue = ANY (p_queue))
       )
-    ORDER BY (LEAST(s.next_run_at, s.timing_out_at))
+    ORDER BY (LEAST(CASE WHEN s.sticky_host_id IS NOT NULL THEN s.next_run_at - interval '180 seconds' ELSE s.next_run_at END, s.timing_out_at))
     FOR UPDATE SKIP LOCKED
     LIMIT 1;
 

data/lib/postjob/queue/notifications.rb

@@ -8,14 +8,14 @@ module Postjob::Queue::Notifications
   SCHEMA_NAME = ::Postjob::Queue::SCHEMA_NAME
   MAX_WAIT_TIME = 120
 
-  def wait_for_new_job(worker_session_id)
+  def wait_for_new_job(worker_session_id, queue:)
     started_at = Time.now
 
     start_listening
 
     # Determine when the next job is up. If we don't have a next job within MAX_WAIT_TIME
     # we wake up regardless.
-    wait_time = time_to_next_job(worker_session_id)
+    wait_time = time_to_next_job(worker_session_id, queue: queue)
     return if wait_time && wait_time <= 0
 
     wait_time = MAX_WAIT_TIME if !wait_time || wait_time > MAX_WAIT_TIME
@@ -43,8 +43,11 @@ module Postjob::Queue::Notifications
 
   # returns the maximum number of seconds to wait until the
   # next runnable or timeoutable job comes up.
-  def time_to_next_job(worker_session_id)
+  def time_to_next_job(worker_session_id, queue:)
     expect! worker_session_id => /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i
-    Simple::SQL.ask "SELECT * FROM #{SCHEMA_NAME}.time_to_next_job($1::uuid)", worker_session_id
+    expect! queue => [ String, Array, nil ]
+
+    queue = Array(queue)
+    Simple::SQL.ask "SELECT * FROM #{SCHEMA_NAME}.time_to_next_job($1::uuid, $2)", worker_session_id, queue
   end
 end

data/lib/postjob/queue.rb

@@ -157,9 +157,14 @@ module Postjob::Queue
 
   UUID_REGEXP = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i
 
-  def checkout(worker_session_id)
+  def checkout(worker_session_id, queue:)
     expect! worker_session_id => UUID_REGEXP
-    SQL.ask "SELECT * FROM #{SCHEMA_NAME}.checkout($1::uuid, $2::boolean)", worker_session_id, Postjob.fast_mode, into: Job
+    expect! queue => [ nil, Array, String ]
+
+    queue = Array(queue) if queue
+
+    SQL.ask "SELECT * FROM #{SCHEMA_NAME}.checkout($1::uuid, $2::boolean, $3)",
+      worker_session_id, Postjob.fast_mode, queue, into: Job
   end
 
   def find_or_create_token(job)

data/lib/postjob/registry.rb

@@ -42,7 +42,7 @@ class Postjob::Registry
         timeout: 15 * 60,
         sticky: false,
         cron_interval: nil,
-        queue: "q"
+        queue: "ruby"
       }
 
       attr_reader :version

data/lib/postjob/runner.rb

@@ -35,9 +35,11 @@ module Postjob::Runner
 
     # returns a subjob within the current job, for a +runner+
     # description and +args+.
-    def async(workflow, *args, timeout: nil, max_attempts:)
+    def async(workflow, *args, timeout: nil, max_attempts: nil, queue: nil)
       worker_session_id = Postjob.current_worker_session.id
 
+      queue = current_job.queue if queue.nil?
+
       # if the workflow is a symbol, then we change it into "__manual__"
       # - there should never be a workflow with that name - or into
       # "CurrentWorkshop.#{workflow}", denoting the \a workflow method of the
@@ -52,12 +54,13 @@ module Postjob::Runner
       end
 
       ::Postjob::Queue.find_or_create_childjob(worker_session_id, self.current_job, workflow, args,
+                                               queue: queue,
                                                timeout: timeout,
                                                max_attempts: max_attempts)
     end
 
     # tries to resolve a job.
-    def await(job, *args, timeout: nil, max_attempts: nil)
+    def await(job, *args, timeout: nil, max_attempts: nil, queue: nil)
       case job
       when :all
         unresolved_childjobs = Postjob::Queue.unresolved_childjobs(current_job)
@@ -75,7 +78,7 @@ module Postjob::Runner
         throw :pending, :pending if r == :pending
         r
       else
-        job = async(job, *args, timeout: timeout, max_attempts: max_attempts)
+        job = async(job, *args, timeout: timeout, max_attempts: max_attempts, queue: queue)
         await(job)
       end
     end

data/lib/postjob/workflow.rb

@@ -15,8 +15,8 @@ module Postjob::Workflow
   # we cannot run such a test.
 
   # see Postjob::Runner.async
-  def async(runner, *args, timeout: nil, max_attempts: nil)
-    ::Postjob::Runner.async(runner, *args, timeout: timeout, max_attempts: max_attempts)
+  def async(runner, *args, timeout: nil, max_attempts: nil, queue: nil)
+    ::Postjob::Runner.async(runner, *args, timeout: timeout, max_attempts: max_attempts, queue: queue)
   end
 
   # see Postjob::Runner.await

data/lib/postjob.rb

@@ -49,6 +49,8 @@ module Postjob
   # - cron_interval
   # - queue
   #
+  #
+  # Returns a job id
   def enqueue!(workflow, *args, queue: nil,
                parent_id: nil,
                max_attempts: nil,
@@ -156,7 +158,7 @@ module Postjob
   # b) the block yielded into returns false.
   #
   # This method returns the number of processed jobs.
-  def run(count: nil, &block)
+  def run(count: nil, queue: nil, &block)
     # to run 10^12 jobs that would take 1 msecs each we would need, at least,
     # 760 years - so this default should be fine. Also, someone should update
     # the machine in the meantime :)
@@ -165,7 +167,7 @@ module Postjob
     processed_jobs_count = 0
 
     loop do
-      processed_job_id, shutdown = Postjob.step
+      processed_job_id, shutdown = Postjob.step(queue: queue)
       processed_jobs_count += 1 if processed_job_id
 
       break if processed_jobs_count >= count
@@ -173,7 +175,7 @@ module Postjob
       break if shutdown == :shutdown
 
       next if processed_job_id
-      Queue::Notifications.wait_for_new_job(current_worker_session.id)
+      Queue::Notifications.wait_for_new_job(current_worker_session.id, queue: queue)
     end
 
     processed_jobs_count
@@ -191,8 +193,9 @@ module Postjob
   #   self.run to terminate the run loop.
   #
   # or nil, when no job could be checked out.
-  def step
-    job = Postjob::Queue.checkout(current_worker_session.id)
+  def step(queue: nil)
+    job = Postjob::Queue.checkout(current_worker_session.id, queue: queue)
+
     [ job.id, process_job(job) ] if job
   end
 
@@ -251,6 +254,7 @@ module Postjob
   def process_job(job) # :nodoc:
     expect! job => Job
 
+    #    ap job
     version, status, value, shutdown = Runner.process_job(job)
 
     expect! version => String
@@ -268,6 +272,23 @@ module Postjob
     else            raise ArgumentError, "Invalid status #{status.inspect}"
     end
 
+    if (status == :ok || status == :failed) && job.workflow_method == "run"
+      Simple::SQL.transaction do
+        job_id = enqueue! "#{job.workflow}.cleanup", *job.args, queue: job.queue,
+                                                                parent_id: nil,
+                                                                max_attempts: nil,
+                                                                timeout: nil,
+                                                                version: version,
+                                                                tags: job.tags,
+                                                                cron_interval: nil,
+                                                                sticky: job.is_sticky
+
+        if job.is_sticky
+          Simple::SQL.ask "UPDATE postjob.postjobs SET sticky_host_id=$1 WHERE id=$2", job.sticky_host_id, job_id
+        end
+      end
+    end
+
     shutdown
   end
 end

data/spec/postjob/enqueue_spec.rb

@@ -53,10 +53,10 @@ describe "Postjob.enqueue!" do
       expect(job2.args).to eq([])
     end
 
-    it "sets queue" do
+    it "sets the queue to the default queue name ('ruby')" do
       id1 = Postjob.enqueue! workflow_name
       job1 = load_job id1
-      expect(job1.queue).to eq("q")
+      expect(job1.queue).to eq("ruby")
 
       id1 = Postjob.enqueue! workflow_name, queue: "bla"
       job1 = load_job id1
@@ -82,8 +82,8 @@ describe "Postjob.enqueue!" do
 
   context "with invalid arguments" do
     it "raises an error with missing args" do
-      expect { Postjob.enqueue! queue: "q" }.to raise_error(ArgumentError)
-      expect { Postjob.enqueue! workflow: "q" }.to raise_error(ArgumentError)
+      expect { Postjob.enqueue! queue: "ruby" }.to raise_error(ArgumentError)
+      expect { Postjob.enqueue! workflow: "ruby" }.to raise_error(ArgumentError)
     end
 
     it "raises an error with invalid args" do

data/spec/postjob/run_spec.rb

@@ -23,7 +23,7 @@ describe "Postjob.process_all" do
   let!(:id) { Postjob.enqueue! "TwoLevelWorkflow" }
 
   it "runs the job returning the result" do
-    expect(Postjob.process_all).to eq(3)
+    expect(Postjob.process_all).to eq(5)
     processed_job = load_job(id)
     expect(processed_job.status).to eq("ok")
     expect(processed_job.resolve).to eq("my string")

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: postjob
 version: !ruby/object:Gem::Version
-  version: 0.5.9
+  version: 0.5.10
 platform: ruby
 authors:
 - radiospiel