postjob 0.5.10 → 0.5.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3a27daa3a8a98739c5b78fe5b74cab7fb9f8d16b
-  data.tar.gz: 41eab0fe1314594cbeaaa06b364e476855f7839c
+  metadata.gz: e26be7454c3a0ddb3d26d67b4e51918e8e16e306
+  data.tar.gz: f837746f80bba5cebbdd0fea7c7ad7c26eacbc5b
 SHA512:
-  metadata.gz: f89717293ad0920aef170e08a9fbca783e95d509c47153c2e217f9d13f445eb49725b84d349aa1e868858f4a53584f1247e3b0934ceec88ec63ff1e0e37c5f22
-  data.tar.gz: c600c26793d24e26f51626b0de1e7c741fe0cb11db3f85562f6308ae2184738cd82bdef7d9b8a0bdcb9841987f1b1186c37fddac89a7cc343de0adf5239f1688
+  metadata.gz: 361995efdeda5d1ca51de93c0807e37737581c32f53c25f20dc45c50c65dc008260abccc0d421776d59186997071fd2aa14cc23828f95145de6536bcf7859589
+  data.tar.gz: f374453e87325c9ddb05d4b1cad18f20d26b3a6c64a596de23ab471bc6bbc57017b20df5c589944e1cc90994ad717e4d1e57b7fa8ec6ab953ef48e6ec0a33adc

data/lib/postjob.rb

@@ -88,6 +88,7 @@ module Postjob
       sticky          = spec.options.sticky if sticky.nil?
       cron_interval   = spec.options.cron_interval if cron_interval.nil?
       queue           = spec.options.queue if queue.nil?
+      greedy          = spec.options.greedy
     end
 
     if cron_interval && cron_interval < 300
@@ -111,7 +112,8 @@ module Postjob
                                                                         tags: tags,
                                                                         version: version,
                                                                         cron_interval: cron_interval,
-                                                                        sticky: sticky
+                                                                        sticky: sticky,
+                                                                        greedy: greedy
 
     logger.info "Generated process #{job}"
     job.id
@@ -272,23 +274,30 @@ module Postjob
     else            raise ArgumentError, "Invalid status #{status.inspect}"
     end
 
-    if (status == :ok || status == :failed) && job.workflow_method == "run"
-      Simple::SQL.transaction do
-        job_id = enqueue! "#{job.workflow}.cleanup", *job.args, queue: job.queue,
-                                                                parent_id: nil,
-                                                                max_attempts: nil,
-                                                                timeout: nil,
-                                                                version: version,
-                                                                tags: job.tags,
-                                                                cron_interval: nil,
-                                                                sticky: job.is_sticky
-
-        if job.is_sticky
-          Simple::SQL.ask "UPDATE postjob.postjobs SET sticky_host_id=$1 WHERE id=$2", job.sticky_host_id, job_id
-        end
-      end
-    end
+    enqueue_cleanup(job) if (status == :ok || status == :failed) && job.workflow_method == "run"
 
     shutdown
   end
+
+  def enqueue_cleanup(job)
+    spec = Postjob::Registry.lookup!(name: job.workflow, version: job.workflow_version)
+    return unless spec.supports_cleanup?
+
+    Postjob.logger.info "Enqueuing cleanup job #{spec.name}.cleanup(#{job.args.map(&:inspect).join(", ")})"
+
+    Simple::SQL.transaction do
+      job_id = enqueue! "#{job.workflow}.cleanup", *job.args, queue: job.queue,
+                                                              parent_id: nil,
+                                                              max_attempts: nil,
+                                                              timeout: nil,
+                                                              version: job.workflow_version,
+                                                              tags: job.tags,
+                                                              cron_interval: nil,
+                                                              sticky: job.is_sticky
+
+      if job.is_sticky
+        Simple::SQL.ask "UPDATE postjob.postjobs SET sticky_host_id=$1 WHERE id=$2", job.sticky_host_id, job_id
+      end
+    end
+  end
 end

data/lib/postjob/cli/enqueue.rb

@@ -6,9 +6,13 @@ module Postjob::CLI
   #
   # Note that the workflow will receive the arguments as strings and must be
   # prepared to handle these.
-  def enqueue(workflow, *args, queue: nil, tags: nil)
+  def enqueue(workflow, *args, queue: nil, tags: nil, count: "1")
+    count = Integer(count)
+
     connect_to_database!
 
-    Postjob.enqueue! workflow, *args, queue: queue, tags: parse_tags(tags)
+    count.times do
+      Postjob.enqueue! workflow, *args, queue: queue, tags: parse_tags(tags)
+    end
   end
 end

data/lib/postjob/cli/ps.rb

@@ -2,11 +2,12 @@
 # rubocop:disable Metrics/MethodLength
 # rubocop:disable Metrics/ModuleLength
 # rubocop:disable Metrics/PerceivedComplexity
+# rubocop:disable Metrics/ParameterLists
 
 module Postjob::CLI
   private
 
-  def ps_query(tags:, limit:, queue:)
+  def ps_query(tags:, limit:, queue:, only_root: false)
     limit = Integer(limit)
 
     sql = <<-SQL
@@ -31,6 +32,7 @@ module Postjob::CLI
           WHEN is_sticky THEN COALESCE(substring(sticky_host_id::varchar for 6) || '...', 'yes')
           ELSE 'no'
           END AS sticky,
+        is_greedy AS greedy,
         next_run_at,
         to_char(EXTRACT(EPOCH FROM (next_run_at - now() at time zone 'utc')), '999999999.99') AS next_run_in,
         to_char(EXTRACT(EPOCH FROM (now() at time zone 'utc' - postjobs.created_at)), '999999999.99') AS age,
@@ -42,6 +44,7 @@ module Postjob::CLI
     scope = Simple::SQL::Scope.new(sql)
     scope = scope.where("tags @> ?", Postjob::Queue::Encoder.encode(parse_tags(tags))) if tags
     scope = scope.where(queue: queue) if queue
+    scope = scope.where("id = root_id") if only_root
 
     scope
       .paginate(per: limit, page: 1)
@@ -61,7 +64,7 @@ module Postjob::CLI
   #
   # For a listing of all jobs in the system use ps:full, see 'postjob help ps:full'
   # for details.
-  def ps(*ids, limit: "100", tags: nil, queue: nil)
+  def ps(*ids, limit: "100", tags: nil, queue: nil, only_root: nil)
     expect! limit => /\A\d+\z/
     limit = Integer(limit)
     queue = queue.split(",") if queue
@@ -76,7 +79,7 @@ module Postjob::CLI
       return
     end
 
-    query = ps_query(tags: tags, limit: limit, queue: queue)
+    query = ps_query(tags: tags, limit: limit, queue: queue, only_root: only_root)
     query = query.where("root_id=id OR status NOT IN ('sleep', 'ok') OR failed_attempts > 0")
 
     print_results query: query
@@ -98,13 +101,13 @@ module Postjob::CLI
   end
 
   # Show up-to-date information once per second
-  def ps_top(*ids, limit: "100", tags: nil, full: false, queue: nil)
+  def ps_top(*ids, limit: "100", tags: nil, full: false, queue: nil, only_root: false)
     loop do
       system "clear"
       if full
         ps_full(*ids, limit: limit, tags: tags, queue: queue)
       else
-        ps(*ids, limit: limit, tags: tags, queue: queue)
+        ps(*ids, limit: limit, tags: tags, queue: queue, only_root: only_root)
       end
       sleep 1
     end

data/lib/postjob/cli/run.rb

@@ -14,10 +14,12 @@ module Postjob::CLI
   # - queue run only the specified queue.
   # - quiet don't show progress.
   #
-  def run(count: nil, queue: nil, quiet: false)
+  def run(count: nil, queue: nil, quiet: false, fast: false)
     count = Integer(count) if count
     queue = queue.split(",") if queue
 
+    Postjob.fast_mode = (fast ? true : false)
+
     connect_to_database!
 
     logger.success "Starting runner with pid #{$$}"

data/lib/postjob/migrations/003_postjobs.sql

@@ -63,6 +63,16 @@ CREATE TABLE IF NOT EXISTS {SCHEMA_NAME}.postjobs (
   -- processing_max_duration float            -- maximum expected duration of processing. Afterwards the
                                               -- processing is considered failed for unknown reasons, and
                                               -- potentially restarted.
+
+  -- process information ------------------------------------------------------------------------
+  --
+  -- Initially these columns didn't exist, and have been created via another migration.
+
+  -- last_worker_session_id uuid NOT NULL,      -- session_id of last worker that touched this job
+  -- cron_interval integer,                     -- for cron jobs: cron_interval in seconds
+  -- is_sticky boolean DEFAULT false NOT NULL,  -- when set this job is supposed to be sticky
+  -- sticky_host_id uuid                        -- once a sticky job starts running the job's host id
+  -- is_greedy boolean DEFAULT false NOT NULL   -- is this a greedy job? (i.e. sticky, and do not allow any other job in this worker.)
 );
 
 -- [TODO] check indices

data/lib/postjob/migrations/006_enqueue.sql

@@ -23,6 +23,20 @@ DROP FUNCTION IF EXISTS {SCHEMA_NAME}.enqueue(
     timeout DOUBLE PRECISION,
     p_cron_interval integer);
 
+DROP FUNCTION IF EXISTS {SCHEMA_NAME}.enqueue(
+    p_worker_session_id UUID,
+    queue VARCHAR,
+    workflow VARCHAR,
+    workflow_method VARCHAR,
+    workflow_version VARCHAR,
+    args JSONB,
+    parent_id BIGINT,
+    tags JSONB,
+    max_attempts INTEGER,
+    timeout DOUBLE PRECISION,
+    p_cron_interval INTEGER,
+    p_is_sticky BOOLEAN);
+
 CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.enqueue(
     p_worker_session_id UUID,
     queue VARCHAR,
@@ -35,7 +49,8 @@ CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.enqueue(
     max_attempts INTEGER,
     timeout DOUBLE PRECISION,
     p_cron_interval INTEGER,
-    p_is_sticky BOOLEAN)
+    p_is_sticky BOOLEAN,
+    p_is_greedy BOOLEAN)
   RETURNS SETOF {SCHEMA_NAME}.postjobs
 AS $$
   DECLARE
@@ -62,7 +77,8 @@ AS $$
     workflow_version  := COALESCE(workflow_version, '');
     queue             := COALESCE(queue, 'q');
     max_attempts      := COALESCE(max_attempts, 5);
-    p_is_sticky       := COALESCE(p_is_sticky, FALSE);
+    p_is_greedy       := COALESCE(p_is_greedy, FALSE);
+    p_is_sticky       := p_is_greedy OR COALESCE(p_is_sticky, FALSE);
 
     -- create postjobs entry ------------------------------------------
     INSERT INTO {SCHEMA_NAME}.postjobs (
@@ -71,7 +87,8 @@ AS $$
       parent_id, tags, max_attempts,
       timing_out_at,
       cron_interval,
-      is_sticky
+      is_sticky,
+      is_greedy
     )
     VALUES(
       p_worker_session_id,
@@ -79,7 +96,8 @@ AS $$
       parent_id, tags, max_attempts,
       (now() at time zone 'utc') + timeout * interval '1 second',
       p_cron_interval,
-      p_is_sticky
+      p_is_sticky,
+      p_is_greedy
     ) RETURNING {SCHEMA_NAME}.postjobs.id INTO job_id;
 
     -- fill in root_id and full_id ------------------------------------

data/lib/postjob/migrations/008a_childjobs.sql

@@ -72,10 +72,11 @@ BEGIN
       v_args,                           -- args JSONB,
       v_parent_id,                      -- parent_id BIGINT,
       parent.tags,                      -- tags JSONB,
-      v_max_attempts,                   -- max_attempts INTEGER,
+      COALESCE(v_max_attempts, parent.max_attempts), -- max_attempts INTEGER,
       v_timeout,                        -- timeout
       NULL,                             -- cron_interval,
-      parent.is_sticky                  -- is_sticky?
+      parent.is_sticky,                 -- is_sticky?,
+      parent.is_greedy                  -- is_greedy?,
     );
 
     UPDATE {SCHEMA_NAME}.postjobs

data/lib/postjob/migrations/013a_checkout_runnable.sql

@@ -6,10 +6,18 @@ AS $$
 DECLARE
   p_processable_at timestamp;
   session {SCHEMA_NAME}.worker_sessions;
+  p_current_greedy_job {SCHEMA_NAME}.postjobs;
 BEGIN
   SELECT * INTO session
   FROM {SCHEMA_NAME}.worker_sessions WHERE id=p_worker_session_id;
 
+  SELECT * INTO p_current_greedy_job
+  FROM {SCHEMA_NAME}.postjobs WHERE
+    status NOT IN ('ok', 'failed') AND
+    id=root_id AND
+    is_greedy
+  LIMIT 1;
+
   SELECT MIN(processable_at) INTO p_processable_at FROM (
       SELECT MIN(timing_out_at) AS processable_at
       FROM {SCHEMA_NAME}.postjobs
@@ -20,7 +28,12 @@ BEGIN
       WHERE status IN ('ready', 'err')
         AND (p_queue IS NULL OR queue = ANY (p_queue))
         AND (workflow || workflow_version) = ANY (session.workflows)
-        AND COALESCE(sticky_host_id, {SCHEMA_NAME}._null_uuid()) IN (session.host_id, {SCHEMA_NAME}._null_uuid())
+        AND COALESCE(sticky_host_id, {SCHEMA_NAME}._null_uuid()) IN (session.host_id, {SCHEMA_NAME}._null_uuid()) -- matches sticky_host_id for started sticky jobs and _null_uuid for 
+                                                                                                                  -- non-started or non-sticky jobs against the host_id and _null_uuid
+        AND (
+          p_current_greedy_job.id IS NULL OR root_id=p_current_greedy_job.root_id                                 -- if there is a greedy job on this host_id which is not finished yet,
+                                                                                                                  -- only jobs belonging to this root jobs are allowed.
+        )
   ) sq;
 
   RETURN EXTRACT(EPOCH FROM p_processable_at - (now() at time zone 'utc'));
@@ -39,10 +52,28 @@ AS $$
 DECLARE
   job {SCHEMA_NAME}.postjobs;
   session {SCHEMA_NAME}.worker_sessions;
+  p_current_greedy_job {SCHEMA_NAME}.postjobs;
 BEGIN
   SELECT * INTO session
   FROM {SCHEMA_NAME}.worker_sessions WHERE id=p_worker_session_id;
 
+  --
+  -- We don't want multiple sessions to run this function in parallel. This can lead to a situation
+  -- where multiple greedy root jobs could be selected for different workers with identical host ids
+  -- at the same time. We therefore lock the function here against parallel usage - and we use the
+  -- hosts table for locking. This look will be released automatically with the current transaction,
+  -- i.e. typically after the "SELECT * FROM checkout(..)" returns.
+  --
+  PERFORM * FROM {SCHEMA_NAME}.hosts WHERE id=session.host_id FOR UPDATE;
+
+  SELECT * INTO p_current_greedy_job
+  FROM {SCHEMA_NAME}.postjobs WHERE
+    status NOT IN ('ok', 'failed')
+    AND id=root_id
+    AND sticky_host_id=session.host_id
+    AND is_greedy
+  LIMIT 1;
+
   LOOP
     -- try to checkout a job. Each of the conditions here is matching
     -- one of the CASE .. WHEN clauses below.
@@ -56,12 +87,16 @@ BEGIN
       OR
       (
         s.status IN ('ready', 'err')
+        AND (p_queue IS NULL OR queue = ANY (p_queue))
         AND s.next_run_at <= (now() at time zone 'utc')
         AND (s.workflow || s.workflow_version) = ANY (session.workflows)
         AND COALESCE(s.sticky_host_id, {SCHEMA_NAME}._null_uuid()) IN (session.host_id, {SCHEMA_NAME}._null_uuid())
-        AND (p_queue IS NULL OR queue = ANY (p_queue))
+        AND (
+          p_current_greedy_job.id IS NULL OR s.root_id=p_current_greedy_job.root_id                               -- if there is a greedy job on this host_id which is not finished yet,
+                                                                                                                  -- only jobs belonging to this root jobs are allowed.
+        )
       )
-    ORDER BY (LEAST(CASE WHEN s.sticky_host_id IS NOT NULL THEN s.next_run_at - interval '180 seconds' ELSE s.next_run_at END, s.timing_out_at))
+    ORDER BY (LEAST(s.next_run_at, s.timing_out_at))
     FOR UPDATE SKIP LOCKED
     LIMIT 1;
 

data/lib/postjob/migrations/023_sticky_jobs.sql

@@ -14,3 +14,11 @@ DO $$
     WHEN duplicate_column THEN RAISE DEBUG 'column {SCHEMA_NAME}.postjobs.sticky_host_id already exists';
   END;
 $$;
+
+DO $$
+  BEGIN
+    ALTER TABLE {SCHEMA_NAME}.postjobs ADD COLUMN is_greedy BOOLEAN NOT NULL DEFAULT FALSE;
+  EXCEPTION
+    WHEN duplicate_column THEN RAISE DEBUG 'column {SCHEMA_NAME}.postjobs.is_greedy already exists';
+  END;
+$$;

data/lib/postjob/queue.rb

@@ -43,17 +43,23 @@ module Postjob::Queue
       tags: [Hash, nil],
       timeout: [Numeric, nil],
       max_attempts: [Integer, nil],
-      sticky: [true, false, nil]
+      sticky: [true, false, nil],
+      greedy: [true, false, nil]
     }
 
     workflow, workflow_method = parse_workflow(workflow)
 
+    if options[:greedy] && !options[:sticky]
+      raise ArgumentError, "#{workflow}: A greedy job must also be sticky" unless options[:sticky].nil?
+      options[:sticky] = true if options[:greedy]
+    end
+
     # The use of a `SELECT * FROM function()` here is due to
     #
     # a) a limitation in Simple::SQL which would not be able to unpack a
     #    "SELECT function()" usefully when the return value is a record;
     # b) and/or my inability to write better SQL functions;
-    SQL.ask "SELECT * FROM #{SCHEMA_NAME}.enqueue($1::uuid, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)",
+    SQL.ask "SELECT * FROM #{SCHEMA_NAME}.enqueue($1::uuid, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)",
                   worker_session_id,
                   options[:queue],
                   workflow,
@@ -66,6 +72,7 @@ module Postjob::Queue
                   options[:timeout],
                   options[:cron_interval],
                   options[:sticky],
+                  options[:greedy],
                   into: Job
   end
 
@@ -162,7 +169,6 @@ module Postjob::Queue
     expect! queue => [ nil, Array, String ]
 
     queue = Array(queue) if queue
-
     SQL.ask "SELECT * FROM #{SCHEMA_NAME}.checkout($1::uuid, $2::boolean, $3)",
       worker_session_id, Postjob.fast_mode, queue, into: Job
   end

data/lib/postjob/registry.rb

@@ -41,6 +41,7 @@ class Postjob::Registry
         max_attempts: 5,
         timeout: 15 * 60,
         sticky: false,
+        greedy: false,
         cron_interval: nil,
         queue: "ruby"
       }
@@ -49,6 +50,7 @@ class Postjob::Registry
       attr_reader :max_attempts
       attr_reader :timeout
       attr_reader :sticky
+      attr_reader :greedy
       attr_reader :cron_interval
       attr_reader :queue
 
@@ -64,10 +66,13 @@ class Postjob::Registry
 
         options = DEFAULTS.merge(options)
 
+        options[:sticky] ||= options[:greedy]
+
         @version        = options[:version]
         @max_attempts   = options[:max_attempts]
         @timeout        = options[:timeout]
         @sticky         = options[:sticky]
+        @greedy         = options[:greedy]
         @cron_interval  = options[:cron_interval]
         @queue          = options[:queue]
       end
@@ -78,6 +83,7 @@ class Postjob::Registry
         r[:max_attempts]    = @max_attempts
         r[:timeout]         = @timeout
         r[:sticky]          = @sticky
+        r[:greedy]          = @greedy
         r[:cron_interval]   = @cron_interval
         r[:queue]           = @queue
 
@@ -109,6 +115,22 @@ class Postjob::Registry
       @workflow = workflow
       @options  = Options.new(options)
     end
+
+    def supports_cleanup?
+      return false unless lookup_workflow_method(:run)
+      return false unless lookup_workflow_method(:cleanup)
+
+      true
+    end
+
+    private
+
+    def lookup_workflow_method(name)
+      return unless workflow.is_a?(Module)
+      workflow.method(name)
+    rescue NameError
+      nil
+    end
   end
 
   # looks up a specific version of a specific workflow. Returns a WorkflowSpec
@@ -132,6 +154,10 @@ class Postjob::Registry
   end
 
   def register(workflow, options)
+    if options[:greedy] && options[:sticky] == false
+      raise ArgumentError, "#{workflow}: a greedy job must also be sticky"
+    end
+
     spec = WorkflowSpec.new(workflow, options)
 
     @workflows_with_versions << spec.name << "#{spec.name}#{spec.options.version}"

data/lib/postjob/runner.rb

@@ -2,6 +2,7 @@
 # rubocop:disable Style/BlockDelimiters
 # rubocop:disable Lint/RescueException
 # rubocop:disable Metrics/MethodLength
+# rubocop:disable Metrics/AbcSize
 
 # Base implementations for Runners
 #
@@ -155,6 +156,10 @@ module Postjob::Runner
       return_exception :failed, $!
     rescue Postjob::Error::Nonrecoverable
       return_exception :failed, $!
+    rescue PG::Error
+      Postjob.logger.error "#{$!}, from\n\t#{$!.backtrace[0, 10].join("\n\t")}"
+      STDERR.puts "#{$!}, from\n\t#{$!.backtrace[0, 10].join("\n\t")}"
+      return_exception :failed, $!
     rescue Exception
       return_exception :err, $!
     end

data/spec/postjob/run_spec.rb

@@ -23,7 +23,7 @@ describe "Postjob.process_all" do
   let!(:id) { Postjob.enqueue! "TwoLevelWorkflow" }
 
   it "runs the job returning the result" do
-    expect(Postjob.process_all).to eq(5)
+    expect(Postjob.process_all).to eq(3)
     processed_job = load_job(id)
     expect(processed_job.status).to eq("ok")
     expect(processed_job.resolve).to eq("my string")

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: postjob
 version: !ruby/object:Gem::Version
-  version: 0.5.10
+  version: 0.5.11
 platform: ruby
 authors:
 - radiospiel
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-08-15 00:00:00.000000000 Z
+date: 2018-08-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec