postjob 0.2.2 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b1ab40be9d1e5d2a5dce344ed912d837bf3ae567
-  data.tar.gz: 09ee97d4ce95a137525d8efee7e0f7c24a21cb95
+  metadata.gz: 586b34b133d67462d64f913cf678ae14ef7954b4
+  data.tar.gz: 4ecb46ab647b4337c56af775110d64cc16a36cb5
 SHA512:
-  metadata.gz: 2930f208eb34cda94e2570a823f4d32132e6ba58c078a24d0f0803c7bbccf8068b51d3832779a1e1b7eb4c40c0034f1e951970e847bcd1f73806582f3031c735
-  data.tar.gz: 15ee51d44d1341b538808e4cb875a1c98cb518c34e68b355ef678860de8d1737ee20197fe49dd627d84b4b8803b928714df9a73e60b0a0074548cc930db4d1db
+  metadata.gz: a0a946eed383baa9e3c23a0bacc0a52175d917f7f976d6dc6f10b1ffe7e87ab8d3da7add93145f426f2e637613c3a0417dfd5a11404e6ac227d57a765e26a08b
+  data.tar.gz: 132c173dc2da7630433511078c8eb7b75766fa00da4d3e52bb88e8a05c84254f1ae27490e7d3dc9ab762af962d6f2e813d42440d1f6e204cd1305e2e16225f48

data/lib/postjob/cli/db.rb

@@ -48,6 +48,9 @@ module Postjob::CLI
   USE_ACTIVE_RECORD = false
 
   def connect_to_database!
+    return if @connected_to_database
+
+    @connected_to_database = true
     if USE_ACTIVE_RECORD
       require "active_record"
       abc = ::Simple::SQL::Config.read_database_yml

data/lib/postjob/cli/job.rb

@@ -1,5 +1,3 @@
-# rubocop:disable Metrics/MethodLength
-
 module Postjob::CLI
   # Enqueues a workflow
   #
@@ -37,15 +35,11 @@ module Postjob::CLI
     Simple::SQL.ask <<~SQL, job_ids
       UPDATE postjob.postjobs
       SET
-        status='ready', next_run_at=now(),
+        status='ready', next_run_at=(now() at time zone 'utc'),
         results=null, failed_attempts=0, error=NULL, error_message=NULL, error_backtrace=NULL
       WHERE id = ANY($1);
     SQL
 
-    Simple::SQL.ask <<~SQL
-      NOTIFY postjob_notifications
-    SQL
-
     logger.warn "The following jobs have been reset: #{job_ids.join(', ')}"
   end
 
@@ -66,6 +60,20 @@ module Postjob::CLI
     SQL
   end
 
+  # runs a job as soon as possible
+  def job_force(job_id)
+    connect_to_database!
+
+    job_id = Integer(job_id)
+
+    Simple::SQL.ask <<~SQL, job_id
+      UPDATE postjob.postjobs
+      SET
+        next_run_at=now() at time zone 'utc'
+      WHERE id = $1 AND status IN ('ready', 'err', 'sleep')
+    SQL
+  end
+
   private
 
   # parses "foo:bar,baz:quibble" into { "foo" => "bar", "baz" => "quibble"}

data/lib/postjob/cli/ps.rb

@@ -1,3 +1,6 @@
+# rubocop:disable Lint/HandleExceptions
+# rubocop:disable Metrics/ModuleLength
+
 module Postjob::CLI
   private
 
@@ -24,7 +27,16 @@ module Postjob::CLI
         error,
         COALESCE((results->0)::varchar, error_message) AS result,
         next_run_at,
-        (now() at time zone 'utc') - created_at AS age,
+        next_run_at - (now() at time zone 'utc') AS next_run_in,
+        to_char(EXTRACT(EPOCH FROM (now() at time zone 'utc') - created_at), '999999999.99') AS age,
+        CASE WHEN processing_started_at IS NOT NULL THEN
+        format(
+          '%s/%s',
+          to_char(EXTRACT(EPOCH FROM (now() at time zone 'utc') - processing_started_at), '999999999.99'),
+          processing_max_duration
+        )
+        END AS processing,
+        COALESCE(processing_client, '') || COALESCE('/' || processing_client_identifier, '') AS worker,
         tags
       FROM postjob.postjobs
       WHERE #{condition_fragment}
@@ -56,6 +68,11 @@ module Postjob::CLI
     expect! limit => /\A\d+\z/
     limit = Integer(limit)
 
+    connect_to_database!
+
+    # check for timed out and zombie processes
+    # ::Postjob::Queue.checkout(nil)
+
     unless ids.empty?
       ps_full(*ids, limit: limit, tags: tags)
       return
@@ -70,11 +87,20 @@ module Postjob::CLI
     print_sql limit: limit, query: query
   end
 
+  def ps_top(*ids, limit: "100", tags: nil)
+    loop do
+      system "clear"
+      ps(*ids, limit: limit, tags: tags)
+      sleep 1
+    end
+  rescue Interrupt
+  end
+
   # Show all information about this job
   def ps_show(id, *ids)
     ids = ([id] + ids).map { |s| Integer(s) }
 
-    jobs = Simple::SQL.all <<~SQL, ids, into: Postjob::Job
+    jobs = Simple::SQL.records <<~SQL, ids, into: Postjob::Job
       SELECT * FROM postjob.postjobs WHERE id = ANY($1)
     SQL
 
@@ -84,6 +110,8 @@ module Postjob::CLI
   end
 
   def ps_full(*ids, limit: 100, tags: nil)
+    connect_to_database!
+
     conditions = []
     conditions << tags_condition(tags)
     conditions << ids_condition(ids)
@@ -110,8 +138,7 @@ module Postjob::CLI
   end
 
   def print_sql(limit:, query:)
-    connect_to_database!
-    records = Simple::SQL.all("#{query} LIMIT $1+1", limit, into: Hash)
+    records = Simple::SQL.records("#{query} LIMIT $1+1", limit)
 
     tp records[0, limit]
     if records.length > limit

data/lib/postjob/cli/run.rb

@@ -20,8 +20,10 @@ module Postjob::CLI
 
     connect_to_database!
 
-    processed = Postjob.run(count: count) do |job|
-      logger.info "Processed job w/id #{job.id}" if job
+    logger.success "Starting runner with pid #{$$}"
+
+    processed = Postjob.run(count: count) do |job_id|
+      logger.info "Processed job w/id #{job_id}" if job_id
       STDERR.print "." unless quiet
     end
 

data/lib/postjob/migrations/002_statuses.rb

@@ -0,0 +1,18 @@
+pg_types = <<~SQL
+  SELECT pg_namespace.nspname AS schema, pg_type.typname AS name
+  FROM pg_type
+  LEFT JOIN pg_namespace on pg_namespace.oid=pg_type.typnamespace
+SQL
+
+unless SQL.ask("SELECT 1 FROM (#{pg_types}) sq WHERE (schema,name) = ($1, $2)", SCHEMA_NAME, "statuses")
+  SQL.exec <<~SQL
+    CREATE TYPE #{SCHEMA_NAME}.statuses AS ENUM (
+      'ready',    -- process can run
+      'sleep',    -- process has external dependencies to wait for.
+      'failed',   -- process failed, with nonrecoverable error
+      'err',      -- process errored (with recoverable error)
+      'timeout',  -- process timed out
+      'ok'        -- process succeeded
+    );
+  SQL
+end

data/lib/postjob/migrations/003_postjobs.sql

@@ -0,0 +1,41 @@
+CREATE TABLE IF NOT EXISTS {SCHEMA_NAME}.postjobs (
+  -- id values, readonly once created
+  id          BIGSERIAL PRIMARY KEY,                                        -- process id
+  parent_id   BIGINT REFERENCES {SCHEMA_NAME}.postjobs ON DELETE CASCADE,  -- parent process id
+  full_id     VARCHAR,                                                      -- full process id
+  root_id     BIGINT,                                                       -- root process id
+
+  created_at  timestamp NOT NULL DEFAULT (now() at time zone 'utc'),        -- creation timestamp
+  updated_at  timestamp NOT NULL DEFAULT (now() at time zone 'utc'),        -- update timestamp
+
+  queue VARCHAR,                                                            -- queue name. (readonly)
+  workflow VARCHAR NOT NULL,                                                -- e.g. "MyJobModule" (readonly)
+  workflow_method VARCHAR NOT NULL DEFAULT 'run',                           -- e.g. "run" (readonly)
+  workflow_version VARCHAR NOT NULL DEFAULT '',                             -- e.g. "1.0"
+  args JSONB,                                                               -- args
+
+  -- process state ----------------------------------------------------
+
+  status {SCHEMA_NAME}.statuses DEFAULT 'ready',
+  next_run_at timestamp DEFAULT (now() at time zone 'utc'),                 -- when possible to run next?
+  timing_out_at timestamp,                                                  -- job times out after this timestamp
+  failed_attempts INTEGER NOT NULL DEFAULT 0,                               -- failed how often?
+  max_attempts INTEGER NOT NULL DEFAULT 1,                                  -- maximum attempts before failing
+
+  -- process result ---------------------------------------------------
+
+  results JSONB,
+  error VARCHAR,
+  error_message VARCHAR,
+  error_backtrace JSONB,
+
+  -- custom fields
+  workflow_status VARCHAR,
+  tags JSONB
+);
+
+-- [TODO] check indices
+CREATE INDEX IF NOT EXISTS postjobs_tags_idx
+  ON {SCHEMA_NAME}.postjobs USING GIN (tags jsonb_path_ops);
+CREATE INDEX IF NOT EXISTS postjobs_parent_id_idx
+  ON {SCHEMA_NAME}.postjobs(parent_id);

data/lib/postjob/migrations/003a_processing.sql

@@ -0,0 +1,3 @@
+-- Apparently this line must be kept in a file of its own.
+ALTER TYPE {SCHEMA_NAME}.statuses
+  ADD value IF NOT EXISTS 'processing' AFTER 'ready';

data/lib/postjob/migrations/003b_processing_columns.sql

@@ -0,0 +1,31 @@
+DO $$
+  BEGIN
+    ALTER TABLE {SCHEMA_NAME}.postjobs ADD COLUMN processing_client varchar;
+  EXCEPTION
+    WHEN duplicate_column THEN RAISE NOTICE 'column {SCHEMA_NAME}.postjobs.processing_client already exists';
+  END;
+$$;
+
+DO $$
+  BEGIN
+    ALTER TABLE {SCHEMA_NAME}.postjobs ADD COLUMN processing_client_identifier varchar;
+  EXCEPTION
+    WHEN duplicate_column THEN RAISE NOTICE 'column {SCHEMA_NAME}.postjobs.processing_client_identifier already exists';
+  END;
+$$;
+
+DO $$
+  BEGIN
+    ALTER TABLE {SCHEMA_NAME}.postjobs ADD COLUMN processing_started_at timestamp;
+  EXCEPTION
+    WHEN duplicate_column THEN RAISE NOTICE 'column {SCHEMA_NAME}.postjobs.processing_started_at already exists';
+  END;
+$$;
+
+DO $$
+  BEGIN
+    ALTER TABLE {SCHEMA_NAME}.postjobs ADD COLUMN processing_max_duration float;
+  EXCEPTION
+    WHEN duplicate_column THEN RAISE NOTICE 'column {SCHEMA_NAME}.postjobs.processing_max_duration already exists';
+  END;
+$$;

data/lib/postjob/migrations/004_tokens.sql

@@ -0,0 +1,9 @@
+CREATE TABLE IF NOT EXISTS {SCHEMA_NAME}.tokens (
+  id          BIGSERIAL PRIMARY KEY,
+  postjob_id  BIGINT REFERENCES {SCHEMA_NAME}.postjobs ON DELETE CASCADE,
+  token       UUID NOT NULL,
+  created_at  timestamp NOT NULL DEFAULT (now() at time zone 'utc')
+);
+
+CREATE INDEX IF NOT EXISTS tokens_postjob_id_idx ON {SCHEMA_NAME}.tokens(postjob_id);
+CREATE INDEX IF NOT EXISTS tokens_token_idx ON {SCHEMA_NAME}.tokens(token);

data/lib/postjob/migrations/005_helpers.sql

@@ -0,0 +1,28 @@
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}._wakeup_runners() RETURNS TRIGGER AS $$
+  BEGIN
+    NOTIFY {CHANNEL};
+    RETURN NEW;
+  END;
+$$ LANGUAGE plpgsql;
+
+BEGIN;
+  DROP TRIGGER IF EXISTS _wakeup_runners ON {TABLE_NAME};
+
+  CREATE TRIGGER _wakeup_runners AFTER INSERT OR UPDATE
+    ON {TABLE_NAME}
+    FOR EACH STATEMENT
+    EXECUTE PROCEDURE {SCHEMA_NAME}._wakeup_runners();
+COMMIT;
+
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}._wakeup_parent_job(job_id BIGINT) RETURNS VOID AS $$
+BEGIN
+  UPDATE postjobs
+  SET
+    status='ready',
+    next_run_at=(now() at time zone 'utc'),
+    updated_at=(now() at time zone 'utc')
+  WHERE
+    status='sleep'
+    AND id=(SELECT parent_id FROM postjobs WHERE id=job_id);
+END;
+$$ LANGUAGE plpgsql;

data/lib/postjob/migrations/006_enqueue.sql

@@ -0,0 +1,44 @@
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.enqueue(
+    queue VARCHAR,
+    workflow VARCHAR,
+    workflow_method VARCHAR,
+    workflow_version VARCHAR,
+    args JSONB,
+    parent_id BIGINT,
+    tags JSONB,
+    max_attempts INTEGER,
+    timeout DOUBLE PRECISION)
+  RETURNS SETOF {SCHEMA_NAME}.postjobs
+AS $$
+  DECLARE
+    job_id BIGINT;
+  BEGIN
+    -- set defaults ---------------------------------------------------
+    workflow_version  := COALESCE(workflow_version, '');
+    queue             := COALESCE(queue, 'q');
+    max_attempts      := COALESCE(max_attempts, 5);
+
+    -- create postjobs entry ------------------------------------------
+    INSERT INTO {SCHEMA_NAME}.postjobs (
+      queue, workflow, workflow_method, workflow_version, args,
+      parent_id, tags, max_attempts,
+      timing_out_at
+    )
+    VALUES(
+      queue, workflow, workflow_method, workflow_version, args,
+      parent_id, tags, max_attempts,
+      (now() at time zone 'utc') + timeout * interval '1 second'
+    ) RETURNING {SCHEMA_NAME}.postjobs.id INTO job_id;
+
+    -- fill in root_id and full_id ------------------------------------
+    UPDATE postjobs
+    SET
+      root_id=COALESCE((SELECT root_id FROM postjobs s WHERE s.id=postjobs.parent_id), id),
+      full_id=COALESCE((SELECT full_id FROM postjobs s WHERE s.id=postjobs.parent_id) || '.' || id, id::varchar)
+    WHERE id=job_id;
+
+    -- return the job -------------------------------------------------
+    RETURN QUERY
+      SELECT * FROM postjobs WHERE id=job_id;
+  END;
+$$ LANGUAGE plpgsql;

data/lib/postjob/migrations/006a_processing.sql

@@ -0,0 +1,48 @@
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.set_client_identifier(client_identifier varchar) RETURNS VOID AS $$
+BEGIN
+  PERFORM set_config('{SCHEMA_NAME}.client_identifier', client_identifier, false);
+END;
+$$ LANGUAGE plpgsql;
+
+--SELECT {SCHEMA_NAME}.set_client_identifier('nope');
+
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}._client_identifier() RETURNS varchar AS $$
+BEGIN
+  RETURN current_setting('{SCHEMA_NAME}.client_identifier');
+EXCEPTION
+  WHEN OTHERS THEN RETURN NULL;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}._set_job_processing(job_id BIGINT) RETURNS VOID AS $$
+DECLARE
+  v_pid int;
+BEGIN
+  v_pid := pg_backend_pid();
+
+  UPDATE postjobs
+  SET
+    status='processing',
+    processing_client=(SELECT client_addr || ':' || client_port FROM pg_stat_activity WHERE pid = v_pid),
+    processing_client_identifier={SCHEMA_NAME}._client_identifier(),
+    processing_started_at=now() at time zone 'utc',
+    processing_max_duration=30 * 60, -- default max duration of processing: 30 minutes.
+    error=NULL,
+    error_message=NULL,
+    error_backtrace=NULL,
+    next_run_at=NULL
+   WHERE id=job_id;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}._reset_job_processing(job_id BIGINT) RETURNS VOID AS $$
+BEGIN
+  UPDATE postjobs
+  SET
+    processing_client=NULL,
+    processing_client_identifier=NULL,
+    processing_started_at=NULL,
+    processing_max_duration=NULL
+  WHERE id=job_id;
+END;
+$$ LANGUAGE plpgsql;

data/lib/postjob/migrations/007_job_results.sql

@@ -0,0 +1,157 @@
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}._update_job_version(job_id BIGINT, p_version VARCHAR) RETURNS VOID AS $$
+BEGIN
+  IF p_version IS NOT NULL THEN
+    UPDATE postjobs
+    SET
+      workflow_version=p_version,
+      updated_at=(now() at time zone 'utc')
+    WHERE id=job_id;
+  END IF;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.set_job_result(job_id BIGINT, p_results JSONB, p_version VARCHAR) RETURNS VOID AS $$
+BEGIN
+  PERFORM _update_job_version(job_id, p_version);
+  PERFORM _reset_job_processing(job_id);
+
+  UPDATE postjobs
+  SET
+    results=p_results,
+    error=NULL,
+    error_message=NULL,
+    error_backtrace=NULL,
+    status='ok',
+    next_run_at=NULL
+  WHERE id=job_id;
+
+  PERFORM _wakeup_parent_job(job_id);
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.set_job_pending(job_id BIGINT, p_version VARCHAR) RETURNS VOID AS $$
+BEGIN
+  PERFORM _update_job_version(job_id, p_version);
+  PERFORM _reset_job_processing(job_id);
+
+  UPDATE postjobs
+  SET status='sleep', next_run_at=NULL
+  WHERE id=job_id;
+END;
+$$ LANGUAGE plpgsql;
+
+-- If this is a recoverable error and if we have another run possible
+-- we'll set next_run_at, and the status to "err", otherwise
+-- next_run_at will be NULL and the status would be "failed" or "timeout"
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}._prepare_next_run(
+  job_id BIGINT,
+  p_status {SCHEMA_NAME}.statuses,
+  p_fast_mode BOOLEAN) RETURNS VOID AS $$
+DECLARE
+  next_run_at_basetime DOUBLE PRECISION;
+BEGIN
+  IF p_status NOT IN ('err', 'timeout', 'failed') THEN
+    RAISE 'Invalid status value %', p_status;
+  END IF;
+
+  -- If this is a recoverable error we check if we have any remaining attempts.
+  IF p_status = 'err' THEN
+    IF (SELECT max_attempts - failed_attempts FROM postjobs WHERE id=job_id) > 0 THEN
+      p_status = 'err';
+    ELSE
+      p_status = 'failed';
+    END IF;
+  END IF;
+
+  -- set status, clear next_run_at
+  UPDATE postjobs
+  SET
+    status=p_status,
+    next_run_at=NULL
+  WHERE id=job_id;
+
+  -- fill in next_run_at
+  IF p_status != 'failed' AND p_status != 'timeout' THEN
+    next_run_at_basetime := CASE WHEN p_fast_mode THEN 0.01 ELSE 10 END;
+
+    UPDATE postjobs
+    SET
+      next_run_at=(now() at time zone 'utc') + next_run_at_basetime * pow(1.5, failed_attempts) * interval '1 second'
+    WHERE id=job_id;
+  END IF;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.set_job_error(
+  job_id BIGINT,
+  p_error VARCHAR,
+  p_error_message VARCHAR,
+  p_error_backtrace JSONB,
+  p_status {SCHEMA_NAME}.statuses,
+  p_version VARCHAR,
+  p_fast_mode BOOLEAN) RETURNS VOID AS $$
+BEGIN
+  PERFORM _update_job_version(job_id, p_version);
+  PERFORM _reset_job_processing(job_id);
+
+  -- write error info
+  UPDATE postjobs
+  SET
+    error=p_error,
+    error_message=p_error_message,
+    error_backtrace=p_error_backtrace,
+    failed_attempts=failed_attempts+1,
+    next_run_at=NULL
+  WHERE id=job_id;
+
+  -- prepare next run, if any
+  PERFORM _prepare_next_run(job_id, p_status, p_fast_mode);
+  PERFORM _wakeup_parent_job(job_id);
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}._set_job_timeout(
+  job_id BIGINT,
+  p_fast_mode BOOLEAN) RETURNS VOID AS $$
+BEGIN
+  PERFORM _reset_job_processing(job_id);
+
+  -- write error info
+  UPDATE postjobs
+  SET
+    error='Timeout',
+    error_message='timeout',
+    error_backtrace=NULL,
+    failed_attempts=failed_attempts+1,
+    next_run_at=NULL
+  WHERE id=job_id;
+
+  -- prepare next run, if any
+  PERFORM _prepare_next_run(job_id, 'timeout', p_fast_mode);
+  PERFORM _wakeup_parent_job(job_id);
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}._set_job_zombie(
+  job_id BIGINT,
+  p_fast_mode BOOLEAN) RETURNS VOID AS $$
+BEGIN
+  PERFORM _reset_job_processing(job_id);
+
+  RAISE NOTICE 'job % is a zombie', job_id;
+
+  -- write error info
+  UPDATE postjobs
+  SET
+    error='Zombie',
+    error_message='zombie',
+    error_backtrace=NULL,
+    failed_attempts=failed_attempts+1,
+    next_run_at=NULL
+  WHERE id=job_id;
+
+  -- prepare next run, if any
+  PERFORM _prepare_next_run(job_id, 'err', p_fast_mode);
+  PERFORM _wakeup_parent_job(job_id);
+END;
+$$ LANGUAGE plpgsql;

data/lib/postjob/migrations/008_checkout_runnable.sql

@@ -0,0 +1,78 @@
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.time_to_next_job(workflows_with_versions varchar[])
+  RETURNS float
+AS $$
+DECLARE
+  p_processable_at timestamp;
+BEGIN
+  SELECT MIN(processable_at) INTO p_processable_at FROM (
+      SELECT MIN(processing_started_at + processing_max_duration * interval '1 second') AS processable_at
+      FROM {TABLE_NAME}
+      WHERE status IN ('processing')
+    UNION
+      SELECT MIN(timing_out_at) AS processable_at
+      FROM {TABLE_NAME}
+      WHERE status IN ('ready', 'err', 'sleep')
+    UNION
+      SELECT MIN(next_run_at) AS processable_at
+      FROM {TABLE_NAME}
+      WHERE status IN ('ready', 'err')
+        AND workflow || workflow_version = ANY ($1)
+  ) sq;
+
+  RETURN EXTRACT(EPOCH FROM  p_processable_at - (now() at time zone 'utc'));
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.checkout(workflows_with_versions varchar[], p_fast_mode BOOLEAN)
+  RETURNS SETOF {TABLE_NAME}
+AS $$
+DECLARE
+  job {TABLE_NAME};
+BEGIN
+  LOOP
+    -- try to checkout a job. Each of the conditions here is matching
+    -- one of the CASE .. WHEN clauses below.
+    SELECT INTO job *
+    FROM {TABLE_NAME} s
+    WHERE
+      (
+        s.status IN ('processing')
+        AND (s.processing_started_at + s.processing_max_duration * interval '1 second') <= (now() at time zone 'utc')
+      )
+      OR
+      (
+        s.status IN ('ready', 'err', 'sleep')
+        AND s.timing_out_at <= (now() at time zone 'utc')
+      )
+      OR
+      (
+        s.status IN ('ready', 'err')
+        AND s.next_run_at <= (now() at time zone 'utc')
+        AND s.workflow || s.workflow_version = ANY ($1)
+      )
+    ORDER BY (LEAST(s.next_run_at, s.timing_out_at, (s.processing_started_at + s.processing_max_duration * interval '1 second')))
+    FOR UPDATE SKIP LOCKED
+    LIMIT 1;
+
+    CASE
+      WHEN job.id IS NULL THEN
+        -- couldn't find a job?
+        EXIT;
+      WHEN job.status='processing' AND
+        (job.processing_started_at + job.processing_max_duration * interval '1 second') <= (now() at time zone 'utc') THEN
+        -- a zombie: the worker probably died.
+        PERFORM {SCHEMA_NAME}._set_job_zombie(job.id, p_fast_mode);
+        CONTINUE;
+      WHEN job.status IN ('ready', 'err', 'sleep') AND job.timing_out_at <= (now() at time zone 'utc') THEN
+        -- job timed out? mark it as such, and try next one.
+        PERFORM {SCHEMA_NAME}._set_job_timeout(job.id, p_fast_mode);
+        CONTINUE;
+      ELSE
+        -- set job to processing
+        PERFORM _set_job_processing(job.id);
+        RETURN QUERY SELECT * FROM {TABLE_NAME} WHERE id=job.id;
+        EXIT;
+    END CASE;
+  END LOOP;
+END;
+$$ LANGUAGE plpgsql;