postjob 0.4.5 → 0.5.0

data/lib/postjob/migrations/003b_processing_columns.sql

@@ -1,31 +1,31 @@
 DO $$
   BEGIN
-    ALTER TABLE {SCHEMA_NAME}.postjobs ADD COLUMN processing_client varchar;
+    ALTER TABLE {SCHEMA_NAME}.postjobs DROP COLUMN processing_client;
   EXCEPTION
-    WHEN duplicate_column THEN RAISE DEBUG 'column {SCHEMA_NAME}.postjobs.processing_client already exists';
+    WHEN undefined_column THEN RAISE DEBUG 'column {SCHEMA_NAME}.postjobs.processing_client already dropped';
   END;
 $$;
 
 DO $$
   BEGIN
-    ALTER TABLE {SCHEMA_NAME}.postjobs ADD COLUMN processing_client_identifier varchar;
+    ALTER TABLE {SCHEMA_NAME}.postjobs DROP COLUMN processing_client_identifier;
   EXCEPTION
-    WHEN duplicate_column THEN RAISE DEBUG 'column {SCHEMA_NAME}.postjobs.processing_client_identifier already exists';
+    WHEN undefined_column THEN RAISE DEBUG 'column {SCHEMA_NAME}.postjobs.processing_client_identifier already dropped';
   END;
 $$;
 
 DO $$
   BEGIN
-    ALTER TABLE {SCHEMA_NAME}.postjobs ADD COLUMN processing_started_at timestamp;
+    ALTER TABLE {SCHEMA_NAME}.postjobs DROP COLUMN processing_started_at;
   EXCEPTION
-    WHEN duplicate_column THEN RAISE DEBUG 'column {SCHEMA_NAME}.postjobs.processing_started_at already exists';
+    WHEN undefined_column THEN RAISE DEBUG 'column {SCHEMA_NAME}.postjobs.processing_started_at already dropped';
   END;
 $$;
 
 DO $$
   BEGIN
-    ALTER TABLE {SCHEMA_NAME}.postjobs ADD COLUMN processing_max_duration float;
+    ALTER TABLE {SCHEMA_NAME}.postjobs DROP COLUMN processing_max_duration;
   EXCEPTION
-    WHEN duplicate_column THEN RAISE DEBUG 'column {SCHEMA_NAME}.postjobs.processing_max_duration already exists';
+    WHEN undefined_column THEN RAISE DEBUG 'column {SCHEMA_NAME}.postjobs.processing_max_duration already dropped';
   END;
 $$;

data/lib/postjob/migrations/005_helpers.sql

@@ -14,10 +14,12 @@ BEGIN;
     EXECUTE PROCEDURE {SCHEMA_NAME}._wakeup_runners();
 COMMIT;
 
-CREATE OR REPLACE FUNCTION {SCHEMA_NAME}._wakeup_parent_job(job_id BIGINT) RETURNS VOID AS $$
+DROP FUNCTION IF EXISTS {SCHEMA_NAME}._wakeup_parent_job(job_id BIGINT);  -- removed in 0.5.0
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}._wakeup_parent_job(worker_session_id UUID, job_id BIGINT) RETURNS VOID AS $$
 BEGIN
   UPDATE {SCHEMA_NAME}.postjobs
   SET
+    last_worker_session_id=worker_session_id,
     status='ready',
     next_run_at=(now() at time zone 'utc'),
     updated_at=(now() at time zone 'utc')

data/lib/postjob/migrations/006_enqueue.sql

@@ -1,4 +1,5 @@
 CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.enqueue(
+    p_worker_session_id UUID,
     queue VARCHAR,
     workflow VARCHAR,
     workflow_method VARCHAR,
@@ -25,11 +26,13 @@ AS $$
 
     -- create postjobs entry ------------------------------------------
     INSERT INTO {SCHEMA_NAME}.postjobs (
+      last_worker_session_id,
       queue, workflow, workflow_method, workflow_version, args,
       parent_id, tags, max_attempts,
       timing_out_at
     )
     VALUES(
+      p_worker_session_id,
       queue, workflow, workflow_method, workflow_version, args,
       parent_id, tags, max_attempts,
       (now() at time zone 'utc') + timeout * interval '1 second'

data/lib/postjob/migrations/006a_processing.sql

@@ -1,20 +1,5 @@
-CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.set_client_identifier(client_identifier varchar) RETURNS VOID AS $$
-BEGIN
-  PERFORM set_config('{SCHEMA_NAME}.client_identifier', client_identifier, false);
-END;
-$$ LANGUAGE plpgsql;
-
---SELECT {SCHEMA_NAME}.set_client_identifier('nope');
-
-CREATE OR REPLACE FUNCTION {SCHEMA_NAME}._client_identifier() RETURNS varchar AS $$
-BEGIN
-  RETURN current_setting('{SCHEMA_NAME}.client_identifier');
-EXCEPTION
-  WHEN OTHERS THEN RETURN NULL;
-END;
-$$ LANGUAGE plpgsql;
-
-CREATE OR REPLACE FUNCTION {SCHEMA_NAME}._set_job_processing(job_id BIGINT) RETURNS VOID AS $$
+DROP FUNCTION IF EXISTS {SCHEMA_NAME}._set_job_processing(job_id BIGINT);  -- removed in 0.5.0
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}._set_job_processing(worker_session_id UUID, job_id BIGINT) RETURNS VOID AS $$
 DECLARE
   v_pid int;
 BEGIN
@@ -22,11 +7,8 @@ BEGIN
 
   UPDATE {SCHEMA_NAME}.postjobs
   SET
+    last_worker_session_id=worker_session_id,
     status='processing',
-    processing_client=(SELECT client_addr || ':' || client_port FROM pg_stat_activity WHERE pid = v_pid),
-    processing_client_identifier={SCHEMA_NAME}._client_identifier(),
-    processing_started_at=now() at time zone 'utc',
-    processing_max_duration=30 * 60, -- default max duration of processing: 30 minutes.
     error=NULL,
     error_message=NULL,
     error_backtrace=NULL,
@@ -35,14 +17,12 @@ BEGIN
 END;
 $$ LANGUAGE plpgsql;
 
-CREATE OR REPLACE FUNCTION {SCHEMA_NAME}._reset_job_processing(job_id BIGINT) RETURNS VOID AS $$
+DROP FUNCTION IF EXISTS {SCHEMA_NAME}._reset_job_processing(job_id BIGINT);  -- removed in 0.5.0
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}._reset_job_processing(worker_session_id UUID, job_id BIGINT) RETURNS VOID AS $$
 BEGIN
   UPDATE {SCHEMA_NAME}.postjobs
   SET
-    processing_client=NULL,
-    processing_client_identifier=NULL,
-    processing_started_at=NULL,
-    processing_max_duration=NULL
+    last_worker_session_id=worker_session_id
   WHERE id=job_id;
 END;
 $$ LANGUAGE plpgsql;

data/lib/postjob/migrations/007_job_results.sql

@@ -10,10 +10,11 @@ BEGIN
 END;
 $$ LANGUAGE plpgsql;
 
-CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.set_job_result(job_id BIGINT, p_results JSONB, p_version VARCHAR) RETURNS VOID AS $$
+DROP FUNCTION IF EXISTS {SCHEMA_NAME}.set_job_result(job_id BIGINT, p_results JSONB, p_version VARCHAR);  -- removed in 0.5.0
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.set_job_result(p_worker_session_id UUID, job_id BIGINT, p_results JSONB, p_version VARCHAR) RETURNS VOID AS $$
 BEGIN
   PERFORM {SCHEMA_NAME}._update_job_version(job_id, p_version);
-  PERFORM {SCHEMA_NAME}._reset_job_processing(job_id);
+  PERFORM {SCHEMA_NAME}._reset_job_processing(p_worker_session_id, job_id);
 
   UPDATE {SCHEMA_NAME}.postjobs
   SET
@@ -25,17 +26,21 @@ BEGIN
     next_run_at=NULL
   WHERE id=job_id;
 
-  PERFORM {SCHEMA_NAME}._wakeup_parent_job(job_id);
+  PERFORM {SCHEMA_NAME}._wakeup_parent_job(p_worker_session_id, job_id);
 END;
 $$ LANGUAGE plpgsql;
 
-CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.set_job_pending(job_id BIGINT, p_version VARCHAR) RETURNS VOID AS $$
+DROP FUNCTION IF EXISTS {SCHEMA_NAME}.set_job_pending(job_id BIGINT, p_version VARCHAR);  -- removed in 0.5.0
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.set_job_pending(p_worker_session_id UUID, job_id BIGINT, p_version VARCHAR) RETURNS VOID AS $$
 BEGIN
   PERFORM {SCHEMA_NAME}._update_job_version(job_id, p_version);
-  PERFORM {SCHEMA_NAME}._reset_job_processing(job_id);
+  PERFORM {SCHEMA_NAME}._reset_job_processing(p_worker_session_id, job_id);
 
   UPDATE {SCHEMA_NAME}.postjobs
-  SET status='sleep', next_run_at=NULL
+  SET
+    last_worker_session_id=p_worker_session_id,
+    status='sleep',
+    next_run_at=NULL
   WHERE id=job_id;
 END;
 $$ LANGUAGE plpgsql;
@@ -82,7 +87,16 @@ BEGIN
 END;
 $$ LANGUAGE plpgsql;
 
+DROP FUNCTION IF EXISTS {SCHEMA_NAME}.set_job_error(
+  job_id BIGINT,
+  p_error VARCHAR,
+  p_error_message VARCHAR,
+  p_error_backtrace JSONB,
+  p_status {SCHEMA_NAME}.statuses,
+  p_version VARCHAR,
+  p_fast_mode BOOLEAN);  -- removed in 0.5.0
 CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.set_job_error(
+  p_worker_session_id UUID,
   job_id BIGINT,
   p_error VARCHAR,
   p_error_message VARCHAR,
@@ -92,11 +106,12 @@ CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.set_job_error(
   p_fast_mode BOOLEAN) RETURNS VOID AS $$
 BEGIN
   PERFORM {SCHEMA_NAME}._update_job_version(job_id, p_version);
-  PERFORM {SCHEMA_NAME}._reset_job_processing(job_id);
+  PERFORM {SCHEMA_NAME}._reset_job_processing(p_worker_session_id, job_id);
 
   -- write error info
   UPDATE {SCHEMA_NAME}.postjobs
   SET
+    last_worker_session_id=p_worker_session_id,
     error=p_error,
     error_message=p_error_message,
     error_backtrace=p_error_backtrace,
@@ -106,15 +121,17 @@ BEGIN
 
   -- prepare next run, if any
   PERFORM {SCHEMA_NAME}._prepare_next_run(job_id, p_status, p_fast_mode);
-  PERFORM {SCHEMA_NAME}._wakeup_parent_job(job_id);
+  PERFORM {SCHEMA_NAME}._wakeup_parent_job(p_worker_session_id, job_id);
 END;
 $$ LANGUAGE plpgsql;
 
+DROP FUNCTION IF EXISTS {SCHEMA_NAME}._set_job_timeout(job_id BIGINT, p_fast_mode BOOLEAN);  -- removed in 0.5.0
 CREATE OR REPLACE FUNCTION {SCHEMA_NAME}._set_job_timeout(
+  p_worker_session_id UUID,
   job_id BIGINT,
   p_fast_mode BOOLEAN) RETURNS VOID AS $$
 BEGIN
-  PERFORM {SCHEMA_NAME}._reset_job_processing(job_id);
+  PERFORM {SCHEMA_NAME}._reset_job_processing(p_worker_session_id, job_id);
 
   -- write error info
   UPDATE {SCHEMA_NAME}.postjobs
@@ -128,17 +145,19 @@ BEGIN
 
   -- prepare next run, if any
   PERFORM {SCHEMA_NAME}._prepare_next_run(job_id, 'timeout', p_fast_mode);
-  PERFORM {SCHEMA_NAME}._wakeup_parent_job(job_id);
+  PERFORM {SCHEMA_NAME}._wakeup_parent_job(p_worker_session_id, job_id);
 END;
 $$ LANGUAGE plpgsql;
 
 CREATE OR REPLACE FUNCTION {SCHEMA_NAME}._set_job_zombie(
   job_id BIGINT,
   p_fast_mode BOOLEAN) RETURNS VOID AS $$
+DECLARE
+  p_worker_session_id UUID;
 BEGIN
-  PERFORM {SCHEMA_NAME}._reset_job_processing(job_id);
+  p_worker_session_id := {SCHEMA_NAME}._null_uuid();
 
-  RAISE NOTICE 'job % is a zombie', job_id;
+  PERFORM {SCHEMA_NAME}._reset_job_processing(p_worker_session_id, job_id);
 
   -- write error info
   UPDATE {SCHEMA_NAME}.postjobs
@@ -152,6 +171,6 @@ BEGIN
 
   -- prepare next run, if any
   PERFORM {SCHEMA_NAME}._prepare_next_run(job_id, 'err', p_fast_mode);
-  PERFORM {SCHEMA_NAME}._wakeup_parent_job(job_id);
+  PERFORM {SCHEMA_NAME}._wakeup_parent_job(p_worker_session_id, job_id);
 END;
 $$ LANGUAGE plpgsql;

data/lib/postjob/migrations/008_checkout_runnable.sql

@@ -1,14 +1,14 @@
-CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.time_to_next_job(workflows_with_versions varchar[])
+DROP FUNCTION IF EXISTS {SCHEMA_NAME}.time_to_next_job(workflows_with_versions varchar[]);  -- removed in 0.5.0
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.time_to_next_job(p_worker_session_id UUID)
   RETURNS float
 AS $$
 DECLARE
   p_processable_at timestamp;
+  p_workflows_with_version varchar[];
 BEGIN
+  SELECT workflows INTO p_workflows_with_version FROM {SCHEMA_NAME}.worker_sessions WHERE id=p_worker_session_id;
+
   SELECT MIN(processable_at) INTO p_processable_at FROM (
-      SELECT MIN(processing_started_at + processing_max_duration * interval '1 second') AS processable_at
-      FROM {SCHEMA_NAME}.postjobs
-      WHERE status IN ('processing')
-    UNION
       SELECT MIN(timing_out_at) AS processable_at
       FROM {SCHEMA_NAME}.postjobs
       WHERE status IN ('ready', 'err', 'sleep')
@@ -16,30 +16,29 @@ BEGIN
       SELECT MIN(next_run_at) AS processable_at
       FROM {SCHEMA_NAME}.postjobs
       WHERE status IN ('ready', 'err')
-        AND workflow || workflow_version = ANY ($1)
+        AND workflow || workflow_version = ANY (p_workflows_with_version)
   ) sq;
 
   RETURN EXTRACT(EPOCH FROM  p_processable_at - (now() at time zone 'utc'));
 END;
 $$ LANGUAGE plpgsql;
 
-CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.checkout(workflows_with_versions varchar[], p_fast_mode BOOLEAN)
+DROP FUNCTION IF EXISTS {SCHEMA_NAME}.checkout(workflows_with_versions varchar[], p_fast_mode BOOLEAN);  -- removed in 0.5.0
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.checkout(p_worker_session_id UUID, p_fast_mode BOOLEAN)
   RETURNS SETOF {SCHEMA_NAME}.postjobs
 AS $$
 DECLARE
   job {SCHEMA_NAME}.postjobs;
+  p_workflows_with_version varchar[];
 BEGIN
+  SELECT workflows INTO p_workflows_with_version FROM {SCHEMA_NAME}.worker_sessions WHERE id=p_worker_session_id;
+
   LOOP
     -- try to checkout a job. Each of the conditions here is matching
     -- one of the CASE .. WHEN clauses below.
     SELECT INTO job *
     FROM {SCHEMA_NAME}.postjobs s
     WHERE
-      (
-        s.status IN ('processing')
-        AND (s.processing_started_at + s.processing_max_duration * interval '1 second') <= (now() at time zone 'utc')
-      )
-      OR
       (
         s.status IN ('ready', 'err', 'sleep')
         AND s.timing_out_at <= (now() at time zone 'utc')
@@ -48,9 +47,9 @@ BEGIN
       (
         s.status IN ('ready', 'err')
         AND s.next_run_at <= (now() at time zone 'utc')
-        AND s.workflow || s.workflow_version = ANY ($1)
+        AND (s.workflow || s.workflow_version) = ANY (p_workflows_with_version)
       )
-    ORDER BY (LEAST(s.next_run_at, s.timing_out_at, (s.processing_started_at + s.processing_max_duration * interval '1 second')))
+    ORDER BY (LEAST(s.next_run_at, s.timing_out_at))
     FOR UPDATE SKIP LOCKED
     LIMIT 1;
 
@@ -58,18 +57,13 @@ BEGIN
       WHEN job.id IS NULL THEN
         -- couldn't find a job?
         EXIT;
-      WHEN job.status='processing' AND
-        (job.processing_started_at + job.processing_max_duration * interval '1 second') <= (now() at time zone 'utc') THEN
-        -- a zombie: the worker probably died.
-        PERFORM {SCHEMA_NAME}._set_job_zombie(job.id, p_fast_mode);
-        CONTINUE;
       WHEN job.status IN ('ready', 'err', 'sleep') AND job.timing_out_at <= (now() at time zone 'utc') THEN
         -- job timed out? mark it as such, and try next one.
-        PERFORM {SCHEMA_NAME}._set_job_timeout(job.id, p_fast_mode);
+        PERFORM {SCHEMA_NAME}._set_job_timeout(p_worker_session_id, job.id, p_fast_mode);
         CONTINUE;
       ELSE
         -- set job to processing
-        PERFORM {SCHEMA_NAME}._set_job_processing(job.id);
+        PERFORM {SCHEMA_NAME}._set_job_processing(p_worker_session_id, job.id);
         RETURN QUERY SELECT * FROM {SCHEMA_NAME}.postjobs WHERE id=job.id;
         EXIT;
     END CASE;

data/lib/postjob/migrations/008a_childjobs.sql

@@ -18,7 +18,19 @@ BEGIN
 END;
 $$ LANGUAGE plpgsql;
 
+DROP FUNCTION IF EXISTS {SCHEMA_NAME}.find_or_create_childjob(
+  v_queue VARCHAR,
+  v_workflow VARCHAR,
+  v_workflow_method VARCHAR,
+  v_workflow_version VARCHAR,
+  v_args JSONB,
+  v_parent_id BIGINT,
+  v_tags JSONB,
+  v_max_attempts INTEGER,
+  v_timeout DOUBLE PRECISION);  -- removed in 0.5.0
+
 CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.find_or_create_childjob(
+  p_worker_session_id UUID,
   v_queue VARCHAR,
   v_workflow VARCHAR,
   v_workflow_method VARCHAR,
@@ -67,6 +79,7 @@ BEGIN
 
       RETURN QUERY
         SELECT * FROM {SCHEMA_NAME}.enqueue(
+          p_worker_session_id,
           COALESCE(v_queue, parent.queue),  -- queue VARCHAR,
           v_workflow,                       -- workflow VARCHAR,
           v_workflow_method,                -- workflow_method VARCHAR,

data/lib/postjob/migrations/010_settings.sql

@@ -3,7 +3,9 @@ CREATE TABLE IF NOT EXISTS {SCHEMA_NAME}.settings (
   value       VARCHAR
 );
 
-CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.update_settings(p_name VARCHAR, p_value VARCHAR)
+DROP FUNCTION IF EXISTS {SCHEMA_NAME}.update_settings(p_name VARCHAR, p_value VARCHAR);
+
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.settings_set(p_name VARCHAR, p_value VARCHAR)
   RETURNS VOID
 AS $$
   BEGIN
@@ -12,5 +14,18 @@ AS $$
   END;
 $$ LANGUAGE plpgsql;
 
-SELECT {SCHEMA_NAME}.update_settings('version', '0.4.2');
-SELECT {SCHEMA_NAME}.update_settings('client_version', '{CLIENT_VERSION}');
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.settings_get(p_name VARCHAR)
+  RETURNS varchar
+AS $$
+  DECLARE
+    p_result varchar;
+  BEGIN
+    SELECT value INTO p_result FROM {SCHEMA_NAME}.settings WHERE name=$1;
+    RETURN p_result;
+  END;
+$$ LANGUAGE plpgsql;
+
+-- define version settings ----------------------------------------------------
+
+SELECT {SCHEMA_NAME}.settings_set('version', '0.5.0');
+SELECT {SCHEMA_NAME}.settings_set('client_version', '{CLIENT_VERSION}');

data/lib/postjob/migrations/011_null_uuid.sql

@@ -0,0 +1,7 @@
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}._null_uuid()
+  RETURNS UUID IMMUTABLE
+AS $$
+  BEGIN
+    RETURN '00000000-0000-0000-0000-000000000000'::uuid;
+  END;
+$$ LANGUAGE plpgsql;

data/lib/postjob/migrations/012_hosts.sql

@@ -0,0 +1,42 @@
+-- hosts ----------------------------------------------------------------------
+
+-- The hosts table records available hosts. hosts are the basis for the
+-- sticky jobs feature - a sticky job is a job which can only be checked
+-- out by a specific host.
+--
+
+CREATE TABLE IF NOT EXISTS {SCHEMA_NAME}.hosts (
+  id UUID PRIMARY KEY  DEFAULT (gen_random_uuid()),     -- UUID identifying a worker **host**
+  attributes JSONB NOT NULL DEFAULT '{}'::JSONB,
+  created_at timestamp NOT NULL DEFAULT (now() at time zone 'utc')
+);
+
+CREATE INDEX IF NOT EXISTS hosts_attributes_idx
+  ON {SCHEMA_NAME}.hosts USING GIN (attributes jsonb_path_ops);
+
+-- returns _null_host_id ------------------------------------------------------
+
+DO $$
+  DECLARE
+    null_uuid UUID := {SCHEMA_NAME}._null_uuid();
+  BEGIN
+    IF NOT EXISTS (SELECT 1 FROM {SCHEMA_NAME}.hosts WHERE id = null_uuid) THEN
+      INSERT INTO {SCHEMA_NAME}.hosts(id, attributes) VALUES(null_uuid, '{}');
+    END IF;
+  END;
+$$ LANGUAGE plpgsql;
+
+-- host_register: registers a host --------------------------------------------
+
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.host_register(p_attrs JSONB)
+  RETURNS UUID
+AS $$
+  DECLARE
+    v_id UUID;
+  BEGIN
+    v_id := gen_random_uuid();
+    INSERT INTO {SCHEMA_NAME}.hosts (id, attributes)
+      VALUES (v_id, p_attrs);
+    RETURN v_id;
+  END;
+$$ LANGUAGE plpgsql;