postjob 0.4.5 → 0.5.0

data/lib/postjob/migrations/013_worker_sessions.sql

@@ -0,0 +1,44 @@
+-- worker_sessions ------------------------------------------------------------
+
+-- The worker_sessions table records available "worker worker_sessions". The
+-- following information is generated/recorded:
+--
+-- - id:            a UUID, which is unique per worker_session.
+-- - host_id:       a UUID, which is unique across all workers on a machine.
+-- - client_socket: contains "host:port" of a connection
+-- - workflows:     an array of workflows with and without version numbers.
+--                  This describes which workflows can be checked out in this
+--                  session.
+--
+-- The host_id value will be used for sticky workflows: A workflow with
+-- `sticky: true` will only be checked out to workers with the same host_id.
+-- This is useful for workflows that use temporary files to share state
+-- between jobs.
+--
+-- The +id+ and +host_id+ UUIDs are generated on the queue. The client is
+-- supposed to get a worker_session via +worker_session_start(id, host_id, workflows)+
+--
+
+CREATE TABLE IF NOT EXISTS {SCHEMA_NAME}.worker_sessions (
+  id UUID PRIMARY KEY  DEFAULT (gen_random_uuid()),                       -- UUID identifying a worker **process**
+  host_id UUID NOT NULL REFERENCES {SCHEMA_NAME}.hosts ON DELETE CASCADE, -- UUID identifying a worker **host**
+  client_socket VARCHAR,                                                  -- host:port of connection (from pg_stat_activity)
+  workflows VARCHAR[] NOT NULL,                                           -- array of workflow versions available on that worker
+  attributes JSONB NOT NULL DEFAULT '{}'::JSONB,
+  created_at timestamp NOT NULL DEFAULT (now() at time zone 'utc')
+);
+
+CREATE INDEX IF NOT EXISTS worker_sessions_attributes_idx
+  ON {SCHEMA_NAME}.worker_sessions USING GIN (attributes jsonb_path_ops);
+
+-- worker_session_start: starts or reuses a worker_session ----------------------------------
+
+DO $$
+  DECLARE
+    null_uuid UUID := {SCHEMA_NAME}._null_uuid();
+  BEGIN
+    IF NOT EXISTS (SELECT 1 FROM {SCHEMA_NAME}.worker_sessions WHERE id = null_uuid) THEN
+      INSERT INTO {SCHEMA_NAME}.worker_sessions(id, host_id, workflows) VALUES(null_uuid, null_uuid, '{}');
+    END IF;
+  END;
+$$ LANGUAGE plpgsql;

data/lib/postjob/migrations/014_postjob_session_id.sql

@@ -0,0 +1,17 @@
+DO $$
+  BEGIN
+    ALTER TABLE {SCHEMA_NAME}.postjobs ADD COLUMN last_worker_session_id UUID REFERENCES {SCHEMA_NAME}.worker_sessions ON DELETE CASCADE;
+    UPDATE {SCHEMA_NAME}.postjobs SET last_worker_session_id={SCHEMA_NAME}._null_uuid();
+    ALTER TABLE {SCHEMA_NAME}.postjobs ALTER COLUMN last_worker_session_id SET NOT NULL;
+  EXCEPTION
+    WHEN duplicate_column THEN RAISE DEBUG 'column {SCHEMA_NAME}.postjobs.last_worker_session_id already exists';
+  END;
+$$;
+
+DO $$
+  BEGIN
+    ALTER TABLE {SCHEMA_NAME}.postjobs DROP COLUMN workflow_status;
+  EXCEPTION
+    WHEN undefined_column THEN RAISE DEBUG 'column {SCHEMA_NAME}.postjobs.workflow_status already dropped';
+  END;
+$$;

data/lib/postjob/migrations/015_events.sql

@@ -0,0 +1,76 @@
+CREATE TABLE IF NOT EXISTS {SCHEMA_NAME}.events (
+  id          BIGSERIAL PRIMARY KEY,
+  postjob_id  BIGINT REFERENCES {SCHEMA_NAME}.postjobs ON DELETE SET NULL,
+  host_id     UUID NOT NULL REFERENCES {SCHEMA_NAME}.hosts ON DELETE CASCADE,
+  worker_session_id  UUID REFERENCES {SCHEMA_NAME}.worker_sessions ON DELETE SET NULL,
+  name        VARCHAR NOT NULL,
+  attributes  JSONB not null DEFAULT '{}'::JSONB,
+  created_at  timestamp NOT NULL DEFAULT (now() at time zone 'utc')
+);
+
+-- We do not create an index on events.attributes just yet. events has a high
+-- write rate, and until we know for sure that we'll need this index we
+-- better not have it.
+--
+-- CREATE INDEX IF NOT EXISTS events_attributes_idx
+--  ON {SCHEMA_NAME}.events USING GIN (attributes jsonb_path_ops);
+
+CREATE INDEX IF NOT EXISTS events_postjob_id_idx
+  ON {SCHEMA_NAME}.events(postjob_id);
+
+--- define triggers to automatically create events -----------------------------
+
+--
+-- An event is created whenever a job's status changes.
+--
+
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}._create_postjob_event() RETURNS TRIGGER AS $$
+  BEGIN
+    IF TG_OP = 'UPDATE' AND (OLD.status = NEW.status) THEN
+      RETURN NEW;
+    END IF;
+
+    INSERT INTO {SCHEMA_NAME}.events (postjob_id, name, host_id, worker_session_id)
+      VALUES (
+        NEW.id,
+        NEW.status,
+        (SELECT host_id FROM {SCHEMA_NAME}.worker_sessions WHERE id=NEW.last_worker_session_id),
+        NEW.last_worker_session_id
+      );
+    RETURN NEW;
+  END;
+$$ LANGUAGE plpgsql;
+
+BEGIN;
+  DROP TRIGGER IF EXISTS _create_postjob_event ON {SCHEMA_NAME}.postjobs;
+
+  CREATE TRIGGER _create_postjob_event AFTER INSERT OR UPDATE
+    ON {SCHEMA_NAME}.postjobs
+    FOR EACH ROW
+    EXECUTE PROCEDURE {SCHEMA_NAME}._create_postjob_event();
+COMMIT;
+
+--
+-- An event is also created whenever a worker sends in a heartbeat, see
+-- 013_worker_sessions_functions.sql for details.
+--
+
+--- create an initial set of events -------------------------------------------
+--
+-- This is a best-effort procedure; we cannot generate data that isn't here.
+--
+
+DO $$
+DECLARE
+  null_uuid UUID := {SCHEMA_NAME}._null_uuid();
+BEGIN
+  IF NOT EXISTS (SELECT id FROM {SCHEMA_NAME}.events LIMIT 1) THEN
+    INSERT INTO {SCHEMA_NAME}.events (postjob_id, name, created_at, host_id, worker_session_id)
+    SELECT * FROM (
+      SELECT id, 'ready',         created_at AS created_at, null_uuid, null_uuid FROM {SCHEMA_NAME}.postjobs
+        UNION
+      SELECT id, status::varchar, updated_at AS created_at, null_uuid, null_uuid FROM {SCHEMA_NAME}.postjobs WHERE status != 'ready'
+    ) sq ORDER BY created_at ;
+  END IF;
+END
+$$;

data/lib/postjob/migrations/016_sessions_functions.sql

@@ -0,0 +1,16 @@
+-- worker_session_start: starts or reuses a worker_session ----------------------------------
+
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.worker_session_start(p_host_id UUID, p_workflows VARCHAR[])
+RETURNS SETOF {SCHEMA_NAME}.worker_sessions AS $$
+DECLARE
+  v_worker_session_id UUID;
+  v_client_socket VARCHAR;
+BEGIN
+  SELECT client_addr || ':' || client_port INTO v_client_socket FROM pg_stat_activity WHERE pid = pg_backend_pid();
+
+  INSERT INTO {SCHEMA_NAME}.worker_sessions (host_id, client_socket, workflows)
+    VALUES (p_host_id, v_client_socket, p_workflows) RETURNING id INTO v_worker_session_id;
+
+  RETURN QUERY SELECT * FROM {SCHEMA_NAME}.worker_sessions WHERE id = v_worker_session_id;
+END;
+$$ LANGUAGE plpgsql;

data/lib/postjob/migrations/017_zombie_check.sql

@@ -0,0 +1,58 @@
+-- zombie checks --------------------------------------------------------------
+
+-- This method runs a zombie check. Its result is written into the database as
+-- a 'zombie' event. A zombie check should only happen once per minute.
+--
+-- A zombie is a job whichs runs on a host which hasn't checked in for a while.
+-- (This is currently at 5 minutes)
+
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}._zombie_check(p_fast_mode BOOLEAN) RETURNS int AS $$
+DECLARE
+  zombie_id bigint;
+  _one int;
+  zombie_count int;
+  zombie_threshold interval := interval '5 minutes';
+BEGIN
+  zombie_count := 0;
+  FOR zombie_id, _one IN
+    SELECT jobs.id, 1
+    FROM {SCHEMA_NAME}.postjobs jobs
+    LEFT JOIN {SCHEMA_NAME}.worker_sessions sessions ON jobs.last_worker_session_id=sessions.id
+    LEFT JOIN
+    (
+      SELECT host_id, MAX(created_at) AS created_at
+      FROM {SCHEMA_NAME}.events
+      WHERE name = 'heartbeat'
+      GROUP BY host_id
+    ) heartbeat ON sessions.host_id = heartbeat.host_id
+    WHERE
+      jobs.status IN ('processing')
+      AND (
+        heartbeat.created_at IS NULL OR
+        heartbeat.created_at < ((now() at time zone 'utc') - zombie_threshold)
+      )
+  LOOP
+    PERFORM {SCHEMA_NAME}._set_job_zombie(zombie_id, p_fast_mode);
+    zombie_count := zombie_count + 1;
+  END LOOP;
+
+  RETURN zombie_count;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.zombie_check(p_fast_mode BOOLEAN)
+RETURNS VOID AS $$
+DECLARE
+  zombie_check_interval interval := '1 minute';
+  p_zombie_count int;
+BEGIN
+
+  -- once per minute run a zombie check. This is marked in the database as a zombie
+  -- event, which has a zombie count value in its attributes.
+  IF NOT EXISTS (SELECT 1 FROM {SCHEMA_NAME}.events WHERE name='zombie' AND created_at > (now() at time zone 'utc') - zombie_check_interval) THEN
+    p_zombie_count := {SCHEMA_NAME}._zombie_check(p_fast_mode);
+    INSERT INTO {SCHEMA_NAME}.events(name, host_id, attributes)
+      VALUES('zombie', {SCHEMA_NAME}._null_uuid(), jsonb_build_object('zombie_count', p_zombie_count));
+  END IF;
+END;
+$$ LANGUAGE plpgsql;

data/lib/postjob/migrations/018_heartbeat.sql

@@ -0,0 +1,28 @@
+-- host_heartbeat: sends a heartbeat ------------------------------------------
+
+-- This method is called regularily by workers to pass in a hosts heartbeat
+-- metrics. The database also uses this to regularily run a zombie check.
+
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.host_heartbeat(
+  p_host_id UUID,
+  p_metrics JSONB,
+  p_fast_mode BOOLEAN)
+RETURNS VOID AS $$
+DECLARE
+  p_latest_zombie_event_id bigint;
+  p_zombie_count int;
+BEGIN
+  -- We perform the zombie_check first. This should not make any difference,
+  -- functionality-wise, but helps us test this.
+  PERFORM {SCHEMA_NAME}.zombie_check(p_fast_mode);
+
+  IF NOT EXISTS (
+      SELECT 1 FROM {SCHEMA_NAME}.events
+      WHERE (name,host_id)=('heartbeat', p_host_id) AND created_at > (now() at time zone 'utc') - interval '1 minute'
+    )
+  THEN
+    INSERT INTO {SCHEMA_NAME}.events(name, host_id, attributes)
+      VALUES ('heartbeat', p_host_id, p_metrics);
+  END IF;
+END;
+$$ LANGUAGE plpgsql;

data/lib/postjob/migrations/019_heartbeat_indices.sql

@@ -0,0 +1,5 @@
+-- These indices might be useful for zombie detection.
+
+CREATE INDEX IF NOT EXISTS events_created_at_idx ON {SCHEMA_NAME}.events(created_at);
+CREATE INDEX IF NOT EXISTS events_host_id_idx ON {SCHEMA_NAME}.events(host_id);
+CREATE INDEX IF NOT EXISTS events_worker_session_id_idx ON {SCHEMA_NAME}.events(worker_session_id);

data/lib/postjob/queue.rb

@@ -31,7 +31,7 @@ module Postjob::Queue
   # - parent_id - the id of the parent job, if any
   # - tags       - # a Hash[String => String]
   #
-  def enqueue_job(workflow, *args, options)
+  def enqueue_job(worker_session_id, workflow, *args, options)
     expect! workflow => String
     expect! options => {
       queue: [String, nil],
@@ -49,7 +49,8 @@ module Postjob::Queue
     # a) a limitation in Simple::SQL which would not be able to unpack a
     #    "SELECT function()" usefully when the return value is a record;
     # b) and/or my inability to write better SQL functions;
-    SQL.ask "SELECT * FROM #{SCHEMA_NAME}.enqueue($1, $2, $3, $4, $5, $6, $7, $8, $9)",
+    SQL.ask "SELECT * FROM #{SCHEMA_NAME}.enqueue($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)",
+                  worker_session_id,
                   options[:queue],
                   workflow,
                   workflow_method,
@@ -62,20 +63,20 @@ module Postjob::Queue
                   into: Job
   end
 
-  def set_job_result(job, value, version:)
+  def set_job_result(worker_session_id, job, value, version:)
     value = Encoder.encode([value]) unless value.nil?
-    SQL.ask "SELECT #{SCHEMA_NAME}.set_job_result($1, $2, $3)", job.id, value, version
+    SQL.ask "SELECT #{SCHEMA_NAME}.set_job_result($1, $2, $3, $4)", worker_session_id, job.id, value, version
   end
 
-  def set_job_pending(job, version:)
-    SQL.ask "SELECT #{SCHEMA_NAME}.set_job_pending($1, $2)", job.id, version
+  def set_job_pending(worker_session_id, job, version:)
+    SQL.ask "SELECT #{SCHEMA_NAME}.set_job_pending($1, $2, $3)", worker_session_id, job.id, version
   end
 
-  def set_job_error(job, error, error_message, error_backtrace = nil, status:, version:)
+  def set_job_error(worker_session_id, job, error, error_message, error_backtrace = nil, status:, version:)
     expect! status => [ :failed, :err, :timeout ]
 
-    SQL.ask "SELECT #{SCHEMA_NAME}.set_job_error($1, $2, $3, $4, $5, $6, $7)",
-      job.id, error, error_message, Encoder.encode(error_backtrace), status, version, Postjob.fast_mode
+    SQL.ask "SELECT #{SCHEMA_NAME}.set_job_error($1, $2, $3, $4, $5, $6, $7, $8)",
+      worker_session_id, job.id, error, error_message, Encoder.encode(error_backtrace), status, version, Postjob.fast_mode
   end
 
   def childjobs(parent)
@@ -88,7 +89,7 @@ module Postjob::Queue
     SQL.ask "SELECT COUNT(*) FROM #{SCHEMA_NAME}.unresolved_childjobs($1)", parent.id
   end
 
-  def find_or_create_childjob(parent, workflow, args, timeout:, max_attempts:, queue: nil)
+  def find_or_create_childjob(worker_session_id, parent, workflow, args, timeout:, max_attempts:, queue: nil)
     expect! parent => Job
     expect! workflow => String
     expect! args => Array
@@ -105,7 +106,8 @@ module Postjob::Queue
     # a) a limitation in Simple::SQL which would not be able to unpack a
     #    "SELECT function()" usefully when the return value is a record;
     # b) and/or my inability to write better SQL functions;
-    return SQL.ask "SELECT * FROM #{SCHEMA_NAME}.find_or_create_childjob($1, $2, $3, $4, $5, $6, $7, $8, $9)",
+    return SQL.ask "SELECT * FROM #{SCHEMA_NAME}.find_or_create_childjob($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)",
+                  worker_session_id,
                   queue,
                   workflow,
                   workflow_method,
@@ -118,19 +120,6 @@ module Postjob::Queue
                   into: Job
   end
 
-  def set_workflow_status(job, status)
-    # [TODO] Try to reduce the number of writes.
-    #
-    # The current implementation updates a status potentially multiple times
-    # within a single run of a job (all within the same transaction and therefore
-    # invisible to the outside).
-    SQL.ask <<~SQL, job.id, status
-      UPDATE #{SCHEMA_NAME}.postjobs
-      SET workflow_status=$2
-      WHERE id=$1
-    SQL
-  end
-
   private
 
   def parse_workflow(workflow)
@@ -145,9 +134,11 @@ module Postjob::Queue
 
   public
 
-  def checkout(workflows_with_versions)
-    SQL.ask "SELECT * FROM #{SCHEMA_NAME}.checkout($1, $2)",
-      workflows_with_versions, Postjob.fast_mode, into: Job
+  UUID_REGEXP = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i
+
+  def checkout(worker_session_id)
+    expect! worker_session_id => UUID_REGEXP
+    SQL.ask "SELECT * FROM #{SCHEMA_NAME}.checkout($1, $2)", worker_session_id, Postjob.fast_mode, into: Job
   end
 
   def find_or_create_token(job)
@@ -157,4 +148,27 @@ module Postjob::Queue
   def find_job_by_token(token)
     SQL.ask "SELECT * FROM #{SCHEMA_NAME}.postjobs_by_token($1)", token, into: Job
   end
+
+  # -- registers a host -------------------------------------------------------
+
+  # returns the host id
+  def host_register(attributes)
+    expect! attributes => [ nil, Hash ]
+    Simple::SQL.ask "SELECT postjob.host_register($1)", JSON.generate(attributes)
+  end
+
+  # starts a session
+  WorkerSession = ::Postjob::WorkerSession
+
+  def start_worker_session(workflows_with_versions, host_id:)
+    expect! host_id => UUID_REGEXP
+
+    Simple::SQL.ask "SELECT * FROM postjob.worker_session_start($1, $2)", host_id, workflows_with_versions, into: ::Postjob::WorkerSession
+  end
+
+  # sends in a heartbeat
+  def host_heartbeat(host_id, measurement)
+    Simple::SQL.ask "SELECT postjob.host_heartbeat($1::uuid, $2::jsonb, $3)",
+      host_id, JSON.generate(measurement), ::Postjob.fast_mode
+  end
 end

data/lib/postjob/queue/notifications.rb

@@ -8,14 +8,14 @@ module Postjob::Queue::Notifications
   SCHEMA_NAME = ::Postjob::Queue::SCHEMA_NAME
   MAX_WAIT_TIME = 120
 
-  def wait_for_new_job
+  def wait_for_new_job(worker_session_id)
     started_at = Time.now
 
     start_listening
 
     # Determine when the next job is up. If we don't have a next job within MAX_WAIT_TIME
     # we wake up regardless.
-    wait_time = time_to_next_job
+    wait_time = time_to_next_job(worker_session_id)
     return if wait_time && wait_time <= 0
 
     wait_time = MAX_WAIT_TIME if !wait_time || wait_time > MAX_WAIT_TIME
@@ -43,7 +43,8 @@ module Postjob::Queue::Notifications
 
   # returns the maximum number of seconds to wait until the
   # next runnable or timeoutable job comes up.
-  def time_to_next_job
-    Simple::SQL.ask "SELECT * FROM #{SCHEMA_NAME}.time_to_next_job($1)", ::Postjob::Registry.workflows_with_versions
+  def time_to_next_job(worker_session_id)
+    expect! worker_session_id => /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i
+    Simple::SQL.ask "SELECT * FROM #{SCHEMA_NAME}.time_to_next_job($1)", worker_session_id
   end
 end

data/lib/postjob/queue/search.rb

@@ -14,6 +14,8 @@ module Postjob::Queue
   end
 
   # Builds a search scope (see Simple::SQL::Scope) for the passed in filter criteria.
+  #
+  # Note that the search scope is unsorted.
   def search(filter = {})
     expect! filter => Hash
 

data/lib/postjob/queue/settings.rb

@@ -8,6 +8,16 @@ module Postjob::Queue
   def version
     return "0.3.*" unless settings?
 
-    SQL.ask("SELECT value FROM postjob.settings WHERE name=$1", "version") || "unknown"
+    sql = <<~SQL
+      SELECT 1 from pg_proc
+      left join pg_namespace on pg_proc.pronamespace=pg_namespace.oid
+      where pg_namespace.nspname='postjob' AND pg_proc.proname='settings_get'
+    SQL
+
+    if Simple::SQL.ask(sql)
+      Simple::SQL.ask "SELECT postjob.settings_get('version')"
+    else
+      Simple::SQL.ask("SELECT value FROM postjob.settings WHERE name=$1", "version") || "unknown"
+    end
   end
 end

data/lib/postjob/record.rb

@@ -0,0 +1,17 @@
+# rubocop:disable Style/EvalWithLocation
+# rubocop:disable Security/Eval
+
+#
+# A job class in-memory representation.
+#
+class Postjob::Record < Hash
+  def initialize(hsh)
+    replace hsh.dup
+  end
+
+  def self.attribute(sym)
+    eval <<~RUBY
+      define_method(:#{sym}) { self[:#{sym}] }
+    RUBY
+  end
+end