postjob 0.4.5 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 78a70bee1c76f0285da58ecc0e1f38f4d420d9f1
-  data.tar.gz: 4fe406b37ebe91e317c00a6c8cb84fceb6b5ce55
+  metadata.gz: 00a785d7f9bd7a640601fb385902e157a76bf02a
+  data.tar.gz: 7368f6f79e2f0392979656ff05bd12acc20751e9
 SHA512:
-  metadata.gz: 3e55f152396f288ee5f51ccbc08ab23cb54e82f8afac7b8b69a248d38ff2ada757c075292008810ed749ba585f01591d87cb2780ab49289b46948a83b7360296
-  data.tar.gz: b7ccda0358d5377e8d4bcb089e736fd3f06efe5149f82a6ee792870dcfc05f5966f739313fd53834f501fa9901afdd76473b75be22fd70a780b751cc621a0241
+  metadata.gz: a09cd7a9014e1dddf95f493b064ae8e818ff65b98fd0db3ba15bf8de50e499b5d8c45ec919db52f994692c14b0550aa184dd74d9d4b76ba033c4109de59562f5
+  data.tar.gz: 748047574250f33ea51cad3a823ecb2c7d66afa7263c4388d66157a0e58948433676935816794dde089b0c8f1cbcf10c24de6310723d12f61e578a6a08c4986e

data/lib/postjob.rb

@@ -12,6 +12,7 @@ end
 require_relative "postjob/workflow"
 require_relative "postjob/registry"
 require_relative "postjob/job"
+require_relative "postjob/worker_session"
 require_relative "postjob/error"
 require_relative "postjob/queue"
 require_relative "postjob/runner"
@@ -52,12 +53,12 @@ module Postjob
     end
 
     tags = stringify_hash(tags) if tags
-    job = Queue.enqueue_job workflow, *args, queue: queue,
-                                             parent_id: parent_id,
-                                             max_attempts: max_attempts,
-                                             timeout: timeout,
-                                             tags: tags,
-                                             version: version
+    job = Queue.enqueue_job current_worker_session.id, workflow, *args, queue: queue,
+                                                                        parent_id: parent_id,
+                                                                        max_attempts: max_attempts,
+                                                                        timeout: timeout,
+                                                                        tags: tags,
+                                                                        version: version
     logger.info "Generated process #{job}"
     job.id
   end
@@ -120,7 +121,7 @@ module Postjob
       break if shutdown == :shutdown
 
       next if processed_job_id
-      Queue::Notifications.wait_for_new_job
+      Queue::Notifications.wait_for_new_job(current_worker_session.id)
     end
 
     processed_jobs_count
@@ -139,10 +140,16 @@ module Postjob
   #
   # or nil, when no job could be checked out.
   def step
-    job = Queue.checkout(Registry.workflows_with_versions)
+    job = Postjob::Queue.checkout(current_worker_session.id)
     [ job.id, process_job(job) ] if job
   end
 
+  # This method connects to the queue. This means it registers as a new worker_session,
+  # if there was no worker_session yet.
+  def current_worker_session
+    @worker_session ||= WorkerSession.start!(Registry.workflows_with_versions)
+  end
+
   private
 
   # This method is called from tests. Otherwise it is supposed to be private.
@@ -158,11 +165,13 @@ module Postjob
       raise "Integrity check failed: job's workflow version changed (from #{job.workflow_version} to #{version})"
     end
 
+    worker_session_id = current_worker_session.id
+
     case status
-    when :failed    then Queue.set_job_error job, *value, status: :failed, version: version
-    when :err       then Queue.set_job_error job, *value, status: :err, version: version
-    when :pending   then Queue.set_job_pending job, version: version
-    when :ok        then Queue.set_job_result job, value, version: version
+    when :failed    then Queue.set_job_error worker_session_id, job, *value, status: :failed, version: version
+    when :err       then Queue.set_job_error worker_session_id, job, *value, status: :err, version: version
+    when :pending   then Queue.set_job_pending worker_session_id, job, version: version
+    when :ok        then Queue.set_job_result worker_session_id, job, value, version: version
     else            raise ArgumentError, "Invalid status #{status.inspect}"
     end
 
@@ -175,7 +184,7 @@ module Postjob
     job = Queue.find_job_by_token(token)
     raise "No job with token #{token}" unless job
 
-    Queue.set_job_result job, result, version: nil
+    Queue.set_job_result current_worker_session.id, job, result, version: nil
   end
 
   def register_workflow(workflow, options = {})

data/lib/postjob/cli/events.rb

@@ -0,0 +1,60 @@
+# rubocop:disable Lint/HandleExceptions
+# rubocop:disable Metrics/MethodLength
+
+module Postjob::CLI
+  private
+
+  def events_query(limit:)
+    limit = Integer(limit)
+
+    sql = <<-SQL
+      SELECT
+        events.id,
+        events.name,
+        events.postjob_id AS job_id,
+        postjobs.workflow
+          || (CASE WHEN postjobs.workflow_version != '' THEN '@' ELSE '' END)
+          || postjobs.workflow_version
+          || (CASE WHEN postjobs.workflow_method != 'run' THEN '.' || postjobs.workflow_method ELSE '' END)
+          || postjobs.args AS job,
+          worker_session_id,
+        events.created_at
+      FROM postjob.events events
+      LEFT JOIN postjob.postjobs postjobs ON events.postjob_id=postjobs.id
+      WHERE events.name != 'heartbeat'
+    SQL
+
+    scope = Simple::SQL::Scope.new(sql)
+    scope
+      .order_by("events.id DESC")
+      .paginate(per: limit, page: 1)
+  end
+
+  public
+
+  # Show the latest job event
+  #
+  # Example:
+  #
+  #   postjob events
+  def events(limit: "100")
+    expect! limit => /\A\d+\z/
+    limit = Integer(limit)
+
+    connect_to_database!
+
+    query = events_query(limit: limit)
+
+    print_results query: query
+  end
+
+  # Show up-to-date events information once per second
+  def events_top(limit: "100")
+    loop do
+      system "clear"
+      events(limit: limit)
+      sleep 1
+    end
+  rescue Interrupt
+  end
+end

data/lib/postjob/cli/heartbeat.rb

@@ -0,0 +1,55 @@
+# rubocop:disable Lint/HandleExceptions
+
+module Postjob::CLI
+  private
+
+  def heartbeat_query(limit:)
+    limit = Integer(limit)
+
+    sql = <<-SQL
+      SELECT
+        name,
+        postjob_id AS job_id,
+        host_id,
+        (attributes->>'uptime')::interval AS uptime,
+        to_char((attributes->>'cpu_load_1min')::float, '99D99') AS cpu_load,
+        attributes->>'net_in_1min' AS net_in,
+        attributes->>'net_out_1min' AS net_out,
+        attributes->>'net_errors_1min' AS net_errors,
+        now() at time zone 'utc' - events.created_at AS age
+      FROM postjob.events events
+      LEFT JOIN postjob.worker_sessions worker_sessions ON events.worker_session_id=worker_sessions.id
+      WHERE events.name = 'heartbeat'
+    SQL
+
+    scope = Simple::SQL::Scope.new(sql)
+    scope
+      .order_by("events.id DESC")
+      .paginate(per: limit, page: 1)
+  end
+
+  public
+
+  # Show the latest heartbeat events
+  def heartbeat(limit: "100")
+    expect! limit => /\A\d+\z/
+    limit = Integer(limit)
+
+    connect_to_database!
+
+    query = heartbeat_query(limit: limit)
+
+    Postjob.logger.info "CPU load and friends are for the last minute"
+    print_results query: query
+  end
+
+  # Show up-to-date heartbeat information once per second
+  def heartbeat_top(limit: "100")
+    loop do
+      system "clear"
+      heartbeat(limit: limit)
+      sleep 1
+    end
+  rescue Interrupt
+  end
+end

data/lib/postjob/cli/hosts.rb

@@ -0,0 +1,67 @@
+# rubocop:disable Lint/HandleExceptions
+# rubocop:disable Metrics/MethodLength
+
+module Postjob::CLI
+  private
+
+  def hosts_query(limit:)
+    limit = Integer(limit)
+
+    sql = <<-SQL
+      SELECT
+        hosts.id,
+        hosts.attributes,
+        hosts.created_at,
+        heartbeat.attributes AS heartbeat,
+        heartbeat.created_at AS heartbeat_created_at
+      FROM postjob.hosts hosts
+      LEFT JOIN (
+        SELECT
+          worker_sessions.host_id,
+          MAX(events.id) AS event_id
+        FROM postjob.worker_sessions
+        LEFT JOIN postjob.events events ON events.worker_session_id=worker_sessions.id
+        WHERE events.name = 'heartbeat'
+        GROUP BY worker_sessions.host_id
+      ) q ON q.host_id=hosts.id
+      LEFT JOIN events heartbeat ON heartbeat.id=event_id
+    SQL
+
+    scope = Simple::SQL::Scope.new(sql)
+    scope
+      .order_by("hosts.created_at DESC NULLS LAST")
+      .paginate(per: limit, page: 1)
+  end
+
+  public
+
+  # Show hosts status
+  #
+  # This command lists all worker_sessions currently in the system.
+  #
+  # Example:
+  #
+  #   postjob hosts
+  def hosts(limit: "100")
+    expect! limit => /\A\d+\z/
+    limit = Integer(limit)
+
+    connect_to_database!
+
+    query = hosts_query(limit: limit)
+
+    print_results query: query
+  end
+
+  # Show up-to-date hosts information once per second
+  #
+  #
+  def hosts_top(limit: "100")
+    loop do
+      system "clear"
+      hosts(limit: limit)
+      sleep 1
+    end
+  rescue Interrupt
+  end
+end

data/lib/postjob/cli/ps.rb

@@ -28,18 +28,6 @@ module Postjob::CLI
         next_run_at - (now() at time zone 'utc') AS next_run_in,
         to_char(EXTRACT(EPOCH FROM (now() at time zone 'utc') - postjobs.created_at), '999999999.99') AS age,
 
-        CASE
-        WHEN processing_started_at IS NOT NULL THEN
-          format(
-            '%s/%s',
-            to_char(EXTRACT(EPOCH FROM (now() at time zone 'utc') - processing_started_at), '999999999.99'),
-            processing_max_duration
-          )
-        WHEN status IN ('failed', 'err', 'ok') THEN
-          format('%s', to_char(EXTRACT(EPOCH FROM (updated_at - created_at)), '999999999.99'))
-        END AS processing,
-
-        COALESCE(processing_client, '') || COALESCE('/' || processing_client_identifier, '') AS worker,
         tags
       FROM postjob.postjobs AS postjobs
     SQL
@@ -172,7 +160,7 @@ module Postjob::CLI
     tp records
 
     if records.total_count > records.length
-      logger.warn "Output limited up to limit #{records.length}. Use the --limit command line option for a different limit."
+      logger.warn "Output limited up to limit #{records.length}. Use the --limit=<NN> command line option for a different limit."
     end
 
     if records.empty? && on_empty

data/lib/postjob/cli/sessions.rb

@@ -0,0 +1,83 @@
+# rubocop:disable Lint/HandleExceptions
+# rubocop:disable Metrics/MethodLength
+
+module Postjob::CLI
+  private
+
+  def sessions_query(limit:)
+    limit = Integer(limit)
+
+    sql = <<-SQL
+      SELECT
+        worker_sessions.id,
+        worker_sessions.host_id,
+        worker_sessions.client_socket,
+        worker_sessions.workflows,
+        worker_sessions.created_at,
+        job_event.name AS event_name,
+        job_event.created_at AS event_created_at,
+        heartbeat.attributes AS heartbeat,
+        heartbeat.created_at AS heartbeat_created_at
+      FROM postjob.worker_sessions AS worker_sessions
+      LEFT JOIN (
+        SELECT
+          worker_sessions.id,
+          MAX(events.id) AS event_id
+        FROM postjob.worker_sessions
+        LEFT JOIN postjob.events events ON events.worker_session_id=worker_sessions.id
+        WHERE events.name != 'heartbeat'
+        GROUP BY worker_sessions.id
+      ) last_job_event ON last_job_event.id=worker_sessions.id
+      LEFT JOIN postjob.events job_event ON job_event.id=last_job_event.event_id
+      LEFT JOIN (
+        SELECT
+          worker_sessions.id,
+          MAX(events.id) AS event_id
+        FROM postjob.worker_sessions
+        LEFT JOIN postjob.events events ON events.worker_session_id=worker_sessions.id
+        WHERE events.name = 'heartbeat'
+        GROUP BY worker_sessions.id
+      ) last_heartbeat ON last_heartbeat.id=worker_sessions.id
+      LEFT JOIN postjob.events heartbeat ON heartbeat.id=last_heartbeat.event_id
+    SQL
+
+    scope = Simple::SQL::Scope.new(sql)
+
+    scope
+      .paginate(per: limit, page: 1)
+      .order_by("heartbeat_created_at DESC NULLS LAST")
+  end
+
+  public
+
+  # Show sessions status
+  #
+  # This command lists all worker sessions currently in the system.
+  #
+  # Example:
+  #
+  #   postjob sessions
+  def sessions(limit: "100")
+    expect! limit => /\A\d+\z/
+    limit = Integer(limit)
+
+    connect_to_database!
+
+    # check for timed out and zombie processes
+    # ::Postjob::Queue.checkout(nil)
+
+    query = sessions_query(limit: limit)
+
+    print_results query: query
+  end
+
+  # Show up-to-date session information once per second
+  def sessions_top(limit: "100")
+    loop do
+      system "clear"
+      sessions(limit: limit)
+      sleep 1
+    end
+  rescue Interrupt
+  end
+end

data/lib/postjob/job.rb

@@ -1,25 +1,14 @@
-# rubocop:disable Style/EvalWithLocation
-# rubocop:disable Security/Eval
+require_relative "./record"
 
 #
-# A job class in-memory representation.
+# A job
 #
-class Postjob::Job < Hash
-  def initialize(hsh)
-    replace hsh.dup
-  end
-
+class Postjob::Job < Postjob::Record
   def self.find(job_id)
     scope = Postjob::Queue.search(id: job_id)
     Simple::SQL.ask(scope, into: Postjob::Job)
   end
 
-  def self.attribute(sym)
-    eval <<~RUBY
-      define_method(:#{sym}) { self[:#{sym}] }
-    RUBY
-  end
-
   attribute :id
   attribute :parent_id
   attribute :full_id
@@ -40,9 +29,9 @@ class Postjob::Job < Hash
   attribute :error_message
   attribute :error_backtrace
   attribute :recipients
-  attribute :workflow_status
   attribute :timed_out
   attribute :tags
+  attribute :last_worker_session_id
 
   STATUSES = %w(ok ready processing sleep err failed timeout)
 

data/lib/postjob/migrations/003_postjobs.sql

@@ -38,6 +38,8 @@ CREATE TABLE IF NOT EXISTS {SCHEMA_NAME}.postjobs (
   -- Number of failed attempts so far.
   failed_attempts INTEGER NOT NULL DEFAULT 0,
 
+  -- last_worker_session_id UUID NOT NULL REFERENCES {SCHEMA_NAME}.worker_sessions ON DELETE CASCADE,
+
   -- process result ---------------------------------------------------------------------------------------
 
   results JSONB,            -- The process result, if any. Only valid when status == 'ok'
@@ -46,8 +48,8 @@ CREATE TABLE IF NOT EXISTS {SCHEMA_NAME}.postjobs (
   error_backtrace JSONB,    -- additional error information, for debugging purposes
 
   -- custom fields ----------------------------------------------------------------------------------------
-  workflow_status VARCHAR,
-  tags JSONB,
+  -- workflow_status VARCHAR,
+  tags JSONB
 
   -- processing_client information ------------------------------------------------------------------------
   -- This information is passed along from workers during processing. They are only valid
@@ -55,12 +57,12 @@ CREATE TABLE IF NOT EXISTS {SCHEMA_NAME}.postjobs (
   --
   -- Initially these columns didn't exist, and have been created via another migration
   -- (003b_processing_columns.sql). They are listed here for documentation purposes.
-  processing_client varchar,              -- host:port of client (taken from pg_stat_activity)
-  processing_client_identifier varchar,   -- free text info, set via set_client_identifier()
-  processing_started_at timestamp,        -- when did processing start?
-  processing_max_duration float           -- maximum expected duration of processing. Afterwards the
-                                          -- processing is considered failed for unknown reasons, and
-                                          -- potentially restarted.
+  -- processing_client varchar,               -- host:port of client (taken from pg_stat_activity)
+  -- processing_client_identifier varchar,    -- free text info, set via set_client_identifier()
+  -- processing_started_at timestamp          -- when did processing start?
+  -- processing_max_duration float            -- maximum expected duration of processing. Afterwards the
+                                              -- processing is considered failed for unknown reasons, and
+                                              -- potentially restarted.
 );
 
 -- [TODO] check indices