postjob 0.5.15 → 0.5.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: 4d12e6c2c4ea007c423fd80afd3fbd7647cf4a8d506cfe9361e6ecc315d1dddb
-  data.tar.gz: 0c821070b2d90969ee26282baa3b09358e4903f8d0b338acff884d07cd874aa7
+SHA1:
+  metadata.gz: b740f5e877d921e9581484fd0f4e43d7e5dc6152
+  data.tar.gz: 736ef026988aa18e7d9e1f23c11a0e1876b62815
 SHA512:
-  metadata.gz: 34eb6abcfbae71dcec0f58d15d03992a2d9f20bd85b3901d8942d8a12a06fb55c648fbf9ee954c8eb4a5e599461022b1a6bf5ef55f73f89f30fa534887e99b2f
-  data.tar.gz: a092bb0cceea9e583177d017d43184188fc161c2c7e26f13be892ae3291d17c819a6f12518dfec2042893f5a25c1519befbf5b5f0c954a6753ef7b300bde6c9f
+  metadata.gz: 539d5be9482e50c5b911cb65f5ad689e2dd4a25dbada7adc50c8b96f9d6d06a2527edc09df285aae11185589d6918db6f63352145ba77f7bee0706a49321a48a
+  data.tar.gz: eab7349a9dab72cdf7231d0d88cd2b7ec83085eb12be8063645eda74a6f16dd5d471e843f7eb042008311768a15b987224520d67741ec190d42550342457ee53

data/lib/postjob.rb

@@ -28,6 +28,10 @@ module Postjob
   attr_accessor :logger
   extend self
 
+  def env
+    ENV["POSTJOB_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
+  end
+
   def host_id
     Host.host_id
   end

data/lib/postjob/cli.rb

@@ -22,7 +22,7 @@ module Postjob::CLI
   def load_environment(path)
     return unless File.exist?(path)
 
-    logger.info "#{path}: loading Postjob configuration"
+    logger.debug "#{path}: loading Postjob configuration"
     load path
   end
 

data/lib/postjob/cli/heartbeat.rb

@@ -10,12 +10,12 @@ module Postjob::CLI
       SELECT
         name,
         postjob_id AS job_id,
-        host_id,
-        (attributes->>'uptime')::interval AS uptime,
-        to_char((attributes->>'cpu_load_1min')::float, '99D99') AS cpu_load,
-        attributes->>'net_in_1min' AS net_in,
-        attributes->>'net_out_1min' AS net_out,
-        attributes->>'net_errors_1min' AS net_errors,
+        events.host_id,
+        (events.attributes->>'uptime')::interval AS uptime,
+        to_char((events.attributes->>'cpu_load_1min')::float, '99D99') AS cpu_load,
+        events.attributes->>'net_in_1min' AS net_in,
+        events.attributes->>'net_out_1min' AS net_out,
+        events.attributes->>'net_errors_1min' AS net_errors,
         now() at time zone 'utc' - events.created_at AS age
       FROM postjob.events events
       LEFT JOIN postjob.worker_sessions worker_sessions ON events.worker_session_id=worker_sessions.id

data/lib/postjob/cli/hosts.rb

@@ -72,7 +72,7 @@ module Postjob::CLI
   def host_shutdown
     connect_to_database!
 
-    Simple::SQL.ask "UPDATE postjob.hosts SET status='shutdown' WHERE id=$1::uuid", ::Postjob.host_id
+    Postjob::Host.shutdown!(host_id: ::Postjob.host_id)
   end
 
   # Set the host to running again

data/lib/postjob/cli/run.rb

@@ -1,11 +1,39 @@
-# rubocop:disable Metrics/PerceivedComplexity
-
 module Postjob::CLI
   # Run a single job
   def step(count: 1, queue: nil, host_id: nil)
     run count: count, queue: queue, host_id: host_id, heartbeat: false
   end
 
+  # Start the control connection
+  #
+  # The control connection runs the heartbeat, and checks for jobs on the
+  # control queue.
+  def run_control(host_id: nil)
+    # By running the "control" queue this code is quite efficient, it does
+    # not poll the database.
+    _run(queue: "control", host_id: host_id, heartbeat: true)
+
+    host_id ||= Postjob.host_id
+
+    # Poll until there are no more working sessions.
+    # [TODO] - improve by listening.
+    wait_for_no_running_session(host_id: host_id)
+
+    run_control_shutdown(host_id: host_id)
+  end
+
+  private
+
+  # This is called after the control connection did shut down
+  #
+  # This method can be reimplemented in a plugin to allow shutting down
+  # worker machines.
+  def run_control_shutdown(host_id: nil)
+    Postjob.logger.success "postjob control:shutdown host_id: #{host_id}"
+  end
+
+  public
+
   # Run postjobs.
   #
   # This method runs jobs as they become ready.
@@ -16,12 +44,23 @@ module Postjob::CLI
   # - --queue=queue1,queue2,queue3 run only the specified queues.
   # - --heartbeat=no don't start heartbeat process.
   #
-  def run(count: nil, queue: nil, fast: false, host_id: nil, heartbeat: true)
-    expect! Integer(host_id, 16) => 1..0xffffffff if host_id
+  def run(count: nil, queue: nil, fast: false, host_id: nil)
     count = Integer(count) if count
+    processed = _run(count: count, queue: queue, fast: fast, host_id: host_id, heartbeat: false)
+
+    if !count || processed < count
+      # The runner has been shut down externally. Wait for interrupt.
+      Postjob.logger.success "External shut down initiated."
+      STDERR.puts "External shut down initiated. Press ^C to terminate process."
+      STDIN.read
+    end
+  end
+
+  private
+
+  def _run(count: nil, queue:, fast: false, host_id:, heartbeat:)
+    expect! Integer(host_id, 16) => 1..0xffffffff if host_id
 
-    expect! heartbeat => [ "yes", "no" ] if heartbeat.is_a?(String)
-    heartbeat = %w(yes true).include?(heartbeat) if heartbeat.is_a?(String)
     expect! heartbeat => [ true, false ]
 
     Postjob.fast_mode = (fast ? true : false)
@@ -33,12 +72,25 @@ module Postjob::CLI
       Postjob.logger.info "Using host_id: #{Postjob::Host.host_id}"
     end
 
-    logger.success "Starting runner with pid #{$$}"
-
     processed = Postjob.run(count: count, queues: queue&.split(","), heartbeat: heartbeat) do |job_id|
       logger.info "Processed job w/id #{job_id}" if job_id
     end
 
     logger.info "Processed #{processed} jobs"
+
+    processed
+  end
+
+  def wait_for_no_running_session(host_id:)
+    Postjob.logger.debug "Waiting for shutdown"
+    loop do
+      count = Simple::SQL.ask "SELECT COUNT (*) FROM postjob.worker_sessions WHERE id=$1", host_id
+      Postjob.logger.info "#{host_id}: #{count} running sessions"
+
+      break if count == 0
+      sleep 0.2
+    end
+
+    Postjob.logger.info "#{host_id}: no more running sessions"
   end
 end

data/lib/postjob/cli/sessions.rb

@@ -11,6 +11,7 @@ module Postjob::CLI
       SELECT
         worker_sessions.id,
         (substring(worker_sessions.host_id::varchar for 9) || '...') AS host_id,
+        worker_sessions.status,
         array_to_string(worker_sessions.queues, ', ') AS queues,
         worker_sessions.client_socket,
         worker_sessions.workflows,

data/lib/postjob/host.rb

@@ -1,5 +1,6 @@
 require_relative "./record"
 require "tempfile"
+require "zlib"
 
 class Postjob::Host < Postjob::Record
   attr_reader :id
@@ -29,6 +30,10 @@ class Postjob::Host < Postjob::Record
       @host_id ||= atomic_set_and_get(storage_path) { register_host(host_id: nil) }
     end
 
+    def shutdown!(host_id:)
+      Simple::SQL.ask "UPDATE postjob.hosts SET status='shutdown' WHERE id=$1::uuid", host_id
+    end
+
     private
 
     # This method returns the path to a file which will hold the host_id. Two runners
@@ -40,28 +45,32 @@ class Postjob::Host < Postjob::Record
     # would be even better - however, our systems do not have a user-writable /var).
     def storage_path
       @storage_path ||= begin
-        env = ENV["POSTJOB_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
-        here = Dir.getwd
-        storage_path = File.join Dir.tmpdir, "postjob.#{env}.#{Process.uid}.#{here.hash.abs.to_s(36)}.host_id"
-        Simple::SQL.logger.info "Keeping host identifier in #{storage_path}"
-        storage_path
+        here_hash = Zlib.crc32(Dir.getwd).to_s(36)
+        File.join Dir.tmpdir, "postjob.#{Postjob.env}.#{Process.uid}.#{here_hash}.host_id"
       end
     end
 
     def atomic_set_and_get(path)
-      value = nil
-
       File.open(path, File::RDWR | File::CREAT, 0644) do |f|
         f.flock(File::LOCK_EX)
 
         value = f.read
-        value = yield if value == "" || value.nil?
-        f.rewind
-        f.write(value)
-        f.flush
-        f.truncate(f.pos)
+
+        if value == "" || value.nil?
+          value = yield
+
+          f.rewind
+          f.write(value)
+          f.flush
+          f.truncate(f.pos)
+
+          Postjob.logger.info "Registering new host with host_id #{value}, in #{path}"
+        else
+          Postjob.logger.info "Reusing host_id #{value}, from #{path}"
+        end
+
+        value
       end
-      value
     end
 
     def register_host(host_id:)

data/lib/postjob/migrations/012_hosts.sql

@@ -62,8 +62,8 @@ AS $$
       v_id := gen_random_uuid();
     END IF;
 
-    INSERT INTO {SCHEMA_NAME}.hosts (id, attributes)
-      VALUES (v_id, p_attrs)
+    INSERT INTO {SCHEMA_NAME}.hosts (id, attributes, status)
+      VALUES (v_id, p_attrs, 'running')
     ON CONFLICT(id) DO UPDATE SET attributes=p_attrs;
     RETURN v_id;
   END;

data/lib/postjob/migrations/016_sessions_functions.sql

@@ -13,13 +13,48 @@ BEGIN
   INSERT INTO {SCHEMA_NAME}.worker_sessions (host_id, client_socket, workflows, queues)
     VALUES (p_host_id, v_client_socket, p_workflows, p_queues) RETURNING id INTO v_worker_session_id;
 
+  UPDATE {SCHEMA_NAME}.hosts 
+    SET status = 'running'
+    WHERE id=p_host_id;
+
   RETURN QUERY SELECT * FROM {SCHEMA_NAME}.worker_sessions WHERE id = v_worker_session_id;
 END;
 $$ LANGUAGE plpgsql;
 
 CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.worker_session_stop(p_worker_session_id UUID)
 RETURNS VOID AS $$
+DECLARE
+  v_host_id uuid;
 BEGIN
-  UPDATE {SCHEMA_NAME}.worker_sessions SET status='stopped' WHERE id=p_worker_session_id;
+  UPDATE {SCHEMA_NAME}.worker_sessions
+    SET status='stopped' WHERE id=p_worker_session_id;
 END;
 $$ LANGUAGE plpgsql;
+
+-- wakeup runners after changing hosts
+
+-- when a host changes its status to shutdown, all of its runners should
+-- shutdown quickly.
+
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}._recalculate_host_status() RETURNS TRIGGER AS $$
+  BEGIN
+    IF NOT EXISTS (
+      SELECT 1 FROM {SCHEMA_NAME}.worker_sessions WHERE id=NEW.host_id AND status = 'running'
+    ) THEN
+      UPDATE {SCHEMA_NAME}.hosts 
+        SET status = 'stopped'
+        WHERE id=NEW.host_id;
+    END IF;
+
+    RETURN NEW;
+  END;
+$$ LANGUAGE plpgsql;
+
+BEGIN;
+  DROP TRIGGER IF EXISTS _recalculate_host_status ON {SCHEMA_NAME}.worker_sessions;
+
+  CREATE TRIGGER _recalculate_host_status AFTER UPDATE
+    ON {SCHEMA_NAME}.worker_sessions
+    FOR EACH ROW
+    EXECUTE PROCEDURE {SCHEMA_NAME}._recalculate_host_status();
+COMMIT;

data/lib/postjob/queue/notifications.rb

@@ -20,7 +20,7 @@ module Postjob::Queue::Notifications
     return if wait_time && wait_time <= 0
 
     if !wait_time && ::Postjob::Queue.should_shutdown?(worker_session_id)
-      Postjob.logger.warn "Shutting down runner: host is set to 'shutdown'"
+      Postjob.logger.debug "Shutting down runner: host is set to 'shutdown'"
       return :shutdown
     end
 

data/lib/postjob/registry.rb

@@ -50,6 +50,18 @@ class Postjob::Registry
     instance.register(workflow, options)
   end
 
+  def self.load(glob_pattern)
+    Dir.glob(glob_pattern).sort.each do |path|
+      before = Postjob::Registry.workflow_names
+      Kernel.load path
+      after = Postjob::Registry.workflow_names
+      new_workflows = after - before
+      next if new_workflows.empty?
+
+      Postjob.logger.debug "#{path}: registered workflow(s) #{new_workflows.join(', ')}"
+    end
+  end
+
   class WorkflowSpec
     class Options
       DEFAULTS = {

data/lib/postjob/worker_session.rb

@@ -12,7 +12,7 @@ class Postjob::WorkerSession < Postjob::Record
       host_id = ::Postjob.host_id
       worker_session = ::Postjob::Queue.worker_session_start(workflows_with_versions, host_id: host_id, queues: queues)
 
-      Postjob.logger.info "Starting worker_session #{worker_session.inspect}"
+      Postjob.logger.info "Starting worker_session #{worker_session.inspect}, on pid #{$$}"
 
       start_heartbeat_monitor(host_id) if heartbeat
       worker_session

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: postjob
 version: !ruby/object:Gem::Version
-  version: 0.5.15
+  version: 0.5.16
 platform: ruby
 authors:
 - radiospiel
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-08-22 00:00:00.000000000 Z
+date: 2018-08-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -327,7 +327,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: restartable, asynchronous, and distributed processes