postjob 0.5.5 → 0.5.6

data/lib/postjob/queue.rb

@@ -2,10 +2,11 @@
 # rubocop:disable Metrics/MethodLength
 # rubocop:disable Metrics/ParameterLists
 # rubocop:disable Metrics/LineLength
+# rubocop:disable Metrics/ModuleLength
 
 require "securerandom"
 
-# The Postjob::Queue manages enqueueing and fetching jobs from a job queue.
+# The Postjob::Queue module manages enqueueing and fetching jobs from a job queue.
 module Postjob::Queue
   extend self
 
@@ -20,7 +21,7 @@ require_relative "queue/associations"
 require_relative "queue/settings"
 
 module Postjob::Queue
-  Job = ::Postjob::Job
+  Job = ::Postjob::Job # :nodoc:
 
   # enqueues a new job with the given arguments
   #
@@ -41,7 +42,8 @@ module Postjob::Queue
       parent_id: [Integer, nil],
       tags: [Hash, nil],
       timeout: [Numeric, nil],
-      max_attempts: [Integer, nil]
+      max_attempts: [Integer, nil],
+      sticky: [true, false, nil]
     }
 
     workflow, workflow_method = parse_workflow(workflow)
@@ -51,7 +53,7 @@ module Postjob::Queue
     # a) a limitation in Simple::SQL which would not be able to unpack a
     #    "SELECT function()" usefully when the return value is a record;
     # b) and/or my inability to write better SQL functions;
-    SQL.ask "SELECT * FROM #{SCHEMA_NAME}.enqueue($1::uuid, $2, $3, $4, $5, $6, $7, $8, $9, $10)",
+    SQL.ask "SELECT * FROM #{SCHEMA_NAME}.enqueue($1::uuid, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)",
                   worker_session_id,
                   options[:queue],
                   workflow,
@@ -62,9 +64,26 @@ module Postjob::Queue
                   Encoder.encode(options[:tags]),
                   options[:max_attempts],
                   options[:timeout],
+                  options[:cron_interval],
+                  options[:sticky],
                   into: Job
   end
 
+  # Disable cron'ness for workflow + args combination
+  def disable_cron_jobs(workflow, args)
+    sql = <<~SQL
+      UPDATE postjob.postjobs
+      SET cron_interval = NULL
+      WHERE cron_interval IS NOT NULL
+        AND workflow=$1
+        AND args=$2
+        AND parent_id IS NULL
+        AND status NOT IN ('ok', 'failed', 'timeout')
+      SQL
+
+    Simple::SQL.ask sql, workflow, Encoder.encode(args)
+  end
+
   def set_job_result(worker_session_id, job, value, version:)
     value = Encoder.encode([value]) unless value.nil?
     SQL.ask "SELECT #{SCHEMA_NAME}.set_job_result($1::uuid, $2, $3, $4)", worker_session_id, job.id, value, version

data/lib/postjob/record.rb

@@ -1,17 +1,11 @@
-# rubocop:disable Style/EvalWithLocation
-# rubocop:disable Security/Eval
-
 #
 # A job class in-memory representation.
 #
-class Postjob::Record < Hash
+class Postjob::Record < Hash # :nodoc:
   def initialize(hsh)
     replace hsh.dup
-  end
-
-  def self.attribute(sym)
-    eval <<~RUBY
-      define_method(:#{sym}) { self[:#{sym}] }
-    RUBY
+    each do |key, value|
+      instance_variable_set "@#{key}", value
+    end
   end
 end

data/lib/postjob/registry.rb

@@ -1,37 +1,134 @@
 # The registry holds a list of all available workflows
-module Postjob::Registry
-  extend self
+class Postjob::Registry
+  def self.instance
+    @instance ||= new
+  end
 
-  def workflows
-    instance.values.uniq
+  # Used for tests
+  def self.reset! # :nodoc:
+    @instance = nil
   end
 
-  def workflows_with_versions
-    @workflows_with_versions ||= []
+  def self.workflows
+    instance.workflows
   end
 
-  # Used for tests
-  def reset! # :nodoc:
-    @instance = @workflows_with_versions = nil
+  def self.workflow_names
+    instance.workflows.keys.map(&:first).uniq
   end
 
-  def register(workflow, _options = {})
-    instance[[workflow.name, ""]] = workflow
-    instance[[workflow.name, workflow.workflow_version]] = workflow
+  def self.runnable_workflows_with_versions
+    workflows
+      .select { |_, spec| spec.runnable? }
+      .inject([]) do |ary, (name_and_version, _spec)|
+        name, version = *name_and_version
+        ary << name << "#{name}#{version}"
+      end
+  end
 
-    workflows_with_versions << workflow.name << "#{workflow.name}#{workflow.workflow_version}"
+  def self.workflows_with_versions
+    instance.workflows_with_versions
   end
 
-  def lookup!(name:, version:)
-    expect! name => String
-    expect! version => String
+  def self.register(workflow, options = {})
+    instance.register(workflow, options)
+  end
+
+  class WorkflowSpec
+    class Options
+      DEFAULTS = {
+        version: "0.0",
+        max_attempts: 5,
+        timeout: 15 * 60,
+        sticky: false,
+        cron_interval: nil,
+        queue: "q"
+      }
+
+      attr_reader :version
+      attr_reader :max_attempts
+      attr_reader :timeout
+      attr_reader :sticky
+      attr_reader :cron_interval
+      attr_reader :queue
+
+      def initialize(options)
+        expect! options => {
+          version:        [ nil, /^\d+(\.\d+)*/ ],
+          max_attempts:   [ nil, Integer ],
+          timeout:        [ nil, Integer ],
+          sticky:         [ nil, true, false ],
+          cron_interval:  [ nil, Integer ],
+          queue:          [ nil, String ]
+        }
+
+        options = DEFAULTS.merge(options)
+
+        @version        = options[:version]
+        @max_attempts   = options[:max_attempts]
+        @timeout        = options[:timeout]
+        @sticky         = options[:sticky]
+        @cron_interval  = options[:cron_interval]
+        @queue          = options[:queue]
+      end
+    end
+
+    attr_reader :workflow
+    attr_reader :options
+
+    def runnable?
+      workflow.is_a?(Module)
+    end
 
-    instance.fetch([name, version])
+    def name
+      case workflow
+      when Module then workflow.name
+      when String then workflow
+      end
+    end
+
+    def initialize(workflow, options)
+      expect! workflow => [ Module, String ]
+      @workflow = workflow
+      @options  = Options.new(options)
+    end
+  end
+
+  # looks up a specific version of a specific workflow. Returns a WorkflowSpec
+  # object.
+  def self.lookup!(name:, version:)
+    instance.lookup! name: name, version: version
+  end
+
+  def self.[](name)
+    lookup! name: name, version: ""
+  rescue KeyError
+    nil
+  end
+
+  attr_reader :workflows
+  attr_reader :workflows_with_versions
+
+  def initialize
+    @workflows = {}
+    @workflows_with_versions = []
   end
 
-  private
+  def register(workflow, options)
+    spec = WorkflowSpec.new(workflow, options)
+
+    @workflows_with_versions << spec.name << "#{spec.name}#{spec.options.version}"
+
+    @workflows[[spec.name, ""]] = spec
+    @workflows[[spec.name, spec.options.version]] = spec
+  end
+
+  # looks up a specific version of a specific workflow. Returns the workflow
+  # module itself
+  def lookup!(name:, version:)
+    expect! name => String
+    expect! version => String
 
-  def instance
-    @instance ||= {}
+    @workflows.fetch([name, version])
   end
 end

data/lib/postjob/runner.rb

@@ -3,194 +3,197 @@
 # rubocop:disable Lint/RescueException
 # rubocop:disable Metrics/MethodLength
 
+# Base implementations for Runners
+#
+# This module contains methods for runners.
 module Postjob::Runner
-  extend self
+  class << self
+    extend Forwardable
+    delegate [:logger] => Postjob
+
+    Job = Postjob::Job
+
+    # returns the job that is currently running.
+    #
+    # This value is set by +process_job+ (via +with_current_job+), and
+    # currently only used from <tt>Postjob::Runner.async</tt>
+    def current_job
+      Thread.current[:current_job]
+    end
 
-  extend Forwardable
-  delegate [:logger] => Postjob
+    private
 
-  Job = Postjob::Job
+    def with_current_job(job)
+      expect! current_job => nil
+      Thread.current[:current_job] = job
+      yield
+    ensure
+      Thread.current[:current_job] = nil
+    end
 
-  # returns the job that is currently running.
-  #
-  # This value is set by +process_job+ (via +with_current_job+), and
-  # currently only used from <tt>Postjob::Runner.async</tt>
-  def current_job
-    Thread.current[:current_job]
-  end
+    public
+
+    # returns a subjob within the current job, for a +runner+
+    # description and +args+.
+    def async(workflow, *args, timeout: nil, max_attempts:)
+      worker_session_id = Postjob.current_worker_session.id
+
+      # if the workflow is a symbol, then we change it into "__manual__"
+      # - there should never be a workflow with that name - or into
+      # "CurrentWorkshop.#{workflow}", denoting the \a workflow method of the
+      # current workflow.
+      case workflow
+      when :manual  then workflow = "__manual__"
+      when Symbol   then workflow = "#{current_job.workflow}.#{workflow}"
+      when Module   then workflow = workflow.name
+      when String   then :nop
+      else
+        raise ArgumentError, "Unsupported workflow spec #{workflow.inspect}. Did you run await(fun(a, b)) instead of await(:fun, a, b)"
+      end
 
-  private
+      ::Postjob::Queue.find_or_create_childjob(worker_session_id, self.current_job, workflow, args,
+                                               timeout: timeout,
+                                               max_attempts: max_attempts)
+    end
 
-  def with_current_job(job)
-    expect! current_job => nil
-    Thread.current[:current_job] = job
-    yield
-  ensure
-    Thread.current[:current_job] = nil
-  end
+    # tries to resolve a job.
+    def await(job, *args, timeout: nil, max_attempts: nil)
+      case job
+      when :all
+        unresolved_childjobs = Postjob::Queue.unresolved_childjobs(current_job)
+        if unresolved_childjobs > 0
+          Postjob.logger.debug "await :all: Found #{unresolved_childjobs} unresolved childjobs"
+          throw :pending, :pending
+        else
+          childjobs = Postjob::Queue.childjobs(current_job)
+          childjobs.each(&:resolve).count
+        end
+      when Job
+        expect! args == []
+        expect! timeout => nil, max_attempts => nil
+        r = job.resolve
+        throw :pending, :pending if r == :pending
+        r
+      else
+        job = async(job, *args, timeout: timeout, max_attempts: max_attempts)
+        await(job)
+      end
+    end
 
-  public
-
-  # returns a subjob within the current job, for a +runner+
-  # description and +args+.
-  def async(workflow, *args, timeout: nil, max_attempts:)
-    worker_session_id = Postjob.current_worker_session.id
-
-    # if the workflow is a symbol, then we change it into "__manual__"
-    # - there should never be a workflow with that name - or into
-    # "CurrentWorkshop.#{workflow}", denoting the \a workflow method of the
-    # current workflow.
-    case workflow
-    when :manual  then workflow = "__manual__"
-    when Symbol   then workflow = "#{current_job.workflow}.#{workflow}"
-    when Module   then workflow = workflow.name
-    when String   then :nop
-    else
-      raise ArgumentError, "Unsupported workflow spec #{workflow.inspect}. Did you run await(fun(a, b)) instead of await(:fun, a, b)"
+    STATUSES = [ :sleep, :ok, :err, :failed ]
+
+    #
+    # runs a specific job
+    #
+    # returns a tuple [status, value], which follows the following pattern:
+    #
+    # - <tt>[ <runner-version>, :ok, value, nil ]</tt>: job completed successfully
+    # - <tt>[ <runner-version>, :sleep, nil, nil ]</tt>: job has to wait on a child job
+    # - <tt>[ <runner-version>, :err, <err>, nil ]</tt>: job errored with a recoverable error
+    # - <tt>[ <runner-version>, :failed, <err>, <shutdown> ]</tt>: job failed with a non-recoverable error
+    #
+    # <err> is a tuple [ error-class-name, error-message, stacktrace ].
+    # <shutdown> is either nil or :shutdown
+    #
+    def process_job(job)
+      expect! job => Job
+
+      spec = Postjob::Registry.lookup!(name: job.workflow, version: job.workflow_version)
+      expect! spec.runnable?
+
+      with_current_job(job) do
+        status, value, shutdown = invoke_workflow spec.workflow, job
+        log_result! job, status, value
+        # If the status is ok the job finished processing. In that case
+        # we'll wait for all child jobs to finish.
+        # if status == :ok
+        #   await :all
+        # end
+        [ spec.options.version, status, value, shutdown ]
+      end
     end
 
-    ::Postjob::Queue.find_or_create_childjob(worker_session_id, self.current_job, workflow, args,
-                                             timeout: timeout,
-                                             max_attempts: max_attempts)
-  end
+    private
+
+    # runs a job. Returns a [ status, value, shutdown ] tuple.
+    #
+    # The shutdown value is used by a worker in run mode (i.e. process
+    # indefinetively) to determine whether or not it should cancel
+    # processing. It is usually nil; but if the worker received a
+    # SIGINT it will be :shutdown instead.
+    #
+    # We are catching SIGINT to allow the job status to be updated.
+    def invoke_workflow(workflow, job)
+      value = catch(:pending) {
+        expect! job.args => [Array, nil]
+
+        workflow_method = job.workflow_method
+        args = job.args
+
+        insp_args = args.map(&:inspect).join(", ")
+        logger.info "Running Postjob##{job.id}: #{job.workflow}.#{workflow_method}(#{insp_args})"
 
-  # tries to resolve a job.
-  def await(job, *args, timeout: nil, max_attempts: nil)
-    case job
-    when :all
-      unresolved_childjobs = Postjob::Queue.unresolved_childjobs(current_job)
-      if unresolved_childjobs > 0
-        Postjob.logger.debug "await :all: Found #{unresolved_childjobs} unresolved childjobs"
-        throw :pending, :pending
+        workflow.public_send workflow_method, *args
+      }
+
+      if value == :pending
+        [ :pending, nil, nil ]
       else
-        childjobs = Postjob::Queue.childjobs(current_job)
-        childjobs.each(&:resolve).count
+        # Check that we can encode the value. If we can't the job returned something invalid
+        # i.e. something we cannot encode as JSON. This will raise a Postjob::Queue::Encoder::Error.
+        #
+        # Usually this points to a non-UTF8 string.
+        # This is a fix for https://github.com/mediapeers/postjob/issues/35
+        Postjob::Queue::Encoder.check_encodable!([value])
+        [ :ok, value, nil ]
       end
-    when Job
-      expect! args == []
-      expect! timeout => nil, max_attempts => nil
-      r = job.resolve
-      throw :pending, :pending if r == :pending
-      r
-    else
-      job = async(job, *args, timeout: timeout, max_attempts: max_attempts)
-      await(job)
-    end
-  end
-
-  STATUSES = [ :sleep, :ok, :err, :failed ]
-
-  #
-  # runs a specific job
-  #
-  # returns a tuple [status, value], which follows the following pattern:
-  #
-  # - <tt>[ <runner-version>, :ok, value, nil ]</tt>: job completed successfully
-  # - <tt>[ <runner-version>, :sleep, nil, nil ]</tt>: job has to wait on a child job
-  # - <tt>[ <runner-version>, :err, <err>, nil ]</tt>: job errored with a recoverable error
-  # - <tt>[ <runner-version>, :failed, <err>, <shutdown> ]</tt>: job failed
-  #       with a non-recoverable error
-  #
-  # <err> is a tuple [ error-class-name, error-message, stacktrace ].
-  # <shutdown> is either nil or :shutdown
-  #
-  def process_job(job)
-    expect! job => Job
-
-    workflow = Postjob::Registry.lookup!(name: job.workflow, version: job.workflow_version)
-
-    with_current_job(job) do
-      status, value, shutdown = invoke_workflow workflow, job
-      log_result! job, status, value
-      # If the status is ok the job finished processing. In that case
-      # we'll wait for all child jobs to finish.
-      # if status == :ok
-      #   await :all
-      # end
-      [ workflow.workflow_version, status, value, shutdown ]
+    rescue ArgumentError, LocalJumpError, NameError, RegexpError, ScriptError, TypeError
+      Postjob.logger.error "#{$!}, from\n\t#{$!.backtrace[0, 10].join("\n\t")}"
+      return_exception :failed, $!
+    rescue Postjob::Error::Nonrecoverable
+      return_exception :failed, $!
+    rescue Exception
+      return_exception :err, $!
     end
-  end
 
-  private
-
-  # runs a job. Returns a [ status, value, shutdown ] tuple.
-  #
-  # The shutdown value is used by a worker in run mode (i.e. process
-  # indefinetively) to determine whether or not it should cancel
-  # processing. It is usually nil; but if the worker received a
-  # SIGINT it will be :shutdown instead.
-  #
-  # We are catching SIGINT to allow the job status to be updated.
-  def invoke_workflow(workflow, job)
-    value = catch(:pending) {
-      expect! job.args => [Array, nil]
-
-      workflow_method = job.workflow_method
-      args = job.args
-
-      insp_args = args.map(&:inspect).join(", ")
-      logger.info "Running Postjob##{job.id}: #{job.workflow}.#{workflow_method}(#{insp_args})"
-
-      workflow.public_send workflow_method, *args
-    }
-
-    if value == :pending
-      [ :pending, nil, nil ]
-    else
-      # Check that we can encode the value. If we can't the job returned something invalid
-      # i.e. something we cannot encode as JSON. This will raise a Postjob::Queue::Encoder::Error.
-      #
-      # Usually this points to a non-UTF8 string.
-      # This is a fix for https://github.com/mediapeers/postjob/issues/35
-      Postjob::Queue::Encoder.check_encodable!([value])
-      [ :ok, value, nil ]
+    def should_shutdown?(exception)
+      exception.is_a?(Interrupt)
     end
-  rescue ArgumentError, LocalJumpError, NameError, RegexpError, ScriptError, TypeError
-    Postjob.logger.error "#{$!}, from\n\t#{$!.backtrace[0, 10].join("\n\t")}"
-    return_exception :failed, $!
-  rescue Postjob::Error::Nonrecoverable
-    return_exception :failed, $!
-  rescue Exception
-    return_exception :err, $!
-  end
 
-  def should_shutdown?(exception)
-    exception.is_a?(Interrupt)
-  end
+    def return_exception(state, exception)
+      # get and shorten backtrace.
+      error_backtrace = exception.backtrace[0, 10]
 
-  def return_exception(state, exception)
-    # get and shorten backtrace.
-    error_backtrace = exception.backtrace[0, 10]
+      curdir = "#{Dir.getwd}/"
+      error_backtrace = error_backtrace.map { |path| path.start_with?(curdir) ? path[curdir.length..-1] : path }
 
-    curdir = "#{Dir.getwd}/"
-    error_backtrace = error_backtrace.map { |path| path.start_with?(curdir) ? path[curdir.length..-1] : path }
+      shutdown = should_shutdown?(exception) ? :shutdown : nil
+      [ state, [exception.class.name, exception.message, error_backtrace], shutdown ]
+    end
 
-    shutdown = should_shutdown?(exception) ? :shutdown : nil
-    [ state, [exception.class.name, exception.message, error_backtrace], shutdown ]
-  end
+    def log_result!(job, status, value)
+      case status
+      when :err
+        severity = job.parent_id ? :warn : :error
+        logger.send severity, error_message(job, status, value)
+      when :failed
+        logger.error error_message(job, status, value)
+      when :ok
+        runtime = Time.now.utc - job.created_at
+        runtime = "%.03f secs" % runtime
+        severity = job.parent_id ? :info : :warn
+        msg = "#{job} successful w/result #{value.inspect}: #{runtime}"
+        logger.send severity, msg
+      end
+    end
 
-  def log_result!(job, status, value)
-    case status
-    when :err
-      severity = job.parent_id ? :warn : :error
-      logger.send severity, error_message(job, status, value)
-    when :failed
-      logger.error error_message(job, status, value)
-    when :ok
+    def error_message(job, status, value)
       runtime = Time.now.utc - job.created_at
       runtime = "%.03f secs" % runtime
-      severity = job.parent_id ? :info : :warn
-      msg = "#{job} successful w/result #{value.inspect}: #{runtime}"
-      logger.send severity, msg
-    end
-  end
-
-  def error_message(job, status, value)
-    runtime = Time.now.utc - job.created_at
-    runtime = "%.03f secs" % runtime
-    error_class, err_message, _error_backtrace = value
+      error_class, err_message, _error_backtrace = value
 
-    "#{job} #{status} #{error_class} #{err_message.inspect}: #{runtime}"
-    # + "\n    backtrace information:\n    #{error_backtrace.join("\n    ")}"
+      "#{job} #{status} #{error_class} #{err_message.inspect}: #{runtime}"
+      # + "\n    backtrace information:\n    #{error_backtrace.join("\n    ")}"
+    end
   end
 end

data/lib/postjob/worker_session.rb

@@ -3,34 +3,18 @@
 require_relative "./record"
 
 require "tools/heartbeat"
-require "tools/atomic_store"
-
-class Postjob::Host < Postjob::Record
-  def self.register(attributes = {})
-    Postjob.logger.debug "registering host w/#{attributes.inspect}"
-    ::Postjob::Queue.host_register(attributes)
-  end
-end
 
 # A worker worker_session
 class Postjob::WorkerSession < Postjob::Record
-  HOST_ID_PATH = ".postjob.host_id"
-
   class << self
     # Starts a worker session.
     def start!(workflows_with_versions)
-      worker_session = nil
-
-      AtomicStore.with(HOST_ID_PATH) do |host_id|
-        host_id ||= ::Postjob::Host.register
-        Postjob.logger.debug "Starting worker_session w/host_id #{host_id.inspect}"
-        worker_session = ::Postjob::Queue.start_worker_session(workflows_with_versions, host_id: host_id)
-        host_id
-      end
+      host_id = ::Postjob.host_id
+      worker_session = ::Postjob::Queue.start_worker_session(workflows_with_versions, host_id: host_id)
 
       Postjob.logger.info "Starting worker_session #{worker_session.inspect}"
 
-      start_heartbeat_monitor(worker_session.host_id)
+      start_heartbeat_monitor(host_id)
       worker_session
     end
 
@@ -58,12 +42,12 @@ class Postjob::WorkerSession < Postjob::Record
     end
   end
 
-  attribute :id
-  attribute :host_id
-  attribute :client_socket
-  attribute :workflows
-  attribute :attributes
-  attribute :created_at
+  attr_reader :id
+  attr_reader :host_id
+  attr_reader :client_socket
+  attr_reader :workflows
+  attr_reader :attributes
+  attr_reader :created_at
 
   def to_s
     "Session##{id}"