postjob 0.1.1

data/lib/postjob/queue.rb

@@ -0,0 +1,331 @@
+# rubocop:disable Layout/IndentationWidth
+# rubocop:disable Style/UnneededInterpolation
+# rubocop:disable Metrics/ModuleLength
+# rubocop:disable Metrics/LineLength
+# rubocop:disable Lint/EndAlignment
+# rubocop:disable Layout/SpaceInsideArrayLiteralBrackets
+
+# The Postjob::Queue manages enqueueing and fetching jobs from a job queue.
+module Postjob::Queue
+  extend self
+
+  SCHEMA_NAME = "postjob"
+  TABLE_NAME  = "#{SCHEMA_NAME}.postjobs"
+
+  SQL         = ::Simple::SQL
+end
+
+require_relative "queue/encoder"
+require_relative "queue/notifications"
+require_relative "queue/search"
+
+module Postjob::Queue
+  Job = ::Postjob::Job
+
+  DEFAULT_OPTIONS = {
+    version: "",
+    queue: "q",
+    max_attempts: 5
+  }.freeze
+
+  # enqueues a new job with the given arguments
+  #
+  # Parameters:
+  #
+  # - queue      - the name of the queue
+  # - workflow   - the name of the workflow (e.g. "FooBar", "FooBar#method_name")
+  # - version    - the version of the workflow, e.g. "0.2"
+  # - args       - an array of arguments, must be encodable via Postjob::JSON.encode
+  # - parent_id - the id of the parent job, if any
+  # - tags       - # a Hash[String => String]
+  #
+  def enqueue_job(workflow, *args, options)
+    expect! workflow => String
+    expect! options => {
+      queue: [String, nil],
+      version: [/\A\d(\.\d)+\z/, nil],
+      parent_id: [Integer, nil],
+      tags: [Hash, nil],
+      timeout: [Numeric, nil],
+      max_attempts: [Integer, nil]
+    }
+
+    workflow, workflow_method = parse_workflow(workflow)
+
+    options.update(DEFAULT_OPTIONS) { |_k, v1, v2| v1 || v2 }
+
+    SQL.transaction do
+      queue, version, parent_id, tags, timeout, max_attempts =
+        options.values_at :queue, :version, :parent_id, :tags, :timeout, :max_attempts
+
+      args = Encoder.encode(args) if args
+      tags = Encoder.encode(tags) if tags
+
+      id = SQL.ask <<~SQL, queue, workflow, workflow_method, version, args, parent_id, tags, max_attempts
+        INSERT INTO #{TABLE_NAME}(queue, workflow, workflow_method, workflow_version, args, parent_id, tags, max_attempts)
+        VALUES($1, $2, $3, $4, $5, $6, $7, $8)
+        RETURNING id
+      SQL
+
+      if timeout
+        SQL.ask <<~SQL, id, timeout
+          UPDATE #{TABLE_NAME}
+          SET timing_out_at = (now() at time zone 'utc') + $2 * interval '1 second'
+          WHERE id=$1
+        SQL
+      end
+
+      root_id, parent_full_id = if parent_id
+        SQL.ask "SELECT root_id, full_id FROM #{TABLE_NAME} WHERE id=$1", parent_id
+      end
+
+      root_id ||= id
+      full_id = parent_full_id ? "#{parent_full_id}.#{id}" : "#{id}"
+
+      SQL.ask "UPDATE #{TABLE_NAME} SET full_id=$2, root_id=$3 WHERE id=$1",
+              id, full_id, root_id
+
+      Notifications.notify_listeners
+
+      SQL.record "SELECT * FROM #{TABLE_NAME} WHERE id=$1", id, into: Job
+    end
+  end
+
+  def set_job_result(job, value, version:)
+    update_job(job, version: version)
+    results = Encoder.encode([value]) unless value.nil?
+
+    SQL.ask <<~SQL, job.id, results
+      UPDATE #{TABLE_NAME}
+      SET results=$2, status='ok', next_run_at=NULL
+      WHERE id=$1
+    SQL
+
+    wakeup(job.parent_id)
+  end
+
+  def set_job_pending(job, version:)
+    update_job(job, version: version)
+    SQL.ask <<~SQL, job.id
+      UPDATE #{TABLE_NAME}
+      SET status='sleep', next_run_at=NULL
+      WHERE id=$1
+    SQL
+  end
+
+  private
+
+  def remaining_attempts(job)
+    SQL.ask <<~SQL, job.id
+      SELECT max_attempts - failed_attempts
+      FROM #{TABLE_NAME}
+      WHERE id=$1
+    SQL
+  end
+
+  def next_status_and_next_run_of_failed_job(job, status)
+    # If this is a recoverable error and if we have another run possible we'll
+    # set next_run_at, and the status to "sleep", otherwise next_run_at will be
+    # NULL and the status would be "failed"
+    #
+    # To check if we have another run we check (max_attempts - failed_attempts).
+    #
+    # This is only necessary with a status of :err. Note that we need to
+    # subtract 1, since this check runs *after* the current run was done,
+    # but before it was written to the database.
+    if status == :err && remaining_attempts(job) > 1
+      [ "ready", next_run_at_fragment ]
+    elsif status == :timeout
+      [ "timeout", "NULL" ]
+    else
+      [ "failed", "NULL" ]
+    end
+  end
+
+  public
+
+  def set_job_error(job, error, error_message, error_backtrace = nil, status:, version:)
+    update_job(job, version: version)
+
+    new_status, next_run_at = next_status_and_next_run_of_failed_job job, status
+
+    unless error_backtrace.nil?
+      error_backtrace = error_backtrace.map { |path| make_relative_path(path) }
+      error_backtrace = Encoder.encode(error_backtrace)
+    end
+
+    SQL.ask <<~SQL, job.id, new_status, error, error_message, error_backtrace
+      UPDATE #{TABLE_NAME}
+      SET
+        status=$2, error=$3, error_message=$4, error_backtrace=$5,
+        failed_attempts=failed_attempts+1, next_run_at=#{next_run_at}
+      WHERE id=$1
+    SQL
+
+    wakeup(job.parent_id)
+  end
+
+  private
+
+  def make_relative_path(path)
+    @here ||= "#{Dir.getwd}/"
+    path.start_with?(@here) ? path[@here.length..-1] : path
+  end
+
+  def update_job(job, version:)
+    return unless version
+
+    SQL.ask <<~SQL, job.id, version
+      UPDATE #{TABLE_NAME}
+      SET workflow_version=$2, updated_at=(now() at time zone 'utc')
+      WHERE id=$1
+    SQL
+  end
+
+  #
+  # The timeout until a job can be run next is calculated by running the
+  # +next_run_at_fragment+ sql in the database.
+  #
+  def next_run_at_fragment
+    # The basetime to use with the NEXT_RUN_AT_FRAGMENT below.
+    next_run_at_basetime = Postjob.fast_mode ? 0.01 : 10
+    "(now() at time zone 'utc') + #{next_run_at_basetime} * pow(1.5, failed_attempts) * interval '1 second'"
+  end
+
+  def wakeup(id)
+    return unless id
+
+    SQL.ask <<~SQL, id
+      UPDATE #{TABLE_NAME}
+      SET status='ready', next_run_at=(now() at time zone 'utc'), updated_at=(now() at time zone 'utc')
+      WHERE id=$1 AND status='sleep'
+    SQL
+
+    Notifications.notify_listeners
+  end
+
+  public
+
+  def childjobs(parent)
+    expect! parent => Job
+
+    SQL.records <<~SQL, parent.id, into: Job
+      SELECT * FROM #{TABLE_NAME}
+      WHERE parent_id=$1
+      ORDER BY id
+      SQL
+  end
+
+  def next_unresolved_childjob(parent)
+    expect! parent => Job
+
+    SQL.record <<~SQL, parent.id, into: Job
+      SELECT * FROM #{TABLE_NAME}
+      WHERE parent_id=$1 AND status NOT IN ('ok', 'failed')
+      ORDER BY next_run_at
+      LIMIT 1
+      SQL
+  end
+
+  def find_or_create_childjob(parent, workflow, args, timeout:, max_attempts:)
+    expect! parent => Job, workflow => String, args => Array
+
+    workflow, workflow_method = parse_workflow(workflow)
+
+    job = SQL.record <<~SQL, parent.id, workflow, workflow_method, Encoder.encode(args), into: Job
+      SELECT * FROM #{TABLE_NAME}
+      WHERE parent_id=$1
+        AND workflow=$2
+        AND workflow_method=$3
+        AND args=$4
+      SQL
+
+    return job if job
+
+    enqueue_job("#{workflow}.#{workflow_method}", *args, queue: parent.queue, parent_id: parent.id, timeout: timeout, max_attempts: max_attempts)
+  end
+
+  def set_workflow_status(job, status)
+    # [TODO] Try to reduce the number of writes.
+    #
+    # The current implementation updates a status potentially multiple times
+    # within a single run of a job (all within the same transaction and therefore
+    # invisible to the outside).
+    SQL.ask <<~SQL, job.id, status
+      UPDATE #{TABLE_NAME}
+      SET workflow_status=$2
+      WHERE id=$1
+    SQL
+  end
+
+  private
+
+  def parse_workflow(workflow)
+    workflow, workflow_method = workflow.split(".", 2)
+    workflow_method ||= "run"
+
+    expect! workflow => /./
+    expect! workflow_method => /^[_a-z][_a-z0-9]*$/
+
+    [workflow, workflow_method]
+  end
+
+  def runnable_sql_fragment
+    escaped_workflows_and_versions = Postjob::Registry.sql_escaped_workflows_and_versions
+    return "FALSE" if escaped_workflows_and_versions == ""
+
+    <<~SQL
+      next_run_at <= (now() at time zone 'utc')
+      AND status = 'ready'
+      AND ((workflow, workflow_version) IN (#{escaped_workflows_and_versions}))
+    SQL
+  end
+
+  def timing_out_sql_fragment
+    <<~SQL
+      timing_out_at <= (now() at time zone 'utc')
+      AND status IN ('ready', 'sleep')
+    SQL
+  end
+
+  public
+
+  def checkout_runnable
+    sql = <<~SQL
+      SELECT
+        *,
+        timing_out_at <= (now() at time zone 'utc') AS timed_out
+      FROM #{TABLE_NAME}
+      WHERE
+        (#{runnable_sql_fragment})
+        OR
+        (#{timing_out_sql_fragment})
+      ORDER BY (LEAST(next_run_at, timing_out_at))
+      FOR UPDATE SKIP LOCKED
+      LIMIT 1
+    SQL
+
+    SQL.transaction do
+      job = SQL.record sql, into: Job
+      yield job if job
+      job
+    end
+  end
+
+  def find_or_create_token(job)
+    token = SQL.ask "SELECT token FROM postjob.tokens WHERE postjob_id=$1", job.id
+    return token if token
+
+    token = SecureRandom.uuid
+    SQL.ask "INSERT INTO postjob.tokens(postjob_id, token) VALUES($1, $2)", job.id, token
+    token
+  end
+
+  def find_job_by_token(token)
+    SQL.record <<~SQL, token, into: Job
+      SELECT postjob.postjobs.* FROM postjob.postjobs
+      INNER JOIN postjob.tokens ON postjob.tokens.postjob_id=postjob.postjobs.id
+      WHERE postjob.tokens.token=$1
+    SQL
+  end
+end

data/lib/postjob/registry.rb

@@ -0,0 +1,52 @@
+# The registry holds a list of all available workflows
+module Postjob::Registry
+  extend self
+
+  def workflows
+    instance.values.uniq
+  end
+
+  # [TODO] - it would be nicer if Simple::SQL would properly build a Postgres version
+  # of the workflow names and versions.
+  #
+  def sql_escaped_workflows_and_versions
+    @sql_escaped_workflows_and_versions ||= begin
+      instance.keys.map do |name, workflow_version|
+        escaped_name    = sql_escape(name)
+        escaped_version = sql_escape(workflow_version)
+        "(#{escaped_name}, #{escaped_version})"
+      end.join(", ")
+    end
+  end
+
+  private
+
+  def sql_escape(s)
+    "'" + PG::Connection.escape_string(s) + "'"
+  end
+
+  public
+
+  # Used for tests
+  def reset! # :nodoc:
+    @instance = @sql_escaped_workflows_and_versions = nil
+  end
+
+  def register(workflow, _options = {})
+    instance[[workflow.name, ""]] = workflow
+    instance[[workflow.name, workflow.workflow_version]] = workflow
+  end
+
+  def lookup!(name:, version:)
+    expect! name => String
+    expect! version => String
+
+    instance.fetch([name, version])
+  end
+
+  private
+
+  def instance
+    @instance ||= {}
+  end
+end

data/lib/postjob/runner.rb

@@ -0,0 +1,153 @@
+# rubocop:disable Style/RedundantSelf
+module Postjob::Runner
+  extend self
+
+  extend Forwardable
+  delegate [:logger] => Postjob
+
+  Job = Postjob::Job
+
+  # returns the job that is currently running.
+  #
+  # This value is set by +process_job+ (via +with_current_job+), and
+  # currently only used from <tt>Postjob::Runner.async</tt>
+  def current_job
+    Thread.current[:current_job]
+  end
+
+  private
+
+  def with_current_job(job)
+    expect! current_job => nil
+    Thread.current[:current_job] = job
+    yield
+  ensure
+    Thread.current[:current_job] = nil
+  end
+
+  public
+
+  # returns a subjob within the current job, for a +runner+
+  # description and +args+.
+  def async(workflow, *args, timeout: nil, max_attempts:)
+    # if the workflow is a symbol, then we change it into "__manual__"
+    # - there should never be a workflow with that name - or into
+    # "CurrentWorkshop.#{workflow}", denoting the \a workflow method of the
+    # current workflow.
+    case workflow
+    when :manual  then workflow = "__manual__"
+    when Symbol   then workflow = "#{current_job.workflow}.#{workflow}"
+    when Module   then workflow = workflow.name
+    end
+
+    ::Postjob::Queue.find_or_create_childjob(self.current_job, workflow, args,
+                                             timeout: timeout,
+                                             max_attempts: max_attempts)
+  end
+
+  # tries to resolve a job.
+  def await(job, *args, timeout: nil, max_attempts: nil)
+    case job
+    when :all
+      if Postjob::Queue.next_unresolved_childjob(current_job)
+        Postjob.logger.warn "await :all: Found an unresolved childjob"
+        throw :pending, :pending
+      else
+        childjobs = Postjob::Queue.childjobs(current_job)
+        childjobs.map(&:resolve)
+      end
+    when Job
+      expect! args == []
+      expect! timeout => nil, max_attempts => nil
+      r = job.resolve
+      throw :pending, :pending if r == :pending
+      r
+    else
+      job = async(job, *args, timeout: timeout, max_attempts: max_attempts)
+      await(job)
+    end
+  end
+
+  STATUSES = [ :sleep, :ok, :err, :failed ]
+
+  #
+  # runs a specific job
+  #
+  # returns a tuple [status, value], which follows the following pattern:
+  #
+  # - <tt>[ <runner-version>, :ok, value ]</tt>: job completed successfully
+  # - <tt>[ <runner-version>, :sleep, nil ]</tt>: job has to wait on a child job
+  # - <tt>[ <runner-version>, :err, <err> ]</tt>: job errored with a recoverable error
+  # - <tt>[ <runner-version>, :failed, <err> ]</tt>: job failed with a non-recoverable error
+  #
+  # <err> is a tuple [ error-class-name, error-message, stacktrace ].
+  #
+  def process_job(job)
+    expect! job => Job
+
+    workflow = Postjob::Registry.lookup!(name: job.workflow, version: job.workflow_version)
+
+    with_current_job(job) do
+      status, value = invoke_workflow workflow, job
+      log_result! job, status, value
+      [ workflow.workflow_version, status, value ]
+    end
+  end
+
+  private
+
+  # runs a job. Returns a [ runner, status, value ] tuple.
+  def invoke_workflow(workflow, job)
+    value = catch(:pending) {
+      expect! job.args => [Array, nil]
+
+      workflow_method = job.workflow_method
+      args = job.args
+
+      insp_args = args.map(&:inspect).join(", ")
+      logger.info "Running Postjob##{job.id}: #{job.workflow}.#{workflow_method}(#{insp_args})"
+
+      workflow.public_send workflow_method, *args
+    }
+
+    case value
+    when :pending then [ :pending, nil ]
+    else               [ :ok, value ]
+    end
+  rescue RuntimeError
+    return_exception :err, $!
+  rescue StandardError
+    Postjob.logger.error "#{$!}, from\n\t#{$!.backtrace[0, 10].join("\n\t")}"
+    return_exception :failed, $!
+  end
+
+  def return_exception(state, exception)
+    error_backtrace = exception.backtrace[0, 10]
+    [ state, [exception.class.name, exception.to_s, error_backtrace] ]
+  end
+
+  def log_result!(job, status, value)
+    case status
+    when :err
+      severity = job.parent_id ? :warn : :error
+      logger.send severity, error_message(job, status, value)
+    when :failed
+      logger.error error_message(job, status, value)
+    when :ok
+      runtime = Time.now.utc - job.created_at
+      runtime = "%.03f secs" % runtime
+      severity = job.parent_id ? :info : :warn
+      msg = "#{job} successful w/result #{value.inspect}: #{runtime}"
+      logger.send severity, msg
+    end
+  end
+
+  def error_message(job, status, value)
+    runtime = Time.now.utc - job.created_at
+    runtime = "%.03f secs" % runtime
+    error_class, err_message, error_backtrace = value
+
+    "#{job} #{status} #{error_class} #{err_message.inspect}: #{runtime}"
+    # + "\n    backtrace information:\n    #{error_backtrace.join("\n    ")}"
+  end
+end

data/lib/postjob/workflow.rb

@@ -0,0 +1,60 @@
+
+# including the Workflow module marks a module as a Postjob::Job, and mixes
+# in a couple of methods into the module (see Workflow::JobMethods)
+module Postjob::Workflow
+  def self.included(job)
+    expect! job.class.name => "Module"
+
+    # This method should make sure there is a public run method. This, however,
+    # is not possible before the run method is defined. We could do the check
+    # with a job like
+    #
+    # module Foo
+    #   def self.run(arg); ..; end
+    #   include Postjob::Workflow
+    # end
+    #
+    # but not the other way around:
+    #
+    # module Foo
+    #   include Postjob::Workflow
+    #   def self.run(arg); ..; end
+    # end
+    #
+    # We therefore do not make this test here.
+    job.extend JobMethods
+  end
+
+  module JobMethods
+    def async(runner, *args, timeout: nil, max_attempts: nil)
+      ::Postjob::Runner.async(runner, *args, timeout: timeout, max_attempts: max_attempts)
+    end
+
+    def await(job, *args, timeout: nil, max_attempts: nil)
+      ::Postjob::Runner.await(job, *args, timeout: timeout, max_attempts: max_attempts)
+    end
+
+    def workflow_token(job)
+      expect! job.parent_id => ::Postjob::Runner.current_job.id
+      ::Postjob::Queue.find_or_create_token(job)
+    end
+
+    def set_workflow_status(status)
+      ::Postjob::Queue.set_workflow_status ::Postjob::Runner.current_job, status
+    end
+
+    def workflow_version
+      @workflow_version || "0.0"
+    end
+
+    def workflow_name
+      "#{name}@#{workflow_version}"
+    end
+
+    private
+
+    def set_workflow_version(workflow_version)
+      @workflow_version = workflow_version
+    end
+  end
+end