postjob 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 93400c11de9310d973cb74ef6912d5e4190d5950
+  data.tar.gz: b505261dc9cb3a2aacf905e8f37027c07ff78946
+SHA512:
+  metadata.gz: fe6794cfc9b47039aa6f57381f9465b30a15e7b7369b6eb0ec051acfa9c2786ae8a52fbc0cd71b1e450824e61238e18e892e69b7ef9611247eb7c5a16f0ea0b3
+  data.tar.gz: eeb6632e7cebbd95f96d801c8b67a3aea53e5a5c8960801f7292cf22aa62c2986fbcb60eb8564cf545a29f63c35f311057bfbb36887a298a4db5cabfd6be17e3

data/README.md

@@ -0,0 +1,23 @@
+# Postjob
+
+The `postjob` gem implements a simple way to have restartable, asynchronous, and distributed processes.
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Make sure you have a local postgresql implementation of at least version 9.5. Add a `postqueue` user with a `postqueue` password, and create a `postqueue_test` database for it. The script `./scripts/prepare_pg` can be somewhat helpful in establishing that.
+
+Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. 
+
+To release a new version, run `./scripts/release`, which will bump the version number, create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/postqueue.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/bin/postjob

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+$: << "lib"
+
+require "bundler"
+Bundler.setup
+
+require "postjob"
+require "postjob/cli"
+
+Simple::SQL.logger = Simple::CLI.logger
+Postjob::CLI.run!(*ARGV)

data/lib/postjob/cli/db.rb

@@ -0,0 +1,39 @@
+require "postjob/cli"
+
+module Postjob::CLI
+  def db_migrate
+    require "postjob/migrations"
+
+    connect_to_database!
+    Postjob::Migrations.migrate!
+  end
+
+  def db_unmigrate
+    require "postjob/migrations"
+
+    connect_to_database!
+    Postjob::Migrations.unmigrate!
+  end
+
+  def db_remigrate
+    require "postjob/migrations"
+
+    connect_to_database!
+    Postjob::Migrations.unmigrate!
+    Postjob::Migrations.migrate!
+  end
+
+  private
+
+  USE_ACTIVE_RECORD = false
+
+  def connect_to_database!
+    if USE_ACTIVE_RECORD
+      require "active_record"
+      abc = ::Simple::SQL::Config.read_database_yml
+      ::ActiveRecord::Base.establish_connection abc
+    else
+      ::Simple::SQL.connect!
+    end
+  end
+end

data/lib/postjob/cli/job.rb

@@ -0,0 +1,67 @@
+module Postjob::CLI
+  # Enqueues a workflow
+  #
+  # Adds a workflow to the job table, with name <workflow> and the given
+  # arguments.
+  #
+  # Note that the workflow will receive the arguments as strings and must be 
+  # prepared to handle these.
+  def job_enqueue(workflow, *args, queue: "ruby", tags: nil)
+    connect_to_database!
+
+    Postjob.enqueue! workflow, *args, queue: queue, tags: parse_tags(tags)
+  end
+
+  # Reset failed jobs
+  #
+  # This resets all failed jobs within the job tree, below the passed in
+  # job id.
+  def job_reset(job_id)
+    job_id = Integer(job_id)
+    full_job_id = Simple::SQL.ask "SELECT full_id FROM postjob.postjobs WHERE id=$1", job_id
+    full_job_id || logger.error("No such job: #{job_id}")
+
+    job_ids = Simple::SQL.all <<~SQL
+      SELECT id FROM postjob.postjobs 
+      WHERE (full_id LIKE '#{full_job_id}.%' OR full_id='#{full_job_id}')
+        AND status IN ('failed', 'err', 'timeout')
+    SQL
+
+    logger.warn "Affected jobs: #{job_ids.count}"
+    return if job_ids.empty?
+
+    Simple::SQL.ask <<~SQL, job_ids
+      UPDATE postjob.postjobs
+      SET
+        status='ready', next_run_at=now(),
+        results=null, failed_attempts=0, error=NULL, error_message=NULL, error_backtrace=NULL
+      WHERE id = ANY($1);
+    SQL
+
+    Simple::SQL.ask <<~SQL
+      NOTIFY postjob_notifications
+    SQL
+
+    logger.warn "The following jobs have been reset: #{job_ids.join(", ")}"
+  end
+
+  private
+
+  # parses "foo:bar,baz:quibble" into { "foo" => "bar", "baz" => "quibble"}
+  def parse_tags(tags)
+    return nil unless tags
+    tags.split(",").inject({}) do |hsh, tag|
+      expect! tag => /\A[^:]+:[^:]+\z/
+      k, v = tag.split(":", 2)
+      hsh.update k => v
+    end
+  end
+
+  public
+
+  def registry
+    workflows = Postjob::Registry.workflows
+    names = workflows.map(&:workflow_name)
+    puts names.sort.join("\n")
+  end
+end

data/lib/postjob/cli/ps.rb

@@ -0,0 +1,110 @@
+module Postjob::CLI
+  private
+  
+  def ps_query(conditions = [])
+    conditions.compact!
+
+    conditions << "TRUE"
+    condition_fragment = conditions
+                         .compact
+                         .map { |s| "(#{s})" }
+                         .join(" AND ")
+
+    <<~SQL
+      SELECT
+        id,
+        full_id,
+        workflow
+          || (CASE WHEN workflow_version != '' THEN '@' ELSE '' END)
+          || workflow_version
+          || (CASE WHEN workflow_method != 'run' THEN '.' || workflow_method ELSE '' END)
+          || args AS job,
+        workflow_status,
+        status,
+        error,
+        COALESCE((results->0)::varchar, error_message) AS result,
+        next_run_at,
+        error_backtrace,
+        (now() at time zone 'utc') - created_at AS age,
+        updated_at - created_at AS runtime,
+        tags
+      FROM postjob.postjobs
+      WHERE #{condition_fragment}
+      ORDER BY root_id DESC, id ASC
+    SQL
+  end
+  
+  def tags_condition(tags)
+    return nil unless tags
+
+    kv = parse_tags(tags)
+    "tags @> '#{Postjob::Queue::Encoder.encode(kv)}'"
+  end
+
+  public
+
+  # Show job status
+  #
+  # This command lists the statuses of all jobs that are either root jobs,
+  # i.e. enqueued workflows, or that have failed.
+  #
+  # Example: 
+  # 
+  #   postjob ps --tags=foo:bar,bar:baz --limit=100
+  #
+  # For a listing of all jobs in the system use ps:full, see 'postjob help ps:full'
+  # for details.
+  def ps(*ids, limit: "100", tags: nil)
+    expect! limit => /\A\d+\z/
+    limit = Integer(limit)
+
+    unless ids.empty?
+      ps_full *ids, limit: limit, tags: tags
+      return
+    end
+
+    conditions = []
+    conditions << "root_id=id OR status NOT IN ('ready', 'sleep', 'ok')"
+    conditions << tags_condition(tags)
+    conditions << ids_condition(ids)
+
+    query = ps_query(conditions)
+    print_sql limit: limit, query: query
+  end
+
+  def ps_full(*ids, limit: 100, tags: nil)
+    conditions = []
+    conditions << tags_condition(tags)
+    conditions << ids_condition(ids)
+
+    query = ps_query(conditions)
+
+    limit = Integer(limit)
+    print_sql limit: limit, query: query
+  end
+
+  private
+
+  def parse_ids(*ids)
+    return [] if ids.empty?
+    ids.flatten.inject([]) { |a, ids_string| a.concat ids_string.split(",") }
+       .map { |p| Integer(p) }
+       .uniq
+  end
+
+  def ids_condition(ids)
+    ids = parse_ids(ids)
+    return nil if ids.empty?
+    "root_id IN (#{ids.join(',')})"
+  end
+
+  def print_sql(limit:, query:)
+    connect_to_database!
+    records = Simple::SQL.records("#{query} LIMIT $1+1", limit)
+
+    tp records[0, limit]
+    if records.length > limit
+      logger.warn "Output limited up to limit #{limit}. Use the --limit command line option for a different limit."
+    end
+  end
+end

data/lib/postjob/cli/run.rb

@@ -0,0 +1,19 @@
+module Postjob::CLI
+  def step
+    run count: 1
+  end
+
+  def run(count: nil, fast: false, quiet: false)
+    count = Integer(count) if count
+    Postjob.fast_mode = fast
+
+    connect_to_database!
+
+    processed = Postjob.run(count: count) do |job|
+      logger.info "Processed job w/id #{job.id}" if job
+      STDERR.print "." unless quiet
+    end
+
+    logger.info "Processed #{processed} jobs"
+  end
+end

data/lib/postjob/cli.rb

@@ -0,0 +1,31 @@
+require "simple/cli"
+require "table_print"
+
+Dir.glob("#{File.dirname(__FILE__)}/cli/**/*.rb").sort.each do |path|
+  load(path)
+end
+
+module Postjob::CLI
+  include ::Simple::CLI
+
+  def run!(command, *args)
+    Postjob.logger = logger
+    load_environment!
+
+    super
+  end
+
+  private
+
+  def load_environment(path)
+    return unless File.exist?(path)
+
+    logger.warn "#{path}: loading Postjob configuration"
+    load path
+  end
+
+  def load_environment!
+    load_environment("config/environment.rb")
+    load_environment("config/postjob.rb")
+  end
+end

data/lib/postjob/error.rb

@@ -0,0 +1,16 @@
+module Postjob
+  class Error < RuntimeError
+    def initialize(job)
+      @job = job
+    end
+
+    def message
+      msg = "Failing child job"
+      msg += " [#{@job.result}]" if @job.result
+      msg
+    end
+  end
+
+  class Error::Nonrecoverable < Error
+  end
+end

data/lib/postjob/job.rb

@@ -0,0 +1,66 @@
+#
+# A job class in-memory representation.
+#
+class Postjob::Job < Hash
+  def initialize(hsh)
+    replace hsh.dup
+  end
+
+  def self.attribute(sym)
+    eval <<~RUBY
+      define_method(:#{sym}) { self[:#{sym}] }
+    RUBY
+  end
+
+  attribute :id
+  attribute :parent_id
+  attribute :full_id
+  attribute :root_id
+  attribute :created_at
+  attribute :queue
+  attribute :workflow
+  attribute :workflow_method
+  attribute :workflow_version
+  attribute :args
+  attribute :next_run_at
+  attribute :timing_out_at
+  attribute :failed_attempts
+  attribute :max_attempts
+  attribute :status
+  attribute :results
+  attribute :error
+  attribute :error_message
+  attribute :error_backtrace
+  attribute :recipients
+  attribute :workflow_status
+  attribute :timed_out
+  attribute :tags
+
+  STATUSES = %w(ok ready sleep err failed timeout)
+
+  def resolve
+    expect! status => STATUSES
+
+    case status
+    when "ok"       then result
+    when "ready"    then :pending
+    when "sleep"    then :pending
+    when "timeout"  then raise Timeout::Error
+    when "err"      then :pending
+    when "failed"   then raise Postjob::Error::Nonrecoverable, self
+    end
+  end
+
+  def result
+    results && results.first
+  end
+
+  def to_s
+    full_workflow = workflow
+    full_workflow += "@#{workflow_version}" if workflow_version != ""
+    full_workflow += ".#{workflow_method}" if workflow_method != "run"
+
+    args = (self.args || []).map(&:inspect).join(", ")
+    "Postjob##{full_id}: #{full_workflow}(#{args}) (#{status})"
+  end
+end

data/lib/postjob/migrations.rb

@@ -0,0 +1,97 @@
+module Postjob
+  module Migrations
+    extend self
+
+    SQL = ::Simple::SQL
+    SCHEMA_NAME = Postjob::Queue::SCHEMA_NAME
+
+    def unmigrate!
+      if SCHEMA_NAME != "public"
+        SQL.exec <<~SQL
+          DROP SCHEMA IF EXISTS #{SCHEMA_NAME} CASCADE;
+        SQL
+      end
+    end
+
+    PG_TYPES = <<~SQL
+      SELECT pg_namespace.nspname AS schema, pg_type.typname AS name
+      FROM pg_type
+      LEFT JOIN pg_namespace on pg_namespace.oid=pg_type.typnamespace
+    SQL
+
+    def migrate!
+      SQL.exec <<~SQL
+        CREATE SCHEMA IF NOT EXISTS #{SCHEMA_NAME};
+      SQL
+
+      unless SQL.ask("SELECT 1 FROM (#{PG_TYPES}) sq WHERE (schema,name) = ($1, $2)", SCHEMA_NAME, "statuses")
+        SQL.exec <<~SQL
+          CREATE TYPE #{SCHEMA_NAME}.statuses AS ENUM (
+            'ready',    -- process can run
+            'sleep',    -- process has external dependencies to wait for.
+            'failed',   -- process failed, with nonrecoverable error
+            'err',      -- process errored (with recoverable error)
+            'timeout',  -- process timed out
+            'ok'        -- process succeeded
+          );
+        SQL
+      end
+
+      SQL.exec <<~SQL
+        CREATE TABLE IF NOT EXISTS #{SCHEMA_NAME}.postjobs (
+          -- id values, readonly once created
+          id          BIGSERIAL PRIMARY KEY,                                        -- process id
+          parent_id   BIGINT REFERENCES #{SCHEMA_NAME}.postjobs ON DELETE CASCADE,  -- parent process id
+          full_id     VARCHAR,                                                      -- full process id
+          root_id     BIGINT,                                                       -- root process id
+
+          created_at  timestamp NOT NULL DEFAULT (now() at time zone 'utc'),        -- creation timestamp
+          updated_at  timestamp NOT NULL DEFAULT (now() at time zone 'utc'),        -- update timestamp
+
+          queue VARCHAR,                                                            -- queue name. (readonly)
+          workflow VARCHAR NOT NULL,                                                -- e.g. "MyJobModule" (readonly)
+          workflow_method VARCHAR NOT NULL DEFAULT 'run',                           -- e.g. "run" (readonly)
+          workflow_version VARCHAR NOT NULL DEFAULT '',                             -- e.g. "1.0"
+          args JSONB,                                                               -- args
+
+          -- process state ----------------------------------------------------
+
+          status #{SCHEMA_NAME}.statuses DEFAULT 'ready',
+          next_run_at timestamp DEFAULT (now() at time zone 'utc'),                 -- when possible to run next?
+          timing_out_at timestamp,                                                  -- job times out after this timestamp
+          failed_attempts INTEGER NOT NULL DEFAULT 0,                               -- failed how often?
+          max_attempts INTEGER NOT NULL DEFAULT 1,                                  -- maximum attempts before failing
+
+          -- process result ---------------------------------------------------
+
+          results JSONB,
+          error VARCHAR,
+          error_message VARCHAR,
+          error_backtrace JSONB,
+
+          -- custom fields
+          workflow_status VARCHAR,
+          tags JSONB
+        );
+
+        -- [TODO] check indices
+        CREATE INDEX IF NOT EXISTS postjobs_tags_idx
+          ON #{SCHEMA_NAME}.postjobs USING GIN (tags jsonb_path_ops);
+        CREATE INDEX IF NOT EXISTS postjobs_parent_id_idx
+          ON #{SCHEMA_NAME}.postjobs(parent_id);
+      SQL
+
+      SQL.exec <<~SQL
+        CREATE TABLE IF NOT EXISTS #{SCHEMA_NAME}.tokens (
+          id          BIGSERIAL PRIMARY KEY,
+          postjob_id  BIGINT REFERENCES #{SCHEMA_NAME}.postjobs ON DELETE CASCADE,
+          token       UUID NOT NULL,
+          created_at  timestamp NOT NULL DEFAULT (now() at time zone 'utc')
+        );
+
+        CREATE INDEX IF NOT EXISTS tokens_postjob_id_idx ON #{SCHEMA_NAME}.tokens(postjob_id);
+        CREATE INDEX IF NOT EXISTS tokens_token_idx ON #{SCHEMA_NAME}.tokens(token);
+      SQL
+    end
+  end
+end

data/lib/postjob/queue/encoder.rb

@@ -0,0 +1,40 @@
+# rubocop:disable Style/Documentation
+module Postjob::Queue
+end
+
+require "json"
+
+#
+# The Postjob::Queue::Encoder module wraps the JSON encoder, to ensure that only
+# *our* data is encoded.
+#
+# Workflows should exclusively use Numbers, true, false, nil, Strings, Times and
+# Dates, and Arrays and Hashes built of those.
+#
+# postjob does not support all data types supported by Ruby's "json" library.
+# We do not support Symbols, but might also not support things like ActiveRecord
+# objects etc.
+module Postjob::Queue::Encoder
+  extend self
+
+  def encode(data)
+    verify_encodable!(data)
+    JSON.generate(data)
+  end
+
+  private
+
+  def verify_encodable!(obj)
+    case obj
+    when nil, true, false then :ok
+    when String then :ok
+    when Numeric then :ok
+    when Time, Date, DateTime then :ok
+    when Hash  then verify_encodable!(obj.keys) && verify_encodable!(obj.values)
+    when Array then obj.each { |entry| verify_encodable!(entry) }
+    else
+      msg = "Unencodable #{obj.class.name} object: #{obj.inspect}"
+      raise ArgumentError, msg
+    end
+  end
+end

data/lib/postjob/queue/notifications.rb

@@ -0,0 +1,72 @@
+#
+# The Postjob::Queue manages enqueueing and fetching jobs from a job queue.
+module Postjob::Queue::Notifications
+  extend self
+
+  SQL        = ::Postjob::Queue::SQL
+  TABLE_NAME = ::Postjob::Queue::TABLE_NAME
+  CHANNEL    = "postjob_notifications"
+
+  def notify_listeners
+    SQL.ask "NOTIFY #{CHANNEL}"
+  end
+
+  def wait_for_new_job
+    started_at = Time.now
+
+    start_listening
+
+    loop do
+      wait_time = time_to_next_job
+      break if wait_time && wait_time <= 0
+
+      wait_time ||= 120
+      Postjob.logger.debug "postjob: waiting for notification for up to #{time_to_next_job} seconds"
+      break if Simple::SQL.wait_for_notify(wait_time)
+    end
+
+    # flush notifications. It is possible that a huge number of notifications
+    # piled up while we have been waiting. The following line takes care of
+    # those.
+    while Simple::SQL.wait_for_notify(0.000001)
+      :nop
+    end
+
+    Postjob.logger.debug "postjob: awoke after #{format('%.03f secs', (Time.now - started_at))}"
+  end
+
+  private
+
+  def start_listening
+    return if @is_listening
+
+    Simple::SQL.ask "LISTEN #{CHANNEL}"
+    @is_listening = true
+  end
+
+  # returns the maximum number of seconds to wait until the
+  # next runnable or timeoutable job comes up.
+  def time_to_next_job
+    queries = []
+
+    escaped_workflows_and_versions = Postjob::Registry.sql_escaped_workflows_and_versions
+    if escaped_workflows_and_versions != ""
+      queries.push <<~SQL
+        SELECT
+          EXTRACT(EPOCH FROM MIN(next_run_at) - (now() at time zone 'utc'))
+        FROM #{TABLE_NAME}
+        WHERE status = 'ready' AND ((workflow, workflow_version) IN (#{escaped_workflows_and_versions}))
+      SQL
+    end
+
+    queries.push <<~SQL
+      SELECT
+        EXTRACT(EPOCH FROM MIN(timing_out_at) - (now() at time zone 'utc'))
+      FROM #{TABLE_NAME}
+      WHERE status IN ('ready', 'sleep')
+    SQL
+
+    timestamps = Simple::SQL.all(queries.join(" UNION "))
+    timestamps.compact.min
+  end
+end

data/lib/postjob/queue/search.rb

@@ -0,0 +1,82 @@
+# rubocop:disable Style/Documentation
+module Postjob::Queue
+end
+
+#
+# The Postjob::Queue::Search module is able to inspect the Postjob Queue.
+module Postjob::Queue::Search
+  extend self
+
+  def one(id, filter: {}, into: nil)
+    query = query(page: 0, per: 1, filter: filter, id: id)
+    Simple::SQL.record(query, into: into)
+  end
+
+  def all(page: 0, per: 100, filter: {}, into: nil)
+    query = query(page: page, per: per, filter: filter)
+    Simple::SQL.records(query, into: into)
+  end
+
+  private
+
+  def query(page: nil, per: nil, filter: {}, root_only: true, id: nil)
+    expect! id => [Integer, nil]
+    expect! page => [Integer, nil]
+    expect! per => [Integer, nil]
+    expect! { page >= 0 && per > 0 }
+    expect! filter => Hash
+
+    conditions = []
+    conditions << "id=#{id}" if id
+    conditions << "root_id=id" if root_only
+    conditions << tags_condition(filter)
+
+    query = base_query(conditions)
+    query = paginated_query query, per: per, page: page
+    query
+  end
+
+  def paginated_query(query, per:, page:)
+    expect! per => Integer
+    expect! page => Integer
+
+    <<~SQL
+      SELECT * FROM (#{query}) sq LIMIT #{per} OFFSET #{per * page}
+    SQL
+  end
+
+  def base_query(conditions = [])
+    conditions.compact!
+    conditions << "TRUE"
+    condition_fragment = conditions
+                         .compact
+                         .map { |s| "(#{s})" }
+                         .join(" AND ")
+
+    <<~SQL
+      SELECT
+        id,
+        full_id,
+        workflow || COALESCE('@' || workflow_version, '') || args AS job,
+        workflow_status,
+        status,
+        error,
+        COALESCE((results->0)::varchar, error_message) AS result,
+        next_run_at,
+        error_backtrace,
+        (now() at time zone 'utc') - created_at AS age,
+        updated_at - created_at AS runtime,
+        tags
+      FROM postjob.postjobs
+      WHERE #{condition_fragment}
+      ORDER BY id
+    SQL
+  end
+
+  def tags_condition(keys_and_values)
+    expect! keys_and_values => Hash
+    return nil if keys_and_values.empty?
+
+    "tags @> '#{Postjob::Queue::Encoder.encode(keys_and_values)}'"
+  end
+end