postjob 0.4.4 → 0.4.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3db8f37270f64683adc0ef25167f8198cb93e681
-  data.tar.gz: 323f1a09174b3364e8776c2e59e8a9fad9e22708
+  metadata.gz: 78a70bee1c76f0285da58ecc0e1f38f4d420d9f1
+  data.tar.gz: 4fe406b37ebe91e317c00a6c8cb84fceb6b5ce55
 SHA512:
-  metadata.gz: d22d0304488080812e9f3b967462867b45d1b51c8eb7733bcace9f4f5bfb2a60db95a70c335f71c393e23077a0a30bc0bbca80731783dac5aeebf93f18536b1f
-  data.tar.gz: a261688942b25f1c1c07870d500138d4a601663e18e081ae4a4d09e756ff0e30f246da60fe2669f7b40adaf9dc271998e6d80a130c0dde5001bbaa87f5300ce1
+  metadata.gz: 3e55f152396f288ee5f51ccbc08ab23cb54e82f8afac7b8b69a248d38ff2ada757c075292008810ed749ba585f01591d87cb2780ab49289b46948a83b7360296
+  data.tar.gz: b7ccda0358d5377e8d4bcb089e736fd3f06efe5149f82a6ee792870dcfc05f5966f739313fd53834f501fa9901afdd76473b75be22fd70a780b751cc621a0241

data/lib/postjob/job.rb

@@ -9,6 +9,11 @@ class Postjob::Job < Hash
     replace hsh.dup
   end
 
+  def self.find(job_id)
+    scope = Postjob::Queue.search(id: job_id)
+    Simple::SQL.ask(scope, into: Postjob::Job)
+  end
+
   def self.attribute(sym)
     eval <<~RUBY
       define_method(:#{sym}) { self[:#{sym}] }
@@ -55,6 +60,11 @@ class Postjob::Job < Hash
     end
   end
 
+  def resolved?
+    expect! status => STATUSES
+    %w(ok timeout failed).include?(status)
+  end
+
   def result
     results && results.first
   end

data/lib/postjob/migrations/003_postjobs.sql

@@ -1,37 +1,66 @@
 CREATE TABLE IF NOT EXISTS {SCHEMA_NAME}.postjobs (
-  -- id values, readonly once created
+  -- identifiers ------------------------------------------------------------------------------------------
+  -- These are set when enqueueing a workflow
   id          BIGSERIAL PRIMARY KEY,                                        -- process id
-  parent_id   BIGINT REFERENCES {SCHEMA_NAME}.postjobs ON DELETE CASCADE,  -- parent process id
+  parent_id   BIGINT REFERENCES {SCHEMA_NAME}.postjobs ON DELETE CASCADE,   -- parent process id
   full_id     VARCHAR,                                                      -- full process id
   root_id     BIGINT,                                                       -- root process id
 
+  -- timestamps -------------------------------------------------------------------------------------------
+  -- managed automatically
   created_at  timestamp NOT NULL DEFAULT (now() at time zone 'utc'),        -- creation timestamp
   updated_at  timestamp NOT NULL DEFAULT (now() at time zone 'utc'),        -- update timestamp
 
+  -- workflow specification -------------------------------------------------------------------------------
+  -- Set when enqueueing the workflow, readonly
+
   queue VARCHAR,                                                            -- queue name. (readonly)
   workflow VARCHAR NOT NULL,                                                -- e.g. "MyJobModule" (readonly)
   workflow_method VARCHAR NOT NULL DEFAULT 'run',                           -- e.g. "run" (readonly)
-  workflow_version VARCHAR NOT NULL DEFAULT '',                             -- e.g. "1.0"
   args JSONB,                                                               -- args
+  timing_out_at timestamp,                                                  -- job times out after this timestamp
+  max_attempts INTEGER NOT NULL DEFAULT 1,                                  -- maximum attempts before failing
+
+  -- process state ----------------------------------------------------------------------------------------
+  -- Managed automatically.
 
-  -- process state ----------------------------------------------------
+  -- The workflow version is pinned as soon as the workflow is running for the first time.
+  -- It is readonly afterwards. Note that the default is not NULL, but '', because this allows
+  -- to query via (workflow, workflow_version) IN (('Foo', ''), ('Foo', '1.0'))
+  workflow_version VARCHAR NOT NULL DEFAULT '',                             -- e.g. "1.0"
 
+  -- The workflow status, one of  'ready', 'sleep', 'failed', 'err', 'timeout', 'ok'
   status {SCHEMA_NAME}.statuses DEFAULT 'ready',
-  next_run_at timestamp DEFAULT (now() at time zone 'utc'),                 -- when possible to run next?
-  timing_out_at timestamp,                                                  -- job times out after this timestamp
-  failed_attempts INTEGER NOT NULL DEFAULT 0,                               -- failed how often?
-  max_attempts INTEGER NOT NULL DEFAULT 1,                                  -- maximum attempts before failing
 
-  -- process result ---------------------------------------------------
+  -- Timestamp when consider running this the next time.
+  next_run_at timestamp DEFAULT (now() at time zone 'utc'),
+
+  -- Number of failed attempts so far.
+  failed_attempts INTEGER NOT NULL DEFAULT 0,
 
-  results JSONB,
-  error VARCHAR,
-  error_message VARCHAR,
-  error_backtrace JSONB,
+  -- process result ---------------------------------------------------------------------------------------
 
-  -- custom fields
+  results JSONB,            -- The process result, if any. Only valid when status == 'ok'
+  error VARCHAR,            -- The last error; usually set to the klass of the error.
+  error_message VARCHAR,    -- The human readable error message, for displaying purposes
+  error_backtrace JSONB,    -- additional error information, for debugging purposes
+
+  -- custom fields ----------------------------------------------------------------------------------------
   workflow_status VARCHAR,
-  tags JSONB
+  tags JSONB,
+
+  -- processing_client information ------------------------------------------------------------------------
+  -- This information is passed along from workers during processing. They are only valid
+  -- when status == 'processing'
+  --
+  -- Initially these columns didn't exist, and have been created via another migration
+  -- (003b_processing_columns.sql). They are listed here for documentation purposes.
+  processing_client varchar,              -- host:port of client (taken from pg_stat_activity)
+  processing_client_identifier varchar,   -- free text info, set via set_client_identifier()
+  processing_started_at timestamp,        -- when did processing start?
+  processing_max_duration float           -- maximum expected duration of processing. Afterwards the
+                                          -- processing is considered failed for unknown reasons, and
+                                          -- potentially restarted.
 );
 
 -- [TODO] check indices

data/lib/postjob/migrations/004_tokens.sql

@@ -1,3 +1,6 @@
+-- The tokens table contains tokens for externally resolvable workflows. They are considered
+-- secret, and therefore not part of the main table.
+
 CREATE TABLE IF NOT EXISTS {SCHEMA_NAME}.tokens (
   id          BIGSERIAL PRIMARY KEY,
   postjob_id  BIGINT REFERENCES {SCHEMA_NAME}.postjobs ON DELETE CASCADE,

data/lib/postjob/queue/search.rb

@@ -1,19 +1,18 @@
 module Postjob::Queue
   DEFAULT_ATTRIBUTES = [
-    "id",
-    "full_id",
     "workflow || COALESCE('@' || workflow_version, '') || args AS job",
-    "workflow_status",
-    "status",
-    "error",
     "COALESCE((results->0)::varchar, error_message) AS result",
-    "next_run_at",
-    "error_backtrace",
     "(now() at time zone 'utc') - created_at AS age",
-    "updated_at - created_at AS runtime",
-    "tags"
+    "updated_at - created_at AS runtime"
   ]
 
+  def default_attributes
+    @default_attributes ||= begin
+      column_names = Simple::SQL::Reflection.columns("#{SCHEMA_NAME}.postjobs")
+      column_names + DEFAULT_ATTRIBUTES
+    end
+  end
+
   # Builds a search scope (see Simple::SQL::Scope) for the passed in filter criteria.
   def search(filter = {})
     expect! filter => Hash
@@ -21,7 +20,7 @@ module Postjob::Queue
     # extract options
     filter     = filter.dup
     root_only  = filter.delete(:root_only) || false
-    attributes = filter.delete(:attributes) || DEFAULT_ATTRIBUTES
+    attributes = filter.delete(:attributes) || default_attributes
     expect! attributes => Array
 
     # build Scope
@@ -38,17 +37,46 @@ module Postjob::Queue
       expect! key => [Symbol, String]
     end
 
-    column_names = Simple::SQL::Reflection.columns("#{SCHEMA_NAME}.postjobs")
-    column_names += column_names.map(&:to_sym)
+    converted_filters = filter.inject({}) do |hsh, (key, value)|
+      hsh.update key => convert_filter_value(value, key: key)
+    end
 
-    column_filters, tags_filters = filter.partition { |key, _| column_names.include?(key) }
+    column_filters, tags_filters = converted_filters.partition { |key, _| !column_types[key].nil? }
 
     scope = scope.where(Hash[column_filters])
     scope = scope.where(tags: Hash[tags_filters])
     scope
   end
 
-  def apply_tag_filters(scope, filters)
-    scope.where(tags: Hash[filters])
+  class << self
+    def column_types
+      @column_types ||= _column_types
+    end
+
+    private
+
+    def _column_types
+      column_info = ::Simple::SQL::Reflection.column_info("#{SCHEMA_NAME}.postjobs")
+      hsh = {}
+      column_info.each do |column, rec|
+        hsh[column.to_sym] = hsh[column.to_s] = rec.data_type
+      end
+      hsh
+    end
+  end
+
+  def convert_filter_value(value, key:)
+    value = Array(value)
+
+    case Postjob::Queue.column_types[key]
+    when "bigint"   then simplify_array(value.map(&:to_i))
+    when "integer"  then simplify_array(value.map(&:to_i))
+    when nil        then simplify_array(value.map(&:to_i) + value.map(&:to_s))
+    else                 simplify_array(value.map(&:to_s))
+    end
+  end
+
+  def simplify_array(ary)
+    ary.length == 1 ? ary.first : ary
   end
 end

data/lib/postjob/runner.rb

@@ -107,6 +107,13 @@ module Postjob::Runner
   private
 
   # runs a job. Returns a [ status, value, shutdown ] tuple.
+  #
+  # The shutdown value is used by a worker in run mode (i.e. process
+  # indefinetively) to determine whether or not it should cancel
+  # processing. It is usually nil; but if the worker received a
+  # SIGINT it will be :shutdown instead.
+  #
+  # We are catching SIGINT to allow the job status to be updated.
   def invoke_workflow(workflow, job)
     value = catch(:pending) {
       expect! job.args => [Array, nil]

data/spec/spec_helper.rb

@@ -7,7 +7,7 @@ require "awesome_print"
 
 unless ENV["SKIP_SIMPLE_COV"]
   SimpleCov.start do
-    minimum_coverage 88
+    minimum_coverage 86
     add_filter "/spec/"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: postjob
 version: !ruby/object:Gem::Version
-  version: 0.4.4
+  version: 0.4.5
 platform: ruby
 authors:
 - radiospiel
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-06-20 00:00:00.000000000 Z
+date: 2018-07-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -123,7 +123,7 @@ dependencies:
         version: '0.4'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.4.7
+        version: 0.4.8
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -133,7 +133,7 @@ dependencies:
         version: '0.4'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.4.7
+        version: 0.4.8
 - !ruby/object:Gem::Dependency
   name: simple-cli
   requirement: !ruby/object:Gem::Requirement