postjob 0.4.2 → 0.4.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 79193eb5de0de5ad5a9de485fd40378c66907c22
-  data.tar.gz: 252abc25bfdb766849168de4bef806c5938d9983
+  metadata.gz: 0be2321073faba782e89878c47c6c6770b97e7ff
+  data.tar.gz: c9d3cf35e02a4105587d5888535e4297d4abb9af
 SHA512:
-  metadata.gz: 458b190937d133249530d48d6cc16d96f94c8ced9af36d944e1c71144112045849ea04fdd348346b9dd419106eae4a189a3acd1b6c966fea5c543d7474261d8f
-  data.tar.gz: 6d7e015d27c697b26bd8aa39d045e1efaada633fcd1b2db11a0acdb83c48a69e8e77ab7313bd98e26a1db3412a9107b3f3ca36e4e17a091fae68d0cd2d103579
+  metadata.gz: 208afca0daf608d1ada39ad4fac5539e2ad10388d73e469bc4164eee2f49c4615a0e845578dd1aad5db6fe33f0decdef62c0329b198fa4f44f2e9941363eb2e7
+  data.tar.gz: 11a854fb585336169a6b218884d8b02503c5e52503efac95609b937dc18dd51703868f93a569f3fb66b557c411215bf9ecf797f5ddd33a4781a750092676855f

data/lib/postjob/cli/ps.rb

@@ -18,20 +18,27 @@ module Postjob::CLI
           || workflow_version
           || (CASE WHEN workflow_method != 'run' THEN '.' || workflow_method ELSE '' END)
           || args AS job,
-        workflow_status AS status,
-        status,
+        CASE
+          WHEN (status = 'err') THEN 'err(' || failed_attempts || '/' || max_attempts || ')'
+          ELSE status::varchar
+          END AS status,
         error,
         COALESCE((results->0)::varchar, error_message) AS result,
         next_run_at,
         next_run_at - (now() at time zone 'utc') AS next_run_in,
         to_char(EXTRACT(EPOCH FROM (now() at time zone 'utc') - postjobs.created_at), '999999999.99') AS age,
-        CASE WHEN processing_started_at IS NOT NULL THEN
-        format(
-          '%s/%s',
-          to_char(EXTRACT(EPOCH FROM (now() at time zone 'utc') - processing_started_at), '999999999.99'),
-          processing_max_duration
-        )
+
+        CASE
+        WHEN processing_started_at IS NOT NULL THEN
+          format(
+            '%s/%s',
+            to_char(EXTRACT(EPOCH FROM (now() at time zone 'utc') - processing_started_at), '999999999.99'),
+            processing_max_duration
+          )
+        WHEN status IN ('failed', 'err', 'ok') THEN
+          format('%s', to_char(EXTRACT(EPOCH FROM (updated_at - created_at)), '999999999.99'))
         END AS processing,
+
         COALESCE(processing_client, '') || COALESCE('/' || processing_client_identifier, '') AS worker,
         tags
       FROM postjob.postjobs AS postjobs

data/lib/postjob/cli/run.rb

@@ -11,12 +11,10 @@ module Postjob::CLI
   # Parameters:
   #
   # - count=<count> maximum number of jobs to process. Default: unlimited.
-  # - fast=<count> reduce waiting time after error (for development only!)
   # - quiet don't show progress.
   #
-  def run(count: nil, fast: false, quiet: false)
+  def run(count: nil, quiet: false)
     count = Integer(count) if count
-    Postjob.fast_mode = fast
 
     connect_to_database!
 

data/lib/postjob/error.rb

@@ -1,16 +1,31 @@
 module Postjob
   class Error < RuntimeError
-    def initialize(job)
-      @job = job
+    attr_reader :message
+
+    def initialize(err)
+      case err
+      when ::Postjob::Job then @message = job_error_msg(err)
+      when nil, false then @message = self.class.name
+      else @message = err.to_s
+      end
+    end
+
+    def to_s
+      message
     end
 
-    def message
-      msg = "Failing child job"
-      msg += " [#{@job.result}]" if @job.result
-      msg
+    private
+
+    def job_error_msg(job)
+      error_msg = "Failing child job"
+      error_msg += " [#{job.result}]" if job.result
+      error_msg
     end
   end
 
   class Error::Nonrecoverable < Error
   end
+
+  class Error::Encoding < Error::Nonrecoverable
+  end
 end

data/lib/postjob/migrations/003b_processing_columns.sql

@@ -2,7 +2,7 @@ DO $$
   BEGIN
     ALTER TABLE {SCHEMA_NAME}.postjobs ADD COLUMN processing_client varchar;
   EXCEPTION
-    WHEN duplicate_column THEN RAISE NOTICE 'column {SCHEMA_NAME}.postjobs.processing_client already exists';
+    WHEN duplicate_column THEN RAISE DEBUG 'column {SCHEMA_NAME}.postjobs.processing_client already exists';
   END;
 $$;
 
@@ -10,7 +10,7 @@ DO $$
   BEGIN
     ALTER TABLE {SCHEMA_NAME}.postjobs ADD COLUMN processing_client_identifier varchar;
   EXCEPTION
-    WHEN duplicate_column THEN RAISE NOTICE 'column {SCHEMA_NAME}.postjobs.processing_client_identifier already exists';
+    WHEN duplicate_column THEN RAISE DEBUG 'column {SCHEMA_NAME}.postjobs.processing_client_identifier already exists';
   END;
 $$;
 
@@ -18,7 +18,7 @@ DO $$
   BEGIN
     ALTER TABLE {SCHEMA_NAME}.postjobs ADD COLUMN processing_started_at timestamp;
   EXCEPTION
-    WHEN duplicate_column THEN RAISE NOTICE 'column {SCHEMA_NAME}.postjobs.processing_started_at already exists';
+    WHEN duplicate_column THEN RAISE DEBUG 'column {SCHEMA_NAME}.postjobs.processing_started_at already exists';
   END;
 $$;
 
@@ -26,6 +26,6 @@ DO $$
   BEGIN
     ALTER TABLE {SCHEMA_NAME}.postjobs ADD COLUMN processing_max_duration float;
   EXCEPTION
-    WHEN duplicate_column THEN RAISE NOTICE 'column {SCHEMA_NAME}.postjobs.processing_max_duration already exists';
+    WHEN duplicate_column THEN RAISE DEBUG 'column {SCHEMA_NAME}.postjobs.processing_max_duration already exists';
   END;
 $$;

data/lib/postjob/migrations/006_enqueue.sql

@@ -14,6 +14,11 @@ AS $$
     job_id BIGINT;
   BEGIN
     -- set defaults ---------------------------------------------------
+    IF workflow = '__manual__' AND COALESCE(max_attempts, 1) != 1 THEN
+      RAISE NOTICE 'Adjusting max_attempts of __manual__ job to 1';
+      max_attempts := 1;
+    END IF;
+
     workflow_version  := COALESCE(workflow_version, '');
     queue             := COALESCE(queue, 'q');
     max_attempts      := COALESCE(max_attempts, 5);

data/lib/postjob/migrations/010_settings.sql

@@ -3,5 +3,14 @@ CREATE TABLE IF NOT EXISTS {SCHEMA_NAME}.settings (
   value       VARCHAR
 );
 
-INSERT INTO {SCHEMA_NAME}.settings (name, value) VALUES('version', '0.4.0') ON CONFLICT DO NOTHING;
-INSERT INTO {SCHEMA_NAME}.settings (name, value) VALUES('client_version', '{CLIENT_VERSION}') ON CONFLICT DO NOTHING;
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.update_settings(p_name VARCHAR, p_value VARCHAR)
+  RETURNS VOID
+AS $$
+  BEGIN
+    INSERT INTO {SCHEMA_NAME}.settings (name, value) VALUES(p_name, p_value)
+    ON CONFLICT(name) DO UPDATE SET value = p_value;
+  END;
+$$ LANGUAGE plpgsql;
+
+SELECT {SCHEMA_NAME}.update_settings('version', '0.4.2');
+SELECT {SCHEMA_NAME}.update_settings('client_version', '{CLIENT_VERSION}');

data/lib/postjob/migrations.rb

@@ -16,6 +16,7 @@ module Postjob
     expect! SCHEMA_NAME != "public"
 
     def unmigrate!
+      SQL.exec "SET client_min_messages TO WARNING"
       SQL.exec "DROP SCHEMA IF EXISTS #{SCHEMA_NAME} CASCADE"
     end
 
@@ -30,7 +31,7 @@ module Postjob
     private
 
     def run_migration(file)
-      STDERR.puts "[postjob] Running migration in #{file}"
+      Postjob.logger.debug "[postjob] Running migration in #{file}"
 
       case file
       when /\.rb$/  then run_migration_ruby(file)

data/lib/postjob/queue/encoder.rb

@@ -16,11 +16,17 @@ require "json"
 module Postjob::Queue::Encoder
   extend self
 
+  def check_encodable!(data)
+    encode(data)
+  end
+
   def encode(data)
     return nil if data.nil?
 
     verify_encodable!(data)
-    JSON.generate(data)
+    ::JSON.generate(data)
+  rescue ::JSON::JSONError, ::EncodingError
+    raise ::Postjob::Error::Encoding, $!.to_s
   end
 
   private
@@ -35,7 +41,7 @@ module Postjob::Queue::Encoder
     when Array then obj.each { |entry| verify_encodable!(entry) }
     else
       msg = "Unencodable #{obj.class.name} object: #{obj.inspect}"
-      raise ArgumentError, msg
+      raise ::Postjob::Error::Encoding, msg
     end
   end
 end

data/lib/postjob/queue.rb

@@ -89,7 +89,14 @@ module Postjob::Queue
   end
 
   def find_or_create_childjob(parent, workflow, args, timeout:, max_attempts:, queue: nil)
-    expect! parent => Job, workflow => String, args => Array
+    expect! parent => Job
+    expect! workflow => String
+    expect! args => Array
+
+    if workflow == "__manual__" && max_attempts != 1
+      Postjob.logger.info "Job ##{parent.id} adjusting max_attempts of '__manual__' child job"
+      max_attempts = 1
+    end
 
     workflow, workflow_method = parse_workflow(workflow)
 

data/lib/postjob/runner.rb

@@ -1,6 +1,7 @@
 # rubocop:disable Metrics/ModuleLength
 # rubocop:disable Style/BlockDelimiters
 # rubocop:disable Lint/RescueException
+# rubocop:disable Metrics/MethodLength
 
 module Postjob::Runner
   extend self
@@ -119,13 +120,22 @@ module Postjob::Runner
       workflow.public_send workflow_method, *args
     }
 
-    case value
-    when :pending then [ :pending, nil, nil ]
-    else               [ :ok, value, nil ]
+    if value == :pending
+      [ :pending, nil, nil ]
+    else
+      # Check that we can encode the value. If we can't the job returned something invalid
+      # i.e. something we cannot encode as JSON. This will raise a Postjob::Queue::Encoder::Error.
+      #
+      # Usually this points to a non-UTF8 string.
+      # This is a fix for https://github.com/mediapeers/postjob/issues/35
+      Postjob::Queue::Encoder.check_encodable!([value])
+      [ :ok, value, nil ]
     end
   rescue ArgumentError, LocalJumpError, NameError, RegexpError, ScriptError, TypeError
     Postjob.logger.error "#{$!}, from\n\t#{$!.backtrace[0, 10].join("\n\t")}"
     return_exception :failed, $!
+  rescue Postjob::Error::Nonrecoverable
+    return_exception :failed, $!
   rescue Exception
     return_exception :err, $!
   end

data/lib/postjob.rb

@@ -1,4 +1,5 @@
 # rubocop:disable Metrics/ParameterLists
+# rubocop:disable Metrics/MethodLength
 
 require "expectation"
 require "simple/sql"
@@ -40,6 +41,16 @@ module Postjob
     expect! timeout => [nil, Numeric]
     expect! tags => [nil, Hash]
 
+    if workflow == "__manual__" && max_attempts != 1
+      if parent_id
+        Postjob.logger.info "Job ##{parent_id} adjusting max_attempts of '__manual__' child job"
+      else
+        Postjob.logger.info "Adjusting max_attempts of '__manual__' root job"
+      end
+
+      max_attempts = 1
+    end
+
     tags = stringify_hash(tags) if tags
     job = Queue.enqueue_job workflow, *args, queue: queue,
                                              parent_id: parent_id,

data/spec/postjob/job_control/manual_spec.rb

@@ -1,8 +1,8 @@
 require "spec_helper"
 
 module ManualWorkflow
-  def self.run
-    job = async :manual, timeout: 10
+  def self.run(max_attempts)
+    job = async :manual, timeout: 10, max_attempts: max_attempts
     _token = workflow_token job
     manual_result = await job
     "manual-result:#{manual_result}"
@@ -12,7 +12,7 @@ module ManualWorkflow
 end
 
 describe "manual processing" do
-  let!(:id) { Postjob.enqueue! "ManualWorkflow" }
+  let!(:id) { Postjob.enqueue! "ManualWorkflow", 3 }
 
   include TestHelper
 
@@ -24,6 +24,10 @@ describe "manual processing" do
     TestHelper.load_job "SELECT * FROM postjobs WHERE parent_id=$1", id
   end
 
+  it "sets max_attempts to 1" do
+    expect(load_child_job[:max_attempts]).to eq(1)
+  end
+
   it "creates a UUID token" do
     token = load_token(load_child_job)
     expect! token => /[A-F0-9]{8}-[A-F0-9]{4}-[A-F0-9]{4}-[a-fA-F0-9]{4}-[A-F0-9]{12}/i

data/spec/postjob/queue/encoder_spec.rb

@@ -37,14 +37,27 @@ describe "Postjob::Queue::Encoder" do
   end
 
   describe "invalid input" do
-    it "cannot encode Symbols on input" do
-      expect do
-        run_workflow input: :foo
-      end.to raise_error(ArgumentError)
-
-      expect do
-        run_workflow output: :foo
-      end.to raise_error(ArgumentError)
+    def self.check_value(value)
+      it "cannot encode input" do
+        expect do
+          run_workflow input: value
+        end.to raise_error(::Postjob::Error::Encoding)
+      end
+
+      it "cannot encode output" do
+        expect do
+          run_workflow output: value
+        end.to raise_error(::Postjob::Error::Encoding)
+      end
+    end
+
+    context "when encoding symbols" do
+      check_value :foo
+    end
+
+    context "when encoding invalid utf8 strings" do
+      check_value 255.chr.force_encoding("utf-8")
+      check_value 255.chr
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: postjob
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.4.3
 platform: ruby
 authors:
 - radiospiel
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-06-04 00:00:00.000000000 Z
+date: 2018-06-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec