pallets 0.8.0 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 46a631b6e48f2f1c4efe5bce09c5d74fa2cb064a78d5e546492c6c8ce1dec843
-  data.tar.gz: 61f6414e16941de41defb30ac8dc9121703f75251e47422239c311a8b7749787
+  metadata.gz: ede234db1dfe8c744cf11e6e2fde3eb2e982e7764738715acbbad35710422274
+  data.tar.gz: a55b3a714dcc5db2a294b27ca53724a1236111e08f63078bcef5007bda65532c
 SHA512:
-  metadata.gz: 4b6f77bd93576dc7dc2c5cf945d52c425a63178740c12fa958f0c18c4ca870da7b02d303c7b8e871504d84a13461f79edc52da2869069322f5a81943bdbd79df
-  data.tar.gz: a2a71f4a2343927a94871523b7475f587bb64181e9c6a4654a40111453e2fc2f20e82a02da44adbc32865f68c53a95dd3a9c6f0b641cfd85c7939f1e06d3521f
+  metadata.gz: 74e23087ba192aeffeb7e167d751bfb6f85de4018fb3c16e992ac6119d7670d60862f41a8e1624817a25944409602e41ee38f13adad7b14a38b185d51142b644
+  data.tar.gz: 96ae4f2aa4edf5d27d366e7a35f16527e9d390a45794d306acc109b1be335682c8b1e13e7d708d7eb3451eb8b6ed96db37ba724da08abb13d7b84f3468869826

data/CHANGELOG.md

@@ -6,6 +6,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.9.0] - 2020-07-05
+### Added
+- limit number of jobs in given up set by number (#56)
+- job duration and metadata to all task logs (#57)
+
+### Changed
+- remove all related workflow keys when giving up on a job (#55)
+- support redis-rb ~> 4.2 (#58)
+
+### Removed
+- support for configuring custom loggers (#57)
+
 ## [0.8.0] - 2020-06-09
 ### Added
 - sync output in CLI (#49)

data/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2019 Andrei Horak
+Copyright (c) 2020 Andrei Horak
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -2,7 +2,7 @@
 
 [![Build Status](https://travis-ci.com/linkyndy/pallets.svg?branch=master)](https://travis-ci.com/linkyndy/pallets)
 
-Toy workflow engine, written in Ruby
+Simple and reliable workflow engine, written in Ruby
 
 ## It is plain simple!
 
@@ -11,10 +11,10 @@ Toy workflow engine, written in Ruby
 require 'pallets'
 
 class MyWorkflow < Pallets::Workflow
-  task Foo
-  task Bar => Foo
-  task Baz => Foo
-  task Qux => [Bar, Baz]
+  task 'Foo'
+  task 'Bar' => 'Foo'
+  task 'Baz' => 'Foo'
+  task 'Qux' => ['Bar', 'Baz']
 end
 
 class Foo < Pallets::Task
@@ -120,7 +120,7 @@ end
 
 ## Motivation
 
-The main reason for Pallet's existence was the need of a fast, simple and reliable workflow engine, one that is easily extensible with various backends and serializer, one that does not lose your data and one that is intelligent enough to concurrently schedule a workflow's tasks.
+The main reason for Pallets' existence was the need of a fast, simple and reliable workflow engine, one that is easily extensible with various backends and serializer, one that does not lose your data and one that is intelligent enough to concurrently schedule a workflow's tasks.
 
 ## Status
 

data/examples/config_savvy.rb

@@ -1,4 +1,3 @@
-require 'logger'
 require 'pallets'
 
 class AnnounceProcessing
@@ -32,8 +31,6 @@ Pallets.configure do |c|
   # given up. Retry times are exponential and happen after: 7, 22, 87, 262, ...
   c.max_failures = 5
 
-  # Custom loggers can be used too
-  c.logger = Logger.new(STDOUT)
   # Job execution can be wrapped with middleware to provide custom logic.
   # Anything that responds to `call` would do
   c.middleware << AnnounceProcessing

data/lib/pallets.rb

@@ -38,6 +38,7 @@ module Pallets
       cls.new(
         blocking_timeout: configuration.blocking_timeout,
         failed_job_lifespan: configuration.failed_job_lifespan,
+        failed_job_max_count: configuration.failed_job_max_count,
         job_timeout: configuration.job_timeout,
         pool_size: configuration.pool_size,
         **configuration.backend_args
@@ -57,6 +58,9 @@ module Pallets
   end
 
   def self.logger
-    @logger ||= configuration.logger
+    @logger ||= Pallets::Logger.new(STDOUT,
+      level: Pallets::Logger::INFO,
+      formatter: Pallets::Logger::Formatters::Pretty.new
+    )
   end
 end

data/lib/pallets/backends/base.rb

@@ -20,8 +20,13 @@ module Pallets
         raise NotImplementedError
       end
 
+      # Discards malformed job
+      def discard(job)
+        raise NotImplementedError
+      end
+
       # Gives up job after repeteadly failing to process it
-      def give_up(job, old_job)
+      def give_up(wfid, job, old_job)
         raise NotImplementedError
       end
 

data/lib/pallets/backends/redis.rb

@@ -9,13 +9,15 @@ module Pallets
       RETRY_SET_KEY = 'retry-set'
       GIVEN_UP_SET_KEY = 'given-up-set'
       WORKFLOW_QUEUE_KEY = 'workflow-queue:%s'
+      JOBMASKS_KEY = 'jobmasks:%s'
       JOBMASK_KEY = 'jobmask:%s'
       CONTEXT_KEY = 'context:%s'
       REMAINING_KEY = 'remaining:%s'
 
-      def initialize(blocking_timeout:, failed_job_lifespan:, job_timeout:, pool_size:, **options)
+      def initialize(blocking_timeout:, failed_job_lifespan:, failed_job_max_count:, job_timeout:, pool_size:, **options)
         @blocking_timeout = blocking_timeout
         @failed_job_lifespan = failed_job_lifespan
+        @failed_job_max_count = failed_job_max_count
         @job_timeout = job_timeout
         @pool = Pallets::Pool.new(pool_size) { ::Redis.new(options) }
 
@@ -46,7 +48,7 @@ module Pallets
         @pool.execute do |client|
           client.evalsha(
             @scripts['save'],
-            [WORKFLOW_QUEUE_KEY % wfid, QUEUE_KEY, RELIABILITY_QUEUE_KEY, RELIABILITY_SET_KEY, CONTEXT_KEY % wfid, REMAINING_KEY % wfid, JOBMASK_KEY % jid],
+            [WORKFLOW_QUEUE_KEY % wfid, QUEUE_KEY, RELIABILITY_QUEUE_KEY, RELIABILITY_SET_KEY, CONTEXT_KEY % wfid, REMAINING_KEY % wfid, JOBMASK_KEY % jid, JOBMASKS_KEY % wfid],
             context_buffer.to_a << job
           )
         end
@@ -62,12 +64,22 @@ module Pallets
         end
       end
 
-      def give_up(job, old_job)
+      def discard(job)
         @pool.execute do |client|
           client.evalsha(
-            @scripts['give_up'],
+            @scripts['discard'],
             [GIVEN_UP_SET_KEY, RELIABILITY_QUEUE_KEY, RELIABILITY_SET_KEY],
-            [Time.now.to_f, job, old_job, Time.now.to_f - @failed_job_lifespan]
+            [Time.now.to_f, job, Time.now.to_f - @failed_job_lifespan, @failed_job_max_count]
+          )
+        end
+      end
+
+      def give_up(wfid, job, old_job)
+        @pool.execute do |client|
+          client.evalsha(
+            @scripts['give_up'],
+            [GIVEN_UP_SET_KEY, RELIABILITY_QUEUE_KEY, RELIABILITY_SET_KEY, JOBMASKS_KEY % wfid, WORKFLOW_QUEUE_KEY % wfid, REMAINING_KEY % wfid, CONTEXT_KEY % wfid],
+            [Time.now.to_f, job, old_job, Time.now.to_f - @failed_job_lifespan, @failed_job_max_count]
           )
         end
       end
@@ -86,6 +98,7 @@ module Pallets
         @pool.execute do |client|
           client.multi do
             jobmasks.each { |jid, jobmask| client.zadd(JOBMASK_KEY % jid, jobmask) }
+            client.sadd(JOBMASKS_KEY % wfid, jobmasks.map { |jid, _| JOBMASK_KEY % jid }) unless jobmasks.empty?
             client.evalsha(
               @scripts['run_workflow'],
               [WORKFLOW_QUEUE_KEY % wfid, QUEUE_KEY, REMAINING_KEY % wfid],

data/lib/pallets/backends/scripts/discard.lua

@@ -0,0 +1,11 @@
+-- Remove job from reliability queue
+redis.call("LREM", KEYS[2], 0, ARGV[2])
+redis.call("ZREM", KEYS[3], ARGV[2])
+
+-- Add job and its fail time (score) to failed sorted set
+redis.call("ZADD", KEYS[1], ARGV[1], ARGV[2])
+
+-- Remove any jobs that have been given up long enough ago (their score is
+-- below given value) and make sure the number of jobs is capped
+redis.call("ZREMRANGEBYSCORE", KEYS[1], "-inf", ARGV[3])
+redis.call("ZREMRANGEBYRANK", KEYS[1], 0, -ARGV[4] - 1)

data/lib/pallets/backends/scripts/give_up.lua

@@ -5,6 +5,11 @@ redis.call("ZREM", KEYS[3], ARGV[3])
 -- Add job and its fail time (score) to failed sorted set
 redis.call("ZADD", KEYS[1], ARGV[1], ARGV[2])
 
+-- Remove all related workflow keys
+local keys = redis.call("SMEMBERS", KEYS[4])
+redis.call("DEL", KEYS[4], KEYS[5], KEYS[6], KEYS[7], unpack(keys))
+
 -- Remove any jobs that have been given up long enough ago (their score is
--- below given value)
+-- below given value) and make sure the number of jobs is capped
 redis.call("ZREMRANGEBYSCORE", KEYS[1], "-inf", ARGV[4])
+redis.call("ZREMRANGEBYRANK", KEYS[1], 0, -ARGV[5] - 1)

data/lib/pallets/backends/scripts/save.lua

@@ -22,9 +22,9 @@ if #work > 0 then
   redis.call("ZREM", KEYS[1], unpack(work))
 end
 
--- Decrement ETA and remove it together with the context if all tasks have
--- been processed (ETA is 0)
+-- Decrement ETA and remove it together with the context and jobmasks if all
+-- tasks have been processed (ETA is 0) or if workflow has been given up (ETA is -1)
 local remaining = redis.call("DECR", KEYS[6])
-if remaining == 0 then
-  redis.call("DEL", KEYS[5], KEYS[6])
+if remaining <= 0 then
+  redis.call("DEL", KEYS[5], KEYS[6], KEYS[8])
 end

data/lib/pallets/cli.rb

@@ -67,6 +67,10 @@ module Pallets
           Pallets.configuration.failed_job_lifespan = failed_job_lifespan
         end
 
+        opts.on('-m', '--failed-job-max-count NUM', Integer, 'Maximum number of jobs in the given up set') do |failed_job_max_count|
+          Pallets.configuration.failed_job_max_count = failed_job_max_count
+        end
+
         opts.on('-p', '--pool-size NUM', Integer, 'Size of backend pool') do |pool_size|
           Pallets.configuration.pool_size = pool_size
         end

data/lib/pallets/configuration.rb

@@ -16,13 +16,14 @@ module Pallets
     # this period, jobs will be permanently deleted
     attr_accessor :failed_job_lifespan
 
+    # Maximum number of failed jobs that can be in the given up set. When this
+    # number is reached, the oldest jobs will be permanently deleted
+    attr_accessor :failed_job_max_count
+
     # Number of seconds allowed for a job to be processed. If a job exceeds this
     # period, it is considered failed, and scheduled to be processed again
     attr_accessor :job_timeout
 
-    # Custom logger used throughout Pallets
-    attr_writer :logger
-
     # Maximum number of failures allowed per job. Can also be configured on a
     # per task basis
     attr_accessor :max_failures
@@ -48,19 +49,13 @@ module Pallets
       @blocking_timeout = 5
       @concurrency = 2
       @failed_job_lifespan = 7_776_000 # 3 months
+      @failed_job_max_count = 1_000
       @job_timeout = 1_800 # 30 minutes
       @max_failures = 3
       @serializer = :json
       @middleware = default_middleware
     end
 
-    def logger
-      @logger || Pallets::Logger.new(STDOUT,
-        level: Pallets::Logger::INFO,
-        formatter: Pallets::Logger::Formatters::Pretty.new
-      )
-    end
-
     def pool_size
       @pool_size || @concurrency + 1
     end

data/lib/pallets/logger.rb

@@ -5,14 +5,22 @@ module Pallets
   class Logger < ::Logger
     # Overwrite severity methods to add metadata capabilities
     %i[debug info warn error fatal unknown].each do |severity|
-      define_method severity do |message, metadata = {}|
-        return super(message) if metadata.empty?
+      define_method severity do |message|
+        metadata = Thread.current[:pallets_log_metadata]
+        return super(message) if metadata.nil?
 
         formatted_metadata = ' ' + metadata.map { |k, v| "#{k}=#{v}" }.join(' ')
         super(formatted_metadata) { message }
       end
     end
 
+    def with_metadata(hash)
+      Thread.current[:pallets_log_metadata] = hash
+      yield
+    ensure
+      Thread.current[:pallets_log_metadata] = nil
+    end
+
     module Formatters
       class Pretty < ::Logger::Formatter
         def call(severity, time, metadata, message)

data/lib/pallets/middleware/job_logger.rb

@@ -2,14 +2,21 @@ module Pallets
   module Middleware
     class JobLogger
       def self.call(worker, job, context)
-        Pallets.logger.info 'Started', extract_metadata(worker.id, job)
-        result = yield
-        Pallets.logger.info 'Done', extract_metadata(worker.id, job)
-        result
-      rescue => ex
-        Pallets.logger.warn "#{ex.class.name}: #{ex.message}", extract_metadata(worker.id, job)
-        Pallets.logger.warn ex.backtrace.join("\n"), extract_metadata(worker.id, job) unless ex.backtrace.nil?
-        raise
+        start_time = current_time
+
+        Pallets.logger.with_metadata(extract_metadata(worker.id, job)) do
+          begin
+            Pallets.logger.info 'Started'
+            result = yield
+            Pallets.logger.info "Done in #{(current_time - start_time).round(3)}s"
+            result
+          rescue => ex
+            Pallets.logger.warn "Failed after #{(current_time - start_time).round(3)}s"
+            Pallets.logger.warn "#{ex.class.name}: #{ex.message}"
+            Pallets.logger.warn ex.backtrace.join("\n") unless ex.backtrace.nil?
+            raise
+          end
+        end
       end
 
       def self.extract_metadata(wid, job)
@@ -21,6 +28,10 @@ module Pallets
           tsk:  job['task_class'],
         }
       end
+
+      def self.current_time
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
     end
   end
 end

data/lib/pallets/version.rb

@@ -1,3 +1,3 @@
 module Pallets
-  VERSION = "0.8.0"
+  VERSION = "0.9.0"
 end

data/lib/pallets/worker.rb

@@ -53,8 +53,8 @@ module Pallets
     rescue Pallets::Shutdown
       @manager.remove_worker(self)
     rescue => ex
-      Pallets.logger.error "#{ex.class.name}: #{ex.message}", wid: id
-      Pallets.logger.error ex.backtrace.join("\n"), wid: id unless ex.backtrace.nil?
+      Pallets.logger.error "#{ex.class.name}: #{ex.message}"
+      Pallets.logger.error ex.backtrace.join("\n") unless ex.backtrace.nil?
       @manager.replace_worker(self)
     end
 
@@ -64,8 +64,8 @@ module Pallets
       rescue
         # We ensure only valid jobs are created. If something fishy reaches this
         # point, just give up on it
-        backend.give_up(job, job)
-        Pallets.logger.error "Could not deserialize #{job}. Gave up job", wid: id
+        backend.discard(job)
+        Pallets.logger.error "Could not deserialize #{job}. Gave up job"
         return
       end
 
@@ -103,7 +103,7 @@ module Pallets
         retry_at = Time.now.to_f + backoff_in_seconds(failures)
         backend.retry(new_job, job, retry_at)
       else
-        backend.give_up(new_job, job)
+        backend.give_up(job_hash['wfid'], new_job, job)
       end
     end
 
@@ -112,7 +112,7 @@ module Pallets
         'given_up_at' => Time.now.to_f,
         'reason' => 'returned_false'
       ))
-      backend.give_up(new_job, job)
+      backend.give_up(job_hash['wfid'], new_job, job)
     end
 
     def handle_job_success(context, job, job_hash)

data/pallets.gemspec

@@ -9,8 +9,8 @@ Gem::Specification.new do |spec|
   spec.authors       = ['Andrei Horak']
   spec.email         = ['linkyndy@gmail.com']
 
-  spec.summary       = 'Toy workflow engine, written in Ruby'
-  spec.description   = 'Toy workflow engine, written in Ruby'
+  spec.summary       = 'Simple and reliable workflow engine, written in Ruby'
+  spec.description   = 'Simple and reliable workflow engine, written in Ruby'
   spec.homepage      = 'https://github.com/linkyndy/pallets'
   spec.license       = 'MIT'
 
@@ -20,6 +20,6 @@ Gem::Specification.new do |spec|
 
   spec.required_ruby_version = '>= 2.4'
 
-  spec.add_dependency 'redis'
+  spec.add_dependency 'redis', '~> 4.2'
   spec.add_dependency 'msgpack'
 end

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: pallets
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.9.0
 platform: ruby
 authors:
 - Andrei Horak
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-06-09 00:00:00.000000000 Z
+date: 2020-07-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '4.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '4.2'
 - !ruby/object:Gem::Dependency
   name: msgpack
   requirement: !ruby/object:Gem::Requirement
@@ -38,7 +38,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: Toy workflow engine, written in Ruby
+description: Simple and reliable workflow engine, written in Ruby
 email:
 - linkyndy@gmail.com
 executables:
@@ -67,6 +67,7 @@ files:
 - lib/pallets.rb
 - lib/pallets/backends/base.rb
 - lib/pallets/backends/redis.rb
+- lib/pallets/backends/scripts/discard.lua
 - lib/pallets/backends/scripts/give_up.lua
 - lib/pallets/backends/scripts/reschedule_all.lua
 - lib/pallets/backends/scripts/retry.lua
@@ -98,7 +99,7 @@ homepage: https://github.com/linkyndy/pallets
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -114,7 +115,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.1.2
-signing_key: 
+signing_key:
 specification_version: 4
-summary: Toy workflow engine, written in Ruby
+summary: Simple and reliable workflow engine, written in Ruby
 test_files: []