pallets 0.6.0 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a2894dd7dff299d2cb4f2c9bf67b7be01f27adc93cfc7729fbbac8c3c9988722
-  data.tar.gz: ce6d91fc194a7441557bd3250f994b65dbb35dd29d7adbb0a08c3be699a88cf8
+  metadata.gz: c7ee0fbd91594f23bc84b81116f867490620f580223195f69fa9ff4f72be2558
+  data.tar.gz: b681bc765933ce36d3732a04eae3b9b7d60c2695b3d47f5704ebcfe96e3f884b
 SHA512:
-  metadata.gz: 37ff1a501b2661b0a63d9d0c3844d323952e07e05135661169138dd3ee88f906115c3db13b33205dbcc7faf94af6ac2dc4a8cc2e2e49982789ca5e75cca0ea2a
-  data.tar.gz: 5cc7c495ca3b2463ef2d8d168f0bb43f73c6c7d7a970d66b21ebe22068db389e272819e2d2f7bb49c6fc2835f6b22a8b3991421e5501dc763a94c5aecd5aca88
+  metadata.gz: 8dfe6d23000df032d7f65fd81bd7907a3600240eaa6f111a161d1b2fb6d234440c8cc9219203a094ae139847ba00011939b50943e5eaaa04c11dde9128c3bb94
+  data.tar.gz: c054b1abd6d25b0408fa8427550751c8c8d41cd204bed65a968302d2ad5be98ac5ef020a6d096efc52ec58e9fef6f66456528ea72cdf423666916dd3a1394a5f

data/.github/FUNDING.yml

@@ -0,0 +1 @@
+github: linkyndy

data/.travis.yml

@@ -4,11 +4,8 @@ services:
   - redis-server
 cache: bundler
 rvm:
-  - 2.4.6
-  - 2.5.5
-  - 2.6.3
-before_install:
-  # Bundler 2.0 needs a newer RubyGems
-  - gem update --system
-  - gem install bundler
+  - 2.4.10
+  - 2.5.8
+  - 2.6.6
+  - 2.7.1
 script: bundle exec rspec

data/CHANGELOG.md

@@ -6,6 +6,45 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.11.0] - 2020-10-09
+### Added
+- JSON logger (#62)
+
+### Changed
+- fix for leaking Redis connections (#61)
+
+## [0.10.0] - 2020-08-30
+### Added
+- configure polling interval for scheduler (#60)
+
+### Changed
+- handle persisting unforseen worker errors more gracefully (#59)
+- add initial wait to scheduler startup (#60)
+
+## [0.9.0] - 2020-07-05
+### Added
+- limit number of jobs in given up set by number (#56)
+- job duration and metadata to all task logs (#57)
+
+### Changed
+- remove all related workflow keys when giving up on a job (#55)
+- support redis-rb ~> 4.2 (#58)
+
+### Removed
+- support for configuring custom loggers (#57)
+
+## [0.8.0] - 2020-06-09
+### Added
+- sync output in CLI (#49)
+- support for configuring custom loggers (#50)
+
+### Changed
+- improve job scheduling using jobmasks (#52)
+
+## [0.7.0] - 2020-01-19
+### Added
+- support for Ruby 2.7 (#46)
+
 ## [0.6.0] - 2019-09-02
 ### Added
 - define task aliases in order to reuse tasks within a workflow definition (#44)

data/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2019 Andrei Horak
+Copyright (c) 2020 Andrei Horak
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -2,7 +2,7 @@
 
 [![Build Status](https://travis-ci.com/linkyndy/pallets.svg?branch=master)](https://travis-ci.com/linkyndy/pallets)
 
-Toy workflow engine, written in Ruby
+Simple and reliable workflow engine, written in Ruby
 
 ## It is plain simple!
 
@@ -11,10 +11,10 @@ Toy workflow engine, written in Ruby
 require 'pallets'
 
 class MyWorkflow < Pallets::Workflow
-  task Foo
-  task Bar => Foo
-  task Baz => Foo
-  task Qux => [Bar, Baz]
+  task 'Foo'
+  task 'Bar' => 'Foo'
+  task 'Baz' => 'Foo'
+  task 'Qux' => ['Bar', 'Baz']
 end
 
 class Foo < Pallets::Task
@@ -120,7 +120,7 @@ end
 
 ## Motivation
 
-The main reason for Pallet's existence was the need of a fast, simple and reliable workflow engine, one that is easily extensible with various backends and serializer, one that does not lose your data and one that is intelligent enough to concurrently schedule a workflow's tasks.
+The main reason for Pallets' existence was the need of a fast, simple and reliable workflow engine, one that is easily extensible with various backends and serializer, one that does not lose your data and one that is intelligent enough to concurrently schedule a workflow's tasks.
 
 ## Status
 

data/examples/config_savvy.rb

@@ -43,7 +43,7 @@ end
 
 class Volatile < Pallets::Task
   def run
-    raise 'I am ràndomly failing' if [true, false].sample
+    raise 'I am randomly failing' if [true, false].sample
   end
 end
 

data/lib/pallets.rb

@@ -38,6 +38,7 @@ module Pallets
       cls.new(
         blocking_timeout: configuration.blocking_timeout,
         failed_job_lifespan: configuration.failed_job_lifespan,
+        failed_job_max_count: configuration.failed_job_max_count,
         job_timeout: configuration.job_timeout,
         pool_size: configuration.pool_size,
         **configuration.backend_args
@@ -62,8 +63,4 @@ module Pallets
       formatter: Pallets::Logger::Formatters::Pretty.new
     )
   end
-
-  def self.logger=(logger)
-    @logger = logger
-  end
 end

data/lib/pallets/backends/base.rb

@@ -6,12 +6,12 @@ module Pallets
         raise NotImplementedError
       end
 
-      def get_context(workflow_id)
+      def get_context(wfid)
         raise NotImplementedError
       end
 
       # Saves a job after successfully processing it
-      def save(workflow_id, job, context_buffer)
+      def save(wfid, jid, job, context_buffer)
         raise NotImplementedError
       end
 
@@ -20,8 +20,13 @@ module Pallets
         raise NotImplementedError
       end
 
+      # Discards malformed job
+      def discard(job)
+        raise NotImplementedError
+      end
+
       # Gives up job after repeteadly failing to process it
-      def give_up(job, old_job)
+      def give_up(wfid, job, old_job)
         raise NotImplementedError
       end
 
@@ -29,7 +34,7 @@ module Pallets
         raise NotImplementedError
       end
 
-      def run_workflow(workflow_id, jobs_with_dependencies, context)
+      def run_workflow(wfid, jobs, jobmasks, context)
         raise NotImplementedError
       end
     end

data/lib/pallets/backends/redis.rb

@@ -9,12 +9,15 @@ module Pallets
       RETRY_SET_KEY = 'retry-set'
       GIVEN_UP_SET_KEY = 'given-up-set'
       WORKFLOW_QUEUE_KEY = 'workflow-queue:%s'
+      JOBMASKS_KEY = 'jobmasks:%s'
+      JOBMASK_KEY = 'jobmask:%s'
       CONTEXT_KEY = 'context:%s'
       REMAINING_KEY = 'remaining:%s'
 
-      def initialize(blocking_timeout:, failed_job_lifespan:, job_timeout:, pool_size:, **options)
+      def initialize(blocking_timeout:, failed_job_lifespan:, failed_job_max_count:, job_timeout:, pool_size:, **options)
         @blocking_timeout = blocking_timeout
         @failed_job_lifespan = failed_job_lifespan
+        @failed_job_max_count = failed_job_max_count
         @job_timeout = job_timeout
         @pool = Pallets::Pool.new(pool_size) { ::Redis.new(options) }
 
@@ -41,11 +44,11 @@ module Pallets
         end
       end
 
-      def save(wfid, job, context_buffer)
+      def save(wfid, jid, job, context_buffer)
         @pool.execute do |client|
           client.evalsha(
             @scripts['save'],
-            [WORKFLOW_QUEUE_KEY % wfid, QUEUE_KEY, RELIABILITY_QUEUE_KEY, RELIABILITY_SET_KEY, CONTEXT_KEY % wfid, REMAINING_KEY % wfid],
+            [WORKFLOW_QUEUE_KEY % wfid, QUEUE_KEY, RELIABILITY_QUEUE_KEY, RELIABILITY_SET_KEY, CONTEXT_KEY % wfid, REMAINING_KEY % wfid, JOBMASK_KEY % jid, JOBMASKS_KEY % wfid],
             context_buffer.to_a << job
           )
         end
@@ -61,12 +64,22 @@ module Pallets
         end
       end
 
-      def give_up(job, old_job)
+      def discard(job)
         @pool.execute do |client|
           client.evalsha(
-            @scripts['give_up'],
+            @scripts['discard'],
             [GIVEN_UP_SET_KEY, RELIABILITY_QUEUE_KEY, RELIABILITY_SET_KEY],
-            [Time.now.to_f, job, old_job, Time.now.to_f - @failed_job_lifespan]
+            [Time.now.to_f, job, Time.now.to_f - @failed_job_lifespan, @failed_job_max_count]
+          )
+        end
+      end
+
+      def give_up(wfid, job, old_job)
+        @pool.execute do |client|
+          client.evalsha(
+            @scripts['give_up'],
+            [GIVEN_UP_SET_KEY, RELIABILITY_QUEUE_KEY, RELIABILITY_SET_KEY, JOBMASKS_KEY % wfid, WORKFLOW_QUEUE_KEY % wfid, REMAINING_KEY % wfid, CONTEXT_KEY % wfid],
+            [Time.now.to_f, job, old_job, Time.now.to_f - @failed_job_lifespan, @failed_job_max_count]
           )
         end
       end
@@ -81,13 +94,15 @@ module Pallets
         end
       end
 
-      def run_workflow(wfid, jobs_with_order, context_buffer)
+      def run_workflow(wfid, jobs, jobmasks, context_buffer)
         @pool.execute do |client|
           client.multi do
+            jobmasks.each { |jid, jobmask| client.zadd(JOBMASK_KEY % jid, jobmask) }
+            client.sadd(JOBMASKS_KEY % wfid, jobmasks.map { |jid, _| JOBMASK_KEY % jid }) unless jobmasks.empty?
             client.evalsha(
               @scripts['run_workflow'],
               [WORKFLOW_QUEUE_KEY % wfid, QUEUE_KEY, REMAINING_KEY % wfid],
-              jobs_with_order
+              jobs
             )
             client.hmset(CONTEXT_KEY % wfid, *context_buffer.to_a) unless context_buffer.empty?
           end

data/lib/pallets/backends/scripts/discard.lua

@@ -0,0 +1,11 @@
+-- Remove job from reliability queue
+redis.call("LREM", KEYS[2], 0, ARGV[2])
+redis.call("ZREM", KEYS[3], ARGV[2])
+
+-- Add job and its fail time (score) to failed sorted set
+redis.call("ZADD", KEYS[1], ARGV[1], ARGV[2])
+
+-- Remove any jobs that have been given up long enough ago (their score is
+-- below given value) and make sure the number of jobs is capped
+redis.call("ZREMRANGEBYSCORE", KEYS[1], "-inf", ARGV[3])
+redis.call("ZREMRANGEBYRANK", KEYS[1], 0, -ARGV[4] - 1)

data/lib/pallets/backends/scripts/give_up.lua

@@ -5,6 +5,11 @@ redis.call("ZREM", KEYS[3], ARGV[3])
 -- Add job and its fail time (score) to failed sorted set
 redis.call("ZADD", KEYS[1], ARGV[1], ARGV[2])
 
+-- Remove all related workflow keys
+local keys = redis.call("SMEMBERS", KEYS[4])
+redis.call("DEL", KEYS[4], KEYS[5], KEYS[6], KEYS[7], unpack(keys))
+
 -- Remove any jobs that have been given up long enough ago (their score is
--- below given value)
+-- below given value) and make sure the number of jobs is capped
 redis.call("ZREMRANGEBYSCORE", KEYS[1], "-inf", ARGV[4])
+redis.call("ZREMRANGEBYRANK", KEYS[1], 0, -ARGV[5] - 1)

data/lib/pallets/backends/scripts/run_workflow.lua

@@ -6,9 +6,8 @@ redis.call("SET", KEYS[3], eta)
 
 -- Queue jobs that are ready to be processed (their score is 0) and
 -- remove queued jobs from the sorted set
-local count = redis.call("ZCOUNT", KEYS[1], 0, 0)
-if count > 0 then
-  local work = redis.call("ZRANGEBYSCORE", KEYS[1], 0, 0)
+local work = redis.call("ZRANGEBYSCORE", KEYS[1], 0, 0)
+if #work > 0 then
   redis.call("LPUSH", KEYS[2], unpack(work))
   redis.call("ZREM", KEYS[1], unpack(work))
 end

data/lib/pallets/backends/scripts/save.lua

@@ -10,24 +10,21 @@ if #ARGV > 0 then
   redis.call("HMSET", KEYS[5], unpack(ARGV))
 end
 
--- Decrement all jobs from the sorted set
-local all_pending = redis.call("ZRANGE", KEYS[1], 0, -1)
-for score, task in pairs(all_pending) do
-  redis.call("ZINCRBY", KEYS[1], -1, task)
-end
+-- Decrement jobs from the sorted set by applying a jobmask
+redis.call("ZUNIONSTORE", KEYS[1], 2, KEYS[1], KEYS[7])
+redis.call("DEL", KEYS[7])
 
 -- Queue jobs that are ready to be processed (their score is 0) and
 -- remove queued jobs from sorted set
-local count = redis.call("ZCOUNT", KEYS[1], 0, 0)
-if count > 0 then
-  local work = redis.call("ZRANGEBYSCORE", KEYS[1], 0, 0)
+local work = redis.call("ZRANGEBYSCORE", KEYS[1], 0, 0)
+if #work > 0 then
   redis.call("LPUSH", KEYS[2], unpack(work))
   redis.call("ZREM", KEYS[1], unpack(work))
 end
 
--- Decrement ETA and remove it together with the context if all tasks have
--- been processed (ETA is 0)
-redis.call("DECR", KEYS[6])
-if tonumber(redis.call("GET", KEYS[6])) == 0 then
-  redis.call("DEL", KEYS[5], KEYS[6])
+-- Decrement ETA and remove it together with the context and jobmasks if all
+-- tasks have been processed (ETA is 0) or if workflow has been given up (ETA is -1)
+local remaining = redis.call("DECR", KEYS[6])
+if remaining <= 0 then
+  redis.call("DEL", KEYS[5], KEYS[6], KEYS[8])
 end

data/lib/pallets/cli.rb

@@ -1,5 +1,7 @@
 require 'optparse'
 
+$stdout.sync = true
+
 module Pallets
   class CLI
     def initialize
@@ -61,10 +63,18 @@ module Pallets
           Pallets.configuration.max_failures = max_failures
         end
 
+        opts.on('-i', '--scheduler-polling-interval NUM', Integer, 'Seconds between scheduler backend polls') do |scheduler_polling_interval|
+          Pallets.configuration.scheduler_polling_interval = scheduler_polling_interval
+        end
+
         opts.on('-l', '--failed-job-lifespan NUM', Integer, 'Seconds a job stays in the given up set') do |failed_job_lifespan|
           Pallets.configuration.failed_job_lifespan = failed_job_lifespan
         end
 
+        opts.on('-m', '--failed-job-max-count NUM', Integer, 'Maximum number of jobs in the given up set') do |failed_job_max_count|
+          Pallets.configuration.failed_job_max_count = failed_job_max_count
+        end
+
         opts.on('-p', '--pool-size NUM', Integer, 'Size of backend pool') do |pool_size|
           Pallets.configuration.pool_size = pool_size
         end

data/lib/pallets/configuration.rb

@@ -16,6 +16,10 @@ module Pallets
     # this period, jobs will be permanently deleted
     attr_accessor :failed_job_lifespan
 
+    # Maximum number of failed jobs that can be in the given up set. When this
+    # number is reached, the oldest jobs will be permanently deleted
+    attr_accessor :failed_job_max_count
+
     # Number of seconds allowed for a job to be processed. If a job exceeds this
     # period, it is considered failed, and scheduled to be processed again
     attr_accessor :job_timeout
@@ -27,6 +31,12 @@ module Pallets
     # Number of connections to the backend
     attr_writer :pool_size
 
+    # Number of seconds at which the scheduler checks whether there are jobs
+    # due to be (re)processed. Note that this interval is per process; it might
+    # require tweaking in case of running multiple Pallets instances, so that
+    # the backend is not polled too often
+    attr_accessor :scheduler_polling_interval
+
     # Serializer used for jobs
     attr_accessor :serializer
 
@@ -45,8 +55,10 @@ module Pallets
       @blocking_timeout = 5
       @concurrency = 2
       @failed_job_lifespan = 7_776_000 # 3 months
+      @failed_job_max_count = 1_000
       @job_timeout = 1_800 # 30 minutes
       @max_failures = 3
+      @scheduler_polling_interval = 10
       @serializer = :json
       @middleware = default_middleware
     end

data/lib/pallets/dsl/workflow.rb

@@ -1,7 +1,13 @@
 module Pallets
   module DSL
     module Workflow
-      def task(arg, as: nil, depends_on: nil, max_failures: nil, &block)
+      def task(arg=nil, as: nil, depends_on: nil, max_failures: nil, **kwargs)
+        # Have to work more to keep Pallets' nice DSL valid in Ruby 2.7
+        arg = !kwargs.empty? ? kwargs : arg
+        raise ArgumentError, 'Task is incorrectly defined. It must receive '\
+                             'either a name, or a name => dependencies pair as '\
+                             'the first argument' unless arg
+
         klass, dependencies = case arg
         when Hash
           # The `task Foo => Bar` notation

data/lib/pallets/graph.rb

@@ -24,18 +24,11 @@ module Pallets
       nodes.empty?
     end
 
-    # Returns nodes topologically sorted, together with their order (number of
-    # nodes that have to be executed prior)
-    def sorted_with_order
-      # Identify groups of nodes that can be executed concurrently
-      groups = tsort_each.slice_when { |a, b| parents(a) != parents(b) }
-
-      # Assign order to each node
-      i = 0
-      groups.flat_map do |group|
-        group_with_order = group.product([i])
-        i += group.size
-        group_with_order
+    def each
+      return enum_for(__method__) unless block_given?
+
+      tsort_each do |node|
+        yield(node, parents(node))
       end
     end
 

data/lib/pallets/logger.rb

@@ -5,18 +5,37 @@ module Pallets
   class Logger < ::Logger
     # Overwrite severity methods to add metadata capabilities
     %i[debug info warn error fatal unknown].each do |severity|
-      define_method severity do |message, metadata = {}|
-        return super(message) if metadata.empty?
+      define_method severity do |message|
+        metadata = Thread.current[:pallets_log_metadata]
+        return super(message) if metadata.nil?
 
-        formatted_metadata = ' ' + metadata.map { |k, v| "#{k}=#{v}" }.join(' ')
-        super(formatted_metadata) { message }
+        super(metadata) { message }
       end
     end
 
+    def with_metadata(hash)
+      Thread.current[:pallets_log_metadata] = hash
+      yield
+    ensure
+      Thread.current[:pallets_log_metadata] = nil
+    end
+
     module Formatters
       class Pretty < ::Logger::Formatter
         def call(severity, time, metadata, message)
-          "#{time.utc.iso8601(4)} pid=#{Process.pid}#{metadata} #{severity}: #{message}\n"
+          formatted_metadata = ' ' + metadata.map { |k, v| "#{k}=#{v}" }.join(' ') if metadata
+          "#{time.utc.iso8601(4)} pid=#{Process.pid}#{formatted_metadata} #{severity}: #{message}\n"
+        end
+      end
+
+      class Json < ::Logger::Formatter
+        def call(severity, time, metadata, message)
+          {
+            timestamp: time.utc.iso8601(4),
+            pid: Process.pid,
+            severity: severity,
+            message: message
+          }.merge(metadata || {}).to_json + "\n"
         end
       end
     end

data/lib/pallets/manager.rb

@@ -3,8 +3,9 @@ module Pallets
     attr_reader :workers, :scheduler
 
     def initialize(concurrency: Pallets.configuration.concurrency)
-      @workers = concurrency.times.map { Worker.new(self) }
-      @scheduler = Scheduler.new(self)
+      @backend = Pallets.backend
+      @workers = concurrency.times.map { Worker.new(self, @backend) }
+      @scheduler = Scheduler.new(self, @backend)
       @lock = Mutex.new
       @needs_to_stop = false
     end
@@ -48,7 +49,7 @@ module Pallets
 
         return if @needs_to_stop
 
-        worker = Worker.new(self)
+        worker = Worker.new(self, @backend)
         @workers << worker
         worker.start
       end

data/lib/pallets/middleware/job_logger.rb

@@ -2,14 +2,21 @@ module Pallets
   module Middleware
     class JobLogger
       def self.call(worker, job, context)
-        Pallets.logger.info 'Started', extract_metadata(worker.id, job)
-        result = yield
-        Pallets.logger.info 'Done', extract_metadata(worker.id, job)
-        result
-      rescue => ex
-        Pallets.logger.warn "#{ex.class.name}: #{ex.message}", extract_metadata(worker.id, job)
-        Pallets.logger.warn ex.backtrace.join("\n"), extract_metadata(worker.id, job) unless ex.backtrace.nil?
-        raise
+        start_time = current_time
+
+        Pallets.logger.with_metadata(extract_metadata(worker.id, job)) do
+          begin
+            Pallets.logger.info 'Started'
+            result = yield
+            Pallets.logger.info "Done in #{(current_time - start_time).round(3)}s"
+            result
+          rescue => ex
+            Pallets.logger.warn "Failed after #{(current_time - start_time).round(3)}s"
+            Pallets.logger.warn "#{ex.class.name}: #{ex.message}"
+            Pallets.logger.warn ex.backtrace.join("\n") unless ex.backtrace.nil?
+            raise
+          end
+        end
       end
 
       def self.extract_metadata(wid, job)
@@ -21,6 +28,10 @@ module Pallets
           tsk:  job['task_class'],
         }
       end
+
+      def self.current_time
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
     end
   end
 end

data/lib/pallets/scheduler.rb

@@ -1,13 +1,14 @@
 module Pallets
   class Scheduler
-    def initialize(manager)
+    def initialize(manager, backend)
       @manager = manager
+      @backend = backend
       @needs_to_stop = false
       @thread = nil
     end
 
     def start
-      @thread ||= Thread.new { work }
+      @thread ||= Thread.new { wait_initial_bit; work }
     end
 
     def shutdown
@@ -35,23 +36,25 @@ module Pallets
       loop do
         break if needs_to_stop?
 
-        backend.reschedule_all(Time.now.to_f)
+        @backend.reschedule_all(Time.now.to_f)
         wait_a_bit
       end
     end
 
-    def wait_a_bit
-      # Wait for roughly 10 seconds
+    def wait_initial_bit
+      # Randomly wait a bit before starting working, so that multiple processes
+      # will not hit the backend all at once
+      wait_a_bit(rand(Pallets.configuration.scheduler_polling_interval))
+    end
+
+    def wait_a_bit(seconds = Pallets.configuration.scheduler_polling_interval)
+      # Wait for roughly the configured number of seconds
       # We don't want to block the entire polling interval, since we want to
       # deal with shutdowns synchronously and as fast as possible
-      10.times do
+      seconds.times do
         break if needs_to_stop?
         sleep 1
       end
     end
-
-    def backend
-      @backend ||= Pallets.backend
-    end
   end
 end

data/lib/pallets/version.rb

@@ -1,3 +1,3 @@
 module Pallets
-  VERSION = "0.6.0"
+  VERSION = "0.11.0"
 end

data/lib/pallets/worker.rb

@@ -1,9 +1,8 @@
 module Pallets
   class Worker
-    attr_reader :manager
-
-    def initialize(manager)
+    def initialize(manager, backend)
       @manager = manager
+      @backend = backend
       @current_job = nil
       @needs_to_stop = false
       @thread = nil
@@ -40,7 +39,7 @@ module Pallets
       loop do
         break if needs_to_stop?
 
-        @current_job = backend.pick
+        @current_job = @backend.pick
         # No need to requeue because of the reliability queue
         break if needs_to_stop?
         next if @current_job.nil?
@@ -53,8 +52,10 @@ module Pallets
     rescue Pallets::Shutdown
       @manager.remove_worker(self)
     rescue => ex
-      Pallets.logger.error "#{ex.class.name}: #{ex.message}", wid: id
-      Pallets.logger.error ex.backtrace.join("\n"), wid: id unless ex.backtrace.nil?
+      Pallets.logger.error "#{ex.class.name}: #{ex.message}"
+      Pallets.logger.error ex.backtrace.join("\n") unless ex.backtrace.nil?
+      # Do not flood the process in case of persisting unforeseen errors
+      sleep 1
       @manager.replace_worker(self)
     end
 
@@ -64,13 +65,13 @@ module Pallets
       rescue
         # We ensure only valid jobs are created. If something fishy reaches this
         # point, just give up on it
-        backend.give_up(job, job)
-        Pallets.logger.error "Could not deserialize #{job}. Gave up job", wid: id
+        @backend.discard(job)
+        Pallets.logger.error "Could not deserialize #{job}. Gave up job"
         return
       end
 
       context = Context[
-        serializer.load_context(backend.get_context(job_hash['wfid']))
+        serializer.load_context(@backend.get_context(job_hash['wfid']))
       ]
 
       task_class = Pallets::Util.constantize(job_hash["task_class"])
@@ -101,9 +102,9 @@ module Pallets
       ))
       if failures < job_hash['max_failures']
         retry_at = Time.now.to_f + backoff_in_seconds(failures)
-        backend.retry(new_job, job, retry_at)
+        @backend.retry(new_job, job, retry_at)
       else
-        backend.give_up(new_job, job)
+        @backend.give_up(job_hash['wfid'], new_job, job)
       end
     end
 
@@ -112,21 +113,17 @@ module Pallets
         'given_up_at' => Time.now.to_f,
         'reason' => 'returned_false'
       ))
-      backend.give_up(new_job, job)
+      @backend.give_up(job_hash['wfid'], new_job, job)
     end
 
     def handle_job_success(context, job, job_hash)
-      backend.save(job_hash['wfid'], job, serializer.dump_context(context.buffer))
+      @backend.save(job_hash['wfid'], job_hash['jid'], job, serializer.dump_context(context.buffer))
     end
 
     def backoff_in_seconds(count)
       count ** 4 + rand(6..10)
     end
 
-    def backend
-      @backend ||= Pallets.backend
-    end
-
     def serializer
       @serializer ||= Pallets.serializer
     end

data/lib/pallets/workflow.rb

@@ -20,7 +20,7 @@ module Pallets
       raise WorkflowError, "#{self.class.name} has no tasks. Workflows "\
                            "must contain at least one task" if self.class.graph.empty?
 
-      backend.run_workflow(id, jobs_with_order, serializer.dump_context(context.buffer))
+      backend.run_workflow(id, *prepare_jobs, serializer.dump_context(context.buffer))
       id
     end
 
@@ -30,11 +30,21 @@ module Pallets
 
     private
 
-    def jobs_with_order
-      self.class.graph.sorted_with_order.map do |task_alias, order|
-        job = serializer.dump(construct_job(task_alias))
-        [order, job]
+    def prepare_jobs
+      jobs = []
+      jobmasks = Hash.new { |h, k| h[k] = [] }
+      acc = {}
+
+      self.class.graph.each do |task_alias, dependencies|
+        job_hash = construct_job(task_alias)
+        acc[task_alias] = job_hash['jid']
+        job = serializer.dump(job_hash)
+
+        jobs << [dependencies.size, job]
+        dependencies.each { |d| jobmasks[acc[d]] << [-1, job] }
       end
+
+      [jobs, jobmasks]
     end
 
     def construct_job(task_alias)

data/pallets.gemspec

@@ -9,8 +9,8 @@ Gem::Specification.new do |spec|
   spec.authors       = ['Andrei Horak']
   spec.email         = ['linkyndy@gmail.com']
 
-  spec.summary       = 'Toy workflow engine, written in Ruby'
-  spec.description   = 'Toy workflow engine, written in Ruby'
+  spec.summary       = 'Simple and reliable workflow engine, written in Ruby'
+  spec.description   = 'Simple and reliable workflow engine, written in Ruby'
   spec.homepage      = 'https://github.com/linkyndy/pallets'
   spec.license       = 'MIT'
 
@@ -20,6 +20,6 @@ Gem::Specification.new do |spec|
 
   spec.required_ruby_version = '>= 2.4'
 
-  spec.add_dependency 'redis'
+  spec.add_dependency 'redis', '~> 4.2'
   spec.add_dependency 'msgpack'
 end

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: pallets
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.11.0
 platform: ruby
 authors:
 - Andrei Horak
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-09-02 00:00:00.000000000 Z
+date: 2020-10-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '4.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '4.2'
 - !ruby/object:Gem::Dependency
   name: msgpack
   requirement: !ruby/object:Gem::Requirement
@@ -38,7 +38,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: Toy workflow engine, written in Ruby
+description: Simple and reliable workflow engine, written in Ruby
 email:
 - linkyndy@gmail.com
 executables:
@@ -46,6 +46,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/FUNDING.yml"
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
@@ -66,6 +67,7 @@ files:
 - lib/pallets.rb
 - lib/pallets/backends/base.rb
 - lib/pallets/backends/redis.rb
+- lib/pallets/backends/scripts/discard.lua
 - lib/pallets/backends/scripts/give_up.lua
 - lib/pallets/backends/scripts/reschedule_all.lua
 - lib/pallets/backends/scripts/retry.lua
@@ -97,7 +99,7 @@ homepage: https://github.com/linkyndy/pallets
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -112,8 +114,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
-summary: Toy workflow engine, written in Ruby
+summary: Simple and reliable workflow engine, written in Ruby
 test_files: []