pallets 0.5.0 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e275565357790942bf8c3d92e452921a5650122cf12679a399bafe67818ea75c
-  data.tar.gz: 3f8c9c584b71385fd679c93cdf208b924e45efea43562062ded98b7a83e05d86
+  metadata.gz: ede234db1dfe8c744cf11e6e2fde3eb2e982e7764738715acbbad35710422274
+  data.tar.gz: a55b3a714dcc5db2a294b27ca53724a1236111e08f63078bcef5007bda65532c
 SHA512:
-  metadata.gz: 0e197b5b10ba2e569b8e7df245a607046831af3412b5417dc3084e986af27088240c61de963a9e60e6a1431921014c92a09b9f4dd0aec9003c6dd3ffe0ce84f4
-  data.tar.gz: cdfa840985dfcea75efea45f7dc5bc42e0aed4933cd8e02c996e46f4d8a58300c1feb47fe5fd5a6f46930b50e5b782525ed9c41434cb4cf21afcbe4ad8158c4e
+  metadata.gz: 74e23087ba192aeffeb7e167d751bfb6f85de4018fb3c16e992ac6119d7670d60862f41a8e1624817a25944409602e41ee38f13adad7b14a38b185d51142b644
+  data.tar.gz: 96ae4f2aa4edf5d27d366e7a35f16527e9d390a45794d306acc109b1be335682c8b1e13e7d708d7eb3451eb8b6ed96db37ba724da08abb13d7b84f3468869826

data/.github/FUNDING.yml

@@ -0,0 +1 @@
+github: linkyndy

data/.travis.yml

@@ -4,11 +4,8 @@ services:
   - redis-server
 cache: bundler
 rvm:
-  - 2.4.6
-  - 2.5.5
-  - 2.6.3
-before_install:
-  # Bundler 2.0 needs a newer RubyGems
-  - gem update --system
-  - gem install bundler
+  - 2.4.10
+  - 2.5.8
+  - 2.6.6
+  - 2.7.1
 script: bundle exec rspec

data/CHANGELOG.md

@@ -6,6 +6,39 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.9.0] - 2020-07-05
+### Added
+- limit number of jobs in given up set by number (#56)
+- job duration and metadata to all task logs (#57)
+
+### Changed
+- remove all related workflow keys when giving up on a job (#55)
+- support redis-rb ~> 4.2 (#58)
+
+### Removed
+- support for configuring custom loggers (#57)
+
+## [0.8.0] - 2020-06-09
+### Added
+- sync output in CLI (#49)
+- support for configuring custom loggers (#50)
+
+### Changed
+- improve job scheduling using jobmasks (#52)
+
+## [0.7.0] - 2020-01-19
+### Added
+- support for Ruby 2.7 (#46)
+
+## [0.6.0] - 2019-09-02
+### Added
+- define task aliases in order to reuse tasks within a workflow definition (#44)
+- define anonymous workflows (#45)
+
+## [0.5.1] - 2019-06-01
+### Changed
+- fix transaction completeness in Appsignal instrumenter (#43)
+
 ## [0.5.0] - 2019-05-12
 ### Added
 - wrap job execution with middleware (#38)
@@ -56,7 +89,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## 0.1.0 - 2018-09-29
 - Pallets' inception <3
 
-[Unreleased]: https://github.com/linkyndy/pallets/compare/compare/v0.5.0...HEAD
+[Unreleased]: https://github.com/linkyndy/pallets/compare/compare/v0.5.1...HEAD
+[0.5.1]: https://github.com/linkyndy/pallets/compare/v0.5.0...v0.5.1
 [0.5.0]: https://github.com/linkyndy/pallets/compare/v0.4.0...v0.5.0
 [0.4.0]: https://github.com/linkyndy/pallets/compare/v0.3.0...v0.5.0
 [0.3.0]: https://github.com/linkyndy/pallets/compare/v0.2.0...v0.3.0

data/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2019 Andrei Horak
+Copyright (c) 2020 Andrei Horak
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -2,7 +2,7 @@
 
 [![Build Status](https://travis-ci.com/linkyndy/pallets.svg?branch=master)](https://travis-ci.com/linkyndy/pallets)
 
-Toy workflow engine, written in Ruby
+Simple and reliable workflow engine, written in Ruby
 
 ## It is plain simple!
 
@@ -11,10 +11,10 @@ Toy workflow engine, written in Ruby
 require 'pallets'
 
 class MyWorkflow < Pallets::Workflow
-  task Foo
-  task Bar => Foo
-  task Baz => Foo
-  task Qux => [Bar, Baz]
+  task 'Foo'
+  task 'Bar' => 'Foo'
+  task 'Baz' => 'Foo'
+  task 'Qux' => ['Bar', 'Baz']
 end
 
 class Foo < Pallets::Task
@@ -120,7 +120,7 @@ end
 
 ## Motivation
 
-The main reason for Pallet's existence was the need of a fast, simple and reliable workflow engine, one that is easily extensible with various backends and serializer, one that does not lose your data and one that is intelligent enough to concurrently schedule a workflow's tasks.
+The main reason for Pallets' existence was the need of a fast, simple and reliable workflow engine, one that is easily extensible with various backends and serializer, one that does not lose your data and one that is intelligent enough to concurrently schedule a workflow's tasks.
 
 ## Status
 

data/examples/aliases.rb

@@ -0,0 +1,28 @@
+require 'pallets'
+
+class Aliases < Pallets::Workflow
+  task 'StartSmtpServer'
+  task 'SendEmail', as: 'SayHello', depends_on: 'StartSmtpServer'
+  task 'SendEmail', as: 'SayGoodbye', depends_on: 'StartSmtpServer'
+  task 'StopSmtpServer' => ['SayHello', 'SayGoodbye']
+end
+
+class StartSmtpServer < Pallets::Task
+  def run
+    puts "Starting SMTP server..."
+  end
+end
+
+class SendEmail < Pallets::Task
+  def run
+    puts "* sending e-mail"
+  end
+end
+
+class StopSmtpServer < Pallets::Task
+  def run
+    puts "Stopped SMTP server"
+  end
+end
+
+Aliases.new.run

data/examples/anonymous.rb

@@ -0,0 +1,13 @@
+require 'pallets'
+
+class Anonymous < Pallets::Task
+  def run
+    puts 'This is anonymous!'
+  end
+end
+
+workflow = Pallets::Workflow.build do
+  task 'Anonymous'
+end
+
+workflow.new.run

data/lib/pallets.rb

@@ -38,6 +38,7 @@ module Pallets
       cls.new(
         blocking_timeout: configuration.blocking_timeout,
         failed_job_lifespan: configuration.failed_job_lifespan,
+        failed_job_max_count: configuration.failed_job_max_count,
         job_timeout: configuration.job_timeout,
         pool_size: configuration.pool_size,
         **configuration.backend_args
@@ -62,8 +63,4 @@ module Pallets
       formatter: Pallets::Logger::Formatters::Pretty.new
     )
   end
-
-  def self.logger=(logger)
-    @logger = logger
-  end
 end

data/lib/pallets/backends/base.rb

@@ -6,12 +6,12 @@ module Pallets
         raise NotImplementedError
       end
 
-      def get_context(workflow_id)
+      def get_context(wfid)
         raise NotImplementedError
       end
 
       # Saves a job after successfully processing it
-      def save(workflow_id, job, context_buffer)
+      def save(wfid, jid, job, context_buffer)
         raise NotImplementedError
       end
 
@@ -20,8 +20,13 @@ module Pallets
         raise NotImplementedError
       end
 
+      # Discards malformed job
+      def discard(job)
+        raise NotImplementedError
+      end
+
       # Gives up job after repeteadly failing to process it
-      def give_up(job, old_job)
+      def give_up(wfid, job, old_job)
         raise NotImplementedError
       end
 
@@ -29,7 +34,7 @@ module Pallets
         raise NotImplementedError
       end
 
-      def run_workflow(workflow_id, jobs_with_dependencies, context)
+      def run_workflow(wfid, jobs, jobmasks, context)
         raise NotImplementedError
       end
     end

data/lib/pallets/backends/redis.rb

@@ -9,12 +9,15 @@ module Pallets
       RETRY_SET_KEY = 'retry-set'
       GIVEN_UP_SET_KEY = 'given-up-set'
       WORKFLOW_QUEUE_KEY = 'workflow-queue:%s'
+      JOBMASKS_KEY = 'jobmasks:%s'
+      JOBMASK_KEY = 'jobmask:%s'
       CONTEXT_KEY = 'context:%s'
       REMAINING_KEY = 'remaining:%s'
 
-      def initialize(blocking_timeout:, failed_job_lifespan:, job_timeout:, pool_size:, **options)
+      def initialize(blocking_timeout:, failed_job_lifespan:, failed_job_max_count:, job_timeout:, pool_size:, **options)
         @blocking_timeout = blocking_timeout
         @failed_job_lifespan = failed_job_lifespan
+        @failed_job_max_count = failed_job_max_count
         @job_timeout = job_timeout
         @pool = Pallets::Pool.new(pool_size) { ::Redis.new(options) }
 
@@ -41,11 +44,11 @@ module Pallets
         end
       end
 
-      def save(wfid, job, context_buffer)
+      def save(wfid, jid, job, context_buffer)
         @pool.execute do |client|
           client.evalsha(
             @scripts['save'],
-            [WORKFLOW_QUEUE_KEY % wfid, QUEUE_KEY, RELIABILITY_QUEUE_KEY, RELIABILITY_SET_KEY, CONTEXT_KEY % wfid, REMAINING_KEY % wfid],
+            [WORKFLOW_QUEUE_KEY % wfid, QUEUE_KEY, RELIABILITY_QUEUE_KEY, RELIABILITY_SET_KEY, CONTEXT_KEY % wfid, REMAINING_KEY % wfid, JOBMASK_KEY % jid, JOBMASKS_KEY % wfid],
             context_buffer.to_a << job
           )
         end
@@ -61,12 +64,22 @@ module Pallets
         end
       end
 
-      def give_up(job, old_job)
+      def discard(job)
         @pool.execute do |client|
           client.evalsha(
-            @scripts['give_up'],
+            @scripts['discard'],
             [GIVEN_UP_SET_KEY, RELIABILITY_QUEUE_KEY, RELIABILITY_SET_KEY],
-            [Time.now.to_f, job, old_job, Time.now.to_f - @failed_job_lifespan]
+            [Time.now.to_f, job, Time.now.to_f - @failed_job_lifespan, @failed_job_max_count]
+          )
+        end
+      end
+
+      def give_up(wfid, job, old_job)
+        @pool.execute do |client|
+          client.evalsha(
+            @scripts['give_up'],
+            [GIVEN_UP_SET_KEY, RELIABILITY_QUEUE_KEY, RELIABILITY_SET_KEY, JOBMASKS_KEY % wfid, WORKFLOW_QUEUE_KEY % wfid, REMAINING_KEY % wfid, CONTEXT_KEY % wfid],
+            [Time.now.to_f, job, old_job, Time.now.to_f - @failed_job_lifespan, @failed_job_max_count]
           )
         end
       end
@@ -81,13 +94,15 @@ module Pallets
         end
       end
 
-      def run_workflow(wfid, jobs_with_order, context_buffer)
+      def run_workflow(wfid, jobs, jobmasks, context_buffer)
         @pool.execute do |client|
           client.multi do
+            jobmasks.each { |jid, jobmask| client.zadd(JOBMASK_KEY % jid, jobmask) }
+            client.sadd(JOBMASKS_KEY % wfid, jobmasks.map { |jid, _| JOBMASK_KEY % jid }) unless jobmasks.empty?
             client.evalsha(
               @scripts['run_workflow'],
               [WORKFLOW_QUEUE_KEY % wfid, QUEUE_KEY, REMAINING_KEY % wfid],
-              jobs_with_order
+              jobs
             )
             client.hmset(CONTEXT_KEY % wfid, *context_buffer.to_a) unless context_buffer.empty?
           end

data/lib/pallets/backends/scripts/discard.lua

@@ -0,0 +1,11 @@
+-- Remove job from reliability queue
+redis.call("LREM", KEYS[2], 0, ARGV[2])
+redis.call("ZREM", KEYS[3], ARGV[2])
+
+-- Add job and its fail time (score) to failed sorted set
+redis.call("ZADD", KEYS[1], ARGV[1], ARGV[2])
+
+-- Remove any jobs that have been given up long enough ago (their score is
+-- below given value) and make sure the number of jobs is capped
+redis.call("ZREMRANGEBYSCORE", KEYS[1], "-inf", ARGV[3])
+redis.call("ZREMRANGEBYRANK", KEYS[1], 0, -ARGV[4] - 1)

data/lib/pallets/backends/scripts/give_up.lua

@@ -5,6 +5,11 @@ redis.call("ZREM", KEYS[3], ARGV[3])
 -- Add job and its fail time (score) to failed sorted set
 redis.call("ZADD", KEYS[1], ARGV[1], ARGV[2])
 
+-- Remove all related workflow keys
+local keys = redis.call("SMEMBERS", KEYS[4])
+redis.call("DEL", KEYS[4], KEYS[5], KEYS[6], KEYS[7], unpack(keys))
+
 -- Remove any jobs that have been given up long enough ago (their score is
--- below given value)
+-- below given value) and make sure the number of jobs is capped
 redis.call("ZREMRANGEBYSCORE", KEYS[1], "-inf", ARGV[4])
+redis.call("ZREMRANGEBYRANK", KEYS[1], 0, -ARGV[5] - 1)

data/lib/pallets/backends/scripts/run_workflow.lua

@@ -6,9 +6,8 @@ redis.call("SET", KEYS[3], eta)
 
 -- Queue jobs that are ready to be processed (their score is 0) and
 -- remove queued jobs from the sorted set
-local count = redis.call("ZCOUNT", KEYS[1], 0, 0)
-if count > 0 then
-  local work = redis.call("ZRANGEBYSCORE", KEYS[1], 0, 0)
+local work = redis.call("ZRANGEBYSCORE", KEYS[1], 0, 0)
+if #work > 0 then
   redis.call("LPUSH", KEYS[2], unpack(work))
   redis.call("ZREM", KEYS[1], unpack(work))
 end

data/lib/pallets/backends/scripts/save.lua

@@ -10,24 +10,21 @@ if #ARGV > 0 then
   redis.call("HMSET", KEYS[5], unpack(ARGV))
 end
 
--- Decrement all jobs from the sorted set
-local all_pending = redis.call("ZRANGE", KEYS[1], 0, -1)
-for score, task in pairs(all_pending) do
-  redis.call("ZINCRBY", KEYS[1], -1, task)
-end
+-- Decrement jobs from the sorted set by applying a jobmask
+redis.call("ZUNIONSTORE", KEYS[1], 2, KEYS[1], KEYS[7])
+redis.call("DEL", KEYS[7])
 
 -- Queue jobs that are ready to be processed (their score is 0) and
 -- remove queued jobs from sorted set
-local count = redis.call("ZCOUNT", KEYS[1], 0, 0)
-if count > 0 then
-  local work = redis.call("ZRANGEBYSCORE", KEYS[1], 0, 0)
+local work = redis.call("ZRANGEBYSCORE", KEYS[1], 0, 0)
+if #work > 0 then
   redis.call("LPUSH", KEYS[2], unpack(work))
   redis.call("ZREM", KEYS[1], unpack(work))
 end
 
--- Decrement ETA and remove it together with the context if all tasks have
--- been processed (ETA is 0)
-redis.call("DECR", KEYS[6])
-if tonumber(redis.call("GET", KEYS[6])) == 0 then
-  redis.call("DEL", KEYS[5], KEYS[6])
+-- Decrement ETA and remove it together with the context and jobmasks if all
+-- tasks have been processed (ETA is 0) or if workflow has been given up (ETA is -1)
+local remaining = redis.call("DECR", KEYS[6])
+if remaining <= 0 then
+  redis.call("DEL", KEYS[5], KEYS[6], KEYS[8])
 end

data/lib/pallets/cli.rb

@@ -1,5 +1,7 @@
 require 'optparse'
 
+$stdout.sync = true
+
 module Pallets
   class CLI
     def initialize
@@ -65,6 +67,10 @@ module Pallets
           Pallets.configuration.failed_job_lifespan = failed_job_lifespan
         end
 
+        opts.on('-m', '--failed-job-max-count NUM', Integer, 'Maximum number of jobs in the given up set') do |failed_job_max_count|
+          Pallets.configuration.failed_job_max_count = failed_job_max_count
+        end
+
         opts.on('-p', '--pool-size NUM', Integer, 'Size of backend pool') do |pool_size|
           Pallets.configuration.pool_size = pool_size
         end

data/lib/pallets/configuration.rb

@@ -16,6 +16,10 @@ module Pallets
     # this period, jobs will be permanently deleted
     attr_accessor :failed_job_lifespan
 
+    # Maximum number of failed jobs that can be in the given up set. When this
+    # number is reached, the oldest jobs will be permanently deleted
+    attr_accessor :failed_job_max_count
+
     # Number of seconds allowed for a job to be processed. If a job exceeds this
     # period, it is considered failed, and scheduled to be processed again
     attr_accessor :job_timeout
@@ -45,6 +49,7 @@ module Pallets
       @blocking_timeout = 5
       @concurrency = 2
       @failed_job_lifespan = 7_776_000 # 3 months
+      @failed_job_max_count = 1_000
       @job_timeout = 1_800 # 30 minutes
       @max_failures = 3
       @serializer = :json

data/lib/pallets/dsl/workflow.rb

@@ -1,7 +1,13 @@
 module Pallets
   module DSL
     module Workflow
-      def task(arg, depends_on: nil, max_failures: nil, &block)
+      def task(arg=nil, as: nil, depends_on: nil, max_failures: nil, **kwargs)
+        # Have to work more to keep Pallets' nice DSL valid in Ruby 2.7
+        arg = !kwargs.empty? ? kwargs : arg
+        raise ArgumentError, 'Task is incorrectly defined. It must receive '\
+                             'either a name, or a name => dependencies pair as '\
+                             'the first argument' unless arg
+
         klass, dependencies = case arg
         when Hash
           # The `task Foo => Bar` notation
@@ -12,10 +18,13 @@ module Pallets
         end
 
         task_class = klass.to_s
+        as ||= task_class
+
         dependencies = Array(dependencies).compact.uniq.map(&:to_s)
-        graph.add(task_class, dependencies)
+        graph.add(as, dependencies)
 
-        task_config[task_class] = {
+        task_config[as] = {
+          'workflow_class' => self.name,
           'task_class' => task_class,
           'max_failures' => max_failures || Pallets.configuration.max_failures
         }

data/lib/pallets/graph.rb

@@ -9,40 +9,39 @@ module Pallets
     end
 
     def add(node, dependencies)
-      @nodes[node] = dependencies
+      raise WorkflowError, "Task #{node} is already defined in this workflow. "\
+                           "Use `task '#{node}', as: 'FooBar'` to define an "\
+                           "alias and reuse task" if nodes.key?(node)
+
+      nodes[node] = dependencies
     end
 
     def parents(node)
-      @nodes[node]
+      nodes[node]
     end
 
     def empty?
-      @nodes.empty?
+      nodes.empty?
     end
 
-    # Returns nodes topologically sorted, together with their order (number of
-    # nodes that have to be executed prior)
-    def sorted_with_order
-      # Identify groups of nodes that can be executed concurrently
-      groups = tsort_each.slice_when { |a, b| parents(a) != parents(b) }
-
-      # Assign order to each node
-      i = 0
-      groups.flat_map do |group|
-        group_with_order = group.product([i])
-        i += group.size
-        group_with_order
+    def each
+      return enum_for(__method__) unless block_given?
+
+      tsort_each do |node|
+        yield(node, parents(node))
       end
     end
 
     private
 
+    attr_reader :nodes
+
     def tsort_each_node(&block)
-      @nodes.each_key(&block)
+      nodes.each_key(&block)
     end
 
     def tsort_each_child(node, &block)
-      @nodes.fetch(node).each(&block)
+      nodes.fetch(node).each(&block)
     rescue KeyError
       raise WorkflowError, "Task #{node} is marked as a dependency but not defined"
     end

data/lib/pallets/logger.rb

@@ -5,14 +5,22 @@ module Pallets
   class Logger < ::Logger
     # Overwrite severity methods to add metadata capabilities
     %i[debug info warn error fatal unknown].each do |severity|
-      define_method severity do |message, metadata = {}|
-        return super(message) if metadata.empty?
+      define_method severity do |message|
+        metadata = Thread.current[:pallets_log_metadata]
+        return super(message) if metadata.nil?
 
         formatted_metadata = ' ' + metadata.map { |k, v| "#{k}=#{v}" }.join(' ')
         super(formatted_metadata) { message }
       end
     end
 
+    def with_metadata(hash)
+      Thread.current[:pallets_log_metadata] = hash
+      yield
+    ensure
+      Thread.current[:pallets_log_metadata] = nil
+    end
+
     module Formatters
       class Pretty < ::Logger::Formatter
         def call(severity, time, metadata, message)

data/lib/pallets/middleware/appsignal_instrumenter.rb

@@ -25,10 +25,11 @@ module Pallets
             transaction.params = filtered_context(context)
             formatted_metadata(job).each { |kv| transaction.set_metadata(*kv) }
             transaction.set_http_or_background_queue_start
-            Appsignal::Transaction.complete_current!
             Appsignal.increment_counter('pallets_job_count', 1, status: job_status || :successful)
           end
         end
+      ensure
+        Appsignal::Transaction.complete_current!
       end
 
       def self.filtered_context(context)

data/lib/pallets/middleware/job_logger.rb

@@ -2,14 +2,21 @@ module Pallets
   module Middleware
     class JobLogger
       def self.call(worker, job, context)
-        Pallets.logger.info 'Started', extract_metadata(worker.id, job)
-        result = yield
-        Pallets.logger.info 'Done', extract_metadata(worker.id, job)
-        result
-      rescue => ex
-        Pallets.logger.warn "#{ex.class.name}: #{ex.message}", extract_metadata(worker.id, job)
-        Pallets.logger.warn ex.backtrace.join("\n"), extract_metadata(worker.id, job) unless ex.backtrace.nil?
-        raise
+        start_time = current_time
+
+        Pallets.logger.with_metadata(extract_metadata(worker.id, job)) do
+          begin
+            Pallets.logger.info 'Started'
+            result = yield
+            Pallets.logger.info "Done in #{(current_time - start_time).round(3)}s"
+            result
+          rescue => ex
+            Pallets.logger.warn "Failed after #{(current_time - start_time).round(3)}s"
+            Pallets.logger.warn "#{ex.class.name}: #{ex.message}"
+            Pallets.logger.warn ex.backtrace.join("\n") unless ex.backtrace.nil?
+            raise
+          end
+        end
       end
 
       def self.extract_metadata(wid, job)
@@ -21,6 +28,10 @@ module Pallets
           tsk:  job['task_class'],
         }
       end
+
+      def self.current_time
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
     end
   end
 end

data/lib/pallets/version.rb

@@ -1,3 +1,3 @@
 module Pallets
-  VERSION = "0.5.0"
+  VERSION = "0.9.0"
 end

data/lib/pallets/worker.rb

@@ -53,8 +53,8 @@ module Pallets
     rescue Pallets::Shutdown
       @manager.remove_worker(self)
     rescue => ex
-      Pallets.logger.error "#{ex.class.name}: #{ex.message}", wid: id
-      Pallets.logger.error ex.backtrace.join("\n"), wid: id unless ex.backtrace.nil?
+      Pallets.logger.error "#{ex.class.name}: #{ex.message}"
+      Pallets.logger.error ex.backtrace.join("\n") unless ex.backtrace.nil?
       @manager.replace_worker(self)
     end
 
@@ -64,8 +64,8 @@ module Pallets
       rescue
         # We ensure only valid jobs are created. If something fishy reaches this
         # point, just give up on it
-        backend.give_up(job, job)
-        Pallets.logger.error "Could not deserialize #{job}. Gave up job", wid: id
+        backend.discard(job)
+        Pallets.logger.error "Could not deserialize #{job}. Gave up job"
         return
       end
 
@@ -103,7 +103,7 @@ module Pallets
         retry_at = Time.now.to_f + backoff_in_seconds(failures)
         backend.retry(new_job, job, retry_at)
       else
-        backend.give_up(new_job, job)
+        backend.give_up(job_hash['wfid'], new_job, job)
       end
     end
 
@@ -112,11 +112,11 @@ module Pallets
         'given_up_at' => Time.now.to_f,
         'reason' => 'returned_false'
       ))
-      backend.give_up(new_job, job)
+      backend.give_up(job_hash['wfid'], new_job, job)
     end
 
     def handle_job_success(context, job, job_hash)
-      backend.save(job_hash['wfid'], job, serializer.dump_context(context.buffer))
+      backend.save(job_hash['wfid'], job_hash['jid'], job, serializer.dump_context(context.buffer))
     end
 
     def backoff_in_seconds(count)

data/lib/pallets/workflow.rb

@@ -4,6 +4,12 @@ module Pallets
 
     attr_reader :context
 
+    def self.build(&block)
+      Class.new(self).tap do |workflow_class|
+        workflow_class.instance_eval(&block)
+      end
+    end
+
     def initialize(context_hash = {})
       @id = nil
       # Passed in context hash needs to be buffered
@@ -14,7 +20,7 @@ module Pallets
       raise WorkflowError, "#{self.class.name} has no tasks. Workflows "\
                            "must contain at least one task" if self.class.graph.empty?
 
-      backend.run_workflow(id, jobs_with_order, serializer.dump_context(context.buffer))
+      backend.run_workflow(id, *prepare_jobs, serializer.dump_context(context.buffer))
       id
     end
 
@@ -24,20 +30,29 @@ module Pallets
 
     private
 
-    def jobs_with_order
-      self.class.graph.sorted_with_order.map do |task_class, order|
-        job = serializer.dump(construct_job(task_class))
-        [order, job]
+    def prepare_jobs
+      jobs = []
+      jobmasks = Hash.new { |h, k| h[k] = [] }
+      acc = {}
+
+      self.class.graph.each do |task_alias, dependencies|
+        job_hash = construct_job(task_alias)
+        acc[task_alias] = job_hash['jid']
+        job = serializer.dump(job_hash)
+
+        jobs << [dependencies.size, job]
+        dependencies.each { |d| jobmasks[acc[d]] << [-1, job] }
       end
+
+      [jobs, jobmasks]
     end
 
-    def construct_job(task_class)
-      {}.tap do |job|
+    def construct_job(task_alias)
+      Hash[self.class.task_config[task_alias]].tap do |job|
         job['wfid'] = id
-        job['jid'] = "J#{Pallets::Util.generate_id(task_class)}".upcase
-        job['workflow_class'] = self.class.name
+        job['jid'] = "J#{Pallets::Util.generate_id(job['task_class'])}".upcase
         job['created_at'] = Time.now.to_f
-      end.merge(self.class.task_config[task_class])
+      end
     end
 
     def backend
@@ -48,6 +63,10 @@ module Pallets
       Pallets.serializer
     end
 
+    def self.name
+      @name ||= super || '<Anonymous>'
+    end
+
     def self.task_config
       @task_config ||= {}
     end

data/pallets.gemspec

@@ -9,8 +9,8 @@ Gem::Specification.new do |spec|
   spec.authors       = ['Andrei Horak']
   spec.email         = ['linkyndy@gmail.com']
 
-  spec.summary       = 'Toy workflow engine, written in Ruby'
-  spec.description   = 'Toy workflow engine, written in Ruby'
+  spec.summary       = 'Simple and reliable workflow engine, written in Ruby'
+  spec.description   = 'Simple and reliable workflow engine, written in Ruby'
   spec.homepage      = 'https://github.com/linkyndy/pallets'
   spec.license       = 'MIT'
 
@@ -20,6 +20,6 @@ Gem::Specification.new do |spec|
 
   spec.required_ruby_version = '>= 2.4'
 
-  spec.add_dependency 'redis'
+  spec.add_dependency 'redis', '~> 4.2'
   spec.add_dependency 'msgpack'
 end

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: pallets
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.9.0
 platform: ruby
 authors:
 - Andrei Horak
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-05-12 00:00:00.000000000 Z
+date: 2020-07-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '4.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '4.2'
 - !ruby/object:Gem::Dependency
   name: msgpack
   requirement: !ruby/object:Gem::Requirement
@@ -38,7 +38,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: Toy workflow engine, written in Ruby
+description: Simple and reliable workflow engine, written in Ruby
 email:
 - linkyndy@gmail.com
 executables:
@@ -46,6 +46,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/FUNDING.yml"
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
@@ -56,6 +57,8 @@ files:
 - README.md
 - Rakefile
 - bin/pallets
+- examples/aliases.rb
+- examples/anonymous.rb
 - examples/appsignal.rb
 - examples/config/appsignal.yml
 - examples/config_savvy.rb
@@ -64,6 +67,7 @@ files:
 - lib/pallets.rb
 - lib/pallets/backends/base.rb
 - lib/pallets/backends/redis.rb
+- lib/pallets/backends/scripts/discard.lua
 - lib/pallets/backends/scripts/give_up.lua
 - lib/pallets/backends/scripts/reschedule_all.lua
 - lib/pallets/backends/scripts/retry.lua
@@ -95,7 +99,7 @@ homepage: https://github.com/linkyndy/pallets
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -110,8 +114,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
-summary: Toy workflow engine, written in Ruby
+summary: Simple and reliable workflow engine, written in Ruby
 test_files: []