pallets 0.4.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 6bd6ebdb788c39b7d3cb434b60f94d927dacd47c
-  data.tar.gz: b463178c06885308a2192bbcf3d549c52391acbd
+SHA256:
+  metadata.gz: 46a631b6e48f2f1c4efe5bce09c5d74fa2cb064a78d5e546492c6c8ce1dec843
+  data.tar.gz: 61f6414e16941de41defb30ac8dc9121703f75251e47422239c311a8b7749787
 SHA512:
-  metadata.gz: 7e976084d28bba01327e0a8c41b8a948e27f6b3041d6a3a31665825d66b57225867ec410e9bf4f24afcf13241f41b855e1150601f37de37425fc72ee9582b5a4
-  data.tar.gz: c05e17653dac7716b9f52958b0bfb295517d614f5f97a00ffb91ad7100e83fbbf20f526f08f8315a1344829d01bd992c4920a7b0881aec4fea9b68eb0d259ff3
+  metadata.gz: 4b6f77bd93576dc7dc2c5cf945d52c425a63178740c12fa958f0c18c4ca870da7b02d303c7b8e871504d84a13461f79edc52da2869069322f5a81943bdbd79df
+  data.tar.gz: a2a71f4a2343927a94871523b7475f587bb64181e9c6a4654a40111453e2fc2f20e82a02da44adbc32865f68c53a95dd3a9c6f0b641cfd85c7939f1e06d3521f

data/.github/FUNDING.yml

@@ -0,0 +1 @@
+github: linkyndy

data/.travis.yml

@@ -4,12 +4,8 @@ services:
   - redis-server
 cache: bundler
 rvm:
-  - 2.3.8
-  - 2.4.5
-  - 2.5.3
-  - 2.6.0
-before_install:
-  # Bundler 2.0 needs a newer RubyGems
-  - gem update --system
-  - gem install bundler
+  - 2.4.10
+  - 2.5.8
+  - 2.6.6
+  - 2.7.1
 script: bundle exec rspec

data/CHANGELOG.md

@@ -6,6 +6,36 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.8.0] - 2020-06-09
+### Added
+- sync output in CLI (#49)
+- support for configuring custom loggers (#50)
+
+### Changed
+- improve job scheduling using jobmasks (#52)
+
+## [0.7.0] - 2020-01-19
+### Added
+- support for Ruby 2.7 (#46)
+
+## [0.6.0] - 2019-09-02
+### Added
+- define task aliases in order to reuse tasks within a workflow definition (#44)
+- define anonymous workflows (#45)
+
+## [0.5.1] - 2019-06-01
+### Changed
+- fix transaction completeness in Appsignal instrumenter (#43)
+
+## [0.5.0] - 2019-05-12
+### Added
+- wrap job execution with middleware (#38)
+- use `Middleware::JobLogger` for job logging (#39)
+- allow Appsignal instrumentation using `Middleware::AppsignalInstrumenter` (#40)
+
+### Removed
+- support for Ruby 2.3 (#41)
+
 ## [0.4.0] - 2019-04-07
 ### Added
 - give up workflow before it finishes by returning `false` in any of its tasks (#25)
@@ -47,6 +77,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## 0.1.0 - 2018-09-29
 - Pallets' inception <3
 
-[Unreleased]: https://github.com/linkyndy/pallets/compare/compare/v0.3.0...HEAD
+[Unreleased]: https://github.com/linkyndy/pallets/compare/compare/v0.5.1...HEAD
+[0.5.1]: https://github.com/linkyndy/pallets/compare/v0.5.0...v0.5.1
+[0.5.0]: https://github.com/linkyndy/pallets/compare/v0.4.0...v0.5.0
+[0.4.0]: https://github.com/linkyndy/pallets/compare/v0.3.0...v0.5.0
 [0.3.0]: https://github.com/linkyndy/pallets/compare/v0.2.0...v0.3.0
 [0.2.0]: https://github.com/linkyndy/pallets/compare/v0.1.0...v0.2.0

data/Gemfile

@@ -3,6 +3,7 @@ source 'https://rubygems.org'
 gemspec
 
 gem 'rake'
+gem 'appsignal'
 
 group :test do
   gem 'rspec'

data/examples/aliases.rb

@@ -0,0 +1,28 @@
+require 'pallets'
+
+class Aliases < Pallets::Workflow
+  task 'StartSmtpServer'
+  task 'SendEmail', as: 'SayHello', depends_on: 'StartSmtpServer'
+  task 'SendEmail', as: 'SayGoodbye', depends_on: 'StartSmtpServer'
+  task 'StopSmtpServer' => ['SayHello', 'SayGoodbye']
+end
+
+class StartSmtpServer < Pallets::Task
+  def run
+    puts "Starting SMTP server..."
+  end
+end
+
+class SendEmail < Pallets::Task
+  def run
+    puts "* sending e-mail"
+  end
+end
+
+class StopSmtpServer < Pallets::Task
+  def run
+    puts "Stopped SMTP server"
+  end
+end
+
+Aliases.new.run

data/examples/anonymous.rb

@@ -0,0 +1,13 @@
+require 'pallets'
+
+class Anonymous < Pallets::Task
+  def run
+    puts 'This is anonymous!'
+  end
+end
+
+workflow = Pallets::Workflow.build do
+  task 'Anonymous'
+end
+
+workflow.new.run

data/examples/appsignal.rb

@@ -0,0 +1,32 @@
+require 'pallets'
+require 'pallets/middleware/appsignal_instrumenter'
+
+Appsignal.config = Appsignal::Config.new(
+  File.expand_path(File.dirname(__FILE__)),
+  "development"
+)
+Appsignal.start
+Appsignal.start_logger
+
+Pallets.configure do |c|
+  c.middleware << Pallets::Middleware::AppsignalInstrumenter
+end
+
+class Appsignaling < Pallets::Workflow
+  task 'Signaling'
+  task 'ReturningSignal' => 'Signaling'
+end
+
+class Signaling < Pallets::Task
+  def run
+    puts context['signal']
+  end
+end
+
+class ReturningSignal < Pallets::Task
+  def run
+    puts 'Ho!'
+  end
+end
+
+Appsignaling.new(signal: 'Hey').run

data/examples/config/appsignal.yml

@@ -0,0 +1,12 @@
+default: &defaults
+  push_api_key: "<%= ENV['APPSIGNAL_PUSH_API_KEY'] %>"
+  name: "Pallets"
+  debug: true
+
+development:
+  <<: *defaults
+  active: true
+
+production:
+  <<: *defaults
+  active: true

data/examples/config_savvy.rb

@@ -1,5 +1,13 @@
+require 'logger'
 require 'pallets'
 
+class AnnounceProcessing
+  def self.call(worker, job, context)
+    puts "Starting to process job..."
+    yield
+  end
+end
+
 Pallets.configure do |c|
   # Harness 4 Pallets workers per process
   c.concurrency = 4
@@ -23,6 +31,12 @@ Pallets.configure do |c|
   # Jobs will be retried up to 5 times upon failure. After that, they will be
   # given up. Retry times are exponential and happen after: 7, 22, 87, 262, ...
   c.max_failures = 5
+
+  # Custom loggers can be used too
+  c.logger = Logger.new(STDOUT)
+  # Job execution can be wrapped with middleware to provide custom logic.
+  # Anything that responds to `call` would do
+  c.middleware << AnnounceProcessing
 end
 
 class ConfigSavvy < Pallets::Workflow

data/lib/pallets.rb

@@ -9,6 +9,8 @@ require 'pallets/errors'
 require 'pallets/graph'
 require 'pallets/logger'
 require 'pallets/manager'
+require 'pallets/middleware/job_logger'
+require 'pallets/middleware/stack'
 require 'pallets/pool'
 require 'pallets/scheduler'
 require 'pallets/serializers/base'
@@ -50,18 +52,11 @@ module Pallets
     end
   end
 
-  def self.logger
-    @logger ||= begin
-      logger = Pallets::Logger.new(STDOUT)
-      # TODO: Ruby 2.4 supports Logger initialization with the arguments below, so
-      #       we can drop this after we drop support for Ruby 2.3
-      logger.level = Pallets::Logger::INFO
-      logger.formatter = Pallets::Logger::Formatters::Pretty.new
-      logger
-    end
+  def self.middleware
+    @middleware ||= configuration.middleware
   end
 
-  def self.logger=(logger)
-    @logger = logger
+  def self.logger
+    @logger ||= configuration.logger
   end
 end

data/lib/pallets/backends/base.rb

@@ -6,12 +6,12 @@ module Pallets
         raise NotImplementedError
       end
 
-      def get_context(workflow_id)
+      def get_context(wfid)
         raise NotImplementedError
       end
 
       # Saves a job after successfully processing it
-      def save(workflow_id, job, context_buffer)
+      def save(wfid, jid, job, context_buffer)
         raise NotImplementedError
       end
 
@@ -29,7 +29,7 @@ module Pallets
         raise NotImplementedError
       end
 
-      def run_workflow(workflow_id, jobs_with_dependencies, context)
+      def run_workflow(wfid, jobs, jobmasks, context)
         raise NotImplementedError
       end
     end

data/lib/pallets/backends/redis.rb

@@ -9,6 +9,7 @@ module Pallets
       RETRY_SET_KEY = 'retry-set'
       GIVEN_UP_SET_KEY = 'given-up-set'
       WORKFLOW_QUEUE_KEY = 'workflow-queue:%s'
+      JOBMASK_KEY = 'jobmask:%s'
       CONTEXT_KEY = 'context:%s'
       REMAINING_KEY = 'remaining:%s'
 
@@ -41,11 +42,11 @@ module Pallets
         end
       end
 
-      def save(wfid, job, context_buffer)
+      def save(wfid, jid, job, context_buffer)
         @pool.execute do |client|
           client.evalsha(
             @scripts['save'],
-            [WORKFLOW_QUEUE_KEY % wfid, QUEUE_KEY, RELIABILITY_QUEUE_KEY, RELIABILITY_SET_KEY, CONTEXT_KEY % wfid, REMAINING_KEY % wfid],
+            [WORKFLOW_QUEUE_KEY % wfid, QUEUE_KEY, RELIABILITY_QUEUE_KEY, RELIABILITY_SET_KEY, CONTEXT_KEY % wfid, REMAINING_KEY % wfid, JOBMASK_KEY % jid],
             context_buffer.to_a << job
           )
         end
@@ -81,13 +82,14 @@ module Pallets
         end
       end
 
-      def run_workflow(wfid, jobs_with_order, context_buffer)
+      def run_workflow(wfid, jobs, jobmasks, context_buffer)
         @pool.execute do |client|
           client.multi do
+            jobmasks.each { |jid, jobmask| client.zadd(JOBMASK_KEY % jid, jobmask) }
             client.evalsha(
               @scripts['run_workflow'],
               [WORKFLOW_QUEUE_KEY % wfid, QUEUE_KEY, REMAINING_KEY % wfid],
-              jobs_with_order
+              jobs
             )
             client.hmset(CONTEXT_KEY % wfid, *context_buffer.to_a) unless context_buffer.empty?
           end

data/lib/pallets/backends/scripts/run_workflow.lua

@@ -6,9 +6,8 @@ redis.call("SET", KEYS[3], eta)
 
 -- Queue jobs that are ready to be processed (their score is 0) and
 -- remove queued jobs from the sorted set
-local count = redis.call("ZCOUNT", KEYS[1], 0, 0)
-if count > 0 then
-  local work = redis.call("ZRANGEBYSCORE", KEYS[1], 0, 0)
+local work = redis.call("ZRANGEBYSCORE", KEYS[1], 0, 0)
+if #work > 0 then
   redis.call("LPUSH", KEYS[2], unpack(work))
   redis.call("ZREM", KEYS[1], unpack(work))
 end

data/lib/pallets/backends/scripts/save.lua

@@ -10,24 +10,21 @@ if #ARGV > 0 then
   redis.call("HMSET", KEYS[5], unpack(ARGV))
 end
 
--- Decrement all jobs from the sorted set
-local all_pending = redis.call("ZRANGE", KEYS[1], 0, -1)
-for score, task in pairs(all_pending) do
-  redis.call("ZINCRBY", KEYS[1], -1, task)
-end
+-- Decrement jobs from the sorted set by applying a jobmask
+redis.call("ZUNIONSTORE", KEYS[1], 2, KEYS[1], KEYS[7])
+redis.call("DEL", KEYS[7])
 
 -- Queue jobs that are ready to be processed (their score is 0) and
 -- remove queued jobs from sorted set
-local count = redis.call("ZCOUNT", KEYS[1], 0, 0)
-if count > 0 then
-  local work = redis.call("ZRANGEBYSCORE", KEYS[1], 0, 0)
+local work = redis.call("ZRANGEBYSCORE", KEYS[1], 0, 0)
+if #work > 0 then
   redis.call("LPUSH", KEYS[2], unpack(work))
   redis.call("ZREM", KEYS[1], unpack(work))
 end
 
 -- Decrement ETA and remove it together with the context if all tasks have
 -- been processed (ETA is 0)
-redis.call("DECR", KEYS[6])
-if tonumber(redis.call("GET", KEYS[6])) == 0 then
+local remaining = redis.call("DECR", KEYS[6])
+if remaining == 0 then
   redis.call("DEL", KEYS[5], KEYS[6])
 end

data/lib/pallets/cli.rb

@@ -1,5 +1,7 @@
 require 'optparse'
 
+$stdout.sync = true
+
 module Pallets
   class CLI
     def initialize

data/lib/pallets/configuration.rb

@@ -20,6 +20,9 @@ module Pallets
     # period, it is considered failed, and scheduled to be processed again
     attr_accessor :job_timeout
 
+    # Custom logger used throughout Pallets
+    attr_writer :logger
+
     # Maximum number of failures allowed per job. Can also be configured on a
     # per task basis
     attr_accessor :max_failures
@@ -30,6 +33,15 @@ module Pallets
     # Serializer used for jobs
     attr_accessor :serializer
 
+    # Middleware used to wrap job execution with custom logic. Acts like a stack
+    # and accepts callable objects (lambdas, procs, objects that respond to call)
+    # that take three arguments: the worker handling the job, the job hash and
+    # the context
+    #
+    # A minimal example of a middleware is:
+    #   ->(worker, job, context, &b) { puts 'Hello World!'; b.call }
+    attr_reader :middleware
+
     def initialize
       @backend = :redis
       @backend_args = {}
@@ -39,10 +51,24 @@ module Pallets
       @job_timeout = 1_800 # 30 minutes
       @max_failures = 3
       @serializer = :json
+      @middleware = default_middleware
+    end
+
+    def logger
+      @logger || Pallets::Logger.new(STDOUT,
+        level: Pallets::Logger::INFO,
+        formatter: Pallets::Logger::Formatters::Pretty.new
+      )
     end
 
     def pool_size
       @pool_size || @concurrency + 1
     end
+
+    def default_middleware
+      Middleware::Stack[
+        Middleware::JobLogger
+      ]
+    end
   end
 end

data/lib/pallets/dsl/workflow.rb

@@ -1,7 +1,13 @@
 module Pallets
   module DSL
     module Workflow
-      def task(arg, depends_on: nil, max_failures: nil, &block)
+      def task(arg=nil, as: nil, depends_on: nil, max_failures: nil, **kwargs)
+        # Have to work more to keep Pallets' nice DSL valid in Ruby 2.7
+        arg = !kwargs.empty? ? kwargs : arg
+        raise ArgumentError, 'Task is incorrectly defined. It must receive '\
+                             'either a name, or a name => dependencies pair as '\
+                             'the first argument' unless arg
+
         klass, dependencies = case arg
         when Hash
           # The `task Foo => Bar` notation
@@ -12,10 +18,13 @@ module Pallets
         end
 
         task_class = klass.to_s
+        as ||= task_class
+
         dependencies = Array(dependencies).compact.uniq.map(&:to_s)
-        graph.add(task_class, dependencies)
+        graph.add(as, dependencies)
 
-        task_config[task_class] = {
+        task_config[as] = {
+          'workflow_class' => self.name,
           'task_class' => task_class,
           'max_failures' => max_failures || Pallets.configuration.max_failures
         }

data/lib/pallets/graph.rb

@@ -9,40 +9,39 @@ module Pallets
     end
 
     def add(node, dependencies)
-      @nodes[node] = dependencies
+      raise WorkflowError, "Task #{node} is already defined in this workflow. "\
+                           "Use `task '#{node}', as: 'FooBar'` to define an "\
+                           "alias and reuse task" if nodes.key?(node)
+
+      nodes[node] = dependencies
     end
 
     def parents(node)
-      @nodes[node]
+      nodes[node]
     end
 
     def empty?
-      @nodes.empty?
+      nodes.empty?
     end
 
-    # Returns nodes topologically sorted, together with their order (number of
-    # nodes that have to be executed prior)
-    def sorted_with_order
-      # Identify groups of nodes that can be executed concurrently
-      groups = tsort_each.slice_when { |a, b| parents(a) != parents(b) }
-
-      # Assign order to each node
-      i = 0
-      groups.flat_map do |group|
-        group_with_order = group.product([i])
-        i += group.size
-        group_with_order
+    def each
+      return enum_for(__method__) unless block_given?
+
+      tsort_each do |node|
+        yield(node, parents(node))
       end
     end
 
     private
 
+    attr_reader :nodes
+
     def tsort_each_node(&block)
-      @nodes.each_key(&block)
+      nodes.each_key(&block)
     end
 
     def tsort_each_child(node, &block)
-      @nodes.fetch(node).each(&block)
+      nodes.fetch(node).each(&block)
     rescue KeyError
       raise WorkflowError, "Task #{node} is marked as a dependency but not defined"
     end

data/lib/pallets/middleware/appsignal_instrumenter.rb

@@ -0,0 +1,47 @@
+require 'appsignal'
+
+module Pallets
+  module Middleware
+    class AppsignalInstrumenter
+      extend Appsignal::Hooks::Helpers
+
+      def self.call(worker, job, context)
+        job_status = nil
+        transaction = Appsignal::Transaction.create(
+          SecureRandom.uuid,
+          Appsignal::Transaction::BACKGROUND_JOB,
+          Appsignal::Transaction::GenericRequest.new(queue_start: job['created_at'])
+        )
+
+        Appsignal.instrument('perform_job.pallets') do
+          begin
+            yield
+          rescue Exception => ex
+            job_status = :failed
+            transaction.set_error(ex)
+            raise
+          ensure
+            transaction.set_action_if_nil("#{job['task_class']}#run (#{job['workflow_class']})")
+            transaction.params = filtered_context(context)
+            formatted_metadata(job).each { |kv| transaction.set_metadata(*kv) }
+            transaction.set_http_or_background_queue_start
+            Appsignal.increment_counter('pallets_job_count', 1, status: job_status || :successful)
+          end
+        end
+      ensure
+        Appsignal::Transaction.complete_current!
+      end
+
+      def self.filtered_context(context)
+        Appsignal::Utils::HashSanitizer.sanitize(
+          context,
+          Appsignal.config[:filter_parameters]
+        )
+      end
+
+      def self.formatted_metadata(job)
+        job.map { |k, v| [k, truncate(string_or_inspect(v))] }
+      end
+    end
+  end
+end

data/lib/pallets/middleware/job_logger.rb

@@ -0,0 +1,26 @@
+module Pallets
+  module Middleware
+    class JobLogger
+      def self.call(worker, job, context)
+        Pallets.logger.info 'Started', extract_metadata(worker.id, job)
+        result = yield
+        Pallets.logger.info 'Done', extract_metadata(worker.id, job)
+        result
+      rescue => ex
+        Pallets.logger.warn "#{ex.class.name}: #{ex.message}", extract_metadata(worker.id, job)
+        Pallets.logger.warn ex.backtrace.join("\n"), extract_metadata(worker.id, job) unless ex.backtrace.nil?
+        raise
+      end
+
+      def self.extract_metadata(wid, job)
+        {
+          wid:  wid,
+          wfid: job['wfid'],
+          jid:  job['jid'],
+          wf:   job['workflow_class'],
+          tsk:  job['task_class'],
+        }
+      end
+    end
+  end
+end

data/lib/pallets/middleware/stack.rb

@@ -0,0 +1,13 @@
+module Pallets
+  module Middleware
+    # Array-like class that acts like a stack and additionally provides the
+    # means to wrap an operation with callable objects
+    class Stack < Array
+      def invoke(*args, &block)
+        reverse.inject(block) do |memo, middleware|
+          lambda { middleware.call(*args, &memo) }
+        end.call
+      end
+    end
+  end
+end

data/lib/pallets/serializers/json.rb

@@ -4,12 +4,11 @@ module Pallets
   module Serializers
     class Json < Base
       def dump(data)
-        # TODO: Remove option after dropping support for Ruby 2.3
-        JSON.generate(data, quirks_mode: true)
+        JSON.generate(data)
       end
 
       def load(data)
-        JSON.parse(data, quirks_mode: true)
+        JSON.parse(data)
       end
     end
   end

data/lib/pallets/version.rb

@@ -1,3 +1,3 @@
 module Pallets
-  VERSION = "0.4.0"
+  VERSION = "0.8.0"
 end

data/lib/pallets/worker.rb

@@ -69,8 +69,6 @@ module Pallets
         return
       end
 
-      Pallets.logger.info "Started", extract_metadata(job_hash)
-
       context = Context[
         serializer.load_context(backend.get_context(job_hash['wfid']))
       ]
@@ -78,7 +76,9 @@ module Pallets
       task_class = Pallets::Util.constantize(job_hash["task_class"])
       task = task_class.new(context)
       begin
-        task_result = task.run
+        task_result = middleware.invoke(self, job_hash, context) do
+          task.run
+        end
       rescue => ex
         handle_job_error(ex, job, job_hash)
       else
@@ -91,8 +91,6 @@ module Pallets
     end
 
     def handle_job_error(ex, job, job_hash)
-      Pallets.logger.warn "#{ex.class.name}: #{ex.message}", extract_metadata(job_hash)
-      Pallets.logger.warn ex.backtrace.join("\n"), extract_metadata(job_hash) unless ex.backtrace.nil?
       failures = job_hash.fetch('failures', 0) + 1
       new_job = serializer.dump(job_hash.merge(
         'failures' => failures,
@@ -106,7 +104,6 @@ module Pallets
         backend.retry(new_job, job, retry_at)
       else
         backend.give_up(new_job, job)
-        Pallets.logger.info "Gave up after #{failures} failed attempts", extract_metadata(job_hash)
       end
     end
 
@@ -116,22 +113,10 @@ module Pallets
         'reason' => 'returned_false'
       ))
       backend.give_up(new_job, job)
-      Pallets.logger.info "Gave up after returning false", extract_metadata(job_hash)
     end
 
     def handle_job_success(context, job, job_hash)
-      backend.save(job_hash['wfid'], job, serializer.dump_context(context.buffer))
-      Pallets.logger.info "Done", extract_metadata(job_hash)
-    end
-
-    def extract_metadata(job_hash)
-      {
-        wid:  id,
-        wfid: job_hash['wfid'],
-        jid: job_hash['jid'],
-        wf:   job_hash['workflow_class'],
-        tsk:  job_hash['task_class']
-      }
+      backend.save(job_hash['wfid'], job_hash['jid'], job, serializer.dump_context(context.buffer))
     end
 
     def backoff_in_seconds(count)
@@ -145,5 +130,9 @@ module Pallets
     def serializer
       @serializer ||= Pallets.serializer
     end
+
+    def middleware
+      @middleware ||= Pallets.middleware
+    end
   end
 end

data/lib/pallets/workflow.rb

@@ -4,6 +4,12 @@ module Pallets
 
     attr_reader :context
 
+    def self.build(&block)
+      Class.new(self).tap do |workflow_class|
+        workflow_class.instance_eval(&block)
+      end
+    end
+
     def initialize(context_hash = {})
       @id = nil
       # Passed in context hash needs to be buffered
@@ -14,7 +20,7 @@ module Pallets
       raise WorkflowError, "#{self.class.name} has no tasks. Workflows "\
                            "must contain at least one task" if self.class.graph.empty?
 
-      backend.run_workflow(id, jobs_with_order, serializer.dump_context(context.buffer))
+      backend.run_workflow(id, *prepare_jobs, serializer.dump_context(context.buffer))
       id
     end
 
@@ -24,20 +30,29 @@ module Pallets
 
     private
 
-    def jobs_with_order
-      self.class.graph.sorted_with_order.map do |task_class, order|
-        job = serializer.dump(construct_job(task_class))
-        [order, job]
+    def prepare_jobs
+      jobs = []
+      jobmasks = Hash.new { |h, k| h[k] = [] }
+      acc = {}
+
+      self.class.graph.each do |task_alias, dependencies|
+        job_hash = construct_job(task_alias)
+        acc[task_alias] = job_hash['jid']
+        job = serializer.dump(job_hash)
+
+        jobs << [dependencies.size, job]
+        dependencies.each { |d| jobmasks[acc[d]] << [-1, job] }
       end
+
+      [jobs, jobmasks]
     end
 
-    def construct_job(task_class)
-      {}.tap do |job|
+    def construct_job(task_alias)
+      Hash[self.class.task_config[task_alias]].tap do |job|
         job['wfid'] = id
-        job['jid'] = "J#{Pallets::Util.generate_id(task_class)}".upcase
-        job['workflow_class'] = self.class.name
+        job['jid'] = "J#{Pallets::Util.generate_id(job['task_class'])}".upcase
         job['created_at'] = Time.now.to_f
-      end.merge(self.class.task_config[task_class])
+      end
     end
 
     def backend
@@ -48,6 +63,10 @@ module Pallets
       Pallets.serializer
     end
 
+    def self.name
+      @name ||= super || '<Anonymous>'
+    end
+
     def self.task_config
       @task_config ||= {}
     end

data/pallets.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
   spec.executables   = ['pallets']
   spec.require_paths = ['lib']
 
-  spec.required_ruby_version = '>= 2.3'
+  spec.required_ruby_version = '>= 2.4'
 
   spec.add_dependency 'redis'
   spec.add_dependency 'msgpack'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pallets
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.8.0
 platform: ruby
 authors:
 - Andrei Horak
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-04-07 00:00:00.000000000 Z
+date: 2020-06-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -46,6 +46,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/FUNDING.yml"
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
@@ -56,6 +57,10 @@ files:
 - README.md
 - Rakefile
 - bin/pallets
+- examples/aliases.rb
+- examples/anonymous.rb
+- examples/appsignal.rb
+- examples/config/appsignal.yml
 - examples/config_savvy.rb
 - examples/do_groceries.rb
 - examples/hello_world.rb
@@ -75,6 +80,9 @@ files:
 - lib/pallets/graph.rb
 - lib/pallets/logger.rb
 - lib/pallets/manager.rb
+- lib/pallets/middleware/appsignal_instrumenter.rb
+- lib/pallets/middleware/job_logger.rb
+- lib/pallets/middleware/stack.rb
 - lib/pallets/pool.rb
 - lib/pallets/scheduler.rb
 - lib/pallets/serializers/base.rb
@@ -98,15 +106,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.3'
+      version: '2.4'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.2.3
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Toy workflow engine, written in Ruby