pallets 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 388f932d85cba341084169563b80c6d18a0959f5
-  data.tar.gz: 04bbaa05f08caf6e53eaf632c13ee481abec2399
+SHA256:
+  metadata.gz: 7e0544b5ac0bf2eb5cbf4aa0b1795e4588df46485ed36e7bc5560d26cf47684a
+  data.tar.gz: 95a3e7f6ca57048773ba2f2ac72304090c5899b562766e2e99e6ad1911fcc103
 SHA512:
-  metadata.gz: cde6012d314a8c543cb63a156f2a667cb8b926c9a18c18c89b5c52bed6c680c07e9f8235a9f4b37a36dba17031f8923f6d445a9f019bce4e7c97757954242409
-  data.tar.gz: 9493181596052789e359bdf5e079e23707f740bd9e97b6ebc56ab0974f85f528c4fbc905bb8695c61cfb6b6e1f77ce2b675c0213a9f6f39251e4b95dad7f71fe
+  metadata.gz: 296c3ce03c89e6081e707014d28d8c48d65d22d6adac4ee5534551f39853eba171d12d143b52fa271857266342f12a965639749fd436126a66118fc911f47309
+  data.tar.gz: fccac5f172c6f8b03bf4409cd37592efc81f93d951ac1e5ac0163a6e74deabdecb607f3b5d9b2a5e7e11ab8c2e7fbecb7aa2b48c8544c18d0041e4ebd2b59467

data/.travis.yml

@@ -4,9 +4,12 @@ services:
   - redis-server
 cache: bundler
 rvm:
-  - 2.1.10
-  - 2.2.10
-  - 2.3.7
-  - 2.4.4
-  - 2.5.1
+  - 2.3.8
+  - 2.4.5
+  - 2.5.3
+  - 2.6.0
+before_install:
+  # Bundler 2.0 needs a newer RubyGems
+  - gem update --system
+  - gem install bundler
 script: bundle exec rspec

data/CHANGELOG.md

@@ -0,0 +1,34 @@
+# Changelog
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [0.3.0] - 2019-02-08
+### Added
+- shared contexts (#9)
+- handle TERM and TTIN signals (#15, #17)
+- configure how long failed jobs are kept (#21)
+
+### Changed
+- use a single Redis connection when picking up work (#11)
+- improve logging (#14)
+- fix handling empty workflows and contexts (#18)
+- fix encoding for msgpack serializer (#19)
+- malformed jobs are given up rather than discarded (#22)
+
+### Removed
+- support for Ruby 2.1 & 2.2 (#13)
+
+## [0.2.0] - 2018-10-02
+### Added
+- msgpack serializer (#5)
+
+## 0.1.0 - 2018-09-29
+- Pallets' inception <3
+
+[Unreleased]: https://github.com/linkyndy/pallets/compare/compare/v0.3.0...HEAD
+[0.3.0]: https://github.com/linkyndy/pallets/compare/v0.2.0...v0.3.0
+[0.2.0]: https://github.com/linkyndy/pallets/compare/v0.1.0...v0.2.0

data/Gemfile

@@ -1,4 +1,17 @@
 source 'https://rubygems.org'
 
-# Specify your gem's dependencies in pallets.gemspec
 gemspec
+
+gem 'rake'
+
+group :test do
+  gem 'rspec'
+  # Better RSpec formatting
+  gem 'fuubar'
+end
+
+group :development, :test do
+  gem 'pry-byebug'
+  # Time travel in style
+  gem 'timecop'
+end

data/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2018 Andrei Horak
+Copyright (c) 2019 Andrei Horak
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -27,7 +27,7 @@ end
 MyWorkflow.new.run
 ```
 
-That's basically it! Curious for more? Read on!
+That's basically it! Curious for more? Read on or [check the examples](examples/)!
 
 > Don't forget to run pallets, so it can process your tasks: `bundle exec pallets -r ./my_workflow`
 

data/examples/config_savvy.rb

@@ -0,0 +1,45 @@
+require 'pallets'
+
+Pallets.configure do |c|
+  # Harness 4 Pallets workers per process
+  c.concurrency = 4
+
+  # The default one, though
+  c.backend = :redis
+  # Useful to connect to a hosted Redis instance. Takes all options `Redis.new`
+  # accepts, like `db`, `timeout`, `host`, `port`, `password`, `sentinels`.
+  # Check https://www.rubydoc.info/github/redis/redis-rb/Redis:initialize for
+  # more details
+  c.backend_args = { url: 'redis://127.0.0.1:6379/1' }
+  # Use a maximum of 10 backend connections (Redis, in this case)
+  c.pool_size = 10
+
+  # A tad faster than JSON
+  c.serializer = :msgpack
+
+  # Allow 10 minutes for a job to process. After this, we assume the job did not
+  # finish and we retry it
+  c.job_timeout = 600
+  # Jobs will be retried up to 5 times upon failure. After that, they will be
+  # given up. Retry times are exponential and happen after: 7, 22, 87, 262, ...
+  c.max_failures = 5
+end
+
+class ConfigSavvy < Pallets::Workflow
+  task :volatile
+  task :success => :volatile
+end
+
+class Volatile < Pallets::Task
+  def run
+    raise 'I am ràndomly failing' if [true, false].sample
+  end
+end
+
+class Success < Pallets::Task
+  def run
+    puts 'I am executed after Volatile manages to successfully execute'
+  end
+end
+
+ConfigSavvy.new.run

data/examples/do_groceries.rb

@@ -0,0 +1,62 @@
+require 'pallets'
+
+Pallets.configure do |c|
+  c.backend_args = { url: 'redis://127.0.0.1:6379/13' }
+end
+
+class DoGroceries < Pallets::Workflow
+  task :enter_shop
+  task :get_shopping_cart => :enter_shop
+  task :put_milk => :get_shopping_cart
+  task :put_bread => :get_shopping_cart
+  task :pay => [:put_milk, :put_bread]
+  task :go_home => :pay
+end
+
+class EnterShop < Pallets::Task
+  def run
+    puts "Entering #{context['shop_name']}"
+    raise 'Cannot enter shop!' if (context['i'].to_i % 10).zero?
+  end
+end
+
+class GetShoppingCart < Pallets::Task
+  def run
+    puts "Where's that 50 cent coin??"
+    context['need_to_return_coin'] = true
+  end
+end
+
+class PutMilk < Pallets::Task
+  def run
+    puts "Whole or half? Hmm..."
+    sleep 1
+  end
+end
+
+class PutBread < Pallets::Task
+  def run
+    raise 'Out of bread!' if (context['i'].to_i % 30).zero?
+    puts "Got the bread"
+  end
+end
+
+class Pay < Pallets::Task
+  def run
+    puts "Paying by #{context['pay_by']}"
+    sleep 2
+    raise 'Payment failed!' if (context['i'].to_i % 100).zero?
+  end
+end
+
+class GoHome < Pallets::Task
+  def run
+    puts "Done!!"
+
+    if context['need_to_return_coin']
+      puts '...forgot to get my coin back...'
+    end
+  end
+end
+
+1_000.times { |i| DoGroceries.new(shop_name: 'Pallet Shop', pay_by: :card, i: i).run }

data/examples/hello_world.rb

@@ -0,0 +1,13 @@
+require 'pallets'
+
+class HelloWorld < Pallets::Workflow
+  task :echo
+end
+
+class Echo < Pallets::Task
+  def run
+    puts 'Hello World!'
+  end
+end
+
+HelloWorld.new.run

data/lib/pallets.rb

@@ -3,9 +3,11 @@ require "pallets/version"
 require 'pallets/backends/base'
 require 'pallets/backends/redis'
 require 'pallets/configuration'
+require 'pallets/context'
 require 'pallets/dsl/workflow'
 require 'pallets/errors'
 require 'pallets/graph'
+require 'pallets/logger'
 require 'pallets/manager'
 require 'pallets/pool'
 require 'pallets/scheduler'
@@ -17,7 +19,6 @@ require 'pallets/util'
 require 'pallets/worker'
 require 'pallets/workflow'
 
-require 'logger'
 require 'securerandom'
 
 module Pallets
@@ -35,6 +36,7 @@ module Pallets
       cls.new(
         namespace: configuration.namespace,
         blocking_timeout: configuration.blocking_timeout,
+        failed_job_lifespan: configuration.failed_job_lifespan,
         job_timeout: configuration.job_timeout,
         pool_size: configuration.pool_size,
         **configuration.backend_args
@@ -50,6 +52,17 @@ module Pallets
   end
 
   def self.logger
-    @logger ||= Logger.new(STDOUT)
+    @logger ||= begin
+      logger = Pallets::Logger.new(STDOUT)
+      # TODO: Ruby 2.4 supports Logger initialization with the arguments below, so
+      #       we can drop this after we drop support for Ruby 2.3
+      logger.level = Pallets::Logger::INFO
+      logger.formatter = Pallets::Logger::Formatters::Pretty.new
+      logger
+    end
+  end
+
+  def self.logger=(logger)
+    @logger = logger
   end
 end

data/lib/pallets/backends/base.rb

@@ -6,23 +6,22 @@ module Pallets
         raise NotImplementedError
       end
 
-      # Saves a job after successfully processing it
-      def save(workflow_id, job)
+      def get_context(workflow_id)
         raise NotImplementedError
       end
 
-      # Discard a malformed job
-      def discard(job)
+      # Saves a job after successfully processing it
+      def save(workflow_id, job, context_buffer)
         raise NotImplementedError
       end
 
-      # Schedule a failed job for retry
+      # Schedules a failed job for retry
       def retry(job, old_job, at)
         raise NotImplementedError
       end
 
-      # Give up job after repeteadly failing to process it
-      def give_up(job, old_job, at)
+      # Gives up job after repeteadly failing to process it
+      def give_up(job, old_job)
         raise NotImplementedError
       end
 
@@ -30,7 +29,7 @@ module Pallets
         raise NotImplementedError
       end
 
-      def run_workflow(workflow_id, jobs_with_dependencies)
+      def run_workflow(workflow_id, jobs_with_dependencies, context)
         raise NotImplementedError
       end
     end

data/lib/pallets/backends/redis.rb

@@ -3,9 +3,10 @@ require 'redis'
 module Pallets
   module Backends
     class Redis < Base
-      def initialize(namespace:, blocking_timeout:, job_timeout:, pool_size:, **options)
+      def initialize(namespace:, blocking_timeout:, failed_job_lifespan:, job_timeout:, pool_size:, **options)
         @namespace = namespace
         @blocking_timeout = blocking_timeout
+        @failed_job_lifespan = failed_job_lifespan
         @job_timeout = job_timeout
         @pool = Pallets::Pool.new(pool_size) { ::Redis.new(options) }
 
@@ -13,44 +14,40 @@ module Pallets
         @reliability_queue_key = "#{namespace}:reliability-queue"
         @reliability_set_key = "#{namespace}:reliability-set"
         @retry_set_key = "#{namespace}:retry-set"
-        @fail_set_key = "#{namespace}:fail-set"
+        @given_up_set_key = "#{namespace}:given-up-set"
         @workflow_key = "#{namespace}:workflows:%s"
+        @context_key = "#{namespace}:contexts:%s"
+        @eta_key = "#{namespace}:etas:%s"
 
         register_scripts
       end
 
       def pick
-        job = @pool.execute do |client|
-          client.brpoplpush(@queue_key, @reliability_queue_key, timeout: @blocking_timeout)
-        end
-        if job
-          # We store the job's timeout so we know when to retry jobs that are
-          # still on the reliability queue. We do this separately since there is
-          # no other way to atomically BRPOPLPUSH from the main queue to a
-          # sorted set
-          @pool.execute do |client|
+        @pool.execute do |client|
+          job = client.brpoplpush(@queue_key, @reliability_queue_key, timeout: @blocking_timeout)
+          if job
+            # We store the job's timeout so we know when to retry jobs that are
+            # still on the reliability queue. We do this separately since there is
+            # no other way to atomically BRPOPLPUSH from the main queue to a
+            # sorted set
             client.zadd(@reliability_set_key, Time.now.to_f + @job_timeout, job)
           end
+          job
         end
-        job
       end
 
-      def save(workflow_id, job)
+      def get_context(workflow_id)
         @pool.execute do |client|
-          client.eval(
-            @scripts['save'],
-            [@workflow_key % workflow_id, @queue_key, @reliability_queue_key, @reliability_set_key],
-            [job]
-          )
+          client.hgetall(@context_key % workflow_id)
         end
       end
 
-      def discard(job)
+      def save(workflow_id, job, context_buffer)
         @pool.execute do |client|
           client.eval(
-            @scripts['discard'],
-            [@reliability_queue_key, @reliability_set_key],
-            [job]
+            @scripts['save'],
+            [@workflow_key % workflow_id, @queue_key, @reliability_queue_key, @reliability_set_key, @context_key % workflow_id, @eta_key % workflow_id],
+            context_buffer.to_a << job
           )
         end
       end
@@ -65,12 +62,12 @@ module Pallets
         end
       end
 
-      def give_up(job, old_job, at)
+      def give_up(job, old_job)
         @pool.execute do |client|
           client.eval(
             @scripts['give_up'],
-            [@fail_set_key, @reliability_queue_key, @reliability_set_key],
-            [at, job, old_job]
+            [@given_up_set_key, @reliability_queue_key, @reliability_set_key],
+            [Time.now.to_f, job, old_job, Time.now.to_f - @failed_job_lifespan]
           )
         end
       end
@@ -85,13 +82,16 @@ module Pallets
         end
       end
 
-      def run_workflow(workflow_id, jobs_with_order)
+      def run_workflow(workflow_id, jobs_with_order, context)
         @pool.execute do |client|
-          client.eval(
-            @scripts['run_workflow'],
-            [@workflow_key % workflow_id, @queue_key],
-            jobs_with_order
-          )
+          client.multi do
+            client.eval(
+              @scripts['run_workflow'],
+              [@workflow_key % workflow_id, @queue_key, @eta_key % workflow_id],
+              jobs_with_order
+            )
+            client.hmset(@context_key % workflow_id, *context.to_a) unless context.empty?
+          end
         end
       end
 

data/lib/pallets/backends/scripts/give_up.lua

@@ -4,3 +4,7 @@ redis.call("ZREM", KEYS[3], ARGV[3])
 
 -- Add job and its fail time (score) to failed sorted set
 redis.call("ZADD", KEYS[1], ARGV[1], ARGV[2])
+
+-- Remove any jobs that have been given up long enough ago (their score is
+-- below given value)
+redis.call("ZREMRANGEBYSCORE", KEYS[1], "-inf", ARGV[4])

data/lib/pallets/backends/scripts/run_workflow.lua

@@ -1,5 +1,8 @@
 -- Add all jobs to sorted set
-redis.call("ZADD", KEYS[1], unpack(ARGV))
+local eta = redis.call("ZADD", KEYS[1], unpack(ARGV))
+
+-- Set ETA key; this is merely the number of jobs that need to be processed
+redis.call("SET", KEYS[3], eta)
 
 -- Queue jobs that are ready to be processed (their score is 0) and
 -- remove queued jobs from the sorted set

data/lib/pallets/backends/scripts/save.lua

@@ -1,6 +1,14 @@
+-- NOTE: We store the job as the last argument passed to this script because it
+--       is more efficient to pop in Lua than shift
+local job = table.remove(ARGV)
 -- Remove job from reliability queue
-redis.call("LREM", KEYS[3], 0, ARGV[1])
-redis.call("ZREM", KEYS[4], ARGV[1])
+redis.call("LREM", KEYS[3], 0, job)
+redis.call("ZREM", KEYS[4], job)
+
+-- Update context hash with buffer
+if #ARGV > 0 then
+  redis.call("HMSET", KEYS[5], unpack(ARGV))
+end
 
 -- Decrement all jobs from the sorted set
 local all_pending = redis.call("ZRANGE", KEYS[1], 0, -1)
@@ -16,3 +24,10 @@ if count > 0 then
   redis.call("LPUSH", KEYS[2], unpack(work))
   redis.call("ZREM", KEYS[1], unpack(work))
 end
+
+-- Decrement ETA and remove it together with the context if all tasks have
+-- been processed (ETA is 0)
+redis.call("DECR", KEYS[6])
+if tonumber(redis.call("GET", KEYS[6])) == 0 then
+  redis.call("DEL", KEYS[5], KEYS[6])
+end

data/lib/pallets/cli.rb

@@ -11,7 +11,8 @@ module Pallets
     end
 
     def run
-      Pallets.logger.info 'Starting the awesomeness of Pallets <3'
+      Pallets.logger.info 'Starting the awesome Pallets <3'
+      Pallets.logger.info "Running on #{RUBY_DESCRIPTION}"
 
       @manager.start
 
@@ -30,8 +31,17 @@ module Pallets
 
     def handle_signal(signal)
       case signal
-      when 'INT'
+      when 'INT', 'TERM'
         raise Interrupt
+      when 'TTIN'
+        print_backtraces
+      end
+    end
+
+    def print_backtraces
+      (@manager.workers + [@manager.scheduler]).each do |actor|
+        Pallets.logger.info "Debugging #{actor.id}"
+        Pallets.logger.info actor.debug.join("\n") unless actor.debug.nil?
       end
     end
 
@@ -51,6 +61,10 @@ module Pallets
           Pallets.configuration.max_failures = max_failures
         end
 
+        opts.on('-l', '--failed-job-lifespan NUM', Integer, 'Seconds a job stays in the given up set') do |failed_job_lifespan|
+          Pallets.configuration.failed_job_lifespan = failed_job_lifespan
+        end
+
         opts.on('-n', '--namespace NAME', 'Namespace to use for backend') do |namespace|
           Pallets.configuration.namespace = namespace
         end
@@ -71,6 +85,10 @@ module Pallets
           Pallets.configuration.serializer = serializer
         end
 
+        opts.on('-t', '--job-timeout NUM', Integer, 'Seconds allowed for a job to be processed') do |job_timeout|
+          Pallets.configuration.job_timeout = job_timeout
+        end
+
         opts.on('-u', '--blocking-timeout NUM', Integer, 'Seconds to block while waiting for work') do |blocking_timeout|
           Pallets.configuration.blocking_timeout = blocking_timeout
         end
@@ -92,7 +110,7 @@ module Pallets
     end
 
     def setup_signal_handlers
-      %w(INT).each do |signal|
+      %w(INT TERM TTIN).each do |signal|
         trap signal do
           @signal_queue.push signal
         end

data/lib/pallets/configuration.rb

@@ -12,6 +12,10 @@ module Pallets
     # Number of workers to process jobs
     attr_accessor :concurrency
 
+    # Minimum number of seconds a failed job stays in the given up set. After
+    # this period, jobs will be permanently deleted
+    attr_accessor :failed_job_lifespan
+
     # Number of seconds allowed for a job to be processed. If a job exceeds this
     # period, it is considered failed, and scheduled to be processed again
     attr_accessor :job_timeout
@@ -34,7 +38,8 @@ module Pallets
       @backend_args = {}
       @blocking_timeout = 5
       @concurrency = 2
-      @job_timeout = 1800 # 30 minutes
+      @failed_job_lifespan = 7_776_000 # 3 months
+      @job_timeout = 1_800 # 30 minutes
       @max_failures = 3
       @namespace = 'pallets'
       @pool_size = 5

data/lib/pallets/context.rb

@@ -0,0 +1,13 @@
+module Pallets
+  # Hash-like class that additionally holds a buffer for all write operations
+  class Context < Hash
+    def []=(key, value)
+      buffer[key] = value
+      super
+    end
+
+    def buffer
+      @buffer ||= {}
+    end
+  end
+end

data/lib/pallets/dsl/workflow.rb

@@ -7,7 +7,11 @@ module Pallets
         else
           options.first
         end
-        raise ArgumentError, "A task must have a name" unless name
+
+        unless name
+          raise WorkflowError, "Task has no name. Provide a name using " \
+                               "`task :name, *args` or `task name: :arg` syntax"
+        end
 
         # Handle nils, symbols or arrays consistently
         name = name.to_sym

data/lib/pallets/errors.rb

@@ -1,4 +1,14 @@
 module Pallets
+  # Generic class for all Pallets-related errors
+  class PalletsError < StandardError
+  end
+
+  # Raised when a workflow is not properly defined
+  class WorkflowError < PalletsError
+  end
+
+  # Raised when Pallets needs to shutdown
+  # NOTE: Do not rescue it!
   class Shutdown < Interrupt
   end
 end

data/lib/pallets/graph.rb

@@ -16,6 +16,10 @@ module Pallets
       @nodes[node]
     end
 
+    def empty?
+      @nodes.empty?
+    end
+
     # Returns nodes topologically sorted, together with their order (number of
     # nodes that have to be executed prior)
     def sorted_with_order

data/lib/pallets/logger.rb

@@ -0,0 +1,24 @@
+require 'logger'
+require 'time'
+
+module Pallets
+  class Logger < ::Logger
+    # Overwrite severity methods to add metadata capabilities
+    %i[debug info warn error fatal unknown].each do |severity|
+      define_method severity do |message, metadata = {}|
+        return super(message) if metadata.empty?
+
+        formatted_metadata = ' ' + metadata.map { |k, v| "#{k}=#{v}" }.join(' ')
+        super(formatted_metadata) { message }
+      end
+    end
+
+    module Formatters
+      class Pretty < ::Logger::Formatter
+        def call(severity, time, metadata, message)
+          "#{time.utc.iso8601(4)} pid=#{Process.pid}#{metadata} #{severity}: #{message}\n"
+        end
+      end
+    end
+  end
+end

data/lib/pallets/manager.rb

@@ -1,6 +1,6 @@
 module Pallets
   class Manager
-    attr_reader :workers, :timeout
+    attr_reader :workers, :scheduler
 
     def initialize(concurrency: Pallets.configuration.concurrency)
       @workers = concurrency.times.map { Worker.new(self) }

data/lib/pallets/scheduler.rb

@@ -21,6 +21,10 @@ module Pallets
       @needs_to_stop
     end
 
+    def debug
+      @thread.backtrace
+    end
+
     def id
       "S#{@thread.object_id.to_s(36)}".upcase if @thread
     end

data/lib/pallets/serializers/msgpack.rb

@@ -8,7 +8,9 @@ module Pallets
       end
 
       def load(data)
-        MessagePack.unpack(data)
+        # Strings coming from the backend are UTF-8 (Encoding.default_external)
+        # while msgpack dumps ASCII-8BIT
+        MessagePack.unpack(data.force_encoding('ASCII-8BIT'))
       end
     end
   end

data/lib/pallets/version.rb

@@ -1,3 +1,3 @@
 module Pallets
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/lib/pallets/worker.rb

@@ -26,6 +26,10 @@ module Pallets
       @needs_to_stop
     end
 
+    def debug
+      @thread.backtrace
+    end
+
     def id
       "W#{@thread.object_id.to_s(36)}".upcase if @thread
     end
@@ -49,34 +53,40 @@ module Pallets
     rescue Pallets::Shutdown
       @manager.remove_worker(self)
     rescue => ex
+      Pallets.logger.error "#{ex.class.name}: #{ex.message}", wid: id
+      Pallets.logger.error ex.backtrace.join("\n"), wid: id unless ex.backtrace.nil?
       @manager.replace_worker(self)
     end
 
     def process(job)
-      Pallets.logger.info "[#{id}] Picked job: #{job}"
       begin
         job_hash = serializer.load(job)
       rescue
         # We ensure only valid jobs are created. If something fishy reaches this
-        # point, just discard it
-        backend.discard(job)
+        # point, just give up on it
+        backend.give_up(job, job)
+        Pallets.logger.error "Could not deserialize #{job}. Gave up job", wid: id
         return
       end
 
+      Pallets.logger.info "Started", extract_metadata(job_hash)
+
+      context = Context[backend.get_context(job_hash['workflow_id'])]
+
       task_class = Pallets::Util.constantize(job_hash["class_name"])
-      task = task_class.new(job_hash["context"])
+      task = task_class.new(context)
       begin
         task.run
       rescue => ex
         handle_job_error(ex, job, job_hash)
       else
-        backend.save(job_hash["workflow_id"], job)
-        Pallets.logger.info "[#{id}] Successfully processed #{job}"
+        handle_job_success(context, job, job_hash)
       end
     end
 
     def handle_job_error(ex, job, job_hash)
-      Pallets.logger.error "[#{id}] Error while processing: #{ex}"
+      Pallets.logger.warn "#{ex.class.name}: #{ex.message}", extract_metadata(job_hash)
+      Pallets.logger.warn ex.backtrace.join("\n"), extract_metadata(job_hash) unless ex.backtrace.nil?
       failures = job_hash.fetch('failures', 0) + 1
       new_job = serializer.dump(job_hash.merge(
         'failures' => failures,
@@ -87,13 +97,26 @@ module Pallets
       if failures < job_hash['max_failures']
         retry_at = Time.now.to_f + backoff_in_seconds(failures)
         backend.retry(new_job, job, retry_at)
-        Pallets.logger.info "[#{id}] Scheduled job for retry"
       else
-        backend.give_up(new_job, job, Time.now.to_f)
-        Pallets.logger.info "[#{id}] Given up on job"
+        backend.give_up(new_job, job)
+        Pallets.logger.info "Gave up after #{failures} failed attempts", extract_metadata(job_hash)
       end
     end
 
+    def handle_job_success(context, job, job_hash)
+      backend.save(job_hash['workflow_id'], job, context.buffer)
+      Pallets.logger.info "Done", extract_metadata(job_hash)
+    end
+
+    def extract_metadata(job_hash)
+      {
+        wid:  id,
+        wfid: job_hash['workflow_id'],
+        wf:   job_hash['workflow_class_name'],
+        tsk:  job_hash['class_name']
+      }
+    end
+
     def backoff_in_seconds(count)
       count ** 4 + 6
     end

data/lib/pallets/workflow.rb

@@ -10,7 +10,10 @@ module Pallets
     end
 
     def run
-      backend.run_workflow(id, jobs_with_order)
+      raise WorkflowError, "#{self.class.name} has no tasks. Workflows "\
+                           "must contain at least one task" if self.class.graph.empty?
+
+      backend.run_workflow(id, jobs_with_order, context)
       id
     end
 
@@ -33,9 +36,9 @@ module Pallets
 
     def job_hash
       {
-        'workflow_id' => id,
-        'context'     => context,
-        'created_at'  => Time.now.to_f
+        'workflow_id'         => id,
+        'workflow_class_name' => self.class.name,
+        'created_at'          => Time.now.to_f
       }
     end
 

data/pallets.gemspec

@@ -4,25 +4,22 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'pallets/version'
 
 Gem::Specification.new do |spec|
-  spec.name          = "pallets"
+  spec.name          = 'pallets'
   spec.version       = Pallets::VERSION
-  spec.authors       = ["Andrei Horak"]
-  spec.email         = ["linkyndy@gmail.com"]
+  spec.authors       = ['Andrei Horak']
+  spec.email         = ['linkyndy@gmail.com']
 
   spec.summary       = 'Toy workflow engine, written in Ruby'
   spec.description   = 'Toy workflow engine, written in Ruby'
   spec.homepage      = 'https://github.com/linkyndy/pallets'
+  spec.license       = 'MIT'
 
-  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
-  spec.bindir        = "bin"
-  spec.executables   = ["pallets"]
-  spec.require_paths = ["lib"]
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
+  spec.executables   = ['pallets']
+  spec.require_paths = ['lib']
 
-  spec.add_dependency "redis"
-  spec.add_dependency "msgpack"
-  spec.add_development_dependency "bundler", "~> 1.11"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "rspec", "~> 3.0"
-  spec.add_development_dependency "timecop"
-  spec.add_development_dependency "fuubar"
+  spec.required_ruby_version = '>= 2.3'
+
+  spec.add_dependency 'redis'
+  spec.add_dependency 'msgpack'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pallets
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Andrei Horak
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-10-02 00:00:00.000000000 Z
+date: 2019-02-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -38,76 +38,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.11'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.11'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '10.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '10.0'
-- !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-- !ruby/object:Gem::Dependency
-  name: timecop
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: fuubar
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description: Toy workflow engine, written in Ruby
 email:
 - linkyndy@gmail.com
@@ -119,18 +49,19 @@ files:
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
+- CHANGELOG.md
 - CONTRIBUTING.md
 - Gemfile
 - LICENSE
 - README.md
 - Rakefile
-- bin/console
 - bin/pallets
-- bin/setup
+- examples/config_savvy.rb
+- examples/do_groceries.rb
+- examples/hello_world.rb
 - lib/pallets.rb
 - lib/pallets/backends/base.rb
 - lib/pallets/backends/redis.rb
-- lib/pallets/backends/scripts/discard.lua
 - lib/pallets/backends/scripts/give_up.lua
 - lib/pallets/backends/scripts/reschedule_all.lua
 - lib/pallets/backends/scripts/retry.lua
@@ -138,9 +69,11 @@ files:
 - lib/pallets/backends/scripts/save.lua
 - lib/pallets/cli.rb
 - lib/pallets/configuration.rb
+- lib/pallets/context.rb
 - lib/pallets/dsl/workflow.rb
 - lib/pallets/errors.rb
 - lib/pallets/graph.rb
+- lib/pallets/logger.rb
 - lib/pallets/manager.rb
 - lib/pallets/pool.rb
 - lib/pallets/scheduler.rb
@@ -154,7 +87,8 @@ files:
 - lib/pallets/workflow.rb
 - pallets.gemspec
 homepage: https://github.com/linkyndy/pallets
-licenses: []
+licenses:
+- MIT
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -164,7 +98,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -172,7 +106,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.14.1
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: Toy workflow engine, written in Ruby

data/bin/console

@@ -1,14 +0,0 @@
-#!/usr/bin/env ruby
-
-require "bundler/setup"
-require "pallets"
-
-# You can add fixtures and/or initialization code here to make experimenting
-# with your gem easier. You can also use a different console, if you like.
-
-# (If you use this, don't forget to add pry to your Gemfile!)
-# require "pry"
-# Pry.start
-
-require "irb"
-IRB.start

data/bin/setup

@@ -1,8 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-IFS=$'\n\t'
-set -vx
-
-bundle install
-
-# Do any other automated setup that you need to do here

data/lib/pallets/backends/scripts/discard.lua

@@ -1,3 +0,0 @@
--- Remove job from reliability queue
-redis.call("LREM", KEYS[1], 0, ARGV[1])
-redis.call("ZREM", KEYS[2], ARGV[1])