pallets 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: 7e0544b5ac0bf2eb5cbf4aa0b1795e4588df46485ed36e7bc5560d26cf47684a
-  data.tar.gz: 95a3e7f6ca57048773ba2f2ac72304090c5899b562766e2e99e6ad1911fcc103
+SHA1:
+  metadata.gz: 6bd6ebdb788c39b7d3cb434b60f94d927dacd47c
+  data.tar.gz: b463178c06885308a2192bbcf3d549c52391acbd
 SHA512:
-  metadata.gz: 296c3ce03c89e6081e707014d28d8c48d65d22d6adac4ee5534551f39853eba171d12d143b52fa271857266342f12a965639749fd436126a66118fc911f47309
-  data.tar.gz: fccac5f172c6f8b03bf4409cd37592efc81f93d951ac1e5ac0163a6e74deabdecb607f3b5d9b2a5e7e11ab8c2e7fbecb7aa2b48c8544c18d0041e4ebd2b59467
+  metadata.gz: 7e976084d28bba01327e0a8c41b8a948e27f6b3041d6a3a31665825d66b57225867ec410e9bf4f24afcf13241f41b855e1150601f37de37425fc72ee9582b5a4
+  data.tar.gz: c05e17653dac7716b9f52958b0bfb295517d614f5f97a00ffb91ad7100e83fbbf20f526f08f8315a1344829d01bd992c4920a7b0881aec4fea9b68eb0d259ff3

data/CHANGELOG.md

@@ -6,6 +6,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.4.0] - 2019-04-07
+### Added
+- give up workflow before it finishes by returning `false` in any of its tasks (#25)
+- jobs have a JID (#30)
+- Rails support (#27)
+
+### Changed
+- contexts are serialized and accept basic Ruby types as values (#24)
+- workflow tasks are defined using classes (#26)
+- some job and Redis keys have been renamed (#28)
+- job retry backoff has a random component (#32)
+- missing dependencies raise a `WorkflowError` (#31)
+- Redis backend uses `EVALSHA` for Lua scripts (#34)
+- the `pool_size` configuration is inferred from `concurrency` (#33)
+
+### Removed
+- backend namespaces (#28)
+
 ## [0.3.0] - 2019-02-08
 ### Added
 - shared contexts (#9)

data/README.md

@@ -11,10 +11,10 @@ Toy workflow engine, written in Ruby
 require 'pallets'
 
 class MyWorkflow < Pallets::Workflow
-  task :foo
-  task :bar => :foo
-  task :baz => :foo
-  task :qux => [:bar, :baz]
+  task Foo
+  task Bar => Foo
+  task Baz => Foo
+  task Qux => [Bar, Baz]
 end
 
 class Foo < Pallets::Task
@@ -38,6 +38,7 @@ That's basically it! Curious for more? Read on or [check the examples](examples/
 * retries failed tasks
 * Redis backend out of the box
 * JSON and msgpack serializers out of the box
+* Rails support
 * beautiful DSL
 * convention over configuration
 * thoroughly tested
@@ -74,6 +75,49 @@ end
 
 For the complete set of options, see [pallets/configuration.rb](lib/pallets/configuration.rb)
 
+## Cookbook
+
+### DSL
+
+Pallets is designed for developers' happiness. Its DSL aims to be as beautiful
+and readable as possible, while still enabling complex scenarios to be performed.
+
+```ruby
+# All workflows must subclass Pallets::Workflow
+class WashClothes < Pallets::Workflow
+  # The simplest task
+  task BuyDetergent
+
+  # Another task; since it has no dependencies, it will be executed in parallel
+  # with BuyDetergent
+  # TIP: Use a String argument when task is not _yet_ loaded
+  task 'BuySoftener'
+
+  # We're not doing it in real life, but we use it to showcase our first dependency!
+  task DilluteDetergent => BuyDetergent
+
+  # We're getting more complex here! This is the alternate way of defining
+  # dependencies (which can be several, by the way!). Choose the style that fits
+  # you best
+  task TurnOnWashingMachine, depends_on: [BuyDetergent, 'BuySoftener']
+
+  # Specify how many times a task is allowed to fail. If max_failures is reached
+  # the task is given up
+  task SelectProgram => TurnOnWashingMachine, max_failures: 2
+end
+
+# Every task must be a subclass of Pallets::Task
+class BuyDetergent < Pallets::Task
+  # Tasks must implement this method; here you can define whatever rocket science
+  # your task needs to perform!
+  def run
+    # ...do whatever...
+  end
+end
+
+# We're omitting the other task definitions for now; you shouldn't!
+```
+
 ## Motivation
 
 The main reason for Pallet's existence was the need of a fast, simple and reliable workflow engine, one that is easily extensible with various backends and serializer, one that does not lose your data and one that is intelligent enough to concurrently schedule a workflow's tasks.

data/examples/config_savvy.rb

@@ -26,8 +26,8 @@ Pallets.configure do |c|
 end
 
 class ConfigSavvy < Pallets::Workflow
-  task :volatile
-  task :success => :volatile
+  task 'Volatile'
+  task 'Success' => 'Volatile'
 end
 
 class Volatile < Pallets::Task

data/examples/do_groceries.rb

@@ -1,22 +1,17 @@
 require 'pallets'
 
-Pallets.configure do |c|
-  c.backend_args = { url: 'redis://127.0.0.1:6379/13' }
-end
-
 class DoGroceries < Pallets::Workflow
-  task :enter_shop
-  task :get_shopping_cart => :enter_shop
-  task :put_milk => :get_shopping_cart
-  task :put_bread => :get_shopping_cart
-  task :pay => [:put_milk, :put_bread]
-  task :go_home => :pay
+  task 'EnterShop'
+  task 'GetShoppingCart' => 'EnterShop'
+  task 'PutMilk' => 'GetShoppingCart'
+  task 'PutBread' => 'GetShoppingCart'
+  task 'Pay' => ['PutMilk', 'PutBread']
+  task 'GoHome' => 'Pay'
 end
 
 class EnterShop < Pallets::Task
   def run
     puts "Entering #{context['shop_name']}"
-    raise 'Cannot enter shop!' if (context['i'].to_i % 10).zero?
   end
 end
 
@@ -36,7 +31,6 @@ end
 
 class PutBread < Pallets::Task
   def run
-    raise 'Out of bread!' if (context['i'].to_i % 30).zero?
     puts "Got the bread"
   end
 end
@@ -45,7 +39,6 @@ class Pay < Pallets::Task
   def run
     puts "Paying by #{context['pay_by']}"
     sleep 2
-    raise 'Payment failed!' if (context['i'].to_i % 100).zero?
   end
 end
 
@@ -59,4 +52,4 @@ class GoHome < Pallets::Task
   end
 end
 
-1_000.times { |i| DoGroceries.new(shop_name: 'Pallet Shop', pay_by: :card, i: i).run }
+DoGroceries.new(shop_name: 'Pallet Shop', pay_by: :card).run

data/examples/hello_world.rb

@@ -1,7 +1,7 @@
 require 'pallets'
 
 class HelloWorld < Pallets::Workflow
-  task :echo
+  task 'Echo'
 end
 
 class Echo < Pallets::Task

data/lib/pallets.rb

@@ -34,7 +34,6 @@ module Pallets
     @backend ||= begin
       cls = Pallets::Util.constantize("Pallets::Backends::#{configuration.backend.capitalize}")
       cls.new(
-        namespace: configuration.namespace,
         blocking_timeout: configuration.blocking_timeout,
         failed_job_lifespan: configuration.failed_job_lifespan,
         job_timeout: configuration.job_timeout,

data/lib/pallets/backends/redis.rb

@@ -3,50 +3,49 @@ require 'redis'
 module Pallets
   module Backends
     class Redis < Base
-      def initialize(namespace:, blocking_timeout:, failed_job_lifespan:, job_timeout:, pool_size:, **options)
-        @namespace = namespace
+      QUEUE_KEY = 'queue'
+      RELIABILITY_QUEUE_KEY = 'reliability-queue'
+      RELIABILITY_SET_KEY = 'reliability-set'
+      RETRY_SET_KEY = 'retry-set'
+      GIVEN_UP_SET_KEY = 'given-up-set'
+      WORKFLOW_QUEUE_KEY = 'workflow-queue:%s'
+      CONTEXT_KEY = 'context:%s'
+      REMAINING_KEY = 'remaining:%s'
+
+      def initialize(blocking_timeout:, failed_job_lifespan:, job_timeout:, pool_size:, **options)
         @blocking_timeout = blocking_timeout
         @failed_job_lifespan = failed_job_lifespan
         @job_timeout = job_timeout
         @pool = Pallets::Pool.new(pool_size) { ::Redis.new(options) }
 
-        @queue_key = "#{namespace}:queue"
-        @reliability_queue_key = "#{namespace}:reliability-queue"
-        @reliability_set_key = "#{namespace}:reliability-set"
-        @retry_set_key = "#{namespace}:retry-set"
-        @given_up_set_key = "#{namespace}:given-up-set"
-        @workflow_key = "#{namespace}:workflows:%s"
-        @context_key = "#{namespace}:contexts:%s"
-        @eta_key = "#{namespace}:etas:%s"
-
         register_scripts
       end
 
       def pick
         @pool.execute do |client|
-          job = client.brpoplpush(@queue_key, @reliability_queue_key, timeout: @blocking_timeout)
+          job = client.brpoplpush(QUEUE_KEY, RELIABILITY_QUEUE_KEY, timeout: @blocking_timeout)
           if job
             # We store the job's timeout so we know when to retry jobs that are
             # still on the reliability queue. We do this separately since there is
             # no other way to atomically BRPOPLPUSH from the main queue to a
             # sorted set
-            client.zadd(@reliability_set_key, Time.now.to_f + @job_timeout, job)
+            client.zadd(RELIABILITY_SET_KEY, Time.now.to_f + @job_timeout, job)
           end
           job
         end
       end
 
-      def get_context(workflow_id)
+      def get_context(wfid)
         @pool.execute do |client|
-          client.hgetall(@context_key % workflow_id)
+          client.hgetall(CONTEXT_KEY % wfid)
         end
       end
 
-      def save(workflow_id, job, context_buffer)
+      def save(wfid, job, context_buffer)
         @pool.execute do |client|
-          client.eval(
+          client.evalsha(
             @scripts['save'],
-            [@workflow_key % workflow_id, @queue_key, @reliability_queue_key, @reliability_set_key, @context_key % workflow_id, @eta_key % workflow_id],
+            [WORKFLOW_QUEUE_KEY % wfid, QUEUE_KEY, RELIABILITY_QUEUE_KEY, RELIABILITY_SET_KEY, CONTEXT_KEY % wfid, REMAINING_KEY % wfid],
             context_buffer.to_a << job
           )
         end
@@ -54,9 +53,9 @@ module Pallets
 
       def retry(job, old_job, at)
         @pool.execute do |client|
-          client.eval(
+          client.evalsha(
             @scripts['retry'],
-            [@retry_set_key, @reliability_queue_key, @reliability_set_key],
+            [RETRY_SET_KEY, RELIABILITY_QUEUE_KEY, RELIABILITY_SET_KEY],
             [at, job, old_job]
           )
         end
@@ -64,9 +63,9 @@ module Pallets
 
       def give_up(job, old_job)
         @pool.execute do |client|
-          client.eval(
+          client.evalsha(
             @scripts['give_up'],
-            [@given_up_set_key, @reliability_queue_key, @reliability_set_key],
+            [GIVEN_UP_SET_KEY, RELIABILITY_QUEUE_KEY, RELIABILITY_SET_KEY],
             [Time.now.to_f, job, old_job, Time.now.to_f - @failed_job_lifespan]
           )
         end
@@ -74,23 +73,23 @@ module Pallets
 
       def reschedule_all(earlier_than)
         @pool.execute do |client|
-          client.eval(
+          client.evalsha(
             @scripts['reschedule_all'],
-            [@reliability_set_key, @reliability_queue_key, @retry_set_key, @queue_key],
+            [RELIABILITY_SET_KEY, RELIABILITY_QUEUE_KEY, RETRY_SET_KEY, QUEUE_KEY],
             [earlier_than]
           )
         end
       end
 
-      def run_workflow(workflow_id, jobs_with_order, context)
+      def run_workflow(wfid, jobs_with_order, context_buffer)
         @pool.execute do |client|
           client.multi do
-            client.eval(
+            client.evalsha(
               @scripts['run_workflow'],
-              [@workflow_key % workflow_id, @queue_key, @eta_key % workflow_id],
+              [WORKFLOW_QUEUE_KEY % wfid, QUEUE_KEY, REMAINING_KEY % wfid],
               jobs_with_order
             )
-            client.hmset(@context_key % workflow_id, *context.to_a) unless context.empty?
+            client.hmset(CONTEXT_KEY % wfid, *context_buffer.to_a) unless context_buffer.empty?
           end
         end
       end
@@ -98,11 +97,14 @@ module Pallets
       private
 
       def register_scripts
-        @scripts ||= Dir["#{__dir__}/scripts/*.lua"].map do |file|
-          name = File.basename(file, '.lua')
-          script = File.read(file)
-          [name, script]
-        end.to_h
+        @scripts ||= @pool.execute do |client|
+          Dir["#{__dir__}/scripts/*.lua"].map do |file|
+            name = File.basename(file, '.lua')
+            script = File.read(file)
+            sha = client.script(:load, script)
+            [name, sha]
+          end.to_h
+        end
       end
     end
   end

data/lib/pallets/cli.rb

@@ -65,10 +65,6 @@ module Pallets
           Pallets.configuration.failed_job_lifespan = failed_job_lifespan
         end
 
-        opts.on('-n', '--namespace NAME', 'Namespace to use for backend') do |namespace|
-          Pallets.configuration.namespace = namespace
-        end
-
         opts.on('-p', '--pool-size NUM', Integer, 'Size of backend pool') do |pool_size|
           Pallets.configuration.pool_size = pool_size
         end
@@ -77,8 +73,12 @@ module Pallets
           Pallets.logger.level = Logger::ERROR
         end
 
-        opts.on('-r', '--require PATH', 'Path containing workflow definitions') do |path|
-          require(path)
+        opts.on('-r', '--require PATH', 'Path to file containing workflow definitions or directory containing Rails application') do |path|
+          if File.directory?(path)
+            require File.expand_path("#{path}/config/environment.rb")
+          else
+            require(path)
+          end
         end
 
         opts.on('-s', '--serializer NAME', 'Serializer to use') do |serializer|

data/lib/pallets/configuration.rb

@@ -24,11 +24,8 @@ module Pallets
     # per task basis
     attr_accessor :max_failures
 
-    # Namespace used by the backend to store information
-    attr_accessor :namespace
-
     # Number of connections to the backend
-    attr_accessor :pool_size
+    attr_writer :pool_size
 
     # Serializer used for jobs
     attr_accessor :serializer
@@ -41,9 +38,11 @@ module Pallets
       @failed_job_lifespan = 7_776_000 # 3 months
       @job_timeout = 1_800 # 30 minutes
       @max_failures = 3
-      @namespace = 'pallets'
-      @pool_size = 5
       @serializer = :json
     end
+
+    def pool_size
+      @pool_size || @concurrency + 1
+    end
   end
 end

data/lib/pallets/context.rb

@@ -1,11 +1,17 @@
 module Pallets
   # Hash-like class that additionally holds a buffer for all write operations
+  # that occur after initialization
   class Context < Hash
     def []=(key, value)
       buffer[key] = value
       super
     end
 
+    def merge!(other_hash)
+      buffer.merge!(other_hash)
+      super
+    end
+
     def buffer
       @buffer ||= {}
     end

data/lib/pallets/dsl/workflow.rb

@@ -1,29 +1,23 @@
 module Pallets
   module DSL
     module Workflow
-      def task(*args, **options, &block)
-        name, depends_on = if args.any?
-          [args.first, options[:depends_on]]
+      def task(arg, depends_on: nil, max_failures: nil, &block)
+        klass, dependencies = case arg
+        when Hash
+          # The `task Foo => Bar` notation
+          arg.first
         else
-          options.first
+          # The `task Foo, depends_on: Bar` notation
+          [arg, depends_on]
         end
 
-        unless name
-          raise WorkflowError, "Task has no name. Provide a name using " \
-                               "`task :name, *args` or `task name: :arg` syntax"
-        end
-
-        # Handle nils, symbols or arrays consistently
-        name = name.to_sym
-        dependencies = Array(depends_on).compact.map(&:to_sym)
-        graph.add(name, dependencies)
-
-        class_name = options[:class_name] || Pallets::Util.camelize(name)
-        max_failures = options[:max_failures] || Pallets.configuration.max_failures
+        task_class = klass.to_s
+        dependencies = Array(dependencies).compact.uniq.map(&:to_s)
+        graph.add(task_class, dependencies)
 
-        task_config[name] = {
-          'class_name' => class_name,
-          'max_failures' => max_failures
+        task_config[task_class] = {
+          'task_class' => task_class,
+          'max_failures' => max_failures || Pallets.configuration.max_failures
         }
 
         nil

data/lib/pallets/graph.rb

@@ -43,6 +43,8 @@ module Pallets
 
     def tsort_each_child(node, &block)
       @nodes.fetch(node).each(&block)
+    rescue KeyError
+      raise WorkflowError, "Task #{node} is marked as a dependency but not defined"
     end
   end
 end

data/lib/pallets/serializers/base.rb

@@ -8,6 +8,18 @@ module Pallets
       def load(data)
         raise NotImplementedError
       end
+
+      alias_method :dump_job, :dump
+      alias_method :load_job, :load
+
+      # Context hashes only need their values (de)serialized
+      def dump_context(data)
+        data.map { |k, v| [k.to_s, dump(v)] }.to_h
+      end
+
+      def load_context(data)
+        data.map { |k, v| [k, load(v)] }.to_h
+      end
     end
   end
 end

data/lib/pallets/serializers/json.rb

@@ -2,13 +2,14 @@ require 'json'
 
 module Pallets
   module Serializers
-    class Json
+    class Json < Base
       def dump(data)
-        JSON.generate(data)
+        # TODO: Remove option after dropping support for Ruby 2.3
+        JSON.generate(data, quirks_mode: true)
       end
 
       def load(data)
-        JSON.parse(data)
+        JSON.parse(data, quirks_mode: true)
       end
     end
   end

data/lib/pallets/serializers/msgpack.rb

@@ -2,7 +2,7 @@ require 'msgpack'
 
 module Pallets
   module Serializers
-    class Msgpack
+    class Msgpack < Base
       def dump(data)
         MessagePack.pack(data)
       end

data/lib/pallets/util.rb

@@ -2,8 +2,10 @@ module Pallets
   module Util
     extend self
 
-    def camelize(str)
-      str.to_s.gsub(/(?:^|_)([a-z])/) { $1.upcase }
+    def generate_id(str)
+      initials = str.gsub(/[^A-Z]+([A-Z])/, '\1')[0,3]
+      random = SecureRandom.hex(5)
+      "#{initials}#{random}"
     end
 
     def constantize(str)

data/lib/pallets/version.rb

@@ -1,3 +1,3 @@
 module Pallets
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

data/lib/pallets/worker.rb

@@ -71,16 +71,22 @@ module Pallets
 
       Pallets.logger.info "Started", extract_metadata(job_hash)
 
-      context = Context[backend.get_context(job_hash['workflow_id'])]
+      context = Context[
+        serializer.load_context(backend.get_context(job_hash['wfid']))
+      ]
 
-      task_class = Pallets::Util.constantize(job_hash["class_name"])
+      task_class = Pallets::Util.constantize(job_hash["task_class"])
       task = task_class.new(context)
       begin
-        task.run
+        task_result = task.run
       rescue => ex
         handle_job_error(ex, job, job_hash)
       else
-        handle_job_success(context, job, job_hash)
+        if task_result == false
+          handle_job_return_false(job, job_hash)
+        else
+          handle_job_success(context, job, job_hash)
+        end
       end
     end
 
@@ -90,9 +96,10 @@ module Pallets
       failures = job_hash.fetch('failures', 0) + 1
       new_job = serializer.dump(job_hash.merge(
         'failures' => failures,
-        'failed_at' => Time.now.to_f,
+        'given_up_at' => Time.now.to_f,
         'error_class' => ex.class.name,
-        'error_message' => ex.message
+        'error_message' => ex.message,
+        'reason' => 'error'
       ))
       if failures < job_hash['max_failures']
         retry_at = Time.now.to_f + backoff_in_seconds(failures)
@@ -103,22 +110,32 @@ module Pallets
       end
     end
 
+    def handle_job_return_false(job, job_hash)
+      new_job = serializer.dump(job_hash.merge(
+        'given_up_at' => Time.now.to_f,
+        'reason' => 'returned_false'
+      ))
+      backend.give_up(new_job, job)
+      Pallets.logger.info "Gave up after returning false", extract_metadata(job_hash)
+    end
+
     def handle_job_success(context, job, job_hash)
-      backend.save(job_hash['workflow_id'], job, context.buffer)
+      backend.save(job_hash['wfid'], job, serializer.dump_context(context.buffer))
       Pallets.logger.info "Done", extract_metadata(job_hash)
     end
 
     def extract_metadata(job_hash)
       {
         wid:  id,
-        wfid: job_hash['workflow_id'],
-        wf:   job_hash['workflow_class_name'],
-        tsk:  job_hash['class_name']
+        wfid: job_hash['wfid'],
+        jid: job_hash['jid'],
+        wf:   job_hash['workflow_class'],
+        tsk:  job_hash['task_class']
       }
     end
 
     def backoff_in_seconds(count)
-      count ** 4 + 6
+      count ** 4 + rand(6..10)
     end
 
     def backend

data/lib/pallets/workflow.rb

@@ -4,42 +4,40 @@ module Pallets
 
     attr_reader :context
 
-    def initialize(context = {})
+    def initialize(context_hash = {})
       @id = nil
-      @context = context
+      # Passed in context hash needs to be buffered
+      @context = Context.new.merge!(context_hash)
     end
 
     def run
       raise WorkflowError, "#{self.class.name} has no tasks. Workflows "\
                            "must contain at least one task" if self.class.graph.empty?
 
-      backend.run_workflow(id, jobs_with_order, context)
+      backend.run_workflow(id, jobs_with_order, serializer.dump_context(context.buffer))
       id
     end
 
     def id
-      @id ||= begin
-        initials = self.class.name.gsub(/[^A-Z]+([A-Z])/, '\1')[0,3]
-        random = SecureRandom.hex(5)
-        "P#{initials}#{random}".upcase
-      end
+      @id ||= "P#{Pallets::Util.generate_id(self.class.name)}".upcase
     end
 
     private
 
     def jobs_with_order
-      self.class.graph.sorted_with_order.map do |task_name, order|
-        job = serializer.dump(job_hash.merge(self.class.task_config[task_name]))
+      self.class.graph.sorted_with_order.map do |task_class, order|
+        job = serializer.dump(construct_job(task_class))
         [order, job]
       end
     end
 
-    def job_hash
-      {
-        'workflow_id'         => id,
-        'workflow_class_name' => self.class.name,
-        'created_at'          => Time.now.to_f
-      }
+    def construct_job(task_class)
+      {}.tap do |job|
+        job['wfid'] = id
+        job['jid'] = "J#{Pallets::Util.generate_id(task_class)}".upcase
+        job['workflow_class'] = self.class.name
+        job['created_at'] = Time.now.to_f
+      end.merge(self.class.task_config[task_class])
     end
 
     def backend

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pallets
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Andrei Horak
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-02-08 00:00:00.000000000 Z
+date: 2019-04-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -106,7 +106,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 2.5.2.3
 signing_key: 
 specification_version: 4
 summary: Toy workflow engine, written in Ruby