taskinator 0.5.0 → 0.5.2

data/Gemfile.lock

@@ -1,36 +1,35 @@
 PATH
   remote: .
   specs:
-    taskinator (0.5.0)
+    taskinator (0.5.2)
       builder (>= 3.2.2)
       connection_pool (>= 2.2.0)
-      globalid (~> 0.3)
+      globalid (>= 0.3)
       json (>= 1.8.2)
       redis (>= 3.2.1)
       redis-namespace (>= 1.5.2)
-      statsd-ruby (~> 1.4.0)
-      thwait (~> 0.2)
+      thwait (>= 0.2)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    actionpack (5.2.6.2)
-      actionview (= 5.2.6.2)
-      activesupport (= 5.2.6.2)
+    actionpack (5.2.8.1)
+      actionview (= 5.2.8.1)
+      activesupport (= 5.2.8.1)
       rack (~> 2.0, >= 2.0.8)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.0.2)
-    actionview (5.2.6.2)
-      activesupport (= 5.2.6.2)
+    actionview (5.2.8.1)
+      activesupport (= 5.2.8.1)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.0.3)
-    activejob (5.2.6.2)
-      activesupport (= 5.2.6.2)
+    activejob (5.2.8.1)
+      activesupport (= 5.2.8.1)
       globalid (>= 0.3.6)
-    activesupport (5.2.6.2)
+    activesupport (5.2.8.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
@@ -38,8 +37,8 @@ GEM
     builder (3.2.4)
     byebug (11.1.3)
     coderay (1.1.3)
-    concurrent-ruby (1.1.9)
-    connection_pool (2.2.5)
+    concurrent-ruby (1.1.10)
+    connection_pool (2.3.0)
     coveralls (0.8.23)
       json (>= 1.8, < 3)
       simplecov (~> 0.16.1)
@@ -47,83 +46,82 @@ GEM
       thor (>= 0.19.4, < 2.0)
       tins (~> 1.6)
     crass (1.0.6)
-    delayed_job (4.1.10)
+    delayed_job (4.1.11)
       activesupport (>= 3.0, < 8.0)
     diff-lcs (1.5.0)
     docile (1.4.0)
     e2mmap (0.1.0)
-    erubi (1.10.0)
+    erubi (1.12.0)
     fakeredis (0.7.0)
       redis (>= 3.2, < 5.0)
-    globalid (0.6.0)
+    globalid (1.0.0)
       activesupport (>= 5.0)
-    i18n (1.9.1)
+    i18n (1.12.0)
       concurrent-ruby (~> 1.0)
-    json (2.6.1)
-    loofah (2.14.0)
+    json (2.6.3)
+    loofah (2.19.1)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     method_source (1.0.0)
-    mini_portile2 (2.7.1)
-    minitest (5.15.0)
+    mini_portile2 (2.8.1)
+    minitest (5.17.0)
     mono_logger (1.1.1)
     multi_json (1.15.0)
-    mustermann (1.1.1)
+    mustermann (3.0.0)
       ruby2_keywords (~> 0.0.1)
-    nokogiri (1.13.1)
-      mini_portile2 (~> 2.7.0)
+    nokogiri (1.13.10)
+      mini_portile2 (~> 2.8.0)
       racc (~> 1.4)
-    pry (0.13.1)
+    pry (0.14.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    pry-byebug (3.9.0)
+    pry-byebug (3.10.1)
       byebug (~> 11.0)
-      pry (~> 0.13.0)
-    racc (1.6.0)
-    rack (2.2.3)
-    rack-protection (2.1.0)
+      pry (>= 0.13, < 0.15)
+    racc (1.6.2)
+    rack (2.2.5)
+    rack-protection (3.0.5)
       rack
-    rack-test (1.1.0)
-      rack (>= 1.0, < 3)
+    rack-test (2.0.2)
+      rack (>= 1.3)
     rails-dom-testing (2.0.3)
       activesupport (>= 4.2.0)
       nokogiri (>= 1.6)
-    rails-html-sanitizer (1.4.2)
-      loofah (~> 2.3)
-    railties (5.2.6.2)
-      actionpack (= 5.2.6.2)
-      activesupport (= 5.2.6.2)
+    rails-html-sanitizer (1.4.4)
+      loofah (~> 2.19, >= 2.19.1)
+    railties (5.2.8.1)
+      actionpack (= 5.2.8.1)
+      activesupport (= 5.2.8.1)
       method_source
       rake (>= 0.8.7)
       thor (>= 0.19.0, < 2.0)
     rake (13.0.6)
-    redis (4.6.0)
-    redis-namespace (1.8.1)
-      redis (>= 3.0.4)
-    resque (2.2.0)
+    redis (4.8.0)
+    redis-namespace (1.10.0)
+      redis (>= 4)
+    resque (2.4.0)
       mono_logger (~> 1.0)
       multi_json (~> 1.0)
       redis-namespace (~> 1.6)
       sinatra (>= 0.9.2)
-      vegas (~> 0.1.2)
     resque_spec (0.18.1)
       resque (>= 1.26.0)
       rspec-core (>= 3.0.0)
       rspec-expectations (>= 3.0.0)
       rspec-mocks (>= 3.0.0)
-    rspec (3.11.0)
-      rspec-core (~> 3.11.0)
-      rspec-expectations (~> 3.11.0)
-      rspec-mocks (~> 3.11.0)
-    rspec-core (3.11.0)
-      rspec-support (~> 3.11.0)
-    rspec-expectations (3.11.0)
+    rspec (3.12.0)
+      rspec-core (~> 3.12.0)
+      rspec-expectations (~> 3.12.0)
+      rspec-mocks (~> 3.12.0)
+    rspec-core (3.12.0)
+      rspec-support (~> 3.12.0)
+    rspec-expectations (3.12.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.11.0)
-    rspec-mocks (3.11.0)
+      rspec-support (~> 3.12.0)
+    rspec-mocks (3.12.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.11.0)
-    rspec-rails (5.1.0)
+      rspec-support (~> 3.12.0)
+    rspec-rails (5.1.2)
       actionpack (>= 5.2)
       activesupport (>= 5.2)
       railties (>= 5.2)
@@ -134,23 +132,22 @@ GEM
     rspec-sidekiq (3.1.0)
       rspec-core (~> 3.0, >= 3.0.0)
       sidekiq (>= 2.4.0)
-    rspec-support (3.11.0)
+    rspec-support (3.12.0)
     ruby2_keywords (0.0.5)
-    sidekiq (6.4.1)
-      connection_pool (>= 2.2.2)
+    sidekiq (6.5.8)
+      connection_pool (>= 2.2.5, < 3)
       rack (~> 2.0)
-      redis (>= 4.2.0)
+      redis (>= 4.5.0, < 5)
     simplecov (0.16.1)
       docile (~> 1.1)
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
-    sinatra (2.1.0)
-      mustermann (~> 1.0)
-      rack (~> 2.2)
-      rack-protection (= 2.1.0)
+    sinatra (3.0.5)
+      mustermann (~> 3.0)
+      rack (~> 2.2, >= 2.2.4)
+      rack-protection (= 3.0.5)
       tilt (~> 2.0)
-    statsd-ruby (1.4.0)
     sync (0.5.0)
     term-ansicolor (1.7.1)
       tins (~> 1.0)
@@ -158,13 +155,11 @@ GEM
     thread_safe (0.3.6)
     thwait (0.2.0)
       e2mmap
-    tilt (2.0.10)
-    tins (1.31.0)
+    tilt (2.0.11)
+    tins (1.32.1)
       sync
-    tzinfo (1.2.9)
+    tzinfo (1.2.10)
       thread_safe (~> 0.1)
-    vegas (0.1.11)
-      rack (>= 1.0.0)
 
 PLATFORMS
   ruby
@@ -184,7 +179,7 @@ DEPENDENCIES
   rspec
   rspec-rails (>= 2.0)
   rspec-sidekiq (>= 2.1.0)
-  sidekiq (>= 3.5.0)
+  sidekiq (>= 3.5.0, < 7.0.0)
   taskinator!
 
 BUNDLED WITH

data/README.md

@@ -1,7 +1,7 @@
 # Taskinator
 
 [![Gem Version](https://badge.fury.io/rb/taskinator.svg)](http://badge.fury.io/rb/taskinator)
-[![Build Status](https://img.shields.io/github/workflow/status/virtualstaticvoid/taskinator/Taskinator?style=flat-square)](https://github.com/virtualstaticvoid/taskinator/actions)
+[![Build Status](https://github.com/virtualstaticvoid/taskinator/actions/workflows/build.yml/badge.svg)](https://github.com/virtualstaticvoid/taskinator/actions/workflows/build.yml)
 [![Code Climate](https://codeclimate.com/github/virtualstaticvoid/taskinator.png)](https://codeclimate.com/github/virtualstaticvoid/taskinator)
 
 A simple orchestration library for running complex processes or workflows in Ruby.
@@ -53,8 +53,8 @@ If you are using Taskinator within a Rails application, then add an initializer,
 Taskinator.configure do |config|
 
   # configure the queue adapter to use
-  # can be :active_job, :delayed_job, :redis or :sidekiq
-  config.queue_adapter = :redis
+  # can be :active_job, :delayed_job, :resque or :sidekiq
+  config.queue_adapter = :resque
 
   # configure redis
   config.redis = {
@@ -171,39 +171,6 @@ module MyProcess
 end
 ```
 
-#### Reusing ActiveJob jobs
-
-It is likely that you already have one or more [jobs](https://guides.rubyonrails.org/active_job_basics.html)
-and want to reuse them within the process definition.
-
-Define a `job` step, providing the class of the Active Job to run and then taskinator will
-invoke that job as part of the process.
-
-The `job` step will be queued and executed on same queue as
-[configured by the job](https://guides.rubyonrails.org/active_job_basics.html#queues).
-
-```ruby
-# E.g. A resque worker
-class DoSomeWork
-  queue :high_priority
-
-  def self.perform(arg1, arg2)
-    # code to do the work
-  end
-end
-
-module MyProcess
-  extend Taskinator::Definition
-
-  # when creating the process, supply the same arguments
-  # that the DoSomeWork worker expects
-
-  define_process do
-    job DoSomeWork
-  end
-end
-```
-
 #### Data Driven Process Definitions
 
 You can also define data driven tasks using the `for_each` method, which takes an iterator method
@@ -271,7 +238,7 @@ process2 = MyProcess.create_process
 process2.tasks.count #=> 1
 ```
 
-#### Transformations
+#### Argument Transformations
 
 In addition, it is possible to transform the arguments used by a task or job, by including
 a `transform` step in the definition.
@@ -396,16 +363,54 @@ MyProcess.create_process(1, 2, 3, :send_notification => true)
 
 ```
 
+#### Reusing ActiveJob jobs
+
+It is likely that you already have one or more [jobs](https://guides.rubyonrails.org/active_job_basics.html)
+and want to reuse them within the process definition.
+
+Define a `job` step, providing the class of the Active Job to run and then taskinator will
+invoke that job as part of the process.
+
+The `job` step will be queued and executed on same queue as
+[configured by the job](https://guides.rubyonrails.org/active_job_basics.html#queues).
+
+```ruby
+# E.g. A resque worker
+class DoSomeWork
+  queue :high_priority
+
+  def self.perform(arg1, arg2)
+    # code to do the work
+  end
+end
+
+module MyProcess
+  extend Taskinator::Definition
+
+  # when creating the process, supply the same arguments
+  # that the DoSomeWork worker expects
+
+  define_process do
+    job DoSomeWork
+  end
+end
+```
+
 ### Execution
 
-A process is executed by calling the generated `create_process` method on your "process" module.
+A process is created by calling the generated `create_process` method on your "process" module.
 
 ```ruby
 process = MyProcess.create_process
+```
+
+And then enqueued for execution by calling the `enqueue!` method of the process.
+
+```ruby
 process.enqueue!
 ```
 
-Or, to start immediately, call the `start!` method.
+Or, started immediately by calling the `start!` method of the process.
 
 ```ruby
 process = MyProcess.create_process
@@ -637,6 +642,7 @@ process_id = "SUPPLY-PROCESS-IDENTIFIER"
 process = Taskinator::Api.find_process(process_id)
 
 puts process.inspect
+puts process.definition
 puts process.current_state
 puts process.tasks
 # etc...
@@ -655,6 +661,7 @@ task = Taskinator::Api.find_task(task_id)
 
 puts task.inspect
 puts task.class
+puts task.definition
 puts task.args                # for Step and Job types
 puts task.sub_process.tasks   # for SubProcess type
 # etc...
@@ -708,7 +715,7 @@ To configure the queue adapter to use, set `config.queue_adapter` to one of the
 
 * `:active_job`
 * `:delayed_job`
-* `:redis`
+* `:resque`
 * `:sidekiq`
 
 As follows:
@@ -717,8 +724,8 @@ As follows:
 Taskinator.configure do |config|
 
   # configure the queue adapter to use
-  # can be :active_job, :delayed_job, :redis or :sidekiq
-  config.queue_adapter = :redis
+  # can be :active_job, :delayed_job, :resque or :sidekiq
+  config.queue_adapter = :resque
 
 end
 ```
@@ -787,6 +794,7 @@ For all events, the data included contains the following information:
 | Key                             | Value                                                    |
 |---------------------------------|----------------------------------------------------------|
 | `:type`                         | The type name of the component reporting the event       |
+| `:definition`                   | The type name of the process definition                  |
 | `:process_uuid`                 | The UUID of the root process                             |
 | `:process_options`              | Options hash of the root process                         |
 | `:uuid`                         | The UUID of the respective task, job or sub process      |

data/lib/taskinator/builder.rb

@@ -0,0 +1,129 @@
+module Taskinator
+  class Builder
+
+    attr_reader :process
+    attr_reader :definition
+    attr_reader :args
+    attr_reader :builder_options
+
+    def initialize(process, definition, *args)
+      @process = process
+      @definition = definition
+      @builder_options = args.last.is_a?(Hash) ? args.pop : {}
+      @args = args
+      @executor = Taskinator::Executor.new(@definition)
+    end
+
+    def option?(key, &block)
+      yield if builder_options[key]
+    end
+
+    # defines a sub process of tasks which are executed sequentially
+    def sequential(options={}, &block)
+      raise ArgumentError, 'block' unless block_given?
+
+      sub_process = Process.define_sequential_process_for(@definition, options)
+      task = define_sub_process_task(@process, sub_process, options)
+      Builder.new(sub_process, @definition, *@args).instance_eval(&block)
+      @process.tasks << task if sub_process.tasks.any?
+      nil
+    end
+
+    # defines a sub process of tasks which are executed concurrently
+    def concurrent(complete_on=CompleteOn::Default, options={}, &block)
+      raise ArgumentError, 'block' unless block_given?
+
+      sub_process = Process.define_concurrent_process_for(@definition, complete_on, options)
+      task = define_sub_process_task(@process, sub_process, options)
+      Builder.new(sub_process, @definition, *@args).instance_eval(&block)
+      @process.tasks << task if sub_process.tasks.any?
+      nil
+    end
+
+    # dynamically defines tasks, using the given @iterator method
+    # the definition will be evaluated for each yielded item
+    def for_each(method, options={}, &block)
+      raise ArgumentError, 'method' if method.nil?
+      raise NoMethodError, method unless @executor.respond_to?(method)
+      raise ArgumentError, 'block' unless block_given?
+
+      #
+      # `for_each` is an exception, since it invokes the definition
+      # in order to yield elements to the builder, and any options passed
+      # are included with the builder options
+      #
+      method_args = options.any? ? [*@args, options] : @args
+      @executor.send(method, *method_args) do |*args|
+        Builder.new(@process, @definition, *args).instance_eval(&block)
+      end
+      nil
+    end
+
+    alias_method :transform, :for_each
+
+    # defines a task which executes the given @method
+    def task(method, options={})
+      raise ArgumentError, 'method' if method.nil?
+      raise NoMethodError, method unless @executor.respond_to?(method)
+
+      define_step_task(@process, method, @args, options)
+      nil
+    end
+
+    # defines a task which executes the given @job
+    # which is expected to implement a perform method either as a class or instance method
+    def job(job, options={})
+      raise ArgumentError, 'job' if job.nil?
+      raise ArgumentError, 'job' unless job.methods.include?(:perform) || job.instance_methods.include?(:perform)
+
+      define_job_task(@process, job, @args, options)
+      nil
+    end
+
+    # TODO: add mailer
+    # TODO: add complete!
+    # TODO: add fail!
+
+    # defines a sub process task, for the given @definition
+    # the definition specified must have input compatible arguments
+    # to the current definition
+    def sub_process(definition, options={})
+      raise ArgumentError, 'definition' if definition.nil?
+      raise ArgumentError, "#{definition.name} does not extend the #{Definition.name} module" unless definition.kind_of?(Definition)
+
+      sub_process = definition.create_sub_process(*@args, combine_options(options))
+      task = define_sub_process_task(@process, sub_process, options)
+      Builder.new(sub_process, definition, *@args)
+      @process.tasks << task if sub_process.tasks.any?
+      nil
+    end
+
+  private
+
+    def define_step_task(process, method, args, options={})
+      define_task(process) {
+        Task.define_step_task(process, method, args, combine_options(options))
+      }
+    end
+
+    def define_job_task(process, job, args, options={})
+      define_task(process) {
+        Task.define_job_task(process, job, args, combine_options(options))
+      }
+    end
+
+    def define_sub_process_task(process, sub_process, options={})
+      Task.define_sub_process_task(process, sub_process, combine_options(options))
+    end
+
+    def define_task(process)
+      process.tasks << task = yield
+      task
+    end
+
+    def combine_options(options={})
+      builder_options.merge(options)
+    end
+
+  end
+end

data/lib/taskinator/definition.rb

@@ -1,3 +1,5 @@
+require 'taskinator/builder'
+
 module Taskinator
   module Definition
 
@@ -120,5 +122,3 @@ module Taskinator
 
   end
 end
-
-require 'taskinator/definition/builder'

data/lib/taskinator/instrumentation.rb

@@ -58,6 +58,7 @@ module Taskinator
       return OpenStruct.new(
         {
           :type                   => self.class.name,
+          :definition             => self.definition.name,
           :process_uuid           => process_uuid,
           :process_options        => process_options.dup,
           :uuid                   => uuid,

data/lib/taskinator/persistence.rb

@@ -1,3 +1,5 @@
+require 'builder'
+
 module Taskinator
   module Persistence
 
@@ -113,13 +115,13 @@ module Taskinator
             transaction.hmset(
               self.key,
               :state, new_state,
-              :updated_at, @updated_at
+              :updated_at, @updated_at.iso8601(3)
             )
 
             # also update the "root" process
             transaction.hset(
               process_key,
-              :updated_at, @updated_at
+              :updated_at, @updated_at.iso8601(3)
             )
           end
         end
@@ -136,7 +138,7 @@ module Taskinator
             :error_type, error.class.name,
             :error_message, error.message,
             :error_backtrace, JSON.generate(error.backtrace || []),
-            :updated_at, Time.now.utc
+            :updated_at, Time.now.utc.iso8601(3)
           )
         end
       end
@@ -180,7 +182,7 @@ module Taskinator
             process_key = self.process_key
             conn.multi do |transaction|
               transaction.hincrby process_key, "tasks_#{status}", 1
-              transaction.hset process_key, :updated_at, Time.now.utc
+              transaction.hset process_key, :updated_at, Time.now.utc.iso8601(3)
             end
           end
         end
@@ -292,7 +294,8 @@ module Taskinator
       end
 
       def visit_attribute_time(attribute)
-        visit_attribute(attribute)
+        value = @instance.send(attribute)
+        @hmset += [attribute, value.iso8601(3)] if value
       end
 
       def visit_attribute_enum(attribute, type)
@@ -447,6 +450,46 @@ module Taskinator
       end
     end
 
+    class UnknownTypeError < StandardError
+    end
+
+    module UnknownType
+
+      @unknown_types = {}
+
+      def self.new(type)
+        Taskinator.logger.error("Unknown type '#{type}' while deserializing.")
+        @unknown_types[type] ||= Module.new do
+          extend UnknownType
+          @type = type
+
+          # for unknown definitions
+          def method_missing(_, *_)
+            raise UnknownTypeError.new(to_s)
+          end
+        end
+      end
+
+      attr_reader :type
+
+      def to_s
+        "Unknown type '#{type}'."
+      end
+
+      def allocate
+        self
+      end
+
+      def accept(*_)
+        # nothing doing
+      end
+
+      # for unknown job types
+      def perform(*_)
+        raise UnknownTypeError.new(to_s)
+      end
+    end
+
     class RedisDeserializationVisitor < Taskinator::Visitor::Base
 
       #
@@ -480,7 +523,7 @@ module Taskinator
         return unless @attribute_values.key?(:type)
 
         type = @attribute_values[:type]
-        klass = Kernel.const_get(type)
+        klass = Kernel.const_get(type) rescue UnknownType.new(type)
 
         #
         # NOTE:
@@ -539,16 +582,18 @@ module Taskinator
       def visit_attribute_enum(attribute, type)
         visit_attribute(attribute) do |value|
           const_value = type.constants.select {|c| type.const_get(c) == value.to_i }.first
-          const_value ?
-            type.const_get(const_value) :
-            (defined?(type::Default) ? type::Default : nil)
+          if const_value
+            type.const_get(const_value)
+          else
+            defined?(type::Default) ? type::Default : nil
+          end
         end
       end
 
       def visit_type(attribute)
         value = @attribute_values[attribute]
         if value
-          type = Kernel.const_get(value)
+          type = Kernel.const_get(value) rescue UnknownType.new(value)
           @instance.instance_variable_set("@#{attribute}", type)
         end
       end
@@ -576,6 +621,8 @@ module Taskinator
         type = Taskinator.redis do |conn|
           conn.hget(base.key_for(uuid), :type)
         end
+        # these types are always from taskinator
+        # so no need to rescue for unknown types!
         klass = Kernel.const_get(type)
         LazyLoader.new(klass, uuid, @instance_cache)
       end