taskinator 0.4.7 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0ea786056f74987f21643d392336032d09896df7ddc01d462a78a257a19088d8
-  data.tar.gz: f693f14e2dac9341f1c83c6bf04b12da9d96261498198cc712e71be5b4b51955
+  metadata.gz: bad31a71a3faf459776cdd3babe99eeece9bc81f42ab2bea305e7e4663a21d2c
+  data.tar.gz: '09b1def02fbf8232cd4fba28acc1b0145f0fc90bacc6e184690c312e4296a58d'
 SHA512:
-  metadata.gz: 6151322f48c6215c41ffeeb6fc08d43aa871f9f76d04f628c80d29fa423b22fb0a3ee589803efd1503c3b59491bc4e2ace36879f801638898b9ace0557e4f180
-  data.tar.gz: 65ba806d211f62c8ff12ea3c036b037f53567a17f1a848129ba8f737d4fc0267be7435d6e315591afa81527275042bf22cc04066c74684b6cc54b0e3ce8691f8
+  metadata.gz: 5924489454c34a1c2f6769d379aaa7c0a9824fe7337f201403b67580fb512005fac1d06d9bf740666062c33b5fd2ab5c312e503c1f17d689f1a9a083ea7f8556
+  data.tar.gz: 504b475299c8b34b49e981cee9f325593fa2112151c4e582bebb92400025f22e39e5904c17b81f4d3a4caf6a83e942cebfc057cd545edc44200265ba9643bc55

data/.github/workflows/{taskinator.yml → build.yml}

@@ -1,4 +1,4 @@
-name: Taskinator
+name: Build
 
 on:
   push:
@@ -16,7 +16,7 @@ jobs:
           - '2.7'
           - '3.0'
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Ruby
       uses: ruby/setup-ruby@v1
       with:
@@ -26,16 +26,3 @@ jobs:
       run: bundle exec rake build
     - name: Run tests
       run: bundle exec rake spec
-
-  publish:
-    if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-    needs: build
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: Release Gem
-      uses: discourse/publish-rubygems-action@v2
-      env:
-        RUBYGEMS_API_KEY: ${{secrets.RUBYGEMS_API_KEY}}
-        GIT_NAME: Chris Stefano
-        GIT_EMAIL: virtualstaticvoid@gmail.com

data/.github/workflows/release.yml

@@ -0,0 +1,18 @@
+name: Release
+
+on:
+  release:
+    types: [ published ]
+
+jobs:
+  publish:
+    if: ${{ github.event_name == 'release' && github.event.action == 'published' }}
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Release Gem
+      uses: virtualstaticvoid/publish-rubygems-action@v4
+      env:
+        GIT_NAME: "Chris Stefano"
+        GIT_EMAIL: "virtualstaticvoid@gmail.com"
+        RUBYGEMS_API_KEY: ${{secrets.RUBYGEMS_API_KEY}}

data/CHANGELOG.md

@@ -1,3 +1,35 @@
+v?.?.? - ?? ??? ????
+---
+
+v0.5.1 - 06 Jan 2023
+---
+Include process definition in processes, tasks and payloads to aid debugging.
+Increased test coverage for process and task specs.
+Removed `statsd` instrumentation.
+Removed unused `Taskinator::Visitor::XmlVisitor` implementation.
+Various refactorings and clean ups.
+Bug fixes for process class when used as a sub-process
+Add handling for unknown types when deserializing old processes
+Raises `UnknownTypeError` when trying to invoke processes or Jobs of unknown types
+
+v0.5.0 - 18 Feb 2022
+---
+Removed unused `ProcessWorker` class and related queue methods.
+Refactored `TestQueueAdapter` to correctly implement queue adapter for use in specs.
+Added deprecation for `Taskinator::Process::Concurrent#concurrency_method` option.
+
+v0.4.7 - 17 Feb 2022
+---
+Use newer format for `pipelined` and `multi` requests in Redis.
+
+v0.4.6 - 12 Feb 2022
+---
+Upgrade actionpack for [information vulnerability fix](https://github.com/virtualstaticvoid/taskinator/security/dependabot/3).
+
+v0.4.5 - 30 Jan 2022
+---
+Upgrade sidekiq dependency for [CVE-2022-23837](https://github.com/advisories/GHSA-jrfj-98qg-qjgv).
+
 v0.4.4 - 17 Jan 2022
 ---
 Add support for `ActiveJob`.
@@ -24,7 +56,7 @@ Added support for having `perform` method as a class method.
 
 v0.3.16 - 17 Feb 2021
 ---
-Bug fix to deincrement pending counts for sequential tasks.
+Bug fix to decrement pending counts for sequential tasks.
 Bug fix to allow concurrent tasks to be retried (via Resque) and to complete processes.
 
 v0.3.15 - 22 Nov 2018

data/Gemfile

@@ -5,7 +5,7 @@ gemspec
 
 # queues
 gem 'activejob'       , '>= 5.0.0'
-gem 'sidekiq'         , '>= 3.5.0'
+gem 'sidekiq'         , '>= 3.5.0', '< 7.0.0'
 gem 'rspec-sidekiq'   , '>= 2.1.0'
 
 gem 'delayed_job'     , '~> 4.1.0'

data/Gemfile.lock

@@ -1,36 +1,35 @@
 PATH
   remote: .
   specs:
-    taskinator (0.4.7)
+    taskinator (0.5.1)
       builder (>= 3.2.2)
       connection_pool (>= 2.2.0)
-      globalid (~> 0.3)
+      globalid (>= 0.3)
       json (>= 1.8.2)
       redis (>= 3.2.1)
       redis-namespace (>= 1.5.2)
-      statsd-ruby (~> 1.4.0)
-      thwait (~> 0.2)
+      thwait (>= 0.2)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    actionpack (5.2.6.2)
-      actionview (= 5.2.6.2)
-      activesupport (= 5.2.6.2)
+    actionpack (5.2.8.1)
+      actionview (= 5.2.8.1)
+      activesupport (= 5.2.8.1)
       rack (~> 2.0, >= 2.0.8)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.0.2)
-    actionview (5.2.6.2)
-      activesupport (= 5.2.6.2)
+    actionview (5.2.8.1)
+      activesupport (= 5.2.8.1)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.0.3)
-    activejob (5.2.6.2)
-      activesupport (= 5.2.6.2)
+    activejob (5.2.8.1)
+      activesupport (= 5.2.8.1)
       globalid (>= 0.3.6)
-    activesupport (5.2.6.2)
+    activesupport (5.2.8.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
@@ -38,8 +37,8 @@ GEM
     builder (3.2.4)
     byebug (11.1.3)
     coderay (1.1.3)
-    concurrent-ruby (1.1.9)
-    connection_pool (2.2.5)
+    concurrent-ruby (1.1.10)
+    connection_pool (2.3.0)
     coveralls (0.8.23)
       json (>= 1.8, < 3)
       simplecov (~> 0.16.1)
@@ -47,83 +46,82 @@ GEM
       thor (>= 0.19.4, < 2.0)
       tins (~> 1.6)
     crass (1.0.6)
-    delayed_job (4.1.10)
+    delayed_job (4.1.11)
       activesupport (>= 3.0, < 8.0)
     diff-lcs (1.5.0)
     docile (1.4.0)
     e2mmap (0.1.0)
-    erubi (1.10.0)
+    erubi (1.12.0)
     fakeredis (0.7.0)
       redis (>= 3.2, < 5.0)
-    globalid (0.6.0)
+    globalid (1.0.0)
       activesupport (>= 5.0)
-    i18n (1.9.1)
+    i18n (1.12.0)
       concurrent-ruby (~> 1.0)
-    json (2.6.1)
-    loofah (2.14.0)
+    json (2.6.3)
+    loofah (2.19.1)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     method_source (1.0.0)
-    mini_portile2 (2.7.1)
-    minitest (5.15.0)
+    mini_portile2 (2.8.1)
+    minitest (5.17.0)
     mono_logger (1.1.1)
     multi_json (1.15.0)
-    mustermann (1.1.1)
+    mustermann (3.0.0)
       ruby2_keywords (~> 0.0.1)
-    nokogiri (1.13.1)
-      mini_portile2 (~> 2.7.0)
+    nokogiri (1.13.10)
+      mini_portile2 (~> 2.8.0)
       racc (~> 1.4)
-    pry (0.13.1)
+    pry (0.14.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    pry-byebug (3.9.0)
+    pry-byebug (3.10.1)
       byebug (~> 11.0)
-      pry (~> 0.13.0)
-    racc (1.6.0)
-    rack (2.2.3)
-    rack-protection (2.1.0)
+      pry (>= 0.13, < 0.15)
+    racc (1.6.2)
+    rack (2.2.5)
+    rack-protection (3.0.5)
       rack
-    rack-test (1.1.0)
-      rack (>= 1.0, < 3)
+    rack-test (2.0.2)
+      rack (>= 1.3)
     rails-dom-testing (2.0.3)
       activesupport (>= 4.2.0)
       nokogiri (>= 1.6)
-    rails-html-sanitizer (1.4.2)
-      loofah (~> 2.3)
-    railties (5.2.6.2)
-      actionpack (= 5.2.6.2)
-      activesupport (= 5.2.6.2)
+    rails-html-sanitizer (1.4.4)
+      loofah (~> 2.19, >= 2.19.1)
+    railties (5.2.8.1)
+      actionpack (= 5.2.8.1)
+      activesupport (= 5.2.8.1)
       method_source
       rake (>= 0.8.7)
       thor (>= 0.19.0, < 2.0)
     rake (13.0.6)
-    redis (4.6.0)
-    redis-namespace (1.8.1)
-      redis (>= 3.0.4)
-    resque (2.2.0)
+    redis (4.8.0)
+    redis-namespace (1.10.0)
+      redis (>= 4)
+    resque (2.4.0)
       mono_logger (~> 1.0)
       multi_json (~> 1.0)
       redis-namespace (~> 1.6)
       sinatra (>= 0.9.2)
-      vegas (~> 0.1.2)
     resque_spec (0.18.1)
       resque (>= 1.26.0)
       rspec-core (>= 3.0.0)
       rspec-expectations (>= 3.0.0)
       rspec-mocks (>= 3.0.0)
-    rspec (3.11.0)
-      rspec-core (~> 3.11.0)
-      rspec-expectations (~> 3.11.0)
-      rspec-mocks (~> 3.11.0)
-    rspec-core (3.11.0)
-      rspec-support (~> 3.11.0)
-    rspec-expectations (3.11.0)
+    rspec (3.12.0)
+      rspec-core (~> 3.12.0)
+      rspec-expectations (~> 3.12.0)
+      rspec-mocks (~> 3.12.0)
+    rspec-core (3.12.0)
+      rspec-support (~> 3.12.0)
+    rspec-expectations (3.12.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.11.0)
-    rspec-mocks (3.11.0)
+      rspec-support (~> 3.12.0)
+    rspec-mocks (3.12.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.11.0)
-    rspec-rails (5.1.0)
+      rspec-support (~> 3.12.0)
+    rspec-rails (5.1.2)
       actionpack (>= 5.2)
       activesupport (>= 5.2)
       railties (>= 5.2)
@@ -134,23 +132,22 @@ GEM
     rspec-sidekiq (3.1.0)
       rspec-core (~> 3.0, >= 3.0.0)
       sidekiq (>= 2.4.0)
-    rspec-support (3.11.0)
+    rspec-support (3.12.0)
     ruby2_keywords (0.0.5)
-    sidekiq (6.4.1)
-      connection_pool (>= 2.2.2)
+    sidekiq (6.5.8)
+      connection_pool (>= 2.2.5, < 3)
       rack (~> 2.0)
-      redis (>= 4.2.0)
+      redis (>= 4.5.0, < 5)
     simplecov (0.16.1)
       docile (~> 1.1)
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
-    sinatra (2.1.0)
-      mustermann (~> 1.0)
-      rack (~> 2.2)
-      rack-protection (= 2.1.0)
+    sinatra (3.0.5)
+      mustermann (~> 3.0)
+      rack (~> 2.2, >= 2.2.4)
+      rack-protection (= 3.0.5)
       tilt (~> 2.0)
-    statsd-ruby (1.4.0)
     sync (0.5.0)
     term-ansicolor (1.7.1)
       tins (~> 1.0)
@@ -158,13 +155,11 @@ GEM
     thread_safe (0.3.6)
     thwait (0.2.0)
       e2mmap
-    tilt (2.0.10)
-    tins (1.31.0)
+    tilt (2.0.11)
+    tins (1.32.1)
       sync
-    tzinfo (1.2.9)
+    tzinfo (1.2.10)
       thread_safe (~> 0.1)
-    vegas (0.1.11)
-      rack (>= 1.0.0)
 
 PLATFORMS
   ruby
@@ -184,7 +179,7 @@ DEPENDENCIES
   rspec
   rspec-rails (>= 2.0)
   rspec-sidekiq (>= 2.1.0)
-  sidekiq (>= 3.5.0)
+  sidekiq (>= 3.5.0, < 7.0.0)
   taskinator!
 
 BUNDLED WITH

data/README.md

@@ -1,7 +1,7 @@
 # Taskinator
 
 [![Gem Version](https://badge.fury.io/rb/taskinator.svg)](http://badge.fury.io/rb/taskinator)
-[![Build Status](https://img.shields.io/github/workflow/status/virtualstaticvoid/taskinator/Taskinator?style=flat-square)](https://github.com/virtualstaticvoid/taskinator/actions)
+[![Build Status](https://github.com/virtualstaticvoid/taskinator/actions/workflows/build.yml/badge.svg)](https://github.com/virtualstaticvoid/taskinator/actions/workflows/build.yml)
 [![Code Climate](https://codeclimate.com/github/virtualstaticvoid/taskinator.png)](https://codeclimate.com/github/virtualstaticvoid/taskinator)
 
 A simple orchestration library for running complex processes or workflows in Ruby.
@@ -53,8 +53,8 @@ If you are using Taskinator within a Rails application, then add an initializer,
 Taskinator.configure do |config|
 
   # configure the queue adapter to use
-  # can be :active_job, :delayed_job, :redis or :sidekiq
-  config.queue_adapter = :redis
+  # can be :active_job, :delayed_job, :resque or :sidekiq
+  config.queue_adapter = :resque
 
   # configure redis
   config.redis = {
@@ -171,39 +171,6 @@ module MyProcess
 end
 ```
 
-#### Reusing ActiveJob jobs
-
-It is likely that you already have one or more [jobs](https://guides.rubyonrails.org/active_job_basics.html)
-and want to reuse them within the process definition.
-
-Define a `job` step, providing the class of the Active Job to run and then taskinator will
-invoke that job as part of the process.
-
-The `job` step will be queued and executed on same queue as
-[configured by the job](https://guides.rubyonrails.org/active_job_basics.html#queues).
-
-```ruby
-# E.g. A resque worker
-class DoSomeWork
-  queue :high_priority
-
-  def self.perform(arg1, arg2)
-    # code to do the work
-  end
-end
-
-module MyProcess
-  extend Taskinator::Definition
-
-  # when creating the process, supply the same arguments
-  # that the DoSomeWork worker expects
-
-  define_process do
-    job DoSomeWork
-  end
-end
-```
-
 #### Data Driven Process Definitions
 
 You can also define data driven tasks using the `for_each` method, which takes an iterator method
@@ -271,7 +238,7 @@ process2 = MyProcess.create_process
 process2.tasks.count #=> 1
 ```
 
-#### Transformations
+#### Argument Transformations
 
 In addition, it is possible to transform the arguments used by a task or job, by including
 a `transform` step in the definition.
@@ -396,16 +363,54 @@ MyProcess.create_process(1, 2, 3, :send_notification => true)
 
 ```
 
+#### Reusing ActiveJob jobs
+
+It is likely that you already have one or more [jobs](https://guides.rubyonrails.org/active_job_basics.html)
+and want to reuse them within the process definition.
+
+Define a `job` step, providing the class of the Active Job to run and then taskinator will
+invoke that job as part of the process.
+
+The `job` step will be queued and executed on same queue as
+[configured by the job](https://guides.rubyonrails.org/active_job_basics.html#queues).
+
+```ruby
+# E.g. A resque worker
+class DoSomeWork
+  queue :high_priority
+
+  def self.perform(arg1, arg2)
+    # code to do the work
+  end
+end
+
+module MyProcess
+  extend Taskinator::Definition
+
+  # when creating the process, supply the same arguments
+  # that the DoSomeWork worker expects
+
+  define_process do
+    job DoSomeWork
+  end
+end
+```
+
 ### Execution
 
-A process is executed by calling the generated `create_process` method on your "process" module.
+A process is created by calling the generated `create_process` method on your "process" module.
 
 ```ruby
 process = MyProcess.create_process
+```
+
+And then enqueued for execution by calling the `enqueue!` method of the process.
+
+```ruby
 process.enqueue!
 ```
 
-Or, to start immediately, call the `start!` method.
+Or, started immediately by calling the `start!` method of the process.
 
 ```ruby
 process = MyProcess.create_process
@@ -637,6 +642,7 @@ process_id = "SUPPLY-PROCESS-IDENTIFIER"
 process = Taskinator::Api.find_process(process_id)
 
 puts process.inspect
+puts process.definition
 puts process.current_state
 puts process.tasks
 # etc...
@@ -655,6 +661,7 @@ task = Taskinator::Api.find_task(task_id)
 
 puts task.inspect
 puts task.class
+puts task.definition
 puts task.args                # for Step and Job types
 puts task.sub_process.tasks   # for SubProcess type
 # etc...
@@ -708,7 +715,7 @@ To configure the queue adapter to use, set `config.queue_adapter` to one of the
 
 * `:active_job`
 * `:delayed_job`
-* `:redis`
+* `:resque`
 * `:sidekiq`
 
 As follows:
@@ -717,8 +724,8 @@ As follows:
 Taskinator.configure do |config|
 
   # configure the queue adapter to use
-  # can be :active_job, :delayed_job, :redis or :sidekiq
-  config.queue_adapter = :redis
+  # can be :active_job, :delayed_job, :resque or :sidekiq
+  config.queue_adapter = :resque
 
 end
 ```
@@ -787,6 +794,7 @@ For all events, the data included contains the following information:
 | Key                             | Value                                                    |
 |---------------------------------|----------------------------------------------------------|
 | `:type`                         | The type name of the component reporting the event       |
+| `:definition`                   | The type name of the process definition                  |
 | `:process_uuid`                 | The UUID of the root process                             |
 | `:process_options`              | Options hash of the root process                         |
 | `:uuid`                         | The UUID of the respective task, job or sub process      |

data/lib/taskinator/builder.rb

@@ -0,0 +1,129 @@
+module Taskinator
+  class Builder
+
+    attr_reader :process
+    attr_reader :definition
+    attr_reader :args
+    attr_reader :builder_options
+
+    def initialize(process, definition, *args)
+      @process = process
+      @definition = definition
+      @builder_options = args.last.is_a?(Hash) ? args.pop : {}
+      @args = args
+      @executor = Taskinator::Executor.new(@definition)
+    end
+
+    def option?(key, &block)
+      yield if builder_options[key]
+    end
+
+    # defines a sub process of tasks which are executed sequentially
+    def sequential(options={}, &block)
+      raise ArgumentError, 'block' unless block_given?
+
+      sub_process = Process.define_sequential_process_for(@definition, options)
+      task = define_sub_process_task(@process, sub_process, options)
+      Builder.new(sub_process, @definition, *@args).instance_eval(&block)
+      @process.tasks << task if sub_process.tasks.any?
+      nil
+    end
+
+    # defines a sub process of tasks which are executed concurrently
+    def concurrent(complete_on=CompleteOn::Default, options={}, &block)
+      raise ArgumentError, 'block' unless block_given?
+
+      sub_process = Process.define_concurrent_process_for(@definition, complete_on, options)
+      task = define_sub_process_task(@process, sub_process, options)
+      Builder.new(sub_process, @definition, *@args).instance_eval(&block)
+      @process.tasks << task if sub_process.tasks.any?
+      nil
+    end
+
+    # dynamically defines tasks, using the given @iterator method
+    # the definition will be evaluated for each yielded item
+    def for_each(method, options={}, &block)
+      raise ArgumentError, 'method' if method.nil?
+      raise NoMethodError, method unless @executor.respond_to?(method)
+      raise ArgumentError, 'block' unless block_given?
+
+      #
+      # `for_each` is an exception, since it invokes the definition
+      # in order to yield elements to the builder, and any options passed
+      # are included with the builder options
+      #
+      method_args = options.any? ? [*@args, options] : @args
+      @executor.send(method, *method_args) do |*args|
+        Builder.new(@process, @definition, *args).instance_eval(&block)
+      end
+      nil
+    end
+
+    alias_method :transform, :for_each
+
+    # defines a task which executes the given @method
+    def task(method, options={})
+      raise ArgumentError, 'method' if method.nil?
+      raise NoMethodError, method unless @executor.respond_to?(method)
+
+      define_step_task(@process, method, @args, options)
+      nil
+    end
+
+    # defines a task which executes the given @job
+    # which is expected to implement a perform method either as a class or instance method
+    def job(job, options={})
+      raise ArgumentError, 'job' if job.nil?
+      raise ArgumentError, 'job' unless job.methods.include?(:perform) || job.instance_methods.include?(:perform)
+
+      define_job_task(@process, job, @args, options)
+      nil
+    end
+
+    # TODO: add mailer
+    # TODO: add complete!
+    # TODO: add fail!
+
+    # defines a sub process task, for the given @definition
+    # the definition specified must have input compatible arguments
+    # to the current definition
+    def sub_process(definition, options={})
+      raise ArgumentError, 'definition' if definition.nil?
+      raise ArgumentError, "#{definition.name} does not extend the #{Definition.name} module" unless definition.kind_of?(Definition)
+
+      sub_process = definition.create_sub_process(*@args, combine_options(options))
+      task = define_sub_process_task(@process, sub_process, options)
+      Builder.new(sub_process, definition, *@args)
+      @process.tasks << task if sub_process.tasks.any?
+      nil
+    end
+
+  private
+
+    def define_step_task(process, method, args, options={})
+      define_task(process) {
+        Task.define_step_task(process, method, args, combine_options(options))
+      }
+    end
+
+    def define_job_task(process, job, args, options={})
+      define_task(process) {
+        Task.define_job_task(process, job, args, combine_options(options))
+      }
+    end
+
+    def define_sub_process_task(process, sub_process, options={})
+      Task.define_sub_process_task(process, sub_process, combine_options(options))
+    end
+
+    def define_task(process)
+      process.tasks << task = yield
+      task
+    end
+
+    def combine_options(options={})
+      builder_options.merge(options)
+    end
+
+  end
+end

data/lib/taskinator/definition.rb

@@ -1,3 +1,5 @@
+require 'taskinator/builder'
+
 module Taskinator
   module Definition
 
@@ -120,5 +122,3 @@ module Taskinator
 
   end
 end
-
-require 'taskinator/definition/builder'

data/lib/taskinator/instrumentation.rb

@@ -58,6 +58,7 @@ module Taskinator
       return OpenStruct.new(
         {
           :type                   => self.class.name,
+          :definition             => self.definition.name,
           :process_uuid           => process_uuid,
           :process_options        => process_options.dup,
           :uuid                   => uuid,