taskinator 0.3.16 → 0.4.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 231c6c9f1bdcf8bb1a7fb38c0ebd0c893f61f49051d94ace5714233a207ad651
-  data.tar.gz: 9c66bf91f8420f3fff69487e095a5c36986409f7249224f03ba0b53fdd427190
+  metadata.gz: f1a746ba56e5b6a2f57736990eef9e83eafe8701fcbebaaddea258a010f3ee14
+  data.tar.gz: 5e17bc8e6a61b93d6df0691dc463fb178aca72957ab79eccaea7b9cd6c1af642
 SHA512:
-  metadata.gz: 0e322ccc3214ed171bd75b14b4ebaa7e51c1e1ba33b3f7f292c707059f166566a8b8c08235442d26922b0cd8a8c2affa320af03d069d33b0fce4d9ffe7485e95
-  data.tar.gz: d16a3220d1be357c77e7a47d3e2c426a99549c3d350979f534a176ec3919444551956d4d41e6b7389e245fe39c1286d7ef60e6f07b1155f3aeae7e9cdce1398b
+  metadata.gz: 8f8cd10cc4c8f5d45d5cc781b379927bae7c0ed5c280e2470450a818a6a3132af75bc11a4c16eef6312056f984e912f3fa758bd45e493988cee37fd01ad92d87
+  data.tar.gz: f8b330d3f6e6766f6ddbc33becd3bb6d16826a021dd0ba16c3b2ec8308cdfe31caad40621fcbc542d6662c8aababaa946e369d251c82b24a213f955a942192a3

data/.travis.yml

@@ -1,15 +1,16 @@
+os: linux
+dist: xenial
 language: ruby
-sudo: false  # See http://docs.travis-ci.com/user/migrating-from-legacy
 cache: bundler
 
 services:
-  - redis-server
+  - redis
 
 rvm:
-  - 2.4.10
   - 2.5.8
   - 2.6.6
   - 2.7.2
+  - 3.0.0
 
 script: 'bundle exec rake spec'
 

data/CHANGELOG.md

@@ -1,3 +1,66 @@
+v0.4.3 - 14 Jan 2022
+---
+Add `#find_process` and `#find_task` methods to `Taskinator::Api`.
+Bug fix to API when enumerating processes.
+Updated dependencies.
+
+v0.4.2 - 16 Mar 2021
+---
+Bug fix for process/task keys not expired upon completion.
+
+v0.4.1 - 15 Mar 2021
+---
+Optimisation to exclude sub-processes which don't have any tasks.
+Preparations for upgrade to Ruby 3 and ActiveSupport 6
+
+v0.4.0 - 4 Mar 2021
+---
+Bug fix `job` tasks which have no arguments to the `perform` method.
+Added support for having `perform` method as a class method.
+
+v0.3.16 - 17 Feb 2021
+---
+Bug fix to deincrement pending counts for sequential tasks.
+Bug fix to allow concurrent tasks to be retried (via Resque) and to complete processes.
+
+v0.3.15 - 22 Nov 2018
+---
+Updated dependencies.
+
+v0.3.14 - 13 Jul 2018
+---
+Updated dependencies.
+Removed gemnasium.
+
+v0.3.13 - 23 Sep 2017
+---
+Updated dependencies.
+
+v0.3.12 - 23 Sep 2017
+---
+Spec fixes.
+Updated dependencies.
+
+v0.3.11 - 1 Nov 2016
+---
+Removed `redis-semaphore` gem and use INCRBY to track pending concurrent tasks instead.
+Added instrumentation using statsd.
+Bug fixes to key expiry logic.
+Refactored process and task state transistions.
+
+v0.3.10 - 1 Nov 2016
+---
+Added support for serializing to XML.
+Improvements to process and task states.
+
+v0.3.9 - 12 Sep 2016
+---
+Added benchmark for redis-mutex.
+
+v0.3.7 - 18 Aug 2016
+---
+Bug fix to `option?` method.
+
 v0.3.6 - 11 Nov 2015
 ---
 Added visitor for performing clean up of completed processes/tasks.

data/Gemfile.lock

@@ -1,21 +1,20 @@
 PATH
   remote: .
   specs:
-    taskinator (0.3.16)
+    taskinator (0.4.3)
       builder (>= 3.2.2)
       connection_pool (>= 2.2.0)
       globalid (~> 0.3)
       json (>= 1.8.2)
       redis (>= 3.2.1)
       redis-namespace (>= 1.5.2)
-      redis-semaphore (>= 0.2.4)
       statsd-ruby (~> 1.4.0)
       thwait (~> 0.2)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (5.2.4.5)
+    activesupport (5.2.6)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
@@ -23,8 +22,8 @@ GEM
     builder (3.2.4)
     byebug (11.1.3)
     coderay (1.1.3)
-    concurrent-ruby (1.1.8)
-    connection_pool (2.2.3)
+    concurrent-ruby (1.1.9)
+    connection_pool (2.2.5)
     coveralls (0.8.23)
       json (>= 1.8, < 3)
       simplecov (~> 0.16.1)
@@ -33,38 +32,36 @@ GEM
       tins (~> 1.6)
     delayed_job (4.1.9)
       activesupport (>= 3.0, < 6.2)
-    diff-lcs (1.4.4)
-    docile (1.3.5)
+    diff-lcs (1.5.0)
+    docile (1.4.0)
     e2mmap (0.1.0)
     fakeredis (0.7.0)
       redis (>= 3.2, < 5.0)
-    globalid (0.4.2)
-      activesupport (>= 4.2.0)
-    i18n (1.8.9)
+    globalid (0.6.0)
+      activesupport (>= 5.0)
+    i18n (1.8.11)
       concurrent-ruby (~> 1.0)
-    json (2.5.1)
+    json (2.6.1)
     method_source (1.0.0)
-    minitest (5.14.3)
-    mono_logger (1.1.0)
+    minitest (5.15.0)
+    mono_logger (1.1.1)
     multi_json (1.15.0)
     mustermann (1.1.1)
       ruby2_keywords (~> 0.0.1)
-    pry (0.13.1)
+    pry (0.14.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    pry-byebug (3.9.0)
+    pry-byebug (3.8.0)
       byebug (~> 11.0)
-      pry (~> 0.13.0)
+      pry (~> 0.10)
     rack (2.2.3)
     rack-protection (2.1.0)
       rack
-    rake (13.0.3)
-    redis (4.2.5)
+    rake (13.0.6)
+    redis (4.5.1)
     redis-namespace (1.8.1)
       redis (>= 3.0.4)
-    redis-semaphore (0.3.1)
-      redis
-    resque (2.0.0)
+    resque (2.2.0)
       mono_logger (~> 1.0)
       multi_json (~> 1.0)
       redis-namespace (~> 1.6)
@@ -90,9 +87,9 @@ GEM
     rspec-sidekiq (3.1.0)
       rspec-core (~> 3.0, >= 3.0.0)
       sidekiq (>= 2.4.0)
-    rspec-support (3.10.2)
-    ruby2_keywords (0.0.4)
-    sidekiq (6.1.3)
+    rspec-support (3.10.3)
+    ruby2_keywords (0.0.5)
+    sidekiq (6.3.1)
       connection_pool (>= 2.2.2)
       rack (~> 2.0)
       redis (>= 4.2.0)
@@ -110,12 +107,12 @@ GEM
     sync (0.5.0)
     term-ansicolor (1.7.1)
       tins (~> 1.0)
-    thor (1.1.0)
+    thor (1.2.1)
     thread_safe (0.3.6)
     thwait (0.2.0)
       e2mmap
     tilt (2.0.10)
-    tins (1.28.0)
+    tins (1.31.0)
       sync
     tzinfo (1.2.9)
       thread_safe (~> 0.1)
@@ -142,4 +139,4 @@ DEPENDENCIES
   taskinator!
 
 BUNDLED WITH
-   2.2.11
+   2.2.14

data/README.md

@@ -5,21 +5,25 @@
 [![Code Climate](https://codeclimate.com/github/virtualstaticvoid/taskinator.png)](https://codeclimate.com/github/virtualstaticvoid/taskinator)
 [![Coverage Status](https://coveralls.io/repos/virtualstaticvoid/taskinator/badge.png)](https://coveralls.io/r/virtualstaticvoid/taskinator)
 
-A simple orchestration library for running complex processes or workflows in Ruby. Processes are defined using a simple DSL, where the sequences and
-tasks are defined. Processes can then be queued for execution. Sequences can be synchronous or asynchronous, and the overall process can be monitored
-for completion or failure.
+A simple orchestration library for running complex processes or workflows in Ruby.
+Processes are defined using a simple DSL, where the sequences and tasks are defined.
+Processes can then be queued for execution. Sequences can be synchronous or asynchronous,
+and the overall process can be monitored for completion or failure.
 
-Processes and tasks are executed by background workers and you can use any one of the following gems:
+Processes and tasks are executed by background workers and you can use any one of the
+following gems:
 
 * [resque](https://github.com/resque/resque)
 * [sidekiq](https://github.com/mperham/sidekiq)
 * [delayed_job](https://github.com/collectiveidea/delayed_job)
 
-The configuration and state of each process and their respective tasks is stored using Redis key/values.
+The configuration and state of each process and their respective tasks is stored using
+Redis key/values.
 
 ## Requirements
 
-The latest MRI (2.1, 2.0) version. Other versions/VMs are untested but might work fine. MRI 1.9 is not supported.
+The latest MRI (2.1, 2.0) version. Other versions/VMs are untested but might work fine.
+MRI 1.9 is not supported.
 
 Redis 2.4 or greater is required.
 
@@ -68,7 +72,8 @@ module MyProcess
 end
 ```
 
-The `define_process` method optionally takes the list of expected arguments which are used to validate the arguments supplied when creating a new process.
+The `define_process` method optionally takes the list of expected arguments which are used
+to validate the arguments supplied when creating a new process.
 These should be specified with symbols.
 
 ```ruby
@@ -87,7 +92,8 @@ process = MyProcess.create_process Date.today, :option_1 => true
 
 _NOTE:_ The current implementation performs a naive check on the count of arguments.
 
-Next, specify the tasks with their corresponding implementation methods, that make up the process, using the `task` method and providing the `method` to execute for the task.
+Next, specify the tasks with their corresponding implementation methods, that make up the
+process, using the `task` method and providing the `method` to execute for the task.
 
 ```ruby
 module MyProcess
@@ -108,7 +114,8 @@ module MyProcess
 end
 ```
 
-More complex processes may define sequential or concurrent steps, using the `sequential` and `concurrent` methods respectively.
+More complex processes may define sequential or concurrent steps, using the `sequential`
+and `concurrent` methods respectively.
 
 ```ruby
 module MyProcess
@@ -141,10 +148,15 @@ module MyProcess
 end
 ```
 
-It is likely that you already have worker classes for one of the queueing libraries, such as resque or delayed_job, and wish to reuse them for executing them in the sequence defined by the process definition.
+It is likely that you already have worker classes for one of the queueing libraries,
+such as resque or delayed_job, and wish to reuse them for executing them in the sequence
+defined by the process definition.
 
-Define a `job` step, providing the class of the worker, and then taskinator will execute that worker as part of the process definition.
-The `job` step will be queued and executed on same queue as configured by `delayed_job`, or that of the worker for `resque` and `sidekiq`.
+Define a `job` step, providing the class of the worker, and then taskinator will execute
+that worker as part of the process definition.
+
+The `job` step will be queued and executed on same queue as configured by `delayed_job`, or
+that of the worker for `resque` and `sidekiq`.
 
 ```ruby
 # E.g. A resque worker
@@ -168,8 +180,11 @@ module MyProcess
 end
 ```
 
-You can also define data driven tasks using the `for_each` method, which takes an iterator method name as an argument.
-The iterator method yields the parameters necessary for the task or job. Notice that the task method takes a parameter in this case, which will be the return values provided by the iterator.
+You can also define data driven tasks using the `for_each` method, which takes an iterator method
+name as an argument.
+
+The iterator method yields the parameters necessary for the task or job. Notice that the task
+method takes a parameter in this case, which will be the return values provided by the iterator.
 
 ```ruby
 module MyProcess
@@ -192,8 +207,9 @@ module MyProcess
 end
 ```
 
-It is possible to branch the process logic based on the options hash passed in when creating a process.
-The `options?` method takes the options key as an argument and calls the supplied block if the option is present and it's value is truthy.
+It is possible to branch the process logic based on the options hash passed in when creating
+a process. The `options?` method takes the options key as an argument and calls the supplied
+block if the option is present and it's value is _truthy_.
 
 ```ruby
 module MyProcess
@@ -227,8 +243,11 @@ process2 = MyProcess.create_process
 process2.tasks.count #=> 1
 ```
 
-In addition, it is possible to transform the arguments used by a task or job, by including a `transform` step in the definition.
-Similarly for the `for_each` method, `transform` takes a method name as an argument. The transformer method must yield the new arguments as required.
+In addition, it is possible to transform the arguments used by a task or job, by including
+a `transform` step in the definition.
+
+Similarly for the `for_each` method, `transform` takes a method name as an argument.
+The transformer method must yield the new arguments as required.
 
 ```ruby
 module MyProcess
@@ -273,7 +292,8 @@ module MyProcess
 end
 ```
 
-Any combination or nesting of `task`, `sequential`, `concurrent` and `for_each` steps are possible. E.g.
+Any combination or nesting of `task`, `sequential`, `concurrent` and `for_each` steps are
+possible. E.g.
 
 ```ruby
 module MyProcess
@@ -306,13 +326,17 @@ module MyProcess
 end
 ```
 
-In this example, the `work_step_begin` is executed, followed by the `work_step_all_at_once` steps which are executed concurrently, then
-the sub process `MySubProcess` is created and executed, followed by the `work_step_one_by_one` tasks which are executed sequentially and
+In this example, the `work_step_begin` is executed, followed by the `work_step_all_at_once`
+steps which are executed concurrently, then the sub process `MySubProcess` is created and
+executed, followed by the `work_step_one_by_one` tasks which are executed sequentially and
 finally the `work_step_end` is executed.
 
-It is also possible to embed conditional logic within the process definition stages in order to produce steps based on the required logic.
-All builder methods are available within the scope of the `define_process` block. These methods include `args` and `options`
-which are passed into the `create_process` method of the definition.
+It is also possible to embed conditional logic within the process definition stages in
+order to produce steps based on the required logic.
+
+All builder methods are available within the scope of the `define_process` block. These
+methods include `args` and `options` which are passed into the `create_process` method
+of the definition.
 
 E.g.
 
@@ -332,7 +356,8 @@ module MyProcess
 end
 
 # when creating this proces, you supply to option when calling `create_process`
-# in this example, 'args' will be an array [1,2,3] and options will be a Hash {:send_notification => true}
+# in this example, 'args' will be an array [1,2,3]
+# and options will be a Hash {:send_notification => true}
 MyProcess.create_process(1, 2, 3, :send_notification => true)
 
 ```
@@ -364,8 +389,12 @@ To best understand how arguments are handled, you need to break it down into 3 p
   * Creation and
   * Execution
 
-Firstly, a process definition is declarative in that the `define_process` and a mix of `sequential`, `concurrent`, `for_each`, `task` and `job` directives provide the way to specify the sequencing of the steps for the process.
-Taskinator will interprete this definition and execute each step in the desired sequence or concurrency.
+Firstly, a process definition is declarative in that the `define_process` and a mix of
+`sequential`, `concurrent`, `for_each`, `task` and `job` directives provide the way to
+specify the sequencing of the steps for the process.
+
+Taskinator will interprete this definition and execute each step in the desired sequence
+or concurrency.
 
 Consider the following process definition:
 
@@ -411,11 +440,17 @@ end
 
 There are three tasks; namely `:work_step_1`, `:work_step_2` and `:work_step_3`.
 
-The third task, `:work_step_3`, is built up using the `for_each` iterator, which means that the number of `:work_step_3` tasks will depend on how many times the `additional_step` iterator method yields to the definition.
+The third task, `:work_step_3`, is built up using the `for_each` iterator, which means that
+the number of `:work_step_3` tasks will depend on how many times the `additional_step`
+iterator method yields to the definition.
+
+This brings us to the creation part. When `create_process` is called on the given module,
+you provide arguments to it, which will get passed onto the respective `task` and
+`for_each` iterator methods.
 
-This brings us to the creation part. When `create_process` is called on the given module, you provide arguments to it, which will get passed onto the respective `task` and `for_each` iterator methods.
+So, considering the `MySimpleProcess` module shown above, `work_step_1`, `work_step_2`
+and `work_step_3` methods each expect arguments.
 
-So, considering the `MySimpleProcess` module shown above, `work_step_1`, `work_step_2` and `work_step_3` methods each expect arguments.
 These will ultimately come from the arguments passed into the `create_process` method.
 
 E.g.
@@ -438,7 +473,8 @@ process = MySimpleProcess.create_process(options)
 
 ```
 
-To best understand how the process is created, consider the following "procedural" code for how it could work.
+To best understand how the process is created, consider the following "procedural" code
+for how it could work.
 
 ```ruby
 # A process, which maps the target and a list of steps
@@ -535,11 +571,17 @@ process.execute
 
 ```
 
-In reality, each task is executed by a worker process, possibly on another host, so the execution process isn't as simple, but this example should help you to understand conceptually how the process is executed, and how the arguments are propagated through.
+In reality, each task is executed by a worker process, possibly on another host, so the
+execution process isn't as simple, but this example should help you to understand
+conceptually how the process is executed, and how the arguments are propagated through.
 
 ### Monitoring
 
-To monitor the state of the processes, use the `Taskinator::Api::Processes` class. This is still a work in progress.
+NOTE: This aspect of the library is still a work in progress.
+
+#### Processes
+
+To monitor the state of the processes, use the `Taskinator::Api::Processes` class.
 
 ```ruby
 processes = Taskinator::Api::Processes.new
@@ -549,11 +591,57 @@ processes.each do |process|
 end
 ```
 
+#### Debugging
+
+To aid debugging specific processes and tasks, where the process or task identifier is
+known, it is possible to retrieve the specific task or process using `Taskinator::Api`.
+
+To retrieve a specific process, given the process identifier:
+
+```ruby
+process_id = "SUPPLY-PROCESS-IDENTIFIER"
+process = Taskinator::Api.find_process(process_id)
+
+puts process.inspect
+puts process.current_state
+puts process.tasks
+# etc...
+```
+
+The type of process may be one of the following:
+
+* `Taskinator::Process::Sequential`
+* `Taskinator::Process::Concurrent`
+
+Then, to retrieve a specific task, given the task identifier:
+
+```ruby
+task_id = "SUPPLY-TASK-IDENTIFIER"
+task = Taskinator::Api.find_task(task_id)
+
+puts task.inspect
+puts task.class
+puts task.args                # for Step and Job types
+puts task.sub_process.tasks   # for SubProcess type
+# etc...
+```
+
+Depending on the type of task, different attributes will be available for inspection.
+
+The types include:
+
+* `Taskinator::Task::Step`
+* `Taskinator::Task::Job`
+* `Taskinator::Task::SubProcess`
+
 ## Configuration
 
 ### Redis
 
-By default Taskinator assumes Redis is located at `localhost:6397`. This is fine for development, but for many production environments you will need to point to an external Redis server. You may also what to use a namespace for the Redis keys.
+By default Taskinator assumes Redis is located at `localhost:6397`. This is fine for development,
+but for many production environments you will need to point to an external Redis server.
+You may also what to use a namespace for the Redis keys.
+
 _NOTE:_ The configuration hash _must_ have symbolized keys.
 
 ```ruby
@@ -568,15 +656,19 @@ end
 Or, alternatively, via an `ENV` variable
 
 Set the `REDIS_PROVIDER` environment variable to the Redis server url.
-E.g. On Heroku, with RedisGreen: set REDIS_PROVIDER=REDISGREEN_URL and Taskinator will use the value of the `REDISGREEN_URL` environment variable when connecting to Redis.
+E.g. On Heroku, with RedisGreen: set REDIS_PROVIDER=REDISGREEN_URL and Taskinator will use the
+value of the `REDISGREEN_URL` environment variable when connecting to Redis.
 
 You may also use the generic `REDIS_URL` which may be set to your own private Redis server.
 
-The Redis configuration leverages the same setup as `sidekiq`. For advanced options, checkout the [Sidekiq Advanced Options](https://github.com/mperham/sidekiq/wiki/Advanced-Options#complete-control) wiki for more information.
+The Redis configuration leverages the same setup as `sidekiq`. For advanced options, checkout the
+[Sidekiq Advanced Options](https://github.com/mperham/sidekiq/wiki/Advanced-Options#complete-control)
+wiki page for more information.
 
 ### Queues
 
-By default the queue names for process and task workers is `default`, however, you can specify the queue names as follows:
+By default the queue names for process and task workers is `default`, however, you can specify
+the queue names as follows:
 
 ```ruby
 Taskinator.configure do |config|
@@ -589,7 +681,8 @@ end
 
 ### Instrumentation
 
-It is possible to instrument processes, tasks and jobs by providing an instrumeter such as `ActiveSupport::Notifications`.
+It is possible to instrument processes, tasks and jobs by providing an instrumeter such
+as `ActiveSupport::Notifications`.
 
 ```ruby
 Taskinator.configure do |config|
@@ -607,40 +700,41 @@ end
 
 The following instrumentation events are issued:
 
-| Event                              | When                                                      |
-|------------------------------------|-----------------------------------------------------------|
-| `taskinator.process.created`       | After a root process gets created                         |
-| `taskinator.process.saved`         | After a root process has been persisted to Redis          |
-| `taskinator.process.enqueued`      | After a process or subprocess is enqueued for processing  |
-| `taskinator.process.processing`    | When a process or subprocess is processing                |
-| `taskinator.process.paused`        | When a process or subprocess is paused                    |
-| `taskinator.process.resumed`       | When a process or subprocess is resumed                   |
-| `taskinator.process.completed`     | After a process or subprocess has completed processing    |
-| `taskinator.process.cancelled`     | After a process or subprocess has been cancelled          |
-| `taskinator.process.failed`        | After a process or subprocess has failed                  |
-| `taskinator.task.enqueued`         | After a task has been enqueued                            |
-| `taskinator.task.processing`       | When a task is processing                                 |
-| `taskinator.task.completed`        | After a task has completed                                |
-| `taskinator.task.cancelled`        | After a task has been cancelled                           |
-| `taskinator.task.failed`           | After a task has failed                                   |
+| Event                           | When                                                     |
+|---------------------------------|----------------------------------------------------------|
+| `taskinator.process.created`    | After a root process gets created                        |
+| `taskinator.process.saved`      | After a root process has been persisted to Redis         |
+| `taskinator.process.enqueued`   | After a process or subprocess is enqueued for processing |
+| `taskinator.process.processing` | When a process or subprocess is processing               |
+| `taskinator.process.paused`     | When a process or subprocess is paused                   |
+| `taskinator.process.resumed`    | When a process or subprocess is resumed                  |
+| `taskinator.process.completed`  | After a process or subprocess has completed processing   |
+| `taskinator.process.cancelled`  | After a process or subprocess has been cancelled         |
+| `taskinator.process.failed`     | After a process or subprocess has failed                 |
+| `taskinator.task.enqueued`      | After a task has been enqueued                           |
+| `taskinator.task.processing`    | When a task is processing                                |
+| `taskinator.task.completed`     | After a task has completed                               |
+| `taskinator.task.cancelled`     | After a task has been cancelled                          |
+| `taskinator.task.failed`        | After a task has failed                                  |
 
 For all events, the data included contains the following information:
 
-| Key                      | Value                                                 |
-|--------------------------|-------------------------------------------------------|
-| `:type`                  | The type name of the component reporting the event    |
-| `:process_uuid`          | The UUID of the root process                          |
-| `:process_options`       | Options hash of the root process                      |
-| `:uuid`                  | The UUID of the respective task, job or sub process   |
-| `:options`               | Options hash of the component                         |
-| `:state`                 | State of the component                                |
-| `:percentage_completed`  | The percentage of completed tasks                     |
-| `:percentage_failed`     | The percentage of failed tasks                        |
-| `:percentage_cancelled`  | The percentage of cancelled tasks                     |
+| Key                             | Value                                                    |
+|---------------------------------|----------------------------------------------------------|
+| `:type`                         | The type name of the component reporting the event       |
+| `:process_uuid`                 | The UUID of the root process                             |
+| `:process_options`              | Options hash of the root process                         |
+| `:uuid`                         | The UUID of the respective task, job or sub process      |
+| `:options`                      | Options hash of the component                            |
+| `:state`                        | State of the component                                   |
+| `:percentage_completed`         | The percentage of completed tasks                        |
+| `:percentage_failed`            | The percentage of failed tasks                           |
+| `:percentage_cancelled`         | The percentage of cancelled tasks                        |
 
 ## Notes
 
-The persistence logic is decoupled from the implementation, so it is possible to implement another backing store if required.
+The persistence logic is decoupled from the implementation, so it is possible to implement
+another backing store if required.
 
 ## Contributing
 
@@ -651,12 +745,19 @@ The persistence logic is decoupled from the implementation, so it is possible to
 5. Create new Pull Request
 
 ## License
+
 MIT Copyright (c) 2014 Chris Stefano
 
 Portions of code are from the Sidekiq project, Copyright (c) Contributed Systems LLC.
 
 ## Inspiration
 
-Inspired by the [sidekiq](https://github.com/mperham/sidekiq) and [workflow](https://github.com/geekq/workflow) gems.
+Inspired by the [sidekiq](https://github.com/mperham/sidekiq) and
+[workflow](https://github.com/geekq/workflow) gems.
+
+For other workflow solutions, checkout [Stonepath](https://github.com/bokmann/stonepath),
+the now deprecated [ruote](https://github.com/jmettraux/ruote) gem and
+[workflow](https://github.com/geekq/workflow).
 
-For other workflow solutions, checkout [Stonepath](https://github.com/bokmann/stonepath), the now deprecated [ruote](https://github.com/jmettraux/ruote) gem and [workflow](https://github.com/geekq/workflow). Alternatively, for a robust enterprise ready solution checkout the [AWS Flow Framework for Ruby](http://docs.aws.amazon.com/amazonswf/latest/awsrbflowguide/welcome.html).
+Alternatively, for a robust enterprise ready solution checkout the
+[AWS Flow Framework for Ruby](http://docs.aws.amazon.com/amazonswf/latest/awsrbflowguide/welcome.html).

data/lib/taskinator/api.rb

@@ -13,12 +13,13 @@ module Taskinator
       def each(&block)
         return to_enum(__method__) unless block_given?
 
+        identifiers = Taskinator.redis do |conn|
+          conn.smembers(@processes_list_key)
+        end
+
         instance_cache = {}
-        Taskinator.redis do |conn|
-          uuids = conn.smembers(@processes_list_key)
-          uuids.each do |uuid|
-            yield Process.fetch(uuid, instance_cache)
-          end
+        identifiers.each do |identifier|
+          yield Process.fetch(identifier, instance_cache)
         end
       end
 
@@ -28,5 +29,13 @@ module Taskinator
         end
       end
     end
+
+    def self.find_process(identifier)
+      Process.fetch(identifier)
+    end
+
+    def self.find_task(identifier)
+      Task.fetch(identifier)
+    end
   end
 end

data/lib/taskinator/definition/builder.rb

@@ -24,7 +24,10 @@ module Taskinator
         raise ArgumentError, 'block' unless block_given?
 
         sub_process = Process.define_sequential_process_for(@definition, options)
-        Builder.new(define_sub_process_task(@process, sub_process, options), @definition, *@args).instance_eval(&block)
+        task = define_sub_process_task(@process, sub_process, options)
+        Builder.new(sub_process, @definition, *@args).instance_eval(&block)
+        @process.tasks << task if sub_process.tasks.any?
+        nil
       end
 
       # defines a sub process of tasks which are executed concurrently
@@ -32,7 +35,10 @@ module Taskinator
         raise ArgumentError, 'block' unless block_given?
 
         sub_process = Process.define_concurrent_process_for(@definition, complete_on, options)
-        Builder.new(define_sub_process_task(@process, sub_process, options), @definition, *@args).instance_eval(&block)
+        task = define_sub_process_task(@process, sub_process, options)
+        Builder.new(sub_process, @definition, *@args).instance_eval(&block)
+        @process.tasks << task if sub_process.tasks.any?
+        nil
       end
 
       # dynamically defines tasks, using the given @iterator method
@@ -51,6 +57,7 @@ module Taskinator
         @executor.send(method, *method_args) do |*args|
           Builder.new(@process, @definition, *args).instance_eval(&block)
         end
+        nil
       end
 
       alias_method :transform, :for_each
@@ -61,6 +68,7 @@ module Taskinator
         raise NoMethodError, method unless @executor.respond_to?(method)
 
         define_step_task(@process, method, @args, options)
+        nil
       end
 
       # defines a task which executes the given @job
@@ -70,6 +78,7 @@ module Taskinator
         raise ArgumentError, 'job' unless job.methods.include?(:perform) || job.instance_methods.include?(:perform)
 
         define_job_task(@process, job, @args, options)
+        nil
       end
 
       # defines a sub process task, for the given @definition
@@ -82,7 +91,10 @@ module Taskinator
         # TODO: decide whether the sub process to dynamically receive arguments
 
         sub_process = definition.create_sub_process(*@args, combine_options(options))
-        Builder.new(define_sub_process_task(@process, sub_process, options), definition, *@args)
+        task = define_sub_process_task(@process, sub_process, options)
+        Builder.new(sub_process, definition, *@args)
+        @process.tasks << task if sub_process.tasks.any?
+        nil
       end
 
     private
@@ -100,10 +112,7 @@ module Taskinator
       end
 
       def define_sub_process_task(process, sub_process, options={})
-        define_task(process) {
-          Task.define_sub_process_task(process, sub_process, combine_options(options))
-        }
-        sub_process
+        Task.define_sub_process_task(process, sub_process, combine_options(options))
       end
 
       def define_task(process)

data/lib/taskinator/persistence.rb

@@ -604,6 +604,9 @@ module Taskinator
       end
 
       def visit_tasks(tasks)
+        @conn.expire "#{@key}:tasks", expire_in
+        @conn.expire "#{@key}.count", expire_in
+        @conn.expire "#{@key}.pending", expire_in
         tasks.each do |task|
           RedisCleanupVisitor.new(@conn, task, expire_in).visit
         end

data/lib/taskinator/task.rb

@@ -230,12 +230,12 @@ module Taskinator
         # NNB: if other job types are required, may need to implement how they get invoked here!
         # FIXME: possible implement using ActiveJob instead, so it doesn't matter how the worker is implemented
 
-        if job.instance_of?(Module)
+        if job.respond_to?(:perform)
           # resque
-          job.perform(args)
+          job.perform(*args)
         else
           # delayedjob and sidekiq
-          job.new.perform(args)
+          job.new.perform(*args)
         end
 
         # ASSUMPTION: when the job returns, the task is considered to be complete

data/lib/taskinator/version.rb

@@ -1,3 +1,3 @@
 module Taskinator
-  VERSION = "0.3.16"
+  VERSION = "0.4.3"
 end

data/lib/taskinator.rb

@@ -1,7 +1,6 @@
 require 'json'
 require 'yaml'
 require 'securerandom'
-require 'redis-semaphore'
 require 'benchmark'
 require 'delegate'
 

data/spec/support/test_flows.rb

@@ -132,4 +132,42 @@ module TestFlows
     end
   end
 
+  module NestedTask
+    extend Taskinator::Definition
+    include Support
+
+    define_process :task_count do
+      task :task_1
+
+      concurrent do
+        task :task_2
+        task :task_3
+
+        sequential do
+          task :task_4
+          task :task_5
+
+          concurrent do
+            task :task_6
+            task :task_7
+
+            sequential do
+              task :task_8
+              task :task_9
+
+            end
+
+            task :task_10
+          end
+
+          task :task_11
+        end
+
+        task :task_12
+      end
+
+      task :task_13
+    end
+  end
+
 end

data/spec/taskinator/api_spec.rb

@@ -44,4 +44,20 @@ describe Taskinator::Api, :redis => true do
     end
   end
 
+  describe "#find_process" do
+    it {
+      # fetch method is covered by persistence spec
+      expect(Taskinator::Process).to receive(:fetch) {}
+      subject.find_process 'foo:bar:process'
+    }
+  end
+
+  describe "#find_task" do
+    it {
+      # fetch method is covered by persistence spec
+      expect(Taskinator::Task).to receive(:fetch) {}
+      subject.find_task 'foo:bar:process:baz:task'
+    }
+  end
+
 end

data/spec/taskinator/definition/builder_spec.rb

@@ -75,6 +75,22 @@ describe Taskinator::Definition::Builder do
       expect(Taskinator::Process).to receive(:define_sequential_process_for).with(definition, options).and_call_original
       subject.sequential(options, &define_block)
     end
+
+    it "adds sub-process task" do
+      block = Proc.new {|p|
+        p.task :task_method
+      }
+      expect(process.tasks).to be_empty
+      subject.sequential(options, &block)
+      expect(process.tasks).to_not be_empty
+    end
+
+    it "ignores sub-processes without tasks" do
+      allow(block).to receive(:call)
+      expect(process.tasks).to be_empty
+      subject.sequential(options, &define_block)
+      expect(process.tasks).to be_empty
+    end
   end
 
   describe "#concurrent" do
@@ -100,6 +116,22 @@ describe Taskinator::Definition::Builder do
       expect(Taskinator::Process).to receive(:define_concurrent_process_for).with(definition, Taskinator::CompleteOn::First, options).and_call_original
       subject.concurrent(Taskinator::CompleteOn::First, options, &define_block)
     end
+
+    it "adds sub-process task" do
+      block = Proc.new {|p|
+        p.task :task_method
+      }
+      expect(process.tasks).to be_empty
+      subject.sequential(options, &block)
+      expect(process.tasks).to_not be_empty
+    end
+
+    it "ignores sub-processes without tasks" do
+      allow(block).to receive(:call)
+      expect(process.tasks).to be_empty
+      subject.sequential(options, &define_block)
+      expect(process.tasks).to be_empty
+    end
   end
 
   describe "#for_each" do
@@ -235,6 +267,22 @@ describe Taskinator::Definition::Builder do
       expect(sub_definition).to receive(:create_sub_process).with(*args, builder_options.merge(options)).and_call_original
       subject.sub_process(sub_definition, options)
     end
+
+    it "adds sub-process task" do
+      block = Proc.new {|p|
+        p.task :task_method
+      }
+      expect(process.tasks).to be_empty
+      subject.sequential(options, &block)
+      expect(process.tasks).to_not be_empty
+    end
+
+    it "ignores sub-processes without tasks" do
+      allow(block).to receive(:call)
+      expect(process.tasks).to be_empty
+      subject.sequential(options, &define_block)
+      expect(process.tasks).to be_empty
+    end
   end
 
 end

data/spec/taskinator/persistence_spec.rb

@@ -370,24 +370,27 @@ describe Taskinator::Persistence, :redis => true do
         TestFlows::Job,
         TestFlows::SubProcess,
         TestFlows::Sequential,
-        TestFlows::Concurrent
+        TestFlows::Concurrent,
+        TestFlows::EmptySequentialProcessTest,
+        TestFlows::EmptyConcurrentProcessTest,
+        TestFlows::NestedTask,
       ].each do |definition|
 
         describe "#{definition.name} expire immediately" do
           it {
-            process = definition.create_process(1)
-
             Taskinator.redis do |conn|
-              expect(conn.hget(process.key, :uuid)).to eq(process.uuid)
+              # sanity check
+              expect(conn.keys).to be_empty
 
-              process.cleanup(0) # immediately
+              process = definition.create_process(1)
 
-              expect(conn.hget(process.key, :uuid)).to be_nil
+              # sanity check
+              expect(conn.hget(process.key, :uuid)).to eq(process.uuid)
 
-              recursively_enumerate_tasks(process.tasks) do |task|
-                expect(conn.hget(task.key, :uuid)).to be_nil
-              end
+              process.cleanup(0) # immediately
 
+              # ensure nothing left behind
+              expect(conn.keys).to be_empty
             end
           }
         end
@@ -396,9 +399,14 @@ describe Taskinator::Persistence, :redis => true do
 
       describe "expires in future" do
         it {
-          process = TestFlows::Task.create_process(1)
-
           Taskinator.redis do |conn|
+
+            # sanity check
+            expect(conn.keys).to be_empty
+
+            process = TestFlows::Task.create_process(1)
+
+            # sanity check
             expect(conn.hget(process.key, :uuid)).to eq(process.uuid)
 
             process.cleanup(2)
@@ -411,12 +419,8 @@ describe Taskinator::Persistence, :redis => true do
 
             sleep 3
 
-            # gone!
-            expect(conn.hget(process.key, :uuid)).to be_nil
-            recursively_enumerate_tasks(process.tasks) do |task|
-              expect(conn.hget(task.key, :uuid)).to be_nil
-            end
-
+            # ensure nothing left behind
+            expect(conn.keys).to be_empty
           end
         }
       end

data/spec/taskinator/task_spec.rb

@@ -227,7 +227,7 @@ describe Taskinator::Task do
 
         method = subject.method
 
-        executor.class_eval do
+        executor.singleton_class.class_eval do
           define_method method do |*args|
             # this method executes in the scope of the executor
             # store the context in an instance variable
@@ -334,13 +334,23 @@ describe Taskinator::Task do
       end
     end
 
-    subject { Taskinator::Task.define_job_task(process, TestJob, {:a => 1, :b => 2}) }
+    class TestJobClassNoArgs
+      def perform
+      end
+    end
+
+    module TestJobModuleNoArgs
+      def self.perform
+      end
+    end
+
+    subject { Taskinator::Task.define_job_task(process, TestJob, [1, {:a => 1, :b => 2}]) }
 
     it_should_behave_like "a task", Taskinator::Task::Job
 
     describe ".define_job_task" do
       it "sets the queue to use" do
-        task = Taskinator::Task.define_job_task(process, TestJob, {:a => 1, :b => 2}, :queue => :foo)
+        task = Taskinator::Task.define_job_task(process, TestJob, [1, {:a => 1, :b => 2}], :queue => :foo)
         expect(task.queue).to eq(:foo)
       end
     end
@@ -367,23 +377,37 @@ describe Taskinator::Task do
 
     describe "#start" do
       it {
-        task = Taskinator::Task.define_job_task(process, TestJobClass, {:a => 1, :b => 2})
+        task = Taskinator::Task.define_job_task(process, TestJobClass, [1, {:a => 1, :b => 2}])
+        expect(process).to receive(:task_completed).with(task)
+        expect_any_instance_of(TestJobClass).to receive(:perform).with(1, {:a => 1, :b => 2})
+        task.start!
+      }
+
+      it {
+        task = Taskinator::Task.define_job_task(process, TestJobModule, [2, {:a => 1, :b => 2}])
+        expect(process).to receive(:task_completed).with(task)
+        expect(TestJobModule).to receive(:perform).with(2, {:a => 1, :b => 2})
+        task.start!
+      }
+
+      it {
+        task = Taskinator::Task.define_job_task(process, TestJobClassNoArgs, nil)
         expect(process).to receive(:task_completed).with(task)
-        expect_any_instance_of(TestJobClass).to receive(:perform).with({:a => 1, :b => 2})
+        expect_any_instance_of(TestJobClassNoArgs).to receive(:perform).and_call_original
         task.start!
       }
 
       it {
-        task = Taskinator::Task.define_job_task(process, TestJobModule, {:a => 1, :b => 2})
+        task = Taskinator::Task.define_job_task(process, TestJobModuleNoArgs, nil)
         expect(process).to receive(:task_completed).with(task)
-        expect(TestJobModule).to receive(:perform).with({:a => 1, :b => 2})
+        expect(TestJobModuleNoArgs).to receive(:perform).and_call_original
         task.start!
       }
 
       it "is instrumented" do
         allow(process).to receive(:task_completed).with(subject)
 
-        allow(TestJob).to receive(:perform).with({:a => 1, :b => 2})
+        allow(TestJob).to receive(:perform).with(1, {:a => 1, :b => 2})
 
         instrumentation_block = SpecSupport::Block.new
 

data/taskinator.gemspec

@@ -23,7 +23,6 @@ Gem::Specification.new do |spec|
   # core
   spec.add_dependency 'redis'                       , '>= 3.2.1'
   spec.add_dependency 'redis-namespace'             , '>= 1.5.2'
-  spec.add_dependency 'redis-semaphore'             , '>= 0.2.4'
   spec.add_dependency 'connection_pool'             , '>= 2.2.0'
   spec.add_dependency 'json'                        , '>= 1.8.2'
   spec.add_dependency 'builder'                     , '>= 3.2.2'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: taskinator
 version: !ruby/object:Gem::Version
-  version: 0.3.16
+  version: 0.4.3
 platform: ruby
 authors:
 - Chris Stefano
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-17 00:00:00.000000000 Z
+date: 2022-01-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -38,20 +38,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.5.2
-- !ruby/object:Gem::Dependency
-  name: redis-semaphore
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.2.4
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.2.4
 - !ruby/object:Gem::Dependency
   name: connection_pool
   requirement: !ruby/object:Gem::Requirement
@@ -246,7 +232,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.2.3
 signing_key: 
 specification_version: 4
 summary: A simple orchestration library for running complex processes or workflows