taskinator 0.3.14 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 15afaaa6ba986d581cfb92e8c7a26beba3385f05cae2a1e2c7c29449f7f00151
-  data.tar.gz: 93bb6cfdce126a1b325549715f0dfab942e5746358a29c75509fc2b6664feece
+  metadata.gz: 5bcb4199fc1cea26510008421e3e1a1cdef9efc81eb5392f6a8913c70ae47ed1
+  data.tar.gz: '0266388a941c4e79d39b808d747d31d56dc957f8499ae1d14d553aceab7ac467'
 SHA512:
-  metadata.gz: 2bcb650d7e41ecaff1bccd572c9966e4c19c02f2e1c0b46d41fdf6d1bd7f771a7338545b8c15172928eae14473990ce1e40ca8ad739e6e9e46f733ad23358d5d
-  data.tar.gz: 0faea9c7ed24e3b23dcafb56e84ad15b6b53668d60e7a874c8af273370eaba1e0c29db913f15357b774444d284927cf95daeb3d3f2e5ef0b3d147301527fa332
+  metadata.gz: 55f5bc10cf12986f073f2f3dd590ed17abac7e0ee25b8e4531391fd5d02e0297e6c7ed58961a626d98944c01395223fc75b4d2c49d28f31c85bbeca76bc21efe
+  data.tar.gz: 2a02585f01df20d868ac294c7daef8414ecefe3023d006869ea40c0aea9831361bdc5633e41787e86e20d4e39d749402baee385563adaf7e9405b7efad212a88

data/.ruby-version

@@ -1 +1 @@
-ruby-2.5.1
+ruby-2.7.2

data/.travis.yml

@@ -1,15 +1,16 @@
+os: linux
+dist: xenial
 language: ruby
-sudo: false  # See http://docs.travis-ci.com/user/migrating-from-legacy
 cache: bundler
 
 services:
-  - redis-server
+  - redis
 
 rvm:
-  - 2.2.10
-  - 2.3.7
-  - 2.4.4
-  - 2.5.1
+  - 2.5.8
+  - 2.6.6
+  - 2.7.2
+  - 3.0.0
 
 script: 'bundle exec rake spec'
 

data/CHANGELOG.md

@@ -1,3 +1,60 @@
+v0.4.2 - 16 Mar 2021
+---
+Bug fix for process/task keys not expired upon completion.
+
+v0.4.1 - 15 Mar 2021
+---
+Optimisation to exclude sub-processes which don't have any tasks.
+Preparations for upgrade to Ruby 3 and ActiveSupport 6
+
+v0.4.0 - 4 Mar 2021
+---
+Bug fix `job` tasks which have no arguments to the `perform` method.
+Added support for having `perform` method as a class method.
+
+v0.3.16 - 17 Feb 2021
+---
+Bug fix to deincrement pending counts for sequential tasks.
+Bug fix to allow concurrent tasks to be retried (via Resque) and to complete processes.
+
+v0.3.15 - 22 Nov 2018
+---
+Updated dependencies.
+
+v0.3.14 - 13 Jul 2018
+---
+Updated dependencies.
+Removed gemnasium.
+
+v0.3.13 - 23 Sep 2017
+---
+Updated dependencies.
+
+v0.3.12 - 23 Sep 2017
+---
+Spec fixes.
+Updated dependencies.
+
+v0.3.11 - 1 Nov 2016
+---
+Removed `redis-semaphore` gem and use INCRBY to track pending concurrent tasks instead.
+Added instrumentation using statsd.
+Bug fixes to key expiry logic.
+Refactored process and task state transistions.
+
+v0.3.10 - 1 Nov 2016
+---
+Added support for serializing to XML.
+Improvements to process and task states.
+
+v0.3.9 - 12 Sep 2016
+---
+Added benchmark for redis-mutex.
+
+v0.3.7 - 18 Aug 2016
+---
+Bug fix to `option?` method.
+
 v0.3.6 - 11 Nov 2015
 ---
 Added visitor for performing clean up of completed processes/tasks.

data/Gemfile

@@ -15,10 +15,10 @@ gem 'resque_spec'     , '>= 0.16.0'
 # other
 gem 'bundler'         , '>= 1.6.0'
 gem 'rake'            , '>= 10.3.0'
-gem 'activesupport'   , '~> 4.2.0'
+gem 'activesupport'   , '~> 5.2.0'
 gem 'rspec'
-gem 'coveralls'       , '>= 0.7.0'
+gem 'coveralls'       , '>= 0.8.22'
 gem 'pry'             , '>= 0.9.0'
 gem 'pry-byebug'      , '>= 1.3.0'
 
-gem 'fakeredis'       , '~> 0.6.0'
+gem 'fakeredis'       , '~> 0.7.0'

data/Gemfile.lock

@@ -1,116 +1,120 @@
 PATH
   remote: .
   specs:
-    taskinator (0.3.14)
+    taskinator (0.4.2)
       builder (>= 3.2.2)
       connection_pool (>= 2.2.0)
       globalid (~> 0.3)
       json (>= 1.8.2)
       redis (>= 3.2.1)
       redis-namespace (>= 1.5.2)
-      redis-semaphore (>= 0.2.4)
-      statsd-ruby (~> 1.2.0)
+      statsd-ruby (~> 1.4.0)
+      thwait (~> 0.2)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (4.2.10)
-      i18n (~> 0.7)
+    activesupport (5.2.4.5)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
       minitest (~> 5.1)
-      thread_safe (~> 0.3, >= 0.3.4)
       tzinfo (~> 1.1)
-    builder (3.2.3)
-    byebug (10.0.2)
-    coderay (1.1.2)
-    concurrent-ruby (1.0.5)
-    connection_pool (2.2.2)
-    coveralls (0.8.22)
+    builder (3.2.4)
+    byebug (11.1.3)
+    coderay (1.1.3)
+    concurrent-ruby (1.1.8)
+    connection_pool (2.2.3)
+    coveralls (0.8.23)
       json (>= 1.8, < 3)
       simplecov (~> 0.16.1)
       term-ansicolor (~> 1.3)
-      thor (~> 0.19.4)
+      thor (>= 0.19.4, < 2.0)
       tins (~> 1.6)
-    delayed_job (4.1.5)
-      activesupport (>= 3.0, < 5.3)
-    diff-lcs (1.3)
-    docile (1.3.1)
-    fakeredis (0.6.0)
-      redis (~> 3.2)
-    globalid (0.4.1)
+    delayed_job (4.1.9)
+      activesupport (>= 3.0, < 6.2)
+    diff-lcs (1.4.4)
+    docile (1.3.5)
+    e2mmap (0.1.0)
+    fakeredis (0.7.0)
+      redis (>= 3.2, < 5.0)
+    globalid (0.4.2)
       activesupport (>= 4.2.0)
-    i18n (0.9.5)
+    i18n (1.8.9)
       concurrent-ruby (~> 1.0)
-    json (2.1.0)
-    method_source (0.9.0)
-    minitest (5.11.3)
+    json (2.5.1)
+    method_source (1.0.0)
+    minitest (5.14.4)
     mono_logger (1.1.0)
-    multi_json (1.13.1)
-    mustermann (1.0.2)
-    pry (0.11.3)
-      coderay (~> 1.1.0)
-      method_source (~> 0.9.0)
-    pry-byebug (3.6.0)
-      byebug (~> 10.0)
-      pry (~> 0.10)
-    rack (2.0.5)
-    rack-protection (2.0.3)
+    multi_json (1.15.0)
+    mustermann (1.1.1)
+      ruby2_keywords (~> 0.0.1)
+    pry (0.13.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    pry-byebug (3.9.0)
+      byebug (~> 11.0)
+      pry (~> 0.13.0)
+    rack (2.2.3)
+    rack-protection (2.1.0)
       rack
-    rake (12.3.1)
-    redis (3.3.5)
-    redis-namespace (1.6.0)
+    rake (13.0.3)
+    redis (4.2.5)
+    redis-namespace (1.8.1)
       redis (>= 3.0.4)
-    redis-semaphore (0.3.1)
-      redis
-    resque (1.27.4)
+    resque (2.0.0)
       mono_logger (~> 1.0)
       multi_json (~> 1.0)
-      redis-namespace (~> 1.3)
+      redis-namespace (~> 1.6)
       sinatra (>= 0.9.2)
       vegas (~> 0.1.2)
-    resque_spec (0.17.0)
-      resque (>= 1.19.0)
+    resque_spec (0.18.1)
+      resque (>= 1.26.0)
       rspec-core (>= 3.0.0)
       rspec-expectations (>= 3.0.0)
       rspec-mocks (>= 3.0.0)
-    rspec (3.7.0)
-      rspec-core (~> 3.7.0)
-      rspec-expectations (~> 3.7.0)
-      rspec-mocks (~> 3.7.0)
-    rspec-core (3.7.1)
-      rspec-support (~> 3.7.0)
-    rspec-expectations (3.7.0)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.1)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-mocks (3.7.0)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-sidekiq (3.0.3)
+      rspec-support (~> 3.10.0)
+    rspec-sidekiq (3.1.0)
       rspec-core (~> 3.0, >= 3.0.0)
       sidekiq (>= 2.4.0)
-    rspec-support (3.7.1)
-    sidekiq (5.1.3)
-      concurrent-ruby (~> 1.0)
-      connection_pool (~> 2.2, >= 2.2.0)
-      rack-protection (>= 1.5.0)
-      redis (>= 3.3.5, < 5)
+    rspec-support (3.10.2)
+    ruby2_keywords (0.0.4)
+    sidekiq (6.1.3)
+      connection_pool (>= 2.2.2)
+      rack (~> 2.0)
+      redis (>= 4.2.0)
     simplecov (0.16.1)
       docile (~> 1.1)
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
-    sinatra (2.0.3)
+    sinatra (2.1.0)
       mustermann (~> 1.0)
-      rack (~> 2.0)
-      rack-protection (= 2.0.3)
+      rack (~> 2.2)
+      rack-protection (= 2.1.0)
       tilt (~> 2.0)
-    statsd-ruby (1.2.1)
-    term-ansicolor (1.6.0)
+    statsd-ruby (1.4.0)
+    sync (0.5.0)
+    term-ansicolor (1.7.1)
       tins (~> 1.0)
-    thor (0.19.4)
+    thor (1.1.0)
     thread_safe (0.3.6)
-    tilt (2.0.8)
-    tins (1.16.3)
-    tzinfo (1.2.5)
+    thwait (0.2.0)
+      e2mmap
+    tilt (2.0.10)
+    tins (1.28.0)
+      sync
+    tzinfo (1.2.9)
       thread_safe (~> 0.1)
     vegas (0.1.11)
       rack (>= 1.0.0)
@@ -119,11 +123,11 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
-  activesupport (~> 4.2.0)
+  activesupport (~> 5.2.0)
   bundler (>= 1.6.0)
-  coveralls (>= 0.7.0)
+  coveralls (>= 0.8.22)
   delayed_job (~> 4.1.0)
-  fakeredis (~> 0.6.0)
+  fakeredis (~> 0.7.0)
   pry (>= 0.9.0)
   pry-byebug (>= 1.3.0)
   rake (>= 10.3.0)
@@ -135,4 +139,4 @@ DEPENDENCIES
   taskinator!
 
 BUNDLED WITH
-   1.16.2
+   2.2.14

data/lib/taskinator.rb

@@ -1,7 +1,6 @@
 require 'json'
 require 'yaml'
 require 'securerandom'
-require 'redis-semaphore'
 require 'benchmark'
 require 'delegate'
 

data/lib/taskinator/definition/builder.rb

@@ -24,7 +24,10 @@ module Taskinator
         raise ArgumentError, 'block' unless block_given?
 
         sub_process = Process.define_sequential_process_for(@definition, options)
-        Builder.new(define_sub_process_task(@process, sub_process, options), @definition, *@args).instance_eval(&block)
+        task = define_sub_process_task(@process, sub_process, options)
+        Builder.new(sub_process, @definition, *@args).instance_eval(&block)
+        @process.tasks << task if sub_process.tasks.any?
+        nil
       end
 
       # defines a sub process of tasks which are executed concurrently
@@ -32,7 +35,10 @@ module Taskinator
         raise ArgumentError, 'block' unless block_given?
 
         sub_process = Process.define_concurrent_process_for(@definition, complete_on, options)
-        Builder.new(define_sub_process_task(@process, sub_process, options), @definition, *@args).instance_eval(&block)
+        task = define_sub_process_task(@process, sub_process, options)
+        Builder.new(sub_process, @definition, *@args).instance_eval(&block)
+        @process.tasks << task if sub_process.tasks.any?
+        nil
       end
 
       # dynamically defines tasks, using the given @iterator method
@@ -51,6 +57,7 @@ module Taskinator
         @executor.send(method, *method_args) do |*args|
           Builder.new(@process, @definition, *args).instance_eval(&block)
         end
+        nil
       end
 
       alias_method :transform, :for_each
@@ -61,6 +68,7 @@ module Taskinator
         raise NoMethodError, method unless @executor.respond_to?(method)
 
         define_step_task(@process, method, @args, options)
+        nil
       end
 
       # defines a task which executes the given @job
@@ -70,6 +78,7 @@ module Taskinator
         raise ArgumentError, 'job' unless job.methods.include?(:perform) || job.instance_methods.include?(:perform)
 
         define_job_task(@process, job, @args, options)
+        nil
       end
 
       # defines a sub process task, for the given @definition
@@ -82,7 +91,10 @@ module Taskinator
         # TODO: decide whether the sub process to dynamically receive arguments
 
         sub_process = definition.create_sub_process(*@args, combine_options(options))
-        Builder.new(define_sub_process_task(@process, sub_process, options), definition, *@args)
+        task = define_sub_process_task(@process, sub_process, options)
+        Builder.new(sub_process, definition, *@args)
+        @process.tasks << task if sub_process.tasks.any?
+        nil
       end
 
     private
@@ -100,10 +112,7 @@ module Taskinator
       end
 
       def define_sub_process_task(process, sub_process, options={})
-        define_task(process) {
-          Task.define_sub_process_task(process, sub_process, combine_options(options))
-        }
-        sub_process
+        Task.define_sub_process_task(process, sub_process, combine_options(options))
       end
 
       def define_task(process)

data/lib/taskinator/persistence.rb

@@ -604,6 +604,9 @@ module Taskinator
       end
 
       def visit_tasks(tasks)
+        @conn.expire "#{@key}:tasks", expire_in
+        @conn.expire "#{@key}.count", expire_in
+        @conn.expire "#{@key}.pending", expire_in
         tasks.each do |task|
           RedisCleanupVisitor.new(@conn, task, expire_in).visit
         end

data/lib/taskinator/process.rb

@@ -199,6 +199,12 @@ module Taskinator
       end
 
       def task_completed(task)
+        # deincrement the count of pending sequential tasks
+        pending = deincr_pending_tasks
+
+        Taskinator.statsd_client.count("taskinator.#{definition.name.underscore.parameterize}.pending", pending)
+        Taskinator.logger.info("Completed task for process '#{uuid}'. Pending is #{pending}.")
+
         next_task = task.next
         if next_task
           next_task.enqueue!
@@ -258,9 +264,6 @@ module Taskinator
       end
 
       def task_completed(task)
-        # skip if failed
-        return if failed?
-
         # deincrement the count of pending concurrent tasks
         pending = deincr_pending_tasks
 

data/lib/taskinator/task.rb

@@ -230,12 +230,12 @@ module Taskinator
         # NNB: if other job types are required, may need to implement how they get invoked here!
         # FIXME: possible implement using ActiveJob instead, so it doesn't matter how the worker is implemented
 
-        if job.instance_of?(Module)
+        if job.respond_to?(:perform)
           # resque
-          job.perform(args)
+          job.perform(*args)
         else
           # delayedjob and sidekiq
-          job.new.perform(args)
+          job.new.perform(*args)
         end
 
         # ASSUMPTION: when the job returns, the task is considered to be complete

data/lib/taskinator/version.rb

@@ -1,3 +1,3 @@
 module Taskinator
-  VERSION = "0.3.14"
+  VERSION = "0.4.2"
 end

data/spec/support/test_flows.rb

@@ -132,4 +132,42 @@ module TestFlows
     end
   end
 
+  module NestedTask
+    extend Taskinator::Definition
+    include Support
+
+    define_process :task_count do
+      task :task_1
+
+      concurrent do
+        task :task_2
+        task :task_3
+
+        sequential do
+          task :task_4
+          task :task_5
+
+          concurrent do
+            task :task_6
+            task :task_7
+
+            sequential do
+              task :task_8
+              task :task_9
+
+            end
+
+            task :task_10
+          end
+
+          task :task_11
+        end
+
+        task :task_12
+      end
+
+      task :task_13
+    end
+  end
+
 end

data/spec/taskinator/definition/builder_spec.rb

@@ -75,6 +75,22 @@ describe Taskinator::Definition::Builder do
       expect(Taskinator::Process).to receive(:define_sequential_process_for).with(definition, options).and_call_original
       subject.sequential(options, &define_block)
     end
+
+    it "adds sub-process task" do
+      block = Proc.new {|p|
+        p.task :task_method
+      }
+      expect(process.tasks).to be_empty
+      subject.sequential(options, &block)
+      expect(process.tasks).to_not be_empty
+    end
+
+    it "ignores sub-processes without tasks" do
+      allow(block).to receive(:call)
+      expect(process.tasks).to be_empty
+      subject.sequential(options, &define_block)
+      expect(process.tasks).to be_empty
+    end
   end
 
   describe "#concurrent" do
@@ -100,6 +116,22 @@ describe Taskinator::Definition::Builder do
       expect(Taskinator::Process).to receive(:define_concurrent_process_for).with(definition, Taskinator::CompleteOn::First, options).and_call_original
       subject.concurrent(Taskinator::CompleteOn::First, options, &define_block)
     end
+
+    it "adds sub-process task" do
+      block = Proc.new {|p|
+        p.task :task_method
+      }
+      expect(process.tasks).to be_empty
+      subject.sequential(options, &block)
+      expect(process.tasks).to_not be_empty
+    end
+
+    it "ignores sub-processes without tasks" do
+      allow(block).to receive(:call)
+      expect(process.tasks).to be_empty
+      subject.sequential(options, &define_block)
+      expect(process.tasks).to be_empty
+    end
   end
 
   describe "#for_each" do
@@ -235,6 +267,22 @@ describe Taskinator::Definition::Builder do
       expect(sub_definition).to receive(:create_sub_process).with(*args, builder_options.merge(options)).and_call_original
       subject.sub_process(sub_definition, options)
     end
+
+    it "adds sub-process task" do
+      block = Proc.new {|p|
+        p.task :task_method
+      }
+      expect(process.tasks).to be_empty
+      subject.sequential(options, &block)
+      expect(process.tasks).to_not be_empty
+    end
+
+    it "ignores sub-processes without tasks" do
+      allow(block).to receive(:call)
+      expect(process.tasks).to be_empty
+      subject.sequential(options, &define_block)
+      expect(process.tasks).to be_empty
+    end
   end
 
 end

data/spec/taskinator/persistence_spec.rb

@@ -370,24 +370,27 @@ describe Taskinator::Persistence, :redis => true do
         TestFlows::Job,
         TestFlows::SubProcess,
         TestFlows::Sequential,
-        TestFlows::Concurrent
+        TestFlows::Concurrent,
+        TestFlows::EmptySequentialProcessTest,
+        TestFlows::EmptyConcurrentProcessTest,
+        TestFlows::NestedTask,
       ].each do |definition|
 
         describe "#{definition.name} expire immediately" do
           it {
-            process = definition.create_process(1)
-
             Taskinator.redis do |conn|
-              expect(conn.hget(process.key, :uuid)).to eq(process.uuid)
+              # sanity check
+              expect(conn.keys).to be_empty
 
-              process.cleanup(0) # immediately
+              process = definition.create_process(1)
 
-              expect(conn.hget(process.key, :uuid)).to be_nil
+              # sanity check
+              expect(conn.hget(process.key, :uuid)).to eq(process.uuid)
 
-              recursively_enumerate_tasks(process.tasks) do |task|
-                expect(conn.hget(task.key, :uuid)).to be_nil
-              end
+              process.cleanup(0) # immediately
 
+              # ensure nothing left behind
+              expect(conn.keys).to be_empty
             end
           }
         end
@@ -396,9 +399,14 @@ describe Taskinator::Persistence, :redis => true do
 
       describe "expires in future" do
         it {
-          process = TestFlows::Task.create_process(1)
-
           Taskinator.redis do |conn|
+
+            # sanity check
+            expect(conn.keys).to be_empty
+
+            process = TestFlows::Task.create_process(1)
+
+            # sanity check
             expect(conn.hget(process.key, :uuid)).to eq(process.uuid)
 
             process.cleanup(2)
@@ -411,12 +419,8 @@ describe Taskinator::Persistence, :redis => true do
 
             sleep 3
 
-            # gone!
-            expect(conn.hget(process.key, :uuid)).to be_nil
-            recursively_enumerate_tasks(process.tasks) do |task|
-              expect(conn.hget(task.key, :uuid)).to be_nil
-            end
-
+            # ensure nothing left behind
+            expect(conn.keys).to be_empty
           end
         }
       end

data/spec/taskinator/process_spec.rb

@@ -288,6 +288,23 @@ describe Taskinator::Process do
         subject.task_completed(task1)
       end
 
+      it "deincrements the pending task count" do
+        tasks.each {|t| subject.tasks << t }
+        task1 = tasks[0]
+        task2 = tasks[1]
+
+        allow(task2).to receive(:enqueue!)
+
+        pending_count = tasks.count
+        allow(subject).to receive(:deincr_pending_tasks) { pending_count -= 1 }
+
+        subject.task_completed(task1)
+        expect(pending_count).to eq(tasks.count - 1)
+
+        subject.task_completed(task2)
+        expect(pending_count).to eq(tasks.count - 2)
+      end
+
       it "completes if no more tasks" do
         tasks.each {|t| subject.tasks << t }
         task2 = tasks[1]
@@ -296,6 +313,30 @@ describe Taskinator::Process do
 
         subject.task_completed(task2)
       end
+
+      it "completes if failed task gets retried" do
+        tasks.each {|t| subject.tasks << t }
+        task1 = tasks[0]
+        task2 = tasks[1]
+
+        allow(task2).to receive(:enqueue!)
+
+        expect(subject).to receive(:fail!).and_call_original
+        expect(subject).to receive(:complete!).and_call_original
+
+        subject.task_completed(task1)
+        expect(subject.completed?).to_not be
+        expect(subject.failed?).to_not be
+
+        subject.task_failed(task2, StandardError.new)
+        expect(subject.completed?).to_not be
+        expect(subject.failed?).to be
+
+        # "retry" the task
+        subject.task_completed(task2)
+        expect(subject.completed?).to be
+        expect(subject.failed?).to_not be
+      end
     end
 
     describe "#tasks_completed?" do
@@ -315,6 +356,28 @@ describe Taskinator::Process do
       end
     end
 
+    describe "#task_failed" do
+      it "fails when tasks fail" do
+        tasks.each {|t| subject.tasks << t }
+
+        error = StandardError.new
+
+        expect(subject).to receive(:fail!).with(error)
+
+        subject.task_failed(tasks.first, error)
+      end
+
+      it "doesn't deincement pending task count" do
+        tasks.each {|t| subject.tasks << t }
+
+        expect(subject).to_not receive(:deincr_pending_tasks)
+
+        error = StandardError.new
+
+        subject.task_failed(tasks.first, error)
+      end
+    end
+
     describe "#accept" do
       it { expect(subject).to be_a(Taskinator::Persistence) }
 
@@ -422,7 +485,8 @@ describe Taskinator::Process do
         process = Taskinator::Process.define_concurrent_process_for(definition, Taskinator::CompleteOn::First)
         tasks.each {|t| process.tasks << t }
 
-        allow(process).to receive(:deincr_pending_tasks) { tasks.count - 1 }
+        pending_count = tasks.count
+        allow(process).to receive(:deincr_pending_tasks) { pending_count -= 1 }
 
         expect(process).to receive(:complete!).once.and_call_original
 
@@ -450,6 +514,82 @@ describe Taskinator::Process do
           expect(process.completed?).to be(false) unless pending_count < 1
         end
       end
+
+      it "deincrements the pending task count" do
+        tasks.each {|t| subject.tasks << t }
+        task1 = tasks[0]
+        task2 = tasks[1]
+
+        pending_count = tasks.count
+        allow(subject).to receive(:deincr_pending_tasks) { pending_count -= 1 }
+
+        subject.task_completed(task1)
+        expect(pending_count).to eq(tasks.count - 1)
+
+        subject.task_completed(task2)
+        expect(pending_count).to eq(tasks.count - 2)
+      end
+
+      describe "completes if failed task gets retried" do
+        it "after first task succeeds" do
+          tasks.each {|t| subject.tasks << t }
+          task1 = tasks[0]
+          task2 = tasks[1]
+
+          pending_count = tasks.count
+          allow(subject).to receive(:deincr_pending_tasks) { pending_count -= 1 }
+          allow(task2).to receive(:enqueue!)
+
+          expect(subject).to receive(:fail!).and_call_original
+          expect(subject).to receive(:complete!).and_call_original
+
+          # first task succeeds
+          subject.task_completed(task1)
+          expect(pending_count).to eq(tasks.count - 1)
+
+          # second task fails
+          subject.task_failed(task2, StandardError.new)
+
+          expect(subject.failed?).to be
+          expect(pending_count).to eq(tasks.count - 1)
+
+          # "retry" the task
+          subject.task_completed(task2)
+
+          expect(pending_count).to eq(tasks.count - 2)
+          expect(subject.failed?).to_not be
+          expect(subject.completed?).to be
+        end
+
+        it "after first task fails" do
+          tasks.each {|t| subject.tasks << t }
+          task1 = tasks[0]
+          task2 = tasks[1]
+
+          pending_count = tasks.count
+          allow(subject).to receive(:deincr_pending_tasks) { pending_count -= 1 }
+          allow(task2).to receive(:enqueue!)
+
+          expect(subject).to receive(:fail!).and_call_original
+          expect(subject).to receive(:complete!).and_call_original
+
+          # first task fails
+          subject.task_failed(task2, StandardError.new)
+          expect(subject.failed?).to be
+          expect(pending_count).to eq(tasks.count)
+
+          # second task succeeds
+          subject.task_completed(task1)
+          expect(pending_count).to eq(tasks.count - 1)
+
+          # "retry" the task
+          subject.task_completed(task2)
+
+          expect(pending_count).to eq(tasks.count - 2)
+          expect(subject.failed?).to_not be
+          expect(subject.completed?).to be
+        end
+      end
     end
 
     describe "#task_failed" do
@@ -462,6 +602,16 @@ describe Taskinator::Process do
 
         subject.task_failed(tasks.first, error)
       end
+
+      it "doesn't deincement pending task count" do
+        tasks.each {|t| subject.tasks << t }
+
+        expect(subject).to_not receive(:deincr_pending_tasks)
+
+        error = StandardError.new
+
+        subject.task_failed(tasks.first, error)
+      end
     end
 
     describe "#tasks_completed?" do

data/spec/taskinator/task_spec.rb

@@ -227,7 +227,7 @@ describe Taskinator::Task do
 
         method = subject.method
 
-        executor.class_eval do
+        executor.singleton_class.class_eval do
           define_method method do |*args|
             # this method executes in the scope of the executor
             # store the context in an instance variable
@@ -334,13 +334,23 @@ describe Taskinator::Task do
       end
     end
 
-    subject { Taskinator::Task.define_job_task(process, TestJob, {:a => 1, :b => 2}) }
+    class TestJobClassNoArgs
+      def perform
+      end
+    end
+
+    module TestJobModuleNoArgs
+      def self.perform
+      end
+    end
+
+    subject { Taskinator::Task.define_job_task(process, TestJob, [1, {:a => 1, :b => 2}]) }
 
     it_should_behave_like "a task", Taskinator::Task::Job
 
     describe ".define_job_task" do
       it "sets the queue to use" do
-        task = Taskinator::Task.define_job_task(process, TestJob, {:a => 1, :b => 2}, :queue => :foo)
+        task = Taskinator::Task.define_job_task(process, TestJob, [1, {:a => 1, :b => 2}], :queue => :foo)
         expect(task.queue).to eq(:foo)
       end
     end
@@ -367,23 +377,37 @@ describe Taskinator::Task do
 
     describe "#start" do
       it {
-        task = Taskinator::Task.define_job_task(process, TestJobClass, {:a => 1, :b => 2})
+        task = Taskinator::Task.define_job_task(process, TestJobClass, [1, {:a => 1, :b => 2}])
+        expect(process).to receive(:task_completed).with(task)
+        expect_any_instance_of(TestJobClass).to receive(:perform).with(1, {:a => 1, :b => 2})
+        task.start!
+      }
+
+      it {
+        task = Taskinator::Task.define_job_task(process, TestJobModule, [2, {:a => 1, :b => 2}])
+        expect(process).to receive(:task_completed).with(task)
+        expect(TestJobModule).to receive(:perform).with(2, {:a => 1, :b => 2})
+        task.start!
+      }
+
+      it {
+        task = Taskinator::Task.define_job_task(process, TestJobClassNoArgs, nil)
         expect(process).to receive(:task_completed).with(task)
-        expect_any_instance_of(TestJobClass).to receive(:perform).with({:a => 1, :b => 2})
+        expect_any_instance_of(TestJobClassNoArgs).to receive(:perform).and_call_original
         task.start!
       }
 
       it {
-        task = Taskinator::Task.define_job_task(process, TestJobModule, {:a => 1, :b => 2})
+        task = Taskinator::Task.define_job_task(process, TestJobModuleNoArgs, nil)
         expect(process).to receive(:task_completed).with(task)
-        expect(TestJobModule).to receive(:perform).with({:a => 1, :b => 2})
+        expect(TestJobModuleNoArgs).to receive(:perform).and_call_original
         task.start!
       }
 
       it "is instrumented" do
         allow(process).to receive(:task_completed).with(subject)
 
-        allow(TestJob).to receive(:perform).with({:a => 1, :b => 2})
+        allow(TestJob).to receive(:perform).with(1, {:a => 1, :b => 2})
 
         instrumentation_block = SpecSupport::Block.new
 

data/taskinator.gemspec

@@ -23,10 +23,11 @@ Gem::Specification.new do |spec|
   # core
   spec.add_dependency 'redis'                       , '>= 3.2.1'
   spec.add_dependency 'redis-namespace'             , '>= 1.5.2'
-  spec.add_dependency 'redis-semaphore'             , '>= 0.2.4'
   spec.add_dependency 'connection_pool'             , '>= 2.2.0'
   spec.add_dependency 'json'                        , '>= 1.8.2'
   spec.add_dependency 'builder'                     , '>= 3.2.2'
   spec.add_dependency 'globalid'                    , '~> 0.3'
-  spec.add_dependency 'statsd-ruby'                 , '~> 1.2.0'
+  spec.add_dependency 'statsd-ruby'                 , '~> 1.4.0'
+  spec.add_dependency 'thwait'                      , '~> 0.2'
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: taskinator
 version: !ruby/object:Gem::Version
-  version: 0.3.14
+  version: 0.4.2
 platform: ruby
 authors:
 - Chris Stefano
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-07-13 00:00:00.000000000 Z
+date: 2021-03-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -38,20 +38,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.5.2
-- !ruby/object:Gem::Dependency
-  name: redis-semaphore
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.2.4
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.2.4
 - !ruby/object:Gem::Dependency
   name: connection_pool
   requirement: !ruby/object:Gem::Requirement
@@ -114,14 +100,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2.0
+        version: 1.4.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+- !ruby/object:Gem::Dependency
+  name: thwait
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2.0
+        version: '0.2'
 description: Simple process orchestration
 email:
 - virtualstaticvoid@gmail.com
@@ -232,8 +232,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: A simple orchestration library for running complex processes or workflows