taskinator 0.3.14 → 0.4.2

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 15afaaa6ba986d581cfb92e8c7a26beba3385f05cae2a1e2c7c29449f7f00151
4
- data.tar.gz: 93bb6cfdce126a1b325549715f0dfab942e5746358a29c75509fc2b6664feece
3
+ metadata.gz: 5bcb4199fc1cea26510008421e3e1a1cdef9efc81eb5392f6a8913c70ae47ed1
4
+ data.tar.gz: '0266388a941c4e79d39b808d747d31d56dc957f8499ae1d14d553aceab7ac467'
5
5
  SHA512:
6
- metadata.gz: 2bcb650d7e41ecaff1bccd572c9966e4c19c02f2e1c0b46d41fdf6d1bd7f771a7338545b8c15172928eae14473990ce1e40ca8ad739e6e9e46f733ad23358d5d
7
- data.tar.gz: 0faea9c7ed24e3b23dcafb56e84ad15b6b53668d60e7a874c8af273370eaba1e0c29db913f15357b774444d284927cf95daeb3d3f2e5ef0b3d147301527fa332
6
+ metadata.gz: 55f5bc10cf12986f073f2f3dd590ed17abac7e0ee25b8e4531391fd5d02e0297e6c7ed58961a626d98944c01395223fc75b4d2c49d28f31c85bbeca76bc21efe
7
+ data.tar.gz: 2a02585f01df20d868ac294c7daef8414ecefe3023d006869ea40c0aea9831361bdc5633e41787e86e20d4e39d749402baee385563adaf7e9405b7efad212a88
data/.ruby-version CHANGED
@@ -1 +1 @@
1
- ruby-2.5.1
1
+ ruby-2.7.2
data/.travis.yml CHANGED
@@ -1,15 +1,16 @@
1
+ os: linux
2
+ dist: xenial
1
3
  language: ruby
2
- sudo: false # See http://docs.travis-ci.com/user/migrating-from-legacy
3
4
  cache: bundler
4
5
 
5
6
  services:
6
- - redis-server
7
+ - redis
7
8
 
8
9
  rvm:
9
- - 2.2.10
10
- - 2.3.7
11
- - 2.4.4
12
- - 2.5.1
10
+ - 2.5.8
11
+ - 2.6.6
12
+ - 2.7.2
13
+ - 3.0.0
13
14
 
14
15
  script: 'bundle exec rake spec'
15
16
 
data/CHANGELOG.md CHANGED
@@ -1,3 +1,60 @@
1
+ v0.4.2 - 16 Mar 2021
2
+ ---
3
+ Bug fix for process/task keys not expired upon completion.
4
+
5
+ v0.4.1 - 15 Mar 2021
6
+ ---
7
+ Optimisation to exclude sub-processes which don't have any tasks.
8
+ Preparations for upgrade to Ruby 3 and ActiveSupport 6
9
+
10
+ v0.4.0 - 4 Mar 2021
11
+ ---
12
+ Bug fix `job` tasks which have no arguments to the `perform` method.
13
+ Added support for having `perform` method as a class method.
14
+
15
+ v0.3.16 - 17 Feb 2021
16
+ ---
17
+ Bug fix to deincrement pending counts for sequential tasks.
18
+ Bug fix to allow concurrent tasks to be retried (via Resque) and to complete processes.
19
+
20
+ v0.3.15 - 22 Nov 2018
21
+ ---
22
+ Updated dependencies.
23
+
24
+ v0.3.14 - 13 Jul 2018
25
+ ---
26
+ Updated dependencies.
27
+ Removed gemnasium.
28
+
29
+ v0.3.13 - 23 Sep 2017
30
+ ---
31
+ Updated dependencies.
32
+
33
+ v0.3.12 - 23 Sep 2017
34
+ ---
35
+ Spec fixes.
36
+ Updated dependencies.
37
+
38
+ v0.3.11 - 1 Nov 2016
39
+ ---
40
+ Removed `redis-semaphore` gem and use INCRBY to track pending concurrent tasks instead.
41
+ Added instrumentation using statsd.
42
+ Bug fixes to key expiry logic.
43
+ Refactored process and task state transistions.
44
+
45
+ v0.3.10 - 1 Nov 2016
46
+ ---
47
+ Added support for serializing to XML.
48
+ Improvements to process and task states.
49
+
50
+ v0.3.9 - 12 Sep 2016
51
+ ---
52
+ Added benchmark for redis-mutex.
53
+
54
+ v0.3.7 - 18 Aug 2016
55
+ ---
56
+ Bug fix to `option?` method.
57
+
1
58
  v0.3.6 - 11 Nov 2015
2
59
  ---
3
60
  Added visitor for performing clean up of completed processes/tasks.
data/Gemfile CHANGED
@@ -15,10 +15,10 @@ gem 'resque_spec' , '>= 0.16.0'
15
15
  # other
16
16
  gem 'bundler' , '>= 1.6.0'
17
17
  gem 'rake' , '>= 10.3.0'
18
- gem 'activesupport' , '~> 4.2.0'
18
+ gem 'activesupport' , '~> 5.2.0'
19
19
  gem 'rspec'
20
- gem 'coveralls' , '>= 0.7.0'
20
+ gem 'coveralls' , '>= 0.8.22'
21
21
  gem 'pry' , '>= 0.9.0'
22
22
  gem 'pry-byebug' , '>= 1.3.0'
23
23
 
24
- gem 'fakeredis' , '~> 0.6.0'
24
+ gem 'fakeredis' , '~> 0.7.0'
data/Gemfile.lock CHANGED
@@ -1,116 +1,120 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- taskinator (0.3.14)
4
+ taskinator (0.4.2)
5
5
  builder (>= 3.2.2)
6
6
  connection_pool (>= 2.2.0)
7
7
  globalid (~> 0.3)
8
8
  json (>= 1.8.2)
9
9
  redis (>= 3.2.1)
10
10
  redis-namespace (>= 1.5.2)
11
- redis-semaphore (>= 0.2.4)
12
- statsd-ruby (~> 1.2.0)
11
+ statsd-ruby (~> 1.4.0)
12
+ thwait (~> 0.2)
13
13
 
14
14
  GEM
15
15
  remote: https://rubygems.org/
16
16
  specs:
17
- activesupport (4.2.10)
18
- i18n (~> 0.7)
17
+ activesupport (5.2.4.5)
18
+ concurrent-ruby (~> 1.0, >= 1.0.2)
19
+ i18n (>= 0.7, < 2)
19
20
  minitest (~> 5.1)
20
- thread_safe (~> 0.3, >= 0.3.4)
21
21
  tzinfo (~> 1.1)
22
- builder (3.2.3)
23
- byebug (10.0.2)
24
- coderay (1.1.2)
25
- concurrent-ruby (1.0.5)
26
- connection_pool (2.2.2)
27
- coveralls (0.8.22)
22
+ builder (3.2.4)
23
+ byebug (11.1.3)
24
+ coderay (1.1.3)
25
+ concurrent-ruby (1.1.8)
26
+ connection_pool (2.2.3)
27
+ coveralls (0.8.23)
28
28
  json (>= 1.8, < 3)
29
29
  simplecov (~> 0.16.1)
30
30
  term-ansicolor (~> 1.3)
31
- thor (~> 0.19.4)
31
+ thor (>= 0.19.4, < 2.0)
32
32
  tins (~> 1.6)
33
- delayed_job (4.1.5)
34
- activesupport (>= 3.0, < 5.3)
35
- diff-lcs (1.3)
36
- docile (1.3.1)
37
- fakeredis (0.6.0)
38
- redis (~> 3.2)
39
- globalid (0.4.1)
33
+ delayed_job (4.1.9)
34
+ activesupport (>= 3.0, < 6.2)
35
+ diff-lcs (1.4.4)
36
+ docile (1.3.5)
37
+ e2mmap (0.1.0)
38
+ fakeredis (0.7.0)
39
+ redis (>= 3.2, < 5.0)
40
+ globalid (0.4.2)
40
41
  activesupport (>= 4.2.0)
41
- i18n (0.9.5)
42
+ i18n (1.8.9)
42
43
  concurrent-ruby (~> 1.0)
43
- json (2.1.0)
44
- method_source (0.9.0)
45
- minitest (5.11.3)
44
+ json (2.5.1)
45
+ method_source (1.0.0)
46
+ minitest (5.14.4)
46
47
  mono_logger (1.1.0)
47
- multi_json (1.13.1)
48
- mustermann (1.0.2)
49
- pry (0.11.3)
50
- coderay (~> 1.1.0)
51
- method_source (~> 0.9.0)
52
- pry-byebug (3.6.0)
53
- byebug (~> 10.0)
54
- pry (~> 0.10)
55
- rack (2.0.5)
56
- rack-protection (2.0.3)
48
+ multi_json (1.15.0)
49
+ mustermann (1.1.1)
50
+ ruby2_keywords (~> 0.0.1)
51
+ pry (0.13.1)
52
+ coderay (~> 1.1)
53
+ method_source (~> 1.0)
54
+ pry-byebug (3.9.0)
55
+ byebug (~> 11.0)
56
+ pry (~> 0.13.0)
57
+ rack (2.2.3)
58
+ rack-protection (2.1.0)
57
59
  rack
58
- rake (12.3.1)
59
- redis (3.3.5)
60
- redis-namespace (1.6.0)
60
+ rake (13.0.3)
61
+ redis (4.2.5)
62
+ redis-namespace (1.8.1)
61
63
  redis (>= 3.0.4)
62
- redis-semaphore (0.3.1)
63
- redis
64
- resque (1.27.4)
64
+ resque (2.0.0)
65
65
  mono_logger (~> 1.0)
66
66
  multi_json (~> 1.0)
67
- redis-namespace (~> 1.3)
67
+ redis-namespace (~> 1.6)
68
68
  sinatra (>= 0.9.2)
69
69
  vegas (~> 0.1.2)
70
- resque_spec (0.17.0)
71
- resque (>= 1.19.0)
70
+ resque_spec (0.18.1)
71
+ resque (>= 1.26.0)
72
72
  rspec-core (>= 3.0.0)
73
73
  rspec-expectations (>= 3.0.0)
74
74
  rspec-mocks (>= 3.0.0)
75
- rspec (3.7.0)
76
- rspec-core (~> 3.7.0)
77
- rspec-expectations (~> 3.7.0)
78
- rspec-mocks (~> 3.7.0)
79
- rspec-core (3.7.1)
80
- rspec-support (~> 3.7.0)
81
- rspec-expectations (3.7.0)
75
+ rspec (3.10.0)
76
+ rspec-core (~> 3.10.0)
77
+ rspec-expectations (~> 3.10.0)
78
+ rspec-mocks (~> 3.10.0)
79
+ rspec-core (3.10.1)
80
+ rspec-support (~> 3.10.0)
81
+ rspec-expectations (3.10.1)
82
82
  diff-lcs (>= 1.2.0, < 2.0)
83
- rspec-support (~> 3.7.0)
84
- rspec-mocks (3.7.0)
83
+ rspec-support (~> 3.10.0)
84
+ rspec-mocks (3.10.2)
85
85
  diff-lcs (>= 1.2.0, < 2.0)
86
- rspec-support (~> 3.7.0)
87
- rspec-sidekiq (3.0.3)
86
+ rspec-support (~> 3.10.0)
87
+ rspec-sidekiq (3.1.0)
88
88
  rspec-core (~> 3.0, >= 3.0.0)
89
89
  sidekiq (>= 2.4.0)
90
- rspec-support (3.7.1)
91
- sidekiq (5.1.3)
92
- concurrent-ruby (~> 1.0)
93
- connection_pool (~> 2.2, >= 2.2.0)
94
- rack-protection (>= 1.5.0)
95
- redis (>= 3.3.5, < 5)
90
+ rspec-support (3.10.2)
91
+ ruby2_keywords (0.0.4)
92
+ sidekiq (6.1.3)
93
+ connection_pool (>= 2.2.2)
94
+ rack (~> 2.0)
95
+ redis (>= 4.2.0)
96
96
  simplecov (0.16.1)
97
97
  docile (~> 1.1)
98
98
  json (>= 1.8, < 3)
99
99
  simplecov-html (~> 0.10.0)
100
100
  simplecov-html (0.10.2)
101
- sinatra (2.0.3)
101
+ sinatra (2.1.0)
102
102
  mustermann (~> 1.0)
103
- rack (~> 2.0)
104
- rack-protection (= 2.0.3)
103
+ rack (~> 2.2)
104
+ rack-protection (= 2.1.0)
105
105
  tilt (~> 2.0)
106
- statsd-ruby (1.2.1)
107
- term-ansicolor (1.6.0)
106
+ statsd-ruby (1.4.0)
107
+ sync (0.5.0)
108
+ term-ansicolor (1.7.1)
108
109
  tins (~> 1.0)
109
- thor (0.19.4)
110
+ thor (1.1.0)
110
111
  thread_safe (0.3.6)
111
- tilt (2.0.8)
112
- tins (1.16.3)
113
- tzinfo (1.2.5)
112
+ thwait (0.2.0)
113
+ e2mmap
114
+ tilt (2.0.10)
115
+ tins (1.28.0)
116
+ sync
117
+ tzinfo (1.2.9)
114
118
  thread_safe (~> 0.1)
115
119
  vegas (0.1.11)
116
120
  rack (>= 1.0.0)
@@ -119,11 +123,11 @@ PLATFORMS
119
123
  ruby
120
124
 
121
125
  DEPENDENCIES
122
- activesupport (~> 4.2.0)
126
+ activesupport (~> 5.2.0)
123
127
  bundler (>= 1.6.0)
124
- coveralls (>= 0.7.0)
128
+ coveralls (>= 0.8.22)
125
129
  delayed_job (~> 4.1.0)
126
- fakeredis (~> 0.6.0)
130
+ fakeredis (~> 0.7.0)
127
131
  pry (>= 0.9.0)
128
132
  pry-byebug (>= 1.3.0)
129
133
  rake (>= 10.3.0)
@@ -135,4 +139,4 @@ DEPENDENCIES
135
139
  taskinator!
136
140
 
137
141
  BUNDLED WITH
138
- 1.16.2
142
+ 2.2.14
data/lib/taskinator.rb CHANGED
@@ -1,7 +1,6 @@
1
1
  require 'json'
2
2
  require 'yaml'
3
3
  require 'securerandom'
4
- require 'redis-semaphore'
5
4
  require 'benchmark'
6
5
  require 'delegate'
7
6
 
@@ -24,7 +24,10 @@ module Taskinator
24
24
  raise ArgumentError, 'block' unless block_given?
25
25
 
26
26
  sub_process = Process.define_sequential_process_for(@definition, options)
27
- Builder.new(define_sub_process_task(@process, sub_process, options), @definition, *@args).instance_eval(&block)
27
+ task = define_sub_process_task(@process, sub_process, options)
28
+ Builder.new(sub_process, @definition, *@args).instance_eval(&block)
29
+ @process.tasks << task if sub_process.tasks.any?
30
+ nil
28
31
  end
29
32
 
30
33
  # defines a sub process of tasks which are executed concurrently
@@ -32,7 +35,10 @@ module Taskinator
32
35
  raise ArgumentError, 'block' unless block_given?
33
36
 
34
37
  sub_process = Process.define_concurrent_process_for(@definition, complete_on, options)
35
- Builder.new(define_sub_process_task(@process, sub_process, options), @definition, *@args).instance_eval(&block)
38
+ task = define_sub_process_task(@process, sub_process, options)
39
+ Builder.new(sub_process, @definition, *@args).instance_eval(&block)
40
+ @process.tasks << task if sub_process.tasks.any?
41
+ nil
36
42
  end
37
43
 
38
44
  # dynamically defines tasks, using the given @iterator method
@@ -51,6 +57,7 @@ module Taskinator
51
57
  @executor.send(method, *method_args) do |*args|
52
58
  Builder.new(@process, @definition, *args).instance_eval(&block)
53
59
  end
60
+ nil
54
61
  end
55
62
 
56
63
  alias_method :transform, :for_each
@@ -61,6 +68,7 @@ module Taskinator
61
68
  raise NoMethodError, method unless @executor.respond_to?(method)
62
69
 
63
70
  define_step_task(@process, method, @args, options)
71
+ nil
64
72
  end
65
73
 
66
74
  # defines a task which executes the given @job
@@ -70,6 +78,7 @@ module Taskinator
70
78
  raise ArgumentError, 'job' unless job.methods.include?(:perform) || job.instance_methods.include?(:perform)
71
79
 
72
80
  define_job_task(@process, job, @args, options)
81
+ nil
73
82
  end
74
83
 
75
84
  # defines a sub process task, for the given @definition
@@ -82,7 +91,10 @@ module Taskinator
82
91
  # TODO: decide whether the sub process to dynamically receive arguments
83
92
 
84
93
  sub_process = definition.create_sub_process(*@args, combine_options(options))
85
- Builder.new(define_sub_process_task(@process, sub_process, options), definition, *@args)
94
+ task = define_sub_process_task(@process, sub_process, options)
95
+ Builder.new(sub_process, definition, *@args)
96
+ @process.tasks << task if sub_process.tasks.any?
97
+ nil
86
98
  end
87
99
 
88
100
  private
@@ -100,10 +112,7 @@ module Taskinator
100
112
  end
101
113
 
102
114
  def define_sub_process_task(process, sub_process, options={})
103
- define_task(process) {
104
- Task.define_sub_process_task(process, sub_process, combine_options(options))
105
- }
106
- sub_process
115
+ Task.define_sub_process_task(process, sub_process, combine_options(options))
107
116
  end
108
117
 
109
118
  def define_task(process)
@@ -604,6 +604,9 @@ module Taskinator
604
604
  end
605
605
 
606
606
  def visit_tasks(tasks)
607
+ @conn.expire "#{@key}:tasks", expire_in
608
+ @conn.expire "#{@key}.count", expire_in
609
+ @conn.expire "#{@key}.pending", expire_in
607
610
  tasks.each do |task|
608
611
  RedisCleanupVisitor.new(@conn, task, expire_in).visit
609
612
  end
@@ -199,6 +199,12 @@ module Taskinator
199
199
  end
200
200
 
201
201
  def task_completed(task)
202
+ # deincrement the count of pending sequential tasks
203
+ pending = deincr_pending_tasks
204
+
205
+ Taskinator.statsd_client.count("taskinator.#{definition.name.underscore.parameterize}.pending", pending)
206
+ Taskinator.logger.info("Completed task for process '#{uuid}'. Pending is #{pending}.")
207
+
202
208
  next_task = task.next
203
209
  if next_task
204
210
  next_task.enqueue!
@@ -258,9 +264,6 @@ module Taskinator
258
264
  end
259
265
 
260
266
  def task_completed(task)
261
- # skip if failed
262
- return if failed?
263
-
264
267
  # deincrement the count of pending concurrent tasks
265
268
  pending = deincr_pending_tasks
266
269
 
@@ -230,12 +230,12 @@ module Taskinator
230
230
  # NNB: if other job types are required, may need to implement how they get invoked here!
231
231
  # FIXME: possible implement using ActiveJob instead, so it doesn't matter how the worker is implemented
232
232
 
233
- if job.instance_of?(Module)
233
+ if job.respond_to?(:perform)
234
234
  # resque
235
- job.perform(args)
235
+ job.perform(*args)
236
236
  else
237
237
  # delayedjob and sidekiq
238
- job.new.perform(args)
238
+ job.new.perform(*args)
239
239
  end
240
240
 
241
241
  # ASSUMPTION: when the job returns, the task is considered to be complete
@@ -1,3 +1,3 @@
1
1
  module Taskinator
2
- VERSION = "0.3.14"
2
+ VERSION = "0.4.2"
3
3
  end
@@ -132,4 +132,42 @@ module TestFlows
132
132
  end
133
133
  end
134
134
 
135
+ module NestedTask
136
+ extend Taskinator::Definition
137
+ include Support
138
+
139
+ define_process :task_count do
140
+ task :task_1
141
+
142
+ concurrent do
143
+ task :task_2
144
+ task :task_3
145
+
146
+ sequential do
147
+ task :task_4
148
+ task :task_5
149
+
150
+ concurrent do
151
+ task :task_6
152
+ task :task_7
153
+
154
+ sequential do
155
+ task :task_8
156
+ task :task_9
157
+
158
+ end
159
+
160
+ task :task_10
161
+ end
162
+
163
+ task :task_11
164
+ end
165
+
166
+ task :task_12
167
+ end
168
+
169
+ task :task_13
170
+ end
171
+ end
172
+
135
173
  end
@@ -75,6 +75,22 @@ describe Taskinator::Definition::Builder do
75
75
  expect(Taskinator::Process).to receive(:define_sequential_process_for).with(definition, options).and_call_original
76
76
  subject.sequential(options, &define_block)
77
77
  end
78
+
79
+ it "adds sub-process task" do
80
+ block = Proc.new {|p|
81
+ p.task :task_method
82
+ }
83
+ expect(process.tasks).to be_empty
84
+ subject.sequential(options, &block)
85
+ expect(process.tasks).to_not be_empty
86
+ end
87
+
88
+ it "ignores sub-processes without tasks" do
89
+ allow(block).to receive(:call)
90
+ expect(process.tasks).to be_empty
91
+ subject.sequential(options, &define_block)
92
+ expect(process.tasks).to be_empty
93
+ end
78
94
  end
79
95
 
80
96
  describe "#concurrent" do
@@ -100,6 +116,22 @@ describe Taskinator::Definition::Builder do
100
116
  expect(Taskinator::Process).to receive(:define_concurrent_process_for).with(definition, Taskinator::CompleteOn::First, options).and_call_original
101
117
  subject.concurrent(Taskinator::CompleteOn::First, options, &define_block)
102
118
  end
119
+
120
+ it "adds sub-process task" do
121
+ block = Proc.new {|p|
122
+ p.task :task_method
123
+ }
124
+ expect(process.tasks).to be_empty
125
+ subject.sequential(options, &block)
126
+ expect(process.tasks).to_not be_empty
127
+ end
128
+
129
+ it "ignores sub-processes without tasks" do
130
+ allow(block).to receive(:call)
131
+ expect(process.tasks).to be_empty
132
+ subject.sequential(options, &define_block)
133
+ expect(process.tasks).to be_empty
134
+ end
103
135
  end
104
136
 
105
137
  describe "#for_each" do
@@ -235,6 +267,22 @@ describe Taskinator::Definition::Builder do
235
267
  expect(sub_definition).to receive(:create_sub_process).with(*args, builder_options.merge(options)).and_call_original
236
268
  subject.sub_process(sub_definition, options)
237
269
  end
270
+
271
+ it "adds sub-process task" do
272
+ block = Proc.new {|p|
273
+ p.task :task_method
274
+ }
275
+ expect(process.tasks).to be_empty
276
+ subject.sequential(options, &block)
277
+ expect(process.tasks).to_not be_empty
278
+ end
279
+
280
+ it "ignores sub-processes without tasks" do
281
+ allow(block).to receive(:call)
282
+ expect(process.tasks).to be_empty
283
+ subject.sequential(options, &define_block)
284
+ expect(process.tasks).to be_empty
285
+ end
238
286
  end
239
287
 
240
288
  end
@@ -370,24 +370,27 @@ describe Taskinator::Persistence, :redis => true do
370
370
  TestFlows::Job,
371
371
  TestFlows::SubProcess,
372
372
  TestFlows::Sequential,
373
- TestFlows::Concurrent
373
+ TestFlows::Concurrent,
374
+ TestFlows::EmptySequentialProcessTest,
375
+ TestFlows::EmptyConcurrentProcessTest,
376
+ TestFlows::NestedTask,
374
377
  ].each do |definition|
375
378
 
376
379
  describe "#{definition.name} expire immediately" do
377
380
  it {
378
- process = definition.create_process(1)
379
-
380
381
  Taskinator.redis do |conn|
381
- expect(conn.hget(process.key, :uuid)).to eq(process.uuid)
382
+ # sanity check
383
+ expect(conn.keys).to be_empty
382
384
 
383
- process.cleanup(0) # immediately
385
+ process = definition.create_process(1)
384
386
 
385
- expect(conn.hget(process.key, :uuid)).to be_nil
387
+ # sanity check
388
+ expect(conn.hget(process.key, :uuid)).to eq(process.uuid)
386
389
 
387
- recursively_enumerate_tasks(process.tasks) do |task|
388
- expect(conn.hget(task.key, :uuid)).to be_nil
389
- end
390
+ process.cleanup(0) # immediately
390
391
 
392
+ # ensure nothing left behind
393
+ expect(conn.keys).to be_empty
391
394
  end
392
395
  }
393
396
  end
@@ -396,9 +399,14 @@ describe Taskinator::Persistence, :redis => true do
396
399
 
397
400
  describe "expires in future" do
398
401
  it {
399
- process = TestFlows::Task.create_process(1)
400
-
401
402
  Taskinator.redis do |conn|
403
+
404
+ # sanity check
405
+ expect(conn.keys).to be_empty
406
+
407
+ process = TestFlows::Task.create_process(1)
408
+
409
+ # sanity check
402
410
  expect(conn.hget(process.key, :uuid)).to eq(process.uuid)
403
411
 
404
412
  process.cleanup(2)
@@ -411,12 +419,8 @@ describe Taskinator::Persistence, :redis => true do
411
419
 
412
420
  sleep 3
413
421
 
414
- # gone!
415
- expect(conn.hget(process.key, :uuid)).to be_nil
416
- recursively_enumerate_tasks(process.tasks) do |task|
417
- expect(conn.hget(task.key, :uuid)).to be_nil
418
- end
419
-
422
+ # ensure nothing left behind
423
+ expect(conn.keys).to be_empty
420
424
  end
421
425
  }
422
426
  end
@@ -288,6 +288,23 @@ describe Taskinator::Process do
288
288
  subject.task_completed(task1)
289
289
  end
290
290
 
291
+ it "deincrements the pending task count" do
292
+ tasks.each {|t| subject.tasks << t }
293
+ task1 = tasks[0]
294
+ task2 = tasks[1]
295
+
296
+ allow(task2).to receive(:enqueue!)
297
+
298
+ pending_count = tasks.count
299
+ allow(subject).to receive(:deincr_pending_tasks) { pending_count -= 1 }
300
+
301
+ subject.task_completed(task1)
302
+ expect(pending_count).to eq(tasks.count - 1)
303
+
304
+ subject.task_completed(task2)
305
+ expect(pending_count).to eq(tasks.count - 2)
306
+ end
307
+
291
308
  it "completes if no more tasks" do
292
309
  tasks.each {|t| subject.tasks << t }
293
310
  task2 = tasks[1]
@@ -296,6 +313,30 @@ describe Taskinator::Process do
296
313
 
297
314
  subject.task_completed(task2)
298
315
  end
316
+
317
+ it "completes if failed task gets retried" do
318
+ tasks.each {|t| subject.tasks << t }
319
+ task1 = tasks[0]
320
+ task2 = tasks[1]
321
+
322
+ allow(task2).to receive(:enqueue!)
323
+
324
+ expect(subject).to receive(:fail!).and_call_original
325
+ expect(subject).to receive(:complete!).and_call_original
326
+
327
+ subject.task_completed(task1)
328
+ expect(subject.completed?).to_not be
329
+ expect(subject.failed?).to_not be
330
+
331
+ subject.task_failed(task2, StandardError.new)
332
+ expect(subject.completed?).to_not be
333
+ expect(subject.failed?).to be
334
+
335
+ # "retry" the task
336
+ subject.task_completed(task2)
337
+ expect(subject.completed?).to be
338
+ expect(subject.failed?).to_not be
339
+ end
299
340
  end
300
341
 
301
342
  describe "#tasks_completed?" do
@@ -315,6 +356,28 @@ describe Taskinator::Process do
315
356
  end
316
357
  end
317
358
 
359
+ describe "#task_failed" do
360
+ it "fails when tasks fail" do
361
+ tasks.each {|t| subject.tasks << t }
362
+
363
+ error = StandardError.new
364
+
365
+ expect(subject).to receive(:fail!).with(error)
366
+
367
+ subject.task_failed(tasks.first, error)
368
+ end
369
+
370
+ it "doesn't deincement pending task count" do
371
+ tasks.each {|t| subject.tasks << t }
372
+
373
+ expect(subject).to_not receive(:deincr_pending_tasks)
374
+
375
+ error = StandardError.new
376
+
377
+ subject.task_failed(tasks.first, error)
378
+ end
379
+ end
380
+
318
381
  describe "#accept" do
319
382
  it { expect(subject).to be_a(Taskinator::Persistence) }
320
383
 
@@ -422,7 +485,8 @@ describe Taskinator::Process do
422
485
  process = Taskinator::Process.define_concurrent_process_for(definition, Taskinator::CompleteOn::First)
423
486
  tasks.each {|t| process.tasks << t }
424
487
 
425
- allow(process).to receive(:deincr_pending_tasks) { tasks.count - 1 }
488
+ pending_count = tasks.count
489
+ allow(process).to receive(:deincr_pending_tasks) { pending_count -= 1 }
426
490
 
427
491
  expect(process).to receive(:complete!).once.and_call_original
428
492
 
@@ -450,6 +514,82 @@ describe Taskinator::Process do
450
514
  expect(process.completed?).to be(false) unless pending_count < 1
451
515
  end
452
516
  end
517
+
518
+ it "deincrements the pending task count" do
519
+ tasks.each {|t| subject.tasks << t }
520
+ task1 = tasks[0]
521
+ task2 = tasks[1]
522
+
523
+ pending_count = tasks.count
524
+ allow(subject).to receive(:deincr_pending_tasks) { pending_count -= 1 }
525
+
526
+ subject.task_completed(task1)
527
+ expect(pending_count).to eq(tasks.count - 1)
528
+
529
+ subject.task_completed(task2)
530
+ expect(pending_count).to eq(tasks.count - 2)
531
+ end
532
+
533
+ describe "completes if failed task gets retried" do
534
+ it "after first task succeeds" do
535
+ tasks.each {|t| subject.tasks << t }
536
+ task1 = tasks[0]
537
+ task2 = tasks[1]
538
+
539
+ pending_count = tasks.count
540
+ allow(subject).to receive(:deincr_pending_tasks) { pending_count -= 1 }
541
+ allow(task2).to receive(:enqueue!)
542
+
543
+ expect(subject).to receive(:fail!).and_call_original
544
+ expect(subject).to receive(:complete!).and_call_original
545
+
546
+ # first task succeeds
547
+ subject.task_completed(task1)
548
+ expect(pending_count).to eq(tasks.count - 1)
549
+
550
+ # second task fails
551
+ subject.task_failed(task2, StandardError.new)
552
+
553
+ expect(subject.failed?).to be
554
+ expect(pending_count).to eq(tasks.count - 1)
555
+
556
+ # "retry" the task
557
+ subject.task_completed(task2)
558
+
559
+ expect(pending_count).to eq(tasks.count - 2)
560
+ expect(subject.failed?).to_not be
561
+ expect(subject.completed?).to be
562
+ end
563
+
564
+ it "after first task fails" do
565
+ tasks.each {|t| subject.tasks << t }
566
+ task1 = tasks[0]
567
+ task2 = tasks[1]
568
+
569
+ pending_count = tasks.count
570
+ allow(subject).to receive(:deincr_pending_tasks) { pending_count -= 1 }
571
+ allow(task2).to receive(:enqueue!)
572
+
573
+ expect(subject).to receive(:fail!).and_call_original
574
+ expect(subject).to receive(:complete!).and_call_original
575
+
576
+ # first task fails
577
+ subject.task_failed(task2, StandardError.new)
578
+ expect(subject.failed?).to be
579
+ expect(pending_count).to eq(tasks.count)
580
+
581
+ # second task succeeds
582
+ subject.task_completed(task1)
583
+ expect(pending_count).to eq(tasks.count - 1)
584
+
585
+ # "retry" the task
586
+ subject.task_completed(task2)
587
+
588
+ expect(pending_count).to eq(tasks.count - 2)
589
+ expect(subject.failed?).to_not be
590
+ expect(subject.completed?).to be
591
+ end
592
+ end
453
593
  end
454
594
 
455
595
  describe "#task_failed" do
@@ -462,6 +602,16 @@ describe Taskinator::Process do
462
602
 
463
603
  subject.task_failed(tasks.first, error)
464
604
  end
605
+
606
+ it "doesn't deincement pending task count" do
607
+ tasks.each {|t| subject.tasks << t }
608
+
609
+ expect(subject).to_not receive(:deincr_pending_tasks)
610
+
611
+ error = StandardError.new
612
+
613
+ subject.task_failed(tasks.first, error)
614
+ end
465
615
  end
466
616
 
467
617
  describe "#tasks_completed?" do
@@ -227,7 +227,7 @@ describe Taskinator::Task do
227
227
 
228
228
  method = subject.method
229
229
 
230
- executor.class_eval do
230
+ executor.singleton_class.class_eval do
231
231
  define_method method do |*args|
232
232
  # this method executes in the scope of the executor
233
233
  # store the context in an instance variable
@@ -334,13 +334,23 @@ describe Taskinator::Task do
334
334
  end
335
335
  end
336
336
 
337
- subject { Taskinator::Task.define_job_task(process, TestJob, {:a => 1, :b => 2}) }
337
+ class TestJobClassNoArgs
338
+ def perform
339
+ end
340
+ end
341
+
342
+ module TestJobModuleNoArgs
343
+ def self.perform
344
+ end
345
+ end
346
+
347
+ subject { Taskinator::Task.define_job_task(process, TestJob, [1, {:a => 1, :b => 2}]) }
338
348
 
339
349
  it_should_behave_like "a task", Taskinator::Task::Job
340
350
 
341
351
  describe ".define_job_task" do
342
352
  it "sets the queue to use" do
343
- task = Taskinator::Task.define_job_task(process, TestJob, {:a => 1, :b => 2}, :queue => :foo)
353
+ task = Taskinator::Task.define_job_task(process, TestJob, [1, {:a => 1, :b => 2}], :queue => :foo)
344
354
  expect(task.queue).to eq(:foo)
345
355
  end
346
356
  end
@@ -367,23 +377,37 @@ describe Taskinator::Task do
367
377
 
368
378
  describe "#start" do
369
379
  it {
370
- task = Taskinator::Task.define_job_task(process, TestJobClass, {:a => 1, :b => 2})
380
+ task = Taskinator::Task.define_job_task(process, TestJobClass, [1, {:a => 1, :b => 2}])
381
+ expect(process).to receive(:task_completed).with(task)
382
+ expect_any_instance_of(TestJobClass).to receive(:perform).with(1, {:a => 1, :b => 2})
383
+ task.start!
384
+ }
385
+
386
+ it {
387
+ task = Taskinator::Task.define_job_task(process, TestJobModule, [2, {:a => 1, :b => 2}])
388
+ expect(process).to receive(:task_completed).with(task)
389
+ expect(TestJobModule).to receive(:perform).with(2, {:a => 1, :b => 2})
390
+ task.start!
391
+ }
392
+
393
+ it {
394
+ task = Taskinator::Task.define_job_task(process, TestJobClassNoArgs, nil)
371
395
  expect(process).to receive(:task_completed).with(task)
372
- expect_any_instance_of(TestJobClass).to receive(:perform).with({:a => 1, :b => 2})
396
+ expect_any_instance_of(TestJobClassNoArgs).to receive(:perform).and_call_original
373
397
  task.start!
374
398
  }
375
399
 
376
400
  it {
377
- task = Taskinator::Task.define_job_task(process, TestJobModule, {:a => 1, :b => 2})
401
+ task = Taskinator::Task.define_job_task(process, TestJobModuleNoArgs, nil)
378
402
  expect(process).to receive(:task_completed).with(task)
379
- expect(TestJobModule).to receive(:perform).with({:a => 1, :b => 2})
403
+ expect(TestJobModuleNoArgs).to receive(:perform).and_call_original
380
404
  task.start!
381
405
  }
382
406
 
383
407
  it "is instrumented" do
384
408
  allow(process).to receive(:task_completed).with(subject)
385
409
 
386
- allow(TestJob).to receive(:perform).with({:a => 1, :b => 2})
410
+ allow(TestJob).to receive(:perform).with(1, {:a => 1, :b => 2})
387
411
 
388
412
  instrumentation_block = SpecSupport::Block.new
389
413
 
data/taskinator.gemspec CHANGED
@@ -23,10 +23,11 @@ Gem::Specification.new do |spec|
23
23
  # core
24
24
  spec.add_dependency 'redis' , '>= 3.2.1'
25
25
  spec.add_dependency 'redis-namespace' , '>= 1.5.2'
26
- spec.add_dependency 'redis-semaphore' , '>= 0.2.4'
27
26
  spec.add_dependency 'connection_pool' , '>= 2.2.0'
28
27
  spec.add_dependency 'json' , '>= 1.8.2'
29
28
  spec.add_dependency 'builder' , '>= 3.2.2'
30
29
  spec.add_dependency 'globalid' , '~> 0.3'
31
- spec.add_dependency 'statsd-ruby' , '~> 1.2.0'
30
+ spec.add_dependency 'statsd-ruby' , '~> 1.4.0'
31
+ spec.add_dependency 'thwait' , '~> 0.2'
32
+
32
33
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: taskinator
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.14
4
+ version: 0.4.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Chris Stefano
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-07-13 00:00:00.000000000 Z
11
+ date: 2021-03-16 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: redis
@@ -38,20 +38,6 @@ dependencies:
38
38
  - - ">="
39
39
  - !ruby/object:Gem::Version
40
40
  version: 1.5.2
41
- - !ruby/object:Gem::Dependency
42
- name: redis-semaphore
43
- requirement: !ruby/object:Gem::Requirement
44
- requirements:
45
- - - ">="
46
- - !ruby/object:Gem::Version
47
- version: 0.2.4
48
- type: :runtime
49
- prerelease: false
50
- version_requirements: !ruby/object:Gem::Requirement
51
- requirements:
52
- - - ">="
53
- - !ruby/object:Gem::Version
54
- version: 0.2.4
55
41
  - !ruby/object:Gem::Dependency
56
42
  name: connection_pool
57
43
  requirement: !ruby/object:Gem::Requirement
@@ -114,14 +100,28 @@ dependencies:
114
100
  requirements:
115
101
  - - "~>"
116
102
  - !ruby/object:Gem::Version
117
- version: 1.2.0
103
+ version: 1.4.0
104
+ type: :runtime
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - "~>"
109
+ - !ruby/object:Gem::Version
110
+ version: 1.4.0
111
+ - !ruby/object:Gem::Dependency
112
+ name: thwait
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - "~>"
116
+ - !ruby/object:Gem::Version
117
+ version: '0.2'
118
118
  type: :runtime
119
119
  prerelease: false
120
120
  version_requirements: !ruby/object:Gem::Requirement
121
121
  requirements:
122
122
  - - "~>"
123
123
  - !ruby/object:Gem::Version
124
- version: 1.2.0
124
+ version: '0.2'
125
125
  description: Simple process orchestration
126
126
  email:
127
127
  - virtualstaticvoid@gmail.com
@@ -232,8 +232,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
232
232
  - !ruby/object:Gem::Version
233
233
  version: '0'
234
234
  requirements: []
235
- rubyforge_project:
236
- rubygems_version: 2.7.6
235
+ rubygems_version: 3.1.4
237
236
  signing_key:
238
237
  specification_version: 4
239
238
  summary: A simple orchestration library for running complex processes or workflows