taskinator 0.3.5 → 0.3.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 1cea1440734a7bcbe3faa2198406e3cd7ebfde21
4
- data.tar.gz: ffbb5077202225399db2dc748010654b5fbf8ef6
3
+ metadata.gz: 3646429986b5ac67608dfa8559660acb89b42fe7
4
+ data.tar.gz: 65421c84b806625c2373d7f09250d55835f5dc5b
5
5
  SHA512:
6
- metadata.gz: 3c57fe34871813ecadb16414ca6bb6333e8dcc5d9de4128ed9ba38c6e1c74aed0ff69ebc8a061955301f42770816d4417db833973a7aaf47ff94413668bd0564
7
- data.tar.gz: 99b036277ab71ad28511ad194c4b183efee3472769f22e7abe66a5270c312272fd2b68a62e89046ea566573fd2b86f937088ac55c989e8d6491ec893f639d501
6
+ metadata.gz: 9a36b04b8b6a957afd3054ccc47f9849b2b14f0c18d99f2b90e77564949bb48716697f968b4ec6d4cf1d369392d938a9448dafc377fb768677cce9aa3faa5994
7
+ data.tar.gz: 8a00ace5324b642eaddde723b351cecfa1cf139953de51ecd10972c9f0266057a372ca8ea7907c31895ca19c82d7b2b016d1c3efa5f3d0676564dedd19a98efc
data/CHANGELOG.md CHANGED
@@ -1,3 +1,9 @@
1
+ v0.3.6 - 11 Nov 2015
2
+ ---
3
+ Added visitor for performing clean up of completed processes/tasks.
4
+ Performance improvement to instrumentation payload; removed references to task/process and use intrinsic types.
5
+ Clean up of keys, via `cleanup` method use key expiry.
6
+
1
7
  v0.3.5 - 02 Nov 2015
2
8
  ---
3
9
  Updated the keys used when persisting processes and tasks in Redis, so they fall in the same key space.
data/Gemfile.lock CHANGED
@@ -1,9 +1,9 @@
1
1
  GIT
2
2
  remote: git://github.com/mperham/sidekiq.git
3
- revision: c556c85b689af3a4f6a1da76c9f8b5208b9753c3
3
+ revision: 8cac86b47d294044bc7a277e4dd0b44faf34f624
4
4
  specs:
5
5
  sidekiq (4.0.0.pre2)
6
- concurrent-ruby (= 1.0.0.pre4)
6
+ concurrent-ruby (~> 1.0.0.pre5)
7
7
  connection_pool (~> 2.2, >= 2.2.0)
8
8
  json (~> 1.0)
9
9
  redis (~> 3.2, >= 3.2.1)
@@ -12,7 +12,7 @@ GIT
12
12
  PATH
13
13
  remote: .
14
14
  specs:
15
- taskinator (0.3.5)
15
+ taskinator (0.3.6)
16
16
  builder (>= 3.2.2)
17
17
  connection_pool (>= 2.2.0)
18
18
  json (>= 1.8.2)
@@ -30,13 +30,11 @@ GEM
30
30
  thread_safe (~> 0.3, >= 0.3.4)
31
31
  tzinfo (~> 1.1)
32
32
  builder (3.2.2)
33
- byebug (5.0.0)
34
- columnize (= 0.9.0)
33
+ byebug (8.1.0)
35
34
  coderay (1.1.0)
36
- columnize (0.9.0)
37
- concurrent-ruby (1.0.0.pre4)
35
+ concurrent-ruby (1.0.0.pre5)
38
36
  connection_pool (2.2.0)
39
- coveralls (0.8.3)
37
+ coveralls (0.8.5)
40
38
  json (~> 1.8)
41
39
  rest-client (>= 1.6.8, < 2)
42
40
  simplecov (~> 0.10.0)
@@ -62,8 +60,8 @@ GEM
62
60
  coderay (~> 1.1.0)
63
61
  method_source (~> 0.8.1)
64
62
  slop (~> 3.4)
65
- pry-byebug (3.2.0)
66
- byebug (~> 5.0)
63
+ pry-byebug (3.3.0)
64
+ byebug (~> 8.0)
67
65
  pry (~> 0.10)
68
66
  rack (1.6.4)
69
67
  rack-protection (1.5.3)
@@ -120,7 +118,7 @@ GEM
120
118
  thor (0.19.1)
121
119
  thread_safe (0.3.5)
122
120
  tilt (2.0.1)
123
- tins (1.6.0)
121
+ tins (1.7.0)
124
122
  tzinfo (1.2.2)
125
123
  thread_safe (~> 0.1)
126
124
  unf (0.1.4)
@@ -57,17 +57,16 @@ module Taskinator
57
57
 
58
58
  return OpenStruct.new(
59
59
  {
60
- :type => self.class,
60
+ :type => self.class.name,
61
61
  :process_uuid => process_uuid,
62
- :process_options => process_options,
62
+ :process_options => process_options.dup,
63
63
  :uuid => uuid,
64
- :options => options,
64
+ :options => options.dup,
65
65
  :state => state,
66
66
  :percentage_failed => (tasks_count > 0) ? (failed_count.to_i / tasks_count) * 100.0 : 0.0,
67
67
  :percentage_cancelled => (tasks_count > 0) ? (cancelled_count.to_i / tasks_count) * 100.0 : 0.0,
68
68
  :percentage_processing => (tasks_count > 0) ? (processing_count.to_i / tasks_count) * 100.0 : 0.0,
69
69
  :percentage_completed => (tasks_count > 0) ? (completed_count.to_i / tasks_count) * 100.0 : 0.0,
70
- :instance => self
71
70
  }.merge(additional)
72
71
  ).freeze
73
72
 
@@ -194,28 +194,23 @@ module Taskinator
194
194
  end
195
195
  end
196
196
 
197
- def cleanup
198
- Taskinator.redis do |conn|
199
-
200
- process_key = self.process_key
197
+ EXPIRE_IN = 10 * 60 # 10 minutes
201
198
 
202
- # delete processes/tasks data
203
- conn.scan_each(:match => "#{process_key}:*", :count => 1000) do |key|
204
- conn.del(key)
205
- end
199
+ def cleanup(expire_at=Time.now+EXPIRE_IN)
200
+ Taskinator.redis do |conn|
206
201
 
207
- # remove the process
208
- conn.del process_key
202
+ # use the "clean up" visitor
203
+ RedisCleanupVisitor.new(conn, self, expire_at).visit
209
204
 
210
205
  # remove from the list
211
- conn.srem Persistence.processes_list_key(scope), uuid
206
+ conn.srem(Persistence.processes_list_key(scope), uuid)
212
207
 
213
208
  end
214
209
  end
215
210
 
216
211
  end
217
212
 
218
- class RedisSerializationVisitor < Visitor::Base
213
+ class RedisSerializationVisitor < Taskinator::Visitor::Base
219
214
 
220
215
  #
221
216
  # the redis connection is passed in since it is
@@ -454,6 +449,36 @@ module Taskinator
454
449
  end
455
450
  end
456
451
 
452
+ class RedisCleanupVisitor < Taskinator::Visitor::Base
453
+
454
+ attr_reader :instance
455
+ attr_reader :expire_at
456
+
457
+ def initialize(conn, instance, expire_at)
458
+ @conn = conn
459
+ @instance = instance
460
+ @expire_at = expire_at.utc
461
+ @key = instance.key
462
+ end
463
+
464
+ def visit
465
+ @instance.accept(self)
466
+ @conn.expireat(@key, expire_at.to_i)
467
+ end
468
+
469
+ def visit_process(attribute)
470
+ process = @instance.send(attribute)
471
+ RedisCleanupVisitor.new(@conn, process, expire_at).visit if process
472
+ end
473
+
474
+ def visit_tasks(tasks)
475
+ tasks.each do |task|
476
+ RedisCleanupVisitor.new(@conn, task, expire_at).visit
477
+ end
478
+ end
479
+
480
+ end
481
+
457
482
  # lazily loads the object specified by the type and uuid
458
483
  class LazyLoader < Delegator
459
484
 
@@ -1,3 +1,3 @@
1
1
  module Taskinator
2
- VERSION = "0.3.5"
2
+ VERSION = "0.3.6"
3
3
  end
@@ -74,7 +74,7 @@ describe Taskinator::Instrumentation, :redis => true do
74
74
 
75
75
  expect(subject.completed_payload(:baz => :qux)).to eq(
76
76
  OpenStruct.new({
77
- :type => subject.class,
77
+ :type => subject.class.name,
78
78
  :process_uuid => subject.uuid,
79
79
  :process_options => {:foo => :bar},
80
80
  :uuid => subject.uuid,
@@ -84,7 +84,6 @@ describe Taskinator::Instrumentation, :redis => true do
84
84
  :percentage_completed => 2.0,
85
85
  :percentage_cancelled => 3.0,
86
86
  :percentage_failed => 4.0,
87
- :instance => subject,
88
87
  :baz => :qux
89
88
  })
90
89
  )
@@ -347,5 +347,65 @@ describe Taskinator::Persistence, :redis => true do
347
347
  }
348
348
  end
349
349
 
350
+ describe "#cleanup" do
351
+
352
+ [
353
+ TestFlows::Task,
354
+ TestFlows::Job,
355
+ TestFlows::SubProcess,
356
+ TestFlows::Sequential,
357
+ TestFlows::Concurrent
358
+ ].each do |definition|
359
+
360
+ describe "#{definition.name} expire immediately" do
361
+ it {
362
+ process = definition.create_process(1)
363
+
364
+ Taskinator.redis do |conn|
365
+ expect(conn.hget(process.key, :uuid)).to eq(process.uuid)
366
+
367
+ process.cleanup(Time.now)
368
+
369
+ expect(conn.hget(process.key, :uuid)).to be_nil
370
+
371
+ recursively_enumerate_tasks(process.tasks) do |task|
372
+ expect(conn.hget(task.key, :uuid)).to be_nil
373
+ end
374
+
375
+ end
376
+ }
377
+ end
378
+
379
+ end
380
+
381
+ describe "expires in future" do
382
+ it {
383
+ process = TestFlows::Task.create_process(1)
384
+
385
+ Taskinator.redis do |conn|
386
+ expect(conn.hget(process.key, :uuid)).to eq(process.uuid)
387
+
388
+ process.cleanup(Time.now + 1)
389
+
390
+ # still available...
391
+ expect(conn.hget(process.key, :uuid)).to_not be_nil
392
+ recursively_enumerate_tasks(process.tasks) do |task|
393
+ expect(conn.hget(task.key, :uuid)).to_not be_nil
394
+ end
395
+
396
+ sleep 2
397
+
398
+ # gone!
399
+ expect(conn.hget(process.key, :uuid)).to be_nil
400
+ recursively_enumerate_tasks(process.tasks) do |task|
401
+ expect(conn.hget(task.key, :uuid)).to be_nil
402
+ end
403
+
404
+ end
405
+ }
406
+ end
407
+
408
+ end
409
+
350
410
  end
351
411
  end
@@ -422,6 +422,7 @@ describe Taskinator::Process do
422
422
  allow_any_instance_of(Taskinator::Task).to receive(:completed?) { true }
423
423
 
424
424
  process = Taskinator::Process.define_concurrent_process_for(definition, Taskinator::CompleteOn::First)
425
+
425
426
  tasks.each {|t| process.tasks << t }
426
427
 
427
428
  expect(process).to receive(:complete!).and_call_original
@@ -438,6 +439,7 @@ describe Taskinator::Process do
438
439
  allow_any_instance_of(Taskinator::Task).to receive(:completed?) { true }
439
440
 
440
441
  process = Taskinator::Process.define_concurrent_process_for(definition, Taskinator::CompleteOn::Last)
442
+
441
443
  tasks.each {|t| process.tasks << t }
442
444
 
443
445
  expect(process).to receive(:complete!).and_call_original
@@ -0,0 +1,5 @@
1
+ require 'spec_helper'
2
+
3
+ describe Taskinator::ProcessWorker do
4
+ pending
5
+ end
@@ -360,7 +360,7 @@ describe TestFlows do
360
360
  if name =~ /taskinator.task.processing/
361
361
  expect(payload[:percentage_completed]).to eq( (invoke_count / task_count.to_f) * 100.0 )
362
362
  elsif name =~ /taskinator.task.completed/
363
- unless payload.type >= Taskinator::Task::SubProcess
363
+ unless payload.type.constantize >= Taskinator::Task::SubProcess
364
364
  invoke_count += 1
365
365
  expect(payload[:percentage_completed]).to eq( (invoke_count / task_count.to_f) * 100.0 )
366
366
  end
@@ -0,0 +1,5 @@
1
+ require 'spec_helper'
2
+
3
+ describe Taskinator::Visitor::XmlVisitor do
4
+ pending
5
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: taskinator
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.5
4
+ version: 0.3.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - Chris Stefano
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-11-02 00:00:00.000000000 Z
11
+ date: 2015-11-11 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: redis
@@ -170,6 +170,7 @@ files:
170
170
  - spec/taskinator/instrumentation_spec.rb
171
171
  - spec/taskinator/persistence_spec.rb
172
172
  - spec/taskinator/process_spec.rb
173
+ - spec/taskinator/process_worker_spec.rb
173
174
  - spec/taskinator/queues/delayed_job_spec.rb
174
175
  - spec/taskinator/queues/resque_spec.rb
175
176
  - spec/taskinator/queues/sidekiq_spec.rb
@@ -180,6 +181,7 @@ files:
180
181
  - spec/taskinator/tasks_spec.rb
181
182
  - spec/taskinator/test_flows_spec.rb
182
183
  - spec/taskinator/visitor_spec.rb
184
+ - spec/taskinator/xml_visitor_spec.rb
183
185
  - taskinator.gemspec
184
186
  - tasks_workflow.png
185
187
  homepage: https://github.com/virtualstaticvoid/taskinator
@@ -235,6 +237,7 @@ test_files:
235
237
  - spec/taskinator/instrumentation_spec.rb
236
238
  - spec/taskinator/persistence_spec.rb
237
239
  - spec/taskinator/process_spec.rb
240
+ - spec/taskinator/process_worker_spec.rb
238
241
  - spec/taskinator/queues/delayed_job_spec.rb
239
242
  - spec/taskinator/queues/resque_spec.rb
240
243
  - spec/taskinator/queues/sidekiq_spec.rb
@@ -245,3 +248,4 @@ test_files:
245
248
  - spec/taskinator/tasks_spec.rb
246
249
  - spec/taskinator/test_flows_spec.rb
247
250
  - spec/taskinator/visitor_spec.rb
251
+ - spec/taskinator/xml_visitor_spec.rb