taskinator 0.3.5 → 0.3.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1cea1440734a7bcbe3faa2198406e3cd7ebfde21
-  data.tar.gz: ffbb5077202225399db2dc748010654b5fbf8ef6
+  metadata.gz: 3646429986b5ac67608dfa8559660acb89b42fe7
+  data.tar.gz: 65421c84b806625c2373d7f09250d55835f5dc5b
 SHA512:
-  metadata.gz: 3c57fe34871813ecadb16414ca6bb6333e8dcc5d9de4128ed9ba38c6e1c74aed0ff69ebc8a061955301f42770816d4417db833973a7aaf47ff94413668bd0564
-  data.tar.gz: 99b036277ab71ad28511ad194c4b183efee3472769f22e7abe66a5270c312272fd2b68a62e89046ea566573fd2b86f937088ac55c989e8d6491ec893f639d501
+  metadata.gz: 9a36b04b8b6a957afd3054ccc47f9849b2b14f0c18d99f2b90e77564949bb48716697f968b4ec6d4cf1d369392d938a9448dafc377fb768677cce9aa3faa5994
+  data.tar.gz: 8a00ace5324b642eaddde723b351cecfa1cf139953de51ecd10972c9f0266057a372ca8ea7907c31895ca19c82d7b2b016d1c3efa5f3d0676564dedd19a98efc

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+v0.3.6 - 11 Nov 2015
+---
+Added visitor for performing clean up of completed processes/tasks.
+Performance improvement to instrumentation payload; removed references to task/process and use intrinsic types.
+Clean up of keys, via `cleanup` method use key expiry.
+
 v0.3.5 - 02 Nov 2015
 ---
 Updated the keys used when persisting processes and tasks in Redis, so they fall in the same key space.

data/Gemfile.lock

@@ -1,9 +1,9 @@
 GIT
   remote: git://github.com/mperham/sidekiq.git
-  revision: c556c85b689af3a4f6a1da76c9f8b5208b9753c3
+  revision: 8cac86b47d294044bc7a277e4dd0b44faf34f624
   specs:
     sidekiq (4.0.0.pre2)
-      concurrent-ruby (= 1.0.0.pre4)
+      concurrent-ruby (~> 1.0.0.pre5)
       connection_pool (~> 2.2, >= 2.2.0)
       json (~> 1.0)
       redis (~> 3.2, >= 3.2.1)
@@ -12,7 +12,7 @@ GIT
 PATH
   remote: .
   specs:
-    taskinator (0.3.5)
+    taskinator (0.3.6)
       builder (>= 3.2.2)
       connection_pool (>= 2.2.0)
       json (>= 1.8.2)
@@ -30,13 +30,11 @@ GEM
       thread_safe (~> 0.3, >= 0.3.4)
       tzinfo (~> 1.1)
     builder (3.2.2)
-    byebug (5.0.0)
-      columnize (= 0.9.0)
+    byebug (8.1.0)
     coderay (1.1.0)
-    columnize (0.9.0)
-    concurrent-ruby (1.0.0.pre4)
+    concurrent-ruby (1.0.0.pre5)
     connection_pool (2.2.0)
-    coveralls (0.8.3)
+    coveralls (0.8.5)
       json (~> 1.8)
       rest-client (>= 1.6.8, < 2)
       simplecov (~> 0.10.0)
@@ -62,8 +60,8 @@ GEM
       coderay (~> 1.1.0)
       method_source (~> 0.8.1)
       slop (~> 3.4)
-    pry-byebug (3.2.0)
-      byebug (~> 5.0)
+    pry-byebug (3.3.0)
+      byebug (~> 8.0)
       pry (~> 0.10)
     rack (1.6.4)
     rack-protection (1.5.3)
@@ -120,7 +118,7 @@ GEM
     thor (0.19.1)
     thread_safe (0.3.5)
     tilt (2.0.1)
-    tins (1.6.0)
+    tins (1.7.0)
     tzinfo (1.2.2)
       thread_safe (~> 0.1)
     unf (0.1.4)

data/lib/taskinator/instrumentation.rb

@@ -57,17 +57,16 @@ module Taskinator
 
       return OpenStruct.new(
         {
-          :type                   => self.class,
+          :type                   => self.class.name,
           :process_uuid           => process_uuid,
-          :process_options        => process_options,
+          :process_options        => process_options.dup,
           :uuid                   => uuid,
-          :options                => options,
+          :options                => options.dup,
           :state                  => state,
           :percentage_failed      => (tasks_count > 0) ? (failed_count.to_i     / tasks_count) * 100.0 : 0.0,
           :percentage_cancelled   => (tasks_count > 0) ? (cancelled_count.to_i  / tasks_count) * 100.0 : 0.0,
           :percentage_processing  => (tasks_count > 0) ? (processing_count.to_i / tasks_count) * 100.0 : 0.0,
           :percentage_completed   => (tasks_count > 0) ? (completed_count.to_i  / tasks_count) * 100.0 : 0.0,
-          :instance               => self
         }.merge(additional)
       ).freeze
 

data/lib/taskinator/persistence.rb

@@ -194,28 +194,23 @@ module Taskinator
         end
       end
 
-      def cleanup
-        Taskinator.redis do |conn|
-
-          process_key = self.process_key
+      EXPIRE_IN = 10 * 60 # 10 minutes
 
-          # delete processes/tasks data
-          conn.scan_each(:match => "#{process_key}:*", :count => 1000) do |key|
-            conn.del(key)
-          end
+      def cleanup(expire_at=Time.now+EXPIRE_IN)
+        Taskinator.redis do |conn|
 
-          # remove the process
-          conn.del process_key
+          # use the "clean up" visitor
+          RedisCleanupVisitor.new(conn, self, expire_at).visit
 
           # remove from the list
-          conn.srem Persistence.processes_list_key(scope), uuid
+          conn.srem(Persistence.processes_list_key(scope), uuid)
 
         end
       end
 
     end
 
-    class RedisSerializationVisitor < Visitor::Base
+    class RedisSerializationVisitor < Taskinator::Visitor::Base
 
       #
       # the redis connection is passed in since it is
@@ -454,6 +449,36 @@ module Taskinator
       end
     end
 
+    class RedisCleanupVisitor < Taskinator::Visitor::Base
+
+      attr_reader :instance
+      attr_reader :expire_at
+
+      def initialize(conn, instance, expire_at)
+        @conn = conn
+        @instance = instance
+        @expire_at = expire_at.utc
+        @key = instance.key
+      end
+
+      def visit
+        @instance.accept(self)
+        @conn.expireat(@key, expire_at.to_i)
+      end
+
+      def visit_process(attribute)
+        process = @instance.send(attribute)
+        RedisCleanupVisitor.new(@conn, process, expire_at).visit if process
+      end
+
+      def visit_tasks(tasks)
+        tasks.each do |task|
+          RedisCleanupVisitor.new(@conn, task, expire_at).visit
+        end
+      end
+
+    end
+
     # lazily loads the object specified by the type and uuid
     class LazyLoader < Delegator
 

data/lib/taskinator/version.rb

@@ -1,3 +1,3 @@
 module Taskinator
-  VERSION = "0.3.5"
+  VERSION = "0.3.6"
 end

data/spec/taskinator/instrumentation_spec.rb

@@ -74,7 +74,7 @@ describe Taskinator::Instrumentation, :redis => true do
 
       expect(subject.completed_payload(:baz => :qux)).to eq(
         OpenStruct.new({
-          :type                   => subject.class,
+          :type                   => subject.class.name,
           :process_uuid           => subject.uuid,
           :process_options        => {:foo => :bar},
           :uuid                   => subject.uuid,
@@ -84,7 +84,6 @@ describe Taskinator::Instrumentation, :redis => true do
           :percentage_completed   => 2.0,
           :percentage_cancelled   => 3.0,
           :percentage_failed      => 4.0,
-          :instance               => subject,
           :baz                    => :qux
         })
       )

data/spec/taskinator/persistence_spec.rb

@@ -347,5 +347,65 @@ describe Taskinator::Persistence, :redis => true do
       }
     end
 
+    describe "#cleanup" do
+
+      [
+        TestFlows::Task,
+        TestFlows::Job,
+        TestFlows::SubProcess,
+        TestFlows::Sequential,
+        TestFlows::Concurrent
+      ].each do |definition|
+
+        describe "#{definition.name} expire immediately" do
+          it {
+            process = definition.create_process(1)
+
+            Taskinator.redis do |conn|
+              expect(conn.hget(process.key, :uuid)).to eq(process.uuid)
+
+              process.cleanup(Time.now)
+
+              expect(conn.hget(process.key, :uuid)).to be_nil
+
+              recursively_enumerate_tasks(process.tasks) do |task|
+                expect(conn.hget(task.key, :uuid)).to be_nil
+              end
+
+            end
+          }
+        end
+
+      end
+
+      describe "expires in future" do
+        it {
+          process = TestFlows::Task.create_process(1)
+
+          Taskinator.redis do |conn|
+            expect(conn.hget(process.key, :uuid)).to eq(process.uuid)
+
+            process.cleanup(Time.now + 1)
+
+            # still available...
+            expect(conn.hget(process.key, :uuid)).to_not be_nil
+            recursively_enumerate_tasks(process.tasks) do |task|
+              expect(conn.hget(task.key, :uuid)).to_not be_nil
+            end
+
+            sleep 2
+
+            # gone!
+            expect(conn.hget(process.key, :uuid)).to be_nil
+            recursively_enumerate_tasks(process.tasks) do |task|
+              expect(conn.hget(task.key, :uuid)).to be_nil
+            end
+
+          end
+        }
+      end
+
+    end
+
   end
 end

data/spec/taskinator/process_spec.rb

@@ -422,6 +422,7 @@ describe Taskinator::Process do
         allow_any_instance_of(Taskinator::Task).to receive(:completed?) { true }
 
         process = Taskinator::Process.define_concurrent_process_for(definition, Taskinator::CompleteOn::First)
+
         tasks.each {|t| process.tasks << t }
 
         expect(process).to receive(:complete!).and_call_original
@@ -438,6 +439,7 @@ describe Taskinator::Process do
         allow_any_instance_of(Taskinator::Task).to receive(:completed?) { true }
 
         process = Taskinator::Process.define_concurrent_process_for(definition, Taskinator::CompleteOn::Last)
+
         tasks.each {|t| process.tasks << t }
 
         expect(process).to receive(:complete!).and_call_original

data/spec/taskinator/process_worker_spec.rb

@@ -0,0 +1,5 @@
+require 'spec_helper'
+
+describe Taskinator::ProcessWorker do
+  pending
+end

data/spec/taskinator/test_flows_spec.rb

@@ -360,7 +360,7 @@ describe TestFlows do
           if name =~ /taskinator.task.processing/
             expect(payload[:percentage_completed]).to eq( (invoke_count / task_count.to_f) * 100.0 )
           elsif name =~ /taskinator.task.completed/
-            unless payload.type >= Taskinator::Task::SubProcess
+            unless payload.type.constantize >= Taskinator::Task::SubProcess
               invoke_count += 1
               expect(payload[:percentage_completed]).to eq( (invoke_count / task_count.to_f) * 100.0 )
             end

data/spec/taskinator/xml_visitor_spec.rb

@@ -0,0 +1,5 @@
+require 'spec_helper'
+
+describe Taskinator::Visitor::XmlVisitor do
+  pending
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: taskinator
 version: !ruby/object:Gem::Version
-  version: 0.3.5
+  version: 0.3.6
 platform: ruby
 authors:
 - Chris Stefano
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-02 00:00:00.000000000 Z
+date: 2015-11-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -170,6 +170,7 @@ files:
 - spec/taskinator/instrumentation_spec.rb
 - spec/taskinator/persistence_spec.rb
 - spec/taskinator/process_spec.rb
+- spec/taskinator/process_worker_spec.rb
 - spec/taskinator/queues/delayed_job_spec.rb
 - spec/taskinator/queues/resque_spec.rb
 - spec/taskinator/queues/sidekiq_spec.rb
@@ -180,6 +181,7 @@ files:
 - spec/taskinator/tasks_spec.rb
 - spec/taskinator/test_flows_spec.rb
 - spec/taskinator/visitor_spec.rb
+- spec/taskinator/xml_visitor_spec.rb
 - taskinator.gemspec
 - tasks_workflow.png
 homepage: https://github.com/virtualstaticvoid/taskinator
@@ -235,6 +237,7 @@ test_files:
 - spec/taskinator/instrumentation_spec.rb
 - spec/taskinator/persistence_spec.rb
 - spec/taskinator/process_spec.rb
+- spec/taskinator/process_worker_spec.rb
 - spec/taskinator/queues/delayed_job_spec.rb
 - spec/taskinator/queues/resque_spec.rb
 - spec/taskinator/queues/sidekiq_spec.rb
@@ -245,3 +248,4 @@ test_files:
 - spec/taskinator/tasks_spec.rb
 - spec/taskinator/test_flows_spec.rb
 - spec/taskinator/visitor_spec.rb
+- spec/taskinator/xml_visitor_spec.rb