taskinator 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1905dc0aad01e4bd348ad38a40100df9649e3a18
-  data.tar.gz: d1e4aa938c9d3a0d56fc4090f71024dec84e32fd
+  metadata.gz: 683a1ffedb432057d353d7ebbc31dff766bce83c
+  data.tar.gz: 92c78c8a6e2c87769269147557c9c2cb13be52fa
 SHA512:
-  metadata.gz: 6cd0c7d2fa8d1f133ddce568ae969fe488401ba9f6311c39143456749b4eea1429edbc63ec080a5a8b60fb84bf312bdcd5758084269b16ae83b7b437877e15d1
-  data.tar.gz: dfcbeced7346f8c0bdcc70e5814e2b2e943ecd10182ac0285a88e8e0bdfc0eb1a227c900390328861bb4012e4feffbf8fe7d41033583d68eb1f25d198f1ef027
+  metadata.gz: d9449a313b53581d06109e16d6f1a268a5f786848fc4d725e0852b773dbe99b1a68898a88c0d077bfb48590b976dbc4ca821e9207f20b50343f924493c35d9aa
+  data.tar.gz: 5c621a66e20dfef0f7800482ea5dd308c7470393537022791d3398a4f12c9350ca40a89267de82a149adcaf5c8561ae8b6aa732826ab85843e0f330cb701396f

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+v0.3.0 - 28 Aug 2015
+---
+Added created_at and updated_at to process and task as attributes.
+Improved serialization visitor to include an optional converter block for deserialization of attribute values.
+Corrections to lazy loader logic and speed improvements.
+Removed JobWorker as it's no longer necessary.
+Improvements to instrumentation.
+Removed workflow gem, and refactored process and task to implement the basics instead.
+Several bug fixes.
+
 v0.2.0 - 31 Jul 2015
 ---
 Bug fix for `create_process_remotely` so that it returns the process uuid instead of nil.

data/Gemfile

@@ -3,5 +3,20 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in Taskinator.gemspec
 gemspec
 
-# add as a git gem dependency until version 1.3.0 is released to RubyGems.
-gem 'workflow', :git => 'https://github.com/virtualstaticvoid/workflow.git', :branch => :master
+# queues
+gem 'sidekiq'         , '>= 3.5.0', :github => "mperham/sidekiq"
+gem 'rspec-sidekiq'   , '>= 2.1.0'
+
+gem 'delayed_job'     , '>= 4.0.0'
+
+gem 'resque'          , '>= 1.25.2'
+gem 'resque_spec'     , '>= 0.16.0'
+
+# other
+gem 'bundler'         , '>= 1.6.0'
+gem 'rake'            , '>= 10.3.0'
+gem 'activesupport'   , '>= 4.0.0'
+gem 'rspec'
+gem 'coveralls'       , '>= 0.7.0'
+gem 'pry'             , '>= 0.9.0'
+gem 'pry-byebug'      , '>= 1.3.0'

data/Gemfile.lock

@@ -1,14 +1,18 @@
 GIT
-  remote: https://github.com/virtualstaticvoid/workflow.git
-  revision: 403a9e44bf49f4d154156d5701e3d67b115ed6da
-  branch: master
+  remote: git://github.com/mperham/sidekiq.git
+  revision: b0cdd4cc868e7564b78ee34965067e025ec41689
   specs:
-    workflow (1.3.0)
+    sidekiq (3.5.0)
+      celluloid (~> 0.17.0)
+      connection_pool (~> 2.2, >= 2.2.0)
+      json (~> 1.0)
+      redis (~> 3.2, >= 3.2.1)
+      redis-namespace (~> 1.5, >= 1.5.2)
 
 PATH
   remote: .
   specs:
-    taskinator (0.2.0)
+    taskinator (0.3.0)
       connection_pool (>= 2.2.0)
       json (>= 1.8.2)
       redis (>= 3.2.1)
@@ -25,7 +29,46 @@ GEM
       tzinfo (~> 1.1)
     byebug (5.0.0)
       columnize (= 0.9.0)
-    celluloid (0.16.0)
+    celluloid (0.17.1.1)
+      bundler
+      celluloid-essentials
+      celluloid-extras
+      celluloid-fsm
+      celluloid-pool
+      celluloid-supervision
+      dotenv
+      nenv
+      rspec-logsplit (>= 0.1.2)
+      timers (~> 4.0.0)
+    celluloid-essentials (0.20.2)
+      bundler
+      dotenv
+      nenv
+      rspec-logsplit (>= 0.1.2)
+      timers (~> 4.0.0)
+    celluloid-extras (0.20.0)
+      bundler
+      dotenv
+      nenv
+      rspec-logsplit (>= 0.1.2)
+      timers (~> 4.0.0)
+    celluloid-fsm (0.20.0)
+      bundler
+      dotenv
+      nenv
+      rspec-logsplit (>= 0.1.2)
+      timers (~> 4.0.0)
+    celluloid-pool (0.20.0)
+      bundler
+      dotenv
+      nenv
+      rspec-logsplit (>= 0.1.2)
+      timers (~> 4.0.0)
+    celluloid-supervision (0.20.1)
+      bundler
+      dotenv
+      nenv
+      rspec-logsplit (>= 0.1.2)
       timers (~> 4.0.0)
     coderay (1.1.0)
     columnize (0.9.0)
@@ -42,6 +85,7 @@ GEM
     docile (1.1.5)
     domain_name (0.5.24)
       unf (>= 0.0.5, < 1.0.0)
+    dotenv (2.0.2)
     hitimes (1.2.2)
     http-cookie (1.0.2)
       domain_name (~> 0.5)
@@ -49,9 +93,10 @@ GEM
     json (1.8.3)
     method_source (0.8.2)
     mime-types (2.6.1)
-    minitest (5.7.0)
+    minitest (5.8.0)
     mono_logger (1.1.0)
     multi_json (1.11.2)
+    nenv (0.2.0)
     netrc (0.10.3)
     pry (0.10.1)
       coderay (~> 1.1.0)
@@ -91,19 +136,14 @@ GEM
     rspec-expectations (3.3.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.3.0)
+    rspec-logsplit (0.1.3)
     rspec-mocks (3.3.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.3.0)
-    rspec-sidekiq (2.0.0)
+    rspec-sidekiq (2.1.0)
       rspec (~> 3.0, >= 3.0.0)
       sidekiq (>= 2.4.0)
     rspec-support (3.3.0)
-    sidekiq (3.4.2)
-      celluloid (~> 0.16.0)
-      connection_pool (~> 2.2, >= 2.2.0)
-      json (~> 1.0)
-      redis (~> 3.2, >= 3.2.1)
-      redis-namespace (~> 1.5, >= 1.5.2)
     simplecov (0.10.0)
       docile (~> 1.1.0)
       json (~> 1.8)
@@ -121,7 +161,7 @@ GEM
     tilt (2.0.1)
     timers (4.0.1)
       hitimes
-    tins (1.5.4)
+    tins (1.6.0)
     tzinfo (1.2.2)
       thread_safe (~> 0.1)
     unf (0.1.4)
@@ -144,7 +184,6 @@ DEPENDENCIES
   resque (>= 1.25.2)
   resque_spec (>= 0.16.0)
   rspec
-  rspec-sidekiq (>= 2.0.0)
-  sidekiq (>= 3.0.0)
+  rspec-sidekiq (>= 2.1.0)
+  sidekiq (>= 3.5.0)!
   taskinator!
-  workflow!

data/README.md

@@ -546,7 +546,7 @@ To monitor the state of the processes, use the `Taskinator::Api::Processes` clas
 processes = Taskinator::Api::Processes.new
 processes.each do |process|
   # => output the unique process identifier and current state
-  puts [:process, process.uuid, process.current_state.name]
+  puts [:process, process.uuid, process.current_state]
 end
 ```
 
@@ -608,29 +608,33 @@ end
 
 The following instrumentation events are issued:
 
-| Event                              | When                                        |
-|------------------------------------|---------------------------------------------|
-| `taskinator.process.created`       | After a process gets created                |
-| `taskinator.process.saved`         | After a process has been persisted to Redis |
-| `taskinator.process.enqueued`      | After a process is enqueued for processing  |
-| `taskinator.process.completed`     | After a process has completed processing    |
-| `taskinator.process.cancelled`     | After a process has been cancelled          |
-| `taskinator.process.failed`        | After a process has failed                  |
-| `taskinator.task.enqueued`         | After a task has been enqueued              |
-| `taskinator.task.executed`         | After a task has executed                   |
-| `taskinator.job.enqueued`          | After a job has been enqueued               |
-| `taskinator.job.executed`          | After a job has executed                    |
-| `taskinator.subprocess.enqueued`   | After a sub process has been enqueued       |
-| `taskinator.subprocess.executed`   | After a sub process has executed            |
+| Event                              | When                                                      |
+|------------------------------------|-----------------------------------------------------------|
+| `taskinator.process.created`       | After a root process gets created                         |
+| `taskinator.process.saved`         | After a root process has been persisted to Redis          |
+| `taskinator.process.enqueued`      | After a process or subprocess is enqueued for processing  |
+| `taskinator.process.processing`    | When a process or subprocess is processing                |
+| `taskinator.process.paused`        | When a process or subprocess is paused                    |
+| `taskinator.process.resumed`       | When a process or subprocess is resumed                   |
+| `taskinator.process.completed`     | After a process or subprocess has completed processing    |
+| `taskinator.process.cancelled`     | After a process or subprocess has been cancelled          |
+| `taskinator.process.failed`        | After a process or subprocess has failed                  |
+| `taskinator.task.enqueued`         | After a task has been enqueued                            |
+| `taskinator.task.processing`       | When a task is processing                                 |
+| `taskinator.task.completed`        | After a task has completed                                |
+| `taskinator.task.cancelled`        | After a task has been cancelled                           |
+| `taskinator.task.failed`           | After a task has failed                                   |
 
 For all events, the data included contains the following information:
 
 | Key                      | Value                                                 |
 |--------------------------|-------------------------------------------------------|
+| `:type`                  | The type name of the component reporting the event    |
 | `:process_uuid`          | The UUID of the root process                          |
 | `:process_options`       | Options hash of the root process                      |
 | `:uuid`                  | The UUID of the respective task, job or sub process   |
-| `:tasks_count`           | The total count of tasks for the given process        |
+| `:options`               | Options hash of the component                         |
+| `:state`                 | State of the component                                |
 | `:percentage_completed`  | The percentage of completed tasks                     |
 | `:percentage_failed`     | The percentage of failed tasks                        |
 | `:percentage_cancelled`  | The percentage of cancelled tasks                     |

data/lib/taskinator/definition.rb

@@ -48,7 +48,7 @@ module Taskinator
           process = factory.call(self, options)
 
           # this may take long... up to users definition
-          Taskinator.instrumenter.instrument('taskinator.process.created', :uuid => process.uuid) do
+          Taskinator.instrumenter.instrument('taskinator.process.created', :uuid => process.uuid, :state => :initial) do
             Builder.new(process, self, *args).instance_eval(&block)
           end
 
@@ -56,7 +56,7 @@ module Taskinator
           unless subprocess
 
             # instrument separately
-            Taskinator.instrumenter.instrument('taskinator.process.saved', :uuid => process.uuid) do
+            Taskinator.instrumenter.instrument('taskinator.process.saved', :uuid => process.uuid, :state => :initial) do
 
               # this will visit "sub processes" and persist them too
               process.save

data/lib/taskinator/instrumentation.rb

@@ -0,0 +1,77 @@
+module Taskinator
+  module Instrumentation
+
+    def instrument(event, payload={})
+      Taskinator.instrumenter.instrument(event, payload) do
+        yield
+      end
+    end
+
+    # helper methods for instrumentation payloads
+
+    def enqueued_payload(additional={})
+      payload_for(:enqueued, additional)
+    end
+
+    def processing_payload(additional={})
+      payload_for(:processing, additional)
+    end
+
+    def paused_payload(additional={})
+      payload_for(:paused, additional)
+    end
+
+    def resumed_payload(additional={})
+      payload_for(:resumed, additional)
+    end
+
+    def completed_payload(additional={})
+      payload_for(:completed, additional)
+    end
+
+    def cancelled_payload(additional={})
+      payload_for(:cancelled, additional)
+    end
+
+    def failed_payload(exception, additional={})
+      payload_for(:failed, { :exception => exception.to_s, :backtrace => exception.backtrace }.merge(additional))
+    end
+
+    private
+
+    def payload_for(state, additional={})
+
+      # need to cache here, since this method hits redis, so can't be part of multi statement following
+      process_key = self.process_key
+
+      tasks_count, processing_count, completed_count, cancelled_count, failed_count = Taskinator.redis do |conn|
+        conn.hmget process_key,
+                   :tasks_count,
+                   :tasks_processing,
+                   :tasks_completed,
+                   :tasks_cancelled,
+                   :tasks_failed
+      end
+
+      tasks_count = tasks_count.to_f
+
+      return OpenStruct.new(
+        {
+          :type                   => self.class,
+          :process_uuid           => process_uuid,
+          :process_options        => process_options,
+          :uuid                   => uuid,
+          :options                => options,
+          :state                  => state,
+          :percentage_failed      => (tasks_count > 0) ? (failed_count.to_i     / tasks_count) * 100.0 : 0.0,
+          :percentage_cancelled   => (tasks_count > 0) ? (cancelled_count.to_i  / tasks_count) * 100.0 : 0.0,
+          :percentage_processing  => (tasks_count > 0) ? (processing_count.to_i / tasks_count) * 100.0 : 0.0,
+          :percentage_completed   => (tasks_count > 0) ? (completed_count.to_i  / tasks_count) * 100.0 : 0.0,
+          :instance               => self
+        }.merge(additional)
+      ).freeze
+
+    end
+
+  end
+end

data/lib/taskinator/persistence.rb

@@ -35,17 +35,6 @@ module Taskinator
         "taskinator:#{base_key}:#{uuid}"
       end
 
-      # retrieves the workflow state for the given identifier
-      # this prevents to need to load the entire object when
-      # querying for the status of an instance
-      def state_for(uuid)
-        key = key_for(uuid)
-        state = Taskinator.redis do |conn|
-          conn.hget(key, :state) || 'initial'
-        end
-        state.to_sym
-      end
-
       # fetches the instance for given identifier
       # optionally, provide a hash to use for the instance cache
       # this argument is defaulted, so top level callers don't
@@ -68,10 +57,11 @@ module Taskinator
             visitor = RedisSerializationVisitor.new(conn, self).visit
             conn.hmset(
               Taskinator::Process.key_for(uuid),
-              :tasks_count,     visitor.task_count,
-              :tasks_failed,    0,
-              :tasks_completed, 0,
-              :tasks_cancelled, 0,
+              :tasks_count,      visitor.task_count,
+              :tasks_failed,     0,
+              :tasks_processing, 0,
+              :tasks_completed,  0,
+              :tasks_cancelled,  0,
             )
             true
           end
@@ -99,17 +89,32 @@ module Taskinator
       # this method is called from the workflow gem
       def load_workflow_state
         state = Taskinator.redis do |conn|
-          conn.hget(self.key, :state)
+          conn.hget(self.key, :state) || 'initial'
         end
-        (state || 'initial').to_sym
+        state.to_sym
       end
 
       # persists the workflow state
       # this method is called from the workflow gem
       def persist_workflow_state(new_state)
+        @updated_at = Time.now.utc
         Taskinator.redis do |conn|
-          conn.hset(self.key, :state, new_state)
+          process_key = self.process_key
+          conn.multi do
+            conn.hmset(
+              self.key,
+              :state, new_state,
+              :updated_at, @updated_at
+            )
+
+            # also update the "root" process
+            conn.hset(
+              process_key,
+              :updated_at, @updated_at
+            )
+          end
         end
+        new_state
       end
 
       # persists the error information
@@ -121,7 +126,8 @@ module Taskinator
             self.key,
             :error_type, error.class.name,
             :error_message, error.message,
-            :error_backtrace, JSON.generate(error.backtrace || [])
+            :error_backtrace, JSON.generate(error.backtrace || []),
+            :updated_at, Time.now.utc
           )
         end
       end
@@ -133,7 +139,7 @@ module Taskinator
           error_type, error_message, error_backtrace =
             conn.hmget(self.key, :error_type, :error_message, :error_backtrace)
 
-          [error_type, error_message, JSON.parse(error_backtrace)]
+          [error_type, error_message, JSON.parse(error_backtrace || '[]')]
         end
       end
 
@@ -149,19 +155,24 @@ module Taskinator
       %w(
         failed
         cancelled
+        processing
         completed
       ).each do |status|
 
         define_method "count_#{status}" do
           count = Taskinator.redis do |conn|
-            conn.hget self.process_key, status
+            conn.hget self.process_key, "tasks_#{status}"
           end
           count.to_i
         end
 
         define_method "incr_#{status}" do
           Taskinator.redis do |conn|
-            conn.hincrby self.process_key, status, 1
+            process_key = self.process_key
+            conn.multi do
+              conn.hincrby process_key, "tasks_#{status}", 1
+              conn.hset process_key, :updated_at, Time.now.utc
+            end
           end
         end
 
@@ -183,34 +194,6 @@ module Taskinator
         end
       end
 
-      # prepairs the meta data for instrumentation events
-      def instrumentation_payload(additional={})
-
-        # need to cache here, since this method hits redis, so can't be part of multi statement following
-        process_key = self.process_key
-
-        tasks_count, completed_count, cancelled_count, failed_count = Taskinator.redis do |conn|
-          conn.hmget process_key, :tasks_count, :completed, :cancelled, :failed
-        end
-
-        tasks_count = tasks_count.to_f
-        completed_percent = tasks_count > 0 ? (completed_count.to_i / tasks_count) * 100.0 : 0.0
-        cancelled_percent = tasks_count > 0 ? (cancelled_count.to_i / tasks_count) * 100.0 : 0.0
-        failed_percent    = tasks_count > 0 ? (failed_count.to_i    / tasks_count) * 100.0 : 0.0
-
-        return {
-          :type                  => self.class.name,
-          :process_uuid          => process_uuid,
-          :process_options       => process_options,
-          :uuid                  => uuid,
-          :percentage_failed     => failed_percent,
-          :percentage_cancelled  => cancelled_percent,
-          :percentage_completed  => completed_percent,
-          :tasks_count           => tasks_count
-        }.merge(additional)
-
-      end
-
     end
 
     class RedisSerializationVisitor < Visitor::Base
@@ -244,6 +227,9 @@ module Taskinator
         # add the process uuid and root key, for easy access later!
         @hmset += [:process_uuid, @root.uuid]
 
+        # add the default state
+        @hmset += [:state, :initial]
+
         # NB: splat args
         @conn.hmset(*@hmset)
 
@@ -259,7 +245,6 @@ module Taskinator
       end
 
       def visit_tasks(tasks)
-        @hmset += [:task_count, tasks.count]  # not used currently, but for informational purposes
         tasks.each do |task|
           RedisSerializationVisitor.new(@conn, task, @base_visitor).visit
           @conn.rpush "#{@key}:tasks", task.uuid
@@ -272,6 +257,14 @@ module Taskinator
         @hmset += [attribute, value] if value
       end
 
+      def visit_attribute_time(attribute)
+        visit_attribute(attribute)
+      end
+
+      def visit_attribute_enum(attribute, type)
+        visit_attribute(attribute)
+      end
+
       def visit_process_reference(attribute)
         process = @instance.send(attribute)
         @hmset += [attribute, process.uuid] if process
@@ -374,7 +367,30 @@ module Taskinator
 
       def visit_attribute(attribute)
         value = @attribute_values[attribute]
-        @instance.instance_variable_set("@#{attribute}", value) if value
+        if value
+          # converted block given?
+          if block_given?
+            @instance.instance_variable_set("@#{attribute}", yield(value))
+          else
+            @instance.instance_variable_set("@#{attribute}", value)
+          end
+        end
+      end
+
+      def visit_attribute_time(attribute)
+        visit_attribute(attribute) do |value|
+          Time.parse(value)
+        end
+      end
+
+      # NB: assumes the enum type's members have integer values!
+      def visit_attribute_enum(attribute, type)
+        visit_attribute(attribute) do |value|
+          const_value = type.constants.select {|c| type.const_get(c) == value.to_i }.first
+          const_value ?
+            type.const_get(const_value) :
+            (defined?(type::Default) ? type::Default : nil)
+        end
       end
 
       def visit_type(attribute)
@@ -405,11 +421,11 @@ module Taskinator
       # arbitrary instance to perform it's work
       #
       def lazy_instance_for(base, uuid)
-        type, process_uuid = Taskinator.redis do |conn|
-          conn.hmget(base.key_for(uuid), :type, :process_uuid)
+        type = Taskinator.redis do |conn|
+          conn.hget(base.key_for(uuid), :type)
         end
         klass = Kernel.const_get(type)
-        LazyLoader.new(klass, uuid, process_uuid, @instance_cache)
+        LazyLoader.new(klass, uuid, @instance_cache)
       end
     end
 
@@ -424,17 +440,12 @@ module Taskinator
       # E.g. this is useful for tasks which refer to their parent processes
       #
 
-      def initialize(type, uuid, process_uuid, instance_cache={})
+      def initialize(type, uuid, instance_cache={})
         @type = type
         @uuid = uuid
-        @process_uuid = process_uuid
         @instance_cache = instance_cache
       end
 
-      # shadows the real methods, but will be the same!
-      attr_reader :process_uuid
-      attr_reader :uuid
-
       def __getobj__
         # only fetch the object as needed
         # and memoize for subsequent calls