taskinator 0.0.18 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bd4b644d746cddfe50ed5d3c9d984bcd978d3fee
-  data.tar.gz: 45f8166dbd3f7f1b5e27d52c31c7e0384a433cc7
+  metadata.gz: 1905dc0aad01e4bd348ad38a40100df9649e3a18
+  data.tar.gz: d1e4aa938c9d3a0d56fc4090f71024dec84e32fd
 SHA512:
-  metadata.gz: 34d036f6fedd66b61b73c6530716882a9b019a769431c719273e0b684e216838571cc3c49a7317292662f8dc41b1ff379c0c41480effb3f705ee0e8752a551de
-  data.tar.gz: 9fa6364a483ec5d361881b07f26e266ebebbfba07f1b44cb73470c5658e048448f38eece7d531cbb3785219253f83233b75e5589a05eedf06c1eadaeff74fff3
+  metadata.gz: 6cd0c7d2fa8d1f133ddce568ae969fe488401ba9f6311c39143456749b4eea1429edbc63ec080a5a8b60fb84bf312bdcd5758084269b16ae83b7b437877e15d1
+  data.tar.gz: dfcbeced7346f8c0bdcc70e5814e2b2e943ecd10182ac0285a88e8e0bdfc0eb1a227c900390328861bb4012e4feffbf8fe7d41033583d68eb1f25d198f1ef027

data/CHANGELOG.md

@@ -1,3 +1,28 @@
+v0.2.0 - 31 Jul 2015
+---
+Bug fix for `create_process_remotely` so that it returns the process uuid instead of nil.
+Removed reload functionality, since it isn't used anymore
+Added missing instrumentation events for task, job and subprocess completed events.
+Bug fix for when `sequential` or `concurrent` steps don't have any tasks to still continue processing.
+Refactoring to remove dead code and "reload" functionality.
+Improvements to console and rake to use console instrumenter.
+Consolidation of instrumentation events. Added `type` to payload.
+Improvements to error handling.
+
+v0.1.1 - 23 Jul 2015  [Yanked]
+---
+Bug fix for option parameter handling.
+
+v0.1.0 - 23 Jul 2015 [Yanked]
+---
+Fixed issue with persistence of options passed to `create_process` on the respective `Process` and `Task` instances.
+Improvements to process creation logic.
+Namespaced instrumentation event names.
+Added process completed, cancelled and failed instrumentation events.
+Include additional data in the instrumentation payload. E.g. Process options and percentages.
+Refactored the way processes/tasks get queued, to prevent unnecessary queuing of contained processes/tasks.
+Removed `ProcessWorker` since it isn't needed anymore.
+
 v0.0.18 - 14 Jul 2015
 ---
 Fixed issue with `Taskinator::Api::Processes#each` method, which was causing a Segmentation fault.

data/Gemfile.lock

@@ -8,7 +8,7 @@ GIT
 PATH
   remote: .
   specs:
-    taskinator (0.0.18)
+    taskinator (0.2.0)
       connection_pool (>= 2.2.0)
       json (>= 1.8.2)
       redis (>= 3.2.1)
@@ -17,20 +17,20 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (4.2.1)
+    activesupport (4.2.3)
       i18n (~> 0.7)
       json (~> 1.7, >= 1.7.7)
       minitest (~> 5.1)
       thread_safe (~> 0.3, >= 0.3.4)
       tzinfo (~> 1.1)
-    byebug (4.0.5)
+    byebug (5.0.0)
       columnize (= 0.9.0)
     celluloid (0.16.0)
       timers (~> 4.0.0)
     coderay (1.1.0)
     columnize (0.9.0)
     connection_pool (2.2.0)
-    coveralls (0.8.1)
+    coveralls (0.8.2)
       json (~> 1.8)
       rest-client (>= 1.6.8, < 2)
       simplecov (~> 0.10.0)
@@ -46,21 +46,21 @@ GEM
     http-cookie (1.0.2)
       domain_name (~> 0.5)
     i18n (0.7.0)
-    json (1.8.2)
+    json (1.8.3)
     method_source (0.8.2)
     mime-types (2.6.1)
     minitest (5.7.0)
     mono_logger (1.1.0)
-    multi_json (1.11.0)
+    multi_json (1.11.2)
     netrc (0.10.3)
     pry (0.10.1)
       coderay (~> 1.1.0)
       method_source (~> 0.8.1)
       slop (~> 3.4)
-    pry-byebug (3.1.0)
-      byebug (~> 4.0)
+    pry-byebug (3.2.0)
+      byebug (~> 5.0)
       pry (~> 0.10)
-    rack (1.6.1)
+    rack (1.6.4)
     rack-protection (1.5.3)
       rack
     rake (10.4.2)
@@ -82,28 +82,28 @@ GEM
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 3.0)
       netrc (~> 0.7)
-    rspec (3.2.0)
-      rspec-core (~> 3.2.0)
-      rspec-expectations (~> 3.2.0)
-      rspec-mocks (~> 3.2.0)
-    rspec-core (3.2.3)
-      rspec-support (~> 3.2.0)
-    rspec-expectations (3.2.1)
+    rspec (3.3.0)
+      rspec-core (~> 3.3.0)
+      rspec-expectations (~> 3.3.0)
+      rspec-mocks (~> 3.3.0)
+    rspec-core (3.3.2)
+      rspec-support (~> 3.3.0)
+    rspec-expectations (3.3.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.2.0)
-    rspec-mocks (3.2.1)
+      rspec-support (~> 3.3.0)
+    rspec-mocks (3.3.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.2.0)
+      rspec-support (~> 3.3.0)
     rspec-sidekiq (2.0.0)
       rspec (~> 3.0, >= 3.0.0)
       sidekiq (>= 2.4.0)
-    rspec-support (3.2.2)
-    sidekiq (3.3.4)
-      celluloid (>= 0.16.0)
-      connection_pool (>= 2.1.1)
-      json
-      redis (>= 3.0.6)
-      redis-namespace (>= 1.3.1)
+    rspec-support (3.3.0)
+    sidekiq (3.4.2)
+      celluloid (~> 0.16.0)
+      connection_pool (~> 2.2, >= 2.2.0)
+      json (~> 1.0)
+      redis (~> 3.2, >= 3.2.1)
+      redis-namespace (~> 1.5, >= 1.5.2)
     simplecov (0.10.0)
       docile (~> 1.1.0)
       json (~> 1.8)
@@ -114,14 +114,14 @@ GEM
       rack-protection (~> 1.4)
       tilt (>= 1.3, < 3)
     slop (3.6.0)
-    term-ansicolor (1.3.0)
+    term-ansicolor (1.3.2)
       tins (~> 1.0)
     thor (0.19.1)
     thread_safe (0.3.5)
     tilt (2.0.1)
     timers (4.0.1)
       hitimes
-    tins (1.5.2)
+    tins (1.5.4)
     tzinfo (1.2.2)
       thread_safe (~> 0.1)
     unf (0.1.4)

data/README.md

@@ -606,6 +606,35 @@ Taskinator.configure do |config|
 end
 ```
 
+The following instrumentation events are issued:
+
+| Event                              | When                                        |
+|------------------------------------|---------------------------------------------|
+| `taskinator.process.created`       | After a process gets created                |
+| `taskinator.process.saved`         | After a process has been persisted to Redis |
+| `taskinator.process.enqueued`      | After a process is enqueued for processing  |
+| `taskinator.process.completed`     | After a process has completed processing    |
+| `taskinator.process.cancelled`     | After a process has been cancelled          |
+| `taskinator.process.failed`        | After a process has failed                  |
+| `taskinator.task.enqueued`         | After a task has been enqueued              |
+| `taskinator.task.executed`         | After a task has executed                   |
+| `taskinator.job.enqueued`          | After a job has been enqueued               |
+| `taskinator.job.executed`          | After a job has executed                    |
+| `taskinator.subprocess.enqueued`   | After a sub process has been enqueued       |
+| `taskinator.subprocess.executed`   | After a sub process has executed            |
+
+For all events, the data included contains the following information:
+
+| Key                      | Value                                                 |
+|--------------------------|-------------------------------------------------------|
+| `:process_uuid`          | The UUID of the root process                          |
+| `:process_options`       | Options hash of the root process                      |
+| `:uuid`                  | The UUID of the respective task, job or sub process   |
+| `:tasks_count`           | The total count of tasks for the given process        |
+| `:percentage_completed`  | The percentage of completed tasks                     |
+| `:percentage_failed`     | The percentage of failed tasks                        |
+| `:percentage_cancelled`  | The percentage of cancelled tasks                     |
+
 ## Notes
 
 The persistence logic is decoupled from the implementation, so it is possible to implement another backing store if required.

data/Rakefile

@@ -9,3 +9,8 @@ task :default => :spec
 require 'resque'
 require 'resque/tasks'
 require 'taskinator'
+
+Taskinator.configure do |config|
+  config.logger.level = 0  # DEBUG
+  config.instrumenter = Taskinator::ConsoleInstrumenter.new
+end

data/bin/console

@@ -7,5 +7,10 @@ require "taskinator"
 # You can add fixtures and/or initialization code here to make experimenting
 # with your gem easier. You can also use a different console, if you like.
 
+Taskinator.configure do |config|
+  config.logger.level = 0  # DEBUG
+  config.instrumenter = Taskinator::ConsoleInstrumenter.new
+end
+
 require "pry"
 Pry.start

data/lib/taskinator/create_process_worker.rb

@@ -19,13 +19,16 @@ module Taskinator
     end
 
     def perform
-      @definition._create_process_(*@args, :uuid => @uuid).enqueue!
+      @definition._create_process_(false, *@args, :uuid => @uuid).enqueue!
     end
 
     private
 
-    # borrowed from activesupport/lib/active_support/inflector/methods.rb
+    # :nocov:
     def constantize(camel_cased_word)
+
+      # borrowed from activesupport/lib/active_support/inflector/methods.rb
+
       names = camel_cased_word.split('::')
 
       # Trigger a built-in NameError exception including the ill-formed constant in the message.

data/lib/taskinator/definition/builder.rb

@@ -5,18 +5,18 @@ module Taskinator
       attr_reader :process
       attr_reader :definition
       attr_reader :args
-      attr_reader :options
+      attr_reader :builder_options
 
       def initialize(process, definition, *args)
         @process = process
         @definition = definition
+        @builder_options = args.last.is_a?(Hash) ? args.pop : {}
         @args = args
-        @options = args.last.is_a?(Hash) ? args.last : {}
         @executor = Taskinator::Executor.new(@definition)
       end
 
       def option?(key, &block)
-        yield if @options.key?(key) && @options[key]
+        yield if builder_options.key?(key) && builder_options[key]
       end
 
       # defines a sub process of tasks which are executed sequentially
@@ -81,7 +81,7 @@ module Taskinator
 
         # TODO: decide whether the sub process to dynamically receive arguments
 
-        sub_process = definition.create_sub_process(*@args)
+        sub_process = definition.create_sub_process(*@args, combine_options(options))
         Builder.new(define_sub_process_task(@process, sub_process, options), definition, *@args)
       end
 
@@ -89,19 +89,19 @@ module Taskinator
 
       def define_step_task(process, method, args, options={})
         define_task(process) {
-          Task.define_step_task(process, method, args, options)
+          Task.define_step_task(process, method, args, combine_options(options))
         }
       end
 
       def define_job_task(process, job, args, options={})
         define_task(process) {
-          Task.define_job_task(process, job, args, options)
+          Task.define_job_task(process, job, args, combine_options(options))
         }
       end
 
       def define_sub_process_task(process, sub_process, options={})
         define_task(process) {
-          Task.define_sub_process_task(process, sub_process, options)
+          Task.define_sub_process_task(process, sub_process, combine_options(options))
         }
         sub_process
       end
@@ -110,6 +110,11 @@ module Taskinator
         process.tasks << task = yield
         task
       end
+
+      def combine_options(options={})
+        builder_options.merge(options)
+      end
+
     end
   end
 end

data/lib/taskinator/definition.rb

@@ -33,42 +33,49 @@ module Taskinator
                     Process.define_sequential_process_for(definition, options)
                   }
 
-      define_singleton_method :_create_process_ do |*args|
+      # called from respective "create_process" methods
+      # parameters can contain options as the last parameter
+      define_singleton_method :_create_process_ do |subprocess, *args|
+        begin
 
-        options = args.last.is_a?(Hash) ? args.pop : {}
+          # TODO: better validation of arguments
 
-        # TODO: better validation of arguments
+          # FIXME: arg_list should only contain an array of symbols
 
-        # FIXME: arg_list should only contain an array of symbols
+          raise ArgumentError, "wrong number of arguments (#{args.length} for #{arg_list.length})" if args.length < arg_list.length
 
-        raise ArgumentError, "wrong number of arguments (#{args.length} for #{arg_list.length})" if args.length < arg_list.length
+          options = (args.last.is_a?(Hash) ? args.last : {})
+          process = factory.call(self, options)
 
-        process = factory.call(self, options)
+          # this may take long... up to users definition
+          Taskinator.instrumenter.instrument('taskinator.process.created', :uuid => process.uuid) do
+            Builder.new(process, self, *args).instance_eval(&block)
+          end
 
-        # this may take long... up to users definition
-        Taskinator.instrumenter.instrument(:create_process, :uuid => process.uuid) do
-          Builder.new(process, self, *args).instance_eval(&block)
-        end
+          # only save "root processes"
+          unless subprocess
 
-        # only save "root processes"
-        unless options[:subprocess]
+            # instrument separately
+            Taskinator.instrumenter.instrument('taskinator.process.saved', :uuid => process.uuid) do
 
-          # instrument separately
-          Taskinator.instrumenter.instrument(:save_process, :uuid => process.uuid) do
+              # this will visit "sub processes" and persist them too
+              process.save
 
-            # this will visit "sub processes" and persist them too
-            process.save
+              # add it to the list of "root processes"
+              Persistence.add_process_to_list(process)
 
-            # add it to the list of "root processes"
-            Persistence.add_process_to_list(process)
+            end
 
           end
 
-        end
-
-        # this is the "root" process
-        process
+          # this is the "root" process
+          process
 
+        rescue => e
+          Taskinator.logger.error(e)
+          Taskinator.logger.debug(e.backtrace)
+          raise e
+        end
       end
     end
 
@@ -80,25 +87,26 @@ module Taskinator
     #
     def create_process(*args)
       assert_valid_process_module
-      _create_process_(*args, :sub_process => false)
+      _create_process_(false, *args)
     end
 
     #
-    # returns a placeholder process, with the uuid attribute of the
-    # actual process. the callee can call `reload` if required to
-    # get the actual process, once it has been built by the CreateProcessWorker
+    # returns the process uuid of the process to be created
+    # the process can be retrieved using this uuid by using
+    # Taskinator::Process.fetch(uuid)
     #
     def create_process_remotely(*args)
       assert_valid_process_module
       uuid = SecureRandom.uuid
+
       Taskinator.queue.enqueue_create_process(self, uuid, args)
 
-      Taskinator::Persistence::LazyLoader.new(Taskinator::Process, uuid, uuid)
+      return uuid
     end
 
     def create_sub_process(*args)
       assert_valid_process_module
-      _create_process_(*args, :subprocess => true)
+      _create_process_(true, *args)
     end
 
     private

data/lib/taskinator/executor.rb

@@ -13,16 +13,16 @@ module Taskinator
       eigen.send(:include, definition)
     end
 
-    def root_key
-      @root_key ||= task.root_key
+    def process_uuid
+      task.process_uuid if task
     end
 
     def uuid
-      task.uuid
+      task.uuid if task
     end
 
     def options
-      task.options
+      task.options if task
     end
 
   end

data/lib/taskinator/job_worker.rb

@@ -1,22 +1,17 @@
 module Taskinator
   class JobWorker
+    attr_reader :uuid
+
     def initialize(uuid)
       @uuid = uuid
     end
 
-    # NB: must be provided a block for the implmentation of the job execution
+    # NB: must be provided a block for the implementation of the job execution
     def perform(&block)
       task = Taskinator::Task.fetch(@uuid)
       return if task.paused? || task.cancelled?
-      begin
-        task.start!
-        task.perform(&block)
-        task.complete!
-      rescue => e
-        Taskinator.logger.error(e)
-        task.fail!(e)
-        raise e
-      end
+      task.start!
+      task.perform(&block)
     end
   end
 end

data/lib/taskinator/logger.rb

@@ -8,6 +8,7 @@
 require 'time'
 require 'logger'
 
+# :nocov:
 module Taskinator
   module Logging
 

data/lib/taskinator/persistence.rb

@@ -40,10 +40,10 @@ module Taskinator
       # querying for the status of an instance
       def state_for(uuid)
         key = key_for(uuid)
-        Taskinator.redis do |conn|
-          state = conn.hget(key, :state) || 'initial'
-          state.to_sym
+        state = Taskinator.redis do |conn|
+          conn.hget(key, :state) || 'initial'
         end
+        state.to_sym
       end
 
       # fetches the instance for given identifier
@@ -67,7 +67,7 @@ module Taskinator
           conn.multi do
             visitor = RedisSerializationVisitor.new(conn, self).visit
             conn.hmset(
-              "taskinator:#{self.key}",
+              Taskinator::Process.key_for(uuid),
               :tasks_count,     visitor.task_count,
               :tasks_failed,    0,
               :tasks_completed, 0,
@@ -78,19 +78,23 @@ module Taskinator
         end
       end
 
-      # this is the persistence key
+      # the persistence key
       def key
         @key ||= self.class.key_for(self.uuid)
       end
 
-      # retrieves the root key associated
-      # with the process or task
-      def root_key
-        @root_key ||= Taskinator.redis do |conn|
-          conn.hget(self.key, :root_key)
+      # the root process uuid associated with this process or task
+      def process_uuid
+        @process_uuid ||= Taskinator.redis do |conn|
+          conn.hget(self.key, :process_uuid)
         end
       end
 
+      # the root process persistence key associated with this process or task
+      def process_key
+        @process_key ||= Taskinator::Process.key_for(process_uuid)
+      end
+
       # retrieves the workflow state
       # this method is called from the workflow gem
       def load_workflow_state
@@ -135,9 +139,10 @@ module Taskinator
 
       def tasks_count
         @tasks_count ||= begin
-          Taskinator.redis do |conn|
-            conn.hget "taskinator:#{self.root_key}", :tasks_count
-          end.to_i
+          count = Taskinator.redis do |conn|
+            conn.hget self.process_key, :tasks_count
+          end
+          count.to_i
         end
       end
 
@@ -148,14 +153,15 @@ module Taskinator
       ).each do |status|
 
         define_method "count_#{status}" do
-          Taskinator.redis do |conn|
-            conn.hget "taskinator:#{self.root_key}", status
-          end.to_i
+          count = Taskinator.redis do |conn|
+            conn.hget self.process_key, status
+          end
+          count.to_i
         end
 
         define_method "incr_#{status}" do
           Taskinator.redis do |conn|
-            conn.hincrby "taskinator:#{self.root_key}", status, 1
+            conn.hincrby self.process_key, status, 1
           end
         end
 
@@ -165,6 +171,46 @@ module Taskinator
 
       end
 
+      # retrieves the process options of the root process
+      # this is so that meta data of the process can be maintained
+      # and accessible to instrumentation subscribers
+      def process_options
+        @process_options ||= begin
+          yaml = Taskinator.redis do |conn|
+            conn.hget(self.process_key, :options)
+          end
+          yaml ? Taskinator::Persistence.deserialize(yaml) : {}
+        end
+      end
+
+      # prepairs the meta data for instrumentation events
+      def instrumentation_payload(additional={})
+
+        # need to cache here, since this method hits redis, so can't be part of multi statement following
+        process_key = self.process_key
+
+        tasks_count, completed_count, cancelled_count, failed_count = Taskinator.redis do |conn|
+          conn.hmget process_key, :tasks_count, :completed, :cancelled, :failed
+        end
+
+        tasks_count = tasks_count.to_f
+        completed_percent = tasks_count > 0 ? (completed_count.to_i / tasks_count) * 100.0 : 0.0
+        cancelled_percent = tasks_count > 0 ? (cancelled_count.to_i / tasks_count) * 100.0 : 0.0
+        failed_percent    = tasks_count > 0 ? (failed_count.to_i    / tasks_count) * 100.0 : 0.0
+
+        return {
+          :type                  => self.class.name,
+          :process_uuid          => process_uuid,
+          :process_options       => process_options,
+          :uuid                  => uuid,
+          :percentage_failed     => failed_percent,
+          :percentage_cancelled  => cancelled_percent,
+          :percentage_completed  => completed_percent,
+          :tasks_count           => tasks_count
+        }.merge(additional)
+
+      end
+
     end
 
     class RedisSerializationVisitor < Visitor::Base
@@ -181,7 +227,7 @@ module Taskinator
         @conn         = conn
         @instance     = instance
         @key          = instance.key
-        @root_key     = base_visitor.instance.key
+        @root         = base_visitor.instance
         @base_visitor = base_visitor
         @task_count   = 0
       end
@@ -195,8 +241,8 @@ module Taskinator
 
         @instance.accept(self)
 
-        # add the root key, for easy access later!
-        @hmset += [:root_key, @root_key]
+        # add the process uuid and root key, for easy access later!
+        @hmset += [:process_uuid, @root.uuid]
 
         # NB: splat args
         @conn.hmset(*@hmset)
@@ -359,13 +405,11 @@ module Taskinator
       # arbitrary instance to perform it's work
       #
       def lazy_instance_for(base, uuid)
-        Taskinator.redis do |conn|
-          type = conn.hget(base.key_for(uuid), :type)
-          root_key = conn.hget(base.key_for(uuid), :root_key)
-
-          klass = Kernel.const_get(type)
-          LazyLoader.new(klass, uuid, root_key, @instance_cache)
+        type, process_uuid = Taskinator.redis do |conn|
+          conn.hmget(base.key_for(uuid), :type, :process_uuid)
         end
+        klass = Kernel.const_get(type)
+        LazyLoader.new(klass, uuid, process_uuid, @instance_cache)
       end
     end
 
@@ -380,29 +424,23 @@ module Taskinator
       # E.g. this is useful for tasks which refer to their parent processes
       #
 
-      def initialize(type, uuid, root_key, instance_cache={})
+      def initialize(type, uuid, process_uuid, instance_cache={})
         @type = type
         @uuid = uuid
-        @root_key = root_key
+        @process_uuid = process_uuid
         @instance_cache = instance_cache
       end
 
       # shadows the real methods, but will be the same!
+      attr_reader :process_uuid
       attr_reader :uuid
-      attr_reader :root_key
-
-      # attempts to reload the actual process
-      def reload
-        @instance = nil
-        __getobj__
-        @instance ? true : false
-      end
 
       def __getobj__
         # only fetch the object as needed
         # and memoize for subsequent calls
         @instance ||= @type.fetch(@uuid, @instance_cache)
       end
+
     end
 
     class << self