taskinator 0.0.18 → 0.2.0

data/lib/taskinator/process.rb

@@ -1,3 +1,6 @@
+require 'thread'
+require 'thwait'
+
 module Taskinator
   class Process
     include ::Comparable
@@ -60,24 +63,21 @@ module Taskinator
       state :initial do
         event :enqueue, :transitions_to => :enqueued
         event :start, :transitions_to => :processing
-
-        # need to be able to complete, for when there are no tasks
-        event :complete, :transitions_to => :completed, :if => :no_tasks_defined?
-
+        event :complete, :transitions_to => :completed
         event :cancel, :transitions_to => :cancelled
         event :fail, :transitions_to => :failed
       end
 
       state :enqueued do
         event :start, :transitions_to => :processing
-        event :complete, :transitions_to => :completed, :if => :no_tasks_defined?
+        event :complete, :transitions_to => :completed
         event :cancel, :transitions_to => :cancelled
         event :fail, :transitions_to => :failed
       end
 
       state :processing do
         event :pause, :transitions_to => :paused
-        event :complete, :transitions_to => :completed, :if => :tasks_completed?
+        event :complete, :transitions_to => :completed
         event :fail, :transitions_to => :failed
       end
 
@@ -95,19 +95,21 @@ module Taskinator
         Taskinator.logger.debug("PROCESS: #{self.class.name}:#{uuid} :: #{from} => #{to}")
       end
 
-      on_error do |error, from, to, event, *args|
-        Taskinator.logger.error("PROCESS: #{self.class.name}:#{uuid} :: #{error.message}")
-        Taskinator.logger.debug(error.backtrace)
-        fail!(error)
-      end
     end
 
     def no_tasks_defined?
       tasks.empty?
     end
 
+    # callback for when the process was cancelled
+    def on_cancelled_entry(*args)
+      Taskinator.instrumenter.instrument('taskinator.process.cancelled', instrumentation_payload) do
+        # intentionally left empty
+      end
+    end
+
+    # subclasses must implement this method
     def tasks_completed?(*args)
-      # subclasses must implement this method
       raise NotImplementedError
     end
 
@@ -121,37 +123,42 @@ module Taskinator
     # need to override the ones defined by workflow
     include Persistence
 
-    def enqueue
-      # don't bother if there aren't any tasks!
-      if tasks.empty?
-        # simply complete the process...
-        complete!
-      else
-        Taskinator.queue.enqueue_process(self)
+    def complete
+      Taskinator.instrumenter.instrument('taskinator.process.completed', instrumentation_payload) do
+        # notify the parent task (if there is one) that this process has completed
+        # note: parent may be a proxy, so explicity check for nil?
+        parent.complete! unless parent.nil?
       end
     end
 
-    # callback for when the process has completed
-    def on_completed_entry(*args)
-      # notify the parent task (if there is one) that this process has completed
-      # note: parent may be a proxy, so explicity check for nil?
-      parent.complete! unless parent.nil?
-    end
-
     # callback for when the process has failed
     def on_failed_entry(*args)
-      # notify the parent task (if there is one) that this process has failed
-      # note: parent may be a proxy, so explicity check for nil?
-      parent.fail!(*args) unless parent.nil?
+      Taskinator.instrumenter.instrument('taskinator.process.failed', instrumentation_payload) do
+        # notify the parent task (if there is one) that this process has failed
+        # note: parent may be a proxy, so explicity check for nil?
+        parent.fail!(*args) unless parent.nil?
+      end
     end
 
     class Sequential < Process
+      def enqueue
+        Taskinator.instrumenter.instrument('taskinator.process.enqueued', instrumentation_payload) do
+          if tasks.empty?
+            complete! # weren't any tasks to start with
+          else
+            tasks.first.enqueue!
+          end
+        end
+      end
+
       def start
-        task = tasks.first
-        if task
-          task.enqueue!
-        else
-          complete! # weren't any tasks to start with
+        Taskinator.instrumenter.instrument('taskinator.process.started', instrumentation_payload) do
+          task = tasks.first
+          if task
+            task.start!
+          else
+            complete! # weren't any tasks to start with
+          end
         end
       end
 
@@ -160,7 +167,7 @@ module Taskinator
         if next_task
           next_task.enqueue!
         else
-          complete! if can_complete?
+          complete!
         end
       end
 
@@ -176,28 +183,55 @@ module Taskinator
 
     class Concurrent < Process
       attr_reader :complete_on
+      attr_reader :concurrency_method
 
       def initialize(definition, complete_on=CompleteOn::Default, options={})
         super(definition, options)
         @complete_on = complete_on
+        @concurrency_method = options.delete(:concurrency_method) || :thread
+      end
+
+      def enqueue
+        Taskinator.instrumenter.instrument('taskinator.process.enqueued', instrumentation_payload) do
+          if tasks.empty?
+            complete! # weren't any tasks to start with
+          else
+            tasks.each(&:enqueue!)
+          end
+        end
       end
 
       def start
-        if tasks.empty?
-          complete! # weren't any tasks to start with
-        else
-          tasks.each(&:enqueue!)
+        Taskinator.instrumenter.instrument('taskinator.process.started', instrumentation_payload) do
+          if tasks.empty?
+            complete! # weren't any tasks to start with
+          else
+            if concurrency_method == :fork
+              tasks.each do |task|
+                fork do
+                  task.start!
+                end
+              end
+              Process.waitall
+            else
+              threads = tasks.map do |task|
+                Thread.new do
+                  task.start!
+                end
+              end
+              ThreadsWait.all_waits(*threads)
+            end
+          end
         end
       end
 
       def task_completed(task)
         # when complete on first, then don't bother with subsequent tasks completing
         return if completed? || failed?
-        complete! if can_complete?
+        complete!
       end
 
       def tasks_completed?(*args)
-        # TODO: optimize this
         if (complete_on == CompleteOn::First)
           tasks.any?(&:completed?)
         else
@@ -214,13 +248,5 @@ module Taskinator
         %(#<#{self.class.name}:0x#{self.__id__.to_s(16)} uuid="#{uuid}", state=:#{current_state.name}, complete_on=:#{complete_on}, tasks=[#{tasks.inspect}]>)
       end
     end
-
-    # reloads the process from storage
-    # NB: only implemented by LazyLoader so that
-    #   the process can be lazy loaded, thereafter
-    #    it has no effect
-    def reload
-      false
-    end
   end
 end

data/lib/taskinator/queues/delayed_job.rb

@@ -17,11 +17,6 @@ module Taskinator
         ::Delayed::Job.enqueue CreateProcessWorker.new(definition.name, uuid, Taskinator::Persistence.serialize(args)), :queue => queue
       end
 
-      def enqueue_process(process)
-        queue = process.queue || @config[:process_queue]
-        ::Delayed::Job.enqueue ProcessWorker.new(process.uuid), :queue => queue
-      end
-
       def enqueue_task(task)
         queue = task.queue || @config[:task_queue]
         ::Delayed::Job.enqueue TaskWorker.new(task.uuid), :queue => queue
@@ -39,12 +34,6 @@ module Taskinator
         end
       end
 
-      ProcessWorker = Struct.new(:process_uuid) do
-        def perform
-          Taskinator::ProcessWorker.new(process_uuid).perform
-        end
-      end
-
       TaskWorker = Struct.new(:task_uuid) do
         def perform
           Taskinator::TaskWorker.new(task_uuid).perform
@@ -58,6 +47,7 @@ module Taskinator
           end
         end
       end
+
     end
   end
 end

data/lib/taskinator/queues/resque.rb

@@ -15,10 +15,6 @@ module Taskinator
           @queue = config[:definition_queue]
         end
 
-        ProcessWorker.class_eval do
-          @queue = config[:process_queue]
-        end
-
         TaskWorker.class_eval do
           @queue = config[:task_queue]
         end
@@ -33,11 +29,6 @@ module Taskinator
         Resque.enqueue_to(queue, CreateProcessWorker, definition.name, uuid, Taskinator::Persistence.serialize(args))
       end
 
-      def enqueue_process(process)
-        queue = process.queue || Resque.queue_from_class(ProcessWorker)
-        Resque.enqueue_to(queue, ProcessWorker, process.uuid)
-      end
-
       def enqueue_task(task)
         queue = task.queue || Resque.queue_from_class(TaskWorker)
         Resque.enqueue_to(queue, TaskWorker, task.uuid)
@@ -57,12 +48,6 @@ module Taskinator
         end
       end
 
-      class ProcessWorker
-        def self.perform(process_uuid)
-          Taskinator::ProcessWorker.new(process_uuid).perform
-        end
-      end
-
       class TaskWorker
         def self.perform(task_uuid)
           Taskinator::TaskWorker.new(task_uuid).perform

data/lib/taskinator/queues/sidekiq.rb

@@ -14,12 +14,7 @@ module Taskinator
 
       def enqueue_create_process(definition, uuid, args)
         queue = definition.queue || @config[:definition_queue]
-        ProcessWorker.client_push('class' => CreateProcessWorker, 'args' => [definition.name, uuid, Taskinator::Persistence.serialize(args)], 'queue' => queue)
-      end
-
-      def enqueue_process(process)
-        queue = process.queue || @config[:process_queue]
-        ProcessWorker.client_push('class' => ProcessWorker, 'args' => [process.uuid], 'queue' => queue)
+        CreateProcessWorker.client_push('class' => CreateProcessWorker, 'args' => [definition.name, uuid, Taskinator::Persistence.serialize(args)], 'queue' => queue)
       end
 
       def enqueue_task(task)
@@ -40,14 +35,6 @@ module Taskinator
         end
       end
 
-      class ProcessWorker
-        include ::Sidekiq::Worker
-
-        def perform(process_uuid)
-          Taskinator::ProcessWorker.new(process_uuid).perform
-        end
-      end
-
       class TaskWorker
         include ::Sidekiq::Worker
 

data/lib/taskinator/queues.rb

@@ -34,11 +34,6 @@ module Taskinator
         adapter.enqueue_create_process(definition, uuid, args)
       end
 
-      def enqueue_process(process)
-        Taskinator.logger.info("Enqueuing process #{process}")
-        adapter.enqueue_process(process)
-      end
-
       def enqueue_task(task)
         Taskinator.logger.info("Enqueuing task #{task}")
         adapter.enqueue_task(task)

data/lib/taskinator/redis_connection.rb

@@ -14,6 +14,7 @@ require 'connection_pool'
 require 'redis'
 require 'uri'
 
+# :nocov:
 module Taskinator
   class RedisConnection
     class << self

data/lib/taskinator/task.rb

@@ -58,20 +58,18 @@ module Taskinator
       state :initial do
         event :enqueue, :transitions_to => :enqueued
         event :start, :transitions_to => :processing
+        event :complete, :transitions_to => :completed  # specific to a SubProcess which has no tasks
         event :fail, :transitions_to => :failed
       end
 
       state :enqueued do
         event :start, :transitions_to => :processing
-
-        # need to be able to complete, for when sub-tasks have no tasks
-        event :complete, :transitions_to => :completed, :if => :can_complete_task?
-
+        event :complete, :transitions_to => :completed
         event :fail, :transitions_to => :failed
       end
 
       state :processing do
-        event :complete, :transitions_to => :completed, :if => :can_complete_task?
+        event :complete, :transitions_to => :completed
         event :fail, :transitions_to => :failed
       end
 
@@ -82,16 +80,6 @@ module Taskinator
         Taskinator.logger.debug("TASK: #{self.class.name}:#{uuid} :: #{from} => #{to}")
       end
 
-      on_error do |error, from, to, event, *args|
-        Taskinator.logger.error("TASK: #{self.class.name}:#{uuid} :: #{error.message}")
-        Taskinator.logger.debug(error.backtrace)
-        fail!(error)
-      end
-    end
-
-    def can_complete_task?(*args)
-      # subclasses must implement this method
-      raise NotImplementedError
     end
 
     # include after defining the workflow
@@ -99,27 +87,28 @@ module Taskinator
     # need to override the ones defined by workflow
     include Persistence
 
-    def enqueue
-      Taskinator.queue.enqueue_task(self)
-    end
-
-    # callback for when the task has completed
-    def on_completed_entry(*args)
-      self.incr_completed
-      # notify the process that this task has completed
-      process.task_completed(self)
+    def complete
+      Taskinator.instrumenter.instrument('taskinator.task.completed', instrumentation_payload) do
+        # notify the process that this task has completed
+        process.task_completed(self)
+        self.incr_completed
+      end
     end
 
     # callback for when the task has failed
     def on_failed_entry(*args)
-      self.incr_failed
-      # notify the process that this task has failed
-      process.task_failed(self, args.last)
+      Taskinator.instrumenter.instrument('taskinator.task.failed', instrumentation_payload) do
+        self.incr_failed
+        # notify the process that this task has failed
+        process.task_failed(self, args.last)
+      end
     end
 
     # callback for when the task has cancelled
     def on_cancelled_entry(*args)
-      self.incr_cancelled
+      Taskinator.instrumenter.instrument('taskinator.task.cancelled', instrumentation_payload) do
+        self.incr_cancelled
+      end
     end
 
     # helper method, delegating to process
@@ -150,21 +139,24 @@ module Taskinator
         @args = args
       end
 
-      def executor
-        @executor ||= Taskinator::Executor.new(@definition, self)
+      def enqueue
+        Taskinator.instrumenter.instrument('taskinator.task.enqueued', instrumentation_payload) do
+          Taskinator.queue.enqueue_task(self)
+        end
       end
 
       def start
-        # ASSUMPTION: when the method returns, the task is considered to be complete
-        Taskinator.instrumenter.instrument(:execute_step_task, :uuid => uuid) do
+        Taskinator.instrumenter.instrument('taskinator.task.started', instrumentation_payload) do
           executor.send(method, *args)
         end
-        @is_complete = true
-      end
+        # ASSUMPTION: when the method returns, the task is considered to be complete
+        complete!
 
-      # NOTE: this _does not_ work when checking out-of-process
-      def can_complete_task?
-        defined?(@is_complete) && @is_complete
+      rescue => e
+        Taskinator.logger.error(e)
+        Taskinator.logger.debug(e.backtrace)
+        fail!(e)
+        raise e
       end
 
       def accept(visitor)
@@ -174,6 +166,10 @@ module Taskinator
         visitor.visit_args(:args)
       end
 
+      def executor
+        @executor ||= Taskinator::Executor.new(@definition, self)
+      end
+
       def inspect
         %(#<#{self.class.name}:0x#{self.__id__.to_s(16)} uuid="#{uuid}", method=:#{method}, args=#{args}, state=:#{current_state.name}>)
       end
@@ -198,19 +194,24 @@ module Taskinator
       end
 
       def enqueue
-        Taskinator.queue.enqueue_job(self)
+        Taskinator.instrumenter.instrument('taskinator.task.enqueued', instrumentation_payload) do
+          Taskinator.queue.enqueue_job(self)
+        end
       end
 
+      # can't use the start! method, since a block is required
       def perform
-        Taskinator.instrumenter.instrument(:execute_job_task, :uuid => uuid) do
+        Taskinator.instrumenter.instrument('taskinator.task.started', instrumentation_payload) do
           yield(job, args)
         end
-        @is_complete = true
-      end
+        # ASSUMPTION: when the method returns, the task is considered to be complete
+        complete!
 
-      # NOTE: this _does not_ work when checking out-of-process
-      def can_complete_task?
-        defined?(@is_complete) && @is_complete
+      rescue => e
+        Taskinator.logger.error(e)
+        Taskinator.logger.debug(e.backtrace)
+        fail!(e)
+        raise e
       end
 
       def accept(visitor)
@@ -237,15 +238,22 @@ module Taskinator
         @sub_process.parent = self
       end
 
+      def enqueue
+        Taskinator.instrumenter.instrument('taskinator.task.enqueued', instrumentation_payload) do
+          sub_process.enqueue!
+        end
+      end
+
       def start
-        Taskinator.instrumenter.instrument(:execute_subprocess, :uuid => uuid) do
+        Taskinator.instrumenter.instrument('taskinator.task.started', instrumentation_payload) do
           sub_process.start!
         end
-      end
 
-      def can_complete_task?
-        # NOTE: this works out-of-process, so there isn't any issue
-        sub_process.completed?
+      rescue => e
+        Taskinator.logger.error(e)
+        Taskinator.logger.debug(e.backtrace)
+        fail!(e)
+        raise e
       end
 
       def accept(visitor)
@@ -257,13 +265,5 @@ module Taskinator
         %(#<#{self.class.name}:0x#{self.__id__.to_s(16)} uuid="#{uuid}", sub_process=#{sub_process.inspect}, state=:#{current_state.name}>)
       end
     end
-
-    # reloads the task from storage
-    # NB: only implemented by LazyLoader so that
-    #   the task can be lazy loaded, thereafter
-    #    it has no effect
-    def reload
-      false
-    end
   end
 end

data/lib/taskinator/task_worker.rb

@@ -9,14 +9,7 @@ module Taskinator
     def perform
       task = Taskinator::Task.fetch(@uuid)
       return if task.paused? || task.cancelled?
-      begin
-        task.start!
-        task.complete! if task.can_complete?
-      rescue => e
-        Taskinator.logger.error(e)
-        task.fail!(e)
-        raise e
-      end
+      task.start!
     end
   end
 end

data/lib/taskinator/version.rb

@@ -1,3 +1,3 @@
 module Taskinator
-  VERSION = "0.0.18"
+  VERSION = "0.2.0"
 end

data/lib/taskinator.rb

@@ -20,7 +20,6 @@ require 'taskinator/process'
 
 require 'taskinator/task_worker'
 require 'taskinator/job_worker'
-require 'taskinator/process_worker'
 require 'taskinator/create_process_worker'
 
 require 'taskinator/executor'
@@ -99,9 +98,11 @@ module Taskinator
     end
 
     def queue
-      adapter = self.queue_adapter || :resque
-      config = queue_config || {}
-      @queue ||= Taskinator::Queues.create_adapter(adapter, config)
+      @queue ||= begin
+        adapter = self.queue_adapter || :resque
+        config = queue_config || {}
+        Taskinator::Queues.create_adapter(adapter, config)
+      end
     end
 
     # set the instrumenter to use.
@@ -116,14 +117,14 @@ module Taskinator
   end
 
   class NoOpInstrumenter
-    # just yield to the given block
+    # :nocov:
     def instrument(event, payload={})
       yield(payload) if block_given?
     end
   end
 
   class ConsoleInstrumenter
-    # just yield to the given block
+    # :nocov:
     def instrument(event, payload={})
       puts [event.inspect, payload.to_yaml]
       yield(payload) if block_given?

data/spec/examples/queue_adapter_examples.rb

@@ -20,16 +20,6 @@ shared_examples_for "a queue adapter" do |adapter_name, adapter_type|
     end
   end
 
-  describe "#enqueue_process" do
-    it { expect(subject).to respond_to(:enqueue_process) }
-
-    it "should enqueue a process" do
-      expect {
-        subject.enqueue_process(double('process', :uuid => 'xx-xx-xx-xx', :queue => nil))
-      }.to_not raise_error
-    end
-  end
-
   describe "#enqueue_task" do
     it { expect(subject).to respond_to(:enqueue_task) }
 

data/spec/support/test_definition.rb

@@ -1,3 +1,7 @@
 module TestDefinition
   extend Taskinator::Definition
+
+  def do_task(*args)
+  end
+
 end

data/spec/support/test_flow.rb

@@ -2,6 +2,8 @@ module TestFlow
   extend Taskinator::Definition
 
   define_process :some_arg1, :some_arg2 do
+
+    # TODO: add support for "continue_on_error"
     task :error_task, :continue_on_error => true
 
     task :the_task