taskinator 0.4.7 → 0.5.1

data/lib/taskinator/persistence.rb

@@ -1,3 +1,5 @@
+require 'builder'
+
 module Taskinator
   module Persistence
 
@@ -447,6 +449,46 @@ module Taskinator
       end
     end
 
+    class UnknownTypeError < StandardError
+    end
+
+    module UnknownType
+
+      @unknown_types = {}
+
+      def self.new(type)
+        Taskinator.logger.error("Unknown type '#{type}' while deserializing.")
+        @unknown_types[type] ||= Module.new do
+          extend UnknownType
+          @type = type
+
+          # for unknown definitions
+          def method_missing(_, *_)
+            raise UnknownTypeError.new(to_s)
+          end
+        end
+      end
+
+      attr_reader :type
+
+      def to_s
+        "Unknown type '#{type}'."
+      end
+
+      def allocate
+        self
+      end
+
+      def accept(*_)
+        # nothing doing
+      end
+
+      # for unknown job types
+      def perform(*_)
+        raise UnknownTypeError.new(to_s)
+      end
+    end
+
     class RedisDeserializationVisitor < Taskinator::Visitor::Base
 
       #
@@ -480,7 +522,7 @@ module Taskinator
         return unless @attribute_values.key?(:type)
 
         type = @attribute_values[:type]
-        klass = Kernel.const_get(type)
+        klass = Kernel.const_get(type) rescue UnknownType.new(type)
 
         #
         # NOTE:
@@ -539,16 +581,18 @@ module Taskinator
       def visit_attribute_enum(attribute, type)
         visit_attribute(attribute) do |value|
           const_value = type.constants.select {|c| type.const_get(c) == value.to_i }.first
-          const_value ?
-            type.const_get(const_value) :
-            (defined?(type::Default) ? type::Default : nil)
+          if const_value
+            type.const_get(const_value)
+          else
+            defined?(type::Default) ? type::Default : nil
+          end
         end
       end
 
       def visit_type(attribute)
         value = @attribute_values[attribute]
         if value
-          type = Kernel.const_get(value)
+          type = Kernel.const_get(value) rescue UnknownType.new(value)
           @instance.instance_variable_set("@#{attribute}", type)
         end
       end
@@ -576,6 +620,8 @@ module Taskinator
         type = Taskinator.redis do |conn|
           conn.hget(base.key_for(uuid), :type)
         end
+        # these types are always from taskinator
+        # so no need to rescue for unknown types!
         klass = Kernel.const_get(type)
         LazyLoader.new(klass, uuid, @instance_cache)
       end

data/lib/taskinator/process.rb

@@ -48,6 +48,7 @@ module Taskinator
       @parent = value
       # update the uuid to be "scoped" within the parent task
       @uuid = "#{@parent.uuid}:subprocess"
+      @key = nil # NB: invalidate memoized key
     end
 
     def tasks
@@ -123,7 +124,7 @@ module Taskinator
         instrument('taskinator.process.completed', completed_payload) do
           complete if respond_to?(:complete)
           # notify the parent task (if there is one) that this process has completed
-          # note: parent may be a proxy, so explicity check for nil?
+          # note: parent may be a proxy, so explicitly check for nil?
           unless parent.nil?
             parent.complete!
           else
@@ -133,6 +134,9 @@ module Taskinator
       end
     end
 
+    # TODO: add retry method - to pick up from a failed task
+    #  e.g. like retrying a failed job in Resque Web
+
     def tasks_completed?
       # TODO: optimize this
       tasks.all?(&:completed?)
@@ -151,7 +155,7 @@ module Taskinator
         instrument('taskinator.process.failed', failed_payload(error)) do
           fail(error) if respond_to?(:fail)
           # notify the parent task (if there is one) that this process has failed
-          # note: parent may be a proxy, so explicity check for nil?
+          # note: parent may be a proxy, so explicitly check for nil?
           parent.fail!(error) unless parent.nil?
         end
       end
@@ -166,6 +170,7 @@ module Taskinator
     # subclasses must implement the following methods
     #--------------------------------------------------
 
+    # :nocov:
     def enqueue
       raise NotImplementedError
     end
@@ -177,6 +182,7 @@ module Taskinator
     def task_completed(task)
       raise NotImplementedError
     end
+    # :nocov:
 
     #--------------------------------------------------
 
@@ -199,10 +205,9 @@ module Taskinator
       end
 
       def task_completed(task)
-        # deincrement the count of pending sequential tasks
+        # decrement the count of pending sequential tasks
         pending = deincr_pending_tasks
 
-        Taskinator.statsd_client.count("taskinator.#{definition.name.underscore.parameterize}.pending", pending)
         Taskinator.logger.info("Completed task for process '#{uuid}'. Pending is #{pending}.")
 
         next_task = task.next
@@ -222,19 +227,21 @@ module Taskinator
 
     class Concurrent < Process
       attr_reader :complete_on
+
+      # <b>DEPRECATED:</b> concurrency_method will be removed in a future version.
       attr_reader :concurrency_method
 
       def initialize(definition, complete_on=CompleteOn::Default, options={})
         super(definition, options)
         @complete_on = complete_on
         @concurrency_method = options.delete(:concurrency_method) || :thread
+        warn("[DEPRECATED]: concurrency_method will be removed in a future version.") if @concurrency_method == :fork
       end
 
       def enqueue
         if tasks.empty?
           complete! # weren't any tasks to start with
         else
-          Taskinator.statsd_client.count("taskinator.#{definition.name.underscore.parameterize}.pending", tasks.count)
           Taskinator.logger.info("Enqueuing #{tasks.count} tasks for process '#{uuid}'.")
           tasks.each(&:enqueue!)
         end
@@ -246,12 +253,15 @@ module Taskinator
           complete! # weren't any tasks to start with
         else
           if concurrency_method == :fork
+            # :nocov:
+            warn("[DEPRECATED]: concurrency_method will be removed in a future version.")
             tasks.each do |task|
               fork do
                 task.start!
               end
             end
             Process.waitall
+            # :nocov:
           else
             threads = tasks.map do |task|
               Thread.new do
@@ -264,14 +274,13 @@ module Taskinator
       end
 
       def task_completed(task)
-        # deincrement the count of pending concurrent tasks
+        # decrement the count of pending concurrent tasks
         pending = deincr_pending_tasks
 
-        Taskinator.statsd_client.count("taskinator.#{definition.name.underscore.parameterize}.pending", pending)
         Taskinator.logger.info("Completed task for process '#{uuid}'. Pending is #{pending}.")
 
         # when complete on first, then don't bother with subsequent tasks completing
-        if (complete_on == CompleteOn::First)
+        if complete_on == CompleteOn::First
           complete! unless completed?
         else
           complete! if pending < 1
@@ -279,7 +288,7 @@ module Taskinator
       end
 
       def tasks_completed?
-        if (complete_on == CompleteOn::First)
+        if complete_on == CompleteOn::First
           tasks.any?(&:completed?)
         else
           super # all

data/lib/taskinator/queues/active_job.rb

@@ -18,12 +18,6 @@ module Taskinator
           .perform_later(definition.name, uuid, Taskinator::Persistence.serialize(args))
       end
 
-      def enqueue_process(process)
-        queue = process.queue || @config[:process_queue]
-        ProcessWorker.set(:queue => queue)
-          .perform_later(process.uuid)
-      end
-
       def enqueue_task(task)
         queue = task.queue || @config[:task_queue]
         TaskWorker.set(:queue => queue)
@@ -36,12 +30,6 @@ module Taskinator
         end
       end
 
-      class ProcessWorker < ApplicationJob
-        def perform(process_uuid)
-          Taskinator::ProcessWorker.new(process_uuid).perform
-        end
-      end
-
       class TaskWorker < ApplicationJob
         def perform(task_uuid)
           Taskinator::TaskWorker.new(task_uuid).perform

data/lib/taskinator/queues/delayed_job.rb

@@ -17,11 +17,6 @@ module Taskinator
         ::Delayed::Job.enqueue CreateProcessWorker.new(definition.name, uuid, Taskinator::Persistence.serialize(args)), :queue => queue
       end
 
-      def enqueue_process(process)
-        queue = process.queue || @config[:process_queue]
-        ::Delayed::Job.enqueue ProcessWorker.new(process.uuid), :queue => queue
-      end
-
       def enqueue_task(task)
         queue = task.queue || @config[:task_queue]
         ::Delayed::Job.enqueue TaskWorker.new(task.uuid), :queue => queue
@@ -33,12 +28,6 @@ module Taskinator
         end
       end
 
-      ProcessWorker = Struct.new(:process_uuid) do
-        def perform
-          Taskinator::ProcessWorker.new(process_uuid).perform
-        end
-      end
-
       TaskWorker = Struct.new(:task_uuid) do
         def perform
           Taskinator::TaskWorker.new(task_uuid).perform

data/lib/taskinator/queues/resque.rb

@@ -15,10 +15,6 @@ module Taskinator
           @queue = config[:definition_queue]
         end
 
-        ProcessWorker.class_eval do
-          @queue = config[:process_queue]
-        end
-
         TaskWorker.class_eval do
           @queue = config[:task_queue]
         end
@@ -30,11 +26,6 @@ module Taskinator
         Resque.enqueue_to(queue, CreateProcessWorker, definition.name, uuid, Taskinator::Persistence.serialize(args))
       end
 
-      def enqueue_process(process)
-        queue = process.queue || Resque.queue_from_class(ProcessWorker)
-        Resque.enqueue_to(queue, ProcessWorker, process.uuid)
-      end
-
       def enqueue_task(task)
         queue = task.queue || Resque.queue_from_class(TaskWorker)
         Resque.enqueue_to(queue, TaskWorker, task.uuid)
@@ -46,12 +37,6 @@ module Taskinator
         end
       end
 
-      class ProcessWorker
-        def self.perform(process_uuid)
-          Taskinator::ProcessWorker.new(process_uuid).perform
-        end
-      end
-
       class TaskWorker
         def self.perform(task_uuid)
           Taskinator::TaskWorker.new(task_uuid).perform

data/lib/taskinator/queues/sidekiq.rb

@@ -17,11 +17,6 @@ module Taskinator
         CreateProcessWorker.client_push('class' => CreateProcessWorker, 'args' => [definition.name, uuid, Taskinator::Persistence.serialize(args)], 'queue' => queue)
       end
 
-      def enqueue_process(process)
-        queue = process.queue || @config[:process_queue]
-        TaskWorker.client_push('class' => ProcessWorker, 'args' => [process.uuid], 'queue' => queue)
-      end
-
       def enqueue_task(task)
         queue = task.queue || @config[:task_queue]
         TaskWorker.client_push('class' => TaskWorker, 'args' => [task.uuid], 'queue' => queue)
@@ -35,14 +30,6 @@ module Taskinator
         end
       end
 
-      class ProcessWorker
-        include ::Sidekiq::Worker
-
-        def perform(process_uuid)
-          Taskinator::ProcessWorker.new(process_uuid).perform
-        end
-      end
-
       class TaskWorker
         include ::Sidekiq::Worker
 

data/lib/taskinator/queues.rb

@@ -33,11 +33,6 @@ module Taskinator
         adapter.enqueue_create_process(definition, uuid, args)
       end
 
-      def enqueue_process(process)
-        Taskinator.logger.info("Enqueuing process #{process}")
-        adapter.enqueue_process(process)
-      end
-
       def enqueue_task(task)
         Taskinator.logger.info("Enqueuing task #{task}")
         adapter.enqueue_task(task)

data/lib/taskinator/task.rb

@@ -21,6 +21,7 @@ module Taskinator
     end
 
     attr_reader :process
+    attr_reader :definition
     attr_reader :uuid
     attr_reader :options
     attr_reader :queue
@@ -35,6 +36,7 @@ module Taskinator
 
       @uuid = "#{process.uuid}:task:#{Taskinator.generate_uuid}"
       @process = process
+      @definition = process.definition
       @options = options
       @queue = options.delete(:queue)
       @created_at = Time.now.utc
@@ -45,6 +47,7 @@ module Taskinator
     def accept(visitor)
       visitor.visit_attribute(:uuid)
       visitor.visit_process_reference(:process)
+      visitor.visit_type(:definition)
       visitor.visit_task_reference(:next)
       visitor.visit_args(:options)
       visitor.visit_attribute(:queue)
@@ -134,6 +137,7 @@ module Taskinator
     # subclasses must implement the following methods
     #--------------------------------------------------
 
+    # :nocov:
     def enqueue
       raise NotImplementedError
     end
@@ -141,6 +145,7 @@ module Taskinator
     def start
       raise NotImplementedError
     end
+    # :nocov:
 
     #--------------------------------------------------
     # and optionally, provide methods:
@@ -155,13 +160,11 @@ module Taskinator
     # a task which invokes the specified method on the definition
     # the args must be intrinsic types, since they are serialized to YAML
     class Step < Task
-      attr_reader :definition
       attr_reader :method
       attr_reader :args
 
       def initialize(process, method, args, options={})
         super(process, options)
-        @definition = process.definition  # for convenience
 
         raise ArgumentError, 'method' if method.nil?
         raise NoMethodError, method unless executor.respond_to?(method)
@@ -188,17 +191,16 @@ module Taskinator
 
       def accept(visitor)
         super
-        visitor.visit_type(:definition)
         visitor.visit_attribute(:method)
         visitor.visit_args(:args)
       end
 
       def executor
-        @executor ||= Taskinator::Executor.new(@definition, self)
+        @executor ||= Taskinator::Executor.new(definition, self)
       end
 
       def inspect
-        %(#<#{self.class.name}:0x#{self.__id__.to_s(16)} uuid="#{uuid}", method=:#{method}, args=#{args}, current_state=:#{current_state}>)
+        %(#<#{self.class.name}:0x#{self.__id__.to_s(16)} uuid="#{uuid}", definition=:#{definition}, method=:#{method}, args=#{args}, current_state=:#{current_state}>)
       end
     end
 
@@ -207,13 +209,11 @@ module Taskinator
     # a task which invokes the specified background job
     # the args must be intrinsic types, since they are serialized to YAML
     class Job < Task
-      attr_reader :definition
       attr_reader :job
       attr_reader :args
 
       def initialize(process, job, args, options={})
         super(process, options)
-        @definition = process.definition  # for convenience
 
         raise ArgumentError, 'job' if job.nil?
         raise ArgumentError, 'job' unless job.methods.include?(:perform) || job.instance_methods.include?(:perform)
@@ -228,7 +228,6 @@ module Taskinator
 
       def start
         # NNB: if other job types are required, may need to implement how they get invoked here!
-        # FIXME: possible implement using ActiveJob instead, so it doesn't matter how the worker is implemented
 
         if job.respond_to?(:perform)
           # resque
@@ -250,13 +249,12 @@ module Taskinator
 
       def accept(visitor)
         super
-        visitor.visit_type(:definition)
         visitor.visit_type(:job)
         visitor.visit_args(:args)
       end
 
       def inspect
-        %(#<#{self.class.name}:0x#{self.__id__.to_s(16)} uuid="#{uuid}", job=#{job}, args=#{args}, current_state=:#{current_state}>)
+        %(#<#{self.class.name}:0x#{self.__id__.to_s(16)} uuid="#{uuid}", definition=:#{definition}, job=#{job}, args=#{args}, current_state=:#{current_state}>)
       end
     end
 
@@ -302,7 +300,7 @@ module Taskinator
       end
 
       def inspect
-        %(#<#{self.class.name}:0x#{self.__id__.to_s(16)} uuid="#{uuid}", sub_process=#{sub_process.inspect}, current_state=:#{current_state}>)
+        %(#<#{self.class.name}:0x#{self.__id__.to_s(16)} uuid="#{uuid}", definition=:#{definition}, sub_process=#{sub_process.inspect}, current_state=:#{current_state}>)
       end
     end
   end

data/lib/taskinator/version.rb

@@ -1,3 +1,3 @@
 module Taskinator
-  VERSION = "0.4.7"
+  VERSION = "0.5.1"
 end

data/lib/taskinator.rb

@@ -6,8 +6,6 @@ require 'delegate'
 
 require 'taskinator/version'
 
-require 'taskinator/log_stats'
-
 require 'taskinator/complete_on'
 require 'taskinator/redis_connection'
 require 'taskinator/logger'
@@ -19,14 +17,12 @@ require 'taskinator/workflow'
 require 'taskinator/visitor'
 require 'taskinator/persistence'
 require 'taskinator/instrumentation'
-require 'taskinator/xml_visitor'
 
 require 'taskinator/task'
 require 'taskinator/tasks'
 require 'taskinator/process'
 
 require 'taskinator/task_worker'
-require 'taskinator/process_worker'
 require 'taskinator/create_process_worker'
 
 require 'taskinator/executor'
@@ -88,14 +84,6 @@ module Taskinator
       Taskinator::Logging.logger = log
     end
 
-    def statsd_client
-      Taskinator::LogStats.client
-    end
-
-    def statsd_client=(client)
-      Taskinator::LogStats.client = client
-    end
-
     # the queue adapter to use
     # supported adapters include
     # :active_job, :delayed_job, :redis and :sidekiq
@@ -136,14 +124,12 @@ module Taskinator
   end
 
   class NoOpInstrumenter
-    # :nocov:
     def instrument(event, payload={})
       yield(payload) if block_given?
     end
   end
 
   class ConsoleInstrumenter
-    # :nocov:
     def instrument(event, payload={})
       puts [event.inspect, payload.to_yaml]
       yield(payload) if block_given?

data/spec/support/test_job.rb

@@ -0,0 +1,30 @@
+module TestJob
+  def self.perform(*args)
+  end
+end
+
+class TestJobClass
+  def perform(*args)
+  end
+end
+
+module TestJobModule
+  def self.perform(*args)
+  end
+end
+
+class TestJobClassNoArgs
+  def perform
+  end
+end
+
+module TestJobModuleNoArgs
+  def self.perform
+  end
+end
+
+module TestJobError
+  def self.perform
+    raise ArgumentError
+  end
+end

data/spec/support/test_job_task.rb

@@ -0,0 +1,2 @@
+class TestJobTask < Taskinator::Task::Job
+end

data/spec/support/test_queue.rb

@@ -2,38 +2,42 @@ module Taskinator
   module Queues
 
     def self.create_test_queue_adapter(config={})
-      TestQueueAdapter.new
+      TestQueueAdapter.new(config)
     end
 
     def self.create_test_queue_worker_adapter(config={})
-      QueueWorkerAdapter.new
+      TestQueueWorkerAdapter.new(config)
     end
 
+    #
+    # this is a no-op adapter, it tracks enqueued processes and tasks
+    #
     class TestQueueAdapter
 
-      attr_reader :creates
-      attr_reader :tasks
-
-      def initialize
+      def initialize(config={})
         clear
       end
 
-      def clear
-        @creates = []
-        @tasks = []
-        @jobs = []
-      end
-
       def enqueue_create_process(definition, uuid, args)
-        @creates << [definition, uuid, args]
+        @processes << [definition, uuid, args]
       end
 
       def enqueue_task(task)
         @tasks << task
       end
 
+      # helpers
+
+      attr_reader :processes
+      attr_reader :tasks
+
+      def clear
+        @processes = []
+        @tasks = []
+      end
+
       def empty?
-        @creates.empty? && @tasks.empty? && @jobs.empty?
+        @processes.empty? && @tasks.empty?
       end
 
     end
@@ -41,7 +45,7 @@ module Taskinator
     #
     # this is a "synchronous" implementation for use in testing
     #
-    class QueueWorkerAdapter < TestQueueAdapter
+    class TestQueueWorkerAdapter < TestQueueAdapter
 
       def enqueue_create_process(definition, uuid, args)
         super
@@ -57,6 +61,8 @@ module Taskinator
         end
       end
 
+      private
+
       def invoke(&block)
         block.call
       end

data/spec/support/test_step_task.rb

@@ -0,0 +1,2 @@
+class TestStepTask < Taskinator::Task::Step
+end

data/spec/support/test_subprocess_task.rb

@@ -0,0 +1,2 @@
+class TestSubProcessTask < Taskinator::Task::SubProcess
+end

data/spec/taskinator/{definition/builder_spec.rb → builder_spec.rb}

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-describe Taskinator::Definition::Builder do
+describe Taskinator::Builder do
 
   let(:definition) do
     Module.new do
@@ -26,7 +26,7 @@ describe Taskinator::Definition::Builder do
     Proc.new {|*args| the_block.call }
   }
 
-  subject { Taskinator::Definition::Builder.new(process, definition, *[*args, builder_options]) }
+  subject { Taskinator::Builder.new(process, definition, *[*args, builder_options]) }
 
   it "assign attributes" do
     expect(subject.process).to eq(process)