taskinator 0.5.0 → 0.5.1

data/lib/taskinator/persistence.rb

@@ -1,3 +1,5 @@
+require 'builder'
+
 module Taskinator
   module Persistence
 
@@ -447,6 +449,46 @@ module Taskinator
       end
     end
 
+    class UnknownTypeError < StandardError
+    end
+
+    module UnknownType
+
+      @unknown_types = {}
+
+      def self.new(type)
+        Taskinator.logger.error("Unknown type '#{type}' while deserializing.")
+        @unknown_types[type] ||= Module.new do
+          extend UnknownType
+          @type = type
+
+          # for unknown definitions
+          def method_missing(_, *_)
+            raise UnknownTypeError.new(to_s)
+          end
+        end
+      end
+
+      attr_reader :type
+
+      def to_s
+        "Unknown type '#{type}'."
+      end
+
+      def allocate
+        self
+      end
+
+      def accept(*_)
+        # nothing doing
+      end
+
+      # for unknown job types
+      def perform(*_)
+        raise UnknownTypeError.new(to_s)
+      end
+    end
+
     class RedisDeserializationVisitor < Taskinator::Visitor::Base
 
       #
@@ -480,7 +522,7 @@ module Taskinator
         return unless @attribute_values.key?(:type)
 
         type = @attribute_values[:type]
-        klass = Kernel.const_get(type)
+        klass = Kernel.const_get(type) rescue UnknownType.new(type)
 
         #
         # NOTE:
@@ -539,16 +581,18 @@ module Taskinator
       def visit_attribute_enum(attribute, type)
         visit_attribute(attribute) do |value|
           const_value = type.constants.select {|c| type.const_get(c) == value.to_i }.first
-          const_value ?
-            type.const_get(const_value) :
-            (defined?(type::Default) ? type::Default : nil)
+          if const_value
+            type.const_get(const_value)
+          else
+            defined?(type::Default) ? type::Default : nil
+          end
         end
       end
 
       def visit_type(attribute)
         value = @attribute_values[attribute]
         if value
-          type = Kernel.const_get(value)
+          type = Kernel.const_get(value) rescue UnknownType.new(value)
           @instance.instance_variable_set("@#{attribute}", type)
         end
       end
@@ -576,6 +620,8 @@ module Taskinator
         type = Taskinator.redis do |conn|
           conn.hget(base.key_for(uuid), :type)
         end
+        # these types are always from taskinator
+        # so no need to rescue for unknown types!
         klass = Kernel.const_get(type)
         LazyLoader.new(klass, uuid, @instance_cache)
       end

data/lib/taskinator/process.rb

@@ -48,6 +48,7 @@ module Taskinator
       @parent = value
       # update the uuid to be "scoped" within the parent task
       @uuid = "#{@parent.uuid}:subprocess"
+      @key = nil # NB: invalidate memoized key
     end
 
     def tasks
@@ -123,7 +124,7 @@ module Taskinator
         instrument('taskinator.process.completed', completed_payload) do
           complete if respond_to?(:complete)
           # notify the parent task (if there is one) that this process has completed
-          # note: parent may be a proxy, so explicity check for nil?
+          # note: parent may be a proxy, so explicitly check for nil?
           unless parent.nil?
             parent.complete!
           else
@@ -133,6 +134,9 @@ module Taskinator
       end
     end
 
+    # TODO: add retry method - to pick up from a failed task
+    #  e.g. like retrying a failed job in Resque Web
+
     def tasks_completed?
       # TODO: optimize this
       tasks.all?(&:completed?)
@@ -151,7 +155,7 @@ module Taskinator
         instrument('taskinator.process.failed', failed_payload(error)) do
           fail(error) if respond_to?(:fail)
           # notify the parent task (if there is one) that this process has failed
-          # note: parent may be a proxy, so explicity check for nil?
+          # note: parent may be a proxy, so explicitly check for nil?
           parent.fail!(error) unless parent.nil?
         end
       end
@@ -166,6 +170,7 @@ module Taskinator
     # subclasses must implement the following methods
     #--------------------------------------------------
 
+    # :nocov:
     def enqueue
       raise NotImplementedError
     end
@@ -177,6 +182,7 @@ module Taskinator
     def task_completed(task)
       raise NotImplementedError
     end
+    # :nocov:
 
     #--------------------------------------------------
 
@@ -199,10 +205,9 @@ module Taskinator
       end
 
       def task_completed(task)
-        # deincrement the count of pending sequential tasks
+        # decrement the count of pending sequential tasks
         pending = deincr_pending_tasks
 
-        Taskinator.statsd_client.count("taskinator.#{definition.name.underscore.parameterize}.pending", pending)
         Taskinator.logger.info("Completed task for process '#{uuid}'. Pending is #{pending}.")
 
         next_task = task.next
@@ -237,7 +242,6 @@ module Taskinator
         if tasks.empty?
           complete! # weren't any tasks to start with
         else
-          Taskinator.statsd_client.count("taskinator.#{definition.name.underscore.parameterize}.pending", tasks.count)
           Taskinator.logger.info("Enqueuing #{tasks.count} tasks for process '#{uuid}'.")
           tasks.each(&:enqueue!)
         end
@@ -249,6 +253,7 @@ module Taskinator
           complete! # weren't any tasks to start with
         else
           if concurrency_method == :fork
+            # :nocov:
             warn("[DEPRECATED]: concurrency_method will be removed in a future version.")
             tasks.each do |task|
               fork do
@@ -256,6 +261,7 @@ module Taskinator
               end
             end
             Process.waitall
+            # :nocov:
           else
             threads = tasks.map do |task|
               Thread.new do
@@ -268,14 +274,13 @@ module Taskinator
       end
 
       def task_completed(task)
-        # deincrement the count of pending concurrent tasks
+        # decrement the count of pending concurrent tasks
         pending = deincr_pending_tasks
 
-        Taskinator.statsd_client.count("taskinator.#{definition.name.underscore.parameterize}.pending", pending)
         Taskinator.logger.info("Completed task for process '#{uuid}'. Pending is #{pending}.")
 
         # when complete on first, then don't bother with subsequent tasks completing
-        if (complete_on == CompleteOn::First)
+        if complete_on == CompleteOn::First
           complete! unless completed?
         else
           complete! if pending < 1
@@ -283,7 +288,7 @@ module Taskinator
       end
 
       def tasks_completed?
-        if (complete_on == CompleteOn::First)
+        if complete_on == CompleteOn::First
           tasks.any?(&:completed?)
         else
           super # all

data/lib/taskinator/task.rb

@@ -21,6 +21,7 @@ module Taskinator
     end
 
     attr_reader :process
+    attr_reader :definition
     attr_reader :uuid
     attr_reader :options
     attr_reader :queue
@@ -35,6 +36,7 @@ module Taskinator
 
       @uuid = "#{process.uuid}:task:#{Taskinator.generate_uuid}"
       @process = process
+      @definition = process.definition
       @options = options
       @queue = options.delete(:queue)
       @created_at = Time.now.utc
@@ -45,6 +47,7 @@ module Taskinator
     def accept(visitor)
       visitor.visit_attribute(:uuid)
       visitor.visit_process_reference(:process)
+      visitor.visit_type(:definition)
       visitor.visit_task_reference(:next)
       visitor.visit_args(:options)
       visitor.visit_attribute(:queue)
@@ -134,6 +137,7 @@ module Taskinator
     # subclasses must implement the following methods
     #--------------------------------------------------
 
+    # :nocov:
     def enqueue
       raise NotImplementedError
     end
@@ -141,6 +145,7 @@ module Taskinator
     def start
       raise NotImplementedError
     end
+    # :nocov:
 
     #--------------------------------------------------
     # and optionally, provide methods:
@@ -155,13 +160,11 @@ module Taskinator
     # a task which invokes the specified method on the definition
     # the args must be intrinsic types, since they are serialized to YAML
     class Step < Task
-      attr_reader :definition
       attr_reader :method
       attr_reader :args
 
       def initialize(process, method, args, options={})
         super(process, options)
-        @definition = process.definition  # for convenience
 
         raise ArgumentError, 'method' if method.nil?
         raise NoMethodError, method unless executor.respond_to?(method)
@@ -188,17 +191,16 @@ module Taskinator
 
       def accept(visitor)
         super
-        visitor.visit_type(:definition)
         visitor.visit_attribute(:method)
         visitor.visit_args(:args)
       end
 
       def executor
-        @executor ||= Taskinator::Executor.new(@definition, self)
+        @executor ||= Taskinator::Executor.new(definition, self)
       end
 
       def inspect
-        %(#<#{self.class.name}:0x#{self.__id__.to_s(16)} uuid="#{uuid}", method=:#{method}, args=#{args}, current_state=:#{current_state}>)
+        %(#<#{self.class.name}:0x#{self.__id__.to_s(16)} uuid="#{uuid}", definition=:#{definition}, method=:#{method}, args=#{args}, current_state=:#{current_state}>)
       end
     end
 
@@ -207,13 +209,11 @@ module Taskinator
     # a task which invokes the specified background job
     # the args must be intrinsic types, since they are serialized to YAML
     class Job < Task
-      attr_reader :definition
       attr_reader :job
       attr_reader :args
 
       def initialize(process, job, args, options={})
         super(process, options)
-        @definition = process.definition  # for convenience
 
         raise ArgumentError, 'job' if job.nil?
         raise ArgumentError, 'job' unless job.methods.include?(:perform) || job.instance_methods.include?(:perform)
@@ -228,7 +228,6 @@ module Taskinator
 
       def start
         # NNB: if other job types are required, may need to implement how they get invoked here!
-        # FIXME: possible implement using ActiveJob instead, so it doesn't matter how the worker is implemented
 
         if job.respond_to?(:perform)
           # resque
@@ -250,13 +249,12 @@ module Taskinator
 
       def accept(visitor)
         super
-        visitor.visit_type(:definition)
         visitor.visit_type(:job)
         visitor.visit_args(:args)
       end
 
       def inspect
-        %(#<#{self.class.name}:0x#{self.__id__.to_s(16)} uuid="#{uuid}", job=#{job}, args=#{args}, current_state=:#{current_state}>)
+        %(#<#{self.class.name}:0x#{self.__id__.to_s(16)} uuid="#{uuid}", definition=:#{definition}, job=#{job}, args=#{args}, current_state=:#{current_state}>)
       end
     end
 
@@ -302,7 +300,7 @@ module Taskinator
       end
 
       def inspect
-        %(#<#{self.class.name}:0x#{self.__id__.to_s(16)} uuid="#{uuid}", sub_process=#{sub_process.inspect}, current_state=:#{current_state}>)
+        %(#<#{self.class.name}:0x#{self.__id__.to_s(16)} uuid="#{uuid}", definition=:#{definition}, sub_process=#{sub_process.inspect}, current_state=:#{current_state}>)
       end
     end
   end

data/lib/taskinator/version.rb

@@ -1,3 +1,3 @@
 module Taskinator
-  VERSION = "0.5.0"
+  VERSION = "0.5.1"
 end

data/lib/taskinator.rb

@@ -6,8 +6,6 @@ require 'delegate'
 
 require 'taskinator/version'
 
-require 'taskinator/log_stats'
-
 require 'taskinator/complete_on'
 require 'taskinator/redis_connection'
 require 'taskinator/logger'
@@ -19,7 +17,6 @@ require 'taskinator/workflow'
 require 'taskinator/visitor'
 require 'taskinator/persistence'
 require 'taskinator/instrumentation'
-require 'taskinator/xml_visitor'
 
 require 'taskinator/task'
 require 'taskinator/tasks'
@@ -87,14 +84,6 @@ module Taskinator
       Taskinator::Logging.logger = log
     end
 
-    def statsd_client
-      Taskinator::LogStats.client
-    end
-
-    def statsd_client=(client)
-      Taskinator::LogStats.client = client
-    end
-
     # the queue adapter to use
     # supported adapters include
     # :active_job, :delayed_job, :redis and :sidekiq
@@ -135,14 +124,12 @@ module Taskinator
   end
 
   class NoOpInstrumenter
-    # :nocov:
     def instrument(event, payload={})
       yield(payload) if block_given?
     end
   end
 
   class ConsoleInstrumenter
-    # :nocov:
     def instrument(event, payload={})
       puts [event.inspect, payload.to_yaml]
       yield(payload) if block_given?

data/spec/support/test_job.rb

@@ -0,0 +1,30 @@
+module TestJob
+  def self.perform(*args)
+  end
+end
+
+class TestJobClass
+  def perform(*args)
+  end
+end
+
+module TestJobModule
+  def self.perform(*args)
+  end
+end
+
+class TestJobClassNoArgs
+  def perform
+  end
+end
+
+module TestJobModuleNoArgs
+  def self.perform
+  end
+end
+
+module TestJobError
+  def self.perform
+    raise ArgumentError
+  end
+end

data/spec/support/test_job_task.rb

@@ -0,0 +1,2 @@
+class TestJobTask < Taskinator::Task::Job
+end

data/spec/support/test_step_task.rb

@@ -0,0 +1,2 @@
+class TestStepTask < Taskinator::Task::Step
+end

data/spec/support/test_subprocess_task.rb

@@ -0,0 +1,2 @@
+class TestSubProcessTask < Taskinator::Task::SubProcess
+end

data/spec/taskinator/{definition/builder_spec.rb → builder_spec.rb}

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-describe Taskinator::Definition::Builder do
+describe Taskinator::Builder do
 
   let(:definition) do
     Module.new do
@@ -26,7 +26,7 @@ describe Taskinator::Definition::Builder do
     Proc.new {|*args| the_block.call }
   }
 
-  subject { Taskinator::Definition::Builder.new(process, definition, *[*args, builder_options]) }
+  subject { Taskinator::Builder.new(process, definition, *[*args, builder_options]) }
 
   it "assign attributes" do
     expect(subject.process).to eq(process)

data/spec/taskinator/definition_spec.rb

@@ -106,6 +106,37 @@ describe Taskinator::Definition do
       expect(subject.create_process).to be_a(Taskinator::Process)
     end
 
+    it "handles error" do
+      subject.define_process do
+        raise ArgumentError
+      end
+
+      expect {
+        subject.create_process
+      }.to raise_error(ArgumentError)
+    end
+
+    it "checks defined arguments provided" do
+      subject.define_process :arg1, :arg2 do
+      end
+
+      expect {
+        subject.create_process
+      }.to raise_error(ArgumentError)
+
+      expect {
+        subject.create_process :foo
+      }.to raise_error(ArgumentError)
+
+      expect {
+        subject.create_process :foo, :bar
+      }.not_to raise_error
+
+      expect {
+        subject.create_process :foo, :bar, :baz
+      }.not_to raise_error
+    end
+
     it "defaults the scope to :shared" do
       block = SpecSupport::Block.new
       allow(block).to receive(:to_proc) {

data/spec/taskinator/instrumentation_spec.rb

@@ -2,8 +2,6 @@ require 'spec_helper'
 
 describe Taskinator::Instrumentation, :redis => true do
 
-  let(:definition) { TestDefinition }
-
   subject do
     klass = Class.new do
       include Taskinator::Persistence
@@ -15,10 +13,12 @@ describe Taskinator::Instrumentation, :redis => true do
 
       attr_reader :uuid
       attr_reader :options
+      attr_reader :definition
 
       def initialize
         @uuid = Taskinator.generate_uuid
         @options = { :bar => :baz }
+        @definition = TestDefinition
       end
     end
 
@@ -75,6 +75,7 @@ describe Taskinator::Instrumentation, :redis => true do
       expect(subject.completed_payload(:baz => :qux)).to eq(
         OpenStruct.new({
           :type                   => subject.class.name,
+          :definition             => subject.definition.name,
           :process_uuid           => subject.uuid,
           :process_options        => {:foo => :bar},
           :uuid                   => subject.uuid,

data/spec/taskinator/persistence_spec.rb

@@ -46,6 +46,15 @@ describe Taskinator::Persistence, :redis => true do
         expect(subject.fetch('uuid', cache)).to eq(item)
       end
 
+      it "yields UnknownType" do
+        Taskinator.redis do |conn|
+          conn.hmset(*[subject.key_for("foo"), [:type, 'UnknownFoo']])
+        end
+        instance = subject.fetch("foo")
+        expect(instance).to be_a(Taskinator::Persistence::UnknownType)
+        expect(instance.type).to eq("UnknownFoo")
+      end
+
       describe "for processes" do
         let(:process) { TestProcess.new(definition) }
 
@@ -53,6 +62,19 @@ describe Taskinator::Persistence, :redis => true do
           process.save
           expect(TestProcess.fetch(process.uuid)).to eq(process)
         }
+
+        describe "unknown definition" do
+          it "yields UnknownType" do
+            Taskinator.redis do |conn|
+              conn.hmset(*[process.key, [:type, TestProcess.name], [:uuid, process.uuid], [:definition, 'UnknownFoo']])
+            end
+
+            instance = TestProcess.fetch(process.uuid)
+            expect(instance.uuid).to eq(process.uuid)
+            expect(instance.definition).to be_a(Taskinator::Persistence::UnknownType)
+            expect(instance.definition.type).to eq("UnknownFoo")
+          end
+        end
       end
 
       describe "for tasks" do
@@ -62,9 +84,45 @@ describe Taskinator::Persistence, :redis => true do
         it {
           process.tasks << task
           process.save
-          expect(TestTask.fetch(task.uuid)).to eq(task)
-          expect(TestTask.fetch(task.uuid).process).to eq(process)
+
+          instance = TestTask.fetch(task.uuid)
+          expect(instance).to eq(task)
+          expect(instance.process).to eq(process)
         }
+
+        describe "unknown job" do
+          let(:task) { TestJobTask.new(process, TestJob, []) }
+
+          it "yields UnknownType" do
+            Taskinator.redis do |conn|
+              conn.hmset(*[task.key, [:type, task.class.name], [:uuid, task.uuid], [:job, 'UnknownBar']])
+            end
+
+            instance = TestJobTask.fetch(task.uuid)
+            expect(instance.uuid).to eq(task.uuid)
+            expect(instance.job).to be_a(Taskinator::Persistence::UnknownType)
+            expect(instance.job.type).to eq("UnknownBar")
+          end
+        end
+
+        describe "unknown subprocess" do
+          let(:sub_process) { TestProcess.new(definition) }
+          let(:task) { TestSubProcessTask.new(process, sub_process) }
+
+          it "yields UnknownType" do
+            Taskinator.redis do |conn|
+              conn.multi do |transaction|
+                transaction.hmset(*[task.key, [:type, task.class.name], [:uuid, task.uuid], [:sub_process, sub_process.uuid]])
+                transaction.hmset(*[sub_process.key, [:type, sub_process.class.name], [:uuid, sub_process.uuid], [:definition, 'UnknownBaz']])
+              end
+            end
+
+            instance = TestSubProcessTask.fetch(task.uuid)
+            expect(instance.uuid).to eq(task.uuid)
+            expect(instance.sub_process.definition).to be_a(Taskinator::Persistence::UnknownType)
+            expect(instance.sub_process.definition.type).to eq("UnknownBaz")
+          end
+        end
       end
     end
   end
@@ -188,6 +246,72 @@ describe Taskinator::Persistence, :redis => true do
     end
   end
 
+  describe "unknown type helpers" do
+    subject { Taskinator::Persistence::UnknownType }
+
+    describe "#new" do
+      it "instantiates new module instance" do
+        instance = subject.new("foo")
+        expect(instance).to_not be_nil
+        expect(instance).to be_a(::Module)
+      end
+
+      it "yields same instance for same type" do
+        instance1 = subject.new("foo")
+        instance2 = subject.new("foo")
+        expect(instance1).to eq(instance2)
+      end
+    end
+
+    describe ".type" do
+      it {
+        instance = subject.new("foo")
+        expect(instance.type).to eq("foo")
+      }
+    end
+
+    describe ".to_s" do
+      it {
+        instance = subject.new("foo")
+        expect(instance.to_s).to eq("Unknown type 'foo'.")
+      }
+    end
+
+    describe ".allocate" do
+      it "emulates Object#allocate" do
+        instance = subject.new("foo")
+        expect(instance.allocate).to eq(instance)
+      end
+    end
+
+    describe ".accept" do
+      it {
+        instance = subject.new("foo")
+        expect(instance).to respond_to(:accept)
+      }
+    end
+
+    describe ".perform" do
+      it "raises UnknownTypeError" do
+        instance = subject.new("foo")
+        expect {
+          instance.perform(:foo, 1, false)
+        }.to raise_error(Taskinator::Persistence::UnknownTypeError)
+      end
+    end
+
+    describe "via executor" do
+      it "raises UnknownTypeError" do
+        instance = subject.new("foo")
+        executor = Taskinator::Executor.new(instance)
+
+        expect {
+          executor.foo
+        }.to raise_error(Taskinator::Persistence::UnknownTypeError)
+      end
+    end
+  end
+
   describe "instance methods" do
     subject {
       klass = Class.new do
@@ -207,11 +331,14 @@ describe Taskinator::Persistence, :redis => true do
     }
 
     describe "#save" do
-      pending __FILE__
+      pending
     end
 
     describe "#to_xml" do
-      pending __FILE__
+      it {
+        process = TestFlows::Task.create_process(1)
+        expect(process.to_xml).to match(/xml/)
+      }
     end
 
     describe "#key" do
@@ -426,6 +553,5 @@ describe Taskinator::Persistence, :redis => true do
       end
 
     end
-
   end
 end