taskinator 0.0.15 → 0.0.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bc6244f01294aaf18ec4f1236e7c77f1ed0ef79a
-  data.tar.gz: ba1c09e29ee13118858d9af4e149002a94361018
+  metadata.gz: 1c4844721e58ee539e62099b0cf6a9181bad692b
+  data.tar.gz: 00d8e01dbf3b6170231946bbeb18856bde5a6e8d
 SHA512:
-  metadata.gz: 4130f514ee353149642139cf6c1aac940850c697a42018d5204db7361dc9918f49d270ffb678fd9c99d9c9e53ffe6aea7598f522dd8d7620ec02e40ba58fead1
-  data.tar.gz: 79f411acfc01f1da54bd639f323c93c442312ee374de99b3228c67ebdb4ca0cd8f6b6cb295c329ef54493b3ba2f0793e3a2dc709044be284ed0f1e1c7712a746
+  metadata.gz: e2b21e17464032c8e59cbe5909ba1121b83e9cb77623f0314352563e416f10056b11e36e30717a3091ffa725d5f4ae6f5cc7f337bedf50b7cc41e606721ca672
+  data.tar.gz: 9e342b0f899929985f87c86e063b503a310227adec593e6506a591ac8acc7ffc3cebd81b0159b091264f2da33574d842c05721a2d3c55c43759ec463ad6484e9

data/CHANGELOG.md

@@ -0,0 +1,77 @@
+v0.0.16 - 25 Jun 2015
+---
+Added ability to enqueue the creation of processes; added a new worker, `CreateProcessWorker`
+Added support for instrumentation
+Improvements to error handling
+Bug fix for the persistence of the `queue` attribute for `Process` and `Task`
+Code clean up and additional specs added
+
+v0.0.15 - 28 May 2015
+---
+Added ability to specify the queue to use when enqueing processes, tasks and jobs
+Improvements to specs for testing with sidekiq; added `rspec-sidekiq` as development dependency
+Gem dependencies updated as per Gemnasium advisory
+
+v0.0.14 - 12 May 2015
+---
+Bug fix for fail! methods
+Bug fix to parameter handling by for_each method
+
+v0.0.13 - 11 May 2015
+---
+Bug fix to `Taskinator::Api` for listing of processes; should only include top-level processes
+Gem dependencies updated as per Gemnasium advisory
+
+v0.0.12 - 20 Apr 2015
+---
+Gem dependencies updated as per Gemnasium advisory
+
+v0.0.11 - 2 Mar 2015
+---
+Gem dependencies updated as per Gemnasium advisory
+
+v0.0.10 - 26 Feb 2015
+---
+Documentation updates
+
+v0.0.9 - 19 Dec 2014
+---
+Various bug fixes
+Added error logging
+Workflow states now include `complete` event
+Gem dependencies updated as per Gemnasium advisory
+
+v0.0.8 - 11 Nov 2014
+---
+Added support for argument chaining with `for_each` and `transform`
+Documentation updates
+Gem dependencies updated as per Gemnasium advisory
+
+v0.0.7 - 16 Oct 2014
+---
+Added better option handling; introduced `option?(key)` method
+Added support for definining the expected arguments for a process
+Gem dependencies updated as per Gemnasium advisory
+
+v0.0.5 - 17 Sep 2014
+---
+Various of bug fixes
+Improved error handling
+Added logging for queuing of processes, tasks and jobs
+
+v0.0.4 - 12 Sep 2014
+---
+Improvements to serialization; make use of GlobalID functionality
+Added support for "job" tasks; reusing existing workers as tasks
+
+v0.0.3 - 2 Sep 2014
+---
+Added failure steps to workflow of processes and tasks
+
+v0.0.2 - 12 Aug 2014
+---
+Refactored how tasks are defined in definitions
+
+v0.0.1 - 12 Aug 2014
+---
+Initial release

data/Gemfile.lock

@@ -8,7 +8,7 @@ GIT
 PATH
   remote: .
   specs:
-    taskinator (0.0.15)
+    taskinator (0.0.16)
       connection_pool (>= 2.2.0)
       json (>= 1.8.2)
       redis (>= 3.2.1)
@@ -134,6 +134,7 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  activesupport (>= 4.0.0)
   bundler (>= 1.6.0)
   coveralls (>= 0.7.0)
   delayed_job (>= 4.0.0)

data/README.md

@@ -543,7 +543,7 @@ In reality, each task is executed by a worker process, possibly on another host,
 To monitor the state of the processes, use the `Taskinator::Api::Processes` class. This is still a work in progress.
 
 ```ruby
-processes = Taskinator::Api::Processes.new()
+processes = Taskinator::Api::Processes.new
 processes.each do |process|
   # => output the unique process identifier and current state
   puts [:process, process.uuid, process.current_state.name]

data/lib/taskinator/api.rb

@@ -3,7 +3,9 @@ module Taskinator
     class Processes
       include Enumerable
 
-      def each(&block)
+      def each
+        return to_enum(__method__) unless block_given?
+
         instance_cache = {}
         Taskinator.redis do |conn|
           uuids = conn.smembers("taskinator:processes")

data/lib/taskinator/create_process_worker.rb

@@ -0,0 +1,60 @@
+module Taskinator
+  class CreateProcessWorker
+
+    attr_reader :definition
+    attr_reader :uuid
+    attr_reader :args
+
+    def initialize(definition_name, uuid, args)
+
+      # convert to the module
+      @definition = constantize(definition_name)
+
+      # this will be uuid of the created process
+      @uuid = uuid
+
+      # convert to the typed arguments
+      @args = Taskinator::Persistence.deserialize(args)
+
+    end
+
+    def perform
+      @definition._create_process_(*@args, :uuid => @uuid).enqueue!
+    end
+
+    private
+
+    # borrowed from activesupport/lib/active_support/inflector/methods.rb
+    def constantize(camel_cased_word)
+      names = camel_cased_word.split('::')
+
+      # Trigger a built-in NameError exception including the ill-formed constant in the message.
+      Object.const_get(camel_cased_word) if names.empty?
+
+      # Remove the first blank element in case of '::ClassName' notation.
+      names.shift if names.size > 1 && names.first.empty?
+
+      names.inject(Object) do |constant, name|
+        if constant == Object
+          constant.const_get(name)
+        else
+          candidate = constant.const_get(name)
+          next candidate if constant.const_defined?(name, false)
+          next candidate unless Object.const_defined?(name)
+
+          # Go down the ancestors to check if it is owned directly. The check
+          # stops when we reach Object or the end of ancestors tree.
+          constant = constant.ancestors.inject do |const, ancestor|
+            break const    if ancestor == Object
+            break ancestor if ancestor.const_defined?(name, false)
+            const
+          end
+
+          # owner is in Object, so raise
+          constant.const_get(name, false)
+        end
+      end
+    end
+
+  end
+end

data/lib/taskinator/definition.rb

@@ -1,9 +1,17 @@
 module Taskinator
   module Definition
-    class UndefinedProcessError < RuntimeError; end
+
+    # errors
+    class ProcessUndefinedError < StandardError; end
+    class ProcessAlreadyDefinedError < StandardError; end
+
+    # for backward compatibility
+    UndefinedProcessError = ProcessUndefinedError
 
     # defines a process
     def define_process(*arg_list, &block)
+      raise ProcessAlreadyDefinedError if respond_to?(:_create_process_)
+
       define_singleton_method :_create_process_ do |args, options={}|
 
         # TODO: better validation of arguments
@@ -12,28 +20,59 @@ module Taskinator
 
         raise ArgumentError, "wrong number of arguments (#{args.length} for #{arg_list.length})" if args.length < arg_list.length
 
-        process = Process.define_sequential_process_for(self)
-        Builder.new(process, self, *args).instance_eval(&block)
-        process.save
+        process = Process.define_sequential_process_for(self, options)
 
-        # if this is a root process, then add it to the list
-        Persistence.add_process_to_list(process) unless options[:subprocess]
+        # this may take long... up to users definition
+        Taskinator.instrumenter.instrument(:create_process, :uuid => process.uuid) do
+          Builder.new(process, self, *args).instance_eval(&block)
+        end
+
+        # instrument separately
+        Taskinator.instrumenter.instrument(:save_process, :uuid => process.uuid) do
+          process.save
+          # if this is a root process, then add it to the list
+          Persistence.add_process_to_list(process) unless options[:subprocess]
+        end
 
         process
       end
     end
 
+    attr_accessor :queue
+
+    #
     # creates an instance of the process
     # NOTE: the supplied @args are serialized and ultimately passed to each method of the defined process
+    #
     def create_process(*args)
-      raise UndefinedProcessError unless respond_to?(:_create_process_)
+      assert_valid_process_module
       _create_process_(args)
     end
 
+    #
+    # returns a placeholder process, with the uuid attribute of the
+    # actual process. the callee can call `reload` if required to
+    # get the actual process, once it has been built by the CreateProcessWorker
+    #
+    def create_process_remotely(*args)
+      assert_valid_process_module
+      uuid = SecureRandom.uuid
+      Taskinator.queue.enqueue_create_process(self, uuid, args)
+
+      Taskinator::Persistence::LazyLoader.new(Taskinator::Process, uuid)
+    end
+
     def create_sub_process(*args)
-      raise UndefinedProcessError unless respond_to?(:_create_process_)
+      assert_valid_process_module
       _create_process_(args, :subprocess => true)
     end
+
+    private
+
+    def assert_valid_process_module
+      raise ProcessUndefinedError unless respond_to?(:_create_process_)
+    end
+
   end
 end
 

data/lib/taskinator/job_worker.rb

@@ -10,9 +10,9 @@ module Taskinator
       return if task.paused? || task.cancelled?
       begin
         task.start!
-        task.perform &block
+        task.perform(&block)
         task.complete!
-      rescue Exception => e
+      rescue => e
         Taskinator.logger.error(e)
         task.fail!(e)
         raise e

data/lib/taskinator/logger.rb

@@ -5,11 +5,6 @@
 # the LGPLv3 license.  Please see <http://www.gnu.org/licenses/lgpl-3.0.html>
 # for license text.
 #
-# Sidekiq Pro has a commercial-friendly license allowing private forks
-# and modifications of Sidekiq.  Please see http://sidekiq.org/pro/ for
-# more detail.  You can find the commercial license terms in COMM-LICENSE.
-#
-
 require 'time'
 require 'logger'
 
@@ -28,82 +23,25 @@ module Taskinator
       end
     end
 
-    def self.with_context(msg)
-      begin
-        Thread.current[:taskinator_context] = msg
-        yield
-      ensure
-        Thread.current[:taskinator_context] = nil
-      end
-    end
-
-    def self.initialize_logger(log_target = STDOUT)
-      oldlogger = defined?(@logger) ? @logger : nil
-      @logger = Logger.new(log_target)
-      @logger.level = Logger::INFO
-      @logger.formatter = Pretty.new
-      oldlogger.close if oldlogger && !$TESTING # don't want to close testing's STDOUT logging
-      @logger
-    end
-
-    def self.logger
-      defined?(@logger) ? @logger : initialize_logger
-    end
-
-    def self.logger=(log)
-      @logger = (log ? log : Logger.new('/dev/null'))
-    end
-
-    # This reopens ALL logfiles in the process that have been rotated
-    # using logrotate(8) (without copytruncate) or similar tools.
-    # A +File+ object is considered for reopening if it is:
-    #   1) opened with the O_APPEND and O_WRONLY flags
-    #   2) the current open file handle does not match its original open path
-    #   3) unbuffered (as far as userspace buffering goes, not O_SYNC)
-    # Returns the number of files reopened
-    def self.reopen_logs
-      to_reopen = []
-      append_flags = File::WRONLY | File::APPEND
+    class << self
 
-      ObjectSpace.each_object(File) do |fp|
-        begin
-          if !fp.closed? && fp.stat.file? && fp.sync && (fp.fcntl(Fcntl::F_GETFL) & append_flags) == append_flags
-            to_reopen << fp
-          end
-        rescue IOError, Errno::EBADF
-        end
+      def initialize_logger(log_target = STDOUT)
+        oldlogger = defined?(@logger) ? @logger : nil
+        @logger = Logger.new(log_target)
+        @logger.level = Logger::INFO
+        @logger.formatter = Pretty.new
+        oldlogger.close if oldlogger && !$TESTING # don't want to close testing's STDOUT logging
+        @logger
       end
 
-      nr = 0
-      to_reopen.each do |fp|
-        orig_st = begin
-          fp.stat
-        rescue IOError, Errno::EBADF
-          next
-        end
-
-        begin
-          b = File.stat(fp.path)
-          next if orig_st.ino == b.ino && orig_st.dev == b.dev
-        rescue Errno::ENOENT
-        end
+      def logger
+        defined?(@logger) ? @logger : initialize_logger
+      end
 
-        begin
-          File.open(fp.path, 'a') { |tmpfp| fp.reopen(tmpfp) }
-          fp.sync = true
-          nr += 1
-        rescue IOError, Errno::EBADF
-          # not much we can do...
-        end
+      def logger=(log)
+        @logger = (log ? log : Logger.new('/dev/null'))
       end
-      nr
-    rescue RuntimeError => ex
-      # RuntimeError: ObjectSpace is disabled; each_object will only work with Class, pass -X+O to enable
-      puts "Unable to reopen logs: #{ex.message}"
-    end
 
-    def logger
-      Taskinator::Logging.logger
     end
   end
 end

data/lib/taskinator/persistence.rb

@@ -94,13 +94,14 @@ module Taskinator
 
       # persists the error information
       def fail(error=nil)
-        return unless error
+        return unless error && error.is_a?(Exception)
+
         Taskinator.redis do |conn|
           conn.hmset(
             self.key,
             :error_type, error.class.name,
             :error_message, error.message,
-            :error_backtrace, JSON.generate(error.backtrace)
+            :error_backtrace, JSON.generate(error.backtrace || [])
           )
         end
       end
@@ -182,23 +183,8 @@ module Taskinator
 
       def visit_args(attribute)
         values = @instance.send(attribute)
-
-        # special case, convert models to global id's
-        if values.is_a?(Array)
-
-          values = values.collect {|value|
-            value.respond_to?(:global_id) ? value.global_id : value
-          }
-
-        elsif values.is_a?(Hash)
-
-          values.each {|key, value|
-            values[key] = value.global_id if value.respond_to?(:global_id)
-          }
-
-        end
-
-        @hmset += [attribute, YAML.dump(values)]
+        yaml = Taskinator::Persistence.serialize(values)
+        @hmset += [attribute, yaml]
       end
     end
 
@@ -289,30 +275,12 @@ module Taskinator
       def visit_args(attribute)
         yaml = @attribute_values[attribute]
         if yaml
-          values = YAML.load(yaml)
-
-          # special case for models, so find model
-          if values.is_a?(Array)
-
-            values = values.collect {|value|
-              # is it a global id?
-              value.respond_to?(:model_id) && value.respond_to?(:find) ? value.find : value
-            }
-
-          elsif values.is_a?(Hash)
-
-            values.each {|key, value|
-              # is it a global id?
-              values[key] = value.find if value.respond_to?(:model_id) && value.respond_to?(:find)
-            }
-
-          end
-
+          values = Taskinator::Persistence.deserialize(yaml)
           @instance.instance_variable_set("@#{attribute}", values)
         end
       end
 
-    private
+      private
 
       #
       # creates a proxy for the instance which
@@ -342,17 +310,57 @@ module Taskinator
       # E.g. this is useful for tasks which refer to their parent processes
       #
 
-      def initialize(type, uuid, instance_cache)
+      def initialize(type, uuid, instance_cache={})
         @type = type
         @uuid = uuid
         @instance_cache = instance_cache
       end
 
+      attr_reader :uuid   # shadows the real method, but will be the same!
+
+      # attempts to reload the actual process
+      def reload
+        @instance = nil
+        __getobj__
+        @instance ? true : false
+      end
+
       def __getobj__
         # only fetch the object as needed
         # and memoize for subsequent calls
         @instance ||= @type.fetch(@uuid, @instance_cache)
       end
     end
+
+    class << self
+      def serialize(values)
+        # special case, convert models to global id's
+        if values.is_a?(Array)
+          values = values.collect {|value|
+            value.respond_to?(:global_id) ? value.global_id : value
+          }
+        elsif values.is_a?(Hash)
+          values.each {|key, value|
+            values[key] = value.global_id if value.respond_to?(:global_id)
+          }
+        end
+        YAML.dump(values)
+      end
+
+      def deserialize(yaml)
+        values = YAML.load(yaml)
+        if values.is_a?(Array)
+          values = values.collect {|value|
+            (value.respond_to?(:model_id) && value.respond_to?(:find)) ? value.find : value
+          }
+        elsif values.is_a?(Hash)
+          values.each {|key, value|
+            values[key] = value.find if value.respond_to?(:model_id) && value.respond_to?(:find)
+          }
+        end
+        values
+      end
+    end
+
   end
 end

data/lib/taskinator/process.rb

@@ -20,6 +20,7 @@ module Taskinator
     attr_reader :uuid
     attr_reader :definition
     attr_reader :options
+    attr_reader :queue
 
     # in the case of sub process tasks, the containing task
     attr_accessor :parent
@@ -28,13 +29,14 @@ module Taskinator
       raise ArgumentError, 'definition' if definition.nil?
       raise ArgumentError, "#{definition.name} does not extend the #{Definition.name} module" unless definition.kind_of?(Definition)
 
-      @uuid = SecureRandom.uuid
+      @uuid = options.delete(:uuid) || SecureRandom.uuid
       @definition = definition
       @options = options
+      @queue = options.delete(:queue)
     end
 
     def tasks
-      @tasks ||= Tasks.new()
+      @tasks ||= Tasks.new
     end
 
     def accept(visitor)
@@ -43,6 +45,7 @@ module Taskinator
       visitor.visit_type(:definition)
       visitor.visit_tasks(tasks)
       visitor.visit_args(:options)
+      visitor.visit_attribute(:queue)
     end
 
     def <=>(other)
@@ -53,10 +56,6 @@ module Taskinator
       "#<#{self.class.name}:#{uuid}>"
     end
 
-    def queue
-      options[:queue]
-    end
-
     workflow do
       state :initial do
         event :enqueue, :transitions_to => :enqueued
@@ -201,5 +200,13 @@ module Taskinator
         visitor.visit_attribute(:complete_on)
       end
     end
+
+    # reloads the process from storage
+    # NB: only implemented by LazyLoader so that
+    #   the process can be lazy loaded, thereafter
+    #    it has no effect
+    def reload
+      false
+    end
   end
 end

data/lib/taskinator/process_worker.rb

@@ -1,5 +1,7 @@
 module Taskinator
   class ProcessWorker
+    attr_reader :uuid
+
     def initialize(uuid)
       @uuid = uuid
     end
@@ -9,7 +11,7 @@ module Taskinator
       return if process.paused? || process.cancelled?
       begin
         process.start!
-      rescue Exception => e
+      rescue => e
         Taskinator.logger.error(e)
         process.fail!(e)
         raise e

data/lib/taskinator/queues/delayed_job.rb

@@ -9,11 +9,12 @@ module Taskinator
 
     class DelayedJobAdapter
       def initialize(config={})
-        @config = {
-          :process_queue => :default,
-          :task_queue => :default,
-          :job_queue => :default,
-        }.merge(config)
+        @config = Taskinator::Queues::DefaultConfig.merge(config)
+      end
+
+      def enqueue_create_process(definition, uuid, args)
+        queue = definition.queue || @config[:definition_queue]
+        ::Delayed::Job.enqueue CreateProcessWorker.new(definition.name, uuid, Taskinator::Persistence.serialize(args)), :queue => queue
       end
 
       def enqueue_process(process)
@@ -32,6 +33,12 @@ module Taskinator
         ::Delayed::Job.enqueue JobWorker.new(job.uuid), :queue => queue
       end
 
+      CreateProcessWorker = Struct.new(:definition_name, :uuid, :args) do
+        def perform
+          Taskinator::CreateProcessWorker.new(definition_name, uuid, args).perform
+        end
+      end
+
       ProcessWorker = Struct.new(:process_uuid) do
         def perform
           Taskinator::ProcessWorker.new(process_uuid).perform

data/lib/taskinator/queues/resque.rb

@@ -9,11 +9,11 @@ module Taskinator
 
     class ResqueAdapter
       def initialize(config={})
-        config = {
-          :process_queue => :default,
-          :task_queue => :default,
-          :job_queue => :default
-        }.merge(config)
+        config = Taskinator::Queues::DefaultConfig.merge(config)
+
+        CreateProcessWorker.class_eval do
+          @queue = config[:definition_queue]
+        end
 
         ProcessWorker.class_eval do
           @queue = config[:process_queue]
@@ -28,6 +28,11 @@ module Taskinator
         end
       end
 
+      def enqueue_create_process(definition, uuid, args)
+        queue = definition.queue || Resque.queue_from_class(CreateProcessWorker)
+        Resque.enqueue_to(queue, CreateProcessWorker, definition.name, uuid, Taskinator::Persistence.serialize(args))
+      end
+
       def enqueue_process(process)
         queue = process.queue || Resque.queue_from_class(ProcessWorker)
         Resque.enqueue_to(queue, ProcessWorker, process.uuid)
@@ -46,6 +51,12 @@ module Taskinator
         Resque.enqueue_to(queue, JobWorker, job.uuid)
       end
 
+      class CreateProcessWorker
+        def self.perform(definition_name, uuid, args)
+          Taskinator::CreateProcessWorker.new(definition_name, uuid, args).perform
+        end
+      end
+
       class ProcessWorker
         def self.perform(process_uuid)
           Taskinator::ProcessWorker.new(process_uuid).perform