ultra_marathon 0.1.7 → 0.1.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c7d97a30d4a344932ef8d315457e5072ec08da58
-  data.tar.gz: 1aaf3b10556ce6344fe795438a1d176254b43a77
+  metadata.gz: c86ad7025f579980952497fa5613049fc05f119f
+  data.tar.gz: a2c6d98d6e09a74e071fdb253e76a6831ee7be48
 SHA512:
-  metadata.gz: cb05ac7278f5cb8d6537650dde46c3fb8377d8460afe832e243b87079162e0c20ec9065903257aaf3e34666378099ee6ebc8c871a29ec2b99e675bb3fa39fd11
-  data.tar.gz: 32a14722374518f955594603987ebde23ed1443ee8579164c7021b18e611b032a1f038f200e9a5be1ce068149d4b70613824d898d5f52a370ef4140c4275512f
+  metadata.gz: 98f72f85992e4b2af816575ff76bcec3adeb6e536409c95ecbc49841bd8fe915104f0ee058cb12ed1417eaf904738be1437a2852f94037a5dced3a9f0c45ba23
+  data.tar.gz: 517904e35cd8cec588e7ec2dde635f59c9f9a8dfcc43f54a970bef170ec1ca29ca4183e7b919e3f53ac5976ba2e8042a178446bc8a0aaf885c6109f8877bebd6

data/README.md

@@ -1,4 +1,5 @@
 # UltraMarathon
+[![Build Status](https://travis-ci.org/tyre/ultra_marathon.svg?branch=master)](https://travis-ci.org/tyre/ultra_marathon)
 
 Fault tolerant platform for long running jobs.
 
@@ -86,6 +87,101 @@ In this instance, `bubbles` will not be run until `don_scuba_gear` successfully
 finishes. If `don_scuba_gear` explicitly fails, such as by raising an error,
 `bubbles` will never be run.
 
+### Collections
+
+Sometimes you want to run a given run block once for each of a given set. Just
+pass the `:collection` option and all of your dreams will come true. Each
+iteration will be passed one item along with the index.
+
+```ruby
+class RangeRunner < UltraMarathon::AbstractRunner
+
+  run :counting!, collection: (1..100) do |number, index|
+    if index == 0
+      puts "We start with #{number}"
+    else
+      puts "And then comes #{number}"
+    end
+  end
+
+end
+```
+
+The only requirement is that the `:collection` option responds to #each. But
+what if it doesn't? Just pass in the `:iterator` option! This option was added
+specifically for Rails ActiveRecord::Association instances that can fetch in
+batches using `:find_each`
+
+```ruby
+# Crow inherits from ActiveRecord::Base
+
+class MurderRunner < UltraMarathon::AbstractRunner
+
+  run :coming_of_age, collection: :crows_to_bless, iterator: :find_each do |youngster_crow|
+    youngster_crow.update_attribute(blessed: true)
+  end
+  
+  def crows_to_bless
+    Crow.unblessed.where(age: 10)
+  end
+
+end
+
+```
+
+### Threading
+
+Passing `threaded: true` will run that run block in its own thread. This is particularly useful for collections or run blocks which contain external API calls, hit a database, or any other candidate for concurrency.
+
+```ruby
+class NapRunner < UltraMarathon::AbstractRunner
+  run :mass_nap, collection: (1..100), threaded: true do
+    sleep(0.01)
+  end
+end
+
+```
+
+#### Example
+
+As we will see in the example below, for longer-running processes that Ruby can run concurrently, threading is a muy bueno idea.
+
+
+```ruby
+require 'benchmark'
+require 'ultra_marathon'
+
+class ThreadedNapRunner < UltraMarathon::AbstractRunner
+  run_collection :mass_nap, items: (1..100), threaded: true do |n|
+    sleep(1)
+  end
+end
+
+class UnthreadedNapRunner < UltraMarathon::AbstractRunner
+  run_collection :mass_nap, items: (1..100), threaded: false do |n|
+    sleep(1)
+  end
+end
+
+Benchmark.bmbm do |reporter|
+  reporter.report(:threaded) { ThreadedNapRunner.new.run! }
+  reporter.report(:unthreaded) { UnthreadedNapRunner.new.run! }
+end
+
+# Rehearsal ----------------------------------------------
+# threaded     1.270000   0.080000   1.350000 (  1.346384)
+# unthreaded   0.060000   0.010000   0.070000 (100.141031)
+# ------------------------------------- total: 1.420000sec
+#
+#                  user     system      total        real
+# threaded     1.320000   0.060000   1.380000 (  1.377940)
+# unthreaded   0.060000   0.000000   0.060000 (100.128980)
+```
+
+Note, however, that threading is not free. In the final benchmark, we would expect a runtime of ~1 second but saw over a third of a second slower. If we run the same test with a run block of `(n * 10_000).times { 'derp' }`, for example, the unthreaded version is about 10% faster.
+
+tl;dr don't thread because it sounds cool. Use it when you need it.
+
 ### Callbacks
 
 `UltraMarathon::AbstractRunner` includes numerous life-cycle callbacks for
@@ -183,6 +279,6 @@ class WatRunner < UltraMarathon::AbstractRunner
 end
 
 WatRunner.run!.reset.run!
-#=> boom
+#=> wrong
 #=> all is well in the universe
 ```

data/lib/core_ext/class.rb

@@ -0,0 +1,13 @@
+class Class
+  def attr_memo_reader(name, memoization_block)
+    define_method(name) do
+      instance_variable_get(:"@#{name}") ||
+      instance_variable_set(:"@#{name}", instance_exec(&memoization_block))
+    end
+  end
+
+  def attr_memo_accessor(name, memoization_block)
+    attr_memo_reader(name, memoization_block)
+    attr_writer name
+  end
+end

data/lib/core_ext/extensions.rb

@@ -0,0 +1,4 @@
+require 'core_ext/class'
+require 'core_ext/object'
+require 'core_ext/proc'
+require 'core_ext/string'

data/lib/core_ext/object.rb

@@ -0,0 +1,12 @@
+class Object
+  # Many times an option can either be a callable object (Proc/Lambda) or
+  # not (symbol/string/integer). This will call with the included arguments,
+  # if it is callable, or return the object if not.
+  def try_call(*args)
+    if respond_to? :call
+      call(*args)
+    else
+      self
+    end
+  end
+end

data/lib/core_ext/string.rb

@@ -0,0 +1,9 @@
+class String
+
+  # Included for the kids.
+  def sploin(seperator, joiner, &block)
+    ary = split(seperator)
+    ary.map!(&block) if block_given?
+    ary.join(joiner)
+  end
+end

data/lib/ultra_marathon.rb

@@ -1,10 +1,11 @@
-require 'core_ext/proc'
+require 'core_ext/extensions'
 require 'ultra_marathon/abstract_runner'
 require 'ultra_marathon/callbacks'
 require 'ultra_marathon/instrumentation'
 require 'ultra_marathon/logging'
 require 'ultra_marathon/sub_runner'
 require 'ultra_marathon/store'
+require 'ultra_marathon/collection_runner'
 
 module UltraMarathon
 

data/lib/ultra_marathon/abstract_runner.rb

@@ -1,80 +1,39 @@
-require 'ultra_marathon/callbacks'
-require 'ultra_marathon/instrumentation'
-require 'ultra_marathon/logging'
+require 'ultra_marathon/base_runner'
 require 'ultra_marathon/sub_runner'
 require 'ultra_marathon/store'
 
 module UltraMarathon
-  class AbstractRunner
-    include Logging
-    include Instrumentation
-    include Callbacks
-    attr_accessor :success
-    callbacks :before_run, :after_run, :on_error, :on_reset
-
+  class AbstractRunner < BaseRunner
     after_run :write_log
     on_error lambda { self.success = false }
     on_error lambda { |error| logger.error(error) }
 
-    ## Public Instance Methods
-
-    # Runs the run block safely in the context of the instance
-    def run!
-      if self.class.run_blocks.any?
-        begin
-          self.success = nil
-          invoke_before_run_callbacks
-          instrument(:run_unrun_sub_runners) { run_unrun_sub_runners }
-          # If any of the sub runners explicitly set the success flag, don't override it
-          self.success = failed_sub_runners.empty? if self.success.nil?
-        rescue StandardError => error
-          invoke_on_error_callbacks(error)
-        ensure
-          invoke_after_run_callbacks
-        end
-        self
-      end
-    end
-
-    def success?
-      !!success
-    end
-
-    # Resets success to being true, unsets the failed sub_runners to [], and
-    # sets the unrun sub_runners to be the uncompleted/failed ones
-    def reset
-      reset_failed_runners
-      @success = nil
-      invoke_on_reset_callbacks
-      self
-    end
-
     private
 
     ## Private Class Methods
 
     class << self
+      attr_memo_reader :run_blocks, -> { Hash.new }
 
       # This is where the magic happens.
       # Called in the class context, it will be safely executed in
       # the context of the instance.
       #
-      # E.g.
-      #
-      # class BubblesRunner < AbstractRunner
-      #   run do
-      #     fire_the_missiles
-      #     take_a_nap
-      #   end
+      # @example
+      #  class BubblesRunner < AbstractRunner
+      #    run do
+      #      fire_the_missiles
+      #      take_a_nap
+      #    end
       #
-      #   def fire_the_missiles
-      #     puts 'But I am le tired'
-      #   end
+      #    def fire_the_missiles
+      #      puts 'But I am le tired'
+      #    end
       #
-      #   def take_a_nap
-      #     puts 'zzzzzz'
-      #   end
-      # end
+      #    def take_a_nap
+      #      puts 'zzzzzz'
+      #    end
+      #  end
       #
       #  BubblesRunner.new.run!
       #  # => 'But I am le tired'
@@ -89,8 +48,9 @@ module UltraMarathon
         end
       end
 
-      def run_blocks
-        @run_blocks ||= Hash.new
+      def run_collection(name=:main, items=[], options={}, &block)
+        options.merge!(collection: true, items: items)
+        run(name, options, &block)
       end
     end
 
@@ -109,97 +69,22 @@ module UltraMarathon
 
     # Creates a new sub runner, defaulting the context to `self`
     def new_sub_runner(options, block)
-      defaults = {
-        context: self
-      }
-      options = defaults.merge(options)
-      SubRunner.new(options, block)
-    end
-
-    # Stores sub runners which ran and were a success
-    def successful_sub_runners
-      @successful_sub_runners ||= Store.new
-    end
-
-    # Stores sub runners which ran and failed
-    # Also store children of those which failed
-    def failed_sub_runners
-      @failed_sub_runners ||= Store.new
-    end
-
-    # If all of the parents have been successfully run (or there are no
-    # parents), runs the sub_runner.
-    # If any one of the parents has failed, considers the runner a failure
-    # If some parents have not yet completed, carries on
-    def run_unrun_sub_runners
-      unrun_sub_runners.each do |sub_runner|
-        if sub_runner_can_run? sub_runner
-          run_sub_runner(sub_runner)
-        elsif sub_runner.parents.any? { |name| failed_sub_runners.exists? name }
-          failed_sub_runners << sub_runner
-          unrun_sub_runners.delete sub_runner.name
-        end
-      end
-      run_unrun_sub_runners unless complete?
-    end
-
-    # Runs the sub runner, adding it to the appropriate sub runner store based
-    # on its success or failure and removes it from the unrun_sub_runners
-    def run_sub_runner(sub_runner)
-      sub_runner.run!
-      logger.info sub_runner.logger.contents
-      if sub_runner.success
-        successful_sub_runners << sub_runner
+      options = {
+        context: self,
+        collection: false,
+        items: []
+      }.merge(options)
+      if options[:collection]
+        CollectionRunner.new(options.delete(:items), options, &block)
       else
-        failed_sub_runners << sub_runner
+        SubRunner.new(options, block)
       end
-      unrun_sub_runners.delete sub_runner.name
-    end
-
-    ## TODO: timeout option
-    def complete?
-      unrun_sub_runners.empty?
-    end
-
-    # A sub runner can run if all prerequisites have been satisfied.
-    # This means all parent runners - those specified by name using the
-    # :requires options - have successfully completed.
-    def sub_runner_can_run?(sub_runner)
-      successful_sub_runners.includes_all?(sub_runner.parents)
-    end
-
-    # Resets all failed sub runners, then sets them as
-    # @unrun_sub_runners and @failed_sub_runners to an empty Store
-    def reset_failed_runners
-      failed_sub_runners.each(&:reset)
-      @unrun_sub_runners = failed_sub_runners
-      @failed_sub_runners = Store.new
     end
 
     def write_log
-      logger.info summary
-    end
-
-    def summary
-      run_instrumentation = instrumentations[:run_unrun_sub_runners]
-      """
-
-      Status: #{status}
-      Run Start Time: #{run_instrumentation.formatted_start_time}
-      End Time: #{run_instrumentation.formatted_end_time}
-      Total Time: #{run_instrumentation.formatted_total_time}
-
-      Successful SubRunners: #{successful_sub_runners.size}
-      Failed SubRunners: #{failed_sub_runners.size}
-      """
+      log_all_sub_runners
+      log_summary
     end
 
-    def status
-      if success
-        'Success'
-      else
-        'Failure'
-      end
-    end
   end
 end

data/lib/ultra_marathon/base_runner.rb

@@ -0,0 +1,216 @@
+require 'ultra_marathon/callbacks'
+require 'ultra_marathon/instrumentation'
+require 'ultra_marathon/logging'
+
+module UltraMarathon
+  class BaseRunner
+    RUN_INSTRUMENTATION_NAME = '__run!'.freeze
+    include Logging
+    include Instrumentation
+    include Callbacks
+
+    attr_accessor :success
+    attr_memo_accessor :running_sub_runners, -> { Store.new }
+    attr_memo_reader :successful_sub_runners, -> { Store.new }
+    attr_memo_reader :failed_sub_runners, -> { Store.new }
+
+    callbacks :before_run, :after_run, :on_error, :on_reset, :after_initialize
+
+    ## Public Class Methods
+
+    def self.new(*args, &block)
+      super(*args, &block).tap do |instance|
+        instance.send(:invoke_after_initialize_callbacks)
+      end
+    end
+
+    ## Public Instance Methods
+
+    # Runs the run block safely in the context of the instance
+    def run!
+      if unrun_sub_runners.any?
+        instrument RUN_INSTRUMENTATION_NAME do
+          begin
+            self.success = nil
+            invoke_before_run_callbacks
+            instrument(:__run_unrun_sub_runners) { run_unrun_sub_runners }
+            # If any of the sub runners explicitly set the success flag, don't override it
+            self.success = failed_sub_runners.empty? if self.success.nil?
+          rescue StandardError => error
+            invoke_on_error_callbacks(error)
+          end
+        end
+        invoke_after_run_callbacks
+      end
+      self
+    end
+
+    def success?
+      !!success
+    end
+
+    # Resets success to being true, unsets the failed sub_runners to [], and
+    # sets the unrun sub_runners to be the uncompleted/failed ones
+    def reset
+      reset_failed_runners
+      @success = nil
+      invoke_on_reset_callbacks
+      self
+    end
+
+    def run_instrumentation
+      instrumentations[RUN_INSTRUMENTATION_NAME]
+    end
+
+    private
+
+    ## Private Instance Methods
+
+    # If all of the parents have been successfully run (or there are no
+    # parents), runs the sub_runner.
+    # If any one of the parents has failed, considers the runner a failure
+    # If some parents have not yet completed, carries on
+    def run_unrun_sub_runners
+      until complete?
+        unrun_sub_runners.each do |sub_runner|
+          if sub_runner_can_run? sub_runner
+            running_sub_runners << sub_runner.run!
+          elsif sub_runner.parents.any? { |name| failed_sub_runners.exists? name }
+            failed_sub_runners << sub_runner
+            unrun_sub_runners.delete sub_runner.name
+          end
+        end
+        clean_up_completed_sub_runners
+      end
+    end
+
+    # Cleans up all dead threads, settings
+    def clean_up_completed_sub_runners
+      completed_runners, unfinished_runners = running_sub_runners.partition(&:complete?)
+      completed_runners.each do |sub_runner|
+        clean_up_sub_runner(sub_runner)
+      end
+      self.running_sub_runners = unfinished_runners
+    end
+
+    # Adds a run sub runner to the appropriate sub runner store based
+    # on its success or failure and removes it from the unrun_sub_runners
+    # Also merges its instrumentation to the group's instrumentation
+    def clean_up_sub_runner(sub_runner)
+      if sub_runner.success
+        successful_sub_runners << sub_runner
+      else
+        failed_sub_runners << sub_runner
+      end
+      instrumentations.merge!(sub_runner.instrumentations)
+      unrun_sub_runners.delete sub_runner.name
+    end
+
+    ## TODO: timeout option
+    def complete?
+      running_sub_runners.empty? && unrun_sub_runners.empty?
+    end
+
+    # Resets all failed sub runners, then sets them as
+    # unrun_sub_runners and failed_sub_runners to an empty Store
+    def reset_failed_runners
+      failed_sub_runners.each(&:reset)
+      @unrun_sub_runners = failed_sub_runners
+      @failed_sub_runners = Store.new
+    end
+
+    # A sub runner can run if all prerequisites have been satisfied.
+    # This means all parent runners - those specified by name using the
+    # :requires options - have successfully completed.
+    def sub_runner_can_run?(sub_runner)
+      successful_sub_runners.includes_all?(sub_runner.parents)
+    end
+
+    def status
+      if success?
+        'Success'
+      else
+        'Failure'
+      end
+    end
+
+    def log_all_sub_runners
+      log_failed_sub_runners if failed_sub_runners.any?
+      log_successful_sub_runners if successful_sub_runners.any?
+    end
+
+    def log_failed_sub_runners
+      logger.info """
+
+      == Failed SubRunners ==
+
+      """
+      log_sub_runners(failed_sub_runners)
+    end
+
+    def log_successful_sub_runners
+      logger.info """
+
+      == Successful SubRunners ==
+
+      """
+      log_sub_runners(successful_sub_runners)
+    end
+
+
+    def log_sub_runners(sub_runners)
+      sub_runners.each do |sub_runner|
+        logger.info(sub_runner.logger.contents << "\n")
+      end
+    end
+
+    def log_summary
+      run_profile = instrumentations[:run!]
+      failed_names = failed_sub_runners.names.map(&:to_s).join(', ')
+      succcessful_names = successful_sub_runners.names.map(&:to_s).join(', ')
+      unrun_names = unrun_sub_runners.names.map(&:to_s).join(', ')
+      logger.info """
+
+      Status: #{status}
+
+      Failed (#{failed_sub_runners.size}): #{failed_names}
+      Successful (#{successful_sub_runners.size}): #{succcessful_names}
+      Unrun (#{unrun_sub_runners.size}): #{unrun_names}
+
+      #{time_summary}
+
+      """
+    end
+
+    def sub_runner_instrumentations
+      @sub_runner_instrumentations ||= begin
+        sub_runner_profiles = instrumentations.select do |profile|
+          profile.name.to_s.start_with? 'sub_runner.'
+        end
+        UltraMarathon::Instrumentation::Store.new(sub_runner_profiles)
+      end
+    end
+
+    def time_summary
+      """
+      Run Start Time: #{run_instrumentation.formatted_start_time}
+      End Time: #{run_instrumentation.formatted_end_time}
+      Total Time: #{run_instrumentation.formatted_total_time}
+
+      #{sub_runner_summary if sub_runner_instrumentations.any?}
+      """
+    end
+
+    def sub_runner_summary
+      median_profile = sub_runner_instrumentations.median
+      max_profile = sub_runner_instrumentations.max
+      min_profile = sub_runner_instrumentations.min
+      """
+      Max SubRunner Runtime: #{max_profile.name} (#{max_profile.total_time})
+      Min SubRunner Runtime: #{min_profile.name} (#{min_profile.total_time})
+      Median SubRunner Runtime: #{median_profile.name} (#{median_profile.total_time})
+      SubRunner Runtime Standard Deviation: #{sub_runner_instrumentations.standard_deviation}
+      """
+    end
+  end
+end