serf 0.6.1 → 0.7.0

data/lib/serf/runners/direct.rb

@@ -0,0 +1,52 @@
+require 'serf/runners/helper'
+require 'serf/util/error_handling'
+
+module Serf
+module Runners
+
+  ##
+  # Direct runner drives the execution of a handler for given messages.
+  # This class deals with error handling and pushing handler results
+  # to proper error or results channels.
+  #
+  # NOTES:
+  # * Results returned from handlers are pushed to response channel.
+  # * Errors raised by handlers are pushed to error channel, not response.
+  #
+  class Direct
+    include Serf::Util::ErrorHandling
+    include Serf::Runners::Helper
+
+    def initialize(*args)
+      extract_options! args
+      opts! :response_channel
+    end
+
+    def call(handlers, context)
+      results = []
+      handlers.each do |handler|
+        ok, run_result = with_error_handling(context) do
+          handler.call
+        end
+        run_result = run_result.is_a?(Hash) ? [run_result] : Array(run_result)
+
+        # We only post to the response channel if we didn't catch and error.
+        # But we add both error and success results to the 'results' to
+        # pass back to the caller of the runner. This may be a background
+        # runner, which then the results are ignored. But it may have been
+        # the Serfer, which means all results should go back to the user
+        # as this was a foreground (synchronous) execution.
+        results.concat run_result
+        push_results run_result if ok
+      end
+      return results
+    end
+
+    def self.build(options={})
+      self.new options
+    end
+
+  end
+
+end
+end

data/lib/serf/runners/event_machine.rb

@@ -0,0 +1,71 @@
+require 'eventmachine'
+
+require 'serf/messages/message_accepted_event'
+require 'serf/runners/direct'
+require 'serf/util/error_handling'
+
+module Serf
+module Runners
+
+  ##
+  # This runner simply wraps another runner to execute in the
+  # EventMachine deferred threadpool.
+  #
+  # NOTE: Because the Serfer class validates messages before
+  #  sending them to runners (and handlers), this class simply
+  #  responds to the calling client with an 'MessageAcceptedEvent'
+  #  to signal that the message will be processed later.
+  #
+  # Errors caught here will simply be logged. This is because
+  # the wrapped runner *MUST* handle its own errors. If an error
+  # should propagate up here, then it was most likely an error
+  # that occurred in a rescue block... we don't want to complicate
+  # any extra pushing to error channels because that may have
+  # been the cause of the error.
+  #
+  class EventMachine
+    include Serf::Util::ErrorHandling
+
+    def initialize(*args)
+      extract_options! args
+
+      # Manditory: Need a runner because this class is just a wrapper.
+      @runner = opts! :runner
+
+      @mae_class = opts(
+        :message_accepted_event_class,
+        ::Serf::Messages::MessageAcceptedEvent)
+
+      @evm = opts :event_machine, ::EventMachine
+      @logger = opts :logger, ::Serf::Util::NullObject.new
+    end
+
+    def call(handlers, context)
+      # This queues up each handler to be run separately.
+      handlers.each do |handler|
+        @evm.defer(proc do
+          begin
+            with_error_handling(context) do
+              @runner.call [handler], context
+            end
+          rescue => e
+            @logger.fatal(
+              "EventMachineThread: #{e.inspect}\n\n#{e.backtrace.join("\n")}")
+          end
+        end)
+      end
+      return @mae_class.new(message: context)
+    end
+
+    def self.build(options={})
+      options[:runner] = options.fetch(:runner) {
+        factory = options[:runner_factory] || ::Serf::Runners::Direct
+        factory.build options
+      }
+      self.new options
+    end
+
+  end
+
+end
+end

data/lib/serf/runners/girl_friday.rb

@@ -0,0 +1,73 @@
+require 'girl_friday'
+
+require 'serf/messages/message_accepted_event'
+require 'serf/runners/helper'
+require 'serf/util/error_handling'
+
+module Serf
+module Runners
+
+  ##
+  #
+  class GirlFriday
+    include Serf::Util::ErrorHandling
+    include Serf::Runners::Helper
+
+    def initialize(*args)
+      extract_options! args
+
+      # Mandatory response channel.
+      opts! :response_channel
+
+      # Create our worker queue that will accept tasks (handler and context).
+      # The worker is just a block that passes on the task to the
+      # actual worker method.
+      @queue = ::GirlFriday::WorkQueue.new(
+          opts(:queue_name, :serf_runner),
+          :size => opts(:queue_size, 1)) do |task|
+        perform task
+      end
+    end
+
+    def call(handlers, context)
+      # Create our accepted event before we enqueue the handlers.
+      mae_class = opts(
+        :message_accepted_event_class,
+        ::Serf::Messages::MessageAcceptedEvent)
+      event = mae_class.new message: context
+
+      # Push each handler into the queue along with a copy of the context.
+      handlers.each do |handler|
+        @queue.push(
+          handler: handler,
+          context: context.dup)
+      end
+
+      # We got here, we succeeded pushing all the works.
+      # Now we return our accepted event.
+      return event
+    end
+
+    ##
+    # Builder method
+    #
+    def self.build(*args)
+      self.new *args
+    end
+
+    ##
+    # Actually drives the execution of individual handlers passed to job
+    # queue.
+    #
+    def perform(task)
+      with_error_handling(task[:context]) do
+        task[:handler].call
+        run_result = run_result.is_a?(Hash) ? [run_result] : Array(run_result)
+        push_results run_result
+      end
+    end
+
+  end
+
+end
+end

data/lib/serf/runners/helper.rb

@@ -0,0 +1,23 @@
+module Serf
+module Runners
+
+  module Helper
+
+    ##
+    # Loop over the results and push them to the response channel.
+    # Any error in pushing individual messages will result in
+    # a log event and an error channel event.
+    def push_results(results)
+      response_channel = opts! :response_channel
+      results.each do |message|
+        with_error_handling(message) do
+          response_channel.push message
+        end
+      end
+      return nil
+    end
+
+  end
+
+end
+end

data/lib/serf/serfer.rb

@@ -1,60 +1,147 @@
+require 'active_support/core_ext/hash/keys'
+
 require 'serf/error'
-require 'serf/util/with_error_handling'
+require 'serf/util/error_handling'
 
 module Serf
 
   class Serfer
-    include ::Serf::Util::WithErrorHandling
+    include ::Serf::Util::ErrorHandling
+
+    def initialize(*args)
+      extract_options! args
 
-    def initialize(options={})
-      # Each route_set has a runner.
-      @route_sets = options[:route_sets] || {}
+      # Map of Runners to Registries
+      @registries = opts :registries, {}
 
-      # Optional overrides for WithErrorHandling
-      @error_channel = options[:error_channel]
-      @error_event_class = options[:error_event_class]
-      @logger = options[:logger]
+      # Additional serf infrastructure options to pass to Endpoints#build.
+      @serf_options = opts :serf_options, {}
 
       # Options for handling the requests
-      @not_found = options[:not_found] || proc do
+      @not_found = opts :not_found, lambda { |env|
         raise ArgumentError, 'Handler Not Found'
-      end
+      }
     end
 
     ##
-    # Rack-like call to run set of handlers for a message
+    # Rack-like call to run a set of handlers for a message
     #
     def call(env)
       # We normalize by symbolizing the env keys
       env = env.symbolize_keys
 
-      # We're going to concat all the results
-      matched_routes = false
-      results = []
-      @route_sets.each do |route_set, runner|
-        with_error_handling(env) do
-          endpoints = route_set.match_routes env
-          if endpoints.size > 0
-            matched_routes = true
-            results.concat Array(runner.run(endpoints, env))
-          end
-        end
-      end
+      # Call the processor
+      matched, results = process_request env
 
       # If we don't have any handlers, do the not_found call.
       # NOTE: Purposefully not wrapping this in exception handling.
-      return @not_found.call env unless matched_routes
+      return @not_found.call env unless matched > 0
 
       return results
     rescue => e
       e.extend(::Serf::Error)
       raise e
     end
+    alias_method :push, :call
+    alias_method :<<, :call
 
     def self.build(options={})
       self.new options
     end
 
+    protected
+
+    ##
+    # Do the work of processing the env.
+    # 1. Match our endpoints to run, keep associated with their runner.
+    # 2. Turn endpoints into actual handlers that can be called by runners.
+    # 3. Call runner to process the endpoints
+    # 4. Return results
+    #
+    # NOTES:
+    # * Any error in matching will be raised to the caller, not absorbed by
+    #   the error handler.
+    # * Any error in Handler creation from endpoint will be caught by the
+    #   error handler, (1) pushed to the error channel and (2)
+    #   an error event will included in the results pass back to caller.
+    #   (a) There may be successul handlers created that can complete.
+    # * If the runner raises an error, it will be caught and the error
+    #   event will be appended to the results. This is so one
+    #   runner failure will not affect another runner's run.
+    #   Each runner SHOULD do their own error handling so errors in
+    #   one handler will not affect another in the list of handlers the runner
+    #   is to process.
+    # * RUNNERS MUST push errors they catch to the error channel.
+    #
+    def process_request(env)
+      # This will be the work we need to do.
+      # This is a hash of runners to handlers to run.
+      tasks = {}
+
+      # We're going to concat all the results
+      results = []
+
+      # Figure out which endpoints to run.
+      matches = match_endpoints env
+
+      # Now we go head and create our Tasks (Units of Work)
+      # for each of the matched endpoints (with their runners).
+      matches.each do |runner, endpoints|
+        # We create the unit of work
+        handlers = endpoints.map{ |endpoint|
+          # We try to build the endpoint. Any errors here will
+          # be caught and returned to the caller.
+          # This is so individual building of tasks do not affect other tasks.
+          ok, obj = with_error_handling(env) do
+            endpoint.build env.dup, @serf_options
+          end
+          # The return of this if/else statement will be result of map item.
+          if ok
+            obj
+          else
+            results << obj
+            nil
+          end
+        }.
+        select{ |h| !h.nil? }
+
+        # No we enqueue the units of work into our task queue
+        # List could be empty because all build calls could have error out.
+        tasks[runner] = handlers if handlers.size > 0
+      end
+
+      # We call the runners with the handlers they need to execute.
+      # Errors raised by the runner are pushed to the error channel.
+      # Errors here are also passed back the caller of the SerfApp.
+      #
+      tasks.each do |runner, handlers|
+        ok, run_result = with_error_handling(env) do
+          runner.call handlers, env
+        end
+        # We need to coerce the runner's results (nil, Hash, Array, Object)
+        # into an Array of messages.
+        # now we concat this run's results to our total results list.
+        run_result = run_result.is_a?(Hash) ? [run_result] : Array(run_result)
+        results.concat run_result
+      end
+
+      return matches.size, results
+    end
+
+    ##
+    # Figure out which endpoints to run
+    #
+    def match_endpoints(env)
+      matches = {}
+
+      @registries.each do |runner, registry|
+        endpoints = registry.match env
+        matches[runner] = endpoints if endpoints.size > 0
+      end
+
+      return matches
+    end
+
   end
 
 end

data/lib/serf/util/{with_error_handling.rb → error_handling.rb}

@@ -1,14 +1,18 @@
+require 'active_support/core_ext/string/inflections'
+
 require 'serf/messages/caught_exception_event'
 require 'serf/util/null_object'
+require 'serf/util/options_extraction'
 
 module Serf
 module Util
 
   ##
   # Helper module to rescues exceptions from executing blocks of
-  # code, and then logs+publishes the error event.
+  # code, and then logs+pushes the error event.
   #
-  module WithErrorHandling
+  module ErrorHandling
+    include ::Serf::Util::OptionsExtraction
 
     ##
     # A block wrapper to handle errors when executing a block.
@@ -20,22 +24,31 @@ module Util
     # * @error_channel - ::Serf::Util::NullObject.new
     #
     def with_error_handling(context=nil)
-      yield
+      return true, yield
     rescue => e
-      eec = @error_event_class || ::Serf::Messages::CaughtExceptionEvent
-      logger = @logger || ::Serf::Util::NullObject.new
-      error_channel = @error_channel || ::Serf::Util::NullObject.new
+      eec = opts :error_event_class, ::Serf::Messages::CaughtExceptionEvent
+      logger = opts :logger, ::Serf::Util::NullObject.new
+      error_channel = opts :error_channel, ::Serf::Util::NullObject.new
       error_event = eec.new(
         context: context,
-        error_message: e.inspect,
-        error_backtrace: e.backtrace.join("\n"))
+        error: e.class.to_s.tableize,
+        message: e.message,
+        backtrace: e.backtrace.join("\n"))
 
       # log the error to our logger, and to our error channel.
       logger.error error_event
-      error_channel.publish error_event
+      begin
+        error_channel.push error_event
+      rescue => e1
+        logger.error("
+          Failed pushing to ErrorChannel:
+          #{e1.message}
+          #{e1.backtrace.join('\n')}
+        ")
+      end
 
       # We're done, so just return this error.
-      return error_event
+      return false, error_event
     end
 
   end

data/lib/serf/util/options_extraction.rb

@@ -0,0 +1,106 @@
+module Serf
+module Util
+
+  ##
+  # Module that provides helpers for dealing with options hash passed
+  # to initializers.
+  #
+  #   class Example
+  #     include Serf::Util::OptionsExtraction
+  #
+  #     def initialize(*args, &block)
+  #       extract_options! args
+  #       puts args # Rest of args w/o the options hash.
+  #     end
+  #
+  #     def do_work
+  #       my_option = opts :my_option, 'Default Value'
+  #       puts my_option
+  #     end
+  #   end
+  #
+  #   example = Example.new my_option: 'Another Value'
+  #   example.do_work
+  #   # => 'Another Value'
+  #
+  module OptionsExtraction
+
+    ##
+    # Reader method for the options hash.
+    #
+    def options
+      return @options || {}
+    end
+
+    ##
+    # Helper method to lookup an option from our options hash.
+    #
+    # Examples:
+    #
+    #   # Optional parameter whose default value is nil.
+    #   do_extra = opts :do_extra
+    #
+    #   # Optional params that defaults to [1,2,3]
+    #   start_array = opts :start_array, [1,2,3]
+    #
+    # Returns default value when:
+    # * Key is non-existent in Options Hash.
+    # * OR when the value of the key in the Options Hash is nil.
+    #
+    # Returns nil when:
+    # * Default is nil and the Options Hash lookup returns a nil.
+    #   Options Hash returns a nil because of either a
+    #   non-existent key or a nil value in said hash for said key.
+    #
+    def opts(key, default=nil, &block)
+      value = options[key]
+      value = default if value.nil?
+      value = block.call if value.nil? && block
+      return value
+    end
+
+    ##
+    # Use this lookup helper if a nil value is unacceptable for processing.
+    #
+    # There are cases where an option may have a default value for a feature,
+    # but the caller may just want to disable said feature. To do so,
+    # users of this module should allow for the caller to pass in 'false'
+    # as an option value instead of nil to disable said feature. The
+    # implementer will have to do a boolean check of the returned option value
+    # and disable accordingly.
+    #
+    # Examples:
+    #
+    #   # Has a default logger, but we can disable logging by passing false.
+    #   # If caller had passed in nil for :logger, it would have
+    #   # defaulted to ::Log4r['my_logger'].
+    #   logger = opts! :logger, ::Log4r['my_logger']
+    #   logger = Serf::NullObject.new unless logger
+    #
+    #   # Here we force the end user to pass in a Non-nil option as a
+    #   # mandatory parameter.
+    #   max_threads = opts! :max_threads
+    #
+    # Raises error when `opts` returns a nil.
+    #
+    def opts!(key, default=nil, &block)
+      value = opts key, default, &block
+      raise "Nil value found for option: #{key}, #{default}" if value.nil?
+      return value
+    end
+
+    protected
+
+    ##
+    # Extracts the options from the arguments list.
+    #
+    def extract_options!(args)
+      _, @options = args.last.is_a?(::Hash) ?
+        [true, args.pop.dup] :
+        [false, {}]
+    end
+
+  end
+
+end
+end

data/lib/serf/version.rb

@@ -2,8 +2,8 @@ module Serf
 
   module Version
     MAJOR = 0
-    MINOR = 6
-    PATCH = 1
+    MINOR = 7
+    PATCH = 0
     BUILD = nil
     STRING = [MAJOR, MINOR, PATCH, BUILD].compact.join '.'
   end

data/serf.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "serf"
-  s.version = "0.6.1"
+  s.version = "0.7.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Benjamin Yu"]
-  s.date = "2012-02-09"
+  s.date = "2012-03-13"
   s.description = "Event-Driven SOA with CQRS"
   s.email = "benjaminlyu@gmail.com"
   s.extra_rdoc_files = [
@@ -25,22 +25,27 @@ Gem::Specification.new do |s|
     "NOTICE.txt",
     "README.md",
     "Rakefile",
+    "docs/thread_pools.txt",
     "lib/serf.rb",
     "lib/serf/builder.rb",
+    "lib/serf/command.rb",
     "lib/serf/error.rb",
     "lib/serf/message.rb",
     "lib/serf/messages/caught_exception_event.rb",
     "lib/serf/messages/message_accepted_event.rb",
     "lib/serf/middleware/uuid_tagger.rb",
-    "lib/serf/runners/direct_runner.rb",
-    "lib/serf/runners/em_runner.rb",
+    "lib/serf/routing/endpoint.rb",
+    "lib/serf/routing/registry.rb",
+    "lib/serf/runners/direct.rb",
+    "lib/serf/runners/event_machine.rb",
+    "lib/serf/runners/girl_friday.rb",
+    "lib/serf/runners/helper.rb",
     "lib/serf/serfer.rb",
+    "lib/serf/util/error_handling.rb",
     "lib/serf/util/null_object.rb",
+    "lib/serf/util/options_extraction.rb",
     "lib/serf/util/regexp_matcher.rb",
-    "lib/serf/util/route_endpoint.rb",
-    "lib/serf/util/route_set.rb",
     "lib/serf/util/uuidable.rb",
-    "lib/serf/util/with_error_handling.rb",
     "lib/serf/version.rb",
     "serf.gemspec",
     "spec/serf_spec.rb",
@@ -49,7 +54,7 @@ Gem::Specification.new do |s|
   s.homepage = "http://github.com/byu/serf"
   s.licenses = ["Apache 2.0"]
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.15"
+  s.rubygems_version = "1.8.17"
   s.summary = "Event-Driven SOA with CQRS"
 
   if s.respond_to? :specification_version then
@@ -59,36 +64,42 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<activesupport>, [">= 3.2.0"])
       s.add_runtime_dependency(%q<i18n>, [">= 0.6.0"])
       s.add_runtime_dependency(%q<uuidtools>, [">= 2.1.2"])
-      s.add_development_dependency(%q<rspec>, ["~> 2.3.0"])
-      s.add_development_dependency(%q<yard>, ["~> 0.6.0"])
-      s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
-      s.add_development_dependency(%q<jeweler>, ["~> 1.6.4"])
+      s.add_development_dependency(%q<rspec>, ["~> 2.8.0"])
+      s.add_development_dependency(%q<yard>, ["~> 0.7.5"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.0.22"])
+      s.add_development_dependency(%q<jeweler>, ["~> 1.8.3"])
       s.add_development_dependency(%q<simplecov>, [">= 0"])
+      s.add_development_dependency(%q<log4r>, ["~> 1.1.10"])
       s.add_development_dependency(%q<msgpack>, [">= 0.4.6"])
       s.add_development_dependency(%q<eventmachine>, [">= 0.12.10"])
+      s.add_development_dependency(%q<girl_friday>, ["~> 0.9.7"])
     else
       s.add_dependency(%q<activesupport>, [">= 3.2.0"])
       s.add_dependency(%q<i18n>, [">= 0.6.0"])
       s.add_dependency(%q<uuidtools>, [">= 2.1.2"])
-      s.add_dependency(%q<rspec>, ["~> 2.3.0"])
-      s.add_dependency(%q<yard>, ["~> 0.6.0"])
-      s.add_dependency(%q<bundler>, ["~> 1.0.0"])
-      s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
+      s.add_dependency(%q<rspec>, ["~> 2.8.0"])
+      s.add_dependency(%q<yard>, ["~> 0.7.5"])
+      s.add_dependency(%q<bundler>, ["~> 1.0.22"])
+      s.add_dependency(%q<jeweler>, ["~> 1.8.3"])
       s.add_dependency(%q<simplecov>, [">= 0"])
+      s.add_dependency(%q<log4r>, ["~> 1.1.10"])
       s.add_dependency(%q<msgpack>, [">= 0.4.6"])
       s.add_dependency(%q<eventmachine>, [">= 0.12.10"])
+      s.add_dependency(%q<girl_friday>, ["~> 0.9.7"])
     end
   else
     s.add_dependency(%q<activesupport>, [">= 3.2.0"])
     s.add_dependency(%q<i18n>, [">= 0.6.0"])
     s.add_dependency(%q<uuidtools>, [">= 2.1.2"])
-    s.add_dependency(%q<rspec>, ["~> 2.3.0"])
-    s.add_dependency(%q<yard>, ["~> 0.6.0"])
-    s.add_dependency(%q<bundler>, ["~> 1.0.0"])
-    s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
+    s.add_dependency(%q<rspec>, ["~> 2.8.0"])
+    s.add_dependency(%q<yard>, ["~> 0.7.5"])
+    s.add_dependency(%q<bundler>, ["~> 1.0.22"])
+    s.add_dependency(%q<jeweler>, ["~> 1.8.3"])
     s.add_dependency(%q<simplecov>, [">= 0"])
+    s.add_dependency(%q<log4r>, ["~> 1.1.10"])
     s.add_dependency(%q<msgpack>, [">= 0.4.6"])
     s.add_dependency(%q<eventmachine>, [">= 0.12.10"])
+    s.add_dependency(%q<girl_friday>, ["~> 0.9.7"])
   end
 end