serf 0.6.1 → 0.7.0

data/docs/thread_pools.txt

@@ -0,0 +1,16 @@
+Serf drives the incoming requests using runners. Each runner
+is implemented to either perform tasks synchronously or asynchronously.
+For asynchronous background processing, we rely on third party
+libraries. The following is a list of current and (hopefully) future
+libraries we hope to support.
+
+Implemented Runners:
+  * EventMachine http://rubyeventmachine.com/
+  * GirlFriday https://github.com/mperham/girl_friday
+
+TBD ThreadPool Libraries:
+  * ThreadPool https://github.com/danielbush/ThreadPool
+  * Threadz https://github.com/nanodeath/threadz
+  * Celluloid https://github.com/tarcieri/celluloid
+  * Resque https://github.com/defunkt/resque
+  * DelayedJob https://github.com/tobi/delayed_job

data/lib/serf/builder.rb

@@ -1,14 +1,15 @@
+require 'serf/routing/endpoint'
+require 'serf/routing/registry'
+require 'serf/runners/direct'
 require 'serf/serfer'
-require 'serf/runners/direct_runner'
-require 'serf/runners/em_runner'
 require 'serf/util/null_object'
-require 'serf/util/route_set'
+require 'serf/util/options_extraction'
 
 module Serf
 
   ##
   # A Serf Builder that processes the SerfUp DSL to build a rack-like
-  # app to route and process received messages. This builder is
+  # app to endpoint and process received messages. This builder is
   # implemented based on code from Rack::Builder.
   #
   #   builder = Serf::Builder.parse_file 'examples/config.su'
@@ -22,6 +23,8 @@ module Serf
   #   builder.to_app
   #
   class Builder
+    include Serf::Util::OptionsExtraction
+
     def self.parse_file(config)
       cfgfile = ::File.read(config)
       builder = eval "::Serf::Builder.new {\n" + cfgfile + "\n}",
@@ -29,33 +32,27 @@ module Serf
       return builder
     end
 
-    def initialize(app=nil, &block)
-      # Configuration about the routes and apps to run.
-      @use = []
-      @route_maps = []
-      @handlers = {}
-      @message_parsers = {}
-      @not_found = app || proc do
-        raise ArgumentError, 'Handler Not Found'
-      end
-
-      # Default option in route_configs for background is 'false'
-      @background = false
+    def initialize(*args, &block)
+      extract_options! args
 
-      # Factories to build objects that wire our Serf App together.
-      # Note that these default implementing classes are also factories
-      # of their own objects (i.e. - define a 'build' class method).
-      @serfer_factory = ::Serf::Serfer
-      @foreground_runner_factory = ::Serf::Runners::DirectRunner
-      @background_runner_factory = ::Serf::Runners::EmRunner
-      @route_set_factory = ::Serf::Util::RouteSet
+      # Configuration about the endpoints and apps to run.
+      @use = []
+      @not_found = opts :not_found, lambda { |env|
+        raise ArgumentError, 'Endpoints Not Found'
+      }
 
-      # Utility and messaging channels for our Runners
-      # NOTE: these are only used if the builder needs to instantiage runners.
-      @results_channel = ::Serf::Util::NullObject.new
+      # Utility and messaging channels that get passed as options
+      # when building our Runners and Handlers.
+      @response_channel = ::Serf::Util::NullObject.new
       @error_channel = ::Serf::Util::NullObject.new
       @logger = ::Serf::Util::NullObject.new
 
+      # Set up the starting state for our DSL calls.
+      @runner_matcher_endpoint_map = {}
+      @runner_params = {}
+      runner :direct
+      @matcher = nil
+
       # configure based on a given block.
       instance_eval(&block) if block_given?
     end
@@ -68,126 +65,130 @@ module Serf
       @use << proc { |app| middleware.new(app, *args, &block) }
     end
 
-    def routes(route_map)
-      @route_maps << route_map
-    end
+    def not_found(app); @not_found = app; end
 
-    def handler(handler_name, handler)
-      @handlers[handler_name] = handler
-    end
+    def response_channel(channel); @response_channel = channel; end
+    def error_channel(channel); @error_channel = channel; end
+    def logger(logger); @logger = logger; end
 
-    def message_parser(message_parser_name, message_parser)
-      @message_parsers[message_parser_name] = message_parser
-    end
+    ##
+    # DSL Method to change our current context to use the given matcher.
+    #
+    def match(matcher); @matcher = matcher; end
 
-    def not_found(app)
-      @not_found = app
-    end
+    ##
+    # Mount and endpoint to the current context's Runner and Matcher.
+    # Connected so the endpoint will pass serf_options to the handler's build.
+    def run(*args, &block); mount true, *args, &block; end
 
-    def background(run_in_background)
-      @background = run_in_background
-    end
+    ##
+    # Mount and endpoint to the current context's Runner and Matcher.
+    # Unconnected so the endpoint will omit serf_options to the handler's build.
+    def run_unconnected(*args, &block); mount false, *args, &block; end
 
-    def serfer_factory(serfer_factory)
-      @serfer_factory = serfer_factory
+    ##
+    # The generic mount method used by run & run_unconnected to create an
+    # endpoint to be associated with the current context's Runner and Matcher.
+    def mount(connected, handler_factory, *args, &block)
+      raise 'No matcher defined yet' unless @matcher
+      @runner_matcher_endpoint_map[@runner_factory] ||= {}
+      @runner_matcher_endpoint_map[@runner_factory][@matcher] ||= []
+      @runner_matcher_endpoint_map[@runner_factory][@matcher] <<
+        Serf::Routing::Endpoint.new(
+          connected,
+          handler_factory,
+          *args,
+          &block)
     end
 
-    def foreground_runner_factory(foreground_runner_factory)
-      @foreground_runner_factory = foreground_runner_factory
+    ##
+    # DSL Method to change our current context to use the given runner.
+    #
+    def runner(type)
+      @runner_factory = case type
+      when :direct
+        ::Serf::Runners::Direct
+      when :event_machine
+        begin
+          require 'serf/runners/event_machine'
+          Serf::Runners::EventMachine
+        rescue NameError => e
+          e.extend Serf::Error
+          raise e
+        end
+      when :girl_friday
+        begin
+          require 'serf/runners/girl_friday'
+          Serf::Runners::GirlFriday
+        rescue NameError => e
+          e.extend Serf::Error
+          raise e
+        end
+      else
+        raise 'No callable runner' unless type.respond_to? :build
+        type
+      end
     end
 
-    def background_runner_factory(background_runner_factory)
-      @background_runner_factory = background_runner_factory
+    def params(*args)
+      @runner_params[@runner_factory] = args
     end
 
-    def route_set_factory(route_set_factory)
-      @route_set_factory = route_set_factory
+    ##
+    # Returns a hash of our current serf infrastructure options
+    # to be passed to Endpoint#build methods.
+    def serf_options
+      {
+        response_channel: @error_channel,
+        error_channel: @error_channel,
+        logger: @logger
+      }
     end
 
-    def results_channel(results_channel)
-      @results_channel = results_channel
-    end
+    ##
+    # Create our app.
+    #
+    def to_app
+      # By default, we go to the not_found app.
+      app = @not_found
 
-    def error_channel(error_channel)
-      @error_channel = error_channel
-    end
+      registries = {}
 
-    def logger(logger)
-      @logger = logger
-    end
+      # Set additional options for all the mounts
+      @runner_matcher_endpoint_map.each do |runner_factory, matcher_endpoints|
 
-    def to_app
-      bg_route_set = @route_set_factory.build
-      fg_route_set = @route_set_factory.build
-
-      @route_maps.each do |route_map|
-        route_map.each do |matcher, route_configs|
-          route_configs_iterator(route_configs).each do |route_config|
-            # If the passed in route_config was a String, then we place
-            # it in an route config as the 'target' field and leave all
-            # other options as default.
-            config = (route_config.is_a?(String) ?
-              { target: route_config } :
-              route_config)
-
-            # Get the required handler.
-            # Raises error if handler wasn't declared in config.
-            target = config.fetch :target
-            handler_name, action = handler_and_action target
-
-            # Raises error if handler wasn't registered with builder.
-            handler = @handlers.fetch handler_name
-
-            # Lookup the parser if it was defined.
-            # The Parser MAY be either an object or string.
-            # If String, then we're going to look up in parser map.
-            # Raises an error if a parser (string) was declared, but not
-            # registered with the builder.
-            parser = config[:message_parser]
-            parser = @message_parsers.fetch(parser) if parser.is_a?(String)
-
-            # We have the handler, action and parser.
-            # Now we're going to add that route to either the background
-            # or foreground route_set.
-            background = config.fetch(:background) { @background }
-            (background ? bg_route_set : fg_route_set).add_route(
-              matcher: matcher,
-              handler: handler,
-              action: action,
-              message_parser: parser)
-          end
+        # 1. Create a registry for our given hash of matchers to endpoints.
+        # 2. Convert the hash of matcher to endpoints into a useable registry
+        #   for lookups.
+        registry = ::Serf::Routing::Registry.new
+        matcher_endpoints.each do |matcher, endpoints|
+          registry.add matcher, endpoints
         end
-      end
-
-      # Get or make our foreground runner
-      fg_runner = @foreground_runner_factory.build(
-        results_channel: @results_channel,
-        error_channel: @error_channel,
-        logger: @logger)
-
-      # Get or make our background runner
-      bg_runner = @background_runner_factory.build(
-        results_channel: @results_channel,
-        error_channel: @error_channel,
-        logger: @logger)
 
-      # We create the route_sets dependent on built routes.
-      route_sets = {}
-      if fg_route_set.size > 0
-        route_sets[fg_route_set] = fg_runner
-      end
-      if bg_route_set.size > 0
-        route_sets[bg_route_set] = bg_runner
+        # Ok, we'll create the runner and add it to our registries hash
+        # if we actually have endpoints here.
+        if registry.size > 0
+          runner_params = @runner_params[runner_factory] ?
+            @runner_params[runner_factory] :
+            []
+          runner_params << (runner_params.last.is_a?(Hash) ?
+            runner_params.pop.merge(serf_options) :
+            serf_options)
+          runner = runner_factory.build *runner_params
+          registries[runner] = registry
+        end
       end
 
-      # By default, we go to the not_found app.
-      app = @not_found
-
-      # But if we have routes, then we'll build a serfer to handle it.
-      if route_sets.size > 0
-        app = @serfer_factory.build(
-          route_sets: route_sets,
+      if registries.size > 0
+        app = Serf::Serfer.build(
+          # The registries to match, and their runners to execute.
+          registries: registries,
+          # App if no endpoints were found.
           not_found: app,
+          # Serf infrastructure options to pass to 'connected' Endpoints
+          # to build a handler instance for each env hash received.
+          serf_options: serf_options,
+          # Options to use by serfer because it includes ErrorHandling.
           error_channel: @error_channel,
           logger: @logger)
       end
@@ -197,42 +198,5 @@ module Serf
 
       return app
     end
-
-    private
-
-    ##
-    # This handles route_configs that are Array, Hash or String.
-    # We want to create a proper iterator to run over the route_configs.
-    def route_configs_iterator(route_configs)
-      case route_configs
-      when String
-        return Array(route_configs)
-      when Hash
-        return [route_configs]
-      else
-        return route_configs
-      end
-    end
-
-    ##
-    # Extracts the handler_name and action from the 'target' using
-    # the shortcut convention similar to Rails routing.
-    #
-    #   'my_handler#my_method' => # my_method action.
-    #   'my_handler#' => # action defaults to 'call' method.
-    #   'my_handler'  => # action defaults to 'call' method.
-    #   '#my_method'  => # some registered handler name with empty string.
-    #
-    # @param [String] target the handler and action description.
-    # @return the splat handler and action.
-    #
-    def handler_and_action(target)
-      handler, action = target.split '#', 2
-      handler = handler.to_s.strip
-      action = action.to_s.strip
-      action = :call if action.size == 0
-      return handler, action
-    end
   end
-
 end

data/lib/serf/command.rb

@@ -0,0 +1,113 @@
+require 'active_support/concern'
+require 'active_support/core_ext/class/attribute'
+
+require 'serf/util/options_extraction'
+
+module Serf
+
+  ##
+  # A base class for Serf users to implement a Command pattern.
+  #
+  #   class MyCommand
+  #     include Serf::Command
+  #
+  #     # Set a default Message Parser for the class.
+  #     self.request_factory = MySerfRequestMessage
+  #
+  #     def initialize(*args, &block)
+  #       # Do some validation here, or extra parameter setting with the args
+  #     end
+  #
+  #     def call
+  #       # Do something w/ @request and @opts
+  #       return nil # e.g. MySerfMessage
+  #     end
+  #   end
+  #
+  #   MyCommand.call(REQUEST_ENV, some, extra, params, options_hash, &block)
+  #
+  #   # Built in lambda wrapping to use the MyCommand with GirlFriday.
+  #   worker = MyCommand.worker some, extra, params, options_hash, &block
+  #   work_queue = GirlFriday::WorkQueue.new &worker
+  #   work_queue.push REQUEST_ENV
+  #
+  module Command
+    extend ActiveSupport::Concern
+    include Serf::Util::OptionsExtraction
+
+    included do
+      class_attribute :request_factory
+      attr_reader :request
+    end
+
+    def call
+      raise NotImplementedError
+    end
+
+    def validate_request!
+      # We must verify that the request is valid, but only if the
+      # request object isn't a hash.
+      unless request.is_a?(Hash) || request.valid?
+        raise ArgumentError, request.full_error_messages
+      end
+    end
+
+    module ClassMethods
+
+      ##
+      # Class method that both builds then executes the unit of work.
+      #
+      def call(*args, &block)
+        self.build(*args, &block).call
+      end
+
+      ##
+      # Factory build method that creates an object of the implementing
+      # class' unit of work with the given parameters.
+      #
+      def build(*args, &block)
+        # The very first argument is the Request, we shift it off the args var.
+        req = args.shift
+        req = {} if req.nil?
+
+        # Now we allocate the object, and do some options extraction that may
+        # modify the args array by popping off the last element if it is a hash.
+        obj = allocate
+        obj.send :__send__, :extract_options!, args
+
+        # If the request was a hash, we MAY be able to convert it into a
+        # request object. We only do this if a request_factory was set either
+        # in the options, or if the request_factory class attribute is set.
+        # Otherwise, just give the command the hash, and it is up to them
+        # to understand what was given to it.
+        factory = obj.opts :request_factory, self.request_factory
+        request = (req.is_a?(Hash) && factory ? factory.build(req) : req)
+
+        # Set the request instance variable to whatever type of request we got.
+        obj.instance_variable_set :@request, request
+
+        # Now validate that the request is ok.
+        # Implementing classes MAY override this method to do different
+        # kind of request validation.
+        obj.validate_request!
+
+        # Finalize the object's construction with the rest of the args & block.
+        obj.send :__send__, :initialize, *args, &block
+
+        return obj
+      end
+
+      ##
+      # Generates a curried function to execute a Command's call class method.
+      #
+      # @returns lambda block to execute a call.
+      #
+      def worker(*args, &block)
+        lambda { |message|
+          self.call message, *args, &block
+        }
+      end
+
+    end
+  end
+end

data/lib/serf/message.rb

@@ -15,6 +15,8 @@ module Serf
     included do
       class_attribute :kind
       send 'kind=', self.to_s.tableize.singularize
+      class_attribute :model_name
+      send 'model_name=', self.to_s
     end
 
     def kind
@@ -33,10 +35,22 @@ module Serf
       to_hash.to_json *args
     end
 
+    def model
+      self.class
+    end
+
+    def full_error_messages
+      errors.full_messages.join '. '
+    end
+
     module ClassMethods
 
-      def parse(*args)
-        self.new *args
+      def parse(*args, &block)
+        self.new *args, &block
+      end
+
+      def build(*args, &block)
+        self.new *args, &block
       end
 
     end

data/lib/serf/messages/caught_exception_event.rb

@@ -9,7 +9,7 @@ module Messages
   # exception during the processing of some message, which is
   # represented by the context field.
   #
-  # Instances of this class are norminally published to an
+  # Instances of this class are norminally pushed to an
   # error channel for out of band processing/notification.
   #
   class CaughtExceptionEvent
@@ -17,21 +17,24 @@ module Messages
     include ::Serf::Util::Uuidable
 
     attr_accessor :context
-    attr_accessor :error_message
-    attr_accessor :error_backtrace
+    attr_accessor :error
+    attr_accessor :message
+    attr_accessor :backtrace
 
     def initialize(options={})
       @context = options[:context]
-      @error_message = options[:error_message]
-      @error_backtrace = options[:error_backtrace]
+      @error = options[:error]
+      @message = options[:message]
+      @backtrace = options[:backtrace]
       @uuid = options[:uuid]
     end
 
     def attributes
       {
         'context' => @context,
-        'error_message' => @error_message,
-        'error_backtrace' => @error_backtrace,
+        'error' => @error,
+        'message' => @message,
+        'backtrace' => @backtrace,
         'uuid' => uuid
       }
     end

data/lib/serf/routing/endpoint.rb

@@ -0,0 +1,49 @@
+require 'serf/util/options_extraction'
+
+module Serf
+module Routing
+
+  ##
+  # An endpoint is the description of how to build a Unit of Work
+  # for a given matched message. It builds an instance that
+  # responds to the `call` method that will actually execute the work.
+  # Units of work is built on every received message with the request,
+  # given arguments, options (merged with serf infrastructure options)
+  # and given block.
+  #
+  class Endpoint
+    include Serf::Util::OptionsExtraction
+
+    def initialize(connect, handler_factory, *args, &block)
+      # If we want to connect serf options, then we try to extract
+      # any possible options from the args list. If a hash exists at the
+      # end of the args list, then we'll merge into it. Otherwise a new hash
+      # will be added on.
+      extract_options! args if @connect = connect
+
+      @handler_factory= handler_factory
+      @args = args
+      @block = block
+    end
+
+    ##
+    # Builds a Unit of Work object.
+    #
+    def build(env, serf_options={})
+      # If we are connecting serf options, then we need to pass these
+      # options on to the builder.
+      if @connect
+        @handler_factory.build(
+          env.dup,
+          *@args,
+          options.merge(serf_options),
+          &@block)
+      else
+        @handler_factory.build env.dup, *@args, &@block
+      end
+    end
+
+  end
+
+end
+end

data/lib/serf/routing/registry.rb

@@ -0,0 +1,66 @@
+require 'serf/util/regexp_matcher'
+
+module Serf
+module Routing
+
+  ##
+  # EndpointRegistry returns list of Endpoints to execute that match
+  # criteria based on the Endpoints' associated 'matcher' object
+  # with the input of the ENV Hash (passed to match).
+  #
+  class Registry
+
+    def initialize(options={})
+      @endpoints = {}
+      @matchers = []
+      @regexp_matcher_factory = options.fetch(:regexp_matcher_factory) {
+        ::Serf::Util::RegexpMatcher
+      }
+    end
+
+    ##
+    # Connects a matcher (String or an Object implementing ===) to endpoints.
+    #
+    def add(matcher, endpoints)
+      # Maybe we have an non-String matcher. Handle the Regexp case.
+      # We only keep track of matchers if it isn't a string because
+      # string matchers are just pulled out of endpoints by key lookup.
+      matcher = @regexp_matcher_factory.build matcher if matcher.kind_of? Regexp
+      @matchers << matcher unless matcher.is_a? String
+
+      # We add the (matcher+endpoint) into our endpoints
+      @endpoints[matcher] ||= []
+      @endpoints[matcher].concat endpoints
+    end
+
+    ##
+    # @param [Hash] env The input message environment to match for endpoints.
+    # @return [Array] List of endpoints that matched.
+    #
+    def match(env={})
+      kind = env[:kind]
+      endpoints = []
+      endpoints.concat @endpoints.fetch(kind) { [] }
+      @matchers.each do |matcher|
+        endpoints.concat @endpoints[matcher] if matcher === env
+      end
+      return endpoints
+    end
+
+    ##
+    # @return [Integer] Number of matchers this EndpointsMap tracks.
+    #
+    def size
+      return @endpoints.size
+    end
+
+    ##
+    # Default factory method.
+    #
+    def self.build(options={})
+      self.new options
+    end
+  end
+
+end
+end