langis 0.1.0

data/lib/langis/engine.rb

@@ -0,0 +1,146 @@
+module Langis
+  module Engine
+
+    ##
+    # Instances of this class are executed when a message is pushed to
+    # the intake's EventMachine::Channel that it subscribed to. Its
+    # primary function is to safely execute the Langis sink
+    # (Rackish application) that it has been tasked to manage. To do
+    # this safely with performance considerations, it enqueues a
+    # Proc to be handled by the EvenMachine's deferred thread pool and
+    # protects the thread pool by wrapping the sink call with a rescue block.
+    # Any caught errors will result in an error message with the caught
+    # exception pushed into the given EventMachine error channel. All
+    # successful completions will push the returned Rackish result to the
+    # EventMachine success channel.
+    class EventMachineRunner
+
+      ##
+      #
+      # @param [#call] app The Rackish app to execute.
+      # @param [Hash] options ({})
+      # @option options [EventMachine::Channel] :success_channel (nil) The
+      #   EventMachine::Channel instance to push the Rackish app's return
+      #   results to. This happens when there are no errors raised.
+      # @option options [EventMachine::Channel] :error_channel (nil) The
+      #   EventMachine::Channel instance to push error messages to when the
+      #   runner catches an exception during the execution of the Rackish app.
+      # @option options [Object] :evm (EventMachine) Specify a different
+      #   class/module to use when executing deferred calls. Mainly useful
+      #   for unit testing, or if you want to run the app directly in
+      #   the main thread instead of the deferred thread pool.
+      def initialize(app, options={})
+        @app = app
+        @success_channel = options[:success_channel]
+        @error_channel = options[:error_channel]
+        @evm = options[:evm] || EventMachine
+        @intake_name = options[:intake_name]
+      end
+
+      ##
+      # The method that is called in the EventMachine's main reactor thread
+      # whose job is to enqueue the main app code to be run by EventMachine's
+      # deferred thread pool.
+      #
+      # This method sets up the Rackish environment hash that is passed
+      # as the main input to the Rackish app; it sets up the Proc that will
+      # be actually executed in by the deferred thread pool. The proc
+      # protects the thread pool from exceptions, and pushes respective
+      # error and success results to given EventMachine channels.
+      #
+      # @param [Object] message The message that is getting pushed through
+      #   the Langis pipes to their eventual handlers.
+      def call(message)
+        # Assign local variables to the proper apps, etc for readability.
+        app = @app
+        success_channel = @success_channel
+        error_channel = @error_channel
+        intake_name = @intake_name
+        # Enqueue the proc to be run in by the deferred thread pool.
+        @evm.defer(proc do
+          # Create the base environment that is understood by the Rackish apps.
+          env = {}
+          env[MESSAGE_TYPE_KEY] = message.message_type.to_s if(
+            message.respond_to? :message_type)
+          env[MESSAGE_KEY] = message
+          env[INTAKE_KEY] = intake_name
+          # Actually run the Rackish app, protected by a rescue block.
+          # Push the results to their respective channels when finished.
+          begin
+            results = app.call env
+            success_channel.push(results) if success_channel
+          rescue => e
+            # It was an error, so we have to create a Rackish response array.
+            # We push a SERVER_ERROR status along with an enhanced
+            # headers section: the exception and original message.
+            error_channel.push([
+              SERVER_ERROR,
+              env.merge({ X_EXCEPTION => e}),
+              ['']]) if error_channel
+          end
+        end)
+      end
+    end
+
+    ##
+    # And EventMachine based implementation of a Langis Engine. Its sole
+    # job is to take a pumped message into an intake and broadcast the same
+    # message to all of the intake's registered sinks. In essense these
+    # engines need to execute the sinks handler methods for each message.
+    #
+    # This class leverages EventMachine's features to easily do efficient
+    # publishing to the subscribers, and uses the EventMachineRunner
+    # to do the actual code execution.
+    #
+    # @see EventMachineRunner
+    class EventMachineEngine
+
+      ##
+      # @param [Hash{String=>Array<#call>}] intakes The mapping of intake
+      #   names to the list of Rackish applications (sinks) that subscribed
+      #   to the given intake.
+      # @option options [Object] :evm (EventMachine) Specify a different
+      #   class/module to use when executing deferred calls. Mainly useful
+      #   for unit testing, or if you want to run the app directly in
+      #   the main thread instead of the deferred thread pool.
+      # @option options [Class] :evm_channel (EventMachine::Channel) The
+      #   channel class to instantiate as the underlying pub-sub engine. This
+      #   is useful for unittesting, or if you want to implement a
+      #   non-EventMachine pub-sub mechanism.
+      def initialize(intakes, options={})
+        evm_channel_class = options[:evm_channel] || EventMachine::Channel
+        @intake_channels = {}
+        intakes.each do |intake_name, apps|
+          runner_options = {
+            :success_channel => options[:success_channel],
+            :error_channel => options[:error_channel],
+            :evm => options[:evm],
+            :intake_name => intake_name
+          }
+          @intake_channels[intake_name] = channel = evm_channel_class.new
+          apps.each do |app|
+            channel.subscribe EventMachineRunner.new(app, runner_options)
+          end
+        end
+      end
+
+      ##
+      # Publishes a message into the Langis publish-subscribe bus.
+      #
+      # @overload pump(message)
+      #   Publishes the message to the :default intake.
+      #   @param [Object] message The message to publish.
+      # @overload pump(message, ...)
+      #   Publishes the message to the given list of intakes.
+      #   @param [Object] message The message to publish.
+      #   @param [#to_s] ... Publish the message to these listed intakes
+      def pump(message, *intakes)
+        intakes.unshift :default if intakes.empty?
+        intakes.each do |name|
+          channel = @intake_channels[name.to_s]
+          channel.push(message) if channel
+        end
+      end
+    end
+  end
+end

data/lib/langis/middleware.rb

@@ -0,0 +1,135 @@
+module Langis
+  module Middleware
+
+    ##
+    # Middleware class that modifies the Rackish input environment by
+    # transforming a specific value in the environment before passing
+    # it along to the rest of the Rackish application chain.
+    #
+    # Some useful applications of this transform:
+    # * Serialization or Deserialization of input data.
+    # * Filter or modify the message; to explicitly whitelist the list of
+    #   properties in the message that may be exposed to a service or
+    #   third party.
+    class EnvFieldTransform
+
+      ##
+      #
+      # @param app The Rackish Application for which this instance is acting as
+      #   middleware.
+      # @option options [String] :key (Langis::MESSAGE_KEY) The hash key
+      #   of the Rackish environment whose value is the object that we
+      #   want to transform (transformation object).
+      # @option options [Symbol,String] :to_method (:to_json) The
+      #   transformation object that will respond to the invokation of this
+      #   method name. The return value of that method will replace the
+      #   original transformation object in the Rackish environment as
+      #   the environment is passed on to the rest of the Rackish app chain.
+      # @option options [Array,Object] :to_args ([]) The parameter or
+      #   list of parameters to pass to the transformation method.
+      def initialize(app, options={})
+        @app = app
+        @to_method = options[:to_method] || :to_json
+        @to_args = options[:to_args] || []
+        @to_args = [@to_args] unless @to_args.is_a? Array
+        @key = options[:key] || MESSAGE_KEY
+      end
+
+      ##
+      # Executes the object transformation, and invokes the rest of the
+      # Rackish app chain.
+      #
+      # @param [Hash] env The input Rackish Environment.
+      # @return [Array<Integer,Hash,#each>] The return of the proxied Rackish
+      #   application chain.
+      def call(env)
+        item = env[@key].send @to_method, *@to_args
+        return @app.call env.merge({ @key => item })
+      end
+    end
+
+    ##
+    # Middleware that adds an Array of values to the Rackish Environment
+    # input. This array of values is created by calling callables using the
+    # said Rackish Environment as input, and from static strings.
+    #
+    # The following example creates an Array of size two, and places it
+    # into the Rackish Environment key identified by 'my_key'. The first
+    # item in the Array is the static string, "Hello World". The second value
+    # is whatever was in the Rackish Environment under the key, "name".
+    #
+    #     use Parameterizer,
+    #       'Hello World',
+    #       lambda { |env| env['name'] },
+    #       :env_key => 'my_key'
+    #
+    class Parameterizer
+
+      ##
+      # @param [#call] app The next link in the Rackish Application chain.
+      # @param [String,#call] *args The list of new parameters that the
+      #   Parameterizer middleware creates. String values are used as is,
+      #   and callable objects are executed with the input Rackish Environment
+      #   as the first parameter.
+      # @option options [String] :env_key (::Langis::MESSAGE_KEY)
+      def initialize(app, *args)
+        @app = app
+        @options = args.last.kind_of?(Hash) ? args.pop : {}
+        @args = args
+        @env_key = @options[:env_key] || MESSAGE_KEY
+      end
+
+      ##
+      # The main method of the Parameterizer middleware.
+      #
+      # @param [Hash] env The input Rackish Environment.
+      def call(env={})
+        new_args = @args.map do |value|
+          value.respond_to?(:call) ? value.call(env) : value
+        end
+        new_env = {}.update(env)
+        new_env[@env_key] = new_args
+        return @app.call new_env
+      end
+    end
+
+    ##
+    # Middleware to only continue execution of the Rackish application chain
+    # if the input environment's Langis::MESSAGE_TYPE_KEY is set to a value
+    # that has been whitelisted.
+    class MessageTypeFilter
+
+      ##
+      #
+      # @param app The Rackish application chain to front.
+      # @param [#to_s] ... The whitelist of message types to allow pass.
+      def initialize(app, *args)
+        @app = app
+        @message_types = args.map { |message_type| message_type.to_s }
+      end
+
+      ##
+      # Executes the filtering, and invokes the rest of the Rackish app chain
+      # if the message type is allowed.
+      # 
+      # @param [Hash] env The Rackish input environment.
+      # @return [Array<Integer,Hash,#each>] The return of the proxied Rackish
+      #   application chain, or an OK with the filter reason.
+      # @see Langis::X_FILTERED_BY
+      # @see Langis::X_FILTERED_TYPE
+      def call(env)
+        if @message_types.include? env[MESSAGE_TYPE_KEY]
+          return @app.call(env)
+        else
+          return [
+            OK,
+            {
+              X_FILTERED_BY => self.class.to_s,
+              X_FILTERED_TYPE => env[MESSAGE_TYPE_KEY].class
+            },
+            ['']]
+        end
+      end
+    end
+  end
+end

data/lib/langis/rackish.rb

@@ -0,0 +1,118 @@
+module Langis
+  module Rackish
+
+    ##
+    # Error raised when RackishJob is asked to run an unregistered Rackish
+    # Application.
+    #
+    # @see RackishJob
+    class NotFoundError < LangisError
+    end
+
+    ##
+    # RackishJob is a dual DelayedJob-Resque job that is used to execute
+    # Rackish applications, or Rack applications that are robust against
+    # non-conformant "Rack Environments", in the background.
+    # Rackish Applications are created and registered with this RackishJob
+    # class. Each registration is associated with an app_key that is well
+    # known to any component that wants to execute that particular Rackish
+    # Application. Client components then queue up this job class with
+    # the app_key and the input hash for that application.
+    #
+    # Notes
+    # * This class does not provide a compliant Rack specified environment
+    #   to the Rackish applications it calls. Prepend middleware that
+    #   provides such an environment to the application chain if required.
+    #
+    # For example, to queue up a RackishJob using DelayedJob:
+    #     Delayed::Job.enqueue Langis::Rackish::RackishJob.new(
+    #       'my_app',
+    #       {
+    #         'input_key' => 'value'
+    #       })
+    #
+    # For example, to queue up a RackishJob using Resque:
+    #     Resque.enqueue(
+    #       Langis::Rackish::RackishJob,
+    #       'my_app',
+    #       {
+    #         'input_key' => 'value'
+    #       })
+    #
+    # The my_app job may be registered in the worker process as follows:
+    #     Langis::Rackish::RackishJob.register_rackish_app(
+    #       'my_app',
+    #       lambda { |env|
+    #         # Do something
+    #       })
+    #
+    class RackishJob < Struct.new(:app_key, :env)
+      class << self
+
+        ##
+        # Registers a Rackish Application under a given name so it can be
+        # executed by the RackishJob class via the DelayedJob or Resque
+        # background job libraries.
+        #
+        # For example, the following can be found in a Rails initializer.
+        #     my_app = Rack::Builder.app do
+        #       run MyApp
+        #     end
+        #     RackishJob.register 'my_app', my_app
+        #
+        # @param [String] app_key The name used to lookup which Rackish
+        #  application to call.
+        # @param [#call] app The Rackish Application to call for the requested
+        #  app_key.
+        # @return [#call] The Rackish Application passed in is returned back.
+        def register_rackish_app(app_key, app)
+          @apps ||= {}
+          @apps[app_key.to_s] = app
+        end
+
+        ##
+        # Acts as the Resque starting point.
+        #
+        # For example, the following can be used to execute the 'my_app'
+        # Rackish application using Resque from an ActiveRecord callback:
+        #     def after_create(record)
+        #       Resque.enqueue RackishJob, 'my_app', { 'my.data' => record.id }
+        #     end
+        #
+        # @param [String] app_key The registered application's name that
+        #  is to be called with the given env input.
+        # @param [Hash] env The Rackish input environment. This is the input
+        #  that should be relevant to the called app. There is no guarantee
+        #  that this environment hash is a fully compliant Rack environment.
+        # @raise [RackishAppNotFoundError] Signals that the given app_key
+        #  was not registered with RackishJob. See DelayedJob and Resque
+        #  documentation to understand how to ignore or handle raised
+        #  exceptions for retry.
+        def perform(app_key=nil, env={})
+          app = @apps[app_key]
+          if app.respond_to? :call
+            app.call env 
+          else
+            raise NotFoundError.new "#{app_key} not found"
+          end
+        end
+      end
+
+      ##
+      # Acts as the DelayedJob starting point. All this does is relays the
+      # call to the Resque starting point.
+      #
+      # For example, the following can be used to execute the 'my_app'
+      # Rackish application using DelayedJob from an ActiveRecord callback:
+      #     def after_create(record)
+      #       Delayed::Job.enqueue(
+      #         RackishJob.new('my_app', { 'my.data' => record.id }))
+      #     end
+      #
+      # @see RackishJob.perform
+      def perform
+        self.class.perform(app_key.to_s, env || {})
+      end
+    end
+  end
+end

data/lib/langis/sinks.rb

@@ -0,0 +1,138 @@
+module Langis
+
+  ##
+  # Predefined sinks, destinations for a message pumped into the Langis Engines.
+  module Sinks
+
+    ##
+    # The header key whose value is the DelayedJob enqueue result.
+    DELAYED_JOB_RESULT_KEY = 'langis.sink.delayed_job.result'
+
+    ##
+    # Module function that creates the endpoint Rackish app that will
+    # enqueue a new instance of a DelayedJob job class with instantiation
+    # parameters extracted from the Rackish input environment.
+    #
+    # @param [Class] job_class The DelayedJob job class to enqueue.
+    # @option options [String] :env_key (Langis::MESSAGE_KEY) The Rackish
+    #   input environment key whose value is passed to the given job_class
+    #   constructor. If the value of this key is an array, then the elements
+    #   of that array are passed as though they were individually specified.
+    # @option options [Integer] :priority (0) DelayedJob priority to be used
+    #   for all jobs enqueued with this sink.
+    # @option options [Time] :run_at (nil) DelayedJob run_at to be used for
+    #   all jobs enqueued with this sink.
+    # @option options [Symbol] :transform (nil) method to call on the object
+    #   passed in via :env_key if env_key responds to it. The returned Array's
+    #   elements becomes the initializer argument(s) of the delayed job.
+    #   If the return value is anything but an array, then that value is
+    #   passed on; even explicit nils are passed on to the job_class#new.
+    #   Note that DelayedJob serializes these parameters in Yaml.
+    # @option options [Array] :transform_args ([]) arguments to pass to
+    #   the transform call.
+    # @return [Array<Integer,Hash,#each>] A simple OK return with the header
+    #   hash that contains the delayed job enqueue result.
+    def delayed_job(job_class, options={})
+      priority = options[:priority] || 0
+      run_at = options[:run_at]
+      env_key = options[:env_key] || MESSAGE_KEY
+      xform = options[:transform]
+      xform_args = options[:transform_args] || []
+      xform_args = [xform_args] unless xform_args.is_a? Array
+      lambda { |env|
+        message = env[env_key]
+        if xform.is_a? Symbol and message.respond_to? xform
+          args = message.send(xform, *xform_args)
+        else
+          args = message
+        end
+        args = [args] unless args.is_a? Array
+        result = Delayed::Job.enqueue job_class.new(*args), priority, run_at
+        return [OK, { DELAYED_JOB_RESULT_KEY => result }, ['']]
+      }
+    end
+    module_function :delayed_job
+
+    ##
+    # The header key whose value is the Redis rpush result.
+    REDIS_RESULT_KEY = 'langis.sink.redis.result'
+
+    ##
+    # Module function that creates the endpoint Rackish app that will
+    # rpush an input environment's value into a list stored in a Redis
+    # database.
+    #
+    # @param [Object] connection The redis database connection.
+    # @param [String] key The index key of the list in the Redis database.
+    # @option options [String] :env_key (Langis::MESSAGE_KEY) The Rackish
+    #   input environment key whose value is pushed onto the end of the
+    #   Redis key's list.
+    # @option options [Symbol] :transform (nil) method to call on the object
+    #   passed in via :env_key if env_key responds to it. The returned value
+    #   is saved (after a #to_s by the Redis client) to the Redis database.
+    # @option options [Array] :transform_args ([]) arguments to pass to
+    #   the transform call.
+    # @return [Array<Integer,Hash,#each>] A simple OK return with the header
+    #   hash that contains the Redis#rpush result.
+    def redis(connection, key, options={})
+      env_key = options[:env_key] || MESSAGE_KEY
+      xform = options[:transform]
+      xform_args = options[:transform_args] || []
+      xform_args = [xform_args] unless xform_args.is_a? Array
+      lambda { |env|
+        message = env[env_key]
+        if xform.is_a? Symbol and message.respond_to? xform
+          message = message.send(xform, *xform_args)
+        end
+        result = connection.rpush key, message
+        return [OK, { REDIS_RESULT_KEY  => result }, ['']]
+      }
+    end
+    module_function :redis
+
+    ##
+    # The header key whose value is the Resque enqueue result.
+    RESQUE_RESULT_KEY = 'langis.sink.resque.result'
+
+    ##
+    # Module function that creates the endpoint Rackish app that will
+    # rpush an input environment's value into a list stored in a Redis
+    # database.
+    #
+    # @param [Class] job_class The Resque job class for which we want
+    #   to enqueue the message.
+    # @option options [String] :env_key (Langis::MESSAGE_KEY) The Rackish
+    #   input environment key whose value is passed as the input arguments
+    #   to the actual execution of the Resque job. The found value can be
+    #   an Array, in which case the elements will be used as the execution
+    #   parameters of the given job.
+    # @option options [Symbol] :transform (nil) method to call on the object
+    #   passed in via :env_key if env_key responds to it. The returned Array's
+    #   elements becomes the perform argument(s) of the Resque job.
+    #   If the return value is anything but an array, then that value is
+    #   passed on; even explicit nils are passed as the perform argument.
+    #   Note that these Resque arguments are serialized via #to_json
+    # @option options [Array] :transform_args ([]) arguments to pass to
+    #   the transform call.
+    # @return [Array<Integer,Hash,#each>] A simple OK return with the header
+    #   hash that contains the Resque enqueue result.
+    def resque(job_class, options={})
+      env_key = options[:env_key] || MESSAGE_KEY
+      xform = options[:transform]
+      xform_args = options[:transform_args] || []
+      xform_args = [xform_args] unless xform_args.is_a? Array
+      lambda { |env|
+        message = env[env_key]
+        if xform.is_a? Symbol and message.respond_to? xform
+          args = message.send(xform, *xform_args)
+        else
+          args = message
+        end
+        args = [args] unless args.is_a? Array
+        result = Resque.enqueue job_class, *args
+        return [OK, { RESQUE_RESULT_KEY => result }, ['']]
+      }
+    end
+    module_function :resque
+  end
+end