gilmour 0.3.4 → 0.4.1

data/lib/gilmour/backends/redis.rb

@@ -45,23 +45,23 @@ module Gilmour
       @ident = generate_ident
     end
 
-    def ident
+    def ident #:nodoc:
       @ident
     end
 
-    def generate_ident
+    def generate_ident #:nodoc:
       "#{Socket.gethostname}-pid-#{Process.pid}-uuid-#{SecureRandom.uuid}"
     end
 
-    def report_health?
+    def report_health? #:nodoc:
       @report_health
     end
 
-    def report_errors?
+    def report_errors? #:nodoc:
       @report_errors
     end
 
-    def emit_error(message)
+    def emit_error(message) #:nodoc:
       report = self.report_errors?
 
       if report == false
@@ -73,7 +73,7 @@ module Gilmour
       end
     end
 
-    def setup_pubsub(opts)
+    def setup_pubsub(opts) #:nodoc:
       @publisher = EM::Hiredis.connect(redis_host(opts))
       @subscriber = @publisher.pubsub_client
       register_handlers
@@ -82,7 +82,7 @@ module Gilmour
       GLogger.debug e.backtrace
     end
 
-    def register_handlers
+    def register_handlers #:nodoc:
       @subscriber.on(:pmessage) do |key, topic, payload|
         pmessage_handler(key, topic, payload)
       end
@@ -100,18 +100,18 @@ module Gilmour
       end
     end
 
-    def subscribe_topic(topic)
+    def subscribe_topic(topic) #:nodoc:
       method = topic.index('*') ? :psubscribe : :subscribe
       @subscriber.method(method).call(topic)
     end
 
-    def pmessage_handler(key, matched_topic, payload)
+    def pmessage_handler(key, matched_topic, payload) #:nodoc:
       @subscriptions[key].each do |subscription|
         EM.defer(->{execute_handler(matched_topic, payload, subscription)})
       end
     end
 
-    def register_response(sender, handler, timeout = 600)
+    def register_response(sender, handler, timeout = 600) #:nodoc:
       topic = "gilmour.response.#{sender}"
       timer = EM::Timer.new(timeout) do # Simulate error response
         GLogger.info "Timeout: Killing handler for #{sender}"
@@ -125,11 +125,11 @@ module Gilmour
       GLogger.debug e.backtrace
     end
 
-    def publish_error(messsage)
+    def publish_error(messsage) #:nodoc:
       publish(messsage, Gilmour::ErrorChannel)
     end
 
-    def queue_error(key, message)
+    def queue_error(key, message) #:nodoc:
       @publisher.lpush(key, message) do
         @publisher.ltrim(key, 0, GilmourErrorBufferLen) do
           Glogger.debug "Error queued"
@@ -137,7 +137,7 @@ module Gilmour
       end
     end
 
-    def acquire_ex_lock(sender)
+    def acquire_ex_lock(sender) #:nodoc:
       @publisher.set(sender, sender, 'EX', 600, 'NX') do |val|
         EM.defer do
           yield val if val && block_given?
@@ -145,7 +145,7 @@ module Gilmour
       end
     end
 
-    def response_handler(sender, payload)
+    def response_handler(sender, payload) #:nodoc:
       data, code, _ = Gilmour::Protocol.parse_response(payload)
       handler = @response_handlers.delete(sender)
       @subscriber.unsubscribe(sender)
@@ -158,28 +158,36 @@ module Gilmour
       GLogger.debug e.backtrace
     end
 
-    def send_response(sender, body, code)
+    def send_response(sender, body, code) #:nodoc:
       publish(body, "gilmour.response.#{sender}", {}, code)
     end
 
-    def get_subscribers
+    def get_subscribers #:nodoc:
       @subscriptions.keys
     end
 
-    def setup_subscribers(subs = {})
-      @subscriptions.merge!(subs)
-      EM.defer do
-        subs.keys.each { |topic| subscribe_topic(topic) }
+    def reply_to(topic, opts={}, &blk) #:nodoc:
+      if topic.index('*')
+        raise ArgumentError.new("Subscribers cannot have wildcard topics")
       end
+      super
     end
 
-    def add_listener(topic, &handler)
+    def add_listener(topic, opts = {}, &blk) #:nodoc:
+      if opts[:excl] && exclusive_group(opts).empty?
+        raise ArgumentError.new("Invalid exclusive group")
+      end
+      opts[:handler] ||= blk
       @subscriptions[topic] ||= []
-      @subscriptions[topic] << { handler: handler }
-      subscribe_topic(topic)
+      @subscriptions[topic] << opts
+      EM.next_tick { subscribe_topic(topic) }
     end
 
-    def remove_listener(topic, handler = nil)
+    def listeners(topic) #:nodoc:
+      @subscriptions[topic] || []
+    end
+
+    def remove_listener(topic, handler = nil) #:nodoc:
       if handler
         subs = @subscriptions[topic]
         subs.delete_if { |e| e[:handler] == handler }
@@ -189,30 +197,36 @@ module Gilmour
       @subscriber.unsubscribe(topic) if @subscriptions[topic].empty?
     end
 
-    def send(sender, destination, payload, opts = {}, &blk)
+    def send_message(sender, destination, payload, opts = {}, &blk) #:nodoc:
       timeout = opts[:timeout] || 600
       if opts[:confirm_subscriber]
         confirm_subscriber(destination) do |present|
           if !present
             blk.call(nil, 404) if blk
           else
-            _send(sender, destination, payload, timeout, &blk)
+            _send_message(sender, destination, payload, timeout, &blk)
           end
         end
       else
-        _send(sender, destination, payload, timeout, &blk)
+        _send_message(sender, destination, payload, timeout, &blk)
       end
     rescue Exception => e
       GLogger.debug e.message
       GLogger.debug e.backtrace
     end
 
-    def _send(sender, destination, payload, timeout, &blk)
+    def _send_message(sender, destination, payload, timeout, &blk) #:nodoc:
       register_response(sender, blk, timeout) if block_given?
       @publisher.publish(destination, payload)
       sender
     end
 
+    # Confirms whether an active subscriber is present. 
+    # Params
+    # +dest+:: The destination topic
+    #
+    # The given block is called with a true boolean
+    # if active subscribers exist, else with false
     def confirm_subscriber(dest, &blk)
       @publisher.pubsub('numsub', dest) do |_, num|
         blk.call(num.to_i > 0)
@@ -222,7 +236,7 @@ module Gilmour
       GLogger.debug e.backtrace
     end
 
-    def stop
+    def stop #:nodoc:
       @subscriber.close_connection
     end
 
@@ -234,7 +248,7 @@ module Gilmour
     # before a Gilmour server starts up. To circumvent this dependency, till
     # monitor is stable enough, use Redis to save/share these data structures.
     #
-    def register_health_check
+    def register_health_check #:nodoc:
       @publisher.hset GilmourHealthKey, self.ident, 'active'
 
       # - Start listening on a dyanmic topic that Health Monitor can publish
@@ -255,7 +269,7 @@ module Gilmour
 
     end
 
-    def unregister_health_check
+    def unregister_health_check #:nodoc:
       deleted = false
 
       @publisher.hdel(GilmourHealthKey, self.ident) do

data/lib/gilmour/base.rb

@@ -43,8 +43,7 @@ module Gilmour
       @@subscribers = {} # rubocop:disable all
       @@registered_services = []
 
-      # :nodoc:
-      def inherited(child)
+      def inherited(child) #:nodoc:
         @@registered_services << child
       end
 
@@ -54,23 +53,21 @@ module Gilmour
         @@registered_services
       end
 
-      # Adds a listener for the given topic
-      # topic:: The topic to listen to
-      # opts: Hash of optional arguments.
-      #       Supported options are:
-      #
-      #       excl:: If true, this listener is added to a group of listeners
-      #       with the same name as the name of the class in which this method
-      #       is called. A message sent to the _topic_ will be processed by at
-      #       most one listener from a group
+      # This is the underlying layer of communication. Use if you
+      # know what you are doing. Use "reply_to" and "slot" instead.
       #
-      #       timeout: Maximum duration (seconds) that a subscriber has to
-      #       finish the task. If the execution exceeds the timeout, gilmour
-      #       responds with status {code:409, data: nil}
+      # Adds a listener for the given topic
+      # +topic+:: The topic to listen to
+      # +opts+:: Hash of optional arguments. Supported options are:
+      #          excl:: If true, this listener is added to a group of listeners
+      #          with the same name as the name of the class in which this
+      #          method is called. A message sent to the _topic_ will be
+      #          processed by at most one listener from a group 
+      #          timeout:: Maximum duration (seconds) that a subscriber has to
+      #          finish the task. If the execution exceeds the timeout, gilmour
+      #          responds with status {code:409, data: nil}
       #
-      def listen_to(topic, opts={})
-        handler = Proc.new
-
+      def listen_to(topic, opts={}, &handler)
         opt_defaults = {
           exclusive: false,
           timeout: 600,
@@ -84,8 +81,28 @@ module Gilmour
         @@subscribers[topic] ||= []
         @@subscribers[topic] << opt_defaults
       end
+      alias_method :add_listener, :listen_to
+
+      # Add a reply listener
+      def reply_to(topic, opts={}, &handler)
+        defopts = opts.merge({
+          type: :reply
+        })
+        listen_to(topic, defopts, &handler)
+      end
+
+      # Add a slot listener
+      def slot(topic, opts={}, &handler)
+        defopts = opts.merge({
+          type: :slot
+        })
+        listen_to(topic, defopts, &handler)
+      end
 
       # Returns the list of subscribers for _topic_ or all subscribers if it is nil
+      # Params:
+      # +topic+:: The topic for which to return the subscribers. All subscribers are
+      # returned if this is not provided
       def subscribers(topic = nil)
         if topic
           @@subscribers[topic]
@@ -96,20 +113,19 @@ module Gilmour
 
       # Loads all ruby source files inside _dir_ as subscribers
       # Should only be used inside the parent container class
+      # Params:
+      # +dir+:: relative path of directory to load subscribers from
       def load_all(dir = nil)
         dir ||= (subscribers_path || DEFAULT_SUBSCRIBER_PATH)
         Dir["#{dir}/*.rb"].each { |f| require f }
       end
 
-      # Loads the ruby file at _path_ as a subscriber
-      # Should only be used inside the parent container class
-      def load_subscriber(path)
+      def load_subscriber(path) #:nodoc:
         require path
       end
     end
 
-    # :nodoc:
-    def registered_subscribers
+    def registered_subscribers #:nodoc:
       self.class.registered_subscribers
     end
     ############ End Register ###############
@@ -120,9 +136,13 @@ module Gilmour
     attr_reader :backends
 
     # Enable and return the given backend
-    # Should only be used inside the parent container class
-    # If +opts[:multi_process]+ is true, every request handler will
-    # be run inside a new child process.
+    # Params
+    # +name+:: the backend name (currently only 'redis' is supported)
+    # +opts+:: backend specific options. Options for redis are
+    #          host:: the redis server hostname
+    #          port:: the redis server port
+    #          braodcast_errors:: whether error reorting should be turned on
+    #          health_check:: whether health_check hartbeats should be enabled
     def enable_backend(name, opts = {})
       Gilmour::Backend.load_backend(name)
       @backends ||= {}
@@ -130,11 +150,11 @@ module Gilmour
     end
     alias_method :get_backend, :enable_backend
 
+    # Cleanup the susbcribers and health checks
     def tear_down!
       subs_by_backend = subs_grouped_by_backend
       subs_by_backend.each do |b, subs|
         backend = get_backend(b)
-        backend.setup_subscribers(subs)
         if backend.report_health?
           backend.unregister_health_check
         end
@@ -144,18 +164,29 @@ module Gilmour
     # Starts all the listeners
     # If _startloop_ is true, this method will start it's own
     # event loop and not return till Eventmachine reactor is stopped
+    # Params:
+    # +startloop+:: If true this call with join the eventmachine thred and
+    # block till it is done.
     def start(startloop = false)
       subs_by_backend = subs_grouped_by_backend
       subs_by_backend.each do |b, subs|
         backend = get_backend(b)
-        backend.setup_subscribers(subs)
-
+        subs.each do |topic, handlers|
+          handlers.each do |handler|
+            if handler[:type] == :slot
+              backend.slot(topic, handler)
+            elsif handler[:type] == :reply
+              backend.reply_to(topic, handler)
+            else
+              backend.add_listener(topic, handler)
+            end
+          end
+        end
         if backend.report_health?
           backend.register_health_check
         end
       end
-
-      if startloop
+      if startloop #Move into redis backend
         GLogger.debug 'Joining EM event loop'
         EM.reactor_thread.join
       end

data/lib/gilmour/composers.rb

@@ -0,0 +1,190 @@
+
+module Gilmour
+  module Composers
+    class Request #:nodoc:
+      attr_reader :topic
+      def initialize(backend, spec)
+        @backend = backend
+        @spec = spec
+        if spec.kind_of?(Hash)
+          @message = spec[:message] || spec['message'] || {}
+          @topic = spec[:topic] || spec['topic']
+          if !@topic
+            raise ArgumentError.new("Request topic cannot be empty in a request spec")
+          end
+          @opts = spec[:opts] || spec['opts'] || {}
+          if @opts && !@opts.kind_of?(Hash)
+            raise ArgumentError.new("Request opts must be a Hash")
+          end
+        elsif spec.kind_of?(Proc)
+          @topic = spec.to_s
+        else
+          raise ArgumentError.new("Request spec must be a spec or proc")
+        end
+      end
+
+      def execute(data = {}, &blk)
+        if @spec.kind_of?(Proc)
+          res = @spec.call(data)
+          code = res ? 200 : 500
+          blk.call(res, code)
+        else
+          message = if @message.kind_of?(Hash) && data.kind_of?(Hash)
+                      data.merge(@message)
+                    else
+                      data
+                    end
+          @backend.request!(message, @topic, @opts, &blk)
+        end
+      rescue Exception => e
+        GLogger.debug e.message
+        GLogger.debug e.backtrace
+      end
+    end
+
+    class Pipeline #:nodoc:
+      attr_reader :pipeline
+
+      def initialize(backend, spec)
+        @backend = backend
+        unless spec.kind_of? Array
+          raise ArgumentError.new("Compose spec must be an array")
+        end
+        @pipeline = spec.map do |s|
+          if s.kind_of?(Pipeline) || s.kind_of?(Request)
+            s
+          else
+            Request.new(backend, s)
+          end
+        end
+      end
+
+      def execute
+        raise NotImplementedError.new
+      end
+
+      def continuation(queue)
+        self.class.new(@backend, queue)
+      end
+    end
+
+    class Compose < Pipeline
+      # Execute a pipeline. The output of the last stage of the
+      # pipeline is passed to the block, if all stages are
+      # successful. Otherwise, the output of the last successful stage
+      # is passed, along with the conitnuation which represents the
+      # remaining pipeline
+      def execute(msg = {}, &blk)
+        blk.call(nil, nil) if pipeline.empty?
+        handler = proc do |queue, data, code|
+          if queue.empty? || code != 200
+            blk.call(data, code, continuation(queue))
+          else
+            head = queue.first
+            tail = queue[1..-1]
+            head.execute(data, &handler.curry[tail])
+          end
+        end
+        handler.call(pipeline, msg, 200)
+      end
+    end
+
+    # Create a Compose pipeline as per the spec. This 
+    # is roughly equivalent to the following unix construct
+    #   cmd1 | cmd2 | cmd3 ...
+    # The spec is an array of hashes, each describing
+    # a request topic and message. The message is optional.
+    # If present, this message is merged into the output
+    # of the previous step, before passing it as the input.
+    # Eg, below, if msg2 was a Hash, it would be merged into
+    # the output from the subscriber of topic1, and the merged hash
+    # would be sent as the request body to topic2
+    #    [
+    #      {topic: topic1, message: msg1},
+    #      {topic: topic2, message: msg2},
+    #      ...
+    #    ]
+    #
+    # Instead of a hash, a spec item can also be a callable
+    # (proc or lambda). In that case, the output of the previous step
+    # is passed through the callable, and the return value of the callable
+    # is sent as the input to the next step.
+    #
+    # In place of a hash or callable, it is also possible to have
+    # any kind of composition itself, allowing the creation of
+    # nested compositions. See examples in the source code.
+    #
+    def compose(spec)
+      Compose.new(self, spec)
+    end
+
+    class AndAnd < Pipeline
+      # Execute the andand pipeline. The output of the last successful
+      # step is passed to block, along with the remaining continuation
+      # See the documentation of the #andand method for more details
+      def execute(data={}, &blk)
+        blk.call(nil, nil) if pipeline.empty?
+        handler = proc do |queue, data, code|
+          if queue.empty? || code != 200
+            blk.call(data, code, continuation(queue))
+          else
+            head = queue.first
+            tail = queue[1..-1]
+            head.execute(&handler.curry[tail])
+          end
+        end
+        handler.call(pipeline, nil, 200)
+      end
+    end
+
+    # Same as compose this composition does not pass data from one step to the
+    # next. Execution halts on the first error
+    # It is roughly equivalent to the unix construct
+    #   cmd1 && cmd2 && cmd3
+
+    def andand(spec)
+      AndAnd.new(self, spec)
+    end
+
+    class Batch < Pipeline
+      def initialize(backend, spec, record=false) #:nodoc:
+        super(backend, spec)
+        @record = record
+      end
+
+      # Execute the batch pipeline. This pipeline ignores all errors
+      # step is passed to block.
+      # See the documentation of the #batch method for more details
+      def execute(data={}, &blk)
+        results = []
+        blk.call(nil, nil) if pipeline.empty?
+        handler = proc do |queue, data, code|
+          results << {data: data, code: code} if @record
+          if queue.empty?
+            result = @record ? results[1..-1] : data
+            blk.call(result, code)
+          else
+            head = queue.first
+            tail = queue[1..-1]
+            head.execute(&handler.curry[tail])
+          end
+        end
+        handler.call(pipeline, nil, 200)
+      end
+    end
+
+    # Same a compose, except that no errors are checked
+    # and the pipeline executes all steps unconditionally
+    # and sequentially. It is roughly equivalent to the unix construct
+    #   cmd1; cmd2; cmd3
+    # OR
+    #   (cmd1; cmd2; cmd3)
+    # Params:
+    # +record+:: If this is false, only the output of the last step
+    # is passed to the block passed to execute. If true, all outputs
+    # are collected in an array and passed to the block
+    def batch(spec, record=false)
+      Batch.new(self, spec, record)
+    end
+  end
+end