gilmour 0.2.4

data/lib/gilmour/base.rb

@@ -0,0 +1,178 @@
+# encoding: utf-8
+# This is required to check whether Mash class already exists
+require 'logger'
+require 'securerandom'
+require 'json'
+require 'mash' unless class_exists? 'Mash'
+require 'eventmachine'
+
+require_relative 'protocol'
+require_relative 'responder'
+require_relative 'backends/backend'
+
+# The Gilmour module
+module Gilmour
+  LoggerLevels = {
+    unknown: Logger::UNKNOWN,
+    fatal: Logger::FATAL,
+    error: Logger::ERROR,
+    warn: Logger::WARN,
+    info: Logger::INFO,
+    debug: Logger::DEBUG
+  }
+
+
+  GLogger = Logger.new(STDERR)
+  EnvLoglevel =  ENV["LOG_LEVEL"] ? ENV["LOG_LEVEL"].to_sym : :warn
+  GLogger.level = LoggerLevels[EnvLoglevel] || Logger::DEBUG
+
+  RUNNING = false
+  # This is the base module that should be included into the
+  # container class
+  module Base
+    def self.included(base)
+      base.extend(Registrar)
+    end
+
+    ######### Registration module ###########
+    # This module helps act as a Resistrar for subclasses
+    module Registrar
+      attr_accessor :subscribers_path
+      attr_accessor :backend
+      DEFAULT_SUBSCRIBER_PATH = 'subscribers'
+      @@subscribers = {} # rubocop:disable all
+      @@registered_services = []
+
+      # :nodoc:
+      def inherited(child)
+        @@registered_services << child
+      end
+
+
+      # Returns the subscriber classes registered
+      def registered_subscribers
+        @@registered_services
+      end
+
+      # Adds a listener for the given topic
+      # topic:: The topic to listen to
+      # opts: Hash of optional arguments.
+      #       Supported options are:
+      #
+      #       excl:: If true, this listener is added to a group of listeners
+      #       with the same name as the name of the class in which this method
+      #       is called. A message sent to the _topic_ will be processed by at
+      #       most one listener from a group
+      #
+      #       timeout: Maximum duration (seconds) that a subscriber has to
+      #       finish the task. If the execution exceeds the timeout, gilmour
+      #       responds with status {code:409, data: nil}
+      #
+      def listen_to(topic, opts={})
+        handler = Proc.new
+
+        opt_defaults = {
+          exclusive: false,
+          timeout: 600,
+          fork: false
+        }.merge(opts)
+
+        #Make sure these are not overriden by opts.
+        opt_defaults[:handler] = handler
+        opt_defaults[:subscriber] = self
+
+        @@subscribers[topic] ||= []
+        @@subscribers[topic] << opt_defaults
+      end
+
+      # Returns the list of subscribers for _topic_ or all subscribers if it is nil
+      def subscribers(topic = nil)
+        if topic
+          @@subscribers[topic]
+        else
+          @@subscribers
+        end
+      end
+
+      # Loads all ruby source files inside _dir_ as subscribers
+      # Should only be used inside the parent container class
+      def load_all(dir = nil)
+        dir ||= (subscribers_path || DEFAULT_SUBSCRIBER_PATH)
+        Dir["#{dir}/*.rb"].each { |f| require f }
+      end
+
+      # Loads the ruby file at _path_ as a subscriber
+      # Should only be used inside the parent container class
+      def load_subscriber(path)
+        require path
+      end
+    end
+
+    # :nodoc:
+    def registered_subscribers
+      self.class.registered_subscribers
+    end
+    ############ End Register ###############
+
+    class << self
+      attr_accessor :backend
+    end
+    attr_reader :backends
+
+    # Enable and return the given backend
+    # Should only be used inside the parent container class
+    # If +opts[:multi_process]+ is true, every request handler will
+    # be run inside a new child process.
+    def enable_backend(name, opts = {})
+      Gilmour::Backend.load_backend(name)
+      @backends ||= {}
+      @backends[name] ||= Gilmour::Backend.get(name).new(opts)
+    end
+    alias_method :get_backend, :enable_backend
+
+    def exit!
+      subs_by_backend = subs_grouped_by_backend
+      subs_by_backend.each do |b, subs|
+        backend = get_backend(b)
+        backend.setup_subscribers(subs)
+        if backend.report_health?
+          backend.unregister_health_check
+        end
+      end
+    end
+
+    # Starts all the listeners
+    # If _startloop_ is true, this method will start it's own
+    # event loop and not return till Eventmachine reactor is stopped
+    def start(startloop = false)
+      subs_by_backend = subs_grouped_by_backend
+      subs_by_backend.each do |b, subs|
+        backend = get_backend(b)
+        backend.setup_subscribers(subs)
+
+        if backend.report_health?
+          backend.register_health_check
+        end
+      end
+
+      if startloop
+        GLogger.debug 'Joining EM event loop'
+        EM.reactor_thread.join
+      end
+    end
+
+    private
+
+    def subs_grouped_by_backend
+      subs_by_backend = {}
+      self.class.subscribers.each do |topic, subs|
+        subs.each do |sub|
+          subs_by_backend[sub[:subscriber].backend] ||= {}
+          subs_by_backend[sub[:subscriber].backend][topic] ||= []
+          subs_by_backend[sub[:subscriber].backend][topic] << sub
+        end
+      end
+      subs_by_backend
+    end
+  end
+end

data/lib/gilmour/protocol.rb

@@ -0,0 +1,48 @@
+# encoding: utf-8
+require 'json'
+# Top level Gilmour module
+module Gilmour
+  # This module implements the JSON communication protocol
+  module Protocol
+    def self.parse_request(payload)
+      payload = sanitised_payload(payload)
+      data = sender = nil
+      if payload.kind_of? Hash
+        data = payload['data']
+        sender = payload['sender']
+      else
+        data = payload
+      end
+      [data, sender]
+    end
+
+    def self.parse_response(payload)
+      payload = sanitised_payload(payload)
+      data = sender = nil
+      if payload.kind_of? Hash
+        data = payload['data']
+        sender = payload['sender']
+        code = payload['code']
+      else
+        data = payload
+      end
+      [data, code, sender]
+    end
+
+    def self.create_request(data, code = nil, sender = nil)
+      sender ||= SecureRandom.hex
+      payload = JSON.generate({ 'data' => data, 'code' => code,
+                              'sender' => sender })
+      [payload, sender]
+    end
+
+    def self.sanitised_payload(raw)
+      ret = begin
+              JSON.parse(raw)
+            rescue
+              raw
+            end
+      ret
+    end
+  end
+end

data/lib/gilmour/responder.rb

@@ -0,0 +1,224 @@
+# encoding: utf-8
+
+require "logger"
+
+# Top level module
+module Gilmour
+  # The Responder module that provides the request and respond
+  # DSL
+  # The public methods in this class are available to be called
+  # from the body of the handlers directly
+
+  class Responder
+    attr_reader :logger
+    attr_reader :request
+
+    def make_logger
+      logger = Logger.new(STDERR)
+      original_formatter = Logger::Formatter.new
+      loglevel =  ENV["LOG_LEVEL"] ? ENV["LOG_LEVEL"].to_sym : :warn
+      logger.level = Gilmour::LoggerLevels[loglevel] || Logger::WARN
+      logger.formatter = proc do |severity, datetime, progname, msg|
+        original_formatter.call(severity, datetime, @sender, msg)
+      end
+      logger
+    end
+
+    def initialize(sender, topic, data, backend, timeout=600, forked=false)
+      @sender = sender
+      @request = Mash.new(topic: topic, body: data)
+      @response = { data: nil, code: nil }
+      @backend = backend
+      @timeout = timeout || 600
+      @multi_process = forked || false
+      @pipe = IO.pipe
+      @publish_pipe = IO.pipe
+      @logger = make_logger()
+    end
+
+    def receive_data(data)
+      sender, res_data, res_code = JSON.parse(data)
+      write_response(sender, res_data, res_code) if sender && res_code
+    end
+
+    # Called by parent
+    def write_response(sender, data, code)
+      if code >= 300 && @backend.report_errors?
+        emit_error data, code
+      end
+
+      @backend.send_response(sender, data, code)
+    end
+
+    # Adds a dynamic listener for _topic_
+    def add_listener(topic, &handler)
+      if @multi_process
+        GLogger.error "Dynamic listeners using add_listener not supported \
+        in forked responder. Ignoring!"
+      end
+
+      @backend.add_listener(topic, &handler)
+    end
+
+    # Sends a response with _body_ and _code_
+    # If +opts[:now]+ is true, the response is sent immediately,
+    # else it is defered until the handler finishes executing
+    def respond(body, code = 200, opts = {})
+      @response[:data] = body
+      @response[:code] = code
+      if opts[:now]
+        send_response
+        @response = {}
+      end
+    end
+
+    # Called by parent
+    # :nodoc:
+    def execute(handler)
+      if @multi_process
+        GLogger.debug "Executing #{@sender} in forked moode"
+        @read_pipe = @pipe[0]
+        @write_pipe = @pipe[1]
+
+        @read_publish_pipe = @publish_pipe[0]
+        @write_publish_pipe = @publish_pipe[1]
+
+        pid = Process.fork do
+          @backend.stop
+          EventMachine.stop_event_loop
+          @read_pipe.close
+          @read_publish_pipe.close
+          @response_sent = false
+          _execute(handler)
+        end
+
+        @write_pipe.close
+        @write_publish_pipe.close
+
+        pub_mutex = Mutex.new
+
+        pub_reader = Thread.new {
+          loop {
+            begin
+              data = @read_publish_pipe.readline
+              pub_mutex.synchronize do
+                destination, message = JSON.parse(data)
+                @backend.publish(message, destination)
+              end
+            rescue EOFError
+              # awkward blank rescue block
+            rescue Exception => e
+              GLogger.debug e.message
+              GLogger.debug e.backtrace
+            end
+          }
+        }
+
+        begin
+          receive_data(@read_pipe.readline)
+        rescue EOFError => e
+          logger.debug e.message
+          logger.debug "EOFError caught in responder.rb, because of nil response"
+        end
+
+        pid, status = Process.waitpid2(pid)
+        if !status
+          msg = "Child Process #{pid} crashed without status."
+          logger.error msg
+          # Set the multi-process mode as false, the child has died anyway.
+          @multi_process = false
+          write_response(@sender, msg, 500)
+        elsif status.exitstatus > 0
+          msg = "Child Process #{pid} exited with status #{status.exitstatus}"
+          logger.error msg
+          # Set the multi-process mode as false, the child has died anyway.
+          @multi_process = false
+          write_response(@sender, msg, 500)
+        end
+
+        pub_mutex.synchronize do
+          pub_reader.kill
+        end
+
+        @read_pipe.close
+        @read_publish_pipe.close
+      else
+        _execute(handler)
+      end
+    end
+
+    def emit_error(message, code=500, extra={})
+      # Publish all errors on gilmour.error
+      # This may or may not have a listener based on the configuration
+      # supplied at setup.
+      opts = {
+        :topic => @request[:topic],
+        :data => @request[:data],
+        :description => '',
+        :sender => @sender,
+        :multi_process => @multi_process,
+        :code => 500
+      }.merge(extra || {})
+
+      opts[:timestamp] = Time.now.getutc
+      payload = {:traceback => message, :extra => opts, :code => code}
+      @backend.emit_error payload
+    end
+
+    # Called by child
+    # :nodoc:
+    def _execute(handler)
+      begin
+        Timeout.timeout(@timeout) do
+          instance_eval(&handler)
+        end
+      rescue Timeout::Error => e
+        logger.error e.message
+        logger.error e.backtrace
+        @response[:code] = 504
+        @response[:data] = e.message
+      rescue Exception => e
+        logger.error e.message
+        logger.error e.backtrace
+        @response[:code] = 500
+        @response[:data] = e.message
+      end
+
+      send_response if @response[:code]
+    end
+
+    # Publishes a message. See Backend::publish
+    def publish(message, destination, opts = {}, code=nil)
+      if @multi_process
+        if block_given?
+          GLogger.error "Publish callback not supported in forked responder. Ignoring!"
+#          raise Exception.new("Publish Callback is not supported in forked mode.")
+        end
+
+        msg = JSON.generate([destination, message, code])
+        @write_publish_pipe.write(msg+"\n")
+        @write_publish_pipe.flush
+      elsif block_given?
+        blk = Proc.new
+        @backend.publish(message, destination, opts, &blk)
+      else
+        @backend.publish(message, destination, opts)
+      end
+    end
+
+    # Called by child
+    # :nodoc:
+    def send_response
+      return if @response_sent
+      @response_sent = true
+
+      if @multi_process
+        msg = JSON.generate([@sender, @response[:data], @response[:code]])
+        @write_pipe.write(msg+"\n")
+        @write_pipe.flush # This flush is very important
+      else
+        write_response(@sender, @response[:data], @response[:code])
+      end
+    end
+  end
+end

data/lib/gilmour/waiter.rb

@@ -0,0 +1,20 @@
+module Gilmour
+  class Waiter
+    def initialize
+      @waiter_m = Mutex.new
+      @waiter_c = ConditionVariable.new
+    end
+
+    def synchronize(&blk)
+      @waiter_m.synchronize(&blk)
+    end
+
+    def signal
+      synchronize { @waiter_c.signal }
+    end
+
+    def wait(timeout=nil)
+      synchronize { @waiter_c.wait(@waiter_m, timeout) }
+    end
+  end
+end

data/lib/gilmour.rb

@@ -0,0 +1,11 @@
+# encoding: utf-8
+
+# This is required to check whether Mash class already exists
+def class_exists?(class_name)
+      klass = Module.const_get(class_name)
+      return klass.is_a?(Class)
+rescue NameError
+      return false
+end
+
+require_relative 'gilmour/base'

data/test/spec/helpers/common.rb

@@ -0,0 +1,25 @@
+
+class Waiter
+  def initialize
+    @waiter_m = Mutex.new
+    @waiter_c = ConditionVariable.new
+  end
+
+  def synchronize(&blk)
+    @waiter_m.synchronize(&blk)
+  end
+
+  def signal
+    synchronize { @waiter_c.signal }
+  end
+
+  def wait(timeout=nil)
+    synchronize { @waiter_c.wait(@waiter_m, timeout) }
+  end
+end
+
+RSpec.configure do |config|
+  config.expect_with :rspec do |c|
+    c.syntax = [:should, :expect]
+  end
+end

data/test/spec/helpers/connection.rb

@@ -0,0 +1,58 @@
+require 'yaml'
+require 'json'
+require 'securerandom'
+require 'fiber'
+require '../lib/gilmour/protocol'
+require 'em-hiredis'
+
+require_relative 'common'
+
+def options(which)
+  data = YAML::load(File.open("#{File.dirname(__FILE__)}/data.yml"))
+  data[which]
+end
+
+def redis_connection_options
+  options(:connection)
+end
+
+def redis_ping_options
+  options(:ping)
+end
+
+def redis_wildcard_options
+  options(:wildcard)
+end
+
+def redis_publish_async(options, message, key)
+  operation = proc do
+    redis = EM::Hiredis.connect
+    payload, _ = Gilmour::Protocol.create_request(message)
+    redis.publish(key, payload)
+  end
+  EM.defer(operation)
+end
+
+def redis_send_and_recv(options, message, key)
+  waiter = Waiter.new
+  response = code = nil
+  operation = proc do
+    redis = EM::Hiredis.connect
+    payload, sender = Gilmour::Protocol.create_request(message)
+    response_topic = "gilmour.response.#{sender}"
+    redis.pubsub.subscribe(response_topic)
+    redis.pubsub.on(:message) do |topic, data|
+      begin
+        response, code, _ = Gilmour::Protocol.parse_response(data)
+        waiter.signal
+      rescue Exception => e
+        $stderr.puts e.message
+      end
+    end
+    redis.publish(key, payload)
+  end
+  EM.defer(operation)
+  waiter.wait
+  [response, code]
+end
+

data/test/spec/helpers/data.yml

@@ -0,0 +1,12 @@
+---
+:connection:
+  :host: localhost
+  :exchange: testtopicexchange
+:ping:
+  :message: Ping!
+  :response: Pong!
+:wildcard:
+  :message: Wild!
+  :topic: test.wildcard.foo
+
+

data/test/spec/test_service_base.rb

@@ -0,0 +1,48 @@
+# encoding: utf-8
+
+require 'rspec/given'
+require 'amqp'
+require 'redis'
+
+require_relative 'helpers/connection'
+require './testservice/test_service_base'
+
+describe TestServiceBase do
+  Given(:subscriber) { TestServiceBase }
+  Then { subscriber.should respond_to(:subscribers) }
+  Then { subscriber.subscribers.should be_kind_of(Hash) }
+
+  context 'Load existing subscribers' do
+    modules_dir = './testservice/subscribers'
+    modules = Dir["#{modules_dir}/*.rb"]
+    When { subscriber.load_all(modules_dir) }
+    Then do
+      subscribers = subscriber.subscribers.map do |topic, handlers|
+        handlers.map { |handler| handler[:subscriber] }
+      end.flatten.uniq
+      subscribers.size.should == modules.size
+    end
+  end
+#  context 'Connect to AMQP' do
+#    after(:all) do
+#      AMQP.stop
+#      EM.stop
+#    end
+#    Given(:subscriber) { TestServiceBase.new(amqp_connection_options, 'amqp') }
+#    Given(:backend) { subscriber.backends['amqp'] }
+#    Then { backend.connection.should be_kind_of AMQP::Session }
+#    And  { backend.connection.connected?.should be_true }
+#    And  { backend.channel.should be_kind_of AMQP::Channel }
+#    And  { backend.exchange.should be_kind_of AMQP::Exchange }
+#    And  { backend.exchange.type.should == :topic }
+#  end
+
+  context 'Connect to Redis' do
+    Given(:subscriber) do
+      TestServiceBase.new(redis_connection_options, 'redis')
+    end
+    Given(:backend) { subscriber.backends['redis'] }
+    Then { backend.subscriber.should be_kind_of EM::Hiredis::PubsubClient }
+    And  { backend.publisher.should be_kind_of EM::Hiredis::Client }
+  end
+end