gilmour 0.2.4

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ed168f421b891494b1f49ad0a572009604d4fe1c
+  data.tar.gz: 7a9fad0b5dfc987d2fd08cb3f19b3d611fd3938a
+SHA512:
+  metadata.gz: acf59c9b55719ae57545085728705a1d300485e6a7c9d539655807eb1055a7cb826a2255f4617da775cb1853b9d61a0185bd5cddbdcb272f3fd011b5d2e147f1
+  data.tar.gz: 0ca69b252a271f54da89da5fdd6749ab9f655262f65c9593e2c7253e29605cb98695c6233def5568e464a48ab823bd0832b7daac7553cf8e15752064dd03afba

data/.gitignore

@@ -0,0 +1,5 @@
+.ruby-version
+*.rdb
+*.swp
+*.gem
+

data/.travis.yml

@@ -0,0 +1,16 @@
+language: ruby
+
+rvm:
+  - 2.1.5
+
+services:
+  - redis-server
+
+install:
+  - bundle install
+
+script:
+  - cd test/
+  - bundle exec rspec spec/test_service_base.rb -b --format documentation
+  - bundle exec rspec spec/test_subscriber_redis.rb -b --format documentation
+  - bundle exec rspec spec/test_subscriber_redis_forked.rb -b --format documentation

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+gemspec
+

data/LICENSE

@@ -0,0 +1,19 @@
+Copyright (C) 2013 Aditya Godbole
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,32 @@
+# Gilmour
+
+Gilmour is a framework for writing micro-services that exchange data over
+non-http transports. Currently the supported backend is Redis PubSub.
+Redis pubsub channels are used like "routes".
+The DSL provided is similar to Sinatra.
+
+## Protocol
+
+Gilmour uses it's own simple protocol to send request and response "headers".
+The structure of the payload is a simple JSON as shown below:
+
+    {
+	  data: The actual payload,
+	  sender: The origin of the request (unique for each request),
+	  code: The respoonse code if this is a response
+    }
+
+The `sender` field actually represents a unique sender key. Any reponse that
+is to be sent to the request is sent on the "reservered" topic
+`response.<sender>` on the same exchange.
+
+## Usage Examples
+
+See the `examples` directory for examples of usage
+
+## Specs
+
+To run the specs, set `REDIS_HOST` (default is localhost) and `REDIS_PORT` (default is 6379)
+to point to your rabbitmq or redis server.
+Then, from within the `test` directory, run `rspec spec/*`
+

data/examples/echoclient.rb

@@ -0,0 +1,22 @@
+require 'securerandom'
+require 'gilmour/backends/redis'
+
+def redis_send_and_recv(message, key)
+  redis = Gilmour::RedisBackend.new({})
+  redis.setup_subscribers({})
+  count = 0
+  loop do
+    waiter = Thread.new { loop { sleep 1 } }
+    newkey = "#{key}.#{SecureRandom.hex(2)}"
+    redis.publish(count, newkey) do |data, code|
+      puts "Client got response: #{code}: #{data}"
+      waiter.kill
+    end
+    count = count + 1
+    waiter.join
+    sleep 1
+  end
+end
+
+redis_send_and_recv('Ping', 'echo')
+

data/examples/server.rb

@@ -0,0 +1,47 @@
+# encoding: utf-8
+require 'gilmour'
+
+class EventServer
+  include Gilmour::Base
+
+  def initialize
+    backend = 'redis'
+    enable_backend(backend, { })
+    registered_subscribers.each do |sub|
+      sub.backend = backend
+    end
+    $stderr.puts "Starting server. To see messaging in action run clients."
+    start(true)
+  end
+end
+
+class EchoSubscriber < EventServer
+  # Passing second parameter as true makes only one instance of this handler handle a request
+  listen_to 'echo.*', {"exclusive" => true} do
+    if request.body == 'Palmolive'
+      respond nil
+    else
+      $stderr.puts request.body
+      respond "#{request.topic}"
+    end
+  end
+end
+
+class FibonacciSubscriber < EventServer
+  class << self
+    attr_accessor :last
+  end
+
+  listen_to 'fib.next' do
+    old = FibonacciSubscriber.last
+    FibonacciSubscriber.last = new = request.body
+    respond(old + new)
+  end
+
+  listen_to 'fib.init' do
+    FibonacciSubscriber.last = request.body
+  end
+
+end
+
+EventServer.new

data/examples/thread_example.rb

@@ -0,0 +1,40 @@
+require 'securerandom'
+require 'gilmour/backends/redis'
+
+def redis_send_and_recv(message, key, ident)
+  redis = Gilmour::RedisBackend.new({})
+  redis.setup_subscribers({})
+  #loop do
+    waiter = Thread.new { loop { sleep 1 } }
+    newkey = "#{key}.#{SecureRandom.hex(2)}"
+    puts "Process: #{ident} Sending: #{newkey}"
+    redis.publish(ident, newkey) do |data, code|
+      puts "Process: #{ident} Received: #{data}"
+      waiter.kill
+    end
+    waiter.join
+  #end
+end
+
+def fork_and_run(num)
+  pid_array = []
+
+  num.times do |i|
+    pid = Process.fork do
+      puts "Process #{i}"
+      yield i
+    end
+
+    pid_array.push(pid)
+  end
+
+  pid_array.each do |pid|
+    Process.waitpid(pid)
+  end
+
+end
+
+fork_and_run(5) do |i|
+  # Start echo server first
+  redis_send_and_recv('Ping', 'echo', i)
+end

data/gilmour.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "./version"
+
+Gem::Specification.new do |s|
+	s.name        = "gilmour"
+	s.version     = Gilmour::VERSION
+	s.platform    = Gem::Platform::RUBY
+	s.authors     = ["Aditya Godbole"]
+	s.email       = ["code.aa@gdbl.me"]
+	s.homepage    = ""
+	s.summary     = %q{A Sinatra like DSL for implementing AMQP services}
+	s.description = %q{This gem provides a Sinatra like DSL and a simple protocol to enable writing services that communicate over AMQP}
+
+	s.add_development_dependency "rspec"
+	s.add_development_dependency "rspec-given"
+	s.add_dependency "mash"
+	s.add_dependency "redis"
+	s.add_dependency "gilmour-em-hiredis"
+	s.add_dependency "amqp"
+
+	s.files         = `git ls-files`.split("\n")
+	s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+	s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+	s.require_paths = ["lib"]
+end

data/lib/gilmour/backends/backend.rb

@@ -0,0 +1,148 @@
+# encoding: utf-8
+require 'socket'
+require 'securerandom'
+
+require_relative '../protocol'
+
+module Gilmour
+  ErrorChannel = "gilmour.error"
+
+  # Base class for loading backends
+  class Backend
+    SUPPORTED_BACKENDS = %w(redis)
+    @@registry = {}
+
+    def ident
+      @ident
+    end
+
+    def generate_ident
+      "#{Socket.gethostname}-pid-#{Process.pid}-uuid-#{SecureRandom.uuid}"
+    end
+
+    def report_errors?
+      #Override this method to adjust if you want errors to be reported.
+      return true
+    end
+
+    def initialize(opts={})
+      @ident = generate_ident
+    end
+
+    def register_health_check
+      raise NotImplementedError.new
+    end
+
+    def unregister_health_check
+      raise NotImplementedError.new
+    end
+
+    def self.implements(backend_name)
+      @@registry[backend_name] = self
+    end
+
+    def self.get(backend_name)
+      @@registry[backend_name]
+    end
+
+    # :nodoc:
+    # This should be implemented by the derived class
+    # subscriptions is a hash in the format -
+    # { topic => [handler1, handler2, ...],
+    #   topic2 => [handler3, handler4, ...],
+    #   ...
+    # }
+    # where handler is a hash
+    # { :handler => handler_proc,
+    #   :subscriber => subscriber_derived_class
+    # }
+    def setup_subscribers(subscriptions)
+    end
+
+    # Sends a message
+    # If optional block is given, it will be executed when a response is received
+    # or if timeout occurs
+    # +message+:: The body of the message (any object that is serialisable)
+    # +destination+:: The channel to post to
+    # +opts+::
+    #   ++timeout+:: Sender side timeout
+    #
+    def publish(message, destination, opts = {}, code = 0, &blk)
+      payload, sender = Gilmour::Protocol.create_request(message, code)
+
+      EM.defer do # Because publish can be called from outside the event loop
+        begin
+          send(sender, destination, payload, opts, &blk)
+        rescue Exception => e
+          GLogger.debug e.message
+          GLogger.debug e.backtrace
+        end
+      end
+      sender
+    end
+
+    def emit_error(message)
+      raise NotImplementedError.new
+    end
+
+    # Adds a new handler for the given _topic_
+    def add_listener(topic, &handler)
+      raise "Not implemented by child class"
+    end
+
+    # Removes existing _handler_ for the _topic_
+    def remove_listener(topic, &handler)
+      raise "Not implemented by child class"
+    end
+
+    def acquire_ex_lock(sender)
+      raise "Not implemented by child class"
+    end
+
+    def send_response(sender, body, code)
+      raise "Not implemented by child class"
+    end
+
+    def execute_handler(topic, payload, sub)
+      data, sender = Gilmour::Protocol.parse_request(payload)
+      if sub[:exclusive]
+        lock_key = sender + sub[:subscriber].to_s
+        acquire_ex_lock(lock_key) { _execute_handler(topic, data, sender, sub) }
+      else
+        _execute_handler(topic, data, sender, sub)
+      end
+    rescue Exception => e
+      GLogger.debug e.message
+      GLogger.debug e.backtrace
+    end
+
+    def _execute_handler(topic, data, sender, sub)
+      Gilmour::Responder.new(
+        sender, topic, data, self, sub[:timeout], sub[:fork]
+      ).execute(sub[:handler])
+    rescue Exception => e
+      GLogger.debug e.message
+      GLogger.debug e.backtrace
+    end
+
+    def send
+      raise "Not implemented by child class"
+    end
+
+    def self.load_backend(name)
+      require_relative name
+    end
+
+    def self.load_all_backends
+      SUPPORTED_BACKENDS.each do |f|
+        load_backend f
+      end
+    end
+
+    def stop(sender, body, code)
+      raise "Not implemented by child class"
+    end
+
+  end
+end
+

data/lib/gilmour/backends/redis.rb

@@ -0,0 +1,254 @@
+require 'em-hiredis'
+require_relative 'backend'
+require_relative '../waiter'
+
+module Gilmour
+  # Redis backend implementation
+  class RedisBackend < Backend
+    GilmourHealthKey = "gilmour.known_host.health"
+    GilmourErrorBufferLen = 9999
+
+    implements 'redis'
+
+    attr_writer :report_errors
+    attr_reader :subscriber
+    attr_reader :publisher
+
+    def redis_host(opts)
+      host = opts[:host] || '127.0.0.1'
+      port = opts[:port] || 6379
+      db = opts[:db] || 0
+      "redis://#{host}:#{port}/#{db}"
+    end
+
+    def initialize(opts)
+      @response_handlers = {}
+      @subscriptions = {}
+
+      waiter = Waiter.new
+
+      Thread.new do
+        EM.run do
+          setup_pubsub(opts)
+          waiter.signal
+        end
+      end
+
+      waiter.wait
+
+      @report_health = opts["health_check"] || opts[:health_check]
+      @report_health = false if @report_health != true
+
+      @report_errors = opts["broadcast_errors"] || opts[:broadcast_errors]
+      @report_errors = true if @report_errors != false
+    end
+
+    def report_health?
+      @report_health
+    end
+
+    def report_errors?
+      @report_errors
+    end
+
+    def emit_error(message)
+      report = self.report_errors?
+
+      if report == false
+        Glogger.debug "Skipping because report_errors is false"
+      elsif report == true
+        publish_error message
+      elsif report.is_a? String and !report.empty?
+        queue_error report, message
+      end
+    end
+
+    def setup_pubsub(opts)
+      @publisher = EM::Hiredis.connect(redis_host(opts))
+      @subscriber = @publisher.pubsub_client
+      register_handlers
+    rescue Exception => e
+      GLogger.debug e.message
+      GLogger.debug e.backtrace
+    end
+
+    def register_handlers
+      @subscriber.on(:pmessage) do |key, topic, payload|
+        pmessage_handler(key, topic, payload)
+      end
+      @subscriber.on(:message) do |topic, payload|
+        begin
+        if topic.start_with? 'gilmour.response.'
+          response_handler(topic, payload)
+        else
+          pmessage_handler(topic, topic, payload)
+        end
+        rescue Exception => e
+          GLogger.debug e.message
+          GLogger.debug e.backtrace
+        end
+      end
+    end
+
+    def subscribe_topic(topic)
+      method = topic.index('*') ? :psubscribe : :subscribe
+      @subscriber.method(method).call(topic)
+    end
+
+    def pmessage_handler(key, matched_topic, payload)
+      @subscriptions[key].each do |subscription|
+        EM.defer(->{execute_handler(matched_topic, payload, subscription)})
+      end
+    end
+
+    def register_response(sender, handler, timeout = 600)
+      topic = "gilmour.response.#{sender}"
+      timer = EM::Timer.new(timeout) do # Simulate error response
+        GLogger.info "Timeout: Killing handler for #{sender}"
+        payload, _ = Gilmour::Protocol.create_request({}, 499)
+        response_handler(topic, payload)
+      end
+      @response_handlers[topic] = {handler: handler, timer: timer}
+      subscribe_topic(topic)
+    rescue Exception => e
+      GLogger.debug e.message
+      GLogger.debug e.backtrace
+    end
+
+    def publish_error(messsage)
+      @publisher.publish(Gilmour::ErrorChannel, messsage)
+    end
+
+    def queue_error(key, message)
+      @publisher.lpush(key, message) do
+        @publisher.ltrim(key, 0, GilmourErrorBufferLen) do
+          Glogger.debug "Error queued"
+        end
+      end
+    end
+
+    def acquire_ex_lock(sender)
+      @publisher.set(sender, sender, 'EX', 600, 'NX') do |val|
+        EM.defer do
+          yield val if val && block_given?
+        end
+      end
+    end
+
+    def response_handler(sender, payload)
+      data, code, _ = Gilmour::Protocol.parse_response(payload)
+      handler = @response_handlers.delete(sender)
+      @subscriber.unsubscribe(sender)
+      if handler
+        handler[:timer].cancel
+        handler[:handler].call(data, code)
+      end
+    rescue Exception => e
+      GLogger.debug e.message
+      GLogger.debug e.backtrace
+    end
+
+    def send_response(sender, body, code)
+      publish(body, "gilmour.response.#{sender}", {}, code)
+    end
+
+    def setup_subscribers(subs = {})
+      @subscriptions.merge!(subs)
+      EM.defer do
+        subs.keys.each { |topic| subscribe_topic(topic) }
+      end
+    end
+
+    def add_listener(topic, &handler)
+      @subscriptions[topic] ||= []
+      @subscriptions[topic] << { handler: handler }
+      subscribe_topic(topic)
+    end
+
+    def remove_listener(topic, handler = nil)
+      if handler
+        subs = @subscriptions[topic]
+        subs.delete_if { |e| e[:handler] == handler }
+      else
+        @subscriptions[topic] = []
+      end
+      @subscriber.unsubscribe(topic) if @subscriptions[topic].empty?
+    end
+
+    def send(sender, destination, payload, opts = {}, &blk)
+      timeout = opts[:timeout] || 600
+      if opts[:confirm_subscriber]
+        confirm_subscriber(destination) do |present|
+          if !present
+            blk.call(nil, 404) if blk
+          else
+            _send(sender, destination, payload, timeout, &blk)
+          end
+        end
+      else
+        _send(sender, destination, payload, timeout, &blk)
+      end
+    rescue Exception => e
+      GLogger.debug e.message
+      GLogger.debug e.backtrace
+    end
+
+    def _send(sender, destination, payload, timeout, &blk)
+      register_response(sender, blk, timeout) if block_given?
+      @publisher.publish(destination, payload)
+      sender
+    end
+
+    def confirm_subscriber(dest, &blk)
+      @publisher.pubsub('numsub', dest) do |_, num|
+        blk.call(num.to_i > 0)
+      end
+    rescue Exception => e
+      GLogger.debug e.message
+      GLogger.debug e.backtrace
+    end
+
+    def stop
+      @subscriber.close_connection
+    end
+
+    # TODO: Health checks currently use Redis to keep keys in a data structure.
+    # An alternate approach would be that monitor subscribes to a topic
+    # and records nodenames that request to be monitored. The publish method
+    # should fail if there is no definite health monitor listening. However,
+    # that would require the health node to be running at all points of time
+    # before a Gilmour server starts up. To circumvent this dependency, till
+    # monitor is stable enough, use Redis to save/share these data structures.
+    #
+    def register_health_check
+      @publisher.hset GilmourHealthKey, self.ident, 'active'
+
+      # - Start listening on a dyanmic topic that Health Monitor can publish
+      # on.
+      #
+      # NOTE: Health checks are not run as forks, to ensure that event-machine's
+      # ThreadPool has sufficient resources to handle new requests.
+      #
+      topic = "gilmour.health.#{self.ident}"
+      add_listener(topic) do
+        respond @subscriptions.keys
+      end
+
+      # TODO: Need to do these manually. Alternate is to return the handler
+      # hash from add_listener.
+      @subscriptions[topic][0][:exclusive] = true
+
+    end
+
+    def unregister_health_check
+      waiter = Waiter.new
+
+      @publisher.hdel(GilmourHealthKey, self.ident) do
+        waiter.signal
+      end
+
+      waiter.wait(5)
+    end
+
+  end
+end