mqrpc 0.0.1

data/lib/mqrpc/agent.rb

@@ -0,0 +1,232 @@
+require 'rubygems'
+require 'amqp'
+require 'mq'
+require 'mqrpc/logger'
+require 'mqrpc/operation'
+require 'thread'
+require 'uuid'
+
+# http://github.com/tmm1/amqp/issues/#issue/3
+# This is our (lame) hack to at least notify the user that something is
+# wrong.
+module AMQP
+  module Client
+    alias :original_reconnect :reconnect 
+    def reconnect(*args)
+      $logger.warn "reconnecting to broker (bad MQ settings?)"
+
+      # some rate limiting
+      sleep(5)
+
+      original_reconnect(*args)
+    end
+  end
+end
+
+module MQRPC
+  # TODO: document this class
+  class Agent
+    MAXBUF = 20
+
+    def initialize(config)
+      Thread::abort_on_exception = true
+      @config = config
+      @handler = self
+      @id = UUID::generate
+      @outbuffer = Hash.new { |h,k| h[k] = [] }
+      @queues = []
+      @receive_queue = Queue.new
+      @topics = []
+      @want_queues = []
+      @want_topics = []
+      #@slidingwindow = LogStash::SlidingWindowSet.new
+
+      @mq = nil
+      @message_operations = Hash.new
+
+      @startup_mutex = Mutex.new
+      @startup_condvar = ConditionVariable.new
+      @amqp_ready = false
+
+      start_amqp
+      @startup_mutex.synchronize do
+        MQRPC::logger.debug "Waiting for @mq ..."
+        @startup_condvar.wait(@startup_mutex) if !@amqp_ready
+      end
+
+      start_receiver
+    end # def initialize
+
+    def start_amqp
+      @amqpthread = Thread.new do 
+        # Create connection to AMQP, and in turn, the main EventMachine loop.
+        amqp_config = {:host => @config.mqhost,
+                       :port => @config.mqport,
+                       :user => @config.mquser,
+                       :pass => @config.mqpass,
+                       :vhost => @config.mqvhost,
+                      }
+        AMQP.start(amqp_config) do
+          @startup_mutex.synchronize do
+            @mq = MQ.new
+            # Notify the main calling thread (MessageSocket#initialize) that
+            # we can continue
+            @amqp_ready = true
+            @startup_condvar.signal
+          end
+
+          MQRPC::logger.info "Subscribing to main queue #{@id}"
+          mq_q = @mq.queue(@id, :auto_delete => true)
+          mq_q.subscribe(:ack =>true) { |hdr, msg| @receive_queue << [hdr, msg] }
+          handle_new_subscriptions
+          
+          # TODO(sissel): make this a deferred thread that reads from a Queue
+          EM.add_periodic_timer(5) { handle_new_subscriptions }
+
+          EM.add_periodic_timer(1) do
+            # TODO(sissel): add locking
+            @outbuffer.each_key { |dest| flushout(dest) }
+            @outbuffer.clear
+          end
+        end # AMQP.start
+      end
+    end # def start_amqp
+
+    def start_receiver
+      Thread.new do 
+        while true
+          header, message = @receive_queue.pop
+          handle_message(header, message)
+        end
+      end
+    end # def start_receiver
+
+    def subscribe(name)
+      @want_queues << name 
+    end # def subscribe
+
+    def subscribe_topic(name)
+      @want_topics << name 
+    end # def subscribe_topic
+
+    def handle_message(hdr, msg_body)
+      obj = JSON::load(msg_body)
+      if !obj.is_a?(Array)
+        obj = [obj]
+      end
+
+      obj.each do |item|
+        message = Message.new_from_data(item)
+        #if @slidingwindow.include?(message.id)
+          #puts "Removing ack for #{message.id}"
+          #@slidingwindow.delete(message.id)
+        #end
+        name = message.class.name.split(":")[-1]
+        func = "#{name}Handler"
+
+        # Check if we have a specific operation looking for this
+        # message.
+        if (message.respond_to?(:in_reply_to) and
+            @message_operations.has_key?(message.in_reply_to))
+          operation = @message_operations[message.in_reply_to]
+          operation.call message
+        elsif @handler.respond_to?(func) 
+          @handler.send(func, message) do |response|
+            reply = message.reply_to
+            sendmsg(reply, response)
+          end
+
+          # We should allow the message handler to defer acking if they want
+          # For instance, if we want to index things, but only want to ack
+          # things once we actually flush to disk.
+        else
+          $stderr.puts "#{@handler.class.name} does not support #{func}"
+        end # if @handler.respond_to?(func)
+      end
+      hdr.ack
+    end # def handle_message
+
+    def run
+      @amqpthread.join
+    end # run
+
+    def handle_new_subscriptions
+      todo = @want_queues - @queues
+      todo.each do |queue|
+        MQRPC::logger.info "Subscribing to queue #{queue}"
+        mq_q = @mq.queue(queue, :durable => true)
+        mq_q.subscribe(:ack => true) { |hdr, msg| @receive_queue << [hdr, msg] }
+        @queues << queue
+      end # todo.each
+
+      todo = @want_topics - @topics
+      todo.each do |topic|
+        MQRPC::logger.info "Subscribing to topic #{topic}"
+        exchange = @mq.topic("amq.topic")
+        mq_q = @mq.queue("#{@id}-#{topic}",
+                         :exclusive => true,
+                         :auto_delete => true).bind(exchange, :key => topic)
+        mq_q.subscribe { |hdr, msg| @receive_queue << [hdr, msg] }
+        @topics << topic
+      end # todo.each
+    end # handle_new_subscriptions
+
+    def flushout(destination)
+      return unless @mq    # wait until we are connected
+
+      msgs = @outbuffer[destination]
+      return if msgs.length == 0
+      data = msgs.to_json
+      @mq.queue(destination, :durable => true).publish(data, :persistent => true)
+      msgs.clear
+    end
+
+    def sendmsg_topic(key, msg)
+      return unless @mq    # wait until we are connected
+      if (msg.is_a?(RequestMessage) and msg.id == nil)
+        msg.generate_id!
+      end
+      msg.timestamp = Time.now.to_f
+
+      data = msg.to_json
+      @mq.topic("amq.topic").publish(data, :key => key)
+    end
+
+    def sendmsg(destination, msg, &callback)
+      return unless @mq    # wait until we are connected
+      if (msg.is_a?(RequestMessage) and msg.id == nil)
+        msg.generate_id!
+      end
+      msg.timestamp = Time.now.to_f
+      msg.reply_to = @id
+
+      #if (msg.is_a?(RequestMessage) and !msg.is_a?(ResponseMessage))
+        #MQRPC::logger.info "Tracking #{msg.class.name}##{msg.id}"
+        #@slidingwindow << msg.id
+      #end
+
+      if msg.buffer?
+        @outbuffer[destination] << msg
+        if @outbuffer[destination].length > MAXBUF
+          flushout(destination)
+        end
+      else
+        @mq.queue(destination, :durable => true).publish([msg].to_json, :persistent => true)
+      end
+
+      if block_given?
+        op = Operation.new(callback)
+        @message_operations[msg.id] = op
+        return op
+      end
+    end
+
+    def handler=(handler)
+      @handler = handler
+    end
+
+    def close
+      EM.stop_event_loop
+    end
+  end # class Agent
+end # module MQRPC

data/lib/mqrpc/config.rb

@@ -0,0 +1,17 @@
+module MQRPC
+  class Config
+    attr_reader :mqhost
+    attr_reader :mqport
+    attr_reader :mquser
+    attr_reader :mqpass
+    attr_reader :mqvhost
+
+    def initialize(options = {})
+      @mqhost = options["mqhost"] || "localhost"
+      @mqport = options["mqport"] || 5672
+      @mquser = options["mquser"] || "guest"
+      @mqpass = options["mqpass"] || "guest"
+      @mqvhost = options["mqvhost"] || "/"
+    end # def initialize
+  end # class Config
+end # module MQRPC

data/lib/mqrpc/functions/ping.rb

@@ -0,0 +1,12 @@
+require 'rubygems'
+require 'mqrpc/messages/ping'
+
+module MQRPC; module Functions; module Ping
+  def PingRequestHandler(request)
+    MQRPC::logger.debug "received PingRequest (#{request.pingdata})"
+    response = MQRPC::Messages::PingResponse.new
+    response.id = request.id
+    response.pingdata = request.pingdata
+    yield response
+  end
+end; end; end

data/lib/mqrpc/logger.rb

@@ -0,0 +1,15 @@
+require 'rubygems'
+require 'logger'
+
+module MQRPC
+  @logger = Logger.new(STDOUT)
+  @logger.level = Logger::WARN
+
+  def self.logger
+    return @logger
+  end
+
+  def self.logger=(logger)
+    @logger = logger
+  end
+end

data/lib/mqrpc/message.rb

@@ -0,0 +1,142 @@
+require 'json'
+require 'thread'
+require 'mqrpc/logger'
+
+module BindToHash
+  def header(method, key=nil)
+    key = method.to_s if key == nil
+    hashbind(method, "/#{key}")
+  end
+
+  def argument(method, key=nil)
+    key = method.to_s if key == nil
+    hashbind(method, "/args/#{key}")
+  end
+
+  private
+  def hashbind(method, key)
+    hashpath = __genhashpath(key)
+    self.class_eval %(
+      def #{method}
+        return #{hashpath}
+      end
+      def #{method}=(val)
+        #{hashpath} = val
+      end
+    )
+  end
+
+  def __genhashpath(key)
+    # TODO(sissel): enforce 'key' needs to be a string or symbol?
+    path = key.split("/").select { |x| x.length > 0 }\
+               .map { |x| "[#{x.inspect}]" }
+    return "@data#{path.join("")}"
+  end
+end # modules BindToHash
+
+module MQRPC
+  class Message
+    extend BindToHash
+
+    @@idseq = 0
+    @@idlock = Mutex.new
+    @@knowntypes = Hash.new
+    attr_accessor :data
+
+    # Message attributes
+    header :id
+    header :message_class
+    header :reply_to
+    header :timestamp
+
+    def initialize
+      generate_id!
+    end
+
+    def generate_id!
+      @@idlock.synchronize do
+        self.id = @@idseq
+        @@idseq += 1
+      end
+    end
+
+    def age
+      return Time.now.to_f - timestamp
+    end
+
+    def buffer?
+      return @buffer
+    end
+
+    def want_buffer(want_buffer=true)
+      @buffer = want_buffer
+    end
+
+    def self.inherited(subclass)
+      MQRPC::logger.debug "Message '#{subclass.name}' subclasses #{self.name}"
+      @@knowntypes[subclass.name] = subclass
+
+      # Call the class initializer if it has one.
+      if subclass.respond_to?(:class_initialize)
+        subclass.class_initialize
+      end
+    end # def self.inherited
+
+    def initialize
+      @data = Hash.new
+      want_buffer(false)
+      self.message_class = self.class.name
+    end
+
+    def self.new_from_data(data)
+      obj = nil
+      name = data["message_class"]
+      if @@knowntypes.has_key?(name)
+        obj = @@knowntypes[name].new
+      else
+        $stderr.puts "No known message class: #{name}, #{data.inspect}"
+        obj = Message.new
+      end
+      obj.data = data
+      return obj
+    end
+
+    def to_json(*args)
+      return @data.to_json(*args)
+    end
+
+    protected
+    attr :data
+  end # class Message
+
+  class RequestMessage < Message
+    header :args
+
+    def initialize
+      super
+      self.args = Hash.new
+    end
+
+  end # class RequestMessage
+
+  class ResponseMessage < Message
+    header :in_reply_to
+    header :args
+
+    def initialize(source_request=nil)
+      super()
+
+      # Copy the request id if we are given a source_request
+      if source_request.is_a?(RequestMessage)
+        self.in_reply_to = source_request.id
+      end
+      self.args = Hash.new
+    end
+
+    # Report the success of the request this response is for.
+    # Should be implemented by subclasses.
+    def success?
+      raise NotImplementedError
+    end
+  end # class ResponseMessage
+end # module MQRPC

data/lib/mqrpc/messages/ping.rb

@@ -0,0 +1,17 @@
+require 'rubygems'
+require 'lib/mqrpc'
+
+module MQRPC::Messages
+  class PingRequest < MQRPC::RequestMessage
+    def initialize
+      super
+      self.pingdata = Time.now.to_f
+    end
+
+    hashbind :pingdata, "/args/pingdata"
+  end # class PingRequest < RequestMessage
+
+  class PingResponse < MQRPC::ResponseMessage
+    hashbind :pingdata, "/args/pingdata"
+  end
+end

data/lib/mqrpc/operation.rb

@@ -0,0 +1,49 @@
+require 'rubygems'
+require 'thread'
+require 'mqrpc/logger'
+
+module MQRPC
+  # A single message operation
+  # * Takes a callback to call when a message is received
+  # * Allows you to wait for the operation to complete.
+  # * An operation is 'complete' when the callback returns :finished
+  class Operation
+    def initialize(callback)
+      @mutex = Mutex.new
+      @callback = callback
+      @cv = ConditionVariable.new
+      @finished = false
+    end # def initialize
+
+    def call(*args)
+      # TODO(sissel): allow the callback to simply invoke 'finished' on this
+      # operation rather than requiring it to emit ':finished'
+      @mutex.synchronize do
+        ret = @callback.call(*args)
+        if ret == :finished
+          MQRPC::logger.debug "operation #{self} finished"
+          @finished = true
+          @cv.signal
+        else
+          return ret
+        end
+      end
+    end # def call
+
+    # Block until the operation has finished.
+    # If the operation has already finished, this method will return
+    # immediately.
+    def wait_until_finished
+      @mutex.synchronize do
+        if !finished?
+          @cv.wait(@mutex)
+        end
+      end
+    end # def wait_until_finished
+
+    protected
+    def finished?
+      return @finished
+    end # def finished?
+  end # class Operation
+end # module MQRPC

data/lib/mqrpc.rb

@@ -0,0 +1,4 @@
+require 'mqrpc/agent'
+require 'mqrpc/config'
+require 'mqrpc/logger'
+require 'mqrpc/message'

metadata

@@ -0,0 +1,92 @@
+--- !ruby/object:Gem::Specification 
+name: mqrpc
+version: !ruby/object:Gem::Version 
+  version: 0.0.1
+platform: ruby
+authors: 
+- Jordan Sissel, Pete Fritchman
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-11-05 00:00:00 -08:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: amqp
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.6.0
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: json
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.1.7
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: uuid
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 2.0.2
+    version: 
+description: RPC mechanism using AMQP as the transport
+email: logstash-dev@googlegroups.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/mqrpc/agent.rb
+- lib/mqrpc/config.rb
+- lib/mqrpc/logger.rb
+- lib/mqrpc/functions/ping.rb
+- lib/mqrpc/message.rb
+- lib/mqrpc/operation.rb
+- lib/mqrpc/messages/ping.rb
+- lib/mqrpc.rb
+has_rdoc: true
+homepage: http://code.google.com/p/logstash/wiki/MQRPC
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: mqrpc - RPC over Message Queue (AMQP)
+test_files: []
+