marconi 0.1.0

data/lib/marconi/broadcaster.rb

@@ -0,0 +1,63 @@
+
+require 'active_support/core_ext/string/inflections'
+
+module Marconi
+  module Broadcaster
+
+    def self.included(base)
+      base.extend ClassMethods
+      base.setup
+    end
+
+    module ClassMethods
+      def setup
+        @master_model_name = self.name.underscore
+
+        # Callbacks
+        after_update :publish_update
+        after_create :publish_create
+        after_destroy :publish_destroy
+      end
+
+      def master_model_name
+        @master_model_name
+      end
+    end
+
+    def publish_update
+      publish('update')
+    end
+
+    def publish_create
+      publish('create')
+    end
+
+    def publish_destroy
+      publish('destroy')
+    end
+
+    def publish(operation)
+
+      # This is set in Receiver if it's included.  Intent is to 
+      # prevent sending messages in response to incoming messages and thus
+      # creating infinite loops.
+      return if self.class.respond_to?(:broadcasts_suppressed?) &&
+        self.class.broadcasts_suppressed?
+
+      fmt = "%28s %9s %s Published" % [guid,operation,Time.now.to_s]
+      logger.debug fmt
+      Marconi.log(fmt)
+
+      e = Envelope.new { |e| e.send(operation, self) }
+      topic = "#{Marconi.application_name}.#{self.class.master_model_name}.#{operation}"
+      exchange.publish(e.to_s, :topic => topic)
+    end
+
+    private
+
+    def exchange
+      Marconi.inbound
+    end
+
+  end
+end

data/lib/marconi/config.rb

@@ -0,0 +1,33 @@
+
+module Marconi
+  class Config
+    attr_accessor :name, :short_name, :keepalive, :bunny_params, :listeners
+    def initialize(config_file = nil, env = 'test')
+      
+      if Object.const_defined?(:Rails)
+        config_file ||= "#{Rails.root}/config/queues.yml"
+        env = Rails.env
+      end
+
+      unless File.exists?(config_file)
+        raise "Could not find #{config_file}"
+      end
+
+      params = YAML.load_file(config_file)
+
+      self.name = params['name'] || raise("Wait... Who am I?")
+      self.short_name = params['short_name'] || raise("w8... hu m i?")
+      self.keepalive = !!params['keepalive']
+      self.listeners = params['listeners'] || []
+      self.bunny_params = params['bunny'] && params['bunny'][env]
+
+      if bunny_params
+        self.bunny_params.symbolize_keys!
+      else
+        puts "Warning: No config specified for #{env}"
+        self.bunny_params = {}
+      end
+
+    end
+  end
+end

data/lib/marconi/envelope.rb

@@ -0,0 +1,79 @@
+
+require 'active_support/core_ext/hash/indifferent_access'
+require 'active_support/core_ext/hash/conversions'
+require 'marconi/smart_xml'
+
+module Marconi
+  class Envelope
+
+    attr_accessor :topic
+
+    class << self
+
+      def from_xml(xml)
+        return nil unless xml
+        Envelope.new(:xml => xml)
+      end
+
+    end
+
+    def initialize(options = {}, &block)
+      if xml = options[:xml]
+        hash = SmartXML.parse(xml)
+        @topic = hash[:headers][:topic]
+        hash[:payload].each do |hsh|
+          messages << { :meta => hsh[:meta], :data => HashWithIndifferentAccess.new(hsh[:data]) }
+        end
+      else
+        @timestamp = options[:timestamp] || Time.now
+        @topic = options[:topic]
+        block.call(self) if block_given?
+      end
+    end
+
+    def messages
+      @messages ||= []
+    end
+
+    def contents
+      { :headers => headers, :payload => messages }
+    end
+
+    def create(model)
+      add_message(model, :create)
+    end
+
+    def update(model)
+      add_message(model, :update)
+    end
+
+    def destroy(model)
+      add_message(model, :destroy)
+    end
+
+    def override(old_guid, model)
+      add_message(model, :override, old_guid)
+    end
+
+    def to_s
+      contents.to_xml
+    end
+
+    private
+
+    def headers
+      { :topic => @topic }
+    end
+
+    def add_message(model, operation, guid = model.guid)
+      meta = {
+        :operation => operation.to_s,
+        :guid => guid,
+        :version => model.version,
+        :timestamp => @timestamp
+      }
+      messages << { :meta => meta, :data => HashWithIndifferentAccess.new(model.attributes) }
+    end
+
+  end
+end

data/lib/marconi/exchange.rb

@@ -0,0 +1,179 @@
+require 'bunny'
+
+module Marconi
+  class Exchange
+
+    def initialize(exchange_name)
+      @exchange_name = exchange_name
+    end
+
+    def exchange_name
+      @exchange_name
+    end
+
+    def name
+      Marconi.config.name
+    end
+
+    def keepalive
+      Marconi.config.keepalive
+    end
+
+    def bunny_params
+      Marconi.config.bunny_params
+    end
+
+    DEFAULT_PUBLISH_OPTIONS = {
+      # Tells the server how to react if the message cannot be routed to a queue.
+      # If set to true, the server will return an unroutable message with a Return method.
+      # If set to false, the server silently drops the message.
+      :mandatory => true,
+
+      # Tells the server how to react if the message cannot be routed to a
+      # queue consumer immediately.  If set to true, the server will return an
+      # undeliverable message with a Return method.  If set to false, the
+      # server will queue the message, but with no guarantee that it will ever
+      # be consumed.
+      :immediate => false,
+
+      # Tells the server whether to persist the message.  If set to true, the
+      # message will be persisted to disk and not lost if the server restarts.
+      # Setting to true incurs a performance penalty as there is an extra cost
+      # associated with disk access.
+      :persistent => true
+    }
+
+    # Example: Marconi.inbound.publish("Howdy!", :topic => 'deals.member.create')
+    def publish(msg, options = {})
+      topic = ensure_valid_publish_topic(options)
+      begin
+        connect
+        @exchange.publish(msg, DEFAULT_PUBLISH_OPTIONS.merge(:key => topic))
+        retmsg = @bunny.returned_message
+        raise "Invalid return payload: #{retmsg[:payload]}" unless retmsg[:payload] == :no_return
+        true
+      rescue Exception => e
+        Marconi.log(e)
+        if Marconi.backup_queue_class && !options[:recovering]
+          Marconi.backup_queue_class.create!(:exchange_name => exchange_name,
+                                             :topic => topic,
+                                             :body => msg)
+        end
+        false
+      end
+    end
+
+    # Example: Marconi.inbound.subscribe('foo_q', :key => 'deals.member.*') { |msg| puts msg[:payload] }
+    def subscribe(q_name, options = {}, &block)
+      q, key = get_q(q_name, options)
+      q.subscribe(options, &block)
+    end
+
+    # Example: Marconi.inbound.pop('foo_q', :key => 'deals.member.*')
+    def pop(q_name, options = {})
+      q, key = get_q(q_name, options)
+      msg = q.pop[:payload]
+      msg == :queue_empty ? nil : msg
+    end
+
+    # Example: Marconi.inbound.purge_q('foo_q')
+    # Use judiciously - this tosses all messages in the Q!
+    def purge_q(q_name)
+      connect
+      unless q_name.blank?
+        if o = exists?(:queue, q_name)
+          o.purge
+        end
+      end
+    rescue Bunny::ForcedChannelCloseError
+      connect(true) # Connection is fucked after this error, so it must be refreshed
+      raise
+    end
+
+    # Example: Marconi.inbound.nuke_q('foo_q')
+    # Use judiciously - this tosses all messages in the Q *and* nukes it
+    def nuke_q(q_name)
+      generic_nuke(:queue, q_name) 
+    end
+
+    private
+
+    def connected?
+      @bunny && @bunny.status == :connected && @exchange
+    end
+
+    def connect(reconnect = false)
+      reconnect = true unless keepalive
+      @bunny.stop if reconnect && connected? rescue nil
+      if reconnect || !connected?
+        @bunny = Bunny.new(bunny_params)
+        result = @bunny.start
+        if result == :connected
+          @exchange =
+            @bunny.exchange(
+              exchange_name,
+
+              # Topic queues are broadcast queues that allow wildcard subscriptions
+              :type => :topic,
+
+              # Durable exchanges remain active when a server restarts.
+              # Non-durable exchanges (transient exchanges) are purged if/when
+              # a server restarts.
+              :durable => true
+            )
+          result
+        else
+          raise "Unable to connect to RabbitMQ: #{result}"
+        end
+      end
+    rescue Bunny::ProtocolError # raised by @bunny.start
+      @bunny = nil
+      raise
+    end
+
+    def ensure_valid_publish_topic(options)
+      raise "You must specify a topic to publish to!" unless topic = options[:topic]
+      raise "You may not publish to a topic name with a wildcard" if topic =~ /[*#]/
+      topic
+    end
+
+    DEFAULT_Q_OPTIONS = {
+      :durable => true,
+      :exclusive => false,
+      :auto_delete => false
+    }
+
+    def get_q(q_name, options)
+      connect
+      raise "You must specify a topic to subscribe to!" unless key = options[:key]
+      raise "You must specify a Queue Name" if q_name.blank?
+      q = @bunny.queue(q_name, DEFAULT_Q_OPTIONS)
+      q.bind(@exchange, :key => key)
+      [q, key]
+    rescue Bunny::ForcedChannelCloseError, Bunny::ProtocolError
+      connect(true) # Connection is fucked after this error, so it must be refreshed
+      raise
+    end
+
+    def exists?(type, name)
+      @bunny.send(type, name, :passive => true)
+      # Annoyingly, this error is thrown if the q or exchange can't be found ... yet :passive isn't
+      #  supposed to err!!  Bah.
+    rescue Bunny::ForcedChannelCloseError
+      connect(true) # Connection is fucked after this error, so it must be refreshed
+      nil
+    end
+
+    def generic_nuke(type, name)
+      connect
+      unless name.blank?
+        if o = exists?(type, name)
+          o.delete
+        end
+      end
+    rescue Bunny::ForcedChannelCloseError
+      connect(true) # Connection is fucked after this error, so it must be refreshed
+      raise
+    end
+  end
+end

data/lib/marconi/guid_generator.rb

@@ -0,0 +1,16 @@
+
+require 'uuidtools'
+
+module Marconi
+  module GUIDGenerator
+
+    extend self
+
+    def next_guid
+      shortname = Marconi.short_application_name[0,2]
+      uid = UUIDTools::UUID.random_create.to_i.to_s(36).rjust(25,'0')
+      "#{shortname}-#{uid}"
+    end
+
+  end
+end

data/lib/marconi/receiver.rb

@@ -0,0 +1,84 @@
+
+require 'active_support/core_ext/hash/indifferent_access'
+
+module Marconi
+  module Receiver
+
+    def self.included(base)
+      base.extend ClassMethods
+      base.setup
+    end
+
+    module ClassMethods
+      
+      def setup
+        @master_model_name = self.name.underscore
+      end
+
+      def master_model_name
+        @master_model_name
+      end
+
+      def register(operation, &block)
+        self.handlers[operation] ||= []
+        self.handlers[operation] << block
+      end
+
+      def handlers
+        @handlers ||= HashWithIndifferentAccess.new
+      end
+
+      def log_message_action(amqp_mesg, info_mesg)
+        fmt = "%28s %9s %s %s" %
+          [amqp_mesg[:meta][:guid],
+           amqp_mesg[:meta][:operation],
+           Time.now().to_s, info_mesg]
+        logger.debug(fmt)
+        Marconi.log(fmt)
+      end
+
+      def listen(max = nil)
+        q_name = "#{Marconi.application_name}.#{self.master_model_name}"
+        topic = "#.#{self.master_model_name}.#"
+        exchange.subscribe(q_name, :key => topic, :message_max => max) do |amqp_msg|
+          e = Envelope.from_xml(amqp_msg[:payload])
+          suppress_broadcasts do
+            e.messages.each do |message|
+              log_message_action(message, "Receiving")
+              operation = message[:meta][:operation]
+              next unless self.handlers[operation]
+              self.handlers[operation].each do |h|
+                h.call(message)
+              end
+            end
+          end
+        end
+      end
+
+      def suppress_broadcasts
+        @suppress_broadcasts = true
+        yield
+      ensure
+        @suppress_broadcasts = false
+      end
+
+      def broadcasts_suppressed?
+        @suppress_broadcasts
+      end
+
+      def purge_handlers
+        @handlers = nil
+      end
+
+      def exchange 
+        Marconi.outbound
+      end
+    end
+
+    def log_message_action(amqp_mesg, info_mesg)
+      self.class.log_message_action(amqp_mesg, info_mesg)
+    end
+
+  end
+
+end

data/lib/marconi/smart_xml.rb

@@ -0,0 +1,40 @@
+
+require 'active_support/core_ext/hash/indifferent_access'
+
+module Marconi
+  module SmartXML
+
+    class << self
+      def parse(xml)
+        return nil unless xml
+        hash = Hash.from_xml(xml)
+        return nil unless hash
+
+        # If the top-level object is a hash, rails's deserializer puts it 
+        # under the key "hash".  If it's an array, it uses "object".  Simple
+        # values like numbers and strings can't be XML docs all by themselves
+        # so we don't need to consider any other cases.
+
+        indifferentize(hash['hash'] || hash['objects'])
+      end
+
+      private
+
+      # Convert all hashes to indifferent hashes, because otherwise we get
+      # subtle bugs.
+      def indifferentize(hash)
+        case hash
+        when Hash
+          new_hash = HashWithIndifferentAccess.new
+          hash.each do |key, value|
+            new_hash[key] = indifferentize(value)
+          end
+          new_hash
+        else
+          hash
+        end
+      end
+    end
+
+  end
+end

data/lib/marconi.rb

@@ -0,0 +1,89 @@
+
+module Marconi
+
+  extend self
+
+  attr_accessor :backup_queue_class_name, :logger
+
+  def init(config_file, env)
+    @config = Config.new(config_file, env)
+  end
+
+  # this uses eval to get the class from its name because
+  # otherwise rails class reloading in dev will screw everything up
+  def backup_queue_class
+    eval(backup_queue_class_name)
+  end
+
+  def inbound
+    @inbound ||= exchange('marconi.events.inbound')
+  end
+
+  def outbound
+    @outbound ||= exchange('marconi.events.outbound')
+  end
+
+  def error
+    @error ||= exchange('marconi.events.error')
+  end
+
+  def config
+    @config ||= Config.new
+  end
+
+  def application_name
+    config.name
+  end
+
+  def short_application_name
+    config.short_name
+  end
+
+  def listen
+    config.listeners.each do |class_name|
+      fork do
+
+        # this is to ditch any exit handlers.  Minitest in ruby 1.9
+        # for some reason sets itself to run an empty suite at exit.
+        at_exit { exit! }
+
+        # Change the process name as it appears in ps/top so the proc 
+        # is easy to identify
+        $0 = "ruby #{$0} Marconi.listen [#{class_name}]"
+
+        class_name.constantize.listen
+      end
+    end
+
+    Process.waitall
+  rescue Interrupt
+    Process.kill("-INT",0) # interrupt the whole process group
+  end
+
+  def run_recovery_loop
+    if backup_queue_class
+      backup_queue_class.find_each do |bc|
+        if exchange(bc.exchange_name).publish(bc.body, :topic => bc.topic, :recovering => true)
+          bc.destroy
+        end
+      end
+    end
+  end
+
+  def exchange(name)
+    @registry ||= {}
+    @registry[name] ||= Exchange.new(name)
+  end
+
+  def log(message)
+    logger.info(message) if logger
+  end
+
+end
+
+require 'marconi/guid_generator'
+require 'marconi/exchange'
+require 'marconi/config'
+require 'marconi/envelope'
+require 'marconi/receiver'
+require 'marconi/broadcaster'

metadata

@@ -0,0 +1,106 @@
+--- !ruby/object:Gem::Specification 
+name: marconi
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 0.1.0
+platform: ruby
+authors: 
+- Ken Miller
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-08-02 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: bunny
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - "="
+      - !ruby/object:Gem::Version 
+        version: 0.6.0
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: activesupport
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: mocha
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :development
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: shoulda
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :development
+  version_requirements: *id004
+description: Implements asychronous, distributed state broadcasting for ActiveModel-like objects.
+email: ken.miller@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/marconi/broadcaster.rb
+- lib/marconi/config.rb
+- lib/marconi/envelope.rb
+- lib/marconi/exchange.rb
+- lib/marconi/guid_generator.rb
+- lib/marconi/receiver.rb
+- lib/marconi/smart_xml.rb
+- lib/marconi.rb
+has_rdoc: true
+homepage: https://github.com/kemiller/marconi
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+requirements: []
+
+rubyforge_project: nowarning
+rubygems_version: 1.6.2
+signing_key: 
+specification_version: 3
+summary: Message-based distributed state updates for Active Record
+test_files: []
+