rabbit_feed 0.3.1

data/lib/rabbit_feed/configuration.rb

@@ -0,0 +1,50 @@
+module RabbitFeed
+  class Configuration
+    include ActiveModel::Validations
+
+    attr_reader :host, :port, :user, :password, :application, :environment, :exchange, :pool_size, :pool_timeout, :heartbeat, :connect_timeout, :network_recovery_interval, :auto_delete_queue, :auto_delete_exchange
+    validates_presence_of :host, :port, :user, :password, :application, :environment, :exchange, :pool_size, :pool_timeout, :heartbeat, :connect_timeout, :network_recovery_interval
+
+    def initialize options
+      RabbitFeed.log.debug "RabbitFeed initialising with options: #{options}..."
+
+      @host                      = options[:host]                      || 'localhost'
+      @port                      = options[:port]                      || 5672
+      @user                      = options[:user]                      || 'guest'
+      @password                  = options[:password]                  || 'guest'
+      @exchange                  = options[:exchange]                  || 'amq.topic'
+      @pool_size                 = options[:pool_size]                 || 1
+      @pool_timeout              = options[:pool_timeout]              || 5
+      @heartbeat                 = options[:heartbeat]                 || 5
+      @connect_timeout           = options[:connect_timeout]           || 10
+      @network_recovery_interval = options[:network_recovery_interval] || 1
+      @auto_delete_queue         = !!(options[:auto_delete_queue] || false)
+      @auto_delete_exchange      = !!(options[:auto_delete_exchange] || false)
+      @application               = options[:application]
+      @environment               = options[:environment]
+      validate!
+    end
+
+    def self.load file_path, environment
+      RabbitFeed.log.debug "Reading configurations from #{file_path} in #{environment}..."
+
+      options = read_configuration_file file_path, environment
+      new options.merge(environment: environment)
+    end
+
+    def queue
+      "#{environment}.#{application}"
+    end
+
+    private
+
+    def self.read_configuration_file file_path, environment
+      raw_configuration = YAML.load(ERB.new(File.read(file_path)).result)
+      HashWithIndifferentAccess.new (raw_configuration[environment] || {})
+    end
+
+    def validate!
+      raise ConfigurationError.new errors.messages if invalid?
+    end
+  end
+end

data/lib/rabbit_feed/connection_concern.rb

@@ -0,0 +1,95 @@
+module RabbitFeed
+  module ConnectionConcern
+    extend ActiveSupport::Concern
+
+    module ClassMethods
+
+      def default_connection_options
+        {
+          heartbeat:                 RabbitFeed.configuration.heartbeat,
+          connect_timeout:           RabbitFeed.configuration.connect_timeout,
+          host:                      RabbitFeed.configuration.host,
+          user:                      RabbitFeed.configuration.user,
+          password:                  RabbitFeed.configuration.password,
+          port:                      RabbitFeed.configuration.port,
+          network_recovery_interval: RabbitFeed.configuration.network_recovery_interval,
+          logger:                    RabbitFeed.log,
+        }
+      end
+
+      def with_connection &block
+        connection_pool.with do |connection|
+          yield connection
+        end
+      end
+
+      def retry_on_exception tries=3, &block
+        yield
+      rescue Bunny::ConnectionClosedError
+        raise # There is no point in retrying if the connection is closed
+      rescue => e
+        RabbitFeed.log.warn "Exception encountered; #{tries - 1} tries remaining. #{self.to_s}: #{e.message} #{e.backtrace}"
+        unless (tries -= 1).zero?
+          retry
+        end
+        raise
+      end
+
+      def retry_on_closed_connection tries=3, &block
+        yield
+      rescue Bunny::ConnectionClosedError => e
+        RabbitFeed.log.warn "Closed connection exception encountered; #{tries - 1} tries remaining. #{self.to_s}: #{e.message} #{e.backtrace}"
+        unless (tries -= 1).zero?
+          unset_connection
+          sleep RabbitFeed.configuration.network_recovery_interval
+          retry
+        end
+        raise
+      end
+
+      def close
+        RabbitFeed.log.debug "Closing connection: #{self.to_s}..."
+        @bunny_connection.close if @bunny_connection.present? && !closed?
+        unset_connection
+      rescue => e
+        RabbitFeed.log.warn "Exception encountered whilst closing #{self.to_s}: #{e.message} #{e.backtrace}"
+      end
+
+      def bunny_connection
+        if @bunny_connection.nil?
+          retry_on_exception do
+            RabbitFeed.log.debug "Opening connection: #{self.to_s}..."
+            @bunny_connection = Bunny.new connection_options
+            @bunny_connection.start
+          end
+        end
+
+        @bunny_connection
+      end
+      private :bunny_connection
+
+      def connection_pool
+        @connection_pool ||= ConnectionPool.new(
+          size:    RabbitFeed.configuration.pool_size,
+          timeout: RabbitFeed.configuration.pool_timeout
+        ) do
+          new bunny_connection.create_channel
+        end
+      end
+      private :connection_pool
+
+      def closed?
+        @bunny_connection.present? && @bunny_connection.closed?
+      end
+      private :closed?
+
+      def unset_connection
+        RabbitFeed.log.debug "Unsetting connection: #{self.to_s}..."
+        @connection_pool  = nil
+        @bunny_connection = nil
+      end
+      private :unset_connection
+
+    end
+  end
+end

data/lib/rabbit_feed/consumer.rb

@@ -0,0 +1,14 @@
+module RabbitFeed
+  module Consumer
+    extend self
+
+    attr_accessor :event_routing
+
+    def run
+      ConsumerConnection.consume do |raw_event|
+        event = Event.deserialize raw_event
+        event_routing.handle_event event
+      end
+    end
+  end
+end

data/lib/rabbit_feed/consumer_connection.rb

@@ -0,0 +1,108 @@
+module RabbitFeed
+  class ConsumerConnection
+    include ConnectionConcern
+
+    SUBSCRIPTION_OPTIONS = {
+      consumer_tag: Socket.gethostname, # Use the host name of the server
+      manual_ack:   true, # Manually acknowledge messages once they've been processed
+      block:        false, # Don't block the thread whilst consuming from the queue, as this breaks during connection recovery
+    }.freeze
+
+    SEVEN_DAYS_IN_MS = 7.days * 1000
+
+    QUEUE_OPTIONS = {
+      durable:     true,  # Persist across server restart
+      no_declare:  false, # Create the queue if it does not exist
+      arguments:   {
+        'x-ha-policy' => 'all', # Apply the queue on all mirrors
+        'x-expires'   => SEVEN_DAYS_IN_MS, # Auto-delete the queue after a period of inactivity (in ms)
+        },
+    }.freeze
+
+    attr_reader :queue
+
+    def initialize channel
+      channel.prefetch(1) # Fetch one message at a time to preserve order
+      RabbitFeed.log.debug "Declaring queue on #{self.to_s} (channel #{channel.id}) named: #{RabbitFeed.configuration.queue} with options: #{queue_options}..."
+      @queue = channel.queue RabbitFeed.configuration.queue, queue_options
+      bind_on_accepted_routes
+    end
+
+    def self.consume &block
+      with_connection do |consumer_connection|
+        consumer_connection.consume(&block)
+      end
+    end
+
+    def consume &block
+      RabbitFeed.log.info "Consuming messages on #{self.to_s} from queue: #{RabbitFeed.configuration.queue}..."
+
+      consumer = queue.subscribe(SUBSCRIPTION_OPTIONS) do |delivery_info, properties, payload|
+        handle_message delivery_info, payload, &block
+      end
+
+      sleep # Sleep indefinitely, as the consumer runs in its own thread
+    rescue
+      (cancel_consumer consumer) if consumer.present?
+      raise
+    end
+
+    private
+
+    def queue_options
+      {
+        auto_delete: RabbitFeed.configuration.auto_delete_queue,
+      }.merge QUEUE_OPTIONS
+    end
+
+    def self.connection_options
+      default_connection_options.merge({
+        threaded: true,
+      })
+    end
+
+    def bind_on_accepted_routes
+      if RabbitFeed::Consumer.event_routing.present?
+        RabbitFeed::Consumer.event_routing.accepted_routes.each do |accepted_route|
+          queue.bind(RabbitFeed.configuration.exchange, { routing_key: accepted_route })
+        end
+      else
+        queue.bind(RabbitFeed.configuration.exchange)
+      end
+    end
+
+    def acknowledge delivery_info
+      queue.channel.ack(delivery_info.delivery_tag)
+      RabbitFeed.log.debug "Message acknowledged on #{self.to_s} from queue: #{RabbitFeed.configuration.queue}..."
+    end
+
+    def handle_message delivery_info, payload, &block
+      RabbitFeed.log.debug "Message received on #{self.to_s} from queue: #{RabbitFeed.configuration.queue}..."
+
+      begin
+        yield payload
+        acknowledge delivery_info
+      rescue => e
+        handle_processing_exception delivery_info, e
+      end
+    end
+
+    def cancel_consumer consumer
+      cancel_ok = consumer.cancel
+      RabbitFeed.log.debug "Consumer: #{cancel_ok.consumer_tag} cancelled on #{self.to_s} from queue: #{RabbitFeed.configuration.queue}..."
+    end
+
+    def negative_acknowledge delivery_info
+      # Tell rabbit that we were unable to process the message
+      # This will re-queue the message
+      queue.channel.nack(delivery_info.delivery_tag, false, true)
+      RabbitFeed.log.debug "Message negatively acknowledged on #{self.to_s} from queue: #{RabbitFeed.configuration.queue}..."
+    end
+
+    def handle_processing_exception delivery_info, exception
+      negative_acknowledge delivery_info
+      RabbitFeed.log.error "Exception encountered while consuming message on #{self.to_s} from queue #{RabbitFeed.configuration.queue}: #{exception.message} #{exception.backtrace}"
+      RabbitFeed.exception_notify exception
+    end
+  end
+end

data/lib/rabbit_feed/event.rb

@@ -0,0 +1,43 @@
+module RabbitFeed
+  class Event
+    include ActiveModel::Validations
+
+    attr_reader :schema, :payload
+    validates_presence_of :schema, :payload
+
+    def initialize schema, payload
+      @schema  = schema
+      @payload = payload
+      validate!
+    end
+
+    def serialize
+      buffer = StringIO.new
+      writer = Avro::DataFile::Writer.new buffer, (Avro::IO::DatumWriter.new schema), schema
+      writer << payload
+      writer.close
+      buffer.string
+    end
+
+    def self.deserialize event
+      datum_reader = Avro::IO::DatumReader.new
+      reader       = Avro::DataFile::Reader.new (StringIO.new event), datum_reader
+      payload      = nil
+      reader.each do |datum|
+        payload = datum
+      end
+      reader.close
+      Event.new datum_reader.readers_schema, payload
+    end
+
+    def method_missing(method_name, *args, &block)
+      payload[method_name.to_s]
+    end
+
+    private
+
+    def validate!
+      raise Error.new errors.messages if invalid?
+    end
+  end
+end

data/lib/rabbit_feed/event_definitions.rb

@@ -0,0 +1,98 @@
+module RabbitFeed
+  class EventDefinitions
+
+    class Field
+      include ActiveModel::Validations
+
+      attr_reader :name, :type, :definition
+      validates_presence_of :name, :type, :definition
+
+      def initialize name, type, definition
+        @name       = name
+        @type       = type
+        @definition = definition
+        validate!
+      end
+
+      def schema
+        { name: name, type: type, doc: definition }
+      end
+
+      private
+
+      def validate!
+        raise ConfigurationError.new "Bad field specification for #{name}: #{errors.messages}" if invalid?
+      end
+    end
+
+    class Event
+      include ActiveModel::Validations
+
+      attr_reader :name, :definition, :version, :fields
+      validates_presence_of :name, :definition, :version
+      validate :schema_parseable
+      validates :version, format: { with: /\A\d+\.\d+\.\d+\z/, message: 'must be in *.*.* format' }
+
+      def initialize name, version
+        @name    = name
+        @version = version
+        @fields  = []
+      end
+
+      def payload_contains &block
+        self.instance_eval(&block)
+      end
+
+      def field name, options
+        fields << (Field.new name, options[:type], options[:definition])
+      end
+
+      def defined_as &block
+        @definition = block.call if block.present?
+      end
+
+      def payload
+        ([
+          (Field.new 'application',    'string', 'The name of the application that created the event'),
+          (Field.new 'host',           'string', 'The hostname of the server on which the event was created'),
+          (Field.new 'environment',    'string', 'The environment in which the event was created'),
+          (Field.new 'version',        'string', 'The version of the event'),
+          (Field.new 'name',           'string', 'The name of the event'),
+          (Field.new 'created_at_utc', 'string', 'The UTC time that the event was created')
+        ] + fields).map(&:schema)
+      end
+
+      def schema
+        @schema ||= (Avro::Schema.parse ({ name: name, type: 'record', doc: definition, fields: payload }.to_json))
+      end
+
+      def validate!
+        raise ConfigurationError.new "Bad event specification for #{name}: #{errors.messages}" if invalid?
+      end
+
+      private
+
+      def schema_parseable
+        schema
+      rescue => e
+        errors.add(:fields, "could not be parsed into a schema, reason: #{e.message}")
+      end
+    end
+
+    attr_reader :events
+
+    def initialize
+      @events = {}
+    end
+
+    def define_event name, options, &block
+      events[name] = Event.new name, options[:version]
+      events[name].instance_eval(&block)
+      events[name].validate!
+    end
+
+    def [] name
+      events[name]
+    end
+  end
+end

data/lib/rabbit_feed/event_routing.rb

@@ -0,0 +1,90 @@
+module RabbitFeed
+  class EventRouting
+
+    class Event
+      include ActiveModel::Validations
+
+      attr_reader :name, :action
+      validates_presence_of :name, :action
+      validate :action_arity
+
+      def initialize name, block
+        @name   = name
+        @action = block
+
+        validate!
+      end
+
+      def handle_event event
+        action.call event
+      end
+
+      private
+
+      def action_arity
+        errors.add(:action, 'arity should be 1') if action.present? && action.arity != 1
+      end
+
+      def validate!
+        raise ConfigurationError.new "Bad event specification for #{name}: #{errors.messages}" if invalid?
+      end
+    end
+
+    class Application
+      include ActiveModel::Validations
+
+      attr_reader :events, :name
+      validates_presence_of :name
+
+      def initialize name
+        @name    = name
+        @events  = {}
+
+        validate!
+      end
+
+      def event name, &block
+        event = (Event.new name, block)
+        events[event.name] = event
+      end
+
+      def accepted_routes
+        events.values.map do |event|
+          "#{RabbitFeed.environment}.#{name}.#{event.name}"
+        end
+      end
+
+      def handle_event event
+        event_rule = events[event.name]
+        raise RoutingError.new "No routing defined for application with name: #{event.application} for events named: #{event.name}" unless event_rule.present?
+        event_rule.handle_event event
+      end
+
+      def validate!
+        raise ConfigurationError.new "Bad application specification for #{name}: #{errors.messages}" if invalid?
+      end
+    end
+
+    attr_reader :applications
+
+    def initialize
+      @applications = {}
+    end
+
+    def accept_from name, &block
+      application = Application.new name
+      application.instance_eval(&block)
+      applications[application.name] = application
+    end
+
+    def accepted_routes
+      applications.values.map{|application| application.accepted_routes }.flatten
+    end
+
+    def handle_event event
+      application = applications[event.application]
+      raise RoutingError.new "No routing defined for application with name: #{event.application}" unless application.present?
+      application.handle_event event
+    end
+  end
+end