rabbit_feed 0.3.1

data/lib/rabbit_feed/producer.rb

@@ -0,0 +1,47 @@
+module RabbitFeed
+  module Producer
+    extend self
+
+    attr_accessor :event_definitions
+
+    def publish_event name, payload
+      raise (Error.new 'Unable to publish event. No event definitions set.') unless event_definitions.present?
+      event_definition = event_definitions[name] or raise (Error.new "definition for event: #{name} not found")
+      timestamp        = Time.now.utc
+      payload          = (enriched_payload payload, event_definition.version, name, timestamp)
+      event            = Event.new event_definition.schema, payload
+      ProducerConnection.publish event.serialize, (options name, timestamp)
+      event
+    end
+
+    def stub!
+      ProducerConnection.stub(:publish)
+    end
+
+    private
+
+    def enriched_payload payload, version, name, timestamp
+      payload.merge ({
+              'application'    => RabbitFeed.configuration.application,
+              'host'           => Socket.gethostname,
+              'environment'    => RabbitFeed.environment,
+              'created_at_utc' => timestamp.iso8601(6),
+              'version'        => version,
+              'name'           => name,
+            })
+    end
+
+    def routing_key event_name
+      "#{RabbitFeed.environment}.#{RabbitFeed.configuration.application}.#{event_name}"
+    end
+
+    def options event_name, timestamp
+      {
+        routing_key: (routing_key event_name),
+        type:        event_name,
+        app_id:      RabbitFeed.configuration.application,
+        timestamp:   timestamp.to_i,
+      }
+    end
+  end
+end

data/lib/rabbit_feed/producer_connection.rb

@@ -0,0 +1,65 @@
+module RabbitFeed
+  class ProducerConnection
+    include ConnectionConcern
+
+    PUBLISH_OPTIONS = {
+      persistent: true, # Persist the message to disk
+      mandatory:  true, # Return the message if it can't be routed to a queue
+    }.freeze
+
+    EXCHANGE_OPTIONS = {
+      type:        :topic, # Allow wildcard routing keys
+      durable:     true,   # Persist across server restart
+      no_declare:  false,  # Create the exchange if it does not exist
+    }.freeze
+
+    attr_reader :exchange
+
+    def self.handle_returned_message return_info, content
+      RabbitFeed.log.error "Handling returned message on #{self.to_s} details: #{return_info}..."
+      RabbitFeed.exception_notify (ReturnedMessageError.new return_info)
+    end
+
+    def initialize channel
+      RabbitFeed.log.debug "Declaring exchange on #{self.to_s} (channel #{channel.id}) named: #{RabbitFeed.configuration.exchange} with options: #{exchange_options}..."
+      @exchange = channel.exchange RabbitFeed.configuration.exchange, exchange_options
+
+      exchange.on_return do |return_info, properties, content|
+        RabbitFeed::ProducerConnection.handle_returned_message return_info, content
+      end
+    end
+
+    def self.publish message, options
+      retry_on_closed_connection do
+        with_connection do |producer_connection|
+          retry_on_exception do
+            producer_connection.publish message, options
+          end
+        end
+      end
+    end
+
+    def publish message, options
+      # It's critical to dup the options for the sake of retries, as bunny modifies this hash
+      bunny_options = (options.merge PUBLISH_OPTIONS)
+
+      RabbitFeed.log.debug "Publishing message on #{self.to_s} with options: #{options} to exchange: #{RabbitFeed.configuration.exchange}..."
+
+      exchange.publish message, bunny_options
+    end
+
+    private
+
+    def exchange_options
+      {
+        auto_delete: RabbitFeed.configuration.auto_delete_exchange,
+      }.merge EXCHANGE_OPTIONS
+    end
+
+    def self.connection_options
+      default_connection_options.merge({
+        threaded: false, # With threading enabled, there is a chance of losing an event during connection recovery
+      })
+    end
+  end
+end

data/lib/rabbit_feed/testing_support/rspec_matchers/publish_event.rb

@@ -0,0 +1,90 @@
+module RabbitFeed
+  module TestingSupport
+    module RSpecMatchers
+      class PublishEvent
+        attr_reader :expected_event, :expected_payload, :received_events
+
+        def initialize(expected_event, expected_payload)
+          @expected_event   = expected_event
+          @expected_payload = expected_payload
+          @received_events  = []
+          stub_publish
+        end
+
+        def matches?(given_proc, negative_expectation = false)
+          unless given_proc.respond_to?(:call)
+            ::Kernel.warn "`publish_event` was called with non-proc object #{given_proc.inspect}"
+            return false
+          end
+
+          begin
+            given_proc.call
+          rescue
+          end
+
+          actual_event = received_events.detect do |event|
+            event.name == expected_event
+          end
+
+          received_expected_event = actual_event.present?
+
+          with_expected_payload = negative_expectation
+          if received_expected_event && !with_expected_payload
+            actual_payload        = (strip_defaults_from actual_event.payload)
+            with_expected_payload = actual_payload == expected_payload
+          end
+
+          return received_expected_event && with_expected_payload
+        end
+
+        alias == matches?
+
+        def does_not_match?(given_proc)
+          !matches?(given_proc, :negative_expectation)
+        end
+
+        def failure_message
+          "expected #{expected_event} with #{expected_payload} but instead received #{received_events_message}"
+        end
+
+        def negative_failure_message
+          "expected no #{expected_event} event, but received one anyways"
+        end
+
+        alias failure_message_when_negated negative_failure_message
+
+        def description
+          "publish_event #{expected_event}"
+        end
+
+        private
+
+        def strip_defaults_from payload
+          payload.reject do |key, value|
+            ['application', 'host', 'environment', 'created_at_utc', 'version', 'name'].include? key
+          end
+        end
+
+        def received_events_message
+          if received_events.any?
+            received_events.map do |received_event|
+              "#{received_event.name} with #{strip_defaults_from received_event.payload}"
+            end
+          else
+            'no events'
+          end
+        end
+
+        def stub_publish
+          ProducerConnection.stub(:publish) do |serialized_event, routing_key|
+            @received_events << (Event.deserialize serialized_event)
+          end
+        end
+      end
+
+      def publish_event(expected_event, expected_payload)
+        PublishEvent.new(expected_event, expected_payload)
+      end
+    end
+  end
+end

data/lib/rabbit_feed/testing_support/testing_helpers.rb

@@ -0,0 +1,16 @@
+module RabbitFeed
+  module TestingSupport
+    module TestingHelpers
+      def rabbit_feed_consumer
+        TestRabbitFeedConsumer.new
+      end
+    end
+
+    class TestRabbitFeedConsumer
+      def consume_event(event)
+        event = Event.new('no schema',event)
+        RabbitFeed::Consumer.event_routing.handle_event(event)
+      end
+    end
+  end
+end

data/lib/rabbit_feed/version.rb

@@ -0,0 +1,3 @@
+module RabbitFeed
+  VERSION = '0.3.1'
+end

data/rabbit_feed.gemspec

@@ -0,0 +1,35 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'rabbit_feed/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'rabbit_feed'
+  spec.version       = RabbitFeed::VERSION
+  spec.authors       = ['Simply Business']
+  spec.email         = ['tech@simplybusiness.co.uk']
+  spec.description   = %q{A gem allowing your application to publish messages to RabbitMQ}
+  spec.summary       = %q{This will enable your application to publish messages to a bus to be processed by other services}
+  spec.homepage      = 'https://github.com/simplybusiness/rabbit_feed'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  # Gem for interfacing with RabbitMq
+  spec.add_dependency 'bunny'
+  # We use some helpers from ActiveSupport
+  spec.add_dependency 'activesupport'
+  # We use validations from ActiveModel
+  spec.add_dependency 'activemodel'
+  # Provides connection pooling for the producer connections
+  spec.add_dependency 'connection_pool'
+  # Manages process pidfile
+  spec.add_dependency 'pidfile'
+  # Schema definitions and serialization for events
+  spec.add_dependency 'avro'
+  # For stubbing and custom matchers
+  spec.add_development_dependency 'rspec'
+end

data/run_benchmark

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -e
+
+echo 'Starting test of rails application...'
+# Start the rails application
+echo 'Starting rails application...'
+pushd example/rails_app >/dev/null
+nodenv local v0.10 >/dev/null
+bundle exec rake db:reset RAILS_ENV=development
+bundle exec unicorn -c config/unicorn.rb -E development -D
+sleep 1
+popd >/dev/null
+echo 'Rails application started'
+
+# Test publishing via the rails application
+siege -c 10 -r 10 -q -b "http://localhost:8080/beavers POST beaver[name]=`date '+%m/%d/%y %H:%M:%S'`"
+sleep 4
+
+# Stop the rails application
+echo 'Stopping rails application...'
+pushd example/rails_app >/dev/null
+kill `cat tmp/pids/unicorn.pid`
+popd >/dev/null
+echo 'Rails application stopped'
+echo 'Rails application test complete'
+echo ''
+echo ''
+
+# Test standalone publishing and consuming
+echo 'Starting standalone publishing and consuming benchmark...'
+pushd example/non_rails_app >/dev/null
+bin/benchmark
+popd >/dev/null
+echo 'Benchmark complete'

data/run_example

@@ -0,0 +1,62 @@
+#!/bin/bash
+
+set -e
+
+# Start consumer on the non-rails application
+echo 'Starting non rails application consumer...'
+pushd example/non_rails_app >/dev/null
+bundle >/dev/null
+bundle exec rake
+bundle exec rabbit_feed consume --environment development --require `pwd`/lib/non_rails_app.rb --daemon --verbose
+sleep 1
+popd >/dev/null
+echo 'Non rails application consumer started'
+
+# Start consumer on the rails application
+echo 'Starting rails application consumer...'
+pushd example/rails_app >/dev/null
+bundle >/dev/null
+bundle exec rake db:reset
+bundle exec rake db:reset RAILS_ENV=test
+bundle exec rake
+bundle exec rabbit_feed consume --environment development --daemon --verbose
+sleep 1
+popd >/dev/null
+echo 'Rails application consumer started'
+
+# Start the rails application
+echo 'Starting rails application...'
+pushd example/rails_app >/dev/null
+nodenv local v0.10 >/dev/null
+bundle exec unicorn -c config/unicorn.rb -D
+sleep 1
+popd >/dev/null
+echo 'Rails application started'
+
+# Trigger an event
+echo 'Triggering event...'
+curl -silent -H -X POST http://localhost:8080/beavers -d "beaver[name]=`date '+%m/%d/%y %H:%M:%S'`" >/dev/null
+echo 'Event triggered'
+
+sleep 3
+
+# Stop consumer on the non-rails application
+echo 'Stopping non rails application consumer...'
+pushd example/non_rails_app >/dev/null
+kill `cat tmp/pids/rabbit_feed.pid`
+popd >/dev/null
+echo 'Non rails application consumer stopped'
+
+# Stop consumer on the rails application
+echo 'Stopping rails application consumer...'
+pushd example/rails_app >/dev/null
+kill `cat tmp/pids/rabbit_feed.pid`
+popd >/dev/null
+echo 'Rails application consumer stopped'
+
+# Stop the rails application
+echo 'Stopping rails application...'
+pushd example/rails_app >/dev/null
+kill `cat tmp/pids/unicorn.pid`
+popd >/dev/null
+echo 'Rails application stopped'

data/run_recovery_test

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -e
+
+# Grant the session sudo access
+sudo date
+
+# In a separate process enable/disable the loopback interface
+echo 'Starting process that will periodically disable lo0...'
+sudo sh -c "while true; do ifconfig lo0 down; sleep 0.1; ifconfig lo0 up; sleep 0.5; done" &
+disabling_pid=$!
+
+# Test standalone publishing and consuming
+echo 'Starting standalone publishing and consuming benchmark...'
+pushd example/non_rails_app >/dev/null
+bin/benchmark
+popd >/dev/null
+echo 'Benchmark complete'
+
+# Kill the process that was toggling lo0 and ensure lo0 is enabled
+function cleanup {
+  sudo kill $disabling_pid
+  sudo ifconfig lo0 up
+  echo 'Restored lo0'
+}
+trap cleanup EXIT

data/spec/features/connectivity.feature

@@ -0,0 +1,13 @@
+Feature:
+  As a developer
+  I want to be able to publish and consume events
+
+Scenario: I can publish and consume events
+  Given I am consuming
+   When I publish an event
+   Then I receive that event
+
+Scenario: When an event cannot be consumed it remains on the queue
+  Given I am consuming
+   When I publish an event that cannot be processed by the consumer
+   Then the event remains on the queue

data/spec/features/step_definitions/connectivity_steps.rb

@@ -0,0 +1,96 @@
+require 'spec_helper'
+
+step 'I am consuming' do
+  set_event_routing
+  initialize_queue
+  @consumer_thread = Thread.new{ RabbitFeed::Consumer.run }
+end
+
+step 'I publish an event' do
+  set_event_definitions
+  publish 'test'
+end
+
+step 'I receive that event' do
+  event = wait_for_event
+  assert_event_presence event
+end
+
+step 'I publish an event that cannot be processed by the consumer' do
+  set_event_definitions
+  publish 'test_failure'
+end
+
+step 'the event remains on the queue' do
+  event = nil
+  2.times { event = wait_for_event }
+  assert_event_presence event
+end
+
+module Turnip::Steps
+
+  def initialize_queue
+    RabbitFeed::ProducerConnection.with_connection{|connection|}
+    RabbitFeed::ConsumerConnection.with_connection{|connection|}
+  end
+
+  def publish event_name
+    @event_text = "#{event_name}_#{Time.now.iso8601(6)}"
+    RabbitFeed::Producer.publish_event event_name, { 'field' => @event_text }
+  end
+
+  def assert_event_presence event
+    expect(event).to_not be_nil
+    expect(event.field).to eq @event_text
+  end
+
+  def wait_for_event
+    begin
+      Timeout::timeout(5.0) do
+        until @consumed_events.any? do
+          sleep 0.1
+        end
+      end
+    rescue Timeout::Error
+    end
+    @consumed_events.pop
+  end
+
+  def set_event_definitions
+    EventDefinitions do
+      define_event('test', version: '1.0.0') do
+        defined_as do
+          'The test event'
+        end
+        payload_contains do
+          field('field', type: 'string', definition: 'The field')
+        end
+      end
+      define_event('test_failure', version: '1.0.0') do
+        defined_as do
+          'The test failure event'
+        end
+        payload_contains do
+          field('field', type: 'string', definition: 'The field')
+        end
+      end
+    end
+  end
+
+  def set_event_routing
+    consumed_events  = []
+    @consumed_events = consumed_events
+
+    EventRouting do
+      accept_from('rabbit_feed') do
+        event('test') do |event|
+          consumed_events << event
+        end
+        event('test_failure') do |event|
+          consumed_events << event
+          raise 'event processing failure'
+        end
+      end
+    end
+  end
+end