rabbit_feed 1.0.2 → 2.0.0

data/lib/rabbit_feed/event.rb

@@ -2,42 +2,82 @@ module RabbitFeed
   class Event
     include ActiveModel::Validations
 
-    attr_reader :schema, :payload
-    validates_presence_of :schema, :payload
+    SCHEMA_VERSION = '2.0.0'
 
-    def initialize schema, payload
-      @schema  = schema
-      @payload = payload
+    attr_reader :schema, :payload, :metadata
+    validates :metadata, presence: true
+    validates :payload, length: { minimum: 0, allow_nil: false, message: 'can\'t be nil' }
+    validate  :required_metadata
+
+    def initialize metadata, payload={}, schema=nil
+      @schema   = schema
+      @payload  = payload.with_indifferent_access if payload
+      @metadata = metadata.with_indifferent_access if metadata
       validate!
     end
 
     def serialize
       buffer = StringIO.new
       writer = Avro::DataFile::Writer.new buffer, (Avro::IO::DatumWriter.new schema), schema
-      writer << payload
+      writer << { 'metadata' => metadata, 'payload' => payload }
       writer.close
       buffer.string
     end
 
-    def self.deserialize event
-      datum_reader = Avro::IO::DatumReader.new
-      reader       = Avro::DataFile::Reader.new (StringIO.new event), datum_reader
-      payload      = nil
-      reader.each do |datum|
-        payload = datum
-      end
-      reader.close
-      Event.new datum_reader.readers_schema, payload
+    def application
+      metadata[:application]
+    end
+
+    def name
+      metadata[:name]
     end
 
-    def method_missing(method_name, *args, &block)
-      payload[method_name.to_s]
+    def created_at_utc
+      (Time.iso8601 metadata[:created_at_utc]) if metadata[:created_at_utc].present?
+    end
+
+    class << self
+
+      def deserialize serialized_event
+        datum_reader = Avro::IO::DatumReader.new
+        reader       = Avro::DataFile::Reader.new (StringIO.new serialized_event), datum_reader
+        event_hash   = nil
+        reader.each do |datum|
+          event_hash = datum
+        end
+        reader.close
+        if (version_1? event_hash)
+          new_from_version_1 event_hash, datum_reader.readers_schema
+        else
+          new event_hash['metadata'], event_hash['payload'], datum_reader.readers_schema
+        end
+      end
+
+      private
+
+      def version_1? event_hash
+        %w(metadata payload).none?{|key| event_hash.has_key? key}
+      end
+
+      def new_from_version_1 metadata_and_payload, schema
+        metadata = {}
+        %w(application name host version environment created_at_utc).each do |field|
+          metadata[field] = metadata_and_payload.delete field
+        end
+        new metadata, metadata_and_payload, schema
+      end
     end
 
     private
 
     def validate!
-      raise Error.new errors.messages if invalid?
+      raise Error.new "Invalid event: #{errors.messages}" if invalid?
+    end
+
+    def required_metadata
+      if metadata
+        errors.add(:metadata, 'name field is required') if metadata[:name].blank?
+      end
     end
   end
 end

data/lib/rabbit_feed/event_definitions.rb

@@ -51,19 +51,31 @@ module RabbitFeed
         @definition = block.call if block.present?
       end
 
-      def payload
-        ([
+      def payload_schema
+        { name: "#{name}_payload", type: 'record', fields: fields.map(&:schema) }
+      end
+
+      def metadata_schema
+        { name: 'event_metadata', type: 'record', fields: [
           (Field.new 'application',    'string', 'The name of the application that created the event'),
           (Field.new 'host',           'string', 'The hostname of the server on which the event was created'),
           (Field.new 'environment',    'string', 'The environment in which the event was created'),
-          (Field.new 'version',        'string', 'The version of the event'),
+          (Field.new 'version',        'string', 'The version of the event payload'),
+          (Field.new 'schema_version', 'string', 'The version of the event schema'),
           (Field.new 'name',           'string', 'The name of the event'),
-          (Field.new 'created_at_utc', 'string', 'The UTC time that the event was created')
-        ] + fields).map(&:schema)
+          (Field.new 'created_at_utc', 'string', 'The UTC time that the event was created'),
+        ].map(&:schema) }
+      end
+
+      def event_schema
+        [
+          { name: 'payload', type: payload_schema, doc: 'The event payload (defined by the source system)' },
+          { name: 'metadata', type: metadata_schema, doc: 'The event metadata (defined by rabbit feed)' },
+        ]
       end
 
       def schema
-        @schema ||= (Avro::Schema.parse ({ name: name, type: 'record', doc: definition, fields: payload }.to_json))
+        @schema ||= (Avro::Schema.parse ({ name: name, type: 'record', doc: definition, fields: event_schema }.to_json))
       end
 
       def validate!

data/lib/rabbit_feed/producer.rb

@@ -8,23 +8,24 @@ module RabbitFeed
       raise (Error.new 'Unable to publish event. No event definitions set.') unless event_definitions.present?
       event_definition = event_definitions[name] or raise (Error.new "definition for event: #{name} not found")
       timestamp        = Time.now.utc
-      payload          = (enriched_payload payload, event_definition.version, name, timestamp)
-      event            = Event.new event_definition.schema, payload
+      metadata         = (metadata event_definition.version, name, timestamp)
+      event            = Event.new metadata, payload, event_definition.schema
       ProducerConnection.publish event.serialize, (options name, timestamp)
       event
     end
 
     private
 
-    def enriched_payload payload, version, name, timestamp
-      payload.merge ({
-              'application'    => RabbitFeed.configuration.application,
-              'host'           => Socket.gethostname,
-              'environment'    => RabbitFeed.environment,
-              'created_at_utc' => timestamp.iso8601(6),
-              'version'        => version,
-              'name'           => name,
-            })
+    def metadata version, name, timestamp
+      {
+        'application'    => RabbitFeed.configuration.application,
+        'host'           => Socket.gethostname,
+        'environment'    => RabbitFeed.environment,
+        'created_at_utc' => timestamp.iso8601(6),
+        'version'        => version,
+        'name'           => name,
+        'schema_version' => Event::SCHEMA_VERSION,
+      }
     end
 
     def routing_key event_name

data/lib/rabbit_feed/producer_connection.rb

@@ -57,7 +57,7 @@ module RabbitFeed
     end
 
     def self.connection_options
-      default_connection_options.merge({
+      super.merge({
         threaded: false, # With threading enabled, there is a chance of losing an event during connection recovery
       })
     end

data/lib/rabbit_feed/testing_support/rspec_matchers/publish_event.rb

@@ -28,7 +28,7 @@ module RabbitFeed
 
           with_expected_payload = negative_expectation
           if received_expected_event && !with_expected_payload
-            actual_payload        = (strip_defaults_from actual_event.payload)
+            actual_payload        = actual_event.payload
             with_expected_payload = expected_payload.nil? || actual_payload == expected_payload
           end
 
@@ -61,16 +61,10 @@ module RabbitFeed
 
         private
 
-        def strip_defaults_from payload
-          payload.reject do |key, value|
-            ['application', 'host', 'environment', 'created_at_utc', 'version', 'name'].include? key
-          end
-        end
-
         def received_events_message
           if TestingSupport.published_events.any?
             TestingSupport.published_events.map do |received_event|
-              "#{received_event.name} with #{strip_defaults_from received_event.payload}"
+              "#{received_event.name} with #{received_event.payload}"
             end
           else
             'no events'

data/lib/rabbit_feed/testing_support/test_rabbit_feed_consumer.rb

@@ -1,9 +1,9 @@
 module RabbitFeed
   module TestingSupport
     class TestRabbitFeedConsumer
-      def consume_event(event)
-        event = Event.new('no schema',event)
-        RabbitFeed::Consumer.event_routing.handle_event(event)
+
+      def consume_event event
+        RabbitFeed::Consumer.event_routing.handle_event event
       end
     end
   end

data/lib/rabbit_feed/version.rb

@@ -1,3 +1,3 @@
 module RabbitFeed
-  VERSION = '1.0.2'
+  VERSION = '2.0.0'
 end

data/lib/rabbit_feed.rb

@@ -17,6 +17,7 @@ require 'rabbit_feed/producer_connection'
 require 'rabbit_feed/producer'
 require 'rabbit_feed/event_definitions'
 require 'rabbit_feed/testing_support'
+require 'rabbit_feed/version'
 
 module RabbitFeed
   extend self

data/rabbit_feed.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   # Gem for interfacing with RabbitMq
-  spec.add_dependency 'bunny', '>= 1.1.9', '< 1.7.0'
+  spec.add_dependency 'bunny', '>= 1.1.9', '< 1.8.0'
   # We use some helpers from ActiveSupport
   spec.add_dependency 'activesupport', '>= 3.2.0', '< 5.0.0'
   # We use validations from ActiveModel
@@ -31,5 +31,5 @@ Gem::Specification.new do |spec|
   # Schema definitions and serialization for events
   spec.add_dependency 'avro', '>= 1.5.4', '< 1.8.0'
   # For stubbing and custom matchers
-  spec.add_development_dependency 'rspec', '>=2.14.0', '< 3.2.0'
+  spec.add_development_dependency 'rspec', '>=2.14.0', '< 3.3.0'
 end

data/spec/features/step_definitions/connectivity_steps.rb

@@ -41,7 +41,7 @@ module Turnip::Steps
 
   def assert_event_presence event
     expect(event).to_not be_nil
-    expect(event.field).to eq @event_text
+    expect(event.payload[:field]).to eq @event_text
   end
 
   def wait_for_event

data/spec/fixtures/configuration.yml

@@ -5,7 +5,6 @@ test:
   password: guest
   application: rabbit_feed
   exchange: rabbit_feed_exchange
-  pool_size: 1
   pool_timeout: 1
   heartbeat: 60
   connect_timeout: 1

data/spec/lib/rabbit_feed/configuration_spec.rb

@@ -15,6 +15,57 @@ module RabbitFeed
       it { should eq 'test.rabbit_feed' }
     end
 
+    describe '#connection_options' do
+      subject { (described_class.new options).connection_options }
+
+      context 'when there are no defaults' do
+        let(:options) do
+          {
+            application: 'rabbit_feed',
+            environment: 'test',
+          }
+        end
+
+        it { should include(logger: RabbitFeed.log) }
+        it { should include(recover_from_connection_close: true) }
+        it { should include(threaded: true) }
+        it { should_not include(:heartbeat) }
+        it { should_not include(:network_recovery_interval) }
+        it { should_not include(:connect_timeout) }
+        it { should_not include(:host) }
+        it { should_not include(:user) }
+        it { should_not include(:password) }
+        it { should_not include(:port) }
+      end
+
+      context 'when there are defaults' do
+        let(:options) do
+          {
+            application:    'rabbit_feed',
+            environment:    'test',
+            heartbeat:       5,
+            connect_timeout: 1,
+            host:            'localhost',
+            user:            'guest',
+            password:        'guest',
+            port:            1234,
+            network_recovery_interval: 1,
+          }
+        end
+
+        it { should include(heartbeat: 5) }
+        it { should include(connect_timeout: 1) }
+        it { should include(host: 'localhost') }
+        it { should include(user: 'guest') }
+        it { should include(password: 'guest') }
+        it { should include(port: 1234) }
+        it { should include(network_recovery_interval: 1) }
+        it { should include(logger: RabbitFeed.log) }
+        it { should include(recover_from_connection_close: true) }
+        it { should include(threaded: true) }
+      end
+    end
+
     describe '.load' do
       let(:file_path)   { 'spec/fixtures/configuration.yml' }
       let(:environment) { 'test' }
@@ -45,7 +96,6 @@ module RabbitFeed
         its(:application)               { should eq 'rabbit_feed' }
         its(:environment)               { should eq 'test' }
         its(:exchange)                  { should eq 'rabbit_feed_exchange' }
-        its(:pool_size)                 { should eq 1 }
         its(:pool_timeout)              { should eq 1 }
         its(:heartbeat)                 { should eq 60 }
         its(:connect_timeout)           { should eq 1 }
@@ -68,16 +118,15 @@ module RabbitFeed
           }
         end
 
-        its(:host)                      { should eq 'localhost' }
-        its(:port)                      { should eq 5672 }
-        its(:user)                      { should eq 'guest' }
-        its(:password)                  { should eq 'guest' }
+        its(:host)                      { should be_nil }
+        its(:port)                      { should be_nil }
+        its(:user)                      { should be_nil }
+        its(:password)                  { should be_nil }
+        its(:heartbeat)                 { should be_nil }
+        its(:network_recovery_interval) { should be_nil }
         its(:exchange)                  { should eq 'amq.topic' }
-        its(:pool_size)                 { should eq 1 }
-        its(:pool_timeout)              { should eq 5 }
-        its(:heartbeat)                 { should eq 5 }
-        its(:connect_timeout)           { should eq 10 }
-        its(:network_recovery_interval) { should eq 1 }
+        its(:pool_timeout)              { should eq 120 }
+        its(:connect_timeout)           { should be_nil }
         its(:auto_delete_queue)         { should be_falsey }
         its(:auto_delete_exchange)      { should be_falsey }
       end
@@ -92,7 +141,6 @@ module RabbitFeed
             application:               'rabbit_feed',
             environment:               'test',
             exchange:                  'exchange_name',
-            pool_size:                 2,
             pool_timeout:              6,
             heartbeat:                 3,
             connect_timeout:           4,
@@ -109,7 +157,6 @@ module RabbitFeed
         its(:application)               { should eq 'rabbit_feed' }
         its(:environment)               { should eq 'test' }
         its(:exchange)                  { should eq 'exchange_name' }
-        its(:pool_size)                 { should eq 2 }
         its(:pool_timeout)              { should eq 6 }
         its(:heartbeat)                 { should eq 3 }
         its(:connect_timeout)           { should eq 4 }

data/spec/lib/rabbit_feed/connection_concern_spec.rb

@@ -30,6 +30,11 @@ module RabbitFeed
         expect(subject.instance_variable_get(:@connection_pool)).to be_a ConnectionPool
       end
 
+      it 'creates a connection pool of one connection' do
+        expect(ConnectionPool).to receive(:new).with(hash_including({size: 1})).and_call_original
+        subject.with_connection{|connection| connection }
+      end
+
       it 'provides an instance of the class' do
         actual = subject.with_connection{|connection| connection }
         expect(actual).to be_a subject
@@ -97,6 +102,7 @@ module RabbitFeed
     describe '.retry_on_closed_connection' do
       before do
         subject.with_connection{|connection| connection }
+        allow(subject).to receive(:sleep).at_least(:once)
       end
 
       it_behaves_like 'an operation that retries on exception', :retry_on_closed_connection, Bunny::ConnectionClosedError
@@ -111,6 +117,11 @@ module RabbitFeed
         expect { subject.retry_on_closed_connection { raise Bunny::ConnectionClosedError.new 'blah' } }.to raise_error
         expect(subject.instance_variable_get(:@connection_pool)).to be_nil
       end
+
+      it 'waits between retries' do
+        expect(subject).to receive(:sleep).with(1).twice
+        begin; subject.retry_on_closed_connection { raise Bunny::ConnectionClosedError.new 'blah' }; rescue; end
+      end
     end
   end
 end

data/spec/lib/rabbit_feed/event_definitions_spec.rb

@@ -38,17 +38,31 @@ module RabbitFeed
       end
 
       it { should be_valid }
-      its(:fields) { should_not be_empty }
-      its(:schema) { should be_a Avro::Schema }
-      its(:payload){ should =~ [
-        {name: 'application', type: 'string', doc: 'The name of the application that created the event'},
-        {name: 'host', type: 'string', doc: 'The hostname of the server on which the event was created'},
-        {name: 'environment', type: 'string', doc: 'The environment in which the event was created'},
-        {name: 'version', type: 'string', doc: 'The version of the event'},
-        {name: 'created_at_utc', type: 'string', doc: 'The UTC time that the event was created'},
-        {name: 'field', type: 'string', doc: 'field definition'},
-        {name: 'name', type: 'string', doc: 'The name of the event'}
-        ]
+      its(:fields)          { should_not be_empty }
+      its(:schema)          { should be_a Avro::Schema }
+      its(:metadata_schema) { should eq({
+          name:   'event_metadata',
+          type:   'record',
+          fields: [
+            {name: 'application', type: 'string', doc: 'The name of the application that created the event'},
+            {name: 'host', type: 'string', doc: 'The hostname of the server on which the event was created'},
+            {name: 'environment', type: 'string', doc: 'The environment in which the event was created'},
+            {name: 'version', type: 'string', doc: 'The version of the event payload'},
+            {name: 'schema_version', type: 'string', doc: 'The version of the event schema'},
+            {name: 'name', type: 'string', doc: 'The name of the event'},
+            {name: 'created_at_utc', type: 'string', doc: 'The UTC time that the event was created'},
+        ]})
+      }
+      its(:payload_schema) { should eq({
+          name:   'event_name_payload',
+          type:   'record',
+          fields: [{name: 'field', type: 'string', doc: 'field definition'}],
+        })
+      }
+      its(:event_schema) { should match([
+          { name: 'payload', type: an_instance_of(Hash), doc: 'The event payload (defined by the source system)' },
+          { name: 'metadata', type: an_instance_of(Hash), doc: 'The event metadata (defined by rabbit feed)' },
+        ])
       }
 
       context 'when the name is nil' do

data/spec/lib/rabbit_feed/event_routing_spec.rb

@@ -41,11 +41,11 @@ module RabbitFeed
 
     it 'routes the event to the correct action, preferring named applications' do
       events = [
-        double(:event, application: 'dummy_1', name: 'event_1', payload: 1),
-        double(:event, application: 'dummy_1', name: 'event_2', payload: 2),
-        double(:event, application: 'dummy_1', name: 'event_4', payload: 4),
-        double(:event, application: 'dummy_2', name: 'event_3', payload: 3),
-        double(:event, application: 'none',    name: 'event_4', payload: 4),
+        Event.new({application: 'dummy_1', name: 'event_1'}, {payload: 1}),
+        Event.new({application: 'dummy_1', name: 'event_2'}, {payload: 2}),
+        Event.new({application: 'dummy_1', name: 'event_4'}, {payload: 4}),
+        Event.new({application: 'dummy_2', name: 'event_3'}, {payload: 3}),
+        Event.new({application: 'none',    name: 'event_4'}, {payload: 4}),
       ]
       events.each do |event|
         (RabbitFeed::Consumer.event_routing.handle_event event).should eq event.payload
@@ -54,8 +54,8 @@ module RabbitFeed
 
     it 'raises a routing error when the event cannot be routed' do
       events = [
-        double(:event, application: 'dummy_9', name: 'event_1', payload: 1),
-        double(:event, application: 'dummy_1', name: 'event_9', payload: 3),
+        Event.new({application: 'dummy_9', name: 'event_1'}, {payload: 1}),
+        Event.new({application: 'dummy_1', name: 'event_9'}, {payload: 3}),
       ]
       events.each do |event|
         expect{ RabbitFeed::Consumer.event_routing.handle_event event }.to raise_error RoutingError

data/spec/lib/rabbit_feed/event_spec.rb

@@ -2,32 +2,140 @@ require 'spec_helper'
 
 module RabbitFeed
   describe Event do
-    let(:schema)  { double(:schema) }
-    let(:payload) { { 'customer_id' => '123' } }
+    let(:schema)   { double(:schema) }
+    let(:payload)  { { 'customer_id' => '123' } }
+    let(:metadata) { { 'name' => 'test_event' } }
 
-    subject { described_class.new schema, payload }
+    subject { described_class.new metadata, payload, schema  }
 
     describe '.new' do
 
       it { should be_valid }
-      its(:schema)  { should eq schema }
-      its(:payload) { should eq({ 'customer_id' => '123' }) }
+      its(:schema)   { should eq schema }
+      its(:payload)  { should eq({ 'customer_id' => '123' }) }
+      its(:metadata) { should eq({ 'name' => 'test_event' }) }
 
-      context 'when schema is nil' do
-        let(:schema) {}
+      context 'when payload is nil' do
+        let(:payload) {}
 
         it 'should raise an error' do
-          expect{ subject }.to raise_error Error
+          expect{ subject }.to raise_error 'Invalid event: {:payload=>["can\'t be nil"]}'
         end
       end
 
-      context 'when payload is nil' do
-        let(:payload) {}
+      context 'when metadata is blank' do
+        let(:metadata) {}
+
+        it 'should raise an error' do
+          expect{ subject }.to raise_error 'Invalid event: {:metadata=>["can\'t be blank"]}'
+        end
+      end
+
+      context 'when name is blank' do
+        let(:metadata) {{ 'name' => '' }}
 
         it 'should raise an error' do
-          expect{ subject }.to raise_error Error
+          expect{ subject }.to raise_error 'Invalid event: {:metadata=>["name field is required"]}'
         end
       end
     end
+
+    describe '#name' do
+      let(:metadata) { { 'name' => 'test_event' } }
+
+      its(:name) { should eq('test_event') }
+    end
+
+    describe '#application' do
+      let(:metadata) { { 'name' => 'test_event', 'application' => 'test_application' } }
+
+      its(:application) { should eq('test_application') }
+    end
+
+    describe '#created_at_utc' do
+
+      context 'when the created_at_utc is in the metadata' do
+        let(:metadata) { { 'name' => 'test_event', 'created_at_utc' => '2015-03-02T15:55:19.411299Z' } }
+
+        its(:created_at_utc) { should eq(Time.iso8601('2015-03-02T15:55:19.411299Z')) }
+      end
+
+      context 'when the created_at_utc is not in the metadata' do
+        let(:metadata) { { 'name' => 'test_event', 'created_at_utc' => '' } }
+
+        its(:created_at_utc) { should be_nil }
+      end
+    end
+
+    describe '.deserialize' do
+      subject { described_class.deserialize serialized_event }
+
+      context 'from a post-1.0 event' do
+        let(:schema) do
+          Avro::Schema.parse ({ name: 'post-1.0', type: 'record', fields: [
+            { name: 'payload', type: {
+              name: 'event_payload', type: 'record', fields: [
+                { name: 'customer_id', type: 'string' },
+                ]
+              },
+            },
+            { name: 'metadata', type: {
+              name: 'event_metadata', type: 'record', fields: [
+                { name: 'name', type: 'string' },
+                ]
+              },
+            },
+          ]}.to_json)
+        end
+        let(:serialized_event) { (described_class.new metadata, payload, schema).serialize }
+
+        its(:schema)   { should eq schema }
+        its(:payload)  { should eq({ 'customer_id' => '123' }) }
+        its(:metadata) { should eq({ 'name' => 'test_event' }) }
+      end
+
+      context 'from a 1.0 event' do
+        let(:metadata_and_payload) do
+          {
+            'application'    => 'rabbit_feed',
+            'name'           => 'test_event',
+            'host'           => 'localhost',
+            'version'        => '1.0.0',
+            'environment'    => 'test',
+            'created_at_utc' => '2015-02-22',
+            'customer_id'    => '123',
+          }
+        end
+        let(:schema) do
+          Avro::Schema.parse ({ name: '1.0', type: 'record', fields: [
+            { name: 'customer_id', type: 'string' },
+            { name: 'application', type: 'string' },
+            { name: 'name', type: 'string' },
+            { name: 'host', type: 'string' },
+            { name: 'version', type: 'string' },
+            { name: 'environment', type: 'string' },
+            { name: 'created_at_utc', type: 'string' },
+          ]}.to_json)
+        end
+        let(:serialized_event) do
+          buffer = StringIO.new
+          writer = Avro::DataFile::Writer.new buffer, (Avro::IO::DatumWriter.new schema), schema
+          writer << metadata_and_payload
+          writer.close
+          buffer.string
+        end
+
+        its(:schema)   { should eq schema }
+        its(:payload)  { should eq({ 'customer_id' => '123' }) }
+        its(:metadata) { should eq({
+          'application'    => 'rabbit_feed',
+          'name'           => 'test_event',
+          'host'           => 'localhost',
+          'version'        => '1.0.0',
+          'environment'    => 'test',
+          'created_at_utc' => '2015-02-22',
+          })}
+      end
+    end
   end
 end