materialist 2.3.1 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4a5fb544fa815aa0d48535edb7ebb25bc6763477
-  data.tar.gz: d3d505b4e193b8286a8a5f9b889d768a2aab197f
+  metadata.gz: 6c126eb6aa8d3e7fd91c9628f1a9a4a16e6476e8
+  data.tar.gz: 3d20acfd6c6ae59358dcfffb0b6270d438d6f7e4
 SHA512:
-  metadata.gz: c4725e268051a7e8a95e45922d9c6b55c332ab362d8bdd6ffaa9fd5656bb19dd8996ab563b8093a028cff268ff302dac41ec8697bc83a22005416ee40e4327d1
-  data.tar.gz: c8d683122d2671a8667283eacd95c806d31be3136d793a7d131ca7c222c0ad244235f8d8c6c03c322ccb6cc21393b842d3c0d06452a97f904f817c4c360fcce3
+  metadata.gz: 39310ca655d95cd3b51b6765da39f5e7ccb6479ca8df8680c5b6f27100cd7fd5479458d36b5baca80d80b1c8b29dec266453b2e2bd26c98dd4b65e01044b5351
+  data.tar.gz: df9c7c8f9c59a6a1b640f011deb7e848ac2cb9e705b288bbb6d8238c1c08e3db5a4cca8014ddfa061236d52a13d60d6a1d239758a414e352adc09e9b1a6d8b64

data/.travis.yml

@@ -4,6 +4,6 @@ services:
   - redis-server
 rvm:
   - 2.4.0
-  - ruby-head
+  - 2.5.0
 script:
   - bundle exec rspec

data/Gemfile

@@ -1,4 +1,4 @@
 source 'https://rubygems.org'
 
-# Specify your gem's dependencies in routemaster_client.gemspec
+# Specify your gem's dependencies in materialist.gemspec
 gemspec

data/README.md

@@ -52,27 +52,47 @@ class Zone < ApplicationRecord
 end
 ```
 
+### Materialist Configuration
+
+If you need to override any of the materialist configurations,
+you can do so in an `configure/initializers/materialist.rb` file:
+
+
+```ruby
+Materialist.configure do |config|
+  # Configure materialist here. For example:
+  #
+  # config.topics = %w(topic_a topic_b)
+  #
+  # config.sidekiq_options = {
+  #   queue: :routemaster_index,
+  #   retry: 3
+  # }
+  #
+  # config.metrics_client = STATSD
+end
+```
+
+- `topics` (only when using in `.subscribe`): A string array of topics to be used.  
+If not provided nothing would be materialized.
+- `sidekiq_options` (optional, default: `{ retry: 10 }`) -- See [Sidekiq docs](https://github.com/mperham/sidekiq/wiki/Advanced-Options#workers) for list of optiosn
+- `metrics_client` (optional) -- You can pass your `STATSD` instance
+
 ### Routemaster Configuration
 
 First you need an "event handler":
 
 ```ruby
-handler = Materialist::EventHandler.new({ ...options })
+handler = Materialist::EventHandler.new
 ```
 
 Where options could be:
 
-- `topics` (only when using in `.subscribe`): An array of topics to be used.
-If not provided nothing would be materialized.
-- `queue` (optional): name of the queue to be used by sidekiq worker
-- `retry` (default: `10`): sidekiq retry policy to be used.
-
 Then there are two ways to configure materialist in routemaster:
 
 1. **If you DON'T need resources to be cached in redis:** use `handler` as siphon:
 
 ```ruby
-handler = Materialist::EventHandler.new
 siphon_events = {
   zones:               handler,
   rider_domain_riders: handler
@@ -86,15 +106,9 @@ map '/events' do
 end
 ```
 
-2. **You DO need resources cached in redis:** In this case you need to use `handler` to subscribe to the caching pipeline:
+2. **You DO need resources cached in redis:** In this case you need to provide `topics` in `Materialist.configure` and use `handler` to subscribe to routemaster caching pipeline:
 
 ```ruby
-TOPICS = %w(
-  zones
-  rider_domain_riders
-)
-
-handler = Materialist::EventHandler.new({ topics: TOPICS })
 app = Routemaster::Drain::Caching.new # or ::Basic.new
 app.subscribe(handler, prefix: true)
 # ...

data/RELEASE_NOTES.md

@@ -1,3 +1,11 @@
+## 3.0.0
+
+Breaking change:
+- `topics`, `retry` and `queue` options are now configured via `Materialist.configure` block.
+
+Features:
+- Ability to provide metrics client e.g. STATSD
+
 ## 2.3.1
 
 Fixes:

data/lib/configuration.rb

@@ -0,0 +1,30 @@
+module Materialist
+  class << self
+    def configuration
+      @configuration ||= Configuration.new
+    end
+
+    def reset_configuration!
+      @configuration = Configuration.new
+    end
+
+    def configure
+      yield(self.configuration)
+    end
+  end
+
+  class Configuration
+    attr_accessor :topics, :sidekiq_options, :metrics_client
+
+    def initialize
+      @topics = []
+      @sidekiq_options = {}
+      @metrics_client = NullMetricsClient
+    end
+
+    class NullMetricsClient
+      def self.increment(_, tags:); end
+      def self.histogram(_, _v, tags:); end
+    end
+  end
+end

data/lib/materialist.rb

@@ -1,3 +1,5 @@
+require_relative './configuration'
+
 module Materialist
-  VERSION = '2.3.1'
+  VERSION = '3.0.0'
 end

data/lib/materialist/event_handler.rb

@@ -4,15 +4,13 @@ require_relative './event_worker'
 module Materialist
   class EventHandler
 
-    # 10 retries takes approximately 6 hours
-    DEFAULT_OPTIONS = { retry: 10 }
+    DEFAULT_SIDEKIQ_OPTIONS = { retry: 10 }.freeze
 
-    def initialize(options={})
-      @options = DEFAULT_OPTIONS.merge(options)
+    def initialize
     end
 
     def on_events_received(batch)
-      batch.each { |event| call(event) if topics.include?(event['topic'].to_s) }
+      batch.each { |event| call(event) if should_materialize?(event['topic']) }
     end
 
     def call(event)
@@ -21,14 +19,18 @@ module Materialist
 
     private
 
-    attr_reader :options
+    attr_reader :topics
 
-    def topics
-      @_topics ||= options.fetch(:topics, []).map(&:to_s)
+    def should_materialize?(topic)
+      Materialist.configuration.topics.include?(topic.to_s)
+    end
+
+    def sidekiq_options
+      DEFAULT_SIDEKIQ_OPTIONS.merge(Materialist.configuration.sidekiq_options)
     end
 
     def worker
-      Materialist::EventWorker.set(options.slice(:queue, :retry))
+      Materialist::EventWorker.set(sidekiq_options)
     end
   end
 end

data/lib/materialist/event_worker.rb

@@ -7,8 +7,35 @@ module Materialist
 
     def perform(event)
       topic = event['topic']
+      action = event['type'].to_sym
+      timestamp = event['t']
+
       materializer = "#{topic.to_s.singularize.classify}Materializer".constantize
-      materializer.perform(event['url'], event['type'].to_sym)
+      materializer.perform(event['url'], action)
+
+      report_latency(topic, timestamp) if timestamp
+      report_stats(topic, action, :success)
+    rescue
+      report_stats(topic, action, :failure)
+      raise
+    end
+
+    private
+
+    def report_latency(topic, timestamp)
+      t = (Time.now.to_f - (timestamp.to_i / 1e3)).round(1)
+      Materialist.configuration.metrics_client.histogram(
+        "materialist.event_latency",
+        t,
+        tags: ["topic:#{topic}"]
+      )
+    end
+
+    def report_stats(topic, action, kind)
+      Materialist.configuration.metrics_client.increment(
+        "materialist.event_worker.#{kind}",
+        tags: ["action:#{action}", "topic:#{topic}"]
+      )
     end
   end
 end

data/materialist.gemspec

@@ -20,6 +20,8 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'activesupport'
   spec.add_runtime_dependency 'routemaster-drain', '>= 3.0'
 
+  spec.add_development_dependency 'activerecord'
+  spec.add_development_dependency 'sqlite3'
   spec.add_development_dependency 'dotenv'
   spec.add_development_dependency 'simplecov'
   spec.add_development_dependency 'codecov'

data/spec/materialist/event_handler_spec.rb

@@ -3,11 +3,16 @@ require 'materialist/event_handler'
 require 'materialist/event_worker'
 
 RSpec.describe Materialist::EventHandler do
-  let(:options) {{}}
-  subject { described_class.new options }
-
+  let(:topics) {[]}
+  let(:sidekiq_options) {{}}
   let(:worker_double) { double() }
+
   before do
+    Materialist.configure do |config|
+      config.sidekiq_options = sidekiq_options
+      config.topics = topics
+    end
+
     allow(Materialist::EventWorker).to receive(:set)
       .and_return worker_double
   end
@@ -17,7 +22,7 @@ RSpec.describe Materialist::EventHandler do
     let(:perform) { subject.on_events_received events.map() }
 
     context "when no topic is specified" do
-      let(:options) {{ topics: [] }}
+      let(:topics) {[]}
 
       it "doesn't enqueue any event" do
         expect(worker_double).to_not receive(:perform_async)
@@ -26,7 +31,7 @@ RSpec.describe Materialist::EventHandler do
     end
 
     context "when a topic is specified" do
-      let(:options) {{ topics: [:topic_a] }}
+      let(:topics) { %w(topic_a) }
 
       it "enqueues event of that topic" do
         expect(worker_double).to receive(:perform_async).with(events[0])
@@ -35,7 +40,7 @@ RSpec.describe Materialist::EventHandler do
     end
 
     context "when both topics are specified" do
-      let(:options) {{ topics: [:topic_a, :topic_b] }}
+      let(:topics) { %w(topic_a topic_b) }
 
       it "enqueues event of both topics" do
         expect(worker_double).to receive(:perform_async).twice
@@ -55,7 +60,7 @@ RSpec.describe Materialist::EventHandler do
 
     context "if queue name is privided" do
       let(:queue_name) { :some_queue_name }
-      let(:options) {{ queue: queue_name }}
+      let(:sidekiq_options) {{ queue: queue_name }}
 
       it "enqueues the event in the given queue" do
         expect(Materialist::EventWorker).to receive(:set)
@@ -66,7 +71,7 @@ RSpec.describe Materialist::EventHandler do
     end
 
     context "when a retry is specified in options" do
-      let(:options) {{ retry: false }}
+      let(:sidekiq_options) {{ retry: false }}
 
       it "uses the given retry option for sidekiq" do
         expect(Materialist::EventWorker).to receive(:set)

data/spec/materialist/event_worker_spec.rb

@@ -5,17 +5,60 @@ RSpec.describe Materialist::EventWorker do
   describe "#perform" do
     let(:source_url) { 'https://service.dev/foobars/1' }
     let(:event) {{ 'topic' => :foobar, 'url' => source_url, 'type' => 'noop' }}
-    let!(:materializer_class) { class FoobarMaterializer; end }
+    let!(:materializer_class) { FoobarMaterializer = Class.new }
+    let(:metrics_client) { double(increment: true) }
 
     before do
       allow(FoobarMaterializer).to receive(:perform)
+      Materialist.configure { |c| c.metrics_client = metrics_client }
     end
 
-    context "when run synchronously" do
-      let(:perform) { subject.perform(event) }
+    after { Object.send(:remove_const, :FoobarMaterializer) }
 
-      it "calls the relevant materializer" do
-        expect(FoobarMaterializer).to receive(:perform).with(source_url, :noop)
+    let(:perform) { subject.perform(event) }
+
+    it "calls the relevant materializer" do
+      expect(FoobarMaterializer).to receive(:perform).with(source_url, :noop)
+      perform
+    end
+
+    it 'logs success to metrics' do
+      expect(metrics_client).to receive(:increment).with(
+        "materialist.event_worker.success",
+        tags: ["action:noop", "topic:foobar"]
+      )
+      perform
+    end
+
+    it 'does not log latency' do
+      expect(metrics_client).to_not receive(:histogram)
+      perform
+    end
+
+    context 'when there is an error' do
+      let(:error){ StandardError.new }
+      before do
+        expect(FoobarMaterializer).to receive(:perform).and_raise error
+      end
+
+      it 'logs failure to metrics and re-raises the error' do
+        expect(metrics_client).to receive(:increment).with(
+          "materialist.event_worker.failure",
+          tags: ["action:noop", "topic:foobar"]
+        )
+        expect{ perform }.to raise_error error
+      end
+    end
+
+    context 'when event has a timestamp' do
+      let(:event) {{ 'topic' => :foobar, 'url' => source_url, 'type' => 'noop', 't' => '1519659773842' }}
+
+      it 'logs latency to metrics' do
+        expect(metrics_client).to receive(:histogram).with(
+          "materialist.event_latency",
+          instance_of(Float),
+          tags: ["topic:foobar"]
+        )
         perform
       end
     end

data/spec/materialist/materialized_record_spec.rb

@@ -6,7 +6,7 @@ RSpec.describe Materialist::MaterializedRecord do
   uses_redis
 
   let!(:materialized_type) do
-    class Foobar
+    Class.new do
       include Materialist::MaterializedRecord
 
       attr_accessor :source_url
@@ -19,7 +19,7 @@ RSpec.describe Materialist::MaterializedRecord do
   end
 
   let(:record) do
-    Foobar.new.tap { |r| r.source_url = source_url }
+    materialized_type.new.tap { |r| r.source_url = source_url }
   end
 
   let(:country_url) { 'https://service.dev/countries/1' }

data/spec/materialist/materializer_spec.rb

@@ -7,7 +7,7 @@ RSpec.describe Materialist::Materializer do
 
   describe "#perform" do
     let!(:materializer_class) do
-      class FoobarMaterializer
+      FoobarMaterializer = Class.new do
         include Materialist::Materializer
 
         persist_to :foobar
@@ -27,122 +27,16 @@ RSpec.describe Materialist::Materializer do
           end
         end
       end
+    end
 
-      class CityMaterializer
+    let!(:city_materializer) do
+      CityMaterializer = Class.new do
         include Materialist::Materializer
 
         persist_to :city
         source_key :source_url
         capture :name
       end
-
-      class DefinedSourceMaterializer
-        include Materialist::Materializer
-
-        persist_to :defined_source
-
-        source_key :source_id do |url|
-          url.split('/').last.to_i
-        end
-
-        capture :name
-      end
-    end
-
-    # this class mocks active record behaviour
-    class BaseModel
-      def update_attributes!(attrs)
-        attrs.each { |k, v| send("#{k}=", v) }
-        save!
-      end
-
-      def save!
-        self.class.all[source_key_value] = self
-      end
-
-      def destroy!
-        self.class.all.delete source_key_value
-      end
-
-      def reload
-        self.class.all[source_key_value]
-      end
-
-      def actions_called
-        @_actions_called ||= {}
-      end
-
-      private def source_key_value
-        send(self.class.source_key_column)
-      end
-
-      class << self
-        attr_accessor :error_to_throw_once_in_find_or_initialize_by,
-                      :source_key_column
-
-        def find_or_initialize_by(kv_hash)
-          if(err = error_to_throw_once_in_find_or_initialize_by)
-            self.error_to_throw_once_in_find_or_initialize_by = nil
-            raise err
-          end
-
-          key_value = kv_hash[source_key_column]
-
-          (all[key_value] || self.new).tap do |record|
-            record.send("#{source_key_column}=", key_value)
-          end
-        end
-
-        def source_key_column
-          @source_key_column || :source_url
-        end
-
-        def find_by(kv_hash)
-          key_value = kv_hash[source_key_column]
-          all[key_value]
-        end
-
-        def create!(attrs)
-          new.tap do |record|
-            record.update_attributes! attrs
-          end
-        end
-
-        def all
-          store[self.name] ||= {}
-        end
-
-        def destroy_all
-          store[self.name] = {}
-        end
-
-        def store
-          @@_store ||= {}
-        end
-
-        def count
-          all.keys.size
-        end
-      end
-    end
-
-    class Foobar < BaseModel
-      attr_accessor :source_url, :name, :how_old, :age, :timezone,
-        :country_tld, :city_url, :account_url
-    end
-
-    class City < BaseModel
-      attr_accessor :source_url, :name
-    end
-
-    class DefinedSource < BaseModel
-      attr_accessor :source_id, :name
-      self.source_key_column = :source_id
-    end
-
-    module ActiveRecord
-      class RecordNotUnique < StandardError; end
-      class RecordInvalid < StandardError; end
     end
 
     let(:country_url) { 'https://service.dev/countries/1' }
@@ -164,18 +58,19 @@ RSpec.describe Materialist::Materializer do
     end
 
     before do
-      Foobar.destroy_all
-      City.destroy_all
-      DefinedSource.destroy_all
-
       stub_resource source_url, source_body
       stub_resource country_url, country_body
       stub_resource city_url, city_body
       stub_resource defined_source_url, defined_source_body
     end
 
+    after do
+      Object.send(:remove_const, :FoobarMaterializer)
+      Object.send(:remove_const, :CityMaterializer)
+    end
+
     let(:action) { :create }
-    let(:perform) { FoobarMaterializer.perform(source_url, action) }
+    let(:perform) { materializer_class.perform(source_url, action) }
 
     it "materializes record in db" do
       expect{perform}.to change{Foobar.count}.by 1
@@ -212,31 +107,28 @@ RSpec.describe Materialist::Materializer do
     end
 
     context "when there is a race condition between a create and update" do
-      let(:error) {   }
+      let(:error) { nil }
       let!(:record) { Foobar.create!(source_url: source_url, name: 'mo') }
 
-      before { Foobar.error_to_throw_once_in_find_or_initialize_by = error }
-
-      [ ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid ].each do |error_type|
+      [ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid].each do |error_type|
         context "when error of type #{error_type.name} is thrown" do
           let(:error) { error_type }
 
           it "still updates the record" do
+            expect(Foobar).to receive(:find_or_initialize_by).ordered.and_raise(error)
+            expect(Foobar).to receive(:find_or_initialize_by).ordered.and_call_original
             expect{ perform }.to change { record.reload.name }
               .from('mo').to('jack')
           end
 
           context "if error was thrown second time" do
-            before { allow(Foobar).to receive(:find_or_initialize_by).and_raise error }
-
             it "bubbles up the error" do
+              expect(Foobar).to receive(:find_or_initialize_by).and_raise(error).twice
               expect{ perform }.to raise_error error
             end
           end
-
         end
       end
-
     end
 
     %i(create update noop).each do |action_name|
@@ -276,21 +168,18 @@ RSpec.describe Materialist::Materializer do
 
     context "when {after, before}_upsert is configured" do
       let!(:record) { Foobar.create!(source_url: source_url, name: 'mo') }
+      let(:actions_called) { materializer_class.class_variable_get(:@@actions_called) }
       let!(:materializer_class) do
-        class FoobarMaterializer
+        FoobarMaterializer = Class.new do
           include Materialist::Materializer
+          @@actions_called = {}
 
           persist_to :foobar
-          before_upsert :my_before_method
-          after_upsert :my_method
-
-          def my_method(entity)
-            entity.actions_called[:after_upsert] = true
-          end
+          before_upsert :before_hook
+          after_upsert :after_hook
 
-          def my_before_method(entity)
-            entity.actions_called[:before_upsert] = true
-          end
+          def before_hook(entity); @@actions_called[:before_hook] = true; end
+          def after_hook(entity); @@actions_called[:after_hook] = true; end
         end
       end
 
@@ -298,30 +187,36 @@ RSpec.describe Materialist::Materializer do
         context "when action is :#{action_name}" do
           let(:action) { action_name }
           it "calls before_upsert method" do
-            expect{ perform }.to change { record.actions_called[:before_upsert] }
+            expect{ perform }.to change { actions_called[:before_hook] }
           end
 
           it "calls after_upsert method" do
-            expect{ perform }.to change { record.actions_called[:after_upsert] }
+            expect{ perform }.to change { actions_called[:after_hook] }
           end
 
-          it "calls more than one method" do
-            class FoobarMaterializer
-              before_upsert :my_before_method, :my_before_method2
-              after_upsert :my_method, :my_method2
+          context "when configured with more than one hook" do
+            let(:materializer_class) do
+              FoobarMaterializer = Class.new do
+                include Materialist::Materializer
+                @@actions_called = {}
 
-              def my_method2(entity)
-                entity.actions_called[:after_upsert2] = true
-              end
+                persist_to :foobar
+                before_upsert :before_hook, :before_hook2
+                after_upsert :after_hook, :after_hook2
 
-              def my_before_method2(entity)
-                entity.actions_called[:before_upsert2] = true
+                def before_hook(entity); @@actions_called[:before_hook] = true; end
+                def before_hook2(entity); @@actions_called[:before_hook2] = true; end
+                def after_hook(entity); @@actions_called[:after_hook] = true; end
+                def after_hook2(entity); @@actions_called[:after_hook2] = true; end
               end
             end
-            expect{ perform }.to  change { record.actions_called[:after_upsert]   }
-                             .and change { record.actions_called[:after_upsert2]  }
-                             .and change { record.actions_called[:before_upsert]  }
-                             .and change { record.actions_called[:before_upsert2] }
+
+            it "calls more than one method" do
+              expect{ perform }.to  change { actions_called[:before_hook] }
+                               .and change { actions_called[:before_hook2] }
+                               .and change { actions_called[:after_hook] }
+                               .and change { actions_called[:after_hook2] }
+            end
           end
         end
       end
@@ -330,11 +225,11 @@ RSpec.describe Materialist::Materializer do
         let(:action) { :delete }
 
         it "does not call after_upsert method" do
-          expect{ perform }.to_not change { record.actions_called[:after_upsert] }
+          expect{ perform }.to_not change { actions_called[:after_hook] }
         end
 
         it "does call after_upsert method" do
-          expect{ perform }.to_not change { record.actions_called[:before_upsert] }
+          expect{ perform }.to_not change { actions_called[:before_hook] }
         end
       end
 
@@ -342,21 +237,18 @@ RSpec.describe Materialist::Materializer do
 
     context "when {before, after}_destroy is configured" do
       let!(:record) { Foobar.create!(source_url: source_url, name: 'mo') }
+      let(:actions_called) { materializer_class.class_variable_get(:@@actions_called) }
       let!(:materializer_class) do
-        class FoobarMaterializer
+        FoobarMaterializer = Class.new do
           include Materialist::Materializer
+          @@actions_called = {}
 
           persist_to :foobar
-          before_destroy :my_before_method
-          after_destroy :my_method
-
-          def my_method(entity)
-            entity.actions_called[:after_destroy] = true
-          end
+          before_destroy :before_hook
+          after_destroy :after_hook
 
-          def my_before_method(entity)
-            entity.actions_called[:before_destroy] = true unless entity.nil?
-          end
+          def before_hook(entity); @@actions_called[:before_hook] = true; end
+          def after_hook(entity); @@actions_called[:after_hook] = true; end
         end
       end
 
@@ -364,11 +256,11 @@ RSpec.describe Materialist::Materializer do
         context "when action is :#{action_name}" do
           let(:action) { action_name }
           it "does not call after_destroy method" do
-            expect{ perform }.to_not change { record.actions_called[:after_destroy] }
+            expect{ perform }.to_not change { actions_called[:after_hook] }
           end
 
           it "does not call before_destroy method" do
-            expect{ perform }.to_not change { record.actions_called[:before_destroy] }
+            expect{ perform }.to_not change { actions_called[:before_hook] }
           end
         end
       end
@@ -377,54 +269,60 @@ RSpec.describe Materialist::Materializer do
         let(:action) { :delete }
 
         it "calls after_destroy method" do
-          expect{ perform }.to change { record.actions_called[:after_destroy] }
+          expect{ perform }.to change { actions_called[:after_hook] }
         end
 
         it "calls before_destroy method" do
-          expect{ perform }.to change { record.actions_called[:before_destroy] }
+          expect{ perform }.to change { actions_called[:before_hook] }
         end
 
-        it "calls more than one method" do
-          class FoobarMaterializer
-            before_destroy :my_before_method, :my_before_method2
-            after_destroy :my_method, :my_method2
+        context "when configured with more than one hook" do
+          let(:materializer_class) do
+            FoobarMaterializer = Class.new do
+              include Materialist::Materializer
+              @@actions_called = {}
 
-            def my_method2(entity)
-              entity.actions_called[:after_destroy2] = true
-            end
+              persist_to :foobar
+              before_destroy :before_hook, :before_hook2
+              after_destroy :after_hook, :after_hook2
 
-            def my_before_method2(entity)
-              entity.actions_called[:before_destroy2] = true unless entity.nil?
+              def before_hook(entity); @@actions_called[:before_hook] = true; end
+              def before_hook2(entity); @@actions_called[:before_hook2] = true; end
+              def after_hook(entity); @@actions_called[:after_hook] = true; end
+              def after_hook2(entity); @@actions_called[:after_hook2] = true; end
             end
           end
-          expect{ perform }.to change  { record.actions_called[:after_destroy]   }
-                           .and change { record.actions_called[:after_destroy2]  }
-                           .and change { record.actions_called[:before_destroy]  }
-                           .and change { record.actions_called[:before_destroy2] }
+
+          it "calls more than one method" do
+            expect{ perform }.to  change { actions_called[:before_hook] }
+                             .and change { actions_called[:before_hook2] }
+                             .and change { actions_called[:after_hook] }
+                             .and change { actions_called[:after_hook2] }
+          end
         end
 
         context "when resource doesn't exist locally" do
           it "does not raise error" do
-            Foobar.destroy_all
+            Foobar.delete_all
             expect{ perform }.to_not raise_error
           end
         end
       end
-
     end
 
     context "when not materializing self but materializing linked parent" do
-      class CitySettingsMaterializer
-        include Materialist::Materializer
+      subject do
+        Class.new do
+          include Materialist::Materializer
 
-        materialize_link :city
+          materialize_link :city
+        end
       end
-
       let(:city_settings_url) { 'https://service.dev/city_settings/1' }
       let(:city_settings_body) {{ _links: { city: { href: city_url }}}}
       before { stub_resource city_settings_url, city_settings_body }
 
-      let(:perform) { CitySettingsMaterializer.perform(city_settings_url, action) }
+      let(:perform) { subject.perform(city_settings_url, action) }
 
       it "materializes linked parent" do
         expect{perform}.to change{City.count}.by 1
@@ -439,47 +337,63 @@ RSpec.describe Materialist::Materializer do
       end
     end
 
-    context "when creating a new entity based on the source_key column" do
-      let(:perform) { DefinedSourceMaterializer.perform(defined_source_url, action) }
+    context "entity based on the source_key column" do
+      subject do
+        Class.new do
+          include Materialist::Materializer
 
-      it "creates based on source_key" do
-        expect{perform}.to change{DefinedSource.count}.by 1
-      end
+          persist_to :defined_source
+
+          source_key :source_id do |url|
+            url.split('/').last.to_i
+          end
 
-      it "sets the correct source key" do
-        perform
-        inserted = DefinedSource.find_by(source_id: defined_source_id)
-        expect(inserted.source_id).to eq defined_source_id
-        expect(inserted.name).to eq defined_source_body[:name]
+          capture :name
+        end
       end
-    end
 
-    context "when updating a new entity based on the source_key column" do
-      let(:action) { :update }
-      let!(:record) { DefinedSource.create!(source_id: defined_source_id, name: 'mo') }
-      let(:perform) { DefinedSourceMaterializer.perform(defined_source_url, action) }
+      context "when creating" do
+        let(:perform) { subject.perform(defined_source_url, action) }
 
-      it "updates based on source_key" do
-        perform
-        expect(DefinedSource.count).to eq 1
+        it "creates based on source_key" do
+          expect{perform}.to change{DefinedSource.count}.by 1
+        end
+
+        it "sets the correct source key" do
+          perform
+          inserted = DefinedSource.find_by(source_id: defined_source_id)
+          expect(inserted.source_id).to eq defined_source_id
+          expect(inserted.name).to eq defined_source_body[:name]
+        end
       end
 
-      it "updates the existing record" do
-        perform
-        inserted = DefinedSource.find_by(source_id: defined_source_id)
-        expect(inserted.source_id).to eq defined_source_id
-        expect(inserted.name).to eq defined_source_body[:name]
+      context "when updating" do
+        let(:action) { :update }
+        let!(:record) { DefinedSource.create!(source_id: defined_source_id, name: 'mo') }
+        let(:perform) { subject.perform(defined_source_url, action) }
+
+        it "updates based on source_key" do
+          perform
+          expect(DefinedSource.count).to eq 1
+        end
+
+        it "updates the existing record" do
+          perform
+          inserted = DefinedSource.find_by(source_id: defined_source_id)
+          expect(inserted.source_id).to eq defined_source_id
+          expect(inserted.name).to eq defined_source_body[:name]
+        end
       end
-    end
 
-    context "when deleting an entity based on the source_key column" do
-      let(:action) { :delete }
-      let!(:record) { DefinedSource.create!(source_id: defined_source_id, name: 'mo') }
-      let(:perform) { DefinedSourceMaterializer.perform(defined_source_url, action) }
+      context "when deleting" do
+        let(:action) { :delete }
+        let!(:record) { DefinedSource.create!(source_id: defined_source_id, name: 'mo') }
+        let(:perform) { subject.perform(defined_source_url, action) }
 
-      it "deletes based on source_key" do
-        perform
-        expect(DefinedSource.count).to eq 0
+        it "deletes based on source_key" do
+          perform
+          expect(DefinedSource.count).to eq 0
+        end
       end
     end
   end

data/spec/models.rb

@@ -0,0 +1,8 @@
+class Foobar < ActiveRecord::Base
+end
+
+class City < ActiveRecord::Base
+end
+
+class DefinedSource < ActiveRecord::Base
+end

data/spec/schema.rb

@@ -0,0 +1,31 @@
+ActiveRecord::Schema.define do
+  self.verbose = false
+
+  create_table :foobars, :force => true do |t|
+    t.string :source_url, null: false
+    t.string :name
+    t.integer :how_old
+    t.integer :age
+    t.string :timezone
+    t.string :country_tld
+    t.string :city_url
+    t.string :account_url
+
+    t.timestamps
+  end
+
+  create_table :cities, :force => true do |t|
+    t.string :source_url, null: false
+    t.string :name
+
+    t.timestamps
+  end
+
+  create_table :defined_sources, :force => true do |t|
+    t.integer :source_id, null: false
+    t.string :name
+
+    t.timestamps
+  end
+
+end

data/spec/spec_helper.rb

@@ -19,9 +19,24 @@ RSpec.configure do |config|
   config.run_all_when_everything_filtered = true
   config.raise_errors_for_deprecations!
 
+  config.before(:each) do
+    Materialist.reset_configuration!
+    
+    # clear database
+    ActiveRecord::Base.descendants.each(&:delete_all)
+  end
+
   # Run specs in random order to surface order dependencies. If you find an
   # order dependency and want to debug it, you can fix the order by providing
   # the seed, which is printed after each run.
   #     --seed 1234
   config.order = 'random'
 end
+
+# active record
+require 'active_record'
+
+ActiveRecord::Base.establish_connection adapter: "sqlite3", database: ":memory:"
+
+load File.dirname(__FILE__) + '/schema.rb'
+load File.dirname(__FILE__) + '/models.rb'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: materialist
 version: !ruby/object:Gem::Version
-  version: 2.3.1
+  version: 3.0.0
 platform: ruby
 authors:
 - Mo Valipour
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-01-31 00:00:00.000000000 Z
+date: 2018-02-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -52,6 +52,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: dotenv
   requirement: !ruby/object:Gem::Requirement
@@ -170,6 +198,7 @@ files:
 - RELEASE_NOTES.md
 - Rakefile
 - appraise
+- lib/configuration.rb
 - lib/materialist.rb
 - lib/materialist/errors.rb
 - lib/materialist/event_handler.rb
@@ -181,6 +210,8 @@ files:
 - spec/materialist/event_worker_spec.rb
 - spec/materialist/materialized_record_spec.rb
 - spec/materialist/materializer_spec.rb
+- spec/models.rb
+- spec/schema.rb
 - spec/spec_helper.rb
 - spec/support/uses_redis.rb
 homepage: http://github.com/deliveroo/materialist
@@ -212,5 +243,7 @@ test_files:
 - spec/materialist/event_worker_spec.rb
 - spec/materialist/materialized_record_spec.rb
 - spec/materialist/materializer_spec.rb
+- spec/models.rb
+- spec/schema.rb
 - spec/spec_helper.rb
 - spec/support/uses_redis.rb