materialist 3.3.0 → 3.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: f551f35e47a806f9f5e3b475feb35358f4a034b5
-  data.tar.gz: 8134a56b74346d367618b99b09604c0ded38a815
+SHA256:
+  metadata.gz: 3cef103c710bdfcf4b9a5ee36bad6b4474bc88c194a436bb935aa15b090a69f0
+  data.tar.gz: f91fcd7467e60889d34ed33ed57c08ac67a6365a4168d5c52e7dd8564f2ec331
 SHA512:
-  metadata.gz: 2a647bde4177b45a090e8a91a1a34b1dde10f1ac38adbe8d1588c161182113cd8f065511edd59f08a82e38721ebaa1260650af9840d4e1f032160a4739561af1
-  data.tar.gz: c660872899ce6526370260e7852d886ecd34abf2a4c0cdaf7178b11680c524db318ade68d8640ce00aa2a50630158f4242f20eec20f79b49f496b877c6a0c678
+  metadata.gz: cc547ca1823460c517b5b1a8720e3259004bcbf1f8b92c91847d90d9ea6638af4ffb39d2f7a50994fc8070b997b1fb61b69a12ae49e2009125c61af4e2a5ac26
+  data.tar.gz: dd6b84e5541132b052bce821853348a62a0fd6608837f7d141b2bb472a6c32eb0fb01211ad07837c093ed62da9b55788a720034f1d398aef9e9eda7ba0b09d1a

data/README.md

@@ -14,6 +14,9 @@ In your `gemfile`
 
 ```ruby
 gem 'materialist'
+
+# or for sandbox features
+gem 'materialist', github: 'deliveroo/materialist', branch: 'sandbox'
 ```
 
 Then do
@@ -22,6 +25,20 @@ Then do
 bundle
 ```
 
+### Release
+
+After merging all of your PRs:
+
+1. Bump the version in `lib/materialist/version.rb` -- let's say `x.y.z`
+1. Build the gem: `gem build materialist.gemspec`
+1. Push the gem: `gem push materialist-x.y.z.gem`
+1. Commit changes: `git commit -am "Bump version"`
+1. Create a tag: `git tag -a vx.y.z`
+1. Push changes: `git push origin master`
+1. Push the new: `git push origin --tags`
+
+## Usage
+
 ### Entity
 
 Your materialised entity need to have a **unique** `source_url` column, alongside any other field you wish to materialise.
@@ -70,12 +87,14 @@ Materialist.configure do |config|
   # }
   #
   # config.metrics_client = STATSD
+  # config.api_client = Routemaster::APIClient.new(response_class: Routemaster::Responses::HateoasResponse)
 end
 ```
 
-- `topics` (only when using in `.subscribe`): A string array of topics to be used.  
+- `topics` (only when using in `.subscribe`): A string array of topics to be used.
 If not provided nothing would be materialized.
 - `sidekiq_options` (optional, default: `{ retry: 10 }`) -- See [Sidekiq docs](https://github.com/mperham/sidekiq/wiki/Advanced-Options#workers) for list of options
+- `api_client` (optional) -- You can pass your `Routemaster::APIClient` instance
 - `metrics_client` (optional) -- You can pass your `STATSD` instance
 - `notice_error` (optional) -- You can pass a lambda accepting two parameters (`exception` and `event`) -- Typical use case is to enrich error and send to NewRelic APM
 
@@ -136,8 +155,8 @@ class ZoneMaterializer
 
   persist_to :zone
 
-  source_key :source_id do |url|
-    /(\d+)\/?$/.match(url)[1]
+  source_key :source_id do |url, response|
+    /(\d+)\/?$/.match(url)[1] # or response.dig(:some_attr)
   end
 
   capture :id, as: :orderweb_id
@@ -170,14 +189,20 @@ describes the name of the active record model to be used.
 If missing, materialist skips materialising the resource itself, but will continue
 with any other functionality -- such as `materialize_link`.
 
-#### `source_key <column> <url_parser_block> (default: url)`
+#### `source_key <column> <parser_block> (default: url, resource response body[create, update action only])`
 describes the column used to persist the unique identifier parsed from the url_parser_block.
 By default the column used is `:source_url` and the original `url` is used as the identifier.
-Passing an optional block allows you to extract an identifier from the URL.
+Passing an optional block allows you to extract an identifier from the URL and captured attributes.
 
 #### `capture <key>, as: <column> (default: key)`
 describes mapping a resource key to a database column.
 
+You can optionally provide a block for parsing the value:
+
+```ruby
+capture(:location, as: :latitude) { |location| location[:latitude] }
+```
+
 #### `capture_link_href <key>, as: <column>`
 describes mapping a link href (as it appears on the hateous response) to a database column.
 
@@ -218,6 +243,24 @@ class ZoneMaterializer
 end
 ```
 
+#### `before_upsert_with_payload <method> (, <method>(, ...))`
+describes the name of the instance method(s) to be invoked before a record is
+materialized, with the record as it exists in the database, or nil if it has
+not been created yet. The function will get as a second argument the `payload`
+of the HTTP response, this can be used to add additional information/persist
+other objects.
+
+
+```ruby
+class ZoneMaterializer
+  include Materialist::Materializer
+
+  before_upsert_with_payload :my_method
+
+  def my_method(record, payload); end
+end
+```
+
 
 #### `after_upsert <method> (, <method>(, ...))` -- also `after_destroy`
 describes the name of the instance method(s) to be invoked after a record was materialized, with the updated record as a parameter. See above for a similar example implementation.

data/RELEASE_NOTES.md

@@ -1,6 +1,19 @@
 ## Next
 
-_description of next release_
+_list pending release notes here..._
+
+## 3.6.0
+
+- Add support for parsing value when using `capture`
+
+## 3.5.0
+
+- Add support for providing an `Routemaster::APIClient` instance as part of configuration
+
+## 3.4.0
+
+- Add support for providing payload into the materializer
+- Enhance linked resource materialization to avoid `:noop` stats
 
 ## 3.3.0
 

data/lib/configuration.rb

@@ -1,3 +1,5 @@
+require 'routemaster/api_client'
+
 module Materialist
   class << self
     def configuration
@@ -14,11 +16,12 @@ module Materialist
   end
 
   class Configuration
-    attr_accessor :topics, :sidekiq_options, :metrics_client, :notice_error
+    attr_accessor :topics, :sidekiq_options, :api_client, :metrics_client, :notice_error
 
     def initialize
       @topics = []
       @sidekiq_options = {}
+      @api_client = Routemaster::APIClient.new(response_class: ::Routemaster::Responses::HateoasResponse)
       @metrics_client = NullMetricsClient
       @notice_error = nil
     end

data/lib/materialist.rb

@@ -1,5 +1,5 @@
 require_relative './configuration'
+require_relative './materialist/version'
 
 module Materialist
-  VERSION = '3.3.0'
 end

data/lib/materialist/materialized_record.rb

@@ -1,4 +1,3 @@
-require 'routemaster/api_client'
 require_relative './errors'
 
 module Materialist
@@ -35,9 +34,7 @@ module Materialist
     private
 
     def source_raw
-      Routemaster::APIClient.new(
-        response_class: Routemaster::Responses::HateoasResponse
-      ).get(source_url)
+      Materialist.configuration.api_client.get(source_url)
     end
   end
 end

data/lib/materialist/materializer/internals/class_methods.rb

@@ -4,9 +4,8 @@ module Materialist
       module ClassMethods
         attr_reader :__materialist_options, :__materialist_dsl_mapping_stack
 
-        def perform(url, action)
-          materializer = Materializer.new(url, self)
-          action == :delete ? materializer.destroy : materializer.upsert
+        def perform(url, action, *options)
+          Materializer.new(url, self, *options).perform(action)
         end
 
         def _sidekiq_options

data/lib/materialist/materializer/internals/dsl.rb

@@ -6,8 +6,12 @@ module Materialist
           __materialist_options[:links_to_materialize][key] = { topic: topic }
         end
 
-        def capture(key, as: key)
-          __materialist_dsl_mapping_stack.last << FieldMapping.new(key: key, as: as)
+        def capture(key, as: key, &value_parser_block)
+          __materialist_dsl_mapping_stack.last << FieldMapping.new(
+            key: key,
+            as: as,
+            value_parser: value_parser_block
+          )
         end
 
         def capture_link_href(key, as:, &url_parser_block)
@@ -34,9 +38,18 @@ module Materialist
           __materialist_options[:sidekiq_options] = options
         end
 
-        def source_key(key, &url_parser_block)
+        def source_key(key, &source_key_parser)
           __materialist_options[:source_key] = key
-          __materialist_options[:url_parser] = url_parser_block
+          __materialist_options[:source_key_parser] = source_key_parser
+        end
+
+        # This method is meant to be used for cases when the application needs
+        # to have access to the `payload` that is returned on the HTTP call.
+        # Such an example would be if the application logic requires all
+        # relationships to be present before the `resource` is saved in the
+        # database. Introduced in https://github.com/deliveroo/materialist/pull/47
+        def before_upsert_with_payload(*method_array)
+          __materialist_options[:before_upsert_with_payload] = method_array
         end
 
         def before_upsert(*method_array)

data/lib/materialist/materializer/internals/field_mapping.rb

@@ -2,13 +2,14 @@ module Materialist
   module Materializer
     module Internals
       class FieldMapping
-        def initialize(key:, as:)
+        def initialize(key:, as: key, value_parser: nil)
           @key = key
           @as = as
+          @value_parser = value_parser || ->value { value }
         end
 
         def map(resource)
-          { @as => resource.body[@key] }
+          { @as => @value_parser.call(resource.dig(@key)) }
         end
       end
     end

data/lib/materialist/materializer/internals/link_href_mapping.rb

@@ -9,8 +9,8 @@ module Materialist
         end
 
         def map(resource)
-          return unless link = resource.body._links[@key]
-          { @as => url_parser.call(link.href) }
+          return unless link = resource.dig(:_links, @key)
+          { @as => url_parser.call(link[:href]) }
         end
 
         private

data/lib/materialist/materializer/internals/link_mapping.rb

@@ -2,25 +2,22 @@ module Materialist
   module Materializer
     module Internals
       class LinkMapping
-        def initialize(key:, enable_caching: false)
+        def initialize(key:, mapping: [], enable_caching: false)
           @key = key
+          @mapping = mapping
           @enable_caching = enable_caching
-          @mapping = []
         end
 
+        attr_reader :mapping
+
         def map(resource)
           return unless linked_resource = linked_resource(resource)
           mapping.map{ |m| m.map(linked_resource) }.compact.reduce(&:merge)
         end
 
-        attr_reader :mapping
-
-        private
-
         def linked_resource(resource)
-          return unless resource.body._links.include?(@key)
-          return unless sub_resource = resource.send(@key)
-          sub_resource.show(enable_caching: @enable_caching)
+          return unless href = resource.dig(:_links, @key, :href)
+          resource.client.get(href, options: { enable_caching: @enable_caching, response_class: HateoasResource })
         rescue Routemaster::Errors::ResourceNotFound
           nil
         end

data/lib/materialist/materializer/internals/materializer.rb

@@ -1,18 +1,28 @@
-require 'routemaster/api_client'
 require_relative '../../workers/event'
+require_relative './resources'
 
 module Materialist
   module Materializer
     module Internals
       class Materializer
-        def initialize(url, klass)
+        def initialize(url, klass, resource_payload: nil, api_client: nil)
           @url = url
           @instance = klass.new
           @options = klass.__materialist_options
+          @api_client = api_client || Materialist.configuration.api_client
+          if resource_payload
+            @resource = PayloadResource.new(resource_payload, client: @api_client)
+          end
+        end
+
+        def perform(action)
+          action.to_sym == :delete ? destroy : upsert
         end
 
+        private
+
         def upsert(retry_on_race_condition: true)
-          return unless root_resource
+          return unless resource
 
           if materialize_self?
             upsert_record.tap do |entity|
@@ -42,18 +52,17 @@ module Materialist
           end
         end
 
-        private
-
-        attr_reader :url, :instance, :options
+        attr_reader :url, :instance, :options, :api_client
 
         def materialize_self?
           options.include? :model_class
         end
 
         def upsert_record
-          model_class.find_or_initialize_by(source_lookup(url)).tap do |entity|
+          model_class.find_or_initialize_by(source_lookup(url, resource)).tap do |entity|
             send_messages(before_upsert, entity) unless before_upsert.nil?
-            entity.update_attributes! attributes
+            before_upsert_with_payload&.each { |m| instance.send(m, entity, resource) }
+            entity.update_attributes!(attributes)
           end
         end
 
@@ -63,16 +72,10 @@ module Materialist
         end
 
         def materialize_link(key, opts)
-          return unless root_resource.body._links.include?(key)
+          return unless link = resource.dig(:_links, key)
+          return unless materializer_class = MaterializerFactory.class_from_topic(opts.fetch(:topic))
 
-          # this can't happen asynchronously
-          # because the handler options are unavailable in this context
-          # :(
-          ::Materialist::Workers::Event.new.perform({
-            'topic' => opts[:topic],
-            'url' => root_resource.body._links[key].href,
-            'type' => 'noop'
-          })
+          materializer_class.perform(link[:href], :noop)
         end
 
         def mappings
@@ -83,6 +86,10 @@ module Materialist
           options[:before_upsert]
         end
 
+        def before_upsert_with_payload
+          options[:before_upsert_with_payload]
+        end
+
         def after_upsert
           options[:after_upsert]
         end
@@ -103,30 +110,26 @@ module Materialist
           options.fetch(:source_key, :source_url)
         end
 
-        def url_parser
-          options[:url_parser] || ->url { url }
+        def source_key_parser
+          options[:source_key_parser] || ->(url, data) { url }
         end
 
-        def source_lookup(url)
-          @_source_lookup ||= { source_key => url_parser.call(url) }
+        def source_lookup(url, resource={})
+          @_source_lookup ||= { source_key => source_key_parser.call(url, resource) }
         end
 
         def attributes
-          mappings.map{ |m| m.map(root_resource) }.compact.reduce(&:merge) || {}
+          mappings.map{ |m| m.map(resource) }.compact.reduce(&:merge) || {}
         end
 
-        def root_resource
-          @_root_resource ||= begin
-            api_client.get(url, options: { enable_caching: false })
-          rescue Routemaster::Errors::ResourceNotFound
-            nil
-          end
+        def resource
+          @resource ||= fetch_resource
         end
 
-        def api_client
-          @_api_client ||= Routemaster::APIClient.new(
-            response_class: Routemaster::Responses::HateoasResponse
-          )
+        def fetch_resource
+          api_client.get(url, options: { enable_caching: false, response_class: HateoasResource })
+        rescue Routemaster::Errors::ResourceNotFound
+          nil
         end
 
         def send_messages(messages, arguments)

data/lib/materialist/materializer/internals/resources.rb

@@ -0,0 +1,22 @@
+module Materialist
+  module Materializer
+    module Internals
+      class PayloadResource
+        attr_reader :client
+
+        delegate :dig, to: :@payload
+
+        def initialize(payload, client:)
+          @payload = payload
+          @client = client
+        end
+      end
+
+      class HateoasResource < PayloadResource
+        def initialize(response, client:)
+          super(response.body, client: client)
+        end
+      end
+    end
+  end
+end

data/lib/materialist/materializer_factory.rb

@@ -1,9 +1,7 @@
 module Materialist
   class MaterializerFactory
     def self.class_from_topic(topic)
-      "#{topic.to_s.singularize.classify}Materializer".constantize
-    rescue NameError
-      nil
+      "#{topic.to_s.singularize.classify}Materializer".safe_constantize
     end
   end
 end

data/lib/materialist/version.rb

@@ -0,0 +1,3 @@
+module Materialist
+  VERSION = '3.8.0'
+end

data/materialist.gemspec

@@ -1,7 +1,7 @@
 # coding: utf-8
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'materialist'
+require 'materialist/version'
 
 Gem::Specification.new do |spec|
   spec.name          = 'materialist'
@@ -17,7 +17,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = %w(lib)
 
   spec.add_runtime_dependency 'sidekiq', '>= 5.0'
-  spec.add_runtime_dependency 'activesupport'
+  spec.add_runtime_dependency 'activesupport', '< 6.0'
   spec.add_runtime_dependency 'routemaster-drain', '>= 3.0'
 
   spec.add_development_dependency 'activerecord'

data/spec/materialist/materializer/internals/field_mapping_spec.rb

@@ -0,0 +1,36 @@
+require 'spec_helper'
+require 'materialist/materializer/internals'
+
+include Materialist::Materializer::Internals
+
+RSpec.describe Materialist::Materializer::Internals::FieldMapping, type: :internals do
+  let(:instance) { described_class.new(key: key, as: as, value_parser: value_parser_block) }
+
+  describe '#map' do
+    let(:key) { :b }
+    let(:as) { :z }
+    let(:value_parser_block) { nil }
+    let(:resource) do
+      {
+        a: 1,
+        b: {
+          c: 2
+        }
+      }
+    end
+    let(:map) { instance.map(resource) }
+
+    context 'when no parse block is passed' do
+      let(:expected_result) { { z: { c: 2 } } }
+
+      it { expect(map).to eq(expected_result) }
+    end
+
+    context 'when a value parse block is passed' do
+      let(:value_parser_block) { ->value { value[:c] } }
+      let(:expected_result) { { z: 2 } }
+
+      it { expect(map).to eq(expected_result) }
+    end
+  end
+end

data/spec/materialist/materializer/internals/link_mapping_spec.rb

@@ -0,0 +1,86 @@
+require 'spec_helper'
+require 'materialist/materializer/internals'
+
+include Materialist::Materializer::Internals
+
+RSpec.describe Materialist::Materializer::Internals::LinkMapping, type: :internals do
+  let(:key) { :details }
+  subject(:instance) { described_class.new(key: key, mapping: mappings) }
+  let(:mappings) { [ FieldMapping.new(key: :name) ] }
+
+  let(:url_sub) { 'https://deliveroo.co.uk/details/1001' }
+  let(:payload) { { _links: { details: { href: url_sub } } } }
+  let(:payload_sub) { { name: "jack", age: 20 } }
+
+  let(:client) { double }
+  let(:resource_class) { Materialist::Materializer::Internals::PayloadResource }
+  let(:resource) { resource_class.new(payload, client: client) }
+  let(:resource_sub) { resource_class.new(payload_sub, client: client) }
+
+  before do
+    allow(client).to receive(:get).with(url_sub, anything).and_return resource_sub
+  end
+
+  describe '#map' do
+    subject(:perform) { instance.map resource }
+
+    it 'returns a hash corresponding to the mapping' do
+      is_expected.to eq({ name: 'jack' })
+    end
+
+    context 'when multiple mappings given' do
+      let(:mappings) do
+        [
+          FieldMapping.new(key: :age),
+          LinkMapping.new(key: :wont_find_me),
+          FieldMapping.new(key: :name)
+        ]
+      end
+
+      it 'returns a hash corresponding to the mapping' do
+        is_expected.to eq({ age: 20, name: 'jack' })
+      end
+    end
+
+    describe 'missing sub resource' do
+      context 'when given link is not present' do
+        let(:key) { :foo }
+
+        it { is_expected.to be nil }
+      end
+
+      context 'when given link is malformed' do
+        let(:payload) { { _links: { details: { foo: url_sub } } } }
+
+        it { is_expected.to be nil }
+      end
+
+      context 'when client returns nil' do
+        let(:resource_sub) { nil }
+
+        it { is_expected.to be nil }
+      end
+
+      context 'when client throws not-found error' do
+        before do
+          allow(client).to receive(:get)
+            .with(url_sub, anything)
+            .and_raise Routemaster::Errors::ResourceNotFound.new(double(body: nil))
+        end
+
+        it { is_expected.to be nil }
+      end
+
+      context 'when caching enabled' do
+        let(:instance) { described_class.new(key: key, mapping: mappings, enable_caching: true) }
+
+        it 'passes on the option to client' do
+          expect(client).to receive(:get)
+            .with(url_sub, options: { enable_caching: true, response_class: HateoasResource})
+          perform
+        end
+      end
+    end
+  end
+
+end

data/spec/materialist/materializer_spec.rb

@@ -2,7 +2,7 @@ require 'spec_helper'
 require 'support/uses_redis'
 require 'materialist/materializer'
 
-RSpec.describe Materialist::Materializer do
+RSpec.describe Materialist::Materializer::Internals::Materializer do
   uses_redis
 
   describe ".perform" do
@@ -50,7 +50,7 @@ RSpec.describe Materialist::Materializer do
     let(:source_body) {{ _links: { city: { href: city_url }}, name: 'jack', age: 30 }}
     let(:defined_source_id) { 65 }
     let(:defined_source_url) { "https://service.dev/defined_sources/#{defined_source_id}" }
-    let(:defined_source_body) {{ name: 'ben' }}
+    let(:defined_source_body) {{ name: 'ben', id: defined_source_id }}
 
     def stub_resource(url, body)
       stub_request(:get, url).to_return(
@@ -60,8 +60,9 @@ RSpec.describe Materialist::Materializer do
       )
     end
 
+    let!(:source_stub) { stub_resource source_url, source_body }
+
     before do
-      stub_resource source_url, source_body
       stub_resource country_url, country_body
       stub_resource city_url, city_body
       stub_resource defined_source_url, defined_source_body
@@ -77,7 +78,7 @@ RSpec.describe Materialist::Materializer do
     let(:actions_called) { materializer_class.class_variable_get(:@@actions_called) }
 
     it "materializes record in db" do
-      expect{perform}.to change{Foobar.count}.by 1
+      expect{ perform }.to change{ Foobar.count }.by 1
       inserted = Foobar.find_by(source_url: source_url)
       expect(inserted.name).to eq source_body[:name]
       expect(inserted.how_old).to eq source_body[:age]
@@ -95,6 +96,23 @@ RSpec.describe Materialist::Materializer do
       expect(inserted.name).to eq city_body[:name]
     end
 
+    context 'when materializing using payload' do
+      let(:perform) { materializer_class.perform(source_url, action, resource_payload: source_body) }
+
+      it 'does not fetch resource' do
+        perform
+        expect(source_stub).to_not have_been_requested
+      end
+
+      it "materializes record in db" do
+        expect{ perform }.to change{ Foobar.count }.by 1
+      end
+
+      it "materializes linked record in db" do
+        expect{ perform }.to change{ City.count }.by 1
+      end
+    end
+
     context "when record already exists" do
       let!(:record) { Foobar.create!(source_url: source_url, name: 'mo') }
 
@@ -181,8 +199,13 @@ RSpec.describe Materialist::Materializer do
 
           persist_to :foobar
           before_upsert :before_hook
+          before_upsert_with_payload :before_hook_with_payload
           after_upsert :after_hook
 
+          def before_hook_with_payload(entity, payload)
+            self.actions_called[:before_hook_with_payload] = true
+          end
+
           def before_hook(entity); self.actions_called[:before_hook] = true; end
           def after_hook(entity); self.actions_called[:after_hook] = true; end
         end
@@ -191,6 +214,11 @@ RSpec.describe Materialist::Materializer do
       %i(create update noop).each do |action_name|
         context "when action is :#{action_name}" do
           let(:action) { action_name }
+
+          it "calls before_upsert_with_payload method" do
+            expect{ perform }.to change { actions_called[:before_hook_with_payload] }
+          end
+
           it "calls before_upsert method" do
             expect{ perform }.to change { actions_called[:before_hook] }
           end
@@ -207,8 +235,16 @@ RSpec.describe Materialist::Materializer do
 
                 persist_to :foobar
                 before_upsert :before_hook, :before_hook2
+                before_upsert_with_payload :before_hook_with_payload, :before_hook_with_payload2
                 after_upsert :after_hook, :after_hook2
 
+                def before_hook_with_payload(entity, payload)
+                  self.actions_called[:before_hook_with_payload] = true
+                end
+                def before_hook_with_payload2(entity, payload)
+                  self.actions_called[:before_hook_with_payload2] = true
+                end
+
                 def before_hook(entity); self.actions_called[:before_hook] = true; end
                 def before_hook2(entity); self.actions_called[:before_hook2] = true; end
                 def after_hook(entity); self.actions_called[:after_hook] = true; end
@@ -221,6 +257,8 @@ RSpec.describe Materialist::Materializer do
                                .and change { actions_called[:before_hook2] }
                                .and change { actions_called[:after_hook] }
                                .and change { actions_called[:after_hook2] }
+                               .and change { actions_called[:before_hook_with_payload] }
+                               .and change { actions_called[:before_hook_with_payload2] }
             end
           end
         end
@@ -342,62 +380,87 @@ RSpec.describe Materialist::Materializer do
     end
 
     context "entity based on the source_key column" do
-      subject do
-        Class.new do
-          include Materialist::Materializer
-
-          persist_to :defined_source
+      shared_examples 'an upsert materialization event' do
+        context "when creating" do
+          let(:perform) { subject.perform(defined_source_url, action) }
 
-          source_key :source_id do |url|
-            url.split('/').last.to_i
+          it "creates based on source_key" do
+            expect{perform}.to change{DefinedSource.count}.by 1
           end
 
-          capture :name
+          it "sets the correct source key" do
+            perform
+            inserted = DefinedSource.find_by(source_id: defined_source_id)
+            expect(inserted.source_id).to eq defined_source_id
+            expect(inserted.name).to eq defined_source_body[:name]
+          end
         end
-      end
 
-      context "when creating" do
-        let(:perform) { subject.perform(defined_source_url, action) }
+        context "when updating" do
+          let(:action) { :update }
+          let!(:record) { DefinedSource.create!(source_id: defined_source_id, name: 'mo') }
+          let(:perform) { subject.perform(defined_source_url, action) }
 
-        it "creates based on source_key" do
-          expect{perform}.to change{DefinedSource.count}.by 1
-        end
+          it "updates based on source_key" do
+            perform
+            expect(DefinedSource.count).to eq 1
+          end
 
-        it "sets the correct source key" do
-          perform
-          inserted = DefinedSource.find_by(source_id: defined_source_id)
-          expect(inserted.source_id).to eq defined_source_id
-          expect(inserted.name).to eq defined_source_body[:name]
+          it "updates the existing record" do
+            perform
+            inserted = DefinedSource.find_by(source_id: defined_source_id)
+            expect(inserted.source_id).to eq defined_source_id
+            expect(inserted.name).to eq defined_source_body[:name]
+          end
         end
       end
 
-      context "when updating" do
-        let(:action) { :update }
-        let!(:record) { DefinedSource.create!(source_id: defined_source_id, name: 'mo') }
-        let(:perform) { subject.perform(defined_source_url, action) }
+      context 'with url source key parser' do
+        subject do
+          Class.new do
+            include Materialist::Materializer
 
-        it "updates based on source_key" do
-          perform
-          expect(DefinedSource.count).to eq 1
+            persist_to :defined_source
+
+            source_key :source_id do |url|
+              url.split('/').last.to_i
+            end
+
+            capture :name
+          end
         end
 
-        it "updates the existing record" do
-          perform
-          inserted = DefinedSource.find_by(source_id: defined_source_id)
-          expect(inserted.source_id).to eq defined_source_id
-          expect(inserted.name).to eq defined_source_body[:name]
+        context "when deleting" do
+          let(:action) { :delete }
+          let!(:record) { DefinedSource.create!(source_id: defined_source_id, name: 'mo') }
+          let(:perform) { subject.perform(defined_source_url, action) }
+
+          it "deletes based on source_key" do
+            perform
+            expect(DefinedSource.count).to eq 0
+          end
         end
+
+        it_behaves_like 'an upsert materialization event'
       end
 
-      context "when deleting" do
-        let(:action) { :delete }
-        let!(:record) { DefinedSource.create!(source_id: defined_source_id, name: 'mo') }
-        let(:perform) { subject.perform(defined_source_url, action) }
+      context 'with resource source key parser' do
+        subject do
+          Class.new do
+            include Materialist::Materializer
+
+            persist_to :defined_source
 
-        it "deletes based on source_key" do
-          perform
-          expect(DefinedSource.count).to eq 0
+            source_key :source_id do |_, resource|
+              resource.dig(:id)
+            end
+
+            capture :name
+            capture :id
+          end
         end
+
+        it_behaves_like 'an upsert materialization event'
       end
     end
   end

data/spec/spec_helper.rb

@@ -8,6 +8,7 @@ require 'webmock/rspec'
 require 'dotenv'
 require 'pry'
 Dotenv.overload('.env.test')
+require 'materialist'
 
 # This file was generated by the `rspec --init` command. Conventionally, all
 # specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
@@ -21,7 +22,7 @@ RSpec.configure do |config|
 
   config.before(:each) do
     Materialist.reset_configuration!
-    
+
     # clear database
     ActiveRecord::Base.descendants.each(&:delete_all)
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: materialist
 version: !ruby/object:Gem::Version
-  version: 3.3.0
+  version: 3.8.0
 platform: ruby
 authors:
 - Mo Valipour
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-05-02 00:00:00.000000000 Z
+date: 2021-03-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -28,16 +28,16 @@ dependencies:
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '6.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '6.0'
 - !ruby/object:Gem::Dependency
   name: routemaster-drain
   requirement: !ruby/object:Gem::Requirement
@@ -178,7 +178,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: 
+description:
 email:
 - valipour@gmail.com
 executables: []
@@ -212,11 +212,15 @@ files:
 - lib/materialist/materializer/internals/link_href_mapping.rb
 - lib/materialist/materializer/internals/link_mapping.rb
 - lib/materialist/materializer/internals/materializer.rb
+- lib/materialist/materializer/internals/resources.rb
 - lib/materialist/materializer_factory.rb
+- lib/materialist/version.rb
 - lib/materialist/workers/event.rb
 - materialist.gemspec
 - spec/materialist/event_handler_spec.rb
 - spec/materialist/materialized_record_spec.rb
+- spec/materialist/materializer/internals/field_mapping_spec.rb
+- spec/materialist/materializer/internals/link_mapping_spec.rb
 - spec/materialist/materializer_factory_spec.rb
 - spec/materialist/materializer_spec.rb
 - spec/materialist/workers/event_spec.rb
@@ -228,7 +232,7 @@ homepage: http://github.com/deliveroo/materialist
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -243,14 +247,15 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.14.1
-signing_key: 
+rubygems_version: 3.0.9
+signing_key:
 specification_version: 4
 summary: Utilities to materialize routemaster topics
 test_files:
 - spec/materialist/event_handler_spec.rb
 - spec/materialist/materialized_record_spec.rb
+- spec/materialist/materializer/internals/field_mapping_spec.rb
+- spec/materialist/materializer/internals/link_mapping_spec.rb
 - spec/materialist/materializer_factory_spec.rb
 - spec/materialist/materializer_spec.rb
 - spec/materialist/workers/event_spec.rb