materialist 3.6.0 → 3.8.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: dc140c911742867585c275c7b87a08cce340b165
-  data.tar.gz: dd08e90c55127a4b8375f859613ec73cda95970d
+SHA256:
+  metadata.gz: ce1a473a4a182ebbf80364ac65046c146796067e5901d2bec1fac4f24fe9d39f
+  data.tar.gz: 61a6d7e77e9d044490b5a2fc92a9c69ce4ece93237344a6b0332fb0d376c9dd8
 SHA512:
-  metadata.gz: 4bcb195f30deff05ab9c83de55475473f8efee6a599cb694f47fdb55438b5374988a6f12872d5078af57c76c978d45208f0829a962b1d41d79f9ef1c824740eb
-  data.tar.gz: 0d15e15c152b85765a3dca6c96dbe28787e780a15b68c505cf4dfd6d0c4e1b732c17c64275794733a096790999e3a0d59be32ba9e48a8c7d374c5fd1b4c6e33b
+  metadata.gz: b46b98674ee9d5482a505bb6aea3d8683b31b74e25a72fb1b25515d13a8d8c8de4afd7389d20b89ea8bbe0480fcab5a1699347f50bd9f38d807465a88c6816b9
+  data.tar.gz: 72eae6ef4cbe277c1c45a9ad5dff11ada17c8392b3ec709048e6baf1f85e7b195e184d41341b9634513d0520a8c0cd67ddda8f7ba33287f20b3bd918f3f20595

data/README.md

@@ -91,7 +91,7 @@ Materialist.configure do |config|
 end
 ```
 
-- `topics` (only when using in `.subscribe`): A string array of topics to be used.  
+- `topics` (only when using in `.subscribe`): A string array of topics to be used.
 If not provided nothing would be materialized.
 - `sidekiq_options` (optional, default: `{ retry: 10 }`) -- See [Sidekiq docs](https://github.com/mperham/sidekiq/wiki/Advanced-Options#workers) for list of options
 - `api_client` (optional) -- You can pass your `Routemaster::APIClient` instance
@@ -155,8 +155,8 @@ class ZoneMaterializer
 
   persist_to :zone
 
-  source_key :source_id do |url|
-    /(\d+)\/?$/.match(url)[1]
+  source_key :source_id do |url, response|
+    /(\d+)\/?$/.match(url)[1] # or response.dig(:some_attr)
   end
 
   capture :id, as: :orderweb_id
@@ -189,10 +189,10 @@ describes the name of the active record model to be used.
 If missing, materialist skips materialising the resource itself, but will continue
 with any other functionality -- such as `materialize_link`.
 
-#### `source_key <column> <url_parser_block> (default: url)`
+#### `source_key <column> <parser_block> (default: url, resource response body[create, update action only])`
 describes the column used to persist the unique identifier parsed from the url_parser_block.
 By default the column used is `:source_url` and the original `url` is used as the identifier.
-Passing an optional block allows you to extract an identifier from the URL.
+Passing an optional block allows you to extract an identifier from the URL and captured attributes.
 
 #### `capture <key>, as: <column> (default: key)`
 describes mapping a resource key to a database column.
@@ -243,6 +243,24 @@ class ZoneMaterializer
 end
 ```
 
+#### `before_upsert_with_payload <method> (, <method>(, ...))`
+describes the name of the instance method(s) to be invoked before a record is
+materialized, with the record as it exists in the database, or nil if it has
+not been created yet. The function will get as a second argument the `payload`
+of the HTTP response, this can be used to add additional information/persist
+other objects.
+
+
+```ruby
+class ZoneMaterializer
+  include Materialist::Materializer
+
+  before_upsert_with_payload :my_method
+
+  def my_method(record, payload); end
+end
+```
+
 
 #### `after_upsert <method> (, <method>(, ...))` -- also `after_destroy`
 describes the name of the instance method(s) to be invoked after a record was materialized, with the updated record as a parameter. See above for a similar example implementation.

data/RELEASE_NOTES.md

@@ -1,6 +1,13 @@
 ## Next
 
 _list pending release notes here..._
+## 3.8.2
+
+- Fix Rails 6 deprecation warning for `update_attributes!`
+
+## 3.8.1
+
+- ActiveSupport >= 6
 
 ## 3.6.0
 

data/lib/materialist/materializer/internals/dsl.rb

@@ -38,9 +38,18 @@ module Materialist
           __materialist_options[:sidekiq_options] = options
         end
 
-        def source_key(key, &url_parser_block)
+        def source_key(key, &source_key_parser)
           __materialist_options[:source_key] = key
-          __materialist_options[:url_parser] = url_parser_block
+          __materialist_options[:source_key_parser] = source_key_parser
+        end
+
+        # This method is meant to be used for cases when the application needs
+        # to have access to the `payload` that is returned on the HTTP call.
+        # Such an example would be if the application logic requires all
+        # relationships to be present before the `resource` is saved in the
+        # database. Introduced in https://github.com/deliveroo/materialist/pull/47
+        def before_upsert_with_payload(*method_array)
+          __materialist_options[:before_upsert_with_payload] = method_array
         end
 
         def before_upsert(*method_array)

data/lib/materialist/materializer/internals/materializer.rb

@@ -59,9 +59,10 @@ module Materialist
         end
 
         def upsert_record
-          model_class.find_or_initialize_by(source_lookup(url)).tap do |entity|
+          model_class.find_or_initialize_by(source_lookup(url, resource)).tap do |entity|
             send_messages(before_upsert, entity) unless before_upsert.nil?
-            entity.update_attributes!(attributes)
+            before_upsert_with_payload&.each { |m| instance.send(m, entity, resource) }
+            entity.update!(attributes)
           end
         end
 
@@ -74,7 +75,6 @@ module Materialist
           return unless link = resource.dig(:_links, key)
           return unless materializer_class = MaterializerFactory.class_from_topic(opts.fetch(:topic))
 
-          # TODO: perhaps consider doing this asynchronously some how?
           materializer_class.perform(link[:href], :noop)
         end
 
@@ -86,6 +86,10 @@ module Materialist
           options[:before_upsert]
         end
 
+        def before_upsert_with_payload
+          options[:before_upsert_with_payload]
+        end
+
         def after_upsert
           options[:after_upsert]
         end
@@ -106,12 +110,12 @@ module Materialist
           options.fetch(:source_key, :source_url)
         end
 
-        def url_parser
-          options[:url_parser] || ->url { url }
+        def source_key_parser
+          options[:source_key_parser] || ->(url, data) { url }
         end
 
-        def source_lookup(url)
-          @_source_lookup ||= { source_key => url_parser.call(url) }
+        def source_lookup(url, resource={})
+          @_source_lookup ||= { source_key => source_key_parser.call(url, resource) }
         end
 
         def attributes

data/lib/materialist/version.rb

@@ -1,3 +1,3 @@
 module Materialist
-  VERSION = '3.6.0'
+  VERSION = '3.8.2'
 end

data/spec/materialist/materializer_spec.rb

@@ -50,7 +50,7 @@ RSpec.describe Materialist::Materializer::Internals::Materializer do
     let(:source_body) {{ _links: { city: { href: city_url }}, name: 'jack', age: 30 }}
     let(:defined_source_id) { 65 }
     let(:defined_source_url) { "https://service.dev/defined_sources/#{defined_source_id}" }
-    let(:defined_source_body) {{ name: 'ben' }}
+    let(:defined_source_body) {{ name: 'ben', id: defined_source_id }}
 
     def stub_resource(url, body)
       stub_request(:get, url).to_return(
@@ -199,8 +199,13 @@ RSpec.describe Materialist::Materializer::Internals::Materializer do
 
           persist_to :foobar
           before_upsert :before_hook
+          before_upsert_with_payload :before_hook_with_payload
           after_upsert :after_hook
 
+          def before_hook_with_payload(entity, payload)
+            self.actions_called[:before_hook_with_payload] = true
+          end
+
           def before_hook(entity); self.actions_called[:before_hook] = true; end
           def after_hook(entity); self.actions_called[:after_hook] = true; end
         end
@@ -209,6 +214,11 @@ RSpec.describe Materialist::Materializer::Internals::Materializer do
       %i(create update noop).each do |action_name|
         context "when action is :#{action_name}" do
           let(:action) { action_name }
+
+          it "calls before_upsert_with_payload method" do
+            expect{ perform }.to change { actions_called[:before_hook_with_payload] }
+          end
+
           it "calls before_upsert method" do
             expect{ perform }.to change { actions_called[:before_hook] }
           end
@@ -225,8 +235,16 @@ RSpec.describe Materialist::Materializer::Internals::Materializer do
 
                 persist_to :foobar
                 before_upsert :before_hook, :before_hook2
+                before_upsert_with_payload :before_hook_with_payload, :before_hook_with_payload2
                 after_upsert :after_hook, :after_hook2
 
+                def before_hook_with_payload(entity, payload)
+                  self.actions_called[:before_hook_with_payload] = true
+                end
+                def before_hook_with_payload2(entity, payload)
+                  self.actions_called[:before_hook_with_payload2] = true
+                end
+
                 def before_hook(entity); self.actions_called[:before_hook] = true; end
                 def before_hook2(entity); self.actions_called[:before_hook2] = true; end
                 def after_hook(entity); self.actions_called[:after_hook] = true; end
@@ -239,6 +257,8 @@ RSpec.describe Materialist::Materializer::Internals::Materializer do
                                .and change { actions_called[:before_hook2] }
                                .and change { actions_called[:after_hook] }
                                .and change { actions_called[:after_hook2] }
+                               .and change { actions_called[:before_hook_with_payload] }
+                               .and change { actions_called[:before_hook_with_payload2] }
             end
           end
         end
@@ -360,62 +380,87 @@ RSpec.describe Materialist::Materializer::Internals::Materializer do
     end
 
     context "entity based on the source_key column" do
-      subject do
-        Class.new do
-          include Materialist::Materializer
+      shared_examples 'an upsert materialization event' do
+        context "when creating" do
+          let(:perform) { subject.perform(defined_source_url, action) }
 
-          persist_to :defined_source
-
-          source_key :source_id do |url|
-            url.split('/').last.to_i
+          it "creates based on source_key" do
+            expect{perform}.to change{DefinedSource.count}.by 1
           end
 
-          capture :name
+          it "sets the correct source key" do
+            perform
+            inserted = DefinedSource.find_by(source_id: defined_source_id)
+            expect(inserted.source_id).to eq defined_source_id
+            expect(inserted.name).to eq defined_source_body[:name]
+          end
         end
-      end
 
-      context "when creating" do
-        let(:perform) { subject.perform(defined_source_url, action) }
+        context "when updating" do
+          let(:action) { :update }
+          let!(:record) { DefinedSource.create!(source_id: defined_source_id, name: 'mo') }
+          let(:perform) { subject.perform(defined_source_url, action) }
 
-        it "creates based on source_key" do
-          expect{perform}.to change{DefinedSource.count}.by 1
-        end
+          it "updates based on source_key" do
+            perform
+            expect(DefinedSource.count).to eq 1
+          end
 
-        it "sets the correct source key" do
-          perform
-          inserted = DefinedSource.find_by(source_id: defined_source_id)
-          expect(inserted.source_id).to eq defined_source_id
-          expect(inserted.name).to eq defined_source_body[:name]
+          it "updates the existing record" do
+            perform
+            inserted = DefinedSource.find_by(source_id: defined_source_id)
+            expect(inserted.source_id).to eq defined_source_id
+            expect(inserted.name).to eq defined_source_body[:name]
+          end
         end
       end
 
-      context "when updating" do
-        let(:action) { :update }
-        let!(:record) { DefinedSource.create!(source_id: defined_source_id, name: 'mo') }
-        let(:perform) { subject.perform(defined_source_url, action) }
+      context 'with url source key parser' do
+        subject do
+          Class.new do
+            include Materialist::Materializer
+
+            persist_to :defined_source
+
+            source_key :source_id do |url|
+              url.split('/').last.to_i
+            end
 
-        it "updates based on source_key" do
-          perform
-          expect(DefinedSource.count).to eq 1
+            capture :name
+          end
         end
 
-        it "updates the existing record" do
-          perform
-          inserted = DefinedSource.find_by(source_id: defined_source_id)
-          expect(inserted.source_id).to eq defined_source_id
-          expect(inserted.name).to eq defined_source_body[:name]
+        context "when deleting" do
+          let(:action) { :delete }
+          let!(:record) { DefinedSource.create!(source_id: defined_source_id, name: 'mo') }
+          let(:perform) { subject.perform(defined_source_url, action) }
+
+          it "deletes based on source_key" do
+            perform
+            expect(DefinedSource.count).to eq 0
+          end
         end
+
+        it_behaves_like 'an upsert materialization event'
       end
 
-      context "when deleting" do
-        let(:action) { :delete }
-        let!(:record) { DefinedSource.create!(source_id: defined_source_id, name: 'mo') }
-        let(:perform) { subject.perform(defined_source_url, action) }
+      context 'with resource source key parser' do
+        subject do
+          Class.new do
+            include Materialist::Materializer
 
-        it "deletes based on source_key" do
-          perform
-          expect(DefinedSource.count).to eq 0
+            persist_to :defined_source
+
+            source_key :source_id do |_, resource|
+              resource.dig(:id)
+            end
+
+            capture :name
+            capture :id
+          end
         end
+
+        it_behaves_like 'an upsert materialization event'
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: materialist
 version: !ruby/object:Gem::Version
-  version: 3.6.0
+  version: 3.8.2
 platform: ruby
 authors:
 - Mo Valipour
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-06-12 00:00:00.000000000 Z
+date: 2021-11-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -178,7 +178,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: 
+description:
 email:
 - valipour@gmail.com
 executables: []
@@ -232,7 +232,7 @@ homepage: http://github.com/deliveroo/materialist
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -247,9 +247,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.14.1
-signing_key: 
+rubygems_version: 3.0.9
+signing_key:
 specification_version: 4
 summary: Utilities to materialize routemaster topics
 test_files: