materialist 3.3.0 → 3.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/README.md +48 -5
- data/RELEASE_NOTES.md +14 -1
- data/lib/configuration.rb +4 -1
- data/lib/materialist.rb +1 -1
- data/lib/materialist/materialized_record.rb +1 -4
- data/lib/materialist/materializer/internals/class_methods.rb +2 -3
- data/lib/materialist/materializer/internals/dsl.rb +17 -4
- data/lib/materialist/materializer/internals/field_mapping.rb +3 -2
- data/lib/materialist/materializer/internals/link_href_mapping.rb +2 -2
- data/lib/materialist/materializer/internals/link_mapping.rb +6 -9
- data/lib/materialist/materializer/internals/materializer.rb +35 -32
- data/lib/materialist/materializer/internals/resources.rb +22 -0
- data/lib/materialist/materializer_factory.rb +1 -3
- data/lib/materialist/version.rb +3 -0
- data/materialist.gemspec +2 -2
- data/spec/materialist/materializer/internals/field_mapping_spec.rb +36 -0
- data/spec/materialist/materializer/internals/link_mapping_spec.rb +86 -0
- data/spec/materialist/materializer_spec.rb +105 -42
- data/spec/spec_helper.rb +2 -1
- metadata +17 -12
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 3cef103c710bdfcf4b9a5ee36bad6b4474bc88c194a436bb935aa15b090a69f0
|
4
|
+
data.tar.gz: f91fcd7467e60889d34ed33ed57c08ac67a6365a4168d5c52e7dd8564f2ec331
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: cc547ca1823460c517b5b1a8720e3259004bcbf1f8b92c91847d90d9ea6638af4ffb39d2f7a50994fc8070b997b1fb61b69a12ae49e2009125c61af4e2a5ac26
|
7
|
+
data.tar.gz: dd6b84e5541132b052bce821853348a62a0fd6608837f7d141b2bb472a6c32eb0fb01211ad07837c093ed62da9b55788a720034f1d398aef9e9eda7ba0b09d1a
|
data/README.md
CHANGED
@@ -14,6 +14,9 @@ In your `gemfile`
|
|
14
14
|
|
15
15
|
```ruby
|
16
16
|
gem 'materialist'
|
17
|
+
|
18
|
+
# or for sandbox features
|
19
|
+
gem 'materialist', github: 'deliveroo/materialist', branch: 'sandbox'
|
17
20
|
```
|
18
21
|
|
19
22
|
Then do
|
@@ -22,6 +25,20 @@ Then do
|
|
22
25
|
bundle
|
23
26
|
```
|
24
27
|
|
28
|
+
### Release
|
29
|
+
|
30
|
+
After merging all of your PRs:
|
31
|
+
|
32
|
+
1. Bump the version in `lib/materialist/version.rb` -- let's say `x.y.z`
|
33
|
+
1. Build the gem: `gem build materialist.gemspec`
|
34
|
+
1. Push the gem: `gem push materialist-x.y.z.gem`
|
35
|
+
1. Commit changes: `git commit -am "Bump version"`
|
36
|
+
1. Create a tag: `git tag -a vx.y.z`
|
37
|
+
1. Push changes: `git push origin master`
|
38
|
+
1. Push the new: `git push origin --tags`
|
39
|
+
|
40
|
+
## Usage
|
41
|
+
|
25
42
|
### Entity
|
26
43
|
|
27
44
|
Your materialised entity need to have a **unique** `source_url` column, alongside any other field you wish to materialise.
|
@@ -70,12 +87,14 @@ Materialist.configure do |config|
|
|
70
87
|
# }
|
71
88
|
#
|
72
89
|
# config.metrics_client = STATSD
|
90
|
+
# config.api_client = Routemaster::APIClient.new(response_class: Routemaster::Responses::HateoasResponse)
|
73
91
|
end
|
74
92
|
```
|
75
93
|
|
76
|
-
- `topics` (only when using in `.subscribe`): A string array of topics to be used.
|
94
|
+
- `topics` (only when using in `.subscribe`): A string array of topics to be used.
|
77
95
|
If not provided nothing would be materialized.
|
78
96
|
- `sidekiq_options` (optional, default: `{ retry: 10 }`) -- See [Sidekiq docs](https://github.com/mperham/sidekiq/wiki/Advanced-Options#workers) for list of options
|
97
|
+
- `api_client` (optional) -- You can pass your `Routemaster::APIClient` instance
|
79
98
|
- `metrics_client` (optional) -- You can pass your `STATSD` instance
|
80
99
|
- `notice_error` (optional) -- You can pass a lambda accepting two parameters (`exception` and `event`) -- Typical use case is to enrich error and send to NewRelic APM
|
81
100
|
|
@@ -136,8 +155,8 @@ class ZoneMaterializer
|
|
136
155
|
|
137
156
|
persist_to :zone
|
138
157
|
|
139
|
-
source_key :source_id do |url|
|
140
|
-
/(\d+)\/?$/.match(url)[1]
|
158
|
+
source_key :source_id do |url, response|
|
159
|
+
/(\d+)\/?$/.match(url)[1] # or response.dig(:some_attr)
|
141
160
|
end
|
142
161
|
|
143
162
|
capture :id, as: :orderweb_id
|
@@ -170,14 +189,20 @@ describes the name of the active record model to be used.
|
|
170
189
|
If missing, materialist skips materialising the resource itself, but will continue
|
171
190
|
with any other functionality -- such as `materialize_link`.
|
172
191
|
|
173
|
-
#### `source_key <column> <
|
192
|
+
#### `source_key <column> <parser_block> (default: url, resource response body[create, update action only])`
|
174
193
|
describes the column used to persist the unique identifier parsed from the url_parser_block.
|
175
194
|
By default the column used is `:source_url` and the original `url` is used as the identifier.
|
176
|
-
Passing an optional block allows you to extract an identifier from the URL.
|
195
|
+
Passing an optional block allows you to extract an identifier from the URL and captured attributes.
|
177
196
|
|
178
197
|
#### `capture <key>, as: <column> (default: key)`
|
179
198
|
describes mapping a resource key to a database column.
|
180
199
|
|
200
|
+
You can optionally provide a block for parsing the value:
|
201
|
+
|
202
|
+
```ruby
|
203
|
+
capture(:location, as: :latitude) { |location| location[:latitude] }
|
204
|
+
```
|
205
|
+
|
181
206
|
#### `capture_link_href <key>, as: <column>`
|
182
207
|
describes mapping a link href (as it appears on the hateous response) to a database column.
|
183
208
|
|
@@ -218,6 +243,24 @@ class ZoneMaterializer
|
|
218
243
|
end
|
219
244
|
```
|
220
245
|
|
246
|
+
#### `before_upsert_with_payload <method> (, <method>(, ...))`
|
247
|
+
describes the name of the instance method(s) to be invoked before a record is
|
248
|
+
materialized, with the record as it exists in the database, or nil if it has
|
249
|
+
not been created yet. The function will get as a second argument the `payload`
|
250
|
+
of the HTTP response, this can be used to add additional information/persist
|
251
|
+
other objects.
|
252
|
+
|
253
|
+
|
254
|
+
```ruby
|
255
|
+
class ZoneMaterializer
|
256
|
+
include Materialist::Materializer
|
257
|
+
|
258
|
+
before_upsert_with_payload :my_method
|
259
|
+
|
260
|
+
def my_method(record, payload); end
|
261
|
+
end
|
262
|
+
```
|
263
|
+
|
221
264
|
|
222
265
|
#### `after_upsert <method> (, <method>(, ...))` -- also `after_destroy`
|
223
266
|
describes the name of the instance method(s) to be invoked after a record was materialized, with the updated record as a parameter. See above for a similar example implementation.
|
data/RELEASE_NOTES.md
CHANGED
@@ -1,6 +1,19 @@
|
|
1
1
|
## Next
|
2
2
|
|
3
|
-
|
3
|
+
_list pending release notes here..._
|
4
|
+
|
5
|
+
## 3.6.0
|
6
|
+
|
7
|
+
- Add support for parsing value when using `capture`
|
8
|
+
|
9
|
+
## 3.5.0
|
10
|
+
|
11
|
+
- Add support for providing an `Routemaster::APIClient` instance as part of configuration
|
12
|
+
|
13
|
+
## 3.4.0
|
14
|
+
|
15
|
+
- Add support for providing payload into the materializer
|
16
|
+
- Enhance linked resource materialization to avoid `:noop` stats
|
4
17
|
|
5
18
|
## 3.3.0
|
6
19
|
|
data/lib/configuration.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
require 'routemaster/api_client'
|
2
|
+
|
1
3
|
module Materialist
|
2
4
|
class << self
|
3
5
|
def configuration
|
@@ -14,11 +16,12 @@ module Materialist
|
|
14
16
|
end
|
15
17
|
|
16
18
|
class Configuration
|
17
|
-
attr_accessor :topics, :sidekiq_options, :metrics_client, :notice_error
|
19
|
+
attr_accessor :topics, :sidekiq_options, :api_client, :metrics_client, :notice_error
|
18
20
|
|
19
21
|
def initialize
|
20
22
|
@topics = []
|
21
23
|
@sidekiq_options = {}
|
24
|
+
@api_client = Routemaster::APIClient.new(response_class: ::Routemaster::Responses::HateoasResponse)
|
22
25
|
@metrics_client = NullMetricsClient
|
23
26
|
@notice_error = nil
|
24
27
|
end
|
data/lib/materialist.rb
CHANGED
@@ -1,4 +1,3 @@
|
|
1
|
-
require 'routemaster/api_client'
|
2
1
|
require_relative './errors'
|
3
2
|
|
4
3
|
module Materialist
|
@@ -35,9 +34,7 @@ module Materialist
|
|
35
34
|
private
|
36
35
|
|
37
36
|
def source_raw
|
38
|
-
|
39
|
-
response_class: Routemaster::Responses::HateoasResponse
|
40
|
-
).get(source_url)
|
37
|
+
Materialist.configuration.api_client.get(source_url)
|
41
38
|
end
|
42
39
|
end
|
43
40
|
end
|
@@ -4,9 +4,8 @@ module Materialist
|
|
4
4
|
module ClassMethods
|
5
5
|
attr_reader :__materialist_options, :__materialist_dsl_mapping_stack
|
6
6
|
|
7
|
-
def perform(url, action)
|
8
|
-
|
9
|
-
action == :delete ? materializer.destroy : materializer.upsert
|
7
|
+
def perform(url, action, *options)
|
8
|
+
Materializer.new(url, self, *options).perform(action)
|
10
9
|
end
|
11
10
|
|
12
11
|
def _sidekiq_options
|
@@ -6,8 +6,12 @@ module Materialist
|
|
6
6
|
__materialist_options[:links_to_materialize][key] = { topic: topic }
|
7
7
|
end
|
8
8
|
|
9
|
-
def capture(key, as: key)
|
10
|
-
__materialist_dsl_mapping_stack.last << FieldMapping.new(
|
9
|
+
def capture(key, as: key, &value_parser_block)
|
10
|
+
__materialist_dsl_mapping_stack.last << FieldMapping.new(
|
11
|
+
key: key,
|
12
|
+
as: as,
|
13
|
+
value_parser: value_parser_block
|
14
|
+
)
|
11
15
|
end
|
12
16
|
|
13
17
|
def capture_link_href(key, as:, &url_parser_block)
|
@@ -34,9 +38,18 @@ module Materialist
|
|
34
38
|
__materialist_options[:sidekiq_options] = options
|
35
39
|
end
|
36
40
|
|
37
|
-
def source_key(key, &
|
41
|
+
def source_key(key, &source_key_parser)
|
38
42
|
__materialist_options[:source_key] = key
|
39
|
-
__materialist_options[:
|
43
|
+
__materialist_options[:source_key_parser] = source_key_parser
|
44
|
+
end
|
45
|
+
|
46
|
+
# This method is meant to be used for cases when the application needs
|
47
|
+
# to have access to the `payload` that is returned on the HTTP call.
|
48
|
+
# Such an example would be if the application logic requires all
|
49
|
+
# relationships to be present before the `resource` is saved in the
|
50
|
+
# database. Introduced in https://github.com/deliveroo/materialist/pull/47
|
51
|
+
def before_upsert_with_payload(*method_array)
|
52
|
+
__materialist_options[:before_upsert_with_payload] = method_array
|
40
53
|
end
|
41
54
|
|
42
55
|
def before_upsert(*method_array)
|
@@ -2,13 +2,14 @@ module Materialist
|
|
2
2
|
module Materializer
|
3
3
|
module Internals
|
4
4
|
class FieldMapping
|
5
|
-
def initialize(key:, as:)
|
5
|
+
def initialize(key:, as: key, value_parser: nil)
|
6
6
|
@key = key
|
7
7
|
@as = as
|
8
|
+
@value_parser = value_parser || ->value { value }
|
8
9
|
end
|
9
10
|
|
10
11
|
def map(resource)
|
11
|
-
{ @as => resource.
|
12
|
+
{ @as => @value_parser.call(resource.dig(@key)) }
|
12
13
|
end
|
13
14
|
end
|
14
15
|
end
|
@@ -2,25 +2,22 @@ module Materialist
|
|
2
2
|
module Materializer
|
3
3
|
module Internals
|
4
4
|
class LinkMapping
|
5
|
-
def initialize(key:, enable_caching: false)
|
5
|
+
def initialize(key:, mapping: [], enable_caching: false)
|
6
6
|
@key = key
|
7
|
+
@mapping = mapping
|
7
8
|
@enable_caching = enable_caching
|
8
|
-
@mapping = []
|
9
9
|
end
|
10
10
|
|
11
|
+
attr_reader :mapping
|
12
|
+
|
11
13
|
def map(resource)
|
12
14
|
return unless linked_resource = linked_resource(resource)
|
13
15
|
mapping.map{ |m| m.map(linked_resource) }.compact.reduce(&:merge)
|
14
16
|
end
|
15
17
|
|
16
|
-
attr_reader :mapping
|
17
|
-
|
18
|
-
private
|
19
|
-
|
20
18
|
def linked_resource(resource)
|
21
|
-
return unless resource.
|
22
|
-
|
23
|
-
sub_resource.show(enable_caching: @enable_caching)
|
19
|
+
return unless href = resource.dig(:_links, @key, :href)
|
20
|
+
resource.client.get(href, options: { enable_caching: @enable_caching, response_class: HateoasResource })
|
24
21
|
rescue Routemaster::Errors::ResourceNotFound
|
25
22
|
nil
|
26
23
|
end
|
@@ -1,18 +1,28 @@
|
|
1
|
-
require 'routemaster/api_client'
|
2
1
|
require_relative '../../workers/event'
|
2
|
+
require_relative './resources'
|
3
3
|
|
4
4
|
module Materialist
|
5
5
|
module Materializer
|
6
6
|
module Internals
|
7
7
|
class Materializer
|
8
|
-
def initialize(url, klass)
|
8
|
+
def initialize(url, klass, resource_payload: nil, api_client: nil)
|
9
9
|
@url = url
|
10
10
|
@instance = klass.new
|
11
11
|
@options = klass.__materialist_options
|
12
|
+
@api_client = api_client || Materialist.configuration.api_client
|
13
|
+
if resource_payload
|
14
|
+
@resource = PayloadResource.new(resource_payload, client: @api_client)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def perform(action)
|
19
|
+
action.to_sym == :delete ? destroy : upsert
|
12
20
|
end
|
13
21
|
|
22
|
+
private
|
23
|
+
|
14
24
|
def upsert(retry_on_race_condition: true)
|
15
|
-
return unless
|
25
|
+
return unless resource
|
16
26
|
|
17
27
|
if materialize_self?
|
18
28
|
upsert_record.tap do |entity|
|
@@ -42,18 +52,17 @@ module Materialist
|
|
42
52
|
end
|
43
53
|
end
|
44
54
|
|
45
|
-
|
46
|
-
|
47
|
-
attr_reader :url, :instance, :options
|
55
|
+
attr_reader :url, :instance, :options, :api_client
|
48
56
|
|
49
57
|
def materialize_self?
|
50
58
|
options.include? :model_class
|
51
59
|
end
|
52
60
|
|
53
61
|
def upsert_record
|
54
|
-
model_class.find_or_initialize_by(source_lookup(url)).tap do |entity|
|
62
|
+
model_class.find_or_initialize_by(source_lookup(url, resource)).tap do |entity|
|
55
63
|
send_messages(before_upsert, entity) unless before_upsert.nil?
|
56
|
-
|
64
|
+
before_upsert_with_payload&.each { |m| instance.send(m, entity, resource) }
|
65
|
+
entity.update_attributes!(attributes)
|
57
66
|
end
|
58
67
|
end
|
59
68
|
|
@@ -63,16 +72,10 @@ module Materialist
|
|
63
72
|
end
|
64
73
|
|
65
74
|
def materialize_link(key, opts)
|
66
|
-
return unless
|
75
|
+
return unless link = resource.dig(:_links, key)
|
76
|
+
return unless materializer_class = MaterializerFactory.class_from_topic(opts.fetch(:topic))
|
67
77
|
|
68
|
-
|
69
|
-
# because the handler options are unavailable in this context
|
70
|
-
# :(
|
71
|
-
::Materialist::Workers::Event.new.perform({
|
72
|
-
'topic' => opts[:topic],
|
73
|
-
'url' => root_resource.body._links[key].href,
|
74
|
-
'type' => 'noop'
|
75
|
-
})
|
78
|
+
materializer_class.perform(link[:href], :noop)
|
76
79
|
end
|
77
80
|
|
78
81
|
def mappings
|
@@ -83,6 +86,10 @@ module Materialist
|
|
83
86
|
options[:before_upsert]
|
84
87
|
end
|
85
88
|
|
89
|
+
def before_upsert_with_payload
|
90
|
+
options[:before_upsert_with_payload]
|
91
|
+
end
|
92
|
+
|
86
93
|
def after_upsert
|
87
94
|
options[:after_upsert]
|
88
95
|
end
|
@@ -103,30 +110,26 @@ module Materialist
|
|
103
110
|
options.fetch(:source_key, :source_url)
|
104
111
|
end
|
105
112
|
|
106
|
-
def
|
107
|
-
options[:
|
113
|
+
def source_key_parser
|
114
|
+
options[:source_key_parser] || ->(url, data) { url }
|
108
115
|
end
|
109
116
|
|
110
|
-
def source_lookup(url)
|
111
|
-
@_source_lookup ||= { source_key =>
|
117
|
+
def source_lookup(url, resource={})
|
118
|
+
@_source_lookup ||= { source_key => source_key_parser.call(url, resource) }
|
112
119
|
end
|
113
120
|
|
114
121
|
def attributes
|
115
|
-
mappings.map{ |m| m.map(
|
122
|
+
mappings.map{ |m| m.map(resource) }.compact.reduce(&:merge) || {}
|
116
123
|
end
|
117
124
|
|
118
|
-
def
|
119
|
-
@
|
120
|
-
api_client.get(url, options: { enable_caching: false })
|
121
|
-
rescue Routemaster::Errors::ResourceNotFound
|
122
|
-
nil
|
123
|
-
end
|
125
|
+
def resource
|
126
|
+
@resource ||= fetch_resource
|
124
127
|
end
|
125
128
|
|
126
|
-
def
|
127
|
-
|
128
|
-
|
129
|
-
|
129
|
+
def fetch_resource
|
130
|
+
api_client.get(url, options: { enable_caching: false, response_class: HateoasResource })
|
131
|
+
rescue Routemaster::Errors::ResourceNotFound
|
132
|
+
nil
|
130
133
|
end
|
131
134
|
|
132
135
|
def send_messages(messages, arguments)
|
@@ -0,0 +1,22 @@
|
|
1
|
+
module Materialist
|
2
|
+
module Materializer
|
3
|
+
module Internals
|
4
|
+
class PayloadResource
|
5
|
+
attr_reader :client
|
6
|
+
|
7
|
+
delegate :dig, to: :@payload
|
8
|
+
|
9
|
+
def initialize(payload, client:)
|
10
|
+
@payload = payload
|
11
|
+
@client = client
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
class HateoasResource < PayloadResource
|
16
|
+
def initialize(response, client:)
|
17
|
+
super(response.body, client: client)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
data/materialist.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
# coding: utf-8
|
2
2
|
lib = File.expand_path('../lib', __FILE__)
|
3
3
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
-
require 'materialist'
|
4
|
+
require 'materialist/version'
|
5
5
|
|
6
6
|
Gem::Specification.new do |spec|
|
7
7
|
spec.name = 'materialist'
|
@@ -17,7 +17,7 @@ Gem::Specification.new do |spec|
|
|
17
17
|
spec.require_paths = %w(lib)
|
18
18
|
|
19
19
|
spec.add_runtime_dependency 'sidekiq', '>= 5.0'
|
20
|
-
spec.add_runtime_dependency 'activesupport'
|
20
|
+
spec.add_runtime_dependency 'activesupport', '< 6.0'
|
21
21
|
spec.add_runtime_dependency 'routemaster-drain', '>= 3.0'
|
22
22
|
|
23
23
|
spec.add_development_dependency 'activerecord'
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'materialist/materializer/internals'
|
3
|
+
|
4
|
+
include Materialist::Materializer::Internals
|
5
|
+
|
6
|
+
RSpec.describe Materialist::Materializer::Internals::FieldMapping, type: :internals do
|
7
|
+
let(:instance) { described_class.new(key: key, as: as, value_parser: value_parser_block) }
|
8
|
+
|
9
|
+
describe '#map' do
|
10
|
+
let(:key) { :b }
|
11
|
+
let(:as) { :z }
|
12
|
+
let(:value_parser_block) { nil }
|
13
|
+
let(:resource) do
|
14
|
+
{
|
15
|
+
a: 1,
|
16
|
+
b: {
|
17
|
+
c: 2
|
18
|
+
}
|
19
|
+
}
|
20
|
+
end
|
21
|
+
let(:map) { instance.map(resource) }
|
22
|
+
|
23
|
+
context 'when no parse block is passed' do
|
24
|
+
let(:expected_result) { { z: { c: 2 } } }
|
25
|
+
|
26
|
+
it { expect(map).to eq(expected_result) }
|
27
|
+
end
|
28
|
+
|
29
|
+
context 'when a value parse block is passed' do
|
30
|
+
let(:value_parser_block) { ->value { value[:c] } }
|
31
|
+
let(:expected_result) { { z: 2 } }
|
32
|
+
|
33
|
+
it { expect(map).to eq(expected_result) }
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,86 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'materialist/materializer/internals'
|
3
|
+
|
4
|
+
include Materialist::Materializer::Internals
|
5
|
+
|
6
|
+
RSpec.describe Materialist::Materializer::Internals::LinkMapping, type: :internals do
|
7
|
+
let(:key) { :details }
|
8
|
+
subject(:instance) { described_class.new(key: key, mapping: mappings) }
|
9
|
+
let(:mappings) { [ FieldMapping.new(key: :name) ] }
|
10
|
+
|
11
|
+
let(:url_sub) { 'https://deliveroo.co.uk/details/1001' }
|
12
|
+
let(:payload) { { _links: { details: { href: url_sub } } } }
|
13
|
+
let(:payload_sub) { { name: "jack", age: 20 } }
|
14
|
+
|
15
|
+
let(:client) { double }
|
16
|
+
let(:resource_class) { Materialist::Materializer::Internals::PayloadResource }
|
17
|
+
let(:resource) { resource_class.new(payload, client: client) }
|
18
|
+
let(:resource_sub) { resource_class.new(payload_sub, client: client) }
|
19
|
+
|
20
|
+
before do
|
21
|
+
allow(client).to receive(:get).with(url_sub, anything).and_return resource_sub
|
22
|
+
end
|
23
|
+
|
24
|
+
describe '#map' do
|
25
|
+
subject(:perform) { instance.map resource }
|
26
|
+
|
27
|
+
it 'returns a hash corresponding to the mapping' do
|
28
|
+
is_expected.to eq({ name: 'jack' })
|
29
|
+
end
|
30
|
+
|
31
|
+
context 'when multiple mappings given' do
|
32
|
+
let(:mappings) do
|
33
|
+
[
|
34
|
+
FieldMapping.new(key: :age),
|
35
|
+
LinkMapping.new(key: :wont_find_me),
|
36
|
+
FieldMapping.new(key: :name)
|
37
|
+
]
|
38
|
+
end
|
39
|
+
|
40
|
+
it 'returns a hash corresponding to the mapping' do
|
41
|
+
is_expected.to eq({ age: 20, name: 'jack' })
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
describe 'missing sub resource' do
|
46
|
+
context 'when given link is not present' do
|
47
|
+
let(:key) { :foo }
|
48
|
+
|
49
|
+
it { is_expected.to be nil }
|
50
|
+
end
|
51
|
+
|
52
|
+
context 'when given link is malformed' do
|
53
|
+
let(:payload) { { _links: { details: { foo: url_sub } } } }
|
54
|
+
|
55
|
+
it { is_expected.to be nil }
|
56
|
+
end
|
57
|
+
|
58
|
+
context 'when client returns nil' do
|
59
|
+
let(:resource_sub) { nil }
|
60
|
+
|
61
|
+
it { is_expected.to be nil }
|
62
|
+
end
|
63
|
+
|
64
|
+
context 'when client throws not-found error' do
|
65
|
+
before do
|
66
|
+
allow(client).to receive(:get)
|
67
|
+
.with(url_sub, anything)
|
68
|
+
.and_raise Routemaster::Errors::ResourceNotFound.new(double(body: nil))
|
69
|
+
end
|
70
|
+
|
71
|
+
it { is_expected.to be nil }
|
72
|
+
end
|
73
|
+
|
74
|
+
context 'when caching enabled' do
|
75
|
+
let(:instance) { described_class.new(key: key, mapping: mappings, enable_caching: true) }
|
76
|
+
|
77
|
+
it 'passes on the option to client' do
|
78
|
+
expect(client).to receive(:get)
|
79
|
+
.with(url_sub, options: { enable_caching: true, response_class: HateoasResource})
|
80
|
+
perform
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
end
|
@@ -2,7 +2,7 @@ require 'spec_helper'
|
|
2
2
|
require 'support/uses_redis'
|
3
3
|
require 'materialist/materializer'
|
4
4
|
|
5
|
-
RSpec.describe Materialist::Materializer do
|
5
|
+
RSpec.describe Materialist::Materializer::Internals::Materializer do
|
6
6
|
uses_redis
|
7
7
|
|
8
8
|
describe ".perform" do
|
@@ -50,7 +50,7 @@ RSpec.describe Materialist::Materializer do
|
|
50
50
|
let(:source_body) {{ _links: { city: { href: city_url }}, name: 'jack', age: 30 }}
|
51
51
|
let(:defined_source_id) { 65 }
|
52
52
|
let(:defined_source_url) { "https://service.dev/defined_sources/#{defined_source_id}" }
|
53
|
-
let(:defined_source_body) {{ name: 'ben' }}
|
53
|
+
let(:defined_source_body) {{ name: 'ben', id: defined_source_id }}
|
54
54
|
|
55
55
|
def stub_resource(url, body)
|
56
56
|
stub_request(:get, url).to_return(
|
@@ -60,8 +60,9 @@ RSpec.describe Materialist::Materializer do
|
|
60
60
|
)
|
61
61
|
end
|
62
62
|
|
63
|
+
let!(:source_stub) { stub_resource source_url, source_body }
|
64
|
+
|
63
65
|
before do
|
64
|
-
stub_resource source_url, source_body
|
65
66
|
stub_resource country_url, country_body
|
66
67
|
stub_resource city_url, city_body
|
67
68
|
stub_resource defined_source_url, defined_source_body
|
@@ -77,7 +78,7 @@ RSpec.describe Materialist::Materializer do
|
|
77
78
|
let(:actions_called) { materializer_class.class_variable_get(:@@actions_called) }
|
78
79
|
|
79
80
|
it "materializes record in db" do
|
80
|
-
expect{perform}.to change{Foobar.count}.by 1
|
81
|
+
expect{ perform }.to change{ Foobar.count }.by 1
|
81
82
|
inserted = Foobar.find_by(source_url: source_url)
|
82
83
|
expect(inserted.name).to eq source_body[:name]
|
83
84
|
expect(inserted.how_old).to eq source_body[:age]
|
@@ -95,6 +96,23 @@ RSpec.describe Materialist::Materializer do
|
|
95
96
|
expect(inserted.name).to eq city_body[:name]
|
96
97
|
end
|
97
98
|
|
99
|
+
context 'when materializing using payload' do
|
100
|
+
let(:perform) { materializer_class.perform(source_url, action, resource_payload: source_body) }
|
101
|
+
|
102
|
+
it 'does not fetch resource' do
|
103
|
+
perform
|
104
|
+
expect(source_stub).to_not have_been_requested
|
105
|
+
end
|
106
|
+
|
107
|
+
it "materializes record in db" do
|
108
|
+
expect{ perform }.to change{ Foobar.count }.by 1
|
109
|
+
end
|
110
|
+
|
111
|
+
it "materializes linked record in db" do
|
112
|
+
expect{ perform }.to change{ City.count }.by 1
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
98
116
|
context "when record already exists" do
|
99
117
|
let!(:record) { Foobar.create!(source_url: source_url, name: 'mo') }
|
100
118
|
|
@@ -181,8 +199,13 @@ RSpec.describe Materialist::Materializer do
|
|
181
199
|
|
182
200
|
persist_to :foobar
|
183
201
|
before_upsert :before_hook
|
202
|
+
before_upsert_with_payload :before_hook_with_payload
|
184
203
|
after_upsert :after_hook
|
185
204
|
|
205
|
+
def before_hook_with_payload(entity, payload)
|
206
|
+
self.actions_called[:before_hook_with_payload] = true
|
207
|
+
end
|
208
|
+
|
186
209
|
def before_hook(entity); self.actions_called[:before_hook] = true; end
|
187
210
|
def after_hook(entity); self.actions_called[:after_hook] = true; end
|
188
211
|
end
|
@@ -191,6 +214,11 @@ RSpec.describe Materialist::Materializer do
|
|
191
214
|
%i(create update noop).each do |action_name|
|
192
215
|
context "when action is :#{action_name}" do
|
193
216
|
let(:action) { action_name }
|
217
|
+
|
218
|
+
it "calls before_upsert_with_payload method" do
|
219
|
+
expect{ perform }.to change { actions_called[:before_hook_with_payload] }
|
220
|
+
end
|
221
|
+
|
194
222
|
it "calls before_upsert method" do
|
195
223
|
expect{ perform }.to change { actions_called[:before_hook] }
|
196
224
|
end
|
@@ -207,8 +235,16 @@ RSpec.describe Materialist::Materializer do
|
|
207
235
|
|
208
236
|
persist_to :foobar
|
209
237
|
before_upsert :before_hook, :before_hook2
|
238
|
+
before_upsert_with_payload :before_hook_with_payload, :before_hook_with_payload2
|
210
239
|
after_upsert :after_hook, :after_hook2
|
211
240
|
|
241
|
+
def before_hook_with_payload(entity, payload)
|
242
|
+
self.actions_called[:before_hook_with_payload] = true
|
243
|
+
end
|
244
|
+
def before_hook_with_payload2(entity, payload)
|
245
|
+
self.actions_called[:before_hook_with_payload2] = true
|
246
|
+
end
|
247
|
+
|
212
248
|
def before_hook(entity); self.actions_called[:before_hook] = true; end
|
213
249
|
def before_hook2(entity); self.actions_called[:before_hook2] = true; end
|
214
250
|
def after_hook(entity); self.actions_called[:after_hook] = true; end
|
@@ -221,6 +257,8 @@ RSpec.describe Materialist::Materializer do
|
|
221
257
|
.and change { actions_called[:before_hook2] }
|
222
258
|
.and change { actions_called[:after_hook] }
|
223
259
|
.and change { actions_called[:after_hook2] }
|
260
|
+
.and change { actions_called[:before_hook_with_payload] }
|
261
|
+
.and change { actions_called[:before_hook_with_payload2] }
|
224
262
|
end
|
225
263
|
end
|
226
264
|
end
|
@@ -342,62 +380,87 @@ RSpec.describe Materialist::Materializer do
|
|
342
380
|
end
|
343
381
|
|
344
382
|
context "entity based on the source_key column" do
|
345
|
-
|
346
|
-
|
347
|
-
|
348
|
-
|
349
|
-
persist_to :defined_source
|
383
|
+
shared_examples 'an upsert materialization event' do
|
384
|
+
context "when creating" do
|
385
|
+
let(:perform) { subject.perform(defined_source_url, action) }
|
350
386
|
|
351
|
-
source_key
|
352
|
-
|
387
|
+
it "creates based on source_key" do
|
388
|
+
expect{perform}.to change{DefinedSource.count}.by 1
|
353
389
|
end
|
354
390
|
|
355
|
-
|
391
|
+
it "sets the correct source key" do
|
392
|
+
perform
|
393
|
+
inserted = DefinedSource.find_by(source_id: defined_source_id)
|
394
|
+
expect(inserted.source_id).to eq defined_source_id
|
395
|
+
expect(inserted.name).to eq defined_source_body[:name]
|
396
|
+
end
|
356
397
|
end
|
357
|
-
end
|
358
398
|
|
359
|
-
|
360
|
-
|
399
|
+
context "when updating" do
|
400
|
+
let(:action) { :update }
|
401
|
+
let!(:record) { DefinedSource.create!(source_id: defined_source_id, name: 'mo') }
|
402
|
+
let(:perform) { subject.perform(defined_source_url, action) }
|
361
403
|
|
362
|
-
|
363
|
-
|
364
|
-
|
404
|
+
it "updates based on source_key" do
|
405
|
+
perform
|
406
|
+
expect(DefinedSource.count).to eq 1
|
407
|
+
end
|
365
408
|
|
366
|
-
|
367
|
-
|
368
|
-
|
369
|
-
|
370
|
-
|
409
|
+
it "updates the existing record" do
|
410
|
+
perform
|
411
|
+
inserted = DefinedSource.find_by(source_id: defined_source_id)
|
412
|
+
expect(inserted.source_id).to eq defined_source_id
|
413
|
+
expect(inserted.name).to eq defined_source_body[:name]
|
414
|
+
end
|
371
415
|
end
|
372
416
|
end
|
373
417
|
|
374
|
-
context
|
375
|
-
|
376
|
-
|
377
|
-
|
418
|
+
context 'with url source key parser' do
|
419
|
+
subject do
|
420
|
+
Class.new do
|
421
|
+
include Materialist::Materializer
|
378
422
|
|
379
|
-
|
380
|
-
|
381
|
-
|
423
|
+
persist_to :defined_source
|
424
|
+
|
425
|
+
source_key :source_id do |url|
|
426
|
+
url.split('/').last.to_i
|
427
|
+
end
|
428
|
+
|
429
|
+
capture :name
|
430
|
+
end
|
382
431
|
end
|
383
432
|
|
384
|
-
|
385
|
-
|
386
|
-
|
387
|
-
|
388
|
-
|
433
|
+
context "when deleting" do
|
434
|
+
let(:action) { :delete }
|
435
|
+
let!(:record) { DefinedSource.create!(source_id: defined_source_id, name: 'mo') }
|
436
|
+
let(:perform) { subject.perform(defined_source_url, action) }
|
437
|
+
|
438
|
+
it "deletes based on source_key" do
|
439
|
+
perform
|
440
|
+
expect(DefinedSource.count).to eq 0
|
441
|
+
end
|
389
442
|
end
|
443
|
+
|
444
|
+
it_behaves_like 'an upsert materialization event'
|
390
445
|
end
|
391
446
|
|
392
|
-
context
|
393
|
-
|
394
|
-
|
395
|
-
|
447
|
+
context 'with resource source key parser' do
|
448
|
+
subject do
|
449
|
+
Class.new do
|
450
|
+
include Materialist::Materializer
|
451
|
+
|
452
|
+
persist_to :defined_source
|
396
453
|
|
397
|
-
|
398
|
-
|
399
|
-
|
454
|
+
source_key :source_id do |_, resource|
|
455
|
+
resource.dig(:id)
|
456
|
+
end
|
457
|
+
|
458
|
+
capture :name
|
459
|
+
capture :id
|
460
|
+
end
|
400
461
|
end
|
462
|
+
|
463
|
+
it_behaves_like 'an upsert materialization event'
|
401
464
|
end
|
402
465
|
end
|
403
466
|
end
|
data/spec/spec_helper.rb
CHANGED
@@ -8,6 +8,7 @@ require 'webmock/rspec'
|
|
8
8
|
require 'dotenv'
|
9
9
|
require 'pry'
|
10
10
|
Dotenv.overload('.env.test')
|
11
|
+
require 'materialist'
|
11
12
|
|
12
13
|
# This file was generated by the `rspec --init` command. Conventionally, all
|
13
14
|
# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
|
@@ -21,7 +22,7 @@ RSpec.configure do |config|
|
|
21
22
|
|
22
23
|
config.before(:each) do
|
23
24
|
Materialist.reset_configuration!
|
24
|
-
|
25
|
+
|
25
26
|
# clear database
|
26
27
|
ActiveRecord::Base.descendants.each(&:delete_all)
|
27
28
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: materialist
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.8.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Mo Valipour
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-03-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: sidekiq
|
@@ -28,16 +28,16 @@ dependencies:
|
|
28
28
|
name: activesupport
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- - "
|
31
|
+
- - "<"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: '0'
|
33
|
+
version: '6.0'
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- - "
|
38
|
+
- - "<"
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: '0'
|
40
|
+
version: '6.0'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: routemaster-drain
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
@@ -178,7 +178,7 @@ dependencies:
|
|
178
178
|
- - ">="
|
179
179
|
- !ruby/object:Gem::Version
|
180
180
|
version: '0'
|
181
|
-
description:
|
181
|
+
description:
|
182
182
|
email:
|
183
183
|
- valipour@gmail.com
|
184
184
|
executables: []
|
@@ -212,11 +212,15 @@ files:
|
|
212
212
|
- lib/materialist/materializer/internals/link_href_mapping.rb
|
213
213
|
- lib/materialist/materializer/internals/link_mapping.rb
|
214
214
|
- lib/materialist/materializer/internals/materializer.rb
|
215
|
+
- lib/materialist/materializer/internals/resources.rb
|
215
216
|
- lib/materialist/materializer_factory.rb
|
217
|
+
- lib/materialist/version.rb
|
216
218
|
- lib/materialist/workers/event.rb
|
217
219
|
- materialist.gemspec
|
218
220
|
- spec/materialist/event_handler_spec.rb
|
219
221
|
- spec/materialist/materialized_record_spec.rb
|
222
|
+
- spec/materialist/materializer/internals/field_mapping_spec.rb
|
223
|
+
- spec/materialist/materializer/internals/link_mapping_spec.rb
|
220
224
|
- spec/materialist/materializer_factory_spec.rb
|
221
225
|
- spec/materialist/materializer_spec.rb
|
222
226
|
- spec/materialist/workers/event_spec.rb
|
@@ -228,7 +232,7 @@ homepage: http://github.com/deliveroo/materialist
|
|
228
232
|
licenses:
|
229
233
|
- MIT
|
230
234
|
metadata: {}
|
231
|
-
post_install_message:
|
235
|
+
post_install_message:
|
232
236
|
rdoc_options: []
|
233
237
|
require_paths:
|
234
238
|
- lib
|
@@ -243,14 +247,15 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
243
247
|
- !ruby/object:Gem::Version
|
244
248
|
version: '0'
|
245
249
|
requirements: []
|
246
|
-
|
247
|
-
|
248
|
-
signing_key:
|
250
|
+
rubygems_version: 3.0.9
|
251
|
+
signing_key:
|
249
252
|
specification_version: 4
|
250
253
|
summary: Utilities to materialize routemaster topics
|
251
254
|
test_files:
|
252
255
|
- spec/materialist/event_handler_spec.rb
|
253
256
|
- spec/materialist/materialized_record_spec.rb
|
257
|
+
- spec/materialist/materializer/internals/field_mapping_spec.rb
|
258
|
+
- spec/materialist/materializer/internals/link_mapping_spec.rb
|
254
259
|
- spec/materialist/materializer_factory_spec.rb
|
255
260
|
- spec/materialist/materializer_spec.rb
|
256
261
|
- spec/materialist/workers/event_spec.rb
|