materialist 3.7.0 → 3.8.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +4 -4
- data/RELEASE_NOTES.md +7 -0
- data/lib/configuration.rb +4 -3
- data/lib/materialist/materializer/internals/dsl.rb +2 -2
- data/lib/materialist/materializer/internals/materializer.rb +6 -6
- data/lib/materialist/version.rb +1 -1
- data/materialist.gemspec +1 -1
- data/spec/materialist/materialized_record_spec.rb +3 -0
- data/spec/materialist/materializer_spec.rb +67 -39
- data/spec/materialist/workers/event_spec.rb +5 -1
- metadata +11 -11
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: dd1ef7f5805827d74ef21e82711656a04d402a7b8e5e03a636538aced1e6f491
|
4
|
+
data.tar.gz: cf0e8a01fbdc6751f9eadb59307fc46cba53bb4ca04f562fe2c51531a893e051
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: de64d93e429085b0aefe268c346c42a56c669507e867f1b40ab7d0f7f6ddcf4b63023391e7468b8fac472304a1959e5eb0b0a8fda4c00c3ed26395a13f08195b
|
7
|
+
data.tar.gz: b593a01f94002824f1068dc36873ff1ae364d2b2a8147116ee97ecb5c408faf7202c60add6886e1ca5b35fc496c5b04cc32e16e43dc7d2766b3a53bec1594382
|
data/README.md
CHANGED
@@ -155,8 +155,8 @@ class ZoneMaterializer
|
|
155
155
|
|
156
156
|
persist_to :zone
|
157
157
|
|
158
|
-
source_key :source_id do |url|
|
159
|
-
/(\d+)\/?$/.match(url)[1]
|
158
|
+
source_key :source_id do |url, response|
|
159
|
+
/(\d+)\/?$/.match(url)[1] # or response.dig(:some_attr)
|
160
160
|
end
|
161
161
|
|
162
162
|
capture :id, as: :orderweb_id
|
@@ -189,10 +189,10 @@ describes the name of the active record model to be used.
|
|
189
189
|
If missing, materialist skips materialising the resource itself, but will continue
|
190
190
|
with any other functionality -- such as `materialize_link`.
|
191
191
|
|
192
|
-
#### `source_key <column> <
|
192
|
+
#### `source_key <column> <parser_block> (default: url, resource response body[create, update action only])`
|
193
193
|
describes the column used to persist the unique identifier parsed from the url_parser_block.
|
194
194
|
By default the column used is `:source_url` and the original `url` is used as the identifier.
|
195
|
-
Passing an optional block allows you to extract an identifier from the URL.
|
195
|
+
Passing an optional block allows you to extract an identifier from the URL and captured attributes.
|
196
196
|
|
197
197
|
#### `capture <key>, as: <column> (default: key)`
|
198
198
|
describes mapping a resource key to a database column.
|
data/RELEASE_NOTES.md
CHANGED
data/lib/configuration.rb
CHANGED
@@ -1,5 +1,3 @@
|
|
1
|
-
require 'routemaster/api_client'
|
2
|
-
|
3
1
|
module Materialist
|
4
2
|
class << self
|
5
3
|
def configuration
|
@@ -21,11 +19,14 @@ module Materialist
|
|
21
19
|
def initialize
|
22
20
|
@topics = []
|
23
21
|
@sidekiq_options = {}
|
24
|
-
@api_client = Routemaster::APIClient.new(response_class: ::Routemaster::Responses::HateoasResponse)
|
25
22
|
@metrics_client = NullMetricsClient
|
26
23
|
@notice_error = nil
|
27
24
|
end
|
28
25
|
|
26
|
+
def api_client
|
27
|
+
@api_client || (raise StandardError.new('missing HTTP client'))
|
28
|
+
end
|
29
|
+
|
29
30
|
class NullMetricsClient
|
30
31
|
def self.increment(_, tags:); end
|
31
32
|
def self.histogram(_, _v, tags:); end
|
@@ -38,9 +38,9 @@ module Materialist
|
|
38
38
|
__materialist_options[:sidekiq_options] = options
|
39
39
|
end
|
40
40
|
|
41
|
-
def source_key(key, &
|
41
|
+
def source_key(key, &source_key_parser)
|
42
42
|
__materialist_options[:source_key] = key
|
43
|
-
__materialist_options[:
|
43
|
+
__materialist_options[:source_key_parser] = source_key_parser
|
44
44
|
end
|
45
45
|
|
46
46
|
# This method is meant to be used for cases when the application needs
|
@@ -59,10 +59,10 @@ module Materialist
|
|
59
59
|
end
|
60
60
|
|
61
61
|
def upsert_record
|
62
|
-
model_class.find_or_initialize_by(source_lookup(url)).tap do |entity|
|
62
|
+
model_class.find_or_initialize_by(source_lookup(url, resource)).tap do |entity|
|
63
63
|
send_messages(before_upsert, entity) unless before_upsert.nil?
|
64
64
|
before_upsert_with_payload&.each { |m| instance.send(m, entity, resource) }
|
65
|
-
entity.
|
65
|
+
entity.update!(attributes)
|
66
66
|
end
|
67
67
|
end
|
68
68
|
|
@@ -110,12 +110,12 @@ module Materialist
|
|
110
110
|
options.fetch(:source_key, :source_url)
|
111
111
|
end
|
112
112
|
|
113
|
-
def
|
114
|
-
options[:
|
113
|
+
def source_key_parser
|
114
|
+
options[:source_key_parser] || ->(url, data) { url }
|
115
115
|
end
|
116
116
|
|
117
|
-
def source_lookup(url)
|
118
|
-
@_source_lookup ||= { source_key =>
|
117
|
+
def source_lookup(url, resource={})
|
118
|
+
@_source_lookup ||= { source_key => source_key_parser.call(url, resource) }
|
119
119
|
end
|
120
120
|
|
121
121
|
def attributes
|
data/lib/materialist/version.rb
CHANGED
data/materialist.gemspec
CHANGED
@@ -17,7 +17,7 @@ Gem::Specification.new do |spec|
|
|
17
17
|
spec.require_paths = %w(lib)
|
18
18
|
|
19
19
|
spec.add_runtime_dependency 'sidekiq', '>= 5.0'
|
20
|
-
spec.add_runtime_dependency 'activesupport'
|
20
|
+
spec.add_runtime_dependency 'activesupport'
|
21
21
|
spec.add_runtime_dependency 'routemaster-drain', '>= 3.0'
|
22
22
|
|
23
23
|
spec.add_development_dependency 'activerecord'
|
@@ -29,6 +29,9 @@ RSpec.describe Materialist::MaterializedRecord do
|
|
29
29
|
let(:source_url) { 'https://service.dev/foobars/1' }
|
30
30
|
let(:source_body) {{ _links: { city: { href: city_url }}, name: 'jack', age: 30 }}
|
31
31
|
before do
|
32
|
+
Materialist.configure do |c|
|
33
|
+
c.api_client = Routemaster::APIClient.new(response_class: ::Routemaster::Responses::HateoasResponse)
|
34
|
+
end
|
32
35
|
stub_request(:get, source_url).to_return(
|
33
36
|
status: 200,
|
34
37
|
body: source_body.to_json,
|
@@ -50,7 +50,7 @@ RSpec.describe Materialist::Materializer::Internals::Materializer do
|
|
50
50
|
let(:source_body) {{ _links: { city: { href: city_url }}, name: 'jack', age: 30 }}
|
51
51
|
let(:defined_source_id) { 65 }
|
52
52
|
let(:defined_source_url) { "https://service.dev/defined_sources/#{defined_source_id}" }
|
53
|
-
let(:defined_source_body) {{ name: 'ben' }}
|
53
|
+
let(:defined_source_body) {{ name: 'ben', id: defined_source_id }}
|
54
54
|
|
55
55
|
def stub_resource(url, body)
|
56
56
|
stub_request(:get, url).to_return(
|
@@ -63,6 +63,9 @@ RSpec.describe Materialist::Materializer::Internals::Materializer do
|
|
63
63
|
let!(:source_stub) { stub_resource source_url, source_body }
|
64
64
|
|
65
65
|
before do
|
66
|
+
Materialist.configure do |c|
|
67
|
+
c.api_client = Routemaster::APIClient.new(response_class: ::Routemaster::Responses::HateoasResponse)
|
68
|
+
end
|
66
69
|
stub_resource country_url, country_body
|
67
70
|
stub_resource city_url, city_body
|
68
71
|
stub_resource defined_source_url, defined_source_body
|
@@ -380,62 +383,87 @@ RSpec.describe Materialist::Materializer::Internals::Materializer do
|
|
380
383
|
end
|
381
384
|
|
382
385
|
context "entity based on the source_key column" do
|
383
|
-
|
384
|
-
|
385
|
-
|
386
|
-
|
387
|
-
persist_to :defined_source
|
386
|
+
shared_examples 'an upsert materialization event' do
|
387
|
+
context "when creating" do
|
388
|
+
let(:perform) { subject.perform(defined_source_url, action) }
|
388
389
|
|
389
|
-
source_key
|
390
|
-
|
390
|
+
it "creates based on source_key" do
|
391
|
+
expect{perform}.to change{DefinedSource.count}.by 1
|
391
392
|
end
|
392
393
|
|
393
|
-
|
394
|
+
it "sets the correct source key" do
|
395
|
+
perform
|
396
|
+
inserted = DefinedSource.find_by(source_id: defined_source_id)
|
397
|
+
expect(inserted.source_id).to eq defined_source_id
|
398
|
+
expect(inserted.name).to eq defined_source_body[:name]
|
399
|
+
end
|
394
400
|
end
|
395
|
-
end
|
396
401
|
|
397
|
-
|
398
|
-
|
402
|
+
context "when updating" do
|
403
|
+
let(:action) { :update }
|
404
|
+
let!(:record) { DefinedSource.create!(source_id: defined_source_id, name: 'mo') }
|
405
|
+
let(:perform) { subject.perform(defined_source_url, action) }
|
399
406
|
|
400
|
-
|
401
|
-
|
402
|
-
|
407
|
+
it "updates based on source_key" do
|
408
|
+
perform
|
409
|
+
expect(DefinedSource.count).to eq 1
|
410
|
+
end
|
403
411
|
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
408
|
-
|
412
|
+
it "updates the existing record" do
|
413
|
+
perform
|
414
|
+
inserted = DefinedSource.find_by(source_id: defined_source_id)
|
415
|
+
expect(inserted.source_id).to eq defined_source_id
|
416
|
+
expect(inserted.name).to eq defined_source_body[:name]
|
417
|
+
end
|
409
418
|
end
|
410
419
|
end
|
411
420
|
|
412
|
-
context
|
413
|
-
|
414
|
-
|
415
|
-
|
421
|
+
context 'with url source key parser' do
|
422
|
+
subject do
|
423
|
+
Class.new do
|
424
|
+
include Materialist::Materializer
|
425
|
+
|
426
|
+
persist_to :defined_source
|
416
427
|
|
417
|
-
|
418
|
-
|
419
|
-
|
428
|
+
source_key :source_id do |url|
|
429
|
+
url.split('/').last.to_i
|
430
|
+
end
|
431
|
+
|
432
|
+
capture :name
|
433
|
+
end
|
420
434
|
end
|
421
435
|
|
422
|
-
|
423
|
-
|
424
|
-
|
425
|
-
|
426
|
-
|
436
|
+
context "when deleting" do
|
437
|
+
let(:action) { :delete }
|
438
|
+
let!(:record) { DefinedSource.create!(source_id: defined_source_id, name: 'mo') }
|
439
|
+
let(:perform) { subject.perform(defined_source_url, action) }
|
440
|
+
|
441
|
+
it "deletes based on source_key" do
|
442
|
+
perform
|
443
|
+
expect(DefinedSource.count).to eq 0
|
444
|
+
end
|
427
445
|
end
|
446
|
+
|
447
|
+
it_behaves_like 'an upsert materialization event'
|
428
448
|
end
|
429
449
|
|
430
|
-
context
|
431
|
-
|
432
|
-
|
433
|
-
|
450
|
+
context 'with resource source key parser' do
|
451
|
+
subject do
|
452
|
+
Class.new do
|
453
|
+
include Materialist::Materializer
|
434
454
|
|
435
|
-
|
436
|
-
|
437
|
-
|
455
|
+
persist_to :defined_source
|
456
|
+
|
457
|
+
source_key :source_id do |_, resource|
|
458
|
+
resource.dig(:id)
|
459
|
+
end
|
460
|
+
|
461
|
+
capture :name
|
462
|
+
capture :id
|
463
|
+
end
|
438
464
|
end
|
465
|
+
|
466
|
+
it_behaves_like 'an upsert materialization event'
|
439
467
|
end
|
440
468
|
end
|
441
469
|
end
|
@@ -1,5 +1,6 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
require 'materialist/workers/event'
|
3
|
+
require 'routemaster/api_client'
|
3
4
|
|
4
5
|
RSpec.describe Materialist::Workers::Event do
|
5
6
|
describe "#perform" do
|
@@ -10,7 +11,10 @@ RSpec.describe Materialist::Workers::Event do
|
|
10
11
|
|
11
12
|
before do
|
12
13
|
allow(FoobarMaterializer).to receive(:perform)
|
13
|
-
Materialist.configure
|
14
|
+
Materialist.configure do |c|
|
15
|
+
c.metrics_client = metrics_client
|
16
|
+
c.api_client = Routemaster::APIClient.new(response_class: ::Routemaster::Responses::HateoasResponse)
|
17
|
+
end
|
14
18
|
end
|
15
19
|
|
16
20
|
after { Object.send(:remove_const, :FoobarMaterializer) }
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: materialist
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.8.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Mo Valipour
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-11-16 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: sidekiq
|
@@ -28,16 +28,16 @@ dependencies:
|
|
28
28
|
name: activesupport
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- - "
|
31
|
+
- - ">="
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: '
|
33
|
+
version: '0'
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- - "
|
38
|
+
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: '
|
40
|
+
version: '0'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: routemaster-drain
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
@@ -178,7 +178,7 @@ dependencies:
|
|
178
178
|
- - ">="
|
179
179
|
- !ruby/object:Gem::Version
|
180
180
|
version: '0'
|
181
|
-
description:
|
181
|
+
description:
|
182
182
|
email:
|
183
183
|
- valipour@gmail.com
|
184
184
|
executables: []
|
@@ -232,7 +232,7 @@ homepage: http://github.com/deliveroo/materialist
|
|
232
232
|
licenses:
|
233
233
|
- MIT
|
234
234
|
metadata: {}
|
235
|
-
post_install_message:
|
235
|
+
post_install_message:
|
236
236
|
rdoc_options: []
|
237
237
|
require_paths:
|
238
238
|
- lib
|
@@ -247,8 +247,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
247
247
|
- !ruby/object:Gem::Version
|
248
248
|
version: '0'
|
249
249
|
requirements: []
|
250
|
-
rubygems_version: 3.0.
|
251
|
-
signing_key:
|
250
|
+
rubygems_version: 3.0.9
|
251
|
+
signing_key:
|
252
252
|
specification_version: 4
|
253
253
|
summary: Utilities to materialize routemaster topics
|
254
254
|
test_files:
|