pub_sub_model_sync 1.3.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ruby.yml +0 -2
- data/.gitignore +2 -1
- data/CHANGELOG.md +2 -0
- data/Gemfile.lock +20 -16
- data/README.md +18 -33
- data/lib/pub_sub_model_sync/config.rb +3 -2
- data/lib/pub_sub_model_sync/message_processor.rb +7 -2
- data/lib/pub_sub_model_sync/message_publisher.rb +5 -2
- data/lib/pub_sub_model_sync/mock_rabbit_service.rb +8 -0
- data/lib/pub_sub_model_sync/payload.rb +19 -1
- data/lib/pub_sub_model_sync/payload_cache_optimizer.rb +51 -0
- data/lib/pub_sub_model_sync/service_base.rb +12 -8
- data/lib/pub_sub_model_sync/service_google.rb +2 -3
- data/lib/pub_sub_model_sync/service_kafka.rb +2 -1
- data/lib/pub_sub_model_sync/service_rabbit.rb +6 -3
- data/lib/pub_sub_model_sync/version.rb +1 -1
- data/lib/pub_sub_model_sync.rb +1 -0
- data/pub_sub_model_sync.gemspec +1 -1
- metadata +3 -88
- data/samples/README.md +0 -50
- data/samples/app1/Dockerfile +0 -13
- data/samples/app1/Gemfile +0 -37
- data/samples/app1/Gemfile.lock +0 -171
- data/samples/app1/README.md +0 -24
- data/samples/app1/Rakefile +0 -6
- data/samples/app1/app/models/application_record.rb +0 -3
- data/samples/app1/app/models/concerns/.keep +0 -0
- data/samples/app1/app/models/post.rb +0 -19
- data/samples/app1/app/models/user.rb +0 -29
- data/samples/app1/bin/bundle +0 -114
- data/samples/app1/bin/rails +0 -5
- data/samples/app1/bin/rake +0 -5
- data/samples/app1/bin/setup +0 -33
- data/samples/app1/bin/spring +0 -14
- data/samples/app1/config/application.rb +0 -40
- data/samples/app1/config/boot.rb +0 -4
- data/samples/app1/config/credentials.yml.enc +0 -1
- data/samples/app1/config/database.yml +0 -25
- data/samples/app1/config/environment.rb +0 -5
- data/samples/app1/config/environments/development.rb +0 -63
- data/samples/app1/config/environments/production.rb +0 -105
- data/samples/app1/config/environments/test.rb +0 -57
- data/samples/app1/config/initializers/application_controller_renderer.rb +0 -8
- data/samples/app1/config/initializers/backtrace_silencers.rb +0 -8
- data/samples/app1/config/initializers/cors.rb +0 -16
- data/samples/app1/config/initializers/filter_parameter_logging.rb +0 -6
- data/samples/app1/config/initializers/inflections.rb +0 -16
- data/samples/app1/config/initializers/mime_types.rb +0 -4
- data/samples/app1/config/initializers/pubsub.rb +0 -4
- data/samples/app1/config/initializers/wrap_parameters.rb +0 -14
- data/samples/app1/config/locales/en.yml +0 -33
- data/samples/app1/config/master.key +0 -1
- data/samples/app1/config/puma.rb +0 -43
- data/samples/app1/config/routes.rb +0 -3
- data/samples/app1/config/spring.rb +0 -6
- data/samples/app1/config.ru +0 -6
- data/samples/app1/db/migrate/20210513080700_create_users.rb +0 -12
- data/samples/app1/db/migrate/20210513134332_create_posts.rb +0 -11
- data/samples/app1/db/schema.rb +0 -34
- data/samples/app1/db/seeds.rb +0 -7
- data/samples/app1/docker-compose.yml +0 -32
- data/samples/app1/log/.keep +0 -0
- data/samples/app2/Dockerfile +0 -13
- data/samples/app2/Gemfile +0 -37
- data/samples/app2/Gemfile.lock +0 -171
- data/samples/app2/README.md +0 -24
- data/samples/app2/Rakefile +0 -6
- data/samples/app2/app/models/application_record.rb +0 -9
- data/samples/app2/app/models/concerns/.keep +0 -0
- data/samples/app2/app/models/customer.rb +0 -28
- data/samples/app2/app/models/post.rb +0 -10
- data/samples/app2/bin/bundle +0 -114
- data/samples/app2/bin/rails +0 -5
- data/samples/app2/bin/rake +0 -5
- data/samples/app2/bin/setup +0 -33
- data/samples/app2/bin/spring +0 -14
- data/samples/app2/config/application.rb +0 -40
- data/samples/app2/config/boot.rb +0 -4
- data/samples/app2/config/credentials.yml.enc +0 -1
- data/samples/app2/config/database.yml +0 -25
- data/samples/app2/config/environment.rb +0 -5
- data/samples/app2/config/environments/development.rb +0 -63
- data/samples/app2/config/environments/production.rb +0 -105
- data/samples/app2/config/environments/test.rb +0 -57
- data/samples/app2/config/initializers/application_controller_renderer.rb +0 -8
- data/samples/app2/config/initializers/backtrace_silencers.rb +0 -8
- data/samples/app2/config/initializers/cors.rb +0 -16
- data/samples/app2/config/initializers/filter_parameter_logging.rb +0 -6
- data/samples/app2/config/initializers/inflections.rb +0 -16
- data/samples/app2/config/initializers/mime_types.rb +0 -4
- data/samples/app2/config/initializers/pubsub.rb +0 -4
- data/samples/app2/config/initializers/wrap_parameters.rb +0 -14
- data/samples/app2/config/locales/en.yml +0 -33
- data/samples/app2/config/master.key +0 -1
- data/samples/app2/config/puma.rb +0 -43
- data/samples/app2/config/routes.rb +0 -3
- data/samples/app2/config/spring.rb +0 -6
- data/samples/app2/config.ru +0 -6
- data/samples/app2/db/development.sqlite3 +0 -0
- data/samples/app2/db/migrate/20210513080956_create_customers.rb +0 -10
- data/samples/app2/db/migrate/20210513135203_create_posts.rb +0 -10
- data/samples/app2/db/schema.rb +0 -31
- data/samples/app2/db/seeds.rb +0 -7
- data/samples/app2/docker-compose.yml +0 -20
- data/samples/app2/log/.keep +0 -0
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: '0839d268607d201c9f77bbbca404f32aa5af02621b7582256eb384b42b9d3985'
|
|
4
|
+
data.tar.gz: 0ad3d2cfdce3ce99eb5d0b95a4292496927befaca2a8e2d83b865af411be2988
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: c86be499e21572b796060fc07f5d95191971b87366bbebe3b22737d5eb906f25a6e850f4b4b6f2c08f2790541169f36401fb67b95df0c5da365fda1f76e8751f
|
|
7
|
+
data.tar.gz: 3e91e306f83eb4ba8dd5c494aadc90b36a702935d7fb716424af80efee6fe6bce9ff895ae6231aa1c32327d7f468ed2148aa69a5f37842c10e0f50e206dad82a
|
data/.github/workflows/ruby.yml
CHANGED
data/.gitignore
CHANGED
data/CHANGELOG.md
CHANGED
data/Gemfile.lock
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
pub_sub_model_sync (1.
|
|
4
|
+
pub_sub_model_sync (1.5.0)
|
|
5
5
|
rails
|
|
6
6
|
|
|
7
7
|
GEM
|
|
@@ -99,8 +99,8 @@ GEM
|
|
|
99
99
|
googleapis-common-protos-types (>= 1.0.6, < 2.0)
|
|
100
100
|
googleauth (~> 0.15, >= 0.15.1)
|
|
101
101
|
grpc (~> 1.36)
|
|
102
|
-
globalid (0.
|
|
103
|
-
activesupport (>=
|
|
102
|
+
globalid (1.0.0)
|
|
103
|
+
activesupport (>= 5.0)
|
|
104
104
|
google-cloud-core (1.6.0)
|
|
105
105
|
google-cloud-env (~> 1.0)
|
|
106
106
|
google-cloud-errors (~> 1.0)
|
|
@@ -116,6 +116,7 @@ GEM
|
|
|
116
116
|
google-cloud-errors (~> 1.0)
|
|
117
117
|
grpc-google-iam-v1 (>= 0.6.10, < 2.0)
|
|
118
118
|
google-protobuf (3.17.0)
|
|
119
|
+
google-protobuf (3.17.0-x86_64-linux)
|
|
119
120
|
googleapis-common-protos (1.3.11)
|
|
120
121
|
google-protobuf (~> 3.14)
|
|
121
122
|
googleapis-common-protos-types (>= 1.0.6, < 2.0)
|
|
@@ -132,6 +133,9 @@ GEM
|
|
|
132
133
|
grpc (1.37.1)
|
|
133
134
|
google-protobuf (~> 3.15)
|
|
134
135
|
googleapis-common-protos-types (~> 1.0)
|
|
136
|
+
grpc (1.37.1-x86_64-linux)
|
|
137
|
+
google-protobuf (~> 3.15)
|
|
138
|
+
googleapis-common-protos-types (~> 1.0)
|
|
135
139
|
grpc-google-iam-v1 (0.6.11)
|
|
136
140
|
google-protobuf (~> 3.14)
|
|
137
141
|
googleapis-common-protos (>= 1.3.11, < 2.0)
|
|
@@ -139,27 +143,27 @@ GEM
|
|
|
139
143
|
i18n (1.8.10)
|
|
140
144
|
concurrent-ruby (~> 1.0)
|
|
141
145
|
jwt (2.2.3)
|
|
142
|
-
loofah (2.
|
|
146
|
+
loofah (2.15.0)
|
|
143
147
|
crass (~> 1.0.2)
|
|
144
148
|
nokogiri (>= 1.5.9)
|
|
145
149
|
mail (2.7.1)
|
|
146
150
|
mini_mime (>= 0.1.1)
|
|
147
|
-
marcel (1.0.
|
|
151
|
+
marcel (1.0.2)
|
|
148
152
|
memoist (0.16.2)
|
|
149
153
|
method_source (1.0.0)
|
|
150
154
|
mini_mime (1.0.3)
|
|
151
155
|
minitest (5.14.4)
|
|
152
156
|
multi_json (1.15.0)
|
|
153
157
|
multipart-post (2.1.1)
|
|
154
|
-
nio4r (2.5.
|
|
155
|
-
nokogiri (1.
|
|
158
|
+
nio4r (2.5.8)
|
|
159
|
+
nokogiri (1.12.5-x86_64-linux)
|
|
156
160
|
racc (~> 1.4)
|
|
157
161
|
os (1.1.1)
|
|
158
162
|
parallel (1.20.1)
|
|
159
163
|
parser (3.0.1.1)
|
|
160
164
|
ast (~> 2.4.1)
|
|
161
165
|
public_suffix (4.0.6)
|
|
162
|
-
racc (1.
|
|
166
|
+
racc (1.6.0)
|
|
163
167
|
rack (2.2.3)
|
|
164
168
|
rack-test (1.1.0)
|
|
165
169
|
rack (>= 1.0, < 3)
|
|
@@ -181,7 +185,7 @@ GEM
|
|
|
181
185
|
rails-dom-testing (2.0.3)
|
|
182
186
|
activesupport (>= 4.2.0)
|
|
183
187
|
nokogiri (>= 1.6)
|
|
184
|
-
rails-html-sanitizer (1.
|
|
188
|
+
rails-html-sanitizer (1.4.2)
|
|
185
189
|
loofah (~> 2.3)
|
|
186
190
|
railties (6.1.3.2)
|
|
187
191
|
actionpack (= 6.1.3.2)
|
|
@@ -226,19 +230,19 @@ GEM
|
|
|
226
230
|
faraday (>= 0.17.3, < 2.0)
|
|
227
231
|
jwt (>= 1.5, < 3.0)
|
|
228
232
|
multi_json (~> 1.10)
|
|
229
|
-
sprockets (4.0.
|
|
233
|
+
sprockets (4.0.3)
|
|
230
234
|
concurrent-ruby (~> 1.0)
|
|
231
235
|
rack (> 1, < 3)
|
|
232
|
-
sprockets-rails (3.
|
|
233
|
-
actionpack (>=
|
|
234
|
-
activesupport (>=
|
|
236
|
+
sprockets-rails (3.4.2)
|
|
237
|
+
actionpack (>= 5.2)
|
|
238
|
+
activesupport (>= 5.2)
|
|
235
239
|
sprockets (>= 3.0.0)
|
|
236
240
|
sqlite3 (1.4.2)
|
|
237
|
-
thor (1.1
|
|
241
|
+
thor (1.2.1)
|
|
238
242
|
tzinfo (2.0.4)
|
|
239
243
|
concurrent-ruby (~> 1.0)
|
|
240
244
|
unicode-display_width (1.7.0)
|
|
241
|
-
websocket-driver (0.7.
|
|
245
|
+
websocket-driver (0.7.5)
|
|
242
246
|
websocket-extensions (>= 0.1.0)
|
|
243
247
|
websocket-extensions (0.1.5)
|
|
244
248
|
zeitwerk (2.4.2)
|
|
@@ -260,4 +264,4 @@ DEPENDENCIES
|
|
|
260
264
|
sqlite3
|
|
261
265
|
|
|
262
266
|
BUNDLED WITH
|
|
263
|
-
2.
|
|
267
|
+
2.3.9
|
data/README.md
CHANGED
|
@@ -167,7 +167,7 @@ PubSubModelSync::Payload.new({ ids: [my_user.id] }, { klass: 'User', action: :ba
|
|
|
167
167
|
ps_class_subscribe(action, settings)
|
|
168
168
|
end
|
|
169
169
|
```
|
|
170
|
-
- Instance subscriptions: `ps_subscribe(action, mapping, settings)`
|
|
170
|
+
- Instance subscriptions: `ps_subscribe(action, mapping, settings, &block)`
|
|
171
171
|
When model receives the corresponding notification, `action` or `to_action` method will be called on the model. Like: `model.destroy`
|
|
172
172
|
- `action` (Symbol|Array<Symbol>) Only notifications with this action name will be processed by this subscription. Sample: save|create|update|destroy|<any_other_action>
|
|
173
173
|
- `mapping` (Array<String>) Data mapping from payload data into model attributes, sample: ["email", "full_name:name"] (Note: Only these attributes will be assigned/synced to the current model)
|
|
@@ -182,12 +182,14 @@ PubSubModelSync::Payload.new({ ids: [my_user.id] }, { klass: 'User', action: :ba
|
|
|
182
182
|
Sample: `id: :id` will search for a model like: `model_class.where(id: payload.data[:id])`
|
|
183
183
|
Sample: `id: [:id, :email:user_email]` will search for a model like: `model_class.where(id: payload.data[:id], user_email: payload.data[:email])`
|
|
184
184
|
- `if:` (Symbol|Proc|Array<Symbol>) Method(s) or block called for the confirmation before calling the callback
|
|
185
|
-
- `unless:` (Symbol|Proc|Array<Symbol>) Method or block called for the negation before calling the callback
|
|
185
|
+
- `unless:` (Symbol|Proc|Array<Symbol>) Method or block called for the negation before calling the callback
|
|
186
|
+
- `&block` Block to be used as the callback method (ignored if `:to_action` is present). Sample: `ps_subscribe(:send_welcome, %i[email]) { |_data| puts model.email }`
|
|
186
187
|
|
|
187
|
-
- Class subscriptions: `ps_class_subscribe(action, settings)`
|
|
188
|
+
- Class subscriptions: `ps_class_subscribe(action, settings, &block)`
|
|
188
189
|
When current class receives the corresponding notification, `action` or `to_action` method will be called on the Class. Like: `User.hello(data)`
|
|
189
190
|
* `action` (Symbol) Notification.action name
|
|
190
191
|
* `settings` (Hash) refer ps_subscribe.settings except(:id)
|
|
192
|
+
* `&block` Block to be used as the callback method (ignored if `:to_action` is present). Sample: `ps_class_subscribe(:send_welcome) { |data| puts data }`
|
|
191
193
|
|
|
192
194
|
- `ps_processing_payload` a class and instance variable that saves the current payload being processed
|
|
193
195
|
|
|
@@ -313,10 +315,15 @@ Any notification before delivering is transformed as a Payload for a better port
|
|
|
313
315
|
* `headers`: (Hash) Notification settings that defines how the notification will be processed or delivered.
|
|
314
316
|
- `ordering_key`: (String, optional): notifications with the same `ordering_key` are processed in the same order they were delivered, default: `<model.class.name>/<model.id>` when instance notification and `klass_name` when class notification.
|
|
315
317
|
Note: Final `ordering_key` is calculated as: `payload.headers[:forced_ordering_key] || current_transaction&.key || payload.headers[:ordering_key]`
|
|
316
|
-
- `internal_key`: (String, optional) Internal identifier of the payload, default: `<model.class.name>/<action>/<model.id>` when model notification and `<klass_name>/<action>` when class notification (Useful for caching techniques).
|
|
317
318
|
- `topic_name`: (String|Array<String>, optional): Specific topic name where to deliver the notification (default `PubSubModelSync::Config.topic_name`).
|
|
318
319
|
- `forced_ordering_key`: (String, optional): Overrides `ordering_key` with the provided value even withing transactions. Default `nil`.
|
|
319
|
-
- `
|
|
320
|
+
- `cache` (Boolean | Hash, Default false) Cache settings
|
|
321
|
+
- `true`: Skip publishing similar payloads
|
|
322
|
+
- `Hash<required: Array<Symbol>>`: Same as `true` and enables payload optimization to exclude unchanged non important attributes. Sample: `{ required: %i[id email] }`
|
|
323
|
+
|
|
324
|
+
** Read ONLY **
|
|
325
|
+
- `internal_key`: Internal identifier of the payload, default: `<model.class.name>/<action>/<model.id>` when model notification and `<klass_name>/<action>` when class notification (Useful for caching techniques).
|
|
326
|
+
- `app_key`: (Auto calculated): Name of the application who delivered the notification.
|
|
320
327
|
- `uuid`: (Auto calculated): Unique notification identifier (Very useful when debugging).
|
|
321
328
|
Note: To reduce Payload size, some header info are not delivered (Enable debug mode to deliver all payload info).
|
|
322
329
|
|
|
@@ -386,37 +393,15 @@ Note: To reduce Payload size, some header info are not delivered (Enable debug m
|
|
|
386
393
|
## **Testing with RSpec**
|
|
387
394
|
- Config: (spec/rails_helper.rb)
|
|
388
395
|
```ruby
|
|
389
|
-
|
|
390
|
-
# when using google service
|
|
391
|
-
require 'pub_sub_model_sync/mock_google_service'
|
|
392
|
-
config.before(:each) do
|
|
393
|
-
google_mock = PubSubModelSync::MockGoogleService.new
|
|
394
|
-
allow(Google::Cloud::Pubsub).to receive(:new).and_return(google_mock)
|
|
395
|
-
end
|
|
396
|
-
|
|
397
|
-
# when using rabbitmq service
|
|
398
|
-
require 'pub_sub_model_sync/mock_rabbit_service'
|
|
399
|
-
config.before(:each) do
|
|
400
|
-
rabbit_mock = PubSubModelSync::MockRabbitService.new
|
|
401
|
-
allow(Bunny).to receive(:new).and_return(rabbit_mock)
|
|
402
|
-
end
|
|
403
|
-
|
|
404
|
-
# when using apache kafka service
|
|
405
|
-
require 'pub_sub_model_sync/mock_kafka_service'
|
|
406
|
-
config.before(:each) do
|
|
407
|
-
kafka_mock = PubSubModelSync::MockKafkaService.new
|
|
408
|
-
allow(Kafka).to receive(:new).and_return(kafka_mock)
|
|
409
|
-
end
|
|
410
|
-
|
|
411
|
-
# disable all models sync by default (reduces testing time)
|
|
412
396
|
config.before(:each) do
|
|
413
|
-
|
|
414
|
-
allow(PubSubModelSync::MessagePublisher).to receive(:
|
|
397
|
+
# disable delivering notifications to pubsub
|
|
398
|
+
allow(PubSubModelSync::MessagePublisher).to receive(:connector_publish)
|
|
399
|
+
# disable all models sync by default (reduces testing time by avoiding to build payload data)
|
|
400
|
+
allow(PubSubModelSync::MessagePublisher).to receive(:publish_model)
|
|
415
401
|
end
|
|
416
402
|
|
|
417
403
|
# enable all models sync only for tests that includes 'sync: true'
|
|
418
404
|
config.before(:each, sync: true) do
|
|
419
|
-
allow(PubSubModelSync::MessagePublisher).to receive(:publish_data).and_call_original
|
|
420
405
|
allow(PubSubModelSync::MessagePublisher).to receive(:publish_model).and_call_original
|
|
421
406
|
end
|
|
422
407
|
|
|
@@ -426,9 +411,9 @@ Note: To reduce Payload size, some header info are not delivered (Enable debug m
|
|
|
426
411
|
# end
|
|
427
412
|
```
|
|
428
413
|
- Examples:
|
|
429
|
-
- **Publisher**
|
|
414
|
+
- **Publisher**
|
|
415
|
+
Note: **Do not forget to include 'sync: true'** to enable publishing pubsub notifications
|
|
430
416
|
```ruby
|
|
431
|
-
# Do not forget to include 'sync: true' to enable publishing pubsub notifications
|
|
432
417
|
describe 'When publishing sync', truncate: true, sync: true do
|
|
433
418
|
it 'publishes user notification when created' do
|
|
434
419
|
expect_publish_notification(:create, klass: 'User')
|
|
@@ -7,12 +7,13 @@ module PubSubModelSync
|
|
|
7
7
|
|
|
8
8
|
# customizable callbacks
|
|
9
9
|
cattr_accessor(:debug) { false }
|
|
10
|
-
cattr_accessor
|
|
10
|
+
cattr_accessor(:logger) { Rails.logger }
|
|
11
11
|
cattr_accessor(:transactions_max_buffer) { 1 }
|
|
12
|
+
cattr_accessor(:skip_cache) { false }
|
|
12
13
|
|
|
13
14
|
cattr_accessor(:on_before_processing) { ->(_payload, _info) {} } # return :cancel to skip
|
|
14
15
|
cattr_accessor(:on_success_processing) { ->(_payload, _info) {} }
|
|
15
|
-
cattr_accessor(:on_error_processing) { ->(
|
|
16
|
+
cattr_accessor(:on_error_processing) { ->(exception, _info) { raise(exception) } }
|
|
16
17
|
cattr_accessor(:on_before_publish) { ->(_payload) {} } # return :cancel to skip
|
|
17
18
|
cattr_accessor(:on_after_publish) { ->(_payload) {} }
|
|
18
19
|
cattr_accessor(:on_error_publish) { ->(_exception, _info) {} }
|
|
@@ -49,9 +49,14 @@ module PubSubModelSync
|
|
|
49
49
|
|
|
50
50
|
# @param error (StandardError)
|
|
51
51
|
def notify_error(error)
|
|
52
|
-
|
|
52
|
+
error_msg = 'Error processing message: '
|
|
53
|
+
error_details = [payload, error.message, error.backtrace]
|
|
53
54
|
res = config.on_error_processing.call(error, { payload: payload })
|
|
54
|
-
log("
|
|
55
|
+
log("#{error_msg} #{error_details}", :error) if res != :skip_log
|
|
56
|
+
rescue => e
|
|
57
|
+
error_details = [payload, e.message, e.backtrace]
|
|
58
|
+
log("#{error_msg} #{error_details}", :error)
|
|
59
|
+
raise(e)
|
|
55
60
|
end
|
|
56
61
|
|
|
57
62
|
def lost_db_connection?(error)
|
|
@@ -98,8 +98,11 @@ module PubSubModelSync
|
|
|
98
98
|
private
|
|
99
99
|
|
|
100
100
|
def ensure_publish(payload)
|
|
101
|
-
|
|
102
|
-
|
|
101
|
+
cache_klass = PubSubModelSync::PayloadCacheOptimizer
|
|
102
|
+
cancelled = payload.cache_settings ? cache_klass.new(payload).call == :already_sent : false
|
|
103
|
+
cancelled ||= config.on_before_publish.call(payload) == :cancel
|
|
104
|
+
log_msg = "Publish cancelled by config.on_before_publish or cache checker: #{[payload]}"
|
|
105
|
+
log(log_msg) if config.debug && cancelled
|
|
103
106
|
!cancelled
|
|
104
107
|
end
|
|
105
108
|
|
|
@@ -24,6 +24,10 @@ module PubSubModelSync
|
|
|
24
24
|
def publish(*_args)
|
|
25
25
|
true
|
|
26
26
|
end
|
|
27
|
+
|
|
28
|
+
def channel
|
|
29
|
+
MockChannel.new
|
|
30
|
+
end
|
|
27
31
|
end
|
|
28
32
|
|
|
29
33
|
class MockChannel
|
|
@@ -39,6 +43,10 @@ module PubSubModelSync
|
|
|
39
43
|
def close
|
|
40
44
|
true
|
|
41
45
|
end
|
|
46
|
+
|
|
47
|
+
def ack(_delivery_tag)
|
|
48
|
+
true
|
|
49
|
+
end
|
|
42
50
|
end
|
|
43
51
|
|
|
44
52
|
def create_channel(*_args)
|
|
@@ -19,9 +19,17 @@ module PubSubModelSync
|
|
|
19
19
|
# <klass>: when class message
|
|
20
20
|
# <klass/id>: when model message
|
|
21
21
|
# topic_name (String|Array<String>): Specific topic name to be used when delivering the
|
|
22
|
-
# message (default
|
|
22
|
+
# message (default Config.topic_name)
|
|
23
23
|
# forced_ordering_key (String, optional): Will force to use this value as the ordering_key,
|
|
24
24
|
# even withing transactions. Default nil.
|
|
25
|
+
# cache (Boolean | Hash, Default false) Cache settings
|
|
26
|
+
# true: Skip publishing similar payloads
|
|
27
|
+
# Hash<required: Array<Symbol>>: Same as true and enables payload optimization to exclude
|
|
28
|
+
# unchanged non important attributes. Sample: { required: %i[id email] }
|
|
29
|
+
# --- READ ONLY ----
|
|
30
|
+
# app_key: (string) Subscriber-Key of the application who delivered the notification
|
|
31
|
+
# internal_key: (String) "<klass>/<action>"
|
|
32
|
+
# uuid: Unique notification identifier
|
|
25
33
|
def initialize(data, info, headers = {})
|
|
26
34
|
@data = data.deep_symbolize_keys
|
|
27
35
|
@info = info.deep_symbolize_keys
|
|
@@ -75,6 +83,16 @@ module PubSubModelSync
|
|
|
75
83
|
klass.publish(self)
|
|
76
84
|
end
|
|
77
85
|
|
|
86
|
+
# @param attr_keys (Array<Symbol>) List of attributes to be excluded from payload
|
|
87
|
+
def exclude_data_attrs(attr_keys)
|
|
88
|
+
@data = data.except(*attr_keys)
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# Attributes to always be delivered after cache optimization
|
|
92
|
+
def cache_settings
|
|
93
|
+
headers[:cache]
|
|
94
|
+
end
|
|
95
|
+
|
|
78
96
|
# convert payload data into Payload
|
|
79
97
|
# @param data [Hash]: payload data (:data, :info, :headers)
|
|
80
98
|
def self.from_payload_data(data)
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module PubSubModelSync
|
|
4
|
+
class PayloadCacheOptimizer < PubSubModelSync::Base
|
|
5
|
+
# Optimizes payload data to deliver only the required ones and the changed ones and thus avoid
|
|
6
|
+
# delivering unnecessary notifications.
|
|
7
|
+
# Uses Rails.cache to retrieve previous delivered data.
|
|
8
|
+
attr_reader :payload, :required_attrs, :cache_key
|
|
9
|
+
|
|
10
|
+
# @param payload (Payload)
|
|
11
|
+
def initialize(payload)
|
|
12
|
+
@payload = payload
|
|
13
|
+
@cache_key = "pubsub/#{payload.headers[:internal_key]}/#{payload.headers[:topic_name]}"
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
# @return (:already_sent|Payload)
|
|
17
|
+
def call
|
|
18
|
+
backup_data = payload.data.clone
|
|
19
|
+
return payload if cache_disabled?
|
|
20
|
+
return :already_sent if previous_payload_data == payload.data
|
|
21
|
+
|
|
22
|
+
optimize_payload if optimization_enabled?
|
|
23
|
+
Rails.cache.write(cache_key, backup_data, expires_in: 1.week)
|
|
24
|
+
payload
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
private
|
|
28
|
+
|
|
29
|
+
def optimization_enabled?
|
|
30
|
+
previous_payload_data && payload.cache_settings.is_a?(Hash)
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def cache_disabled?
|
|
34
|
+
res = config.skip_cache || Rails.cache.nil?
|
|
35
|
+
log("Skipping cache, it was disabled: #{[payload]}") if res && debug?
|
|
36
|
+
res
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def previous_payload_data
|
|
40
|
+
@previous_payload_data ||= Rails.cache.read(cache_key)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def optimize_payload # rubocop:disable Metrics/AbcSize
|
|
44
|
+
changed_keys = Hash[(payload.data.to_a - previous_payload_data.to_a)].keys.map(&:to_sym)
|
|
45
|
+
required_keys = payload.cache_settings[:required].map(&:to_sym)
|
|
46
|
+
invalid_keys = payload.data.keys - (changed_keys + required_keys)
|
|
47
|
+
log("Excluding non changed attributes: #{invalid_keys} from: #{payload.inspect}") if debug?
|
|
48
|
+
payload.exclude_data_attrs(invalid_keys)
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
|
@@ -24,7 +24,7 @@ module PubSubModelSync
|
|
|
24
24
|
# @return (String): Json Format
|
|
25
25
|
def encode_payload(payload)
|
|
26
26
|
data = payload.to_h
|
|
27
|
-
not_important_keys = %i[forced_ordering_key]
|
|
27
|
+
not_important_keys = %i[forced_ordering_key cache]
|
|
28
28
|
reduce_payload_size = !config.debug
|
|
29
29
|
data[:headers].except!(*not_important_keys) if reduce_payload_size
|
|
30
30
|
data.to_json
|
|
@@ -33,25 +33,29 @@ module PubSubModelSync
|
|
|
33
33
|
# @param (String: Payload in json format)
|
|
34
34
|
def process_message(payload_info)
|
|
35
35
|
payload = decode_payload(payload_info)
|
|
36
|
-
return
|
|
36
|
+
return unless payload
|
|
37
|
+
return if same_app_message?(payload)
|
|
37
38
|
|
|
38
|
-
|
|
39
|
-
rescue => e
|
|
40
|
-
error_payload = [payload, e.message, e.backtrace]
|
|
41
|
-
log("Error while starting to process a message: #{error_payload}", :error)
|
|
39
|
+
payload.process
|
|
42
40
|
end
|
|
43
41
|
|
|
44
|
-
# @return Payload
|
|
42
|
+
# @return [Payload,Nil]
|
|
45
43
|
def decode_payload(payload_info)
|
|
46
44
|
payload = ::PubSubModelSync::Payload.from_payload_data(JSON.parse(payload_info))
|
|
47
45
|
log("Received message: #{[payload]}") if config.debug
|
|
48
46
|
payload
|
|
47
|
+
rescue => e
|
|
48
|
+
error_payload = [payload_info, e.message, e.backtrace]
|
|
49
|
+
log("Error while parsing payload: #{error_payload}", :error)
|
|
50
|
+
nil
|
|
49
51
|
end
|
|
50
52
|
|
|
51
53
|
# @param payload (Payload)
|
|
52
54
|
def same_app_message?(payload)
|
|
53
55
|
key = payload.headers[:app_key]
|
|
54
|
-
key && key == config.subscription_key
|
|
56
|
+
res = key && key == config.subscription_key
|
|
57
|
+
log("Skipping message from same origin: #{[payload]}") if res && config.debug
|
|
58
|
+
res
|
|
55
59
|
end
|
|
56
60
|
end
|
|
57
61
|
end
|
|
@@ -81,8 +81,8 @@ module PubSubModelSync
|
|
|
81
81
|
def subscribe_to_topics
|
|
82
82
|
topics.map do |key, topic|
|
|
83
83
|
subs_name = "#{config.subscription_key}_#{key}"
|
|
84
|
-
subscription = topic.subscription(subs_name) || topic.subscribe(subs_name, SUBSCRIPTION_SETTINGS)
|
|
85
|
-
subscriber = subscription.listen(LISTEN_SETTINGS, &method(:process_message))
|
|
84
|
+
subscription = topic.subscription(subs_name) || topic.subscribe(subs_name, **SUBSCRIPTION_SETTINGS)
|
|
85
|
+
subscriber = subscription.listen(**LISTEN_SETTINGS, &method(:process_message))
|
|
86
86
|
subscriber.start
|
|
87
87
|
log("Subscribed to topic: #{topic.name} as: #{subs_name}")
|
|
88
88
|
subscriber
|
|
@@ -92,7 +92,6 @@ module PubSubModelSync
|
|
|
92
92
|
def process_message(received_message)
|
|
93
93
|
message = received_message.message
|
|
94
94
|
super(message.data) if message.attributes[SERVICE_KEY]
|
|
95
|
-
ensure
|
|
96
95
|
received_message.acknowledge!
|
|
97
96
|
end
|
|
98
97
|
end
|
|
@@ -8,7 +8,7 @@ end
|
|
|
8
8
|
module PubSubModelSync
|
|
9
9
|
class ServiceKafka < ServiceBase
|
|
10
10
|
QTY_WORKERS = 10
|
|
11
|
-
LISTEN_SETTINGS = {}.freeze
|
|
11
|
+
LISTEN_SETTINGS = { automatically_mark_as_processed: false }.freeze
|
|
12
12
|
PUBLISH_SETTINGS = {}.freeze
|
|
13
13
|
PRODUCER_SETTINGS = { delivery_threshold: 200, delivery_interval: 30 }.freeze
|
|
14
14
|
cattr_accessor :producer
|
|
@@ -73,6 +73,7 @@ module PubSubModelSync
|
|
|
73
73
|
|
|
74
74
|
def process_message(message)
|
|
75
75
|
super(message.value) if message.headers[SERVICE_KEY]
|
|
76
|
+
consumer.mark_message_as_processed(message)
|
|
76
77
|
end
|
|
77
78
|
|
|
78
79
|
# Check topic existence, create if missing topic
|
|
@@ -8,7 +8,7 @@ end
|
|
|
8
8
|
module PubSubModelSync
|
|
9
9
|
class ServiceRabbit < ServiceBase
|
|
10
10
|
QUEUE_SETTINGS = { durable: true, auto_delete: false }.freeze
|
|
11
|
-
LISTEN_SETTINGS = { manual_ack:
|
|
11
|
+
LISTEN_SETTINGS = { manual_ack: true }.freeze
|
|
12
12
|
PUBLISH_SETTINGS = {}.freeze
|
|
13
13
|
|
|
14
14
|
# @!attribute topic_names (Array): ['Topic 1', 'Topic 2']
|
|
@@ -25,7 +25,9 @@ module PubSubModelSync
|
|
|
25
25
|
|
|
26
26
|
def listen_messages
|
|
27
27
|
log('Listener starting...')
|
|
28
|
-
subscribe_to_queues
|
|
28
|
+
subscribe_to_queues do |queue|
|
|
29
|
+
queue.subscribe(LISTEN_SETTINGS) { |info, meta, payload| process_message(queue, info, meta, payload) }
|
|
30
|
+
end
|
|
29
31
|
log('Listener started')
|
|
30
32
|
loop { sleep 5 }
|
|
31
33
|
rescue PubSubModelSync::Runner::ShutDown
|
|
@@ -62,8 +64,9 @@ module PubSubModelSync
|
|
|
62
64
|
}.merge(PUBLISH_SETTINGS)
|
|
63
65
|
end
|
|
64
66
|
|
|
65
|
-
def process_message(
|
|
67
|
+
def process_message(queue, delivery_info, meta_info, payload)
|
|
66
68
|
super(payload) if meta_info[:type] == SERVICE_KEY
|
|
69
|
+
queue.channel.ack(delivery_info.delivery_tag)
|
|
67
70
|
end
|
|
68
71
|
|
|
69
72
|
def subscribe_to_queues(&block)
|
data/lib/pub_sub_model_sync.rb
CHANGED
|
@@ -16,6 +16,7 @@ require 'pub_sub_model_sync/message_processor'
|
|
|
16
16
|
require 'pub_sub_model_sync/run_subscriber'
|
|
17
17
|
|
|
18
18
|
require 'pub_sub_model_sync/payload_builder'
|
|
19
|
+
require 'pub_sub_model_sync/payload_cache_optimizer'
|
|
19
20
|
require 'pub_sub_model_sync/subscriber'
|
|
20
21
|
|
|
21
22
|
require 'pub_sub_model_sync/service_base'
|
data/pub_sub_model_sync.gemspec
CHANGED
|
@@ -27,7 +27,7 @@ Gem::Specification.new do |spec|
|
|
|
27
27
|
# into git.
|
|
28
28
|
spec.files = Dir.chdir(File.expand_path(__dir__)) do
|
|
29
29
|
`git ls-files -z`.split("\x0")
|
|
30
|
-
.reject { |f| f.match(%r{^(test|spec|features)/}) }
|
|
30
|
+
.reject { |f| f.match(%r{^(test|spec|features|samples)/}) }
|
|
31
31
|
end
|
|
32
32
|
spec.bindir = 'exe'
|
|
33
33
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|