pub_sub_model_sync 1.3.0 → 1.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ruby.yml +0 -2
- data/.gitignore +2 -1
- data/CHANGELOG.md +2 -0
- data/Gemfile.lock +20 -16
- data/README.md +18 -33
- data/lib/pub_sub_model_sync/config.rb +3 -2
- data/lib/pub_sub_model_sync/message_processor.rb +7 -2
- data/lib/pub_sub_model_sync/message_publisher.rb +5 -2
- data/lib/pub_sub_model_sync/mock_rabbit_service.rb +8 -0
- data/lib/pub_sub_model_sync/payload.rb +19 -1
- data/lib/pub_sub_model_sync/payload_cache_optimizer.rb +51 -0
- data/lib/pub_sub_model_sync/service_base.rb +12 -8
- data/lib/pub_sub_model_sync/service_google.rb +2 -3
- data/lib/pub_sub_model_sync/service_kafka.rb +2 -1
- data/lib/pub_sub_model_sync/service_rabbit.rb +6 -3
- data/lib/pub_sub_model_sync/version.rb +1 -1
- data/lib/pub_sub_model_sync.rb +1 -0
- data/pub_sub_model_sync.gemspec +1 -1
- metadata +3 -88
- data/samples/README.md +0 -50
- data/samples/app1/Dockerfile +0 -13
- data/samples/app1/Gemfile +0 -37
- data/samples/app1/Gemfile.lock +0 -171
- data/samples/app1/README.md +0 -24
- data/samples/app1/Rakefile +0 -6
- data/samples/app1/app/models/application_record.rb +0 -3
- data/samples/app1/app/models/concerns/.keep +0 -0
- data/samples/app1/app/models/post.rb +0 -19
- data/samples/app1/app/models/user.rb +0 -29
- data/samples/app1/bin/bundle +0 -114
- data/samples/app1/bin/rails +0 -5
- data/samples/app1/bin/rake +0 -5
- data/samples/app1/bin/setup +0 -33
- data/samples/app1/bin/spring +0 -14
- data/samples/app1/config/application.rb +0 -40
- data/samples/app1/config/boot.rb +0 -4
- data/samples/app1/config/credentials.yml.enc +0 -1
- data/samples/app1/config/database.yml +0 -25
- data/samples/app1/config/environment.rb +0 -5
- data/samples/app1/config/environments/development.rb +0 -63
- data/samples/app1/config/environments/production.rb +0 -105
- data/samples/app1/config/environments/test.rb +0 -57
- data/samples/app1/config/initializers/application_controller_renderer.rb +0 -8
- data/samples/app1/config/initializers/backtrace_silencers.rb +0 -8
- data/samples/app1/config/initializers/cors.rb +0 -16
- data/samples/app1/config/initializers/filter_parameter_logging.rb +0 -6
- data/samples/app1/config/initializers/inflections.rb +0 -16
- data/samples/app1/config/initializers/mime_types.rb +0 -4
- data/samples/app1/config/initializers/pubsub.rb +0 -4
- data/samples/app1/config/initializers/wrap_parameters.rb +0 -14
- data/samples/app1/config/locales/en.yml +0 -33
- data/samples/app1/config/master.key +0 -1
- data/samples/app1/config/puma.rb +0 -43
- data/samples/app1/config/routes.rb +0 -3
- data/samples/app1/config/spring.rb +0 -6
- data/samples/app1/config.ru +0 -6
- data/samples/app1/db/migrate/20210513080700_create_users.rb +0 -12
- data/samples/app1/db/migrate/20210513134332_create_posts.rb +0 -11
- data/samples/app1/db/schema.rb +0 -34
- data/samples/app1/db/seeds.rb +0 -7
- data/samples/app1/docker-compose.yml +0 -32
- data/samples/app1/log/.keep +0 -0
- data/samples/app2/Dockerfile +0 -13
- data/samples/app2/Gemfile +0 -37
- data/samples/app2/Gemfile.lock +0 -171
- data/samples/app2/README.md +0 -24
- data/samples/app2/Rakefile +0 -6
- data/samples/app2/app/models/application_record.rb +0 -9
- data/samples/app2/app/models/concerns/.keep +0 -0
- data/samples/app2/app/models/customer.rb +0 -28
- data/samples/app2/app/models/post.rb +0 -10
- data/samples/app2/bin/bundle +0 -114
- data/samples/app2/bin/rails +0 -5
- data/samples/app2/bin/rake +0 -5
- data/samples/app2/bin/setup +0 -33
- data/samples/app2/bin/spring +0 -14
- data/samples/app2/config/application.rb +0 -40
- data/samples/app2/config/boot.rb +0 -4
- data/samples/app2/config/credentials.yml.enc +0 -1
- data/samples/app2/config/database.yml +0 -25
- data/samples/app2/config/environment.rb +0 -5
- data/samples/app2/config/environments/development.rb +0 -63
- data/samples/app2/config/environments/production.rb +0 -105
- data/samples/app2/config/environments/test.rb +0 -57
- data/samples/app2/config/initializers/application_controller_renderer.rb +0 -8
- data/samples/app2/config/initializers/backtrace_silencers.rb +0 -8
- data/samples/app2/config/initializers/cors.rb +0 -16
- data/samples/app2/config/initializers/filter_parameter_logging.rb +0 -6
- data/samples/app2/config/initializers/inflections.rb +0 -16
- data/samples/app2/config/initializers/mime_types.rb +0 -4
- data/samples/app2/config/initializers/pubsub.rb +0 -4
- data/samples/app2/config/initializers/wrap_parameters.rb +0 -14
- data/samples/app2/config/locales/en.yml +0 -33
- data/samples/app2/config/master.key +0 -1
- data/samples/app2/config/puma.rb +0 -43
- data/samples/app2/config/routes.rb +0 -3
- data/samples/app2/config/spring.rb +0 -6
- data/samples/app2/config.ru +0 -6
- data/samples/app2/db/development.sqlite3 +0 -0
- data/samples/app2/db/migrate/20210513080956_create_customers.rb +0 -10
- data/samples/app2/db/migrate/20210513135203_create_posts.rb +0 -10
- data/samples/app2/db/schema.rb +0 -31
- data/samples/app2/db/seeds.rb +0 -7
- data/samples/app2/docker-compose.yml +0 -20
- data/samples/app2/log/.keep +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: '0839d268607d201c9f77bbbca404f32aa5af02621b7582256eb384b42b9d3985'
|
4
|
+
data.tar.gz: 0ad3d2cfdce3ce99eb5d0b95a4292496927befaca2a8e2d83b865af411be2988
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c86be499e21572b796060fc07f5d95191971b87366bbebe3b22737d5eb906f25a6e850f4b4b6f2c08f2790541169f36401fb67b95df0c5da365fda1f76e8751f
|
7
|
+
data.tar.gz: 3e91e306f83eb4ba8dd5c494aadc90b36a702935d7fb716424af80efee6fe6bce9ff895ae6231aa1c32327d7f468ed2148aa69a5f37842c10e0f50e206dad82a
|
data/.github/workflows/ruby.yml
CHANGED
data/.gitignore
CHANGED
data/CHANGELOG.md
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
pub_sub_model_sync (1.
|
4
|
+
pub_sub_model_sync (1.5.0)
|
5
5
|
rails
|
6
6
|
|
7
7
|
GEM
|
@@ -99,8 +99,8 @@ GEM
|
|
99
99
|
googleapis-common-protos-types (>= 1.0.6, < 2.0)
|
100
100
|
googleauth (~> 0.15, >= 0.15.1)
|
101
101
|
grpc (~> 1.36)
|
102
|
-
globalid (0.
|
103
|
-
activesupport (>=
|
102
|
+
globalid (1.0.0)
|
103
|
+
activesupport (>= 5.0)
|
104
104
|
google-cloud-core (1.6.0)
|
105
105
|
google-cloud-env (~> 1.0)
|
106
106
|
google-cloud-errors (~> 1.0)
|
@@ -116,6 +116,7 @@ GEM
|
|
116
116
|
google-cloud-errors (~> 1.0)
|
117
117
|
grpc-google-iam-v1 (>= 0.6.10, < 2.0)
|
118
118
|
google-protobuf (3.17.0)
|
119
|
+
google-protobuf (3.17.0-x86_64-linux)
|
119
120
|
googleapis-common-protos (1.3.11)
|
120
121
|
google-protobuf (~> 3.14)
|
121
122
|
googleapis-common-protos-types (>= 1.0.6, < 2.0)
|
@@ -132,6 +133,9 @@ GEM
|
|
132
133
|
grpc (1.37.1)
|
133
134
|
google-protobuf (~> 3.15)
|
134
135
|
googleapis-common-protos-types (~> 1.0)
|
136
|
+
grpc (1.37.1-x86_64-linux)
|
137
|
+
google-protobuf (~> 3.15)
|
138
|
+
googleapis-common-protos-types (~> 1.0)
|
135
139
|
grpc-google-iam-v1 (0.6.11)
|
136
140
|
google-protobuf (~> 3.14)
|
137
141
|
googleapis-common-protos (>= 1.3.11, < 2.0)
|
@@ -139,27 +143,27 @@ GEM
|
|
139
143
|
i18n (1.8.10)
|
140
144
|
concurrent-ruby (~> 1.0)
|
141
145
|
jwt (2.2.3)
|
142
|
-
loofah (2.
|
146
|
+
loofah (2.15.0)
|
143
147
|
crass (~> 1.0.2)
|
144
148
|
nokogiri (>= 1.5.9)
|
145
149
|
mail (2.7.1)
|
146
150
|
mini_mime (>= 0.1.1)
|
147
|
-
marcel (1.0.
|
151
|
+
marcel (1.0.2)
|
148
152
|
memoist (0.16.2)
|
149
153
|
method_source (1.0.0)
|
150
154
|
mini_mime (1.0.3)
|
151
155
|
minitest (5.14.4)
|
152
156
|
multi_json (1.15.0)
|
153
157
|
multipart-post (2.1.1)
|
154
|
-
nio4r (2.5.
|
155
|
-
nokogiri (1.
|
158
|
+
nio4r (2.5.8)
|
159
|
+
nokogiri (1.12.5-x86_64-linux)
|
156
160
|
racc (~> 1.4)
|
157
161
|
os (1.1.1)
|
158
162
|
parallel (1.20.1)
|
159
163
|
parser (3.0.1.1)
|
160
164
|
ast (~> 2.4.1)
|
161
165
|
public_suffix (4.0.6)
|
162
|
-
racc (1.
|
166
|
+
racc (1.6.0)
|
163
167
|
rack (2.2.3)
|
164
168
|
rack-test (1.1.0)
|
165
169
|
rack (>= 1.0, < 3)
|
@@ -181,7 +185,7 @@ GEM
|
|
181
185
|
rails-dom-testing (2.0.3)
|
182
186
|
activesupport (>= 4.2.0)
|
183
187
|
nokogiri (>= 1.6)
|
184
|
-
rails-html-sanitizer (1.
|
188
|
+
rails-html-sanitizer (1.4.2)
|
185
189
|
loofah (~> 2.3)
|
186
190
|
railties (6.1.3.2)
|
187
191
|
actionpack (= 6.1.3.2)
|
@@ -226,19 +230,19 @@ GEM
|
|
226
230
|
faraday (>= 0.17.3, < 2.0)
|
227
231
|
jwt (>= 1.5, < 3.0)
|
228
232
|
multi_json (~> 1.10)
|
229
|
-
sprockets (4.0.
|
233
|
+
sprockets (4.0.3)
|
230
234
|
concurrent-ruby (~> 1.0)
|
231
235
|
rack (> 1, < 3)
|
232
|
-
sprockets-rails (3.
|
233
|
-
actionpack (>=
|
234
|
-
activesupport (>=
|
236
|
+
sprockets-rails (3.4.2)
|
237
|
+
actionpack (>= 5.2)
|
238
|
+
activesupport (>= 5.2)
|
235
239
|
sprockets (>= 3.0.0)
|
236
240
|
sqlite3 (1.4.2)
|
237
|
-
thor (1.1
|
241
|
+
thor (1.2.1)
|
238
242
|
tzinfo (2.0.4)
|
239
243
|
concurrent-ruby (~> 1.0)
|
240
244
|
unicode-display_width (1.7.0)
|
241
|
-
websocket-driver (0.7.
|
245
|
+
websocket-driver (0.7.5)
|
242
246
|
websocket-extensions (>= 0.1.0)
|
243
247
|
websocket-extensions (0.1.5)
|
244
248
|
zeitwerk (2.4.2)
|
@@ -260,4 +264,4 @@ DEPENDENCIES
|
|
260
264
|
sqlite3
|
261
265
|
|
262
266
|
BUNDLED WITH
|
263
|
-
2.
|
267
|
+
2.3.9
|
data/README.md
CHANGED
@@ -167,7 +167,7 @@ PubSubModelSync::Payload.new({ ids: [my_user.id] }, { klass: 'User', action: :ba
|
|
167
167
|
ps_class_subscribe(action, settings)
|
168
168
|
end
|
169
169
|
```
|
170
|
-
- Instance subscriptions: `ps_subscribe(action, mapping, settings)`
|
170
|
+
- Instance subscriptions: `ps_subscribe(action, mapping, settings, &block)`
|
171
171
|
When model receives the corresponding notification, `action` or `to_action` method will be called on the model. Like: `model.destroy`
|
172
172
|
- `action` (Symbol|Array<Symbol>) Only notifications with this action name will be processed by this subscription. Sample: save|create|update|destroy|<any_other_action>
|
173
173
|
- `mapping` (Array<String>) Data mapping from payload data into model attributes, sample: ["email", "full_name:name"] (Note: Only these attributes will be assigned/synced to the current model)
|
@@ -182,12 +182,14 @@ PubSubModelSync::Payload.new({ ids: [my_user.id] }, { klass: 'User', action: :ba
|
|
182
182
|
Sample: `id: :id` will search for a model like: `model_class.where(id: payload.data[:id])`
|
183
183
|
Sample: `id: [:id, :email:user_email]` will search for a model like: `model_class.where(id: payload.data[:id], user_email: payload.data[:email])`
|
184
184
|
- `if:` (Symbol|Proc|Array<Symbol>) Method(s) or block called for the confirmation before calling the callback
|
185
|
-
- `unless:` (Symbol|Proc|Array<Symbol>) Method or block called for the negation before calling the callback
|
185
|
+
- `unless:` (Symbol|Proc|Array<Symbol>) Method or block called for the negation before calling the callback
|
186
|
+
- `&block` Block to be used as the callback method (ignored if `:to_action` is present). Sample: `ps_subscribe(:send_welcome, %i[email]) { |_data| puts model.email }`
|
186
187
|
|
187
|
-
- Class subscriptions: `ps_class_subscribe(action, settings)`
|
188
|
+
- Class subscriptions: `ps_class_subscribe(action, settings, &block)`
|
188
189
|
When current class receives the corresponding notification, `action` or `to_action` method will be called on the Class. Like: `User.hello(data)`
|
189
190
|
* `action` (Symbol) Notification.action name
|
190
191
|
* `settings` (Hash) refer ps_subscribe.settings except(:id)
|
192
|
+
* `&block` Block to be used as the callback method (ignored if `:to_action` is present). Sample: `ps_class_subscribe(:send_welcome) { |data| puts data }`
|
191
193
|
|
192
194
|
- `ps_processing_payload` a class and instance variable that saves the current payload being processed
|
193
195
|
|
@@ -313,10 +315,15 @@ Any notification before delivering is transformed as a Payload for a better port
|
|
313
315
|
* `headers`: (Hash) Notification settings that defines how the notification will be processed or delivered.
|
314
316
|
- `ordering_key`: (String, optional): notifications with the same `ordering_key` are processed in the same order they were delivered, default: `<model.class.name>/<model.id>` when instance notification and `klass_name` when class notification.
|
315
317
|
Note: Final `ordering_key` is calculated as: `payload.headers[:forced_ordering_key] || current_transaction&.key || payload.headers[:ordering_key]`
|
316
|
-
- `internal_key`: (String, optional) Internal identifier of the payload, default: `<model.class.name>/<action>/<model.id>` when model notification and `<klass_name>/<action>` when class notification (Useful for caching techniques).
|
317
318
|
- `topic_name`: (String|Array<String>, optional): Specific topic name where to deliver the notification (default `PubSubModelSync::Config.topic_name`).
|
318
319
|
- `forced_ordering_key`: (String, optional): Overrides `ordering_key` with the provided value even withing transactions. Default `nil`.
|
319
|
-
- `
|
320
|
+
- `cache` (Boolean | Hash, Default false) Cache settings
|
321
|
+
- `true`: Skip publishing similar payloads
|
322
|
+
- `Hash<required: Array<Symbol>>`: Same as `true` and enables payload optimization to exclude unchanged non important attributes. Sample: `{ required: %i[id email] }`
|
323
|
+
|
324
|
+
** Read ONLY **
|
325
|
+
- `internal_key`: Internal identifier of the payload, default: `<model.class.name>/<action>/<model.id>` when model notification and `<klass_name>/<action>` when class notification (Useful for caching techniques).
|
326
|
+
- `app_key`: (Auto calculated): Name of the application who delivered the notification.
|
320
327
|
- `uuid`: (Auto calculated): Unique notification identifier (Very useful when debugging).
|
321
328
|
Note: To reduce Payload size, some header info are not delivered (Enable debug mode to deliver all payload info).
|
322
329
|
|
@@ -386,37 +393,15 @@ Note: To reduce Payload size, some header info are not delivered (Enable debug m
|
|
386
393
|
## **Testing with RSpec**
|
387
394
|
- Config: (spec/rails_helper.rb)
|
388
395
|
```ruby
|
389
|
-
|
390
|
-
# when using google service
|
391
|
-
require 'pub_sub_model_sync/mock_google_service'
|
392
|
-
config.before(:each) do
|
393
|
-
google_mock = PubSubModelSync::MockGoogleService.new
|
394
|
-
allow(Google::Cloud::Pubsub).to receive(:new).and_return(google_mock)
|
395
|
-
end
|
396
|
-
|
397
|
-
# when using rabbitmq service
|
398
|
-
require 'pub_sub_model_sync/mock_rabbit_service'
|
399
|
-
config.before(:each) do
|
400
|
-
rabbit_mock = PubSubModelSync::MockRabbitService.new
|
401
|
-
allow(Bunny).to receive(:new).and_return(rabbit_mock)
|
402
|
-
end
|
403
|
-
|
404
|
-
# when using apache kafka service
|
405
|
-
require 'pub_sub_model_sync/mock_kafka_service'
|
406
|
-
config.before(:each) do
|
407
|
-
kafka_mock = PubSubModelSync::MockKafkaService.new
|
408
|
-
allow(Kafka).to receive(:new).and_return(kafka_mock)
|
409
|
-
end
|
410
|
-
|
411
|
-
# disable all models sync by default (reduces testing time)
|
412
396
|
config.before(:each) do
|
413
|
-
|
414
|
-
allow(PubSubModelSync::MessagePublisher).to receive(:
|
397
|
+
# disable delivering notifications to pubsub
|
398
|
+
allow(PubSubModelSync::MessagePublisher).to receive(:connector_publish)
|
399
|
+
# disable all models sync by default (reduces testing time by avoiding to build payload data)
|
400
|
+
allow(PubSubModelSync::MessagePublisher).to receive(:publish_model)
|
415
401
|
end
|
416
402
|
|
417
403
|
# enable all models sync only for tests that includes 'sync: true'
|
418
404
|
config.before(:each, sync: true) do
|
419
|
-
allow(PubSubModelSync::MessagePublisher).to receive(:publish_data).and_call_original
|
420
405
|
allow(PubSubModelSync::MessagePublisher).to receive(:publish_model).and_call_original
|
421
406
|
end
|
422
407
|
|
@@ -426,9 +411,9 @@ Note: To reduce Payload size, some header info are not delivered (Enable debug m
|
|
426
411
|
# end
|
427
412
|
```
|
428
413
|
- Examples:
|
429
|
-
- **Publisher**
|
414
|
+
- **Publisher**
|
415
|
+
Note: **Do not forget to include 'sync: true'** to enable publishing pubsub notifications
|
430
416
|
```ruby
|
431
|
-
# Do not forget to include 'sync: true' to enable publishing pubsub notifications
|
432
417
|
describe 'When publishing sync', truncate: true, sync: true do
|
433
418
|
it 'publishes user notification when created' do
|
434
419
|
expect_publish_notification(:create, klass: 'User')
|
@@ -7,12 +7,13 @@ module PubSubModelSync
|
|
7
7
|
|
8
8
|
# customizable callbacks
|
9
9
|
cattr_accessor(:debug) { false }
|
10
|
-
cattr_accessor
|
10
|
+
cattr_accessor(:logger) { Rails.logger }
|
11
11
|
cattr_accessor(:transactions_max_buffer) { 1 }
|
12
|
+
cattr_accessor(:skip_cache) { false }
|
12
13
|
|
13
14
|
cattr_accessor(:on_before_processing) { ->(_payload, _info) {} } # return :cancel to skip
|
14
15
|
cattr_accessor(:on_success_processing) { ->(_payload, _info) {} }
|
15
|
-
cattr_accessor(:on_error_processing) { ->(
|
16
|
+
cattr_accessor(:on_error_processing) { ->(exception, _info) { raise(exception) } }
|
16
17
|
cattr_accessor(:on_before_publish) { ->(_payload) {} } # return :cancel to skip
|
17
18
|
cattr_accessor(:on_after_publish) { ->(_payload) {} }
|
18
19
|
cattr_accessor(:on_error_publish) { ->(_exception, _info) {} }
|
@@ -49,9 +49,14 @@ module PubSubModelSync
|
|
49
49
|
|
50
50
|
# @param error (StandardError)
|
51
51
|
def notify_error(error)
|
52
|
-
|
52
|
+
error_msg = 'Error processing message: '
|
53
|
+
error_details = [payload, error.message, error.backtrace]
|
53
54
|
res = config.on_error_processing.call(error, { payload: payload })
|
54
|
-
log("
|
55
|
+
log("#{error_msg} #{error_details}", :error) if res != :skip_log
|
56
|
+
rescue => e
|
57
|
+
error_details = [payload, e.message, e.backtrace]
|
58
|
+
log("#{error_msg} #{error_details}", :error)
|
59
|
+
raise(e)
|
55
60
|
end
|
56
61
|
|
57
62
|
def lost_db_connection?(error)
|
@@ -98,8 +98,11 @@ module PubSubModelSync
|
|
98
98
|
private
|
99
99
|
|
100
100
|
def ensure_publish(payload)
|
101
|
-
|
102
|
-
|
101
|
+
cache_klass = PubSubModelSync::PayloadCacheOptimizer
|
102
|
+
cancelled = payload.cache_settings ? cache_klass.new(payload).call == :already_sent : false
|
103
|
+
cancelled ||= config.on_before_publish.call(payload) == :cancel
|
104
|
+
log_msg = "Publish cancelled by config.on_before_publish or cache checker: #{[payload]}"
|
105
|
+
log(log_msg) if config.debug && cancelled
|
103
106
|
!cancelled
|
104
107
|
end
|
105
108
|
|
@@ -24,6 +24,10 @@ module PubSubModelSync
|
|
24
24
|
def publish(*_args)
|
25
25
|
true
|
26
26
|
end
|
27
|
+
|
28
|
+
def channel
|
29
|
+
MockChannel.new
|
30
|
+
end
|
27
31
|
end
|
28
32
|
|
29
33
|
class MockChannel
|
@@ -39,6 +43,10 @@ module PubSubModelSync
|
|
39
43
|
def close
|
40
44
|
true
|
41
45
|
end
|
46
|
+
|
47
|
+
def ack(_delivery_tag)
|
48
|
+
true
|
49
|
+
end
|
42
50
|
end
|
43
51
|
|
44
52
|
def create_channel(*_args)
|
@@ -19,9 +19,17 @@ module PubSubModelSync
|
|
19
19
|
# <klass>: when class message
|
20
20
|
# <klass/id>: when model message
|
21
21
|
# topic_name (String|Array<String>): Specific topic name to be used when delivering the
|
22
|
-
# message (default
|
22
|
+
# message (default Config.topic_name)
|
23
23
|
# forced_ordering_key (String, optional): Will force to use this value as the ordering_key,
|
24
24
|
# even withing transactions. Default nil.
|
25
|
+
# cache (Boolean | Hash, Default false) Cache settings
|
26
|
+
# true: Skip publishing similar payloads
|
27
|
+
# Hash<required: Array<Symbol>>: Same as true and enables payload optimization to exclude
|
28
|
+
# unchanged non important attributes. Sample: { required: %i[id email] }
|
29
|
+
# --- READ ONLY ----
|
30
|
+
# app_key: (string) Subscriber-Key of the application who delivered the notification
|
31
|
+
# internal_key: (String) "<klass>/<action>"
|
32
|
+
# uuid: Unique notification identifier
|
25
33
|
def initialize(data, info, headers = {})
|
26
34
|
@data = data.deep_symbolize_keys
|
27
35
|
@info = info.deep_symbolize_keys
|
@@ -75,6 +83,16 @@ module PubSubModelSync
|
|
75
83
|
klass.publish(self)
|
76
84
|
end
|
77
85
|
|
86
|
+
# @param attr_keys (Array<Symbol>) List of attributes to be excluded from payload
|
87
|
+
def exclude_data_attrs(attr_keys)
|
88
|
+
@data = data.except(*attr_keys)
|
89
|
+
end
|
90
|
+
|
91
|
+
# Attributes to always be delivered after cache optimization
|
92
|
+
def cache_settings
|
93
|
+
headers[:cache]
|
94
|
+
end
|
95
|
+
|
78
96
|
# convert payload data into Payload
|
79
97
|
# @param data [Hash]: payload data (:data, :info, :headers)
|
80
98
|
def self.from_payload_data(data)
|
@@ -0,0 +1,51 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module PubSubModelSync
|
4
|
+
class PayloadCacheOptimizer < PubSubModelSync::Base
|
5
|
+
# Optimizes payload data to deliver only the required ones and the changed ones and thus avoid
|
6
|
+
# delivering unnecessary notifications.
|
7
|
+
# Uses Rails.cache to retrieve previous delivered data.
|
8
|
+
attr_reader :payload, :required_attrs, :cache_key
|
9
|
+
|
10
|
+
# @param payload (Payload)
|
11
|
+
def initialize(payload)
|
12
|
+
@payload = payload
|
13
|
+
@cache_key = "pubsub/#{payload.headers[:internal_key]}/#{payload.headers[:topic_name]}"
|
14
|
+
end
|
15
|
+
|
16
|
+
# @return (:already_sent|Payload)
|
17
|
+
def call
|
18
|
+
backup_data = payload.data.clone
|
19
|
+
return payload if cache_disabled?
|
20
|
+
return :already_sent if previous_payload_data == payload.data
|
21
|
+
|
22
|
+
optimize_payload if optimization_enabled?
|
23
|
+
Rails.cache.write(cache_key, backup_data, expires_in: 1.week)
|
24
|
+
payload
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
|
29
|
+
def optimization_enabled?
|
30
|
+
previous_payload_data && payload.cache_settings.is_a?(Hash)
|
31
|
+
end
|
32
|
+
|
33
|
+
def cache_disabled?
|
34
|
+
res = config.skip_cache || Rails.cache.nil?
|
35
|
+
log("Skipping cache, it was disabled: #{[payload]}") if res && debug?
|
36
|
+
res
|
37
|
+
end
|
38
|
+
|
39
|
+
def previous_payload_data
|
40
|
+
@previous_payload_data ||= Rails.cache.read(cache_key)
|
41
|
+
end
|
42
|
+
|
43
|
+
def optimize_payload # rubocop:disable Metrics/AbcSize
|
44
|
+
changed_keys = Hash[(payload.data.to_a - previous_payload_data.to_a)].keys.map(&:to_sym)
|
45
|
+
required_keys = payload.cache_settings[:required].map(&:to_sym)
|
46
|
+
invalid_keys = payload.data.keys - (changed_keys + required_keys)
|
47
|
+
log("Excluding non changed attributes: #{invalid_keys} from: #{payload.inspect}") if debug?
|
48
|
+
payload.exclude_data_attrs(invalid_keys)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -24,7 +24,7 @@ module PubSubModelSync
|
|
24
24
|
# @return (String): Json Format
|
25
25
|
def encode_payload(payload)
|
26
26
|
data = payload.to_h
|
27
|
-
not_important_keys = %i[forced_ordering_key]
|
27
|
+
not_important_keys = %i[forced_ordering_key cache]
|
28
28
|
reduce_payload_size = !config.debug
|
29
29
|
data[:headers].except!(*not_important_keys) if reduce_payload_size
|
30
30
|
data.to_json
|
@@ -33,25 +33,29 @@ module PubSubModelSync
|
|
33
33
|
# @param (String: Payload in json format)
|
34
34
|
def process_message(payload_info)
|
35
35
|
payload = decode_payload(payload_info)
|
36
|
-
return
|
36
|
+
return unless payload
|
37
|
+
return if same_app_message?(payload)
|
37
38
|
|
38
|
-
|
39
|
-
rescue => e
|
40
|
-
error_payload = [payload, e.message, e.backtrace]
|
41
|
-
log("Error while starting to process a message: #{error_payload}", :error)
|
39
|
+
payload.process
|
42
40
|
end
|
43
41
|
|
44
|
-
# @return Payload
|
42
|
+
# @return [Payload,Nil]
|
45
43
|
def decode_payload(payload_info)
|
46
44
|
payload = ::PubSubModelSync::Payload.from_payload_data(JSON.parse(payload_info))
|
47
45
|
log("Received message: #{[payload]}") if config.debug
|
48
46
|
payload
|
47
|
+
rescue => e
|
48
|
+
error_payload = [payload_info, e.message, e.backtrace]
|
49
|
+
log("Error while parsing payload: #{error_payload}", :error)
|
50
|
+
nil
|
49
51
|
end
|
50
52
|
|
51
53
|
# @param payload (Payload)
|
52
54
|
def same_app_message?(payload)
|
53
55
|
key = payload.headers[:app_key]
|
54
|
-
key && key == config.subscription_key
|
56
|
+
res = key && key == config.subscription_key
|
57
|
+
log("Skipping message from same origin: #{[payload]}") if res && config.debug
|
58
|
+
res
|
55
59
|
end
|
56
60
|
end
|
57
61
|
end
|
@@ -81,8 +81,8 @@ module PubSubModelSync
|
|
81
81
|
def subscribe_to_topics
|
82
82
|
topics.map do |key, topic|
|
83
83
|
subs_name = "#{config.subscription_key}_#{key}"
|
84
|
-
subscription = topic.subscription(subs_name) || topic.subscribe(subs_name, SUBSCRIPTION_SETTINGS)
|
85
|
-
subscriber = subscription.listen(LISTEN_SETTINGS, &method(:process_message))
|
84
|
+
subscription = topic.subscription(subs_name) || topic.subscribe(subs_name, **SUBSCRIPTION_SETTINGS)
|
85
|
+
subscriber = subscription.listen(**LISTEN_SETTINGS, &method(:process_message))
|
86
86
|
subscriber.start
|
87
87
|
log("Subscribed to topic: #{topic.name} as: #{subs_name}")
|
88
88
|
subscriber
|
@@ -92,7 +92,6 @@ module PubSubModelSync
|
|
92
92
|
def process_message(received_message)
|
93
93
|
message = received_message.message
|
94
94
|
super(message.data) if message.attributes[SERVICE_KEY]
|
95
|
-
ensure
|
96
95
|
received_message.acknowledge!
|
97
96
|
end
|
98
97
|
end
|
@@ -8,7 +8,7 @@ end
|
|
8
8
|
module PubSubModelSync
|
9
9
|
class ServiceKafka < ServiceBase
|
10
10
|
QTY_WORKERS = 10
|
11
|
-
LISTEN_SETTINGS = {}.freeze
|
11
|
+
LISTEN_SETTINGS = { automatically_mark_as_processed: false }.freeze
|
12
12
|
PUBLISH_SETTINGS = {}.freeze
|
13
13
|
PRODUCER_SETTINGS = { delivery_threshold: 200, delivery_interval: 30 }.freeze
|
14
14
|
cattr_accessor :producer
|
@@ -73,6 +73,7 @@ module PubSubModelSync
|
|
73
73
|
|
74
74
|
def process_message(message)
|
75
75
|
super(message.value) if message.headers[SERVICE_KEY]
|
76
|
+
consumer.mark_message_as_processed(message)
|
76
77
|
end
|
77
78
|
|
78
79
|
# Check topic existence, create if missing topic
|
@@ -8,7 +8,7 @@ end
|
|
8
8
|
module PubSubModelSync
|
9
9
|
class ServiceRabbit < ServiceBase
|
10
10
|
QUEUE_SETTINGS = { durable: true, auto_delete: false }.freeze
|
11
|
-
LISTEN_SETTINGS = { manual_ack:
|
11
|
+
LISTEN_SETTINGS = { manual_ack: true }.freeze
|
12
12
|
PUBLISH_SETTINGS = {}.freeze
|
13
13
|
|
14
14
|
# @!attribute topic_names (Array): ['Topic 1', 'Topic 2']
|
@@ -25,7 +25,9 @@ module PubSubModelSync
|
|
25
25
|
|
26
26
|
def listen_messages
|
27
27
|
log('Listener starting...')
|
28
|
-
subscribe_to_queues
|
28
|
+
subscribe_to_queues do |queue|
|
29
|
+
queue.subscribe(LISTEN_SETTINGS) { |info, meta, payload| process_message(queue, info, meta, payload) }
|
30
|
+
end
|
29
31
|
log('Listener started')
|
30
32
|
loop { sleep 5 }
|
31
33
|
rescue PubSubModelSync::Runner::ShutDown
|
@@ -62,8 +64,9 @@ module PubSubModelSync
|
|
62
64
|
}.merge(PUBLISH_SETTINGS)
|
63
65
|
end
|
64
66
|
|
65
|
-
def process_message(
|
67
|
+
def process_message(queue, delivery_info, meta_info, payload)
|
66
68
|
super(payload) if meta_info[:type] == SERVICE_KEY
|
69
|
+
queue.channel.ack(delivery_info.delivery_tag)
|
67
70
|
end
|
68
71
|
|
69
72
|
def subscribe_to_queues(&block)
|
data/lib/pub_sub_model_sync.rb
CHANGED
@@ -16,6 +16,7 @@ require 'pub_sub_model_sync/message_processor'
|
|
16
16
|
require 'pub_sub_model_sync/run_subscriber'
|
17
17
|
|
18
18
|
require 'pub_sub_model_sync/payload_builder'
|
19
|
+
require 'pub_sub_model_sync/payload_cache_optimizer'
|
19
20
|
require 'pub_sub_model_sync/subscriber'
|
20
21
|
|
21
22
|
require 'pub_sub_model_sync/service_base'
|
data/pub_sub_model_sync.gemspec
CHANGED
@@ -27,7 +27,7 @@ Gem::Specification.new do |spec|
|
|
27
27
|
# into git.
|
28
28
|
spec.files = Dir.chdir(File.expand_path(__dir__)) do
|
29
29
|
`git ls-files -z`.split("\x0")
|
30
|
-
.reject { |f| f.match(%r{^(test|spec|features)/}) }
|
30
|
+
.reject { |f| f.match(%r{^(test|spec|features|samples)/}) }
|
31
31
|
end
|
32
32
|
spec.bindir = 'exe'
|
33
33
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|