pub_sub_model_sync 1.3.0 → 1.5.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (106) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/ruby.yml +0 -2
  3. data/.gitignore +2 -1
  4. data/CHANGELOG.md +2 -0
  5. data/Gemfile.lock +20 -16
  6. data/README.md +18 -33
  7. data/lib/pub_sub_model_sync/config.rb +3 -2
  8. data/lib/pub_sub_model_sync/message_processor.rb +7 -2
  9. data/lib/pub_sub_model_sync/message_publisher.rb +5 -2
  10. data/lib/pub_sub_model_sync/mock_rabbit_service.rb +8 -0
  11. data/lib/pub_sub_model_sync/payload.rb +19 -1
  12. data/lib/pub_sub_model_sync/payload_cache_optimizer.rb +51 -0
  13. data/lib/pub_sub_model_sync/service_base.rb +12 -8
  14. data/lib/pub_sub_model_sync/service_google.rb +2 -3
  15. data/lib/pub_sub_model_sync/service_kafka.rb +2 -1
  16. data/lib/pub_sub_model_sync/service_rabbit.rb +6 -3
  17. data/lib/pub_sub_model_sync/version.rb +1 -1
  18. data/lib/pub_sub_model_sync.rb +1 -0
  19. data/pub_sub_model_sync.gemspec +1 -1
  20. metadata +3 -88
  21. data/samples/README.md +0 -50
  22. data/samples/app1/Dockerfile +0 -13
  23. data/samples/app1/Gemfile +0 -37
  24. data/samples/app1/Gemfile.lock +0 -171
  25. data/samples/app1/README.md +0 -24
  26. data/samples/app1/Rakefile +0 -6
  27. data/samples/app1/app/models/application_record.rb +0 -3
  28. data/samples/app1/app/models/concerns/.keep +0 -0
  29. data/samples/app1/app/models/post.rb +0 -19
  30. data/samples/app1/app/models/user.rb +0 -29
  31. data/samples/app1/bin/bundle +0 -114
  32. data/samples/app1/bin/rails +0 -5
  33. data/samples/app1/bin/rake +0 -5
  34. data/samples/app1/bin/setup +0 -33
  35. data/samples/app1/bin/spring +0 -14
  36. data/samples/app1/config/application.rb +0 -40
  37. data/samples/app1/config/boot.rb +0 -4
  38. data/samples/app1/config/credentials.yml.enc +0 -1
  39. data/samples/app1/config/database.yml +0 -25
  40. data/samples/app1/config/environment.rb +0 -5
  41. data/samples/app1/config/environments/development.rb +0 -63
  42. data/samples/app1/config/environments/production.rb +0 -105
  43. data/samples/app1/config/environments/test.rb +0 -57
  44. data/samples/app1/config/initializers/application_controller_renderer.rb +0 -8
  45. data/samples/app1/config/initializers/backtrace_silencers.rb +0 -8
  46. data/samples/app1/config/initializers/cors.rb +0 -16
  47. data/samples/app1/config/initializers/filter_parameter_logging.rb +0 -6
  48. data/samples/app1/config/initializers/inflections.rb +0 -16
  49. data/samples/app1/config/initializers/mime_types.rb +0 -4
  50. data/samples/app1/config/initializers/pubsub.rb +0 -4
  51. data/samples/app1/config/initializers/wrap_parameters.rb +0 -14
  52. data/samples/app1/config/locales/en.yml +0 -33
  53. data/samples/app1/config/master.key +0 -1
  54. data/samples/app1/config/puma.rb +0 -43
  55. data/samples/app1/config/routes.rb +0 -3
  56. data/samples/app1/config/spring.rb +0 -6
  57. data/samples/app1/config.ru +0 -6
  58. data/samples/app1/db/migrate/20210513080700_create_users.rb +0 -12
  59. data/samples/app1/db/migrate/20210513134332_create_posts.rb +0 -11
  60. data/samples/app1/db/schema.rb +0 -34
  61. data/samples/app1/db/seeds.rb +0 -7
  62. data/samples/app1/docker-compose.yml +0 -32
  63. data/samples/app1/log/.keep +0 -0
  64. data/samples/app2/Dockerfile +0 -13
  65. data/samples/app2/Gemfile +0 -37
  66. data/samples/app2/Gemfile.lock +0 -171
  67. data/samples/app2/README.md +0 -24
  68. data/samples/app2/Rakefile +0 -6
  69. data/samples/app2/app/models/application_record.rb +0 -9
  70. data/samples/app2/app/models/concerns/.keep +0 -0
  71. data/samples/app2/app/models/customer.rb +0 -28
  72. data/samples/app2/app/models/post.rb +0 -10
  73. data/samples/app2/bin/bundle +0 -114
  74. data/samples/app2/bin/rails +0 -5
  75. data/samples/app2/bin/rake +0 -5
  76. data/samples/app2/bin/setup +0 -33
  77. data/samples/app2/bin/spring +0 -14
  78. data/samples/app2/config/application.rb +0 -40
  79. data/samples/app2/config/boot.rb +0 -4
  80. data/samples/app2/config/credentials.yml.enc +0 -1
  81. data/samples/app2/config/database.yml +0 -25
  82. data/samples/app2/config/environment.rb +0 -5
  83. data/samples/app2/config/environments/development.rb +0 -63
  84. data/samples/app2/config/environments/production.rb +0 -105
  85. data/samples/app2/config/environments/test.rb +0 -57
  86. data/samples/app2/config/initializers/application_controller_renderer.rb +0 -8
  87. data/samples/app2/config/initializers/backtrace_silencers.rb +0 -8
  88. data/samples/app2/config/initializers/cors.rb +0 -16
  89. data/samples/app2/config/initializers/filter_parameter_logging.rb +0 -6
  90. data/samples/app2/config/initializers/inflections.rb +0 -16
  91. data/samples/app2/config/initializers/mime_types.rb +0 -4
  92. data/samples/app2/config/initializers/pubsub.rb +0 -4
  93. data/samples/app2/config/initializers/wrap_parameters.rb +0 -14
  94. data/samples/app2/config/locales/en.yml +0 -33
  95. data/samples/app2/config/master.key +0 -1
  96. data/samples/app2/config/puma.rb +0 -43
  97. data/samples/app2/config/routes.rb +0 -3
  98. data/samples/app2/config/spring.rb +0 -6
  99. data/samples/app2/config.ru +0 -6
  100. data/samples/app2/db/development.sqlite3 +0 -0
  101. data/samples/app2/db/migrate/20210513080956_create_customers.rb +0 -10
  102. data/samples/app2/db/migrate/20210513135203_create_posts.rb +0 -10
  103. data/samples/app2/db/schema.rb +0 -31
  104. data/samples/app2/db/seeds.rb +0 -7
  105. data/samples/app2/docker-compose.yml +0 -20
  106. data/samples/app2/log/.keep +0 -0
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: e02077ac63ab98c674a0071b440c77da2e269109d8768c738b82362379e2c921
4
- data.tar.gz: 0d29b69fb65290be2a7a437b87b04d53e4c8a394c98775eb1c98e20d96936866
3
+ metadata.gz: '0839d268607d201c9f77bbbca404f32aa5af02621b7582256eb384b42b9d3985'
4
+ data.tar.gz: 0ad3d2cfdce3ce99eb5d0b95a4292496927befaca2a8e2d83b865af411be2988
5
5
  SHA512:
6
- metadata.gz: 7e8877f20626a3404f8bb13a897d9e4d5fc28761a18e1cd88253d6210c323d25ac3c793c742ededb2ca11c5500b6997d5cf2c176300b444528ee206f6fdf3f0b
7
- data.tar.gz: ae8f2610ee4778746b11781cd2f3b1e2bc97352989cc300cbc95b5884568b73f8f9f431d0d2c36d64f720fc777c21897e3f9e579d38e1b622da860dff8a2d75c
6
+ metadata.gz: c86be499e21572b796060fc07f5d95191971b87366bbebe3b22737d5eb906f25a6e850f4b4b6f2c08f2790541169f36401fb67b95df0c5da365fda1f76e8751f
7
+ data.tar.gz: 3e91e306f83eb4ba8dd5c494aadc90b36a702935d7fb716424af80efee6fe6bce9ff895ae6231aa1c32327d7f468ed2148aa69a5f37842c10e0f50e206dad82a
@@ -24,8 +24,6 @@ jobs:
24
24
  exclude: # rails 6 requires ruby >= 2.5
25
25
  - ruby: 2.4
26
26
  rails: 6
27
- - ruby: '3.0'
28
- rails: 7
29
27
 
30
28
  steps:
31
29
  - uses: actions/checkout@v2
data/.gitignore CHANGED
@@ -6,6 +6,7 @@
6
6
  /pkg/
7
7
  /spec/reports/
8
8
  /tmp/
9
-
9
+ /samples/app2/db/**/*
10
+ /samples/app1/db/**/*
10
11
  # rspec failure tracking
11
12
  .rspec_status
data/CHANGELOG.md CHANGED
@@ -1,5 +1,7 @@
1
1
  # Change Log
2
2
 
3
+ For recent releases list go to: https://github.com/owen2345/pub_sub_model_sync/releases
4
+
3
5
  # 1.2.1 (October 28, 2021)
4
6
  chore: improve logs
5
7
 
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- pub_sub_model_sync (1.2.1)
4
+ pub_sub_model_sync (1.5.0)
5
5
  rails
6
6
 
7
7
  GEM
@@ -99,8 +99,8 @@ GEM
99
99
  googleapis-common-protos-types (>= 1.0.6, < 2.0)
100
100
  googleauth (~> 0.15, >= 0.15.1)
101
101
  grpc (~> 1.36)
102
- globalid (0.4.2)
103
- activesupport (>= 4.2.0)
102
+ globalid (1.0.0)
103
+ activesupport (>= 5.0)
104
104
  google-cloud-core (1.6.0)
105
105
  google-cloud-env (~> 1.0)
106
106
  google-cloud-errors (~> 1.0)
@@ -116,6 +116,7 @@ GEM
116
116
  google-cloud-errors (~> 1.0)
117
117
  grpc-google-iam-v1 (>= 0.6.10, < 2.0)
118
118
  google-protobuf (3.17.0)
119
+ google-protobuf (3.17.0-x86_64-linux)
119
120
  googleapis-common-protos (1.3.11)
120
121
  google-protobuf (~> 3.14)
121
122
  googleapis-common-protos-types (>= 1.0.6, < 2.0)
@@ -132,6 +133,9 @@ GEM
132
133
  grpc (1.37.1)
133
134
  google-protobuf (~> 3.15)
134
135
  googleapis-common-protos-types (~> 1.0)
136
+ grpc (1.37.1-x86_64-linux)
137
+ google-protobuf (~> 3.15)
138
+ googleapis-common-protos-types (~> 1.0)
135
139
  grpc-google-iam-v1 (0.6.11)
136
140
  google-protobuf (~> 3.14)
137
141
  googleapis-common-protos (>= 1.3.11, < 2.0)
@@ -139,27 +143,27 @@ GEM
139
143
  i18n (1.8.10)
140
144
  concurrent-ruby (~> 1.0)
141
145
  jwt (2.2.3)
142
- loofah (2.9.1)
146
+ loofah (2.15.0)
143
147
  crass (~> 1.0.2)
144
148
  nokogiri (>= 1.5.9)
145
149
  mail (2.7.1)
146
150
  mini_mime (>= 0.1.1)
147
- marcel (1.0.1)
151
+ marcel (1.0.2)
148
152
  memoist (0.16.2)
149
153
  method_source (1.0.0)
150
154
  mini_mime (1.0.3)
151
155
  minitest (5.14.4)
152
156
  multi_json (1.15.0)
153
157
  multipart-post (2.1.1)
154
- nio4r (2.5.7)
155
- nokogiri (1.11.3-x86_64-darwin)
158
+ nio4r (2.5.8)
159
+ nokogiri (1.12.5-x86_64-linux)
156
160
  racc (~> 1.4)
157
161
  os (1.1.1)
158
162
  parallel (1.20.1)
159
163
  parser (3.0.1.1)
160
164
  ast (~> 2.4.1)
161
165
  public_suffix (4.0.6)
162
- racc (1.5.2)
166
+ racc (1.6.0)
163
167
  rack (2.2.3)
164
168
  rack-test (1.1.0)
165
169
  rack (>= 1.0, < 3)
@@ -181,7 +185,7 @@ GEM
181
185
  rails-dom-testing (2.0.3)
182
186
  activesupport (>= 4.2.0)
183
187
  nokogiri (>= 1.6)
184
- rails-html-sanitizer (1.3.0)
188
+ rails-html-sanitizer (1.4.2)
185
189
  loofah (~> 2.3)
186
190
  railties (6.1.3.2)
187
191
  actionpack (= 6.1.3.2)
@@ -226,19 +230,19 @@ GEM
226
230
  faraday (>= 0.17.3, < 2.0)
227
231
  jwt (>= 1.5, < 3.0)
228
232
  multi_json (~> 1.10)
229
- sprockets (4.0.2)
233
+ sprockets (4.0.3)
230
234
  concurrent-ruby (~> 1.0)
231
235
  rack (> 1, < 3)
232
- sprockets-rails (3.2.2)
233
- actionpack (>= 4.0)
234
- activesupport (>= 4.0)
236
+ sprockets-rails (3.4.2)
237
+ actionpack (>= 5.2)
238
+ activesupport (>= 5.2)
235
239
  sprockets (>= 3.0.0)
236
240
  sqlite3 (1.4.2)
237
- thor (1.1.0)
241
+ thor (1.2.1)
238
242
  tzinfo (2.0.4)
239
243
  concurrent-ruby (~> 1.0)
240
244
  unicode-display_width (1.7.0)
241
- websocket-driver (0.7.3)
245
+ websocket-driver (0.7.5)
242
246
  websocket-extensions (>= 0.1.0)
243
247
  websocket-extensions (0.1.5)
244
248
  zeitwerk (2.4.2)
@@ -260,4 +264,4 @@ DEPENDENCIES
260
264
  sqlite3
261
265
 
262
266
  BUNDLED WITH
263
- 2.2.29
267
+ 2.3.9
data/README.md CHANGED
@@ -167,7 +167,7 @@ PubSubModelSync::Payload.new({ ids: [my_user.id] }, { klass: 'User', action: :ba
167
167
  ps_class_subscribe(action, settings)
168
168
  end
169
169
  ```
170
- - Instance subscriptions: `ps_subscribe(action, mapping, settings)`
170
+ - Instance subscriptions: `ps_subscribe(action, mapping, settings, &block)`
171
171
  When model receives the corresponding notification, `action` or `to_action` method will be called on the model. Like: `model.destroy`
172
172
  - `action` (Symbol|Array<Symbol>) Only notifications with this action name will be processed by this subscription. Sample: save|create|update|destroy|<any_other_action>
173
173
  - `mapping` (Array<String>) Data mapping from payload data into model attributes, sample: ["email", "full_name:name"] (Note: Only these attributes will be assigned/synced to the current model)
@@ -182,12 +182,14 @@ PubSubModelSync::Payload.new({ ids: [my_user.id] }, { klass: 'User', action: :ba
182
182
  Sample: `id: :id` will search for a model like: `model_class.where(id: payload.data[:id])`
183
183
  Sample: `id: [:id, :email:user_email]` will search for a model like: `model_class.where(id: payload.data[:id], user_email: payload.data[:email])`
184
184
  - `if:` (Symbol|Proc|Array<Symbol>) Method(s) or block called for the confirmation before calling the callback
185
- - `unless:` (Symbol|Proc|Array<Symbol>) Method or block called for the negation before calling the callback
185
+ - `unless:` (Symbol|Proc|Array<Symbol>) Method or block called for the negation before calling the callback
186
+ - `&block` Block to be used as the callback method (ignored if `:to_action` is present). Sample: `ps_subscribe(:send_welcome, %i[email]) { |_data| puts model.email }`
186
187
 
187
- - Class subscriptions: `ps_class_subscribe(action, settings)`
188
+ - Class subscriptions: `ps_class_subscribe(action, settings, &block)`
188
189
  When current class receives the corresponding notification, `action` or `to_action` method will be called on the Class. Like: `User.hello(data)`
189
190
  * `action` (Symbol) Notification.action name
190
191
  * `settings` (Hash) refer ps_subscribe.settings except(:id)
192
+ * `&block` Block to be used as the callback method (ignored if `:to_action` is present). Sample: `ps_class_subscribe(:send_welcome) { |data| puts data }`
191
193
 
192
194
  - `ps_processing_payload` a class and instance variable that saves the current payload being processed
193
195
 
@@ -313,10 +315,15 @@ Any notification before delivering is transformed as a Payload for a better port
313
315
  * `headers`: (Hash) Notification settings that defines how the notification will be processed or delivered.
314
316
  - `ordering_key`: (String, optional): notifications with the same `ordering_key` are processed in the same order they were delivered, default: `<model.class.name>/<model.id>` when instance notification and `klass_name` when class notification.
315
317
  Note: Final `ordering_key` is calculated as: `payload.headers[:forced_ordering_key] || current_transaction&.key || payload.headers[:ordering_key]`
316
- - `internal_key`: (String, optional) Internal identifier of the payload, default: `<model.class.name>/<action>/<model.id>` when model notification and `<klass_name>/<action>` when class notification (Useful for caching techniques).
317
318
  - `topic_name`: (String|Array<String>, optional): Specific topic name where to deliver the notification (default `PubSubModelSync::Config.topic_name`).
318
319
  - `forced_ordering_key`: (String, optional): Overrides `ordering_key` with the provided value even withing transactions. Default `nil`.
319
- - `app_key`: (Auto calculated): Name of the application who delivered the notification.
320
+ - `cache` (Boolean | Hash, Default false) Cache settings
321
+ - `true`: Skip publishing similar payloads
322
+ - `Hash<required: Array<Symbol>>`: Same as `true` and enables payload optimization to exclude unchanged non important attributes. Sample: `{ required: %i[id email] }`
323
+
324
+ ** Read ONLY **
325
+ - `internal_key`: Internal identifier of the payload, default: `<model.class.name>/<action>/<model.id>` when model notification and `<klass_name>/<action>` when class notification (Useful for caching techniques).
326
+ - `app_key`: (Auto calculated): Name of the application who delivered the notification.
320
327
  - `uuid`: (Auto calculated): Unique notification identifier (Very useful when debugging).
321
328
  Note: To reduce Payload size, some header info are not delivered (Enable debug mode to deliver all payload info).
322
329
 
@@ -386,37 +393,15 @@ Note: To reduce Payload size, some header info are not delivered (Enable debug m
386
393
  ## **Testing with RSpec**
387
394
  - Config: (spec/rails_helper.rb)
388
395
  ```ruby
389
-
390
- # when using google service
391
- require 'pub_sub_model_sync/mock_google_service'
392
- config.before(:each) do
393
- google_mock = PubSubModelSync::MockGoogleService.new
394
- allow(Google::Cloud::Pubsub).to receive(:new).and_return(google_mock)
395
- end
396
-
397
- # when using rabbitmq service
398
- require 'pub_sub_model_sync/mock_rabbit_service'
399
- config.before(:each) do
400
- rabbit_mock = PubSubModelSync::MockRabbitService.new
401
- allow(Bunny).to receive(:new).and_return(rabbit_mock)
402
- end
403
-
404
- # when using apache kafka service
405
- require 'pub_sub_model_sync/mock_kafka_service'
406
- config.before(:each) do
407
- kafka_mock = PubSubModelSync::MockKafkaService.new
408
- allow(Kafka).to receive(:new).and_return(kafka_mock)
409
- end
410
-
411
- # disable all models sync by default (reduces testing time)
412
396
  config.before(:each) do
413
- allow(PubSubModelSync::MessagePublisher).to receive(:publish_data) # disable class level notif
414
- allow(PubSubModelSync::MessagePublisher).to receive(:publish_model) # disable instance level notif
397
+ # disable delivering notifications to pubsub
398
+ allow(PubSubModelSync::MessagePublisher).to receive(:connector_publish)
399
+ # disable all models sync by default (reduces testing time by avoiding to build payload data)
400
+ allow(PubSubModelSync::MessagePublisher).to receive(:publish_model)
415
401
  end
416
402
 
417
403
  # enable all models sync only for tests that includes 'sync: true'
418
404
  config.before(:each, sync: true) do
419
- allow(PubSubModelSync::MessagePublisher).to receive(:publish_data).and_call_original
420
405
  allow(PubSubModelSync::MessagePublisher).to receive(:publish_model).and_call_original
421
406
  end
422
407
 
@@ -426,9 +411,9 @@ Note: To reduce Payload size, some header info are not delivered (Enable debug m
426
411
  # end
427
412
  ```
428
413
  - Examples:
429
- - **Publisher**
414
+ - **Publisher**
415
+ Note: **Do not forget to include 'sync: true'** to enable publishing pubsub notifications
430
416
  ```ruby
431
- # Do not forget to include 'sync: true' to enable publishing pubsub notifications
432
417
  describe 'When publishing sync', truncate: true, sync: true do
433
418
  it 'publishes user notification when created' do
434
419
  expect_publish_notification(:create, klass: 'User')
@@ -7,12 +7,13 @@ module PubSubModelSync
7
7
 
8
8
  # customizable callbacks
9
9
  cattr_accessor(:debug) { false }
10
- cattr_accessor :logger # LoggerInst
10
+ cattr_accessor(:logger) { Rails.logger }
11
11
  cattr_accessor(:transactions_max_buffer) { 1 }
12
+ cattr_accessor(:skip_cache) { false }
12
13
 
13
14
  cattr_accessor(:on_before_processing) { ->(_payload, _info) {} } # return :cancel to skip
14
15
  cattr_accessor(:on_success_processing) { ->(_payload, _info) {} }
15
- cattr_accessor(:on_error_processing) { ->(_exception, _info) {} }
16
+ cattr_accessor(:on_error_processing) { ->(exception, _info) { raise(exception) } }
16
17
  cattr_accessor(:on_before_publish) { ->(_payload) {} } # return :cancel to skip
17
18
  cattr_accessor(:on_after_publish) { ->(_payload) {} }
18
19
  cattr_accessor(:on_error_publish) { ->(_exception, _info) {} }
@@ -49,9 +49,14 @@ module PubSubModelSync
49
49
 
50
50
  # @param error (StandardError)
51
51
  def notify_error(error)
52
- info = [payload, error.message, error.backtrace]
52
+ error_msg = 'Error processing message: '
53
+ error_details = [payload, error.message, error.backtrace]
53
54
  res = config.on_error_processing.call(error, { payload: payload })
54
- log("Error processing message: #{info}", :error) if res != :skip_log
55
+ log("#{error_msg} #{error_details}", :error) if res != :skip_log
56
+ rescue => e
57
+ error_details = [payload, e.message, e.backtrace]
58
+ log("#{error_msg} #{error_details}", :error)
59
+ raise(e)
55
60
  end
56
61
 
57
62
  def lost_db_connection?(error)
@@ -98,8 +98,11 @@ module PubSubModelSync
98
98
  private
99
99
 
100
100
  def ensure_publish(payload)
101
- cancelled = config.on_before_publish.call(payload) == :cancel
102
- log("Publish cancelled by config.on_before_publish: #{[payload]}") if config.debug && cancelled
101
+ cache_klass = PubSubModelSync::PayloadCacheOptimizer
102
+ cancelled = payload.cache_settings ? cache_klass.new(payload).call == :already_sent : false
103
+ cancelled ||= config.on_before_publish.call(payload) == :cancel
104
+ log_msg = "Publish cancelled by config.on_before_publish or cache checker: #{[payload]}"
105
+ log(log_msg) if config.debug && cancelled
103
106
  !cancelled
104
107
  end
105
108
 
@@ -24,6 +24,10 @@ module PubSubModelSync
24
24
  def publish(*_args)
25
25
  true
26
26
  end
27
+
28
+ def channel
29
+ MockChannel.new
30
+ end
27
31
  end
28
32
 
29
33
  class MockChannel
@@ -39,6 +43,10 @@ module PubSubModelSync
39
43
  def close
40
44
  true
41
45
  end
46
+
47
+ def ack(_delivery_tag)
48
+ true
49
+ end
42
50
  end
43
51
 
44
52
  def create_channel(*_args)
@@ -19,9 +19,17 @@ module PubSubModelSync
19
19
  # <klass>: when class message
20
20
  # <klass/id>: when model message
21
21
  # topic_name (String|Array<String>): Specific topic name to be used when delivering the
22
- # message (default first topic)
22
+ # message (default Config.topic_name)
23
23
  # forced_ordering_key (String, optional): Will force to use this value as the ordering_key,
24
24
  # even withing transactions. Default nil.
25
+ # cache (Boolean | Hash, Default false) Cache settings
26
+ # true: Skip publishing similar payloads
27
+ # Hash<required: Array<Symbol>>: Same as true and enables payload optimization to exclude
28
+ # unchanged non important attributes. Sample: { required: %i[id email] }
29
+ # --- READ ONLY ----
30
+ # app_key: (string) Subscriber-Key of the application who delivered the notification
31
+ # internal_key: (String) "<klass>/<action>"
32
+ # uuid: Unique notification identifier
25
33
  def initialize(data, info, headers = {})
26
34
  @data = data.deep_symbolize_keys
27
35
  @info = info.deep_symbolize_keys
@@ -75,6 +83,16 @@ module PubSubModelSync
75
83
  klass.publish(self)
76
84
  end
77
85
 
86
+ # @param attr_keys (Array<Symbol>) List of attributes to be excluded from payload
87
+ def exclude_data_attrs(attr_keys)
88
+ @data = data.except(*attr_keys)
89
+ end
90
+
91
+ # Attributes to always be delivered after cache optimization
92
+ def cache_settings
93
+ headers[:cache]
94
+ end
95
+
78
96
  # convert payload data into Payload
79
97
  # @param data [Hash]: payload data (:data, :info, :headers)
80
98
  def self.from_payload_data(data)
@@ -0,0 +1,51 @@
1
+ # frozen_string_literal: true
2
+
3
+ module PubSubModelSync
4
+ class PayloadCacheOptimizer < PubSubModelSync::Base
5
+ # Optimizes payload data to deliver only the required ones and the changed ones and thus avoid
6
+ # delivering unnecessary notifications.
7
+ # Uses Rails.cache to retrieve previous delivered data.
8
+ attr_reader :payload, :required_attrs, :cache_key
9
+
10
+ # @param payload (Payload)
11
+ def initialize(payload)
12
+ @payload = payload
13
+ @cache_key = "pubsub/#{payload.headers[:internal_key]}/#{payload.headers[:topic_name]}"
14
+ end
15
+
16
+ # @return (:already_sent|Payload)
17
+ def call
18
+ backup_data = payload.data.clone
19
+ return payload if cache_disabled?
20
+ return :already_sent if previous_payload_data == payload.data
21
+
22
+ optimize_payload if optimization_enabled?
23
+ Rails.cache.write(cache_key, backup_data, expires_in: 1.week)
24
+ payload
25
+ end
26
+
27
+ private
28
+
29
+ def optimization_enabled?
30
+ previous_payload_data && payload.cache_settings.is_a?(Hash)
31
+ end
32
+
33
+ def cache_disabled?
34
+ res = config.skip_cache || Rails.cache.nil?
35
+ log("Skipping cache, it was disabled: #{[payload]}") if res && debug?
36
+ res
37
+ end
38
+
39
+ def previous_payload_data
40
+ @previous_payload_data ||= Rails.cache.read(cache_key)
41
+ end
42
+
43
+ def optimize_payload # rubocop:disable Metrics/AbcSize
44
+ changed_keys = Hash[(payload.data.to_a - previous_payload_data.to_a)].keys.map(&:to_sym)
45
+ required_keys = payload.cache_settings[:required].map(&:to_sym)
46
+ invalid_keys = payload.data.keys - (changed_keys + required_keys)
47
+ log("Excluding non changed attributes: #{invalid_keys} from: #{payload.inspect}") if debug?
48
+ payload.exclude_data_attrs(invalid_keys)
49
+ end
50
+ end
51
+ end
@@ -24,7 +24,7 @@ module PubSubModelSync
24
24
  # @return (String): Json Format
25
25
  def encode_payload(payload)
26
26
  data = payload.to_h
27
- not_important_keys = %i[forced_ordering_key]
27
+ not_important_keys = %i[forced_ordering_key cache]
28
28
  reduce_payload_size = !config.debug
29
29
  data[:headers].except!(*not_important_keys) if reduce_payload_size
30
30
  data.to_json
@@ -33,25 +33,29 @@ module PubSubModelSync
33
33
  # @param (String: Payload in json format)
34
34
  def process_message(payload_info)
35
35
  payload = decode_payload(payload_info)
36
- return payload.process unless same_app_message?(payload)
36
+ return unless payload
37
+ return if same_app_message?(payload)
37
38
 
38
- log("Skipping message from same origin: #{[payload]}") if config.debug
39
- rescue => e
40
- error_payload = [payload, e.message, e.backtrace]
41
- log("Error while starting to process a message: #{error_payload}", :error)
39
+ payload.process
42
40
  end
43
41
 
44
- # @return Payload
42
+ # @return [Payload,Nil]
45
43
  def decode_payload(payload_info)
46
44
  payload = ::PubSubModelSync::Payload.from_payload_data(JSON.parse(payload_info))
47
45
  log("Received message: #{[payload]}") if config.debug
48
46
  payload
47
+ rescue => e
48
+ error_payload = [payload_info, e.message, e.backtrace]
49
+ log("Error while parsing payload: #{error_payload}", :error)
50
+ nil
49
51
  end
50
52
 
51
53
  # @param payload (Payload)
52
54
  def same_app_message?(payload)
53
55
  key = payload.headers[:app_key]
54
- key && key == config.subscription_key
56
+ res = key && key == config.subscription_key
57
+ log("Skipping message from same origin: #{[payload]}") if res && config.debug
58
+ res
55
59
  end
56
60
  end
57
61
  end
@@ -81,8 +81,8 @@ module PubSubModelSync
81
81
  def subscribe_to_topics
82
82
  topics.map do |key, topic|
83
83
  subs_name = "#{config.subscription_key}_#{key}"
84
- subscription = topic.subscription(subs_name) || topic.subscribe(subs_name, SUBSCRIPTION_SETTINGS)
85
- subscriber = subscription.listen(LISTEN_SETTINGS, &method(:process_message))
84
+ subscription = topic.subscription(subs_name) || topic.subscribe(subs_name, **SUBSCRIPTION_SETTINGS)
85
+ subscriber = subscription.listen(**LISTEN_SETTINGS, &method(:process_message))
86
86
  subscriber.start
87
87
  log("Subscribed to topic: #{topic.name} as: #{subs_name}")
88
88
  subscriber
@@ -92,7 +92,6 @@ module PubSubModelSync
92
92
  def process_message(received_message)
93
93
  message = received_message.message
94
94
  super(message.data) if message.attributes[SERVICE_KEY]
95
- ensure
96
95
  received_message.acknowledge!
97
96
  end
98
97
  end
@@ -8,7 +8,7 @@ end
8
8
  module PubSubModelSync
9
9
  class ServiceKafka < ServiceBase
10
10
  QTY_WORKERS = 10
11
- LISTEN_SETTINGS = {}.freeze
11
+ LISTEN_SETTINGS = { automatically_mark_as_processed: false }.freeze
12
12
  PUBLISH_SETTINGS = {}.freeze
13
13
  PRODUCER_SETTINGS = { delivery_threshold: 200, delivery_interval: 30 }.freeze
14
14
  cattr_accessor :producer
@@ -73,6 +73,7 @@ module PubSubModelSync
73
73
 
74
74
  def process_message(message)
75
75
  super(message.value) if message.headers[SERVICE_KEY]
76
+ consumer.mark_message_as_processed(message)
76
77
  end
77
78
 
78
79
  # Check topic existence, create if missing topic
@@ -8,7 +8,7 @@ end
8
8
  module PubSubModelSync
9
9
  class ServiceRabbit < ServiceBase
10
10
  QUEUE_SETTINGS = { durable: true, auto_delete: false }.freeze
11
- LISTEN_SETTINGS = { manual_ack: false }.freeze
11
+ LISTEN_SETTINGS = { manual_ack: true }.freeze
12
12
  PUBLISH_SETTINGS = {}.freeze
13
13
 
14
14
  # @!attribute topic_names (Array): ['Topic 1', 'Topic 2']
@@ -25,7 +25,9 @@ module PubSubModelSync
25
25
 
26
26
  def listen_messages
27
27
  log('Listener starting...')
28
- subscribe_to_queues { |queue| queue.subscribe(LISTEN_SETTINGS, &method(:process_message)) }
28
+ subscribe_to_queues do |queue|
29
+ queue.subscribe(LISTEN_SETTINGS) { |info, meta, payload| process_message(queue, info, meta, payload) }
30
+ end
29
31
  log('Listener started')
30
32
  loop { sleep 5 }
31
33
  rescue PubSubModelSync::Runner::ShutDown
@@ -62,8 +64,9 @@ module PubSubModelSync
62
64
  }.merge(PUBLISH_SETTINGS)
63
65
  end
64
66
 
65
- def process_message(_delivery_info, meta_info, payload)
67
+ def process_message(queue, delivery_info, meta_info, payload)
66
68
  super(payload) if meta_info[:type] == SERVICE_KEY
69
+ queue.channel.ack(delivery_info.delivery_tag)
67
70
  end
68
71
 
69
72
  def subscribe_to_queues(&block)
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module PubSubModelSync
4
- VERSION = '1.3.0'
4
+ VERSION = '1.5.0'
5
5
  end
@@ -16,6 +16,7 @@ require 'pub_sub_model_sync/message_processor'
16
16
  require 'pub_sub_model_sync/run_subscriber'
17
17
 
18
18
  require 'pub_sub_model_sync/payload_builder'
19
+ require 'pub_sub_model_sync/payload_cache_optimizer'
19
20
  require 'pub_sub_model_sync/subscriber'
20
21
 
21
22
  require 'pub_sub_model_sync/service_base'
@@ -27,7 +27,7 @@ Gem::Specification.new do |spec|
27
27
  # into git.
28
28
  spec.files = Dir.chdir(File.expand_path(__dir__)) do
29
29
  `git ls-files -z`.split("\x0")
30
- .reject { |f| f.match(%r{^(test|spec|features)/}) }
30
+ .reject { |f| f.match(%r{^(test|spec|features|samples)/}) }
31
31
  end
32
32
  spec.bindir = 'exe'
33
33
  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }