pub_sub_model_sync 1.3.0 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/ruby.yml +0 -2
  3. data/.gitignore +2 -1
  4. data/CHANGELOG.md +2 -0
  5. data/Gemfile.lock +20 -16
  6. data/README.md +18 -33
  7. data/lib/pub_sub_model_sync/config.rb +3 -2
  8. data/lib/pub_sub_model_sync/message_processor.rb +7 -2
  9. data/lib/pub_sub_model_sync/message_publisher.rb +5 -2
  10. data/lib/pub_sub_model_sync/mock_rabbit_service.rb +8 -0
  11. data/lib/pub_sub_model_sync/payload.rb +19 -1
  12. data/lib/pub_sub_model_sync/payload_cache_optimizer.rb +51 -0
  13. data/lib/pub_sub_model_sync/service_base.rb +12 -8
  14. data/lib/pub_sub_model_sync/service_google.rb +2 -3
  15. data/lib/pub_sub_model_sync/service_kafka.rb +2 -1
  16. data/lib/pub_sub_model_sync/service_rabbit.rb +6 -3
  17. data/lib/pub_sub_model_sync/version.rb +1 -1
  18. data/lib/pub_sub_model_sync.rb +1 -0
  19. data/pub_sub_model_sync.gemspec +1 -1
  20. metadata +3 -88
  21. data/samples/README.md +0 -50
  22. data/samples/app1/Dockerfile +0 -13
  23. data/samples/app1/Gemfile +0 -37
  24. data/samples/app1/Gemfile.lock +0 -171
  25. data/samples/app1/README.md +0 -24
  26. data/samples/app1/Rakefile +0 -6
  27. data/samples/app1/app/models/application_record.rb +0 -3
  28. data/samples/app1/app/models/concerns/.keep +0 -0
  29. data/samples/app1/app/models/post.rb +0 -19
  30. data/samples/app1/app/models/user.rb +0 -29
  31. data/samples/app1/bin/bundle +0 -114
  32. data/samples/app1/bin/rails +0 -5
  33. data/samples/app1/bin/rake +0 -5
  34. data/samples/app1/bin/setup +0 -33
  35. data/samples/app1/bin/spring +0 -14
  36. data/samples/app1/config/application.rb +0 -40
  37. data/samples/app1/config/boot.rb +0 -4
  38. data/samples/app1/config/credentials.yml.enc +0 -1
  39. data/samples/app1/config/database.yml +0 -25
  40. data/samples/app1/config/environment.rb +0 -5
  41. data/samples/app1/config/environments/development.rb +0 -63
  42. data/samples/app1/config/environments/production.rb +0 -105
  43. data/samples/app1/config/environments/test.rb +0 -57
  44. data/samples/app1/config/initializers/application_controller_renderer.rb +0 -8
  45. data/samples/app1/config/initializers/backtrace_silencers.rb +0 -8
  46. data/samples/app1/config/initializers/cors.rb +0 -16
  47. data/samples/app1/config/initializers/filter_parameter_logging.rb +0 -6
  48. data/samples/app1/config/initializers/inflections.rb +0 -16
  49. data/samples/app1/config/initializers/mime_types.rb +0 -4
  50. data/samples/app1/config/initializers/pubsub.rb +0 -4
  51. data/samples/app1/config/initializers/wrap_parameters.rb +0 -14
  52. data/samples/app1/config/locales/en.yml +0 -33
  53. data/samples/app1/config/master.key +0 -1
  54. data/samples/app1/config/puma.rb +0 -43
  55. data/samples/app1/config/routes.rb +0 -3
  56. data/samples/app1/config/spring.rb +0 -6
  57. data/samples/app1/config.ru +0 -6
  58. data/samples/app1/db/migrate/20210513080700_create_users.rb +0 -12
  59. data/samples/app1/db/migrate/20210513134332_create_posts.rb +0 -11
  60. data/samples/app1/db/schema.rb +0 -34
  61. data/samples/app1/db/seeds.rb +0 -7
  62. data/samples/app1/docker-compose.yml +0 -32
  63. data/samples/app1/log/.keep +0 -0
  64. data/samples/app2/Dockerfile +0 -13
  65. data/samples/app2/Gemfile +0 -37
  66. data/samples/app2/Gemfile.lock +0 -171
  67. data/samples/app2/README.md +0 -24
  68. data/samples/app2/Rakefile +0 -6
  69. data/samples/app2/app/models/application_record.rb +0 -9
  70. data/samples/app2/app/models/concerns/.keep +0 -0
  71. data/samples/app2/app/models/customer.rb +0 -28
  72. data/samples/app2/app/models/post.rb +0 -10
  73. data/samples/app2/bin/bundle +0 -114
  74. data/samples/app2/bin/rails +0 -5
  75. data/samples/app2/bin/rake +0 -5
  76. data/samples/app2/bin/setup +0 -33
  77. data/samples/app2/bin/spring +0 -14
  78. data/samples/app2/config/application.rb +0 -40
  79. data/samples/app2/config/boot.rb +0 -4
  80. data/samples/app2/config/credentials.yml.enc +0 -1
  81. data/samples/app2/config/database.yml +0 -25
  82. data/samples/app2/config/environment.rb +0 -5
  83. data/samples/app2/config/environments/development.rb +0 -63
  84. data/samples/app2/config/environments/production.rb +0 -105
  85. data/samples/app2/config/environments/test.rb +0 -57
  86. data/samples/app2/config/initializers/application_controller_renderer.rb +0 -8
  87. data/samples/app2/config/initializers/backtrace_silencers.rb +0 -8
  88. data/samples/app2/config/initializers/cors.rb +0 -16
  89. data/samples/app2/config/initializers/filter_parameter_logging.rb +0 -6
  90. data/samples/app2/config/initializers/inflections.rb +0 -16
  91. data/samples/app2/config/initializers/mime_types.rb +0 -4
  92. data/samples/app2/config/initializers/pubsub.rb +0 -4
  93. data/samples/app2/config/initializers/wrap_parameters.rb +0 -14
  94. data/samples/app2/config/locales/en.yml +0 -33
  95. data/samples/app2/config/master.key +0 -1
  96. data/samples/app2/config/puma.rb +0 -43
  97. data/samples/app2/config/routes.rb +0 -3
  98. data/samples/app2/config/spring.rb +0 -6
  99. data/samples/app2/config.ru +0 -6
  100. data/samples/app2/db/development.sqlite3 +0 -0
  101. data/samples/app2/db/migrate/20210513080956_create_customers.rb +0 -10
  102. data/samples/app2/db/migrate/20210513135203_create_posts.rb +0 -10
  103. data/samples/app2/db/schema.rb +0 -31
  104. data/samples/app2/db/seeds.rb +0 -7
  105. data/samples/app2/docker-compose.yml +0 -20
  106. data/samples/app2/log/.keep +0 -0
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: e02077ac63ab98c674a0071b440c77da2e269109d8768c738b82362379e2c921
4
- data.tar.gz: 0d29b69fb65290be2a7a437b87b04d53e4c8a394c98775eb1c98e20d96936866
3
+ metadata.gz: '0839d268607d201c9f77bbbca404f32aa5af02621b7582256eb384b42b9d3985'
4
+ data.tar.gz: 0ad3d2cfdce3ce99eb5d0b95a4292496927befaca2a8e2d83b865af411be2988
5
5
  SHA512:
6
- metadata.gz: 7e8877f20626a3404f8bb13a897d9e4d5fc28761a18e1cd88253d6210c323d25ac3c793c742ededb2ca11c5500b6997d5cf2c176300b444528ee206f6fdf3f0b
7
- data.tar.gz: ae8f2610ee4778746b11781cd2f3b1e2bc97352989cc300cbc95b5884568b73f8f9f431d0d2c36d64f720fc777c21897e3f9e579d38e1b622da860dff8a2d75c
6
+ metadata.gz: c86be499e21572b796060fc07f5d95191971b87366bbebe3b22737d5eb906f25a6e850f4b4b6f2c08f2790541169f36401fb67b95df0c5da365fda1f76e8751f
7
+ data.tar.gz: 3e91e306f83eb4ba8dd5c494aadc90b36a702935d7fb716424af80efee6fe6bce9ff895ae6231aa1c32327d7f468ed2148aa69a5f37842c10e0f50e206dad82a
@@ -24,8 +24,6 @@ jobs:
24
24
  exclude: # rails 6 requires ruby >= 2.5
25
25
  - ruby: 2.4
26
26
  rails: 6
27
- - ruby: '3.0'
28
- rails: 7
29
27
 
30
28
  steps:
31
29
  - uses: actions/checkout@v2
data/.gitignore CHANGED
@@ -6,6 +6,7 @@
6
6
  /pkg/
7
7
  /spec/reports/
8
8
  /tmp/
9
-
9
+ /samples/app2/db/**/*
10
+ /samples/app1/db/**/*
10
11
  # rspec failure tracking
11
12
  .rspec_status
data/CHANGELOG.md CHANGED
@@ -1,5 +1,7 @@
1
1
  # Change Log
2
2
 
3
+ For recent releases list go to: https://github.com/owen2345/pub_sub_model_sync/releases
4
+
3
5
  # 1.2.1 (October 28, 2021)
4
6
  chore: improve logs
5
7
 
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- pub_sub_model_sync (1.2.1)
4
+ pub_sub_model_sync (1.5.0)
5
5
  rails
6
6
 
7
7
  GEM
@@ -99,8 +99,8 @@ GEM
99
99
  googleapis-common-protos-types (>= 1.0.6, < 2.0)
100
100
  googleauth (~> 0.15, >= 0.15.1)
101
101
  grpc (~> 1.36)
102
- globalid (0.4.2)
103
- activesupport (>= 4.2.0)
102
+ globalid (1.0.0)
103
+ activesupport (>= 5.0)
104
104
  google-cloud-core (1.6.0)
105
105
  google-cloud-env (~> 1.0)
106
106
  google-cloud-errors (~> 1.0)
@@ -116,6 +116,7 @@ GEM
116
116
  google-cloud-errors (~> 1.0)
117
117
  grpc-google-iam-v1 (>= 0.6.10, < 2.0)
118
118
  google-protobuf (3.17.0)
119
+ google-protobuf (3.17.0-x86_64-linux)
119
120
  googleapis-common-protos (1.3.11)
120
121
  google-protobuf (~> 3.14)
121
122
  googleapis-common-protos-types (>= 1.0.6, < 2.0)
@@ -132,6 +133,9 @@ GEM
132
133
  grpc (1.37.1)
133
134
  google-protobuf (~> 3.15)
134
135
  googleapis-common-protos-types (~> 1.0)
136
+ grpc (1.37.1-x86_64-linux)
137
+ google-protobuf (~> 3.15)
138
+ googleapis-common-protos-types (~> 1.0)
135
139
  grpc-google-iam-v1 (0.6.11)
136
140
  google-protobuf (~> 3.14)
137
141
  googleapis-common-protos (>= 1.3.11, < 2.0)
@@ -139,27 +143,27 @@ GEM
139
143
  i18n (1.8.10)
140
144
  concurrent-ruby (~> 1.0)
141
145
  jwt (2.2.3)
142
- loofah (2.9.1)
146
+ loofah (2.15.0)
143
147
  crass (~> 1.0.2)
144
148
  nokogiri (>= 1.5.9)
145
149
  mail (2.7.1)
146
150
  mini_mime (>= 0.1.1)
147
- marcel (1.0.1)
151
+ marcel (1.0.2)
148
152
  memoist (0.16.2)
149
153
  method_source (1.0.0)
150
154
  mini_mime (1.0.3)
151
155
  minitest (5.14.4)
152
156
  multi_json (1.15.0)
153
157
  multipart-post (2.1.1)
154
- nio4r (2.5.7)
155
- nokogiri (1.11.3-x86_64-darwin)
158
+ nio4r (2.5.8)
159
+ nokogiri (1.12.5-x86_64-linux)
156
160
  racc (~> 1.4)
157
161
  os (1.1.1)
158
162
  parallel (1.20.1)
159
163
  parser (3.0.1.1)
160
164
  ast (~> 2.4.1)
161
165
  public_suffix (4.0.6)
162
- racc (1.5.2)
166
+ racc (1.6.0)
163
167
  rack (2.2.3)
164
168
  rack-test (1.1.0)
165
169
  rack (>= 1.0, < 3)
@@ -181,7 +185,7 @@ GEM
181
185
  rails-dom-testing (2.0.3)
182
186
  activesupport (>= 4.2.0)
183
187
  nokogiri (>= 1.6)
184
- rails-html-sanitizer (1.3.0)
188
+ rails-html-sanitizer (1.4.2)
185
189
  loofah (~> 2.3)
186
190
  railties (6.1.3.2)
187
191
  actionpack (= 6.1.3.2)
@@ -226,19 +230,19 @@ GEM
226
230
  faraday (>= 0.17.3, < 2.0)
227
231
  jwt (>= 1.5, < 3.0)
228
232
  multi_json (~> 1.10)
229
- sprockets (4.0.2)
233
+ sprockets (4.0.3)
230
234
  concurrent-ruby (~> 1.0)
231
235
  rack (> 1, < 3)
232
- sprockets-rails (3.2.2)
233
- actionpack (>= 4.0)
234
- activesupport (>= 4.0)
236
+ sprockets-rails (3.4.2)
237
+ actionpack (>= 5.2)
238
+ activesupport (>= 5.2)
235
239
  sprockets (>= 3.0.0)
236
240
  sqlite3 (1.4.2)
237
- thor (1.1.0)
241
+ thor (1.2.1)
238
242
  tzinfo (2.0.4)
239
243
  concurrent-ruby (~> 1.0)
240
244
  unicode-display_width (1.7.0)
241
- websocket-driver (0.7.3)
245
+ websocket-driver (0.7.5)
242
246
  websocket-extensions (>= 0.1.0)
243
247
  websocket-extensions (0.1.5)
244
248
  zeitwerk (2.4.2)
@@ -260,4 +264,4 @@ DEPENDENCIES
260
264
  sqlite3
261
265
 
262
266
  BUNDLED WITH
263
- 2.2.29
267
+ 2.3.9
data/README.md CHANGED
@@ -167,7 +167,7 @@ PubSubModelSync::Payload.new({ ids: [my_user.id] }, { klass: 'User', action: :ba
167
167
  ps_class_subscribe(action, settings)
168
168
  end
169
169
  ```
170
- - Instance subscriptions: `ps_subscribe(action, mapping, settings)`
170
+ - Instance subscriptions: `ps_subscribe(action, mapping, settings, &block)`
171
171
  When model receives the corresponding notification, `action` or `to_action` method will be called on the model. Like: `model.destroy`
172
172
  - `action` (Symbol|Array<Symbol>) Only notifications with this action name will be processed by this subscription. Sample: save|create|update|destroy|<any_other_action>
173
173
  - `mapping` (Array<String>) Data mapping from payload data into model attributes, sample: ["email", "full_name:name"] (Note: Only these attributes will be assigned/synced to the current model)
@@ -182,12 +182,14 @@ PubSubModelSync::Payload.new({ ids: [my_user.id] }, { klass: 'User', action: :ba
182
182
  Sample: `id: :id` will search for a model like: `model_class.where(id: payload.data[:id])`
183
183
  Sample: `id: [:id, :email:user_email]` will search for a model like: `model_class.where(id: payload.data[:id], user_email: payload.data[:email])`
184
184
  - `if:` (Symbol|Proc|Array<Symbol>) Method(s) or block called for the confirmation before calling the callback
185
- - `unless:` (Symbol|Proc|Array<Symbol>) Method or block called for the negation before calling the callback
185
+ - `unless:` (Symbol|Proc|Array<Symbol>) Method or block called for the negation before calling the callback
186
+ - `&block` Block to be used as the callback method (ignored if `:to_action` is present). Sample: `ps_subscribe(:send_welcome, %i[email]) { |_data| puts model.email }`
186
187
 
187
- - Class subscriptions: `ps_class_subscribe(action, settings)`
188
+ - Class subscriptions: `ps_class_subscribe(action, settings, &block)`
188
189
  When current class receives the corresponding notification, `action` or `to_action` method will be called on the Class. Like: `User.hello(data)`
189
190
  * `action` (Symbol) Notification.action name
190
191
  * `settings` (Hash) refer ps_subscribe.settings except(:id)
192
+ * `&block` Block to be used as the callback method (ignored if `:to_action` is present). Sample: `ps_class_subscribe(:send_welcome) { |data| puts data }`
191
193
 
192
194
  - `ps_processing_payload` a class and instance variable that saves the current payload being processed
193
195
 
@@ -313,10 +315,15 @@ Any notification before delivering is transformed as a Payload for a better port
313
315
  * `headers`: (Hash) Notification settings that defines how the notification will be processed or delivered.
314
316
  - `ordering_key`: (String, optional): notifications with the same `ordering_key` are processed in the same order they were delivered, default: `<model.class.name>/<model.id>` when instance notification and `klass_name` when class notification.
315
317
  Note: Final `ordering_key` is calculated as: `payload.headers[:forced_ordering_key] || current_transaction&.key || payload.headers[:ordering_key]`
316
- - `internal_key`: (String, optional) Internal identifier of the payload, default: `<model.class.name>/<action>/<model.id>` when model notification and `<klass_name>/<action>` when class notification (Useful for caching techniques).
317
318
  - `topic_name`: (String|Array<String>, optional): Specific topic name where to deliver the notification (default `PubSubModelSync::Config.topic_name`).
318
319
  - `forced_ordering_key`: (String, optional): Overrides `ordering_key` with the provided value even withing transactions. Default `nil`.
319
- - `app_key`: (Auto calculated): Name of the application who delivered the notification.
320
+ - `cache` (Boolean | Hash, Default false) Cache settings
321
+ - `true`: Skip publishing similar payloads
322
+ - `Hash<required: Array<Symbol>>`: Same as `true` and enables payload optimization to exclude unchanged non important attributes. Sample: `{ required: %i[id email] }`
323
+
324
+ ** Read ONLY **
325
+ - `internal_key`: Internal identifier of the payload, default: `<model.class.name>/<action>/<model.id>` when model notification and `<klass_name>/<action>` when class notification (Useful for caching techniques).
326
+ - `app_key`: (Auto calculated): Name of the application who delivered the notification.
320
327
  - `uuid`: (Auto calculated): Unique notification identifier (Very useful when debugging).
321
328
  Note: To reduce Payload size, some header info are not delivered (Enable debug mode to deliver all payload info).
322
329
 
@@ -386,37 +393,15 @@ Note: To reduce Payload size, some header info are not delivered (Enable debug m
386
393
  ## **Testing with RSpec**
387
394
  - Config: (spec/rails_helper.rb)
388
395
  ```ruby
389
-
390
- # when using google service
391
- require 'pub_sub_model_sync/mock_google_service'
392
- config.before(:each) do
393
- google_mock = PubSubModelSync::MockGoogleService.new
394
- allow(Google::Cloud::Pubsub).to receive(:new).and_return(google_mock)
395
- end
396
-
397
- # when using rabbitmq service
398
- require 'pub_sub_model_sync/mock_rabbit_service'
399
- config.before(:each) do
400
- rabbit_mock = PubSubModelSync::MockRabbitService.new
401
- allow(Bunny).to receive(:new).and_return(rabbit_mock)
402
- end
403
-
404
- # when using apache kafka service
405
- require 'pub_sub_model_sync/mock_kafka_service'
406
- config.before(:each) do
407
- kafka_mock = PubSubModelSync::MockKafkaService.new
408
- allow(Kafka).to receive(:new).and_return(kafka_mock)
409
- end
410
-
411
- # disable all models sync by default (reduces testing time)
412
396
  config.before(:each) do
413
- allow(PubSubModelSync::MessagePublisher).to receive(:publish_data) # disable class level notif
414
- allow(PubSubModelSync::MessagePublisher).to receive(:publish_model) # disable instance level notif
397
+ # disable delivering notifications to pubsub
398
+ allow(PubSubModelSync::MessagePublisher).to receive(:connector_publish)
399
+ # disable all models sync by default (reduces testing time by avoiding to build payload data)
400
+ allow(PubSubModelSync::MessagePublisher).to receive(:publish_model)
415
401
  end
416
402
 
417
403
  # enable all models sync only for tests that includes 'sync: true'
418
404
  config.before(:each, sync: true) do
419
- allow(PubSubModelSync::MessagePublisher).to receive(:publish_data).and_call_original
420
405
  allow(PubSubModelSync::MessagePublisher).to receive(:publish_model).and_call_original
421
406
  end
422
407
 
@@ -426,9 +411,9 @@ Note: To reduce Payload size, some header info are not delivered (Enable debug m
426
411
  # end
427
412
  ```
428
413
  - Examples:
429
- - **Publisher**
414
+ - **Publisher**
415
+ Note: **Do not forget to include 'sync: true'** to enable publishing pubsub notifications
430
416
  ```ruby
431
- # Do not forget to include 'sync: true' to enable publishing pubsub notifications
432
417
  describe 'When publishing sync', truncate: true, sync: true do
433
418
  it 'publishes user notification when created' do
434
419
  expect_publish_notification(:create, klass: 'User')
@@ -7,12 +7,13 @@ module PubSubModelSync
7
7
 
8
8
  # customizable callbacks
9
9
  cattr_accessor(:debug) { false }
10
- cattr_accessor :logger # LoggerInst
10
+ cattr_accessor(:logger) { Rails.logger }
11
11
  cattr_accessor(:transactions_max_buffer) { 1 }
12
+ cattr_accessor(:skip_cache) { false }
12
13
 
13
14
  cattr_accessor(:on_before_processing) { ->(_payload, _info) {} } # return :cancel to skip
14
15
  cattr_accessor(:on_success_processing) { ->(_payload, _info) {} }
15
- cattr_accessor(:on_error_processing) { ->(_exception, _info) {} }
16
+ cattr_accessor(:on_error_processing) { ->(exception, _info) { raise(exception) } }
16
17
  cattr_accessor(:on_before_publish) { ->(_payload) {} } # return :cancel to skip
17
18
  cattr_accessor(:on_after_publish) { ->(_payload) {} }
18
19
  cattr_accessor(:on_error_publish) { ->(_exception, _info) {} }
@@ -49,9 +49,14 @@ module PubSubModelSync
49
49
 
50
50
  # @param error (StandardError)
51
51
  def notify_error(error)
52
- info = [payload, error.message, error.backtrace]
52
+ error_msg = 'Error processing message: '
53
+ error_details = [payload, error.message, error.backtrace]
53
54
  res = config.on_error_processing.call(error, { payload: payload })
54
- log("Error processing message: #{info}", :error) if res != :skip_log
55
+ log("#{error_msg} #{error_details}", :error) if res != :skip_log
56
+ rescue => e
57
+ error_details = [payload, e.message, e.backtrace]
58
+ log("#{error_msg} #{error_details}", :error)
59
+ raise(e)
55
60
  end
56
61
 
57
62
  def lost_db_connection?(error)
@@ -98,8 +98,11 @@ module PubSubModelSync
98
98
  private
99
99
 
100
100
  def ensure_publish(payload)
101
- cancelled = config.on_before_publish.call(payload) == :cancel
102
- log("Publish cancelled by config.on_before_publish: #{[payload]}") if config.debug && cancelled
101
+ cache_klass = PubSubModelSync::PayloadCacheOptimizer
102
+ cancelled = payload.cache_settings ? cache_klass.new(payload).call == :already_sent : false
103
+ cancelled ||= config.on_before_publish.call(payload) == :cancel
104
+ log_msg = "Publish cancelled by config.on_before_publish or cache checker: #{[payload]}"
105
+ log(log_msg) if config.debug && cancelled
103
106
  !cancelled
104
107
  end
105
108
 
@@ -24,6 +24,10 @@ module PubSubModelSync
24
24
  def publish(*_args)
25
25
  true
26
26
  end
27
+
28
+ def channel
29
+ MockChannel.new
30
+ end
27
31
  end
28
32
 
29
33
  class MockChannel
@@ -39,6 +43,10 @@ module PubSubModelSync
39
43
  def close
40
44
  true
41
45
  end
46
+
47
+ def ack(_delivery_tag)
48
+ true
49
+ end
42
50
  end
43
51
 
44
52
  def create_channel(*_args)
@@ -19,9 +19,17 @@ module PubSubModelSync
19
19
  # <klass>: when class message
20
20
  # <klass/id>: when model message
21
21
  # topic_name (String|Array<String>): Specific topic name to be used when delivering the
22
- # message (default first topic)
22
+ # message (default Config.topic_name)
23
23
  # forced_ordering_key (String, optional): Will force to use this value as the ordering_key,
24
24
  # even withing transactions. Default nil.
25
+ # cache (Boolean | Hash, Default false) Cache settings
26
+ # true: Skip publishing similar payloads
27
+ # Hash<required: Array<Symbol>>: Same as true and enables payload optimization to exclude
28
+ # unchanged non important attributes. Sample: { required: %i[id email] }
29
+ # --- READ ONLY ----
30
+ # app_key: (string) Subscriber-Key of the application who delivered the notification
31
+ # internal_key: (String) "<klass>/<action>"
32
+ # uuid: Unique notification identifier
25
33
  def initialize(data, info, headers = {})
26
34
  @data = data.deep_symbolize_keys
27
35
  @info = info.deep_symbolize_keys
@@ -75,6 +83,16 @@ module PubSubModelSync
75
83
  klass.publish(self)
76
84
  end
77
85
 
86
+ # @param attr_keys (Array<Symbol>) List of attributes to be excluded from payload
87
+ def exclude_data_attrs(attr_keys)
88
+ @data = data.except(*attr_keys)
89
+ end
90
+
91
+ # Attributes to always be delivered after cache optimization
92
+ def cache_settings
93
+ headers[:cache]
94
+ end
95
+
78
96
  # convert payload data into Payload
79
97
  # @param data [Hash]: payload data (:data, :info, :headers)
80
98
  def self.from_payload_data(data)
@@ -0,0 +1,51 @@
1
+ # frozen_string_literal: true
2
+
3
+ module PubSubModelSync
4
+ class PayloadCacheOptimizer < PubSubModelSync::Base
5
+ # Optimizes payload data to deliver only the required ones and the changed ones and thus avoid
6
+ # delivering unnecessary notifications.
7
+ # Uses Rails.cache to retrieve previous delivered data.
8
+ attr_reader :payload, :required_attrs, :cache_key
9
+
10
+ # @param payload (Payload)
11
+ def initialize(payload)
12
+ @payload = payload
13
+ @cache_key = "pubsub/#{payload.headers[:internal_key]}/#{payload.headers[:topic_name]}"
14
+ end
15
+
16
+ # @return (:already_sent|Payload)
17
+ def call
18
+ backup_data = payload.data.clone
19
+ return payload if cache_disabled?
20
+ return :already_sent if previous_payload_data == payload.data
21
+
22
+ optimize_payload if optimization_enabled?
23
+ Rails.cache.write(cache_key, backup_data, expires_in: 1.week)
24
+ payload
25
+ end
26
+
27
+ private
28
+
29
+ def optimization_enabled?
30
+ previous_payload_data && payload.cache_settings.is_a?(Hash)
31
+ end
32
+
33
+ def cache_disabled?
34
+ res = config.skip_cache || Rails.cache.nil?
35
+ log("Skipping cache, it was disabled: #{[payload]}") if res && debug?
36
+ res
37
+ end
38
+
39
+ def previous_payload_data
40
+ @previous_payload_data ||= Rails.cache.read(cache_key)
41
+ end
42
+
43
+ def optimize_payload # rubocop:disable Metrics/AbcSize
44
+ changed_keys = Hash[(payload.data.to_a - previous_payload_data.to_a)].keys.map(&:to_sym)
45
+ required_keys = payload.cache_settings[:required].map(&:to_sym)
46
+ invalid_keys = payload.data.keys - (changed_keys + required_keys)
47
+ log("Excluding non changed attributes: #{invalid_keys} from: #{payload.inspect}") if debug?
48
+ payload.exclude_data_attrs(invalid_keys)
49
+ end
50
+ end
51
+ end
@@ -24,7 +24,7 @@ module PubSubModelSync
24
24
  # @return (String): Json Format
25
25
  def encode_payload(payload)
26
26
  data = payload.to_h
27
- not_important_keys = %i[forced_ordering_key]
27
+ not_important_keys = %i[forced_ordering_key cache]
28
28
  reduce_payload_size = !config.debug
29
29
  data[:headers].except!(*not_important_keys) if reduce_payload_size
30
30
  data.to_json
@@ -33,25 +33,29 @@ module PubSubModelSync
33
33
  # @param (String: Payload in json format)
34
34
  def process_message(payload_info)
35
35
  payload = decode_payload(payload_info)
36
- return payload.process unless same_app_message?(payload)
36
+ return unless payload
37
+ return if same_app_message?(payload)
37
38
 
38
- log("Skipping message from same origin: #{[payload]}") if config.debug
39
- rescue => e
40
- error_payload = [payload, e.message, e.backtrace]
41
- log("Error while starting to process a message: #{error_payload}", :error)
39
+ payload.process
42
40
  end
43
41
 
44
- # @return Payload
42
+ # @return [Payload,Nil]
45
43
  def decode_payload(payload_info)
46
44
  payload = ::PubSubModelSync::Payload.from_payload_data(JSON.parse(payload_info))
47
45
  log("Received message: #{[payload]}") if config.debug
48
46
  payload
47
+ rescue => e
48
+ error_payload = [payload_info, e.message, e.backtrace]
49
+ log("Error while parsing payload: #{error_payload}", :error)
50
+ nil
49
51
  end
50
52
 
51
53
  # @param payload (Payload)
52
54
  def same_app_message?(payload)
53
55
  key = payload.headers[:app_key]
54
- key && key == config.subscription_key
56
+ res = key && key == config.subscription_key
57
+ log("Skipping message from same origin: #{[payload]}") if res && config.debug
58
+ res
55
59
  end
56
60
  end
57
61
  end
@@ -81,8 +81,8 @@ module PubSubModelSync
81
81
  def subscribe_to_topics
82
82
  topics.map do |key, topic|
83
83
  subs_name = "#{config.subscription_key}_#{key}"
84
- subscription = topic.subscription(subs_name) || topic.subscribe(subs_name, SUBSCRIPTION_SETTINGS)
85
- subscriber = subscription.listen(LISTEN_SETTINGS, &method(:process_message))
84
+ subscription = topic.subscription(subs_name) || topic.subscribe(subs_name, **SUBSCRIPTION_SETTINGS)
85
+ subscriber = subscription.listen(**LISTEN_SETTINGS, &method(:process_message))
86
86
  subscriber.start
87
87
  log("Subscribed to topic: #{topic.name} as: #{subs_name}")
88
88
  subscriber
@@ -92,7 +92,6 @@ module PubSubModelSync
92
92
  def process_message(received_message)
93
93
  message = received_message.message
94
94
  super(message.data) if message.attributes[SERVICE_KEY]
95
- ensure
96
95
  received_message.acknowledge!
97
96
  end
98
97
  end
@@ -8,7 +8,7 @@ end
8
8
  module PubSubModelSync
9
9
  class ServiceKafka < ServiceBase
10
10
  QTY_WORKERS = 10
11
- LISTEN_SETTINGS = {}.freeze
11
+ LISTEN_SETTINGS = { automatically_mark_as_processed: false }.freeze
12
12
  PUBLISH_SETTINGS = {}.freeze
13
13
  PRODUCER_SETTINGS = { delivery_threshold: 200, delivery_interval: 30 }.freeze
14
14
  cattr_accessor :producer
@@ -73,6 +73,7 @@ module PubSubModelSync
73
73
 
74
74
  def process_message(message)
75
75
  super(message.value) if message.headers[SERVICE_KEY]
76
+ consumer.mark_message_as_processed(message)
76
77
  end
77
78
 
78
79
  # Check topic existence, create if missing topic
@@ -8,7 +8,7 @@ end
8
8
  module PubSubModelSync
9
9
  class ServiceRabbit < ServiceBase
10
10
  QUEUE_SETTINGS = { durable: true, auto_delete: false }.freeze
11
- LISTEN_SETTINGS = { manual_ack: false }.freeze
11
+ LISTEN_SETTINGS = { manual_ack: true }.freeze
12
12
  PUBLISH_SETTINGS = {}.freeze
13
13
 
14
14
  # @!attribute topic_names (Array): ['Topic 1', 'Topic 2']
@@ -25,7 +25,9 @@ module PubSubModelSync
25
25
 
26
26
  def listen_messages
27
27
  log('Listener starting...')
28
- subscribe_to_queues { |queue| queue.subscribe(LISTEN_SETTINGS, &method(:process_message)) }
28
+ subscribe_to_queues do |queue|
29
+ queue.subscribe(LISTEN_SETTINGS) { |info, meta, payload| process_message(queue, info, meta, payload) }
30
+ end
29
31
  log('Listener started')
30
32
  loop { sleep 5 }
31
33
  rescue PubSubModelSync::Runner::ShutDown
@@ -62,8 +64,9 @@ module PubSubModelSync
62
64
  }.merge(PUBLISH_SETTINGS)
63
65
  end
64
66
 
65
- def process_message(_delivery_info, meta_info, payload)
67
+ def process_message(queue, delivery_info, meta_info, payload)
66
68
  super(payload) if meta_info[:type] == SERVICE_KEY
69
+ queue.channel.ack(delivery_info.delivery_tag)
67
70
  end
68
71
 
69
72
  def subscribe_to_queues(&block)
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module PubSubModelSync
4
- VERSION = '1.3.0'
4
+ VERSION = '1.5.0'
5
5
  end
@@ -16,6 +16,7 @@ require 'pub_sub_model_sync/message_processor'
16
16
  require 'pub_sub_model_sync/run_subscriber'
17
17
 
18
18
  require 'pub_sub_model_sync/payload_builder'
19
+ require 'pub_sub_model_sync/payload_cache_optimizer'
19
20
  require 'pub_sub_model_sync/subscriber'
20
21
 
21
22
  require 'pub_sub_model_sync/service_base'
@@ -27,7 +27,7 @@ Gem::Specification.new do |spec|
27
27
  # into git.
28
28
  spec.files = Dir.chdir(File.expand_path(__dir__)) do
29
29
  `git ls-files -z`.split("\x0")
30
- .reject { |f| f.match(%r{^(test|spec|features)/}) }
30
+ .reject { |f| f.match(%r{^(test|spec|features|samples)/}) }
31
31
  end
32
32
  spec.bindir = 'exe'
33
33
  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }