pub_sub_model_sync 1.0.beta1 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/release.yml +43 -0
  3. data/CHANGELOG.md +15 -4
  4. data/Gemfile.lock +11 -15
  5. data/README.md +184 -111
  6. data/docs/notifications-diagram.png +0 -0
  7. data/lib/pub_sub_model_sync/base.rb +0 -20
  8. data/lib/pub_sub_model_sync/config.rb +2 -3
  9. data/lib/pub_sub_model_sync/message_processor.rb +32 -9
  10. data/lib/pub_sub_model_sync/message_publisher.rb +18 -14
  11. data/lib/pub_sub_model_sync/payload.rb +15 -12
  12. data/lib/pub_sub_model_sync/{publisher.rb → payload_builder.rb} +16 -11
  13. data/lib/pub_sub_model_sync/publisher_concern.rb +29 -21
  14. data/lib/pub_sub_model_sync/railtie.rb +1 -1
  15. data/lib/pub_sub_model_sync/run_subscriber.rb +17 -13
  16. data/lib/pub_sub_model_sync/runner.rb +3 -5
  17. data/lib/pub_sub_model_sync/service_base.rb +5 -32
  18. data/lib/pub_sub_model_sync/service_google.rb +2 -2
  19. data/lib/pub_sub_model_sync/service_kafka.rb +2 -2
  20. data/lib/pub_sub_model_sync/service_rabbit.rb +1 -1
  21. data/lib/pub_sub_model_sync/subscriber_concern.rb +11 -9
  22. data/lib/pub_sub_model_sync/transaction.rb +12 -6
  23. data/lib/pub_sub_model_sync/version.rb +1 -1
  24. data/lib/pub_sub_model_sync.rb +1 -1
  25. data/samples/README.md +50 -0
  26. data/samples/app1/Dockerfile +13 -0
  27. data/samples/app1/Gemfile +37 -0
  28. data/samples/app1/Gemfile.lock +171 -0
  29. data/samples/app1/README.md +24 -0
  30. data/samples/app1/Rakefile +6 -0
  31. data/samples/app1/app/models/application_record.rb +3 -0
  32. data/samples/app1/app/models/concerns/.keep +0 -0
  33. data/samples/app1/app/models/post.rb +19 -0
  34. data/samples/app1/app/models/user.rb +29 -0
  35. data/samples/app1/bin/bundle +114 -0
  36. data/samples/app1/bin/rails +5 -0
  37. data/samples/app1/bin/rake +5 -0
  38. data/samples/app1/bin/setup +33 -0
  39. data/samples/app1/bin/spring +14 -0
  40. data/samples/app1/config/application.rb +40 -0
  41. data/samples/app1/config/boot.rb +4 -0
  42. data/samples/app1/config/credentials.yml.enc +1 -0
  43. data/samples/app1/config/database.yml +25 -0
  44. data/samples/app1/config/environment.rb +5 -0
  45. data/samples/app1/config/environments/development.rb +63 -0
  46. data/samples/app1/config/environments/production.rb +105 -0
  47. data/samples/app1/config/environments/test.rb +57 -0
  48. data/samples/app1/config/initializers/application_controller_renderer.rb +8 -0
  49. data/samples/app1/config/initializers/backtrace_silencers.rb +8 -0
  50. data/samples/app1/config/initializers/cors.rb +16 -0
  51. data/samples/app1/config/initializers/filter_parameter_logging.rb +6 -0
  52. data/samples/app1/config/initializers/inflections.rb +16 -0
  53. data/samples/app1/config/initializers/mime_types.rb +4 -0
  54. data/samples/app1/config/initializers/pubsub.rb +4 -0
  55. data/samples/app1/config/initializers/wrap_parameters.rb +14 -0
  56. data/samples/app1/config/locales/en.yml +33 -0
  57. data/samples/app1/config/master.key +1 -0
  58. data/samples/app1/config/puma.rb +43 -0
  59. data/samples/app1/config/routes.rb +3 -0
  60. data/samples/app1/config/spring.rb +6 -0
  61. data/samples/app1/config.ru +6 -0
  62. data/samples/app1/db/migrate/20210513080700_create_users.rb +12 -0
  63. data/samples/app1/db/migrate/20210513134332_create_posts.rb +11 -0
  64. data/samples/app1/db/schema.rb +34 -0
  65. data/samples/app1/db/seeds.rb +7 -0
  66. data/samples/app1/docker-compose.yml +32 -0
  67. data/samples/app1/log/.keep +0 -0
  68. data/samples/app2/Dockerfile +13 -0
  69. data/samples/app2/Gemfile +37 -0
  70. data/samples/app2/Gemfile.lock +171 -0
  71. data/samples/app2/README.md +24 -0
  72. data/samples/app2/Rakefile +6 -0
  73. data/samples/app2/app/models/application_record.rb +9 -0
  74. data/samples/app2/app/models/concerns/.keep +0 -0
  75. data/samples/app2/app/models/customer.rb +28 -0
  76. data/samples/app2/app/models/post.rb +10 -0
  77. data/samples/app2/bin/bundle +114 -0
  78. data/samples/app2/bin/rails +5 -0
  79. data/samples/app2/bin/rake +5 -0
  80. data/samples/app2/bin/setup +33 -0
  81. data/samples/app2/bin/spring +14 -0
  82. data/samples/app2/config/application.rb +40 -0
  83. data/samples/app2/config/boot.rb +4 -0
  84. data/samples/app2/config/credentials.yml.enc +1 -0
  85. data/samples/app2/config/database.yml +25 -0
  86. data/samples/app2/config/environment.rb +5 -0
  87. data/samples/app2/config/environments/development.rb +63 -0
  88. data/samples/app2/config/environments/production.rb +105 -0
  89. data/samples/app2/config/environments/test.rb +57 -0
  90. data/samples/app2/config/initializers/application_controller_renderer.rb +8 -0
  91. data/samples/app2/config/initializers/backtrace_silencers.rb +8 -0
  92. data/samples/app2/config/initializers/cors.rb +16 -0
  93. data/samples/app2/config/initializers/filter_parameter_logging.rb +6 -0
  94. data/samples/app2/config/initializers/inflections.rb +16 -0
  95. data/samples/app2/config/initializers/mime_types.rb +4 -0
  96. data/samples/app2/config/initializers/pubsub.rb +4 -0
  97. data/samples/app2/config/initializers/wrap_parameters.rb +14 -0
  98. data/samples/app2/config/locales/en.yml +33 -0
  99. data/samples/app2/config/master.key +1 -0
  100. data/samples/app2/config/puma.rb +43 -0
  101. data/samples/app2/config/routes.rb +3 -0
  102. data/samples/app2/config/spring.rb +6 -0
  103. data/samples/app2/config.ru +6 -0
  104. data/samples/app2/db/development.sqlite3 +0 -0
  105. data/samples/app2/db/migrate/20210513080956_create_customers.rb +10 -0
  106. data/samples/app2/db/migrate/20210513135203_create_posts.rb +10 -0
  107. data/samples/app2/db/schema.rb +31 -0
  108. data/samples/app2/db/seeds.rb +7 -0
  109. data/samples/app2/docker-compose.yml +20 -0
  110. data/samples/app2/log/.keep +0 -0
  111. metadata +92 -6
  112. data/lib/pub_sub_model_sync/initializers/before_commit.rb +0 -23
data/README.md CHANGED
@@ -1,6 +1,10 @@
1
1
  # **PubSubModelSync**
2
- This gem permits to sync automatically model data, send custom notifications between multiple Rails applications by publishing notifications via pubsub (Google PubSub, RabbitMQ, or Apache Kafka). Out of the scope this gem includes transactions to keep Data consistency by processing notifications in the order they were delivered.
3
- These notifications use JSON format to easily be decoded by subscribers (Rails applications and even other languages)
2
+ ![Rails badge](https://img.shields.io/badge/Rails-4+-success.png)
3
+ ![Ruby badge](https://img.shields.io/badge/Ruby-2.4+-success.png)
4
+ ![Production badge](https://img.shields.io/badge/Production-ready-success.png)
5
+
6
+ This gem permits to sync automatically models and custom data between multiple Rails applications by publishing notifications via pubsub (Google PubSub, RabbitMQ, or Apache Kafka) and automatically processed by all connected applications. Out of the scope, this gem includes transactions to keep Data consistency by processing notifications in the order they were delivered.
7
+ These notifications use JSON format to easily be decoded by subscribers (Rails applications and even other languages, soon for [Cristal-lang](https://crystal-lang.org/))
4
8
 
5
9
  - [**PubSubModelSync**](#pubsubmodelsync)
6
10
  - [**Features**](#features)
@@ -29,12 +33,12 @@ These notifications use JSON format to easily be decoded by subscribers (Rails a
29
33
  - [**Code of Conduct**](#code-of-conduct)
30
34
 
31
35
  ## **Features**
32
- - Sync model data between Rails apps: All changes made on App1, will be immediately reflected on App2, App3, etc.
36
+ - Sync model data between Rails apps: All changes made on App1, will be immediately reflected on App2, App3, etc.
33
37
  Example: If User is created on App1, this user will be created on App2, App3 too with the accepted attributes.
34
- - Ability to send class level communications
38
+ - Ability to send instance and class level notifications
35
39
  Example: If App1 wants to send emails to multiple users, this can be listened on App2, to deliver corresponding emails
36
40
  - Change pub/sub service at any time: Switch between rabbitmq, kafka, google pubsub
37
- - Support for transactions: Permits to keep data consistency between applications by processing notifications in the same order they were delivered.
41
+ - Support for transactions: Permits to keep data consistency between applications by processing notifications in the same order they were delivered (auto included in models transactions).
38
42
  - Ability to send notifications to a specific topic (single application) or multiple topics (multiple applications)
39
43
 
40
44
  ## **Installation**
@@ -56,7 +60,7 @@ And then execute: $ bundle install
56
60
  # initializers/pub_sub_config.rb
57
61
  PubSubModelSync::Config.service_name = :google
58
62
  PubSubModelSync::Config.project = 'google-project-id'
59
- PubSubModelSync::Config.credentials = 'path-to-the-config'
63
+ PubSubModelSync::Config.credentials = 'path-to-google-config.json'
60
64
  PubSubModelSync::Config.topic_name = 'sample-topic'
61
65
  PubSubModelSync::Config.subscription_name = 'my-app1'
62
66
  ```
@@ -85,12 +89,14 @@ And then execute: $ bundle install
85
89
 
86
90
  - Start subscribers to listen for publishers (Only in the app that has subscribers)
87
91
  ```bash
88
- DB_POOL=20 bundle exec rake pub_sub_model_sync:start
92
+ DB_POOL=20 bundle exec rake pub_sub_model_sync:start
89
93
  ```
90
94
  Note: You need more than 15 DB pools to avoid "could not obtain a connection from the pool within 5.000 seconds". https://devcenter.heroku.com/articles/concurrency-and-database-connections
91
95
 
92
96
  - Check the service status with:
93
- ```PubSubModelSync::MessagePublisher.publish_data('Test message', {sample_value: 10}, :create)```
97
+ ```ruby
98
+ PubSubModelSync::Payload.new({ my_data: 'here' }, { klass: 'MyClass', action: :sample_action }).publish!
99
+ ```
94
100
 
95
101
  - More configurations: [here](#extra-configurations)
96
102
 
@@ -98,14 +104,15 @@ And then execute: $ bundle install
98
104
  ![Diagram](/docs/notifications-diagram.png?raw=true)
99
105
 
100
106
  ## **Examples**
107
+ See sample apps in [/samples](/samples/)
101
108
  ### **Basic Example**
102
109
  ```ruby
103
110
  # App 1 (Publisher)
104
111
  class User < ActiveRecord::Base
105
112
  include PubSubModelSync::PublisherConcern
106
- ps_on_crud_event(:create) { ps_publish(:create, mapping: %i[id name email]) }
107
- ps_on_crud_event(:update) { ps_publish(:update, mapping: %i[id name email]) }
108
- ps_on_crud_event(:destroy) { ps_publish(:destroy, mapping: %i[id]) }
113
+ ps_after_action(:create) { ps_publish(:create, mapping: %i[id name email]) }
114
+ ps_after_action(:update) { ps_publish(:update, mapping: %i[id name email]) }
115
+ ps_after_action(:destroy) { ps_publish(:destroy, mapping: %i[id]) }
109
116
  end
110
117
 
111
118
  # App 2 (Subscriber)
@@ -115,9 +122,9 @@ class User < ActiveRecord::Base
115
122
  end
116
123
 
117
124
  # CRUD syncs
118
- my_user = User.create(name: 'test user', email: 'sample@gmail.com') # Publishes `:create` notification (App 2 syncs the new user)
119
- my_user.update(name: 'changed user') # Publishes `:update` notification (App2 updates changes)
120
- my_user.destroy # Publishes `:destroy` notification (App2 destroys the corresponding user)
125
+ my_user = User.create!(name: 'test user', email: 'sample@gmail.com') # Publishes `:create` notification (App 2 syncs the new user)
126
+ my_user.update!(name: 'changed user') # Publishes `:update` notification (App2 updates changes on user with the same id)
127
+ my_user.destroy! # Publishes `:destroy` notification (App2 destroys the corresponding user)
121
128
  ```
122
129
 
123
130
  ### **Advanced Example**
@@ -125,14 +132,16 @@ my_user.destroy # Publishes `:destroy` notification (App2 destroys the correspon
125
132
  # App 1 (Publisher)
126
133
  class User < ActiveRecord::Base
127
134
  include PubSubModelSync::PublisherConcern
128
- ps_on_crud_event([:create, :update]) { ps_publish(:save, mapping: %i[id name:full_name email], as_klass: 'App1User', headers: { topic_name: %i[topic1 topic2] }) }
135
+ ps_after_action([:create, :update]) do |action|
136
+ ps_publish(action, mapping: %i[name:full_name email], as_klass: 'App1User', headers: { topic_name: %i[topic1 topic2] })
137
+ end
129
138
  end
130
139
 
131
140
  # App 2 (Subscriber)
132
141
  class User < ActiveRecord::Base
133
142
  include PubSubModelSync::SubscriberConcern
134
- ps_subscribe(:save, %i[full_name:customer_name], id: [:id, :email], from_klass: 'App1User')
135
- ps_subscribe(:send_welcome, %i[email], to_action: :send_email, if: ->(model) { model.email.present? })
143
+ ps_subscribe([:create, :update], %i[full_name:customer_name], id: :email, from_klass: 'App1User')
144
+ ps_subscribe(:send_welcome, %i[email], id: :email, to_action: :send_email, if: ->(model) { model.email.present? })
136
145
  ps_class_subscribe(:batch_disable) # class subscription
137
146
 
138
147
  def send_email
@@ -143,9 +152,9 @@ class User < ActiveRecord::Base
143
152
  puts "disabling users: #{data[:ids]}"
144
153
  end
145
154
  end
146
- my_user = User.create(name: 'test user', email: 's@gmail.com') # Publishes `:save` notification as class name `App1User` (App2 syncs the new user)
155
+ my_user = User.create!(name: 'test user', email: 's@gmail.com') # Publishes `:create` notification with classname `App1User` (App2 syncs the new user)
147
156
  my_user.ps_publish(:send_welcome, mapping: %i[id email]) # Publishes `:send_welcome` notification (App2 prints "sending email to...")
148
- PubSubModelSync::MessagePublisher.publish_data(User, { ids: [my_user.id] }, :batch_disable) # Publishes class notification (App2 prints "disabling users..")
157
+ PubSubModelSync::Payload.new({ ids: [my_user.id] }, { klass: 'User', action: :batch_disable, mode: :klass }).publish! # Publishes class notification (App2 prints "disabling users..")
149
158
  ```
150
159
 
151
160
  ## **API**
@@ -167,8 +176,8 @@ PubSubModelSync::MessagePublisher.publish_data(User, { ids: [my_user.id] }, :bat
167
176
  - `settings` (Hash<:from_klass, :to_action, :id, :if, :unless>)
168
177
  - `from_klass:` (String, default current class): Only notifications with this class name will be processed by this subscription
169
178
  - `to_action:` (Symbol|Proc, default `action`):
170
- When Symbol: Model method to process the notification
171
- When Proc: Block to process the notification
179
+ When Symbol: Model method to process the notification, sample: `def my_method(data)...end`
180
+ When Proc: Block to process the notification, sample: `{|data| ... }`
172
181
  - `id:` (Symbol|Array<Symbol|String>, default: `:id`) identifier attribute(s) to find the corresponding model instance (Supports for mapping format)
173
182
  Sample: `id: :id` will search for a model like: `model_class.where(id: payload.data[:id])`
174
183
  Sample: `id: [:id, :email:user_email]` will search for a model like: `model_class.where(id: payload.data[:id], user_email: payload.data[:email])`
@@ -205,20 +214,20 @@ PubSubModelSync::MessagePublisher.publish_data(User, { ids: [my_user.id] }, :bat
205
214
  #### **Subscription helpers**
206
215
  - List all configured subscriptions
207
216
  ```ruby
208
- PubSubModelSync::Config.subscribers
217
+ PubSubModelSync::Config.subscribers
209
218
  ```
210
- - Manually process or reprocess a notification (useful when failed)
219
+ - Process or reprocess a notification
211
220
  ```ruby
212
- payload = PubSubModelSync::Payload.new(data, attributes, headers)
213
- payload.process!
221
+ payload = PubSubModelSync::Payload.new(data, attributes, headers)
222
+ payload.process!
214
223
  ```
215
224
 
216
225
 
217
226
  ### **Publishers**
218
227
  ```ruby
219
228
  class MyModel < ActiveRecord::Base
220
- ps_on_crud_event([:create, :update, :destroy], :method_publisher_name) # using method callback
221
- ps_on_crud_event([:create, :update, :destroy]) do |action| # using block callback
229
+ ps_after_action([:create, :update, :destroy], :method_publisher_name) # using method callback
230
+ ps_after_action([:create, :update, :destroy]) do |action| # using block callback
222
231
  ps_publish(action, data: {}, mapping: [], headers: {}, as_klass: nil)
223
232
  ps_class_publish({}, action: :my_action, as_klass: nil, headers: {})
224
233
  end
@@ -231,25 +240,13 @@ PubSubModelSync::MessagePublisher.publish_data(User, { ids: [my_user.id] }, :bat
231
240
 
232
241
  #### **Publishing notifications**
233
242
 
234
- - `ps_on_crud_event(crud_actions, method_name = nil, &block)` Listens for CRUD events and calls provided `block` or `method` to process event callback
243
+ - `ps_after_action(crud_actions, method_name = nil, &block)` Listens for CRUD events and calls provided `block` or `method` to process event callback
235
244
  - `crud_actions` (Symbol|Array<Symbol>) Crud event(s) to be observed (Allowed: `:create, :update, :destroy`)
236
- - `method_name` (Symbol, optional) method to be called to process action callback
237
- - `block` (Proc, optional) Block to be called to process action callback
238
- **Note1**: Due to rails callback ordering, this method uses `before_commit` callback when creating or updating models to ensure expected notifications order, sample:
239
- ```ruby
240
- user = User.create(name: 'asasas', posts_attributes: [{ title: 't1' }, { title: 't2' }])
241
- ```
242
- 1: User notification
243
- 2: First post notification
244
- 3: Second post notification
245
-
246
- **Note2**: Due to rails callback ordering, this method uses `after_destroy` callback when destroying models to ensure the expected notifications ordering.
247
- ```ruby
248
- user.destroy
249
- ```
250
- 1: Second post notification
251
- 2: First post notification
252
- 3: User notification
245
+ - `method_name` (Symbol, optional) method to be called to process action callback, sample: `def my_method(action) ... end`
246
+ - `block` (Proc, optional) Block to be called to process action callback, sample: `{ |action| ... }`
247
+
248
+ **Note1**: Due to rails callback ordering, this method uses `after_commit on: action {...}` callback when creating or updating models to ensure expected notifications order (More details [**here**](#transactions)).
249
+ **Note2**: Due to rails callback ordering, this method uses `after_destroy` callback when destroying models to ensure the expected notifications order.
253
250
 
254
251
  - `ps_publish(action, data: {}, mapping: [], headers: {}, as_klass: nil)` Delivers an instance notification via pubsub
255
252
  - `action` (Sym|String) Action name of the instance notification. Sample: create|update|destroy|<any_other_key>
@@ -266,36 +263,23 @@ PubSubModelSync::MessagePublisher.publish_data(User, { ids: [my_user.id] }, :bat
266
263
  - When Proc: Block to be called to retrieve header values (must return a `hash`, receives `:model, :action` as args)
267
264
  - `as_klass:` (String, default current class name): Output class name used instead of current class name
268
265
 
269
- - `ps_class_publish` Delivers a Class notification via pubsub
266
+ - `ps_class_publish(data, action:, as_klass: nil, headers: {})` Delivers a Class notification via pubsub
270
267
  - `data` (Hash): Data of the notification
271
268
  - `action` (Symbol): action name of the notification
272
269
  - `as_klass:` (String, default current class name): Class name of the notification
273
270
  - `headers:` (Hash, optional): header settings (More in Payload.headers)
271
+
272
+ - `ps_perform_publish(action = :create)` Permits to perform manually the callback of a specific `ps_after_action`
273
+ - `action` (Symbol, default: :create) Only :create|:update|:destroy
274
274
 
275
275
  #### **Publisher helpers**
276
- - Publish a class notification from anywhere
277
- ```ruby
278
- PubSubModelSync::MessagePublisher.publish_data(klass, data, action, headers: )
279
- ```
280
- - `klass`: (String) Class name to be used
281
- - Refer to `ps_class_publish` except `as_klass:`
282
-
283
- - Manually publish or republish a notification (useful when failed)
276
+ - Publish or republish a notification
284
277
  ```ruby
285
- payload = PubSubModelSync::Payload.new(data, attributes, headers)
286
- payload.publish!
278
+ payload = PubSubModelSync::Payload.new(data, attributes, headers)
279
+ payload.publish!
287
280
  ```
288
281
 
289
282
  #### **Publisher callbacks**
290
- - Prevent delivering a notification (called before building payload)
291
- If returns "true", will not publish notification
292
- ```ruby
293
- class MyModel < ActiveRecord::Base
294
- def ps_skip_publish?(action)
295
- # logic here
296
- end
297
- end
298
- ```
299
283
 
300
284
  - Do some actions before publishing notification.
301
285
  If returns ":cancel", notification will not be delivered
@@ -328,7 +312,8 @@ Any notification before delivering is transformed as a Payload for a better port
328
312
  - `mode`: (Symbol: `:model`|`:class`) Kind of notification
329
313
  * `headers`: (Hash) Notification settings that defines how the notification will be processed or delivered.
330
314
  - `key`: (String, optional) identifier of the payload, default: `<klass_name>/<action>` when class message, `<model.class.name>/<action>/<model.id>` when model message (Useful for caching techniques).
331
- - `ordering_key`: (String, optional): messages with the same value are processed in the same order they were delivered, default: `klass_name` when class message, `<model.class.name>/<model.id>` when instance message
315
+ - `ordering_key`: (String, optional): messages with the same value are processed in the same order they were delivered, default: `klass_name` when class message, `<model.class.name>/<model.id>` when instance message.
316
+ Note: Final `ordering_key` is calculated by this way: `payload.headers[:forced_ordering_key] || current_transaction&.key || payload.headers[:ordering_key]`
332
317
  - `topic_name`: (String|Array<String>, optional): Specific topic name (can be seen as a channel) to be used when delivering the message (default first topic from config).
333
318
  - `forced_ordering_key`: (String, optional): Will force to use this value as the `ordering_key`, even withing transactions. Default `nil`.
334
319
 
@@ -345,20 +330,38 @@ Any notification before delivering is transformed as a Payload for a better port
345
330
  * Crud syncs auto includes transactions which works as the following:
346
331
  ```ruby
347
332
  class User
348
- ps_on_crud_event(:create) { ps_publish(:create, mapping: %i[id name]) }
349
- has_many :posts
333
+ ps_after_action([:create, :update, :destroy]) { |action| ps_publish(action, mapping: %i[id name]) }
334
+ has_many :posts, dependent: :destroy
350
335
  accepts_nested_attributes_for :posts
351
336
  end
352
337
 
353
338
  class Post
354
339
  belongs_to :user
355
- ps_on_crud_event(:create) { ps_publish(:create, mapping: %i[id title]) }
340
+ ps_after_action([:create, :update, :destroy]) { |action| ps_publish(action, mapping: %i[id user_id title]) }
356
341
  end
357
-
358
- User.create!(name: 'test', posts_attributes: [{ title: 'Post 1' }, { title: 'Post 2' }])
359
342
  ```
360
- When user is created, `User`:`:save` notification is published with the ordering_key = `User/<user_id>`.
361
- Posts created together with the user model publishes `Post`:`:save` notification each one using its parents (user model) `ordering_key`.
343
+ - When created (all notifications use the same ordering key to be processed in the same order)
344
+ ```ruby
345
+ user = User.create!(name: 'test', posts_attributes: [{ title: 'Post 1' }, { title: 'Post 2' }])
346
+ # notification #1 => <Payload data: {id: 1, name: 'sample'}, info: { klass: 'User', action: :create, mode: :model }, headers: { ordering_key = `User/1` }>
347
+ # notification #2 => <Payload data: {id: 1, title: 'Post 1', user_id: 1}, info: { klass: 'Post', action: :create, mode: :model }, headers: { ordering_key = `User/1` }>
348
+ # notification #3 => <Payload data: {id: 2, title: 'Post 2', user_id: 1}, info: { klass: 'Post', action: :create, mode: :model }, headers: { ordering_key = `User/1` }>
349
+ ```
350
+ - When updated (all notifications use the same ordering key to be processed in the same order)
351
+ ```ruby
352
+ user.update!(name: 'changed', posts_attributes: [{ id: 1, title: 'Post 1C' }, { id: 2, title: 'Post 2C' }])
353
+ # notification #1 => <Payload data: {id: 1, name: 'changed'}, info: { klass: 'User', action: :update, mode: :model }, headers: { ordering_key = `User/1` }>
354
+ # notification #2 => <Payload data: {id: 1, title: 'Post 1C', user_id: 1}, info: { klass: 'Post', action: :update, mode: :model }, headers: { ordering_key = `User/1` }>
355
+ # notification #3 => <Payload data: {id: 2, title: 'Post 2C', user_id: 1}, info: { klass: 'Post', action: :update, mode: :model }, headers: { ordering_key = `User/1` }>
356
+ ```
357
+ - When destroyed (all notifications use the same ordering key to be processed in the same order)
358
+ **Note**: The notifications order were reordered in order to avoid inconsistency in other apps
359
+ ```ruby
360
+ user.destroy!
361
+ # notification #1 => <Payload data: {id: 1, title: 'Post 1C', user_id: 1}, info: { klass: 'Post', action: :destroy, mode: :model }>
362
+ # notification #2 => <Payload data: {id: 2, title: 'Post 2C', user_id: 1}, info: { klass: 'Post', action: :destroy, mode: :model }>
363
+ # notification #3 => <Payload data: {id: 1, name: 'changed'}, info: { klass: 'User', action: :destroy, mode: :model }>
364
+ ```
362
365
  By this way parent notification and all inner notifications are processed in the same order they were published (includes notifications from callbacks like `ps_before_publish`).
363
366
 
364
367
  **Note**: When any error is raised when saving user or posts, the transaction is cancelled and thus all notifications wont be delivered (customizable by `PubSubModelSync::Config.transactions_use_buffer`).
@@ -366,22 +369,20 @@ Any notification before delivering is transformed as a Payload for a better port
366
369
  - Manual transactions
367
370
  `PubSubModelSync::MessagePublisher::transaction(key, max_buffer: , &block)`
368
371
  - `key` (String|nil) Key used as the ordering key for all inner notifications (When nil, will use `ordering_key` of the first notification)
369
- - `max_buffer:` (Boolean, default: `PubSubModelSync::Config.transactions_max_buffer`)
370
- If true: will save all notifications and deliver all them when transaction has successfully finished. If transaction has failed, then all saved notifications will be discarded (not delivered).
371
- If false: will deliver all notifications immediately (no way to rollback notifications if transaction has failed)
372
+ - `max_buffer:` (Integer, default: `PubSubModelSync::Config.transactions_max_buffer`) Transaction buffer size (more details in #transactions_max_buffer).
372
373
  Sample:
373
374
  ```ruby
374
375
  PubSubModelSync::MessagePublisher::transaction('my-custom-key') do
375
376
  user = User.create(name: 'test') # `User`:`:create` notification
376
377
  post = Post.create(title: 'sample') # `Post`:`:create` notification
377
- PubSubModelSync::MessagePublisher.publish_data(User, { ids: [user.id] }, :send_welcome) # `User`:`:send_welcome` notification
378
+ PubSubModelSync::Payload.new({ ids: [user.id] }, { klass: 'User', action: :send_welcome, mode: :klass }).publish! # `User`:`:send_welcome` notification
378
379
  end
379
380
  ```
380
381
  All notifications uses `ordering_key: 'my-custom-key'` and will be processed in the same order they were published.
381
382
 
382
383
  ## **Testing with RSpec**
383
384
  - Config: (spec/rails_helper.rb)
384
- ```ruby
385
+ ```ruby
385
386
 
386
387
  # when using google service
387
388
  require 'pub_sub_model_sync/mock_google_service'
@@ -404,51 +405,119 @@ Any notification before delivering is transformed as a Payload for a better port
404
405
  allow(Kafka).to receive(:new).and_return(kafka_mock)
405
406
  end
406
407
 
407
- #
408
+ # disable all models sync by default (reduces testing time)
408
409
  config.before(:each) do
409
- # **** disable payloads generation, sync callbacks to improve tests speed
410
410
  allow(PubSubModelSync::MessagePublisher).to receive(:publish_data) # disable class level notif
411
411
  allow(PubSubModelSync::MessagePublisher).to receive(:publish_model) # disable instance level notif
412
-
413
- # **** when testing model syncs, it can be re enabled by:
414
- # before do
415
- # allow(PubSubModelSync::MessagePublisher).to receive(:publish_data).and_call_original
416
- # allow(PubSubModelSync::MessagePublisher).to receive(:publish_model).and_call_original
417
- # end
418
412
  end
419
- ```
413
+
414
+ # enable all models sync only for tests that includes 'sync: true'
415
+ config.before(:each, sync: true) do
416
+ allow(PubSubModelSync::MessagePublisher).to receive(:publish_data).and_call_original
417
+ allow(PubSubModelSync::MessagePublisher).to receive(:publish_model).and_call_original
418
+ end
419
+
420
+ # Only when using database cleaner in old versions of rspec (enables after_commit callback)
421
+ # config.before(:each, truncate: true) do
422
+ # DatabaseCleaner.strategy = :truncation
423
+ # end
424
+ ```
420
425
  - Examples:
426
+ - **Publisher**
421
427
  ```ruby
422
- # Subscriber
423
- it 'receive model notification' do
424
- data = { name: 'name', id: 999 }
428
+ # Do not forget to include 'sync: true' to enable publishing pubsub notifications
429
+ describe 'When publishing sync', truncate: true, sync: true do
430
+ it 'publishes user notification when created' do
431
+ expect_publish_notification(:create, klass: 'User')
432
+ create(:user)
433
+ end
434
+
435
+ it 'publishes user notification with all defined data' do
436
+ user = build(:user)
437
+ data = PubSubModelSync::PayloadBuilder.parse_mapping_for(user, %i[id name:full_name email])
438
+ data[:id] = be_a(Integer)
439
+ expect_publish_notification(:create, klass: 'User', data: data)
440
+ user.save!
441
+ end
442
+
443
+ it 'publishes user notification when created' do
444
+ email = 'Newemail@gmail.com'
445
+ user = create(:user)
446
+ expect_publish_notification(:update, klass: 'User', data: { id: user.id, email: email })
447
+ user.update!(email: email)
448
+ end
449
+
450
+ it 'publishes user notification when created' do
451
+ user = create(:user)
452
+ expect_publish_notification(:destroy, klass: 'User', data: { id: user.id })
453
+ user.destroy!
454
+ end
455
+
456
+ private
457
+
458
+ # @param action (Symbol)
459
+ # @param klass (String, default described_class name)
460
+ # @param data (Hash, optional) notification data
461
+ # @param info (Hash, optional) notification info
462
+ # @param headers (Hash, optional) notification headers
463
+ def expect_publish_notification(action, klass: described_class.to_s, data: {}, info: {}, headers: {})
464
+ publisher = PubSubModelSync::MessagePublisher
465
+ exp_data = have_attributes(data: hash_including(data),
466
+ info: hash_including(info.merge(klass: klass, action: action)),
467
+ headers: hash_including(headers))
468
+ allow(publisher).to receive(:publish!).and_call_original
469
+ expect(publisher).to receive(:publish!).with(exp_data)
470
+ end
471
+ end
472
+ ```
473
+ - **Subscriber**
474
+ ```ruby
475
+
476
+ describe 'when syncing data from other apps' do
477
+ it 'creates user when received :create notification' do
478
+ user = build(:user)
479
+ data = user.as_json(only: %i[name email]).merge(id: 999)
425
480
  payload = PubSubModelSync::Payload.new(data, { klass: 'User', action: :create })
481
+ expect { payload.process! }.to change(described_class, :count)
482
+ end
483
+
484
+ it 'updates user when received :update notification' do
485
+ user = create(:user)
486
+ name = 'new name'
487
+ data = user.as_json(only: %i[id email]).merge(name: name)
488
+ payload = PubSubModelSync::Payload.new(data, { klass: 'User', action: :update })
489
+ payload.process!
490
+ expect(user.reload.name).to eq(name)
491
+ end
492
+
493
+ it 'destroys user when received :destroy notification' do
494
+ user = create(:user)
495
+ data = user.as_json(only: %i[id])
496
+ payload = PubSubModelSync::Payload.new(data, { klass: 'User', action: :destroy })
497
+ payload.process!
498
+ expect { user.reload }.to raise_error(ActiveRecord::RecordNotFound)
499
+ end
500
+
501
+
502
+ it 'receive custom model notification' do
503
+ user = create(:user)
504
+ data = { id: user.id, custom_data: {} }
505
+ custom_action = :say_hello
506
+ expect_any_instance_of(User).to receive(custom_action).with(data)
507
+ payload = PubSubModelSync::Payload.new(data, { klass: 'User', action: custom_action })
426
508
  payload.process!
427
- expect(User.where(id: data[:id])).to be_any
428
509
  end
429
510
 
430
511
  it 'receive class notification' do
431
512
  data = { msg: 'hello' }
432
513
  action = :greeting
514
+ expect(User).to receive(action).with(data)
515
+ # Do not forget to include `mode: :klass` for class notifications
433
516
  payload = PubSubModelSync::Payload.new(data, { klass: 'User', action: action, mode: :klass })
434
517
  payload.process!
435
- expect(User).to receive(action)
436
518
  end
437
-
438
- # Publisher
439
- it 'publish model notification' do
440
- publisher = PubSubModelSync::MessagePublisher
441
- user = User.create(name: 'name', email: 'email')
442
- expect(publisher).to receive(:publish_model).with(user, :create, anything)
443
- end
444
-
445
- it 'publish class notification' do
446
- publisher = PubSubModelSync::MessagePublisher
447
- user = User.create(name: 'name', email: 'email')
448
- user.ps_class_publish({msg: 'hello'}, action: :greeting)
449
- expect(publisher).to receive(:publish_data).with('User', data, :greeting)
450
- end
451
- ```
519
+ end
520
+ ```
452
521
 
453
522
  ## **Extra configurations**
454
523
  ```ruby
@@ -479,9 +548,10 @@ config.debug = true
479
548
  (Proc) => called after publishing a message
480
549
  - ```.on_error_publish = ->(exception, {payload:}) { payload.delay(...).publish! }```
481
550
  (Proc) => called when failed publishing a message (delayed_job or similar can be used for retrying)
482
- - ```.transactions_max_buffer = 100``` (Integer) Once this quantity of notifications is reached, then all notifications will immediately be delivered.
483
- Note: There is no way to rollback delivered notifications if current transaction fails
484
- - ```.enable_rails4_before_commit = true``` (true*|false) When false will disable rails 4 hack compatibility and then CRUD notifications will be prepared using `after_commit` callback instead of `before_commit` which will not rollback sql transactions if fails.
551
+ - ```.transactions_max_buffer = 1``` (Integer, default 1) Controls the maximum quantity of notifications to be enqueued to the transaction-buffer before delivering them and thus adds the ability to rollback notifications if the transaction fails.
552
+ Once this quantity of notifications is reached, then all notifications of the current transaction will immediately be delivered (can be customized per transaction).
553
+ Note: There is no way to rollback delivered notifications if current transaction fails later.
554
+ Note2: Only notifications from the buffer can be rollbacked if the current transaction has failed.
485
555
 
486
556
  ## **TODO**
487
557
  - Auto publish update only if payload has changed (see ways to compare previous payload vs new payload)
@@ -490,8 +560,11 @@ config.debug = true
490
560
  - Add DB table to use as a shield to prevent publishing similar notifications and publish partial notifications (similar idea when processing notif)
491
561
  - Last notification is not being delivered immediately in google pubsub (maybe force with timeout 10secs and service.deliver_messages)
492
562
  - Update folder structure
493
- - Support for blocks in ps_publish and ps_subscribe
494
563
  - Services support to deliver multiple payloads from transactions
564
+ - Fix deprecation warnings: pub_sub_model_sync/service_google.rb:39: warning: Splitting the last argument into positional and keyword parameters is deprecated
565
+ - Add if/unless to ps_after_action
566
+ - Add subscription liveness checker using thread without db connection to check periodically pending messages from google pubsub
567
+ - Unify .stop() and 'Listener stopped'
495
568
 
496
569
  ## **Q&A**
497
570
  - I'm getting error "could not obtain a connection from the pool within 5.000 seconds"... what does this mean?
Binary file
@@ -17,25 +17,5 @@ module PubSubModelSync
17
17
  config.debug
18
18
  end
19
19
  end
20
-
21
- # @param errors (Array(Class|String))
22
- def retry_error(errors, qty: 2, &block)
23
- retries ||= 0
24
- block.call
25
- rescue => e
26
- retries += 1
27
- res = errors.find { |e_type| match_error?(e, e_type) }
28
- raise if !res || retries > qty
29
-
30
- sleep(qty * 0.1) && retry
31
- end
32
-
33
- private
34
-
35
- # @param error (Exception)
36
- # @param error_type (Class|String)
37
- def match_error?(error, error_type)
38
- error_type.is_a?(String) ? error.message.include?(error_type) : error.is_a?(error_type)
39
- end
40
20
  end
41
21
  end
@@ -8,8 +8,7 @@ module PubSubModelSync
8
8
  # customizable callbacks
9
9
  cattr_accessor(:debug) { false }
10
10
  cattr_accessor :logger # LoggerInst
11
- cattr_accessor(:transactions_max_buffer) { 100 }
12
- cattr_accessor(:enable_rails4_before_commit) { Rails::VERSION::MAJOR == 4 }
11
+ cattr_accessor(:transactions_max_buffer) { 1 }
13
12
 
14
13
  cattr_accessor(:on_before_processing) { ->(_payload, _info) {} } # return :cancel to skip
15
14
  cattr_accessor(:on_success_processing) { ->(_payload, _info) {} }
@@ -30,7 +29,7 @@ module PubSubModelSync
30
29
  def self.log(msg, kind = :info)
31
30
  msg = "PS_MSYNC ==> #{msg}"
32
31
  if logger == :raise_error
33
- kind == :error ? raise(msg) : puts(msg)
32
+ kind == :error ? raise(StandardError, msg) : puts(msg)
34
33
  else
35
34
  logger ? logger.send(kind, msg) : puts(msg)
36
35
  end
@@ -16,13 +16,17 @@ module PubSubModelSync
16
16
  end
17
17
 
18
18
  def process!
19
- filter_subscribers.each(&method(:run_subscriber))
19
+ subscribers = filter_subscribers
20
+ payload_info = { klass: payload.klass, action: payload.action, mode: payload.mode }
21
+ log("No subscribers found for #{payload_info}", :warn) if config.debug && subscribers.empty?
22
+ subscribers.each(&method(:run_subscriber))
20
23
  end
21
24
 
22
25
  def process
26
+ retries ||= 0
23
27
  process!
24
28
  rescue => e
25
- notify_error(e)
29
+ retry_process?(e, retries += 1) ? retry : notify_error(e)
26
30
  end
27
31
 
28
32
  private
@@ -31,12 +35,10 @@ module PubSubModelSync
31
35
  processor = PubSubModelSync::RunSubscriber.new(subscriber, payload)
32
36
  return unless processable?(subscriber)
33
37
 
34
- errors = [ActiveRecord::ConnectionTimeoutError, 'deadlock detected', 'could not serialize access']
35
- retry_error(errors, qty: 5) do
36
- processor.call
37
- res = config.on_success_processing.call(payload, { subscriber: subscriber })
38
- log "processed message with: #{payload.inspect}" if res != :skip_log
39
- end
38
+ log("Processing message #{[subscriber, payload]}...") if config.debug
39
+ processor.call
40
+ res = config.on_success_processing.call(payload, { subscriber: subscriber })
41
+ log "processed message with: #{payload.inspect}" if res != :skip_log
40
42
  end
41
43
 
42
44
  def processable?(subscriber)
@@ -45,13 +47,34 @@ module PubSubModelSync
45
47
  !cancel
46
48
  end
47
49
 
48
- # @param error (Error)
50
+ # @param error (StandardError)
49
51
  def notify_error(error)
50
52
  info = [payload, error.message, error.backtrace]
51
53
  res = config.on_error_processing.call(error, { payload: payload })
52
54
  log("Error processing message: #{info}", :error) if res != :skip_log
53
55
  end
54
56
 
57
+ def lost_db_connection?(error)
58
+ connection_lost_classes = %w[ActiveRecord::ConnectionTimeoutError PG::UnableToSend]
59
+ connection_lost_classes.include?(error.class.name) || error.message.match?(/lost connection/i)
60
+ end
61
+
62
+ def retry_process?(error, retries) # rubocop:disable Metrics/MethodLength
63
+ error_payload = [payload, error.message, error.backtrace]
64
+ return false unless lost_db_connection?(error)
65
+
66
+ if retries <= 5
67
+ sleep(retries)
68
+ log("Error processing message: (retrying #{retries}/5): #{error_payload}", :error)
69
+ ActiveRecord::Base.connection.reconnect! rescue nil # rubocop:disable Style/RescueModifier
70
+ true
71
+ else
72
+ log("Retried 5 times and error persists, exiting...: #{error_payload}", :error)
73
+ Process.exit!(true)
74
+ end
75
+ end
76
+
77
+ # @return (Array<PubSubModelSync::Subscriber>)
55
78
  def filter_subscribers
56
79
  config.subscribers.select do |subscriber|
57
80
  subscriber.from_klass == payload.klass && subscriber.action == payload.action && payload.mode == subscriber.mode