pub_sub_model_sync 1.0.beta1 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/release.yml +43 -0
- data/CHANGELOG.md +15 -4
- data/Gemfile.lock +11 -15
- data/README.md +184 -111
- data/docs/notifications-diagram.png +0 -0
- data/lib/pub_sub_model_sync/base.rb +0 -20
- data/lib/pub_sub_model_sync/config.rb +2 -3
- data/lib/pub_sub_model_sync/message_processor.rb +32 -9
- data/lib/pub_sub_model_sync/message_publisher.rb +18 -14
- data/lib/pub_sub_model_sync/payload.rb +15 -12
- data/lib/pub_sub_model_sync/{publisher.rb → payload_builder.rb} +16 -11
- data/lib/pub_sub_model_sync/publisher_concern.rb +29 -21
- data/lib/pub_sub_model_sync/railtie.rb +1 -1
- data/lib/pub_sub_model_sync/run_subscriber.rb +17 -13
- data/lib/pub_sub_model_sync/runner.rb +3 -5
- data/lib/pub_sub_model_sync/service_base.rb +5 -32
- data/lib/pub_sub_model_sync/service_google.rb +2 -2
- data/lib/pub_sub_model_sync/service_kafka.rb +2 -2
- data/lib/pub_sub_model_sync/service_rabbit.rb +1 -1
- data/lib/pub_sub_model_sync/subscriber_concern.rb +11 -9
- data/lib/pub_sub_model_sync/transaction.rb +12 -6
- data/lib/pub_sub_model_sync/version.rb +1 -1
- data/lib/pub_sub_model_sync.rb +1 -1
- data/samples/README.md +50 -0
- data/samples/app1/Dockerfile +13 -0
- data/samples/app1/Gemfile +37 -0
- data/samples/app1/Gemfile.lock +171 -0
- data/samples/app1/README.md +24 -0
- data/samples/app1/Rakefile +6 -0
- data/samples/app1/app/models/application_record.rb +3 -0
- data/samples/app1/app/models/concerns/.keep +0 -0
- data/samples/app1/app/models/post.rb +19 -0
- data/samples/app1/app/models/user.rb +29 -0
- data/samples/app1/bin/bundle +114 -0
- data/samples/app1/bin/rails +5 -0
- data/samples/app1/bin/rake +5 -0
- data/samples/app1/bin/setup +33 -0
- data/samples/app1/bin/spring +14 -0
- data/samples/app1/config/application.rb +40 -0
- data/samples/app1/config/boot.rb +4 -0
- data/samples/app1/config/credentials.yml.enc +1 -0
- data/samples/app1/config/database.yml +25 -0
- data/samples/app1/config/environment.rb +5 -0
- data/samples/app1/config/environments/development.rb +63 -0
- data/samples/app1/config/environments/production.rb +105 -0
- data/samples/app1/config/environments/test.rb +57 -0
- data/samples/app1/config/initializers/application_controller_renderer.rb +8 -0
- data/samples/app1/config/initializers/backtrace_silencers.rb +8 -0
- data/samples/app1/config/initializers/cors.rb +16 -0
- data/samples/app1/config/initializers/filter_parameter_logging.rb +6 -0
- data/samples/app1/config/initializers/inflections.rb +16 -0
- data/samples/app1/config/initializers/mime_types.rb +4 -0
- data/samples/app1/config/initializers/pubsub.rb +4 -0
- data/samples/app1/config/initializers/wrap_parameters.rb +14 -0
- data/samples/app1/config/locales/en.yml +33 -0
- data/samples/app1/config/master.key +1 -0
- data/samples/app1/config/puma.rb +43 -0
- data/samples/app1/config/routes.rb +3 -0
- data/samples/app1/config/spring.rb +6 -0
- data/samples/app1/config.ru +6 -0
- data/samples/app1/db/migrate/20210513080700_create_users.rb +12 -0
- data/samples/app1/db/migrate/20210513134332_create_posts.rb +11 -0
- data/samples/app1/db/schema.rb +34 -0
- data/samples/app1/db/seeds.rb +7 -0
- data/samples/app1/docker-compose.yml +32 -0
- data/samples/app1/log/.keep +0 -0
- data/samples/app2/Dockerfile +13 -0
- data/samples/app2/Gemfile +37 -0
- data/samples/app2/Gemfile.lock +171 -0
- data/samples/app2/README.md +24 -0
- data/samples/app2/Rakefile +6 -0
- data/samples/app2/app/models/application_record.rb +9 -0
- data/samples/app2/app/models/concerns/.keep +0 -0
- data/samples/app2/app/models/customer.rb +28 -0
- data/samples/app2/app/models/post.rb +10 -0
- data/samples/app2/bin/bundle +114 -0
- data/samples/app2/bin/rails +5 -0
- data/samples/app2/bin/rake +5 -0
- data/samples/app2/bin/setup +33 -0
- data/samples/app2/bin/spring +14 -0
- data/samples/app2/config/application.rb +40 -0
- data/samples/app2/config/boot.rb +4 -0
- data/samples/app2/config/credentials.yml.enc +1 -0
- data/samples/app2/config/database.yml +25 -0
- data/samples/app2/config/environment.rb +5 -0
- data/samples/app2/config/environments/development.rb +63 -0
- data/samples/app2/config/environments/production.rb +105 -0
- data/samples/app2/config/environments/test.rb +57 -0
- data/samples/app2/config/initializers/application_controller_renderer.rb +8 -0
- data/samples/app2/config/initializers/backtrace_silencers.rb +8 -0
- data/samples/app2/config/initializers/cors.rb +16 -0
- data/samples/app2/config/initializers/filter_parameter_logging.rb +6 -0
- data/samples/app2/config/initializers/inflections.rb +16 -0
- data/samples/app2/config/initializers/mime_types.rb +4 -0
- data/samples/app2/config/initializers/pubsub.rb +4 -0
- data/samples/app2/config/initializers/wrap_parameters.rb +14 -0
- data/samples/app2/config/locales/en.yml +33 -0
- data/samples/app2/config/master.key +1 -0
- data/samples/app2/config/puma.rb +43 -0
- data/samples/app2/config/routes.rb +3 -0
- data/samples/app2/config/spring.rb +6 -0
- data/samples/app2/config.ru +6 -0
- data/samples/app2/db/development.sqlite3 +0 -0
- data/samples/app2/db/migrate/20210513080956_create_customers.rb +10 -0
- data/samples/app2/db/migrate/20210513135203_create_posts.rb +10 -0
- data/samples/app2/db/schema.rb +31 -0
- data/samples/app2/db/seeds.rb +7 -0
- data/samples/app2/docker-compose.yml +20 -0
- data/samples/app2/log/.keep +0 -0
- metadata +92 -6
- data/lib/pub_sub_model_sync/initializers/before_commit.rb +0 -23
data/README.md
CHANGED
@@ -1,6 +1,10 @@
|
|
1
1
|
# **PubSubModelSync**
|
2
|
-
|
3
|
-
|
2
|
+

|
3
|
+

|
4
|
+

|
5
|
+
|
6
|
+
This gem permits to sync automatically models and custom data between multiple Rails applications by publishing notifications via pubsub (Google PubSub, RabbitMQ, or Apache Kafka) and automatically processed by all connected applications. Out of the scope, this gem includes transactions to keep Data consistency by processing notifications in the order they were delivered.
|
7
|
+
These notifications use JSON format to easily be decoded by subscribers (Rails applications and even other languages, soon for [Cristal-lang](https://crystal-lang.org/))
|
4
8
|
|
5
9
|
- [**PubSubModelSync**](#pubsubmodelsync)
|
6
10
|
- [**Features**](#features)
|
@@ -29,12 +33,12 @@ These notifications use JSON format to easily be decoded by subscribers (Rails a
|
|
29
33
|
- [**Code of Conduct**](#code-of-conduct)
|
30
34
|
|
31
35
|
## **Features**
|
32
|
-
- Sync model data between Rails apps: All changes made on App1, will be immediately reflected on App2, App3, etc.
|
36
|
+
- Sync model data between Rails apps: All changes made on App1, will be immediately reflected on App2, App3, etc.
|
33
37
|
Example: If User is created on App1, this user will be created on App2, App3 too with the accepted attributes.
|
34
|
-
- Ability to send class level
|
38
|
+
- Ability to send instance and class level notifications
|
35
39
|
Example: If App1 wants to send emails to multiple users, this can be listened on App2, to deliver corresponding emails
|
36
40
|
- Change pub/sub service at any time: Switch between rabbitmq, kafka, google pubsub
|
37
|
-
- Support for transactions: Permits to keep data consistency between applications by processing notifications in the same order they were delivered.
|
41
|
+
- Support for transactions: Permits to keep data consistency between applications by processing notifications in the same order they were delivered (auto included in models transactions).
|
38
42
|
- Ability to send notifications to a specific topic (single application) or multiple topics (multiple applications)
|
39
43
|
|
40
44
|
## **Installation**
|
@@ -56,7 +60,7 @@ And then execute: $ bundle install
|
|
56
60
|
# initializers/pub_sub_config.rb
|
57
61
|
PubSubModelSync::Config.service_name = :google
|
58
62
|
PubSubModelSync::Config.project = 'google-project-id'
|
59
|
-
PubSubModelSync::Config.credentials = 'path-to-
|
63
|
+
PubSubModelSync::Config.credentials = 'path-to-google-config.json'
|
60
64
|
PubSubModelSync::Config.topic_name = 'sample-topic'
|
61
65
|
PubSubModelSync::Config.subscription_name = 'my-app1'
|
62
66
|
```
|
@@ -85,12 +89,14 @@ And then execute: $ bundle install
|
|
85
89
|
|
86
90
|
- Start subscribers to listen for publishers (Only in the app that has subscribers)
|
87
91
|
```bash
|
88
|
-
|
92
|
+
DB_POOL=20 bundle exec rake pub_sub_model_sync:start
|
89
93
|
```
|
90
94
|
Note: You need more than 15 DB pools to avoid "could not obtain a connection from the pool within 5.000 seconds". https://devcenter.heroku.com/articles/concurrency-and-database-connections
|
91
95
|
|
92
96
|
- Check the service status with:
|
93
|
-
```
|
97
|
+
```ruby
|
98
|
+
PubSubModelSync::Payload.new({ my_data: 'here' }, { klass: 'MyClass', action: :sample_action }).publish!
|
99
|
+
```
|
94
100
|
|
95
101
|
- More configurations: [here](#extra-configurations)
|
96
102
|
|
@@ -98,14 +104,15 @@ And then execute: $ bundle install
|
|
98
104
|

|
99
105
|
|
100
106
|
## **Examples**
|
107
|
+
See sample apps in [/samples](/samples/)
|
101
108
|
### **Basic Example**
|
102
109
|
```ruby
|
103
110
|
# App 1 (Publisher)
|
104
111
|
class User < ActiveRecord::Base
|
105
112
|
include PubSubModelSync::PublisherConcern
|
106
|
-
|
107
|
-
|
108
|
-
|
113
|
+
ps_after_action(:create) { ps_publish(:create, mapping: %i[id name email]) }
|
114
|
+
ps_after_action(:update) { ps_publish(:update, mapping: %i[id name email]) }
|
115
|
+
ps_after_action(:destroy) { ps_publish(:destroy, mapping: %i[id]) }
|
109
116
|
end
|
110
117
|
|
111
118
|
# App 2 (Subscriber)
|
@@ -115,9 +122,9 @@ class User < ActiveRecord::Base
|
|
115
122
|
end
|
116
123
|
|
117
124
|
# CRUD syncs
|
118
|
-
my_user = User.create(name: 'test user', email: 'sample@gmail.com') # Publishes `:create` notification (App 2 syncs the new user)
|
119
|
-
my_user.update(name: 'changed user') # Publishes `:update` notification (App2 updates changes)
|
120
|
-
my_user.destroy # Publishes `:destroy` notification (App2 destroys the corresponding user)
|
125
|
+
my_user = User.create!(name: 'test user', email: 'sample@gmail.com') # Publishes `:create` notification (App 2 syncs the new user)
|
126
|
+
my_user.update!(name: 'changed user') # Publishes `:update` notification (App2 updates changes on user with the same id)
|
127
|
+
my_user.destroy! # Publishes `:destroy` notification (App2 destroys the corresponding user)
|
121
128
|
```
|
122
129
|
|
123
130
|
### **Advanced Example**
|
@@ -125,14 +132,16 @@ my_user.destroy # Publishes `:destroy` notification (App2 destroys the correspon
|
|
125
132
|
# App 1 (Publisher)
|
126
133
|
class User < ActiveRecord::Base
|
127
134
|
include PubSubModelSync::PublisherConcern
|
128
|
-
|
135
|
+
ps_after_action([:create, :update]) do |action|
|
136
|
+
ps_publish(action, mapping: %i[name:full_name email], as_klass: 'App1User', headers: { topic_name: %i[topic1 topic2] })
|
137
|
+
end
|
129
138
|
end
|
130
139
|
|
131
140
|
# App 2 (Subscriber)
|
132
141
|
class User < ActiveRecord::Base
|
133
142
|
include PubSubModelSync::SubscriberConcern
|
134
|
-
ps_subscribe(:
|
135
|
-
ps_subscribe(:send_welcome, %i[email], to_action: :send_email, if: ->(model) { model.email.present? })
|
143
|
+
ps_subscribe([:create, :update], %i[full_name:customer_name], id: :email, from_klass: 'App1User')
|
144
|
+
ps_subscribe(:send_welcome, %i[email], id: :email, to_action: :send_email, if: ->(model) { model.email.present? })
|
136
145
|
ps_class_subscribe(:batch_disable) # class subscription
|
137
146
|
|
138
147
|
def send_email
|
@@ -143,9 +152,9 @@ class User < ActiveRecord::Base
|
|
143
152
|
puts "disabling users: #{data[:ids]}"
|
144
153
|
end
|
145
154
|
end
|
146
|
-
my_user = User.create(name: 'test user', email: 's@gmail.com') # Publishes `:
|
155
|
+
my_user = User.create!(name: 'test user', email: 's@gmail.com') # Publishes `:create` notification with classname `App1User` (App2 syncs the new user)
|
147
156
|
my_user.ps_publish(:send_welcome, mapping: %i[id email]) # Publishes `:send_welcome` notification (App2 prints "sending email to...")
|
148
|
-
PubSubModelSync::
|
157
|
+
PubSubModelSync::Payload.new({ ids: [my_user.id] }, { klass: 'User', action: :batch_disable, mode: :klass }).publish! # Publishes class notification (App2 prints "disabling users..")
|
149
158
|
```
|
150
159
|
|
151
160
|
## **API**
|
@@ -167,8 +176,8 @@ PubSubModelSync::MessagePublisher.publish_data(User, { ids: [my_user.id] }, :bat
|
|
167
176
|
- `settings` (Hash<:from_klass, :to_action, :id, :if, :unless>)
|
168
177
|
- `from_klass:` (String, default current class): Only notifications with this class name will be processed by this subscription
|
169
178
|
- `to_action:` (Symbol|Proc, default `action`):
|
170
|
-
When Symbol: Model method to process the notification
|
171
|
-
When Proc: Block to process the notification
|
179
|
+
When Symbol: Model method to process the notification, sample: `def my_method(data)...end`
|
180
|
+
When Proc: Block to process the notification, sample: `{|data| ... }`
|
172
181
|
- `id:` (Symbol|Array<Symbol|String>, default: `:id`) identifier attribute(s) to find the corresponding model instance (Supports for mapping format)
|
173
182
|
Sample: `id: :id` will search for a model like: `model_class.where(id: payload.data[:id])`
|
174
183
|
Sample: `id: [:id, :email:user_email]` will search for a model like: `model_class.where(id: payload.data[:id], user_email: payload.data[:email])`
|
@@ -205,20 +214,20 @@ PubSubModelSync::MessagePublisher.publish_data(User, { ids: [my_user.id] }, :bat
|
|
205
214
|
#### **Subscription helpers**
|
206
215
|
- List all configured subscriptions
|
207
216
|
```ruby
|
208
|
-
|
217
|
+
PubSubModelSync::Config.subscribers
|
209
218
|
```
|
210
|
-
-
|
219
|
+
- Process or reprocess a notification
|
211
220
|
```ruby
|
212
|
-
|
213
|
-
|
221
|
+
payload = PubSubModelSync::Payload.new(data, attributes, headers)
|
222
|
+
payload.process!
|
214
223
|
```
|
215
224
|
|
216
225
|
|
217
226
|
### **Publishers**
|
218
227
|
```ruby
|
219
228
|
class MyModel < ActiveRecord::Base
|
220
|
-
|
221
|
-
|
229
|
+
ps_after_action([:create, :update, :destroy], :method_publisher_name) # using method callback
|
230
|
+
ps_after_action([:create, :update, :destroy]) do |action| # using block callback
|
222
231
|
ps_publish(action, data: {}, mapping: [], headers: {}, as_klass: nil)
|
223
232
|
ps_class_publish({}, action: :my_action, as_klass: nil, headers: {})
|
224
233
|
end
|
@@ -231,25 +240,13 @@ PubSubModelSync::MessagePublisher.publish_data(User, { ids: [my_user.id] }, :bat
|
|
231
240
|
|
232
241
|
#### **Publishing notifications**
|
233
242
|
|
234
|
-
- `
|
243
|
+
- `ps_after_action(crud_actions, method_name = nil, &block)` Listens for CRUD events and calls provided `block` or `method` to process event callback
|
235
244
|
- `crud_actions` (Symbol|Array<Symbol>) Crud event(s) to be observed (Allowed: `:create, :update, :destroy`)
|
236
|
-
- `method_name` (Symbol, optional) method to be called to process action callback
|
237
|
-
- `block` (Proc, optional) Block to be called to process action callback
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
```
|
242
|
-
1: User notification
|
243
|
-
2: First post notification
|
244
|
-
3: Second post notification
|
245
|
-
|
246
|
-
**Note2**: Due to rails callback ordering, this method uses `after_destroy` callback when destroying models to ensure the expected notifications ordering.
|
247
|
-
```ruby
|
248
|
-
user.destroy
|
249
|
-
```
|
250
|
-
1: Second post notification
|
251
|
-
2: First post notification
|
252
|
-
3: User notification
|
245
|
+
- `method_name` (Symbol, optional) method to be called to process action callback, sample: `def my_method(action) ... end`
|
246
|
+
- `block` (Proc, optional) Block to be called to process action callback, sample: `{ |action| ... }`
|
247
|
+
|
248
|
+
**Note1**: Due to rails callback ordering, this method uses `after_commit on: action {...}` callback when creating or updating models to ensure expected notifications order (More details [**here**](#transactions)).
|
249
|
+
**Note2**: Due to rails callback ordering, this method uses `after_destroy` callback when destroying models to ensure the expected notifications order.
|
253
250
|
|
254
251
|
- `ps_publish(action, data: {}, mapping: [], headers: {}, as_klass: nil)` Delivers an instance notification via pubsub
|
255
252
|
- `action` (Sym|String) Action name of the instance notification. Sample: create|update|destroy|<any_other_key>
|
@@ -266,36 +263,23 @@ PubSubModelSync::MessagePublisher.publish_data(User, { ids: [my_user.id] }, :bat
|
|
266
263
|
- When Proc: Block to be called to retrieve header values (must return a `hash`, receives `:model, :action` as args)
|
267
264
|
- `as_klass:` (String, default current class name): Output class name used instead of current class name
|
268
265
|
|
269
|
-
- `ps_class_publish` Delivers a Class notification via pubsub
|
266
|
+
- `ps_class_publish(data, action:, as_klass: nil, headers: {})` Delivers a Class notification via pubsub
|
270
267
|
- `data` (Hash): Data of the notification
|
271
268
|
- `action` (Symbol): action name of the notification
|
272
269
|
- `as_klass:` (String, default current class name): Class name of the notification
|
273
270
|
- `headers:` (Hash, optional): header settings (More in Payload.headers)
|
271
|
+
|
272
|
+
- `ps_perform_publish(action = :create)` Permits to perform manually the callback of a specific `ps_after_action`
|
273
|
+
- `action` (Symbol, default: :create) Only :create|:update|:destroy
|
274
274
|
|
275
275
|
#### **Publisher helpers**
|
276
|
-
- Publish a
|
277
|
-
```ruby
|
278
|
-
PubSubModelSync::MessagePublisher.publish_data(klass, data, action, headers: )
|
279
|
-
```
|
280
|
-
- `klass`: (String) Class name to be used
|
281
|
-
- Refer to `ps_class_publish` except `as_klass:`
|
282
|
-
|
283
|
-
- Manually publish or republish a notification (useful when failed)
|
276
|
+
- Publish or republish a notification
|
284
277
|
```ruby
|
285
|
-
|
286
|
-
|
278
|
+
payload = PubSubModelSync::Payload.new(data, attributes, headers)
|
279
|
+
payload.publish!
|
287
280
|
```
|
288
281
|
|
289
282
|
#### **Publisher callbacks**
|
290
|
-
- Prevent delivering a notification (called before building payload)
|
291
|
-
If returns "true", will not publish notification
|
292
|
-
```ruby
|
293
|
-
class MyModel < ActiveRecord::Base
|
294
|
-
def ps_skip_publish?(action)
|
295
|
-
# logic here
|
296
|
-
end
|
297
|
-
end
|
298
|
-
```
|
299
283
|
|
300
284
|
- Do some actions before publishing notification.
|
301
285
|
If returns ":cancel", notification will not be delivered
|
@@ -328,7 +312,8 @@ Any notification before delivering is transformed as a Payload for a better port
|
|
328
312
|
- `mode`: (Symbol: `:model`|`:class`) Kind of notification
|
329
313
|
* `headers`: (Hash) Notification settings that defines how the notification will be processed or delivered.
|
330
314
|
- `key`: (String, optional) identifier of the payload, default: `<klass_name>/<action>` when class message, `<model.class.name>/<action>/<model.id>` when model message (Useful for caching techniques).
|
331
|
-
- `ordering_key`: (String, optional): messages with the same value are processed in the same order they were delivered, default: `klass_name` when class message, `<model.class.name>/<model.id>` when instance message
|
315
|
+
- `ordering_key`: (String, optional): messages with the same value are processed in the same order they were delivered, default: `klass_name` when class message, `<model.class.name>/<model.id>` when instance message.
|
316
|
+
Note: Final `ordering_key` is calculated by this way: `payload.headers[:forced_ordering_key] || current_transaction&.key || payload.headers[:ordering_key]`
|
332
317
|
- `topic_name`: (String|Array<String>, optional): Specific topic name (can be seen as a channel) to be used when delivering the message (default first topic from config).
|
333
318
|
- `forced_ordering_key`: (String, optional): Will force to use this value as the `ordering_key`, even withing transactions. Default `nil`.
|
334
319
|
|
@@ -345,20 +330,38 @@ Any notification before delivering is transformed as a Payload for a better port
|
|
345
330
|
* Crud syncs auto includes transactions which works as the following:
|
346
331
|
```ruby
|
347
332
|
class User
|
348
|
-
|
349
|
-
has_many :posts
|
333
|
+
ps_after_action([:create, :update, :destroy]) { |action| ps_publish(action, mapping: %i[id name]) }
|
334
|
+
has_many :posts, dependent: :destroy
|
350
335
|
accepts_nested_attributes_for :posts
|
351
336
|
end
|
352
337
|
|
353
338
|
class Post
|
354
339
|
belongs_to :user
|
355
|
-
|
340
|
+
ps_after_action([:create, :update, :destroy]) { |action| ps_publish(action, mapping: %i[id user_id title]) }
|
356
341
|
end
|
357
|
-
|
358
|
-
User.create!(name: 'test', posts_attributes: [{ title: 'Post 1' }, { title: 'Post 2' }])
|
359
342
|
```
|
360
|
-
When
|
361
|
-
|
343
|
+
- When created (all notifications use the same ordering key to be processed in the same order)
|
344
|
+
```ruby
|
345
|
+
user = User.create!(name: 'test', posts_attributes: [{ title: 'Post 1' }, { title: 'Post 2' }])
|
346
|
+
# notification #1 => <Payload data: {id: 1, name: 'sample'}, info: { klass: 'User', action: :create, mode: :model }, headers: { ordering_key = `User/1` }>
|
347
|
+
# notification #2 => <Payload data: {id: 1, title: 'Post 1', user_id: 1}, info: { klass: 'Post', action: :create, mode: :model }, headers: { ordering_key = `User/1` }>
|
348
|
+
# notification #3 => <Payload data: {id: 2, title: 'Post 2', user_id: 1}, info: { klass: 'Post', action: :create, mode: :model }, headers: { ordering_key = `User/1` }>
|
349
|
+
```
|
350
|
+
- When updated (all notifications use the same ordering key to be processed in the same order)
|
351
|
+
```ruby
|
352
|
+
user.update!(name: 'changed', posts_attributes: [{ id: 1, title: 'Post 1C' }, { id: 2, title: 'Post 2C' }])
|
353
|
+
# notification #1 => <Payload data: {id: 1, name: 'changed'}, info: { klass: 'User', action: :update, mode: :model }, headers: { ordering_key = `User/1` }>
|
354
|
+
# notification #2 => <Payload data: {id: 1, title: 'Post 1C', user_id: 1}, info: { klass: 'Post', action: :update, mode: :model }, headers: { ordering_key = `User/1` }>
|
355
|
+
# notification #3 => <Payload data: {id: 2, title: 'Post 2C', user_id: 1}, info: { klass: 'Post', action: :update, mode: :model }, headers: { ordering_key = `User/1` }>
|
356
|
+
```
|
357
|
+
- When destroyed (all notifications use the same ordering key to be processed in the same order)
|
358
|
+
**Note**: The notifications order were reordered in order to avoid inconsistency in other apps
|
359
|
+
```ruby
|
360
|
+
user.destroy!
|
361
|
+
# notification #1 => <Payload data: {id: 1, title: 'Post 1C', user_id: 1}, info: { klass: 'Post', action: :destroy, mode: :model }>
|
362
|
+
# notification #2 => <Payload data: {id: 2, title: 'Post 2C', user_id: 1}, info: { klass: 'Post', action: :destroy, mode: :model }>
|
363
|
+
# notification #3 => <Payload data: {id: 1, name: 'changed'}, info: { klass: 'User', action: :destroy, mode: :model }>
|
364
|
+
```
|
362
365
|
By this way parent notification and all inner notifications are processed in the same order they were published (includes notifications from callbacks like `ps_before_publish`).
|
363
366
|
|
364
367
|
**Note**: When any error is raised when saving user or posts, the transaction is cancelled and thus all notifications wont be delivered (customizable by `PubSubModelSync::Config.transactions_use_buffer`).
|
@@ -366,22 +369,20 @@ Any notification before delivering is transformed as a Payload for a better port
|
|
366
369
|
- Manual transactions
|
367
370
|
`PubSubModelSync::MessagePublisher::transaction(key, max_buffer: , &block)`
|
368
371
|
- `key` (String|nil) Key used as the ordering key for all inner notifications (When nil, will use `ordering_key` of the first notification)
|
369
|
-
- `max_buffer:` (
|
370
|
-
If true: will save all notifications and deliver all them when transaction has successfully finished. If transaction has failed, then all saved notifications will be discarded (not delivered).
|
371
|
-
If false: will deliver all notifications immediately (no way to rollback notifications if transaction has failed)
|
372
|
+
- `max_buffer:` (Integer, default: `PubSubModelSync::Config.transactions_max_buffer`) Transaction buffer size (more details in #transactions_max_buffer).
|
372
373
|
Sample:
|
373
374
|
```ruby
|
374
375
|
PubSubModelSync::MessagePublisher::transaction('my-custom-key') do
|
375
376
|
user = User.create(name: 'test') # `User`:`:create` notification
|
376
377
|
post = Post.create(title: 'sample') # `Post`:`:create` notification
|
377
|
-
PubSubModelSync::
|
378
|
+
PubSubModelSync::Payload.new({ ids: [user.id] }, { klass: 'User', action: :send_welcome, mode: :klass }).publish! # `User`:`:send_welcome` notification
|
378
379
|
end
|
379
380
|
```
|
380
381
|
All notifications uses `ordering_key: 'my-custom-key'` and will be processed in the same order they were published.
|
381
382
|
|
382
383
|
## **Testing with RSpec**
|
383
384
|
- Config: (spec/rails_helper.rb)
|
384
|
-
|
385
|
+
```ruby
|
385
386
|
|
386
387
|
# when using google service
|
387
388
|
require 'pub_sub_model_sync/mock_google_service'
|
@@ -404,51 +405,119 @@ Any notification before delivering is transformed as a Payload for a better port
|
|
404
405
|
allow(Kafka).to receive(:new).and_return(kafka_mock)
|
405
406
|
end
|
406
407
|
|
407
|
-
#
|
408
|
+
# disable all models sync by default (reduces testing time)
|
408
409
|
config.before(:each) do
|
409
|
-
# **** disable payloads generation, sync callbacks to improve tests speed
|
410
410
|
allow(PubSubModelSync::MessagePublisher).to receive(:publish_data) # disable class level notif
|
411
411
|
allow(PubSubModelSync::MessagePublisher).to receive(:publish_model) # disable instance level notif
|
412
|
-
|
413
|
-
# **** when testing model syncs, it can be re enabled by:
|
414
|
-
# before do
|
415
|
-
# allow(PubSubModelSync::MessagePublisher).to receive(:publish_data).and_call_original
|
416
|
-
# allow(PubSubModelSync::MessagePublisher).to receive(:publish_model).and_call_original
|
417
|
-
# end
|
418
412
|
end
|
419
|
-
|
413
|
+
|
414
|
+
# enable all models sync only for tests that includes 'sync: true'
|
415
|
+
config.before(:each, sync: true) do
|
416
|
+
allow(PubSubModelSync::MessagePublisher).to receive(:publish_data).and_call_original
|
417
|
+
allow(PubSubModelSync::MessagePublisher).to receive(:publish_model).and_call_original
|
418
|
+
end
|
419
|
+
|
420
|
+
# Only when using database cleaner in old versions of rspec (enables after_commit callback)
|
421
|
+
# config.before(:each, truncate: true) do
|
422
|
+
# DatabaseCleaner.strategy = :truncation
|
423
|
+
# end
|
424
|
+
```
|
420
425
|
- Examples:
|
426
|
+
- **Publisher**
|
421
427
|
```ruby
|
422
|
-
|
423
|
-
|
424
|
-
|
428
|
+
# Do not forget to include 'sync: true' to enable publishing pubsub notifications
|
429
|
+
describe 'When publishing sync', truncate: true, sync: true do
|
430
|
+
it 'publishes user notification when created' do
|
431
|
+
expect_publish_notification(:create, klass: 'User')
|
432
|
+
create(:user)
|
433
|
+
end
|
434
|
+
|
435
|
+
it 'publishes user notification with all defined data' do
|
436
|
+
user = build(:user)
|
437
|
+
data = PubSubModelSync::PayloadBuilder.parse_mapping_for(user, %i[id name:full_name email])
|
438
|
+
data[:id] = be_a(Integer)
|
439
|
+
expect_publish_notification(:create, klass: 'User', data: data)
|
440
|
+
user.save!
|
441
|
+
end
|
442
|
+
|
443
|
+
it 'publishes user notification when created' do
|
444
|
+
email = 'Newemail@gmail.com'
|
445
|
+
user = create(:user)
|
446
|
+
expect_publish_notification(:update, klass: 'User', data: { id: user.id, email: email })
|
447
|
+
user.update!(email: email)
|
448
|
+
end
|
449
|
+
|
450
|
+
it 'publishes user notification when created' do
|
451
|
+
user = create(:user)
|
452
|
+
expect_publish_notification(:destroy, klass: 'User', data: { id: user.id })
|
453
|
+
user.destroy!
|
454
|
+
end
|
455
|
+
|
456
|
+
private
|
457
|
+
|
458
|
+
# @param action (Symbol)
|
459
|
+
# @param klass (String, default described_class name)
|
460
|
+
# @param data (Hash, optional) notification data
|
461
|
+
# @param info (Hash, optional) notification info
|
462
|
+
# @param headers (Hash, optional) notification headers
|
463
|
+
def expect_publish_notification(action, klass: described_class.to_s, data: {}, info: {}, headers: {})
|
464
|
+
publisher = PubSubModelSync::MessagePublisher
|
465
|
+
exp_data = have_attributes(data: hash_including(data),
|
466
|
+
info: hash_including(info.merge(klass: klass, action: action)),
|
467
|
+
headers: hash_including(headers))
|
468
|
+
allow(publisher).to receive(:publish!).and_call_original
|
469
|
+
expect(publisher).to receive(:publish!).with(exp_data)
|
470
|
+
end
|
471
|
+
end
|
472
|
+
```
|
473
|
+
- **Subscriber**
|
474
|
+
```ruby
|
475
|
+
|
476
|
+
describe 'when syncing data from other apps' do
|
477
|
+
it 'creates user when received :create notification' do
|
478
|
+
user = build(:user)
|
479
|
+
data = user.as_json(only: %i[name email]).merge(id: 999)
|
425
480
|
payload = PubSubModelSync::Payload.new(data, { klass: 'User', action: :create })
|
481
|
+
expect { payload.process! }.to change(described_class, :count)
|
482
|
+
end
|
483
|
+
|
484
|
+
it 'updates user when received :update notification' do
|
485
|
+
user = create(:user)
|
486
|
+
name = 'new name'
|
487
|
+
data = user.as_json(only: %i[id email]).merge(name: name)
|
488
|
+
payload = PubSubModelSync::Payload.new(data, { klass: 'User', action: :update })
|
489
|
+
payload.process!
|
490
|
+
expect(user.reload.name).to eq(name)
|
491
|
+
end
|
492
|
+
|
493
|
+
it 'destroys user when received :destroy notification' do
|
494
|
+
user = create(:user)
|
495
|
+
data = user.as_json(only: %i[id])
|
496
|
+
payload = PubSubModelSync::Payload.new(data, { klass: 'User', action: :destroy })
|
497
|
+
payload.process!
|
498
|
+
expect { user.reload }.to raise_error(ActiveRecord::RecordNotFound)
|
499
|
+
end
|
500
|
+
|
501
|
+
|
502
|
+
it 'receive custom model notification' do
|
503
|
+
user = create(:user)
|
504
|
+
data = { id: user.id, custom_data: {} }
|
505
|
+
custom_action = :say_hello
|
506
|
+
expect_any_instance_of(User).to receive(custom_action).with(data)
|
507
|
+
payload = PubSubModelSync::Payload.new(data, { klass: 'User', action: custom_action })
|
426
508
|
payload.process!
|
427
|
-
expect(User.where(id: data[:id])).to be_any
|
428
509
|
end
|
429
510
|
|
430
511
|
it 'receive class notification' do
|
431
512
|
data = { msg: 'hello' }
|
432
513
|
action = :greeting
|
514
|
+
expect(User).to receive(action).with(data)
|
515
|
+
# Do not forget to include `mode: :klass` for class notifications
|
433
516
|
payload = PubSubModelSync::Payload.new(data, { klass: 'User', action: action, mode: :klass })
|
434
517
|
payload.process!
|
435
|
-
expect(User).to receive(action)
|
436
518
|
end
|
437
|
-
|
438
|
-
|
439
|
-
it 'publish model notification' do
|
440
|
-
publisher = PubSubModelSync::MessagePublisher
|
441
|
-
user = User.create(name: 'name', email: 'email')
|
442
|
-
expect(publisher).to receive(:publish_model).with(user, :create, anything)
|
443
|
-
end
|
444
|
-
|
445
|
-
it 'publish class notification' do
|
446
|
-
publisher = PubSubModelSync::MessagePublisher
|
447
|
-
user = User.create(name: 'name', email: 'email')
|
448
|
-
user.ps_class_publish({msg: 'hello'}, action: :greeting)
|
449
|
-
expect(publisher).to receive(:publish_data).with('User', data, :greeting)
|
450
|
-
end
|
451
|
-
```
|
519
|
+
end
|
520
|
+
```
|
452
521
|
|
453
522
|
## **Extra configurations**
|
454
523
|
```ruby
|
@@ -479,9 +548,10 @@ config.debug = true
|
|
479
548
|
(Proc) => called after publishing a message
|
480
549
|
- ```.on_error_publish = ->(exception, {payload:}) { payload.delay(...).publish! }```
|
481
550
|
(Proc) => called when failed publishing a message (delayed_job or similar can be used for retrying)
|
482
|
-
- ```.transactions_max_buffer =
|
483
|
-
|
484
|
-
|
551
|
+
- ```.transactions_max_buffer = 1``` (Integer, default 1) Controls the maximum quantity of notifications to be enqueued to the transaction-buffer before delivering them and thus adds the ability to rollback notifications if the transaction fails.
|
552
|
+
Once this quantity of notifications is reached, then all notifications of the current transaction will immediately be delivered (can be customized per transaction).
|
553
|
+
Note: There is no way to rollback delivered notifications if current transaction fails later.
|
554
|
+
Note2: Only notifications from the buffer can be rollbacked if the current transaction has failed.
|
485
555
|
|
486
556
|
## **TODO**
|
487
557
|
- Auto publish update only if payload has changed (see ways to compare previous payload vs new payload)
|
@@ -490,8 +560,11 @@ config.debug = true
|
|
490
560
|
- Add DB table to use as a shield to prevent publishing similar notifications and publish partial notifications (similar idea when processing notif)
|
491
561
|
- Last notification is not being delivered immediately in google pubsub (maybe force with timeout 10secs and service.deliver_messages)
|
492
562
|
- Update folder structure
|
493
|
-
- Support for blocks in ps_publish and ps_subscribe
|
494
563
|
- Services support to deliver multiple payloads from transactions
|
564
|
+
- Fix deprecation warnings: pub_sub_model_sync/service_google.rb:39: warning: Splitting the last argument into positional and keyword parameters is deprecated
|
565
|
+
- Add if/unless to ps_after_action
|
566
|
+
- Add subscription liveness checker using thread without db connection to check periodically pending messages from google pubsub
|
567
|
+
- Unify .stop() and 'Listener stopped'
|
495
568
|
|
496
569
|
## **Q&A**
|
497
570
|
- I'm getting error "could not obtain a connection from the pool within 5.000 seconds"... what does this mean?
|
Binary file
|
@@ -17,25 +17,5 @@ module PubSubModelSync
|
|
17
17
|
config.debug
|
18
18
|
end
|
19
19
|
end
|
20
|
-
|
21
|
-
# @param errors (Array(Class|String))
|
22
|
-
def retry_error(errors, qty: 2, &block)
|
23
|
-
retries ||= 0
|
24
|
-
block.call
|
25
|
-
rescue => e
|
26
|
-
retries += 1
|
27
|
-
res = errors.find { |e_type| match_error?(e, e_type) }
|
28
|
-
raise if !res || retries > qty
|
29
|
-
|
30
|
-
sleep(qty * 0.1) && retry
|
31
|
-
end
|
32
|
-
|
33
|
-
private
|
34
|
-
|
35
|
-
# @param error (Exception)
|
36
|
-
# @param error_type (Class|String)
|
37
|
-
def match_error?(error, error_type)
|
38
|
-
error_type.is_a?(String) ? error.message.include?(error_type) : error.is_a?(error_type)
|
39
|
-
end
|
40
20
|
end
|
41
21
|
end
|
@@ -8,8 +8,7 @@ module PubSubModelSync
|
|
8
8
|
# customizable callbacks
|
9
9
|
cattr_accessor(:debug) { false }
|
10
10
|
cattr_accessor :logger # LoggerInst
|
11
|
-
cattr_accessor(:transactions_max_buffer) {
|
12
|
-
cattr_accessor(:enable_rails4_before_commit) { Rails::VERSION::MAJOR == 4 }
|
11
|
+
cattr_accessor(:transactions_max_buffer) { 1 }
|
13
12
|
|
14
13
|
cattr_accessor(:on_before_processing) { ->(_payload, _info) {} } # return :cancel to skip
|
15
14
|
cattr_accessor(:on_success_processing) { ->(_payload, _info) {} }
|
@@ -30,7 +29,7 @@ module PubSubModelSync
|
|
30
29
|
def self.log(msg, kind = :info)
|
31
30
|
msg = "PS_MSYNC ==> #{msg}"
|
32
31
|
if logger == :raise_error
|
33
|
-
kind == :error ? raise(msg) : puts(msg)
|
32
|
+
kind == :error ? raise(StandardError, msg) : puts(msg)
|
34
33
|
else
|
35
34
|
logger ? logger.send(kind, msg) : puts(msg)
|
36
35
|
end
|
@@ -16,13 +16,17 @@ module PubSubModelSync
|
|
16
16
|
end
|
17
17
|
|
18
18
|
def process!
|
19
|
-
filter_subscribers
|
19
|
+
subscribers = filter_subscribers
|
20
|
+
payload_info = { klass: payload.klass, action: payload.action, mode: payload.mode }
|
21
|
+
log("No subscribers found for #{payload_info}", :warn) if config.debug && subscribers.empty?
|
22
|
+
subscribers.each(&method(:run_subscriber))
|
20
23
|
end
|
21
24
|
|
22
25
|
def process
|
26
|
+
retries ||= 0
|
23
27
|
process!
|
24
28
|
rescue => e
|
25
|
-
notify_error(e)
|
29
|
+
retry_process?(e, retries += 1) ? retry : notify_error(e)
|
26
30
|
end
|
27
31
|
|
28
32
|
private
|
@@ -31,12 +35,10 @@ module PubSubModelSync
|
|
31
35
|
processor = PubSubModelSync::RunSubscriber.new(subscriber, payload)
|
32
36
|
return unless processable?(subscriber)
|
33
37
|
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
log "processed message with: #{payload.inspect}" if res != :skip_log
|
39
|
-
end
|
38
|
+
log("Processing message #{[subscriber, payload]}...") if config.debug
|
39
|
+
processor.call
|
40
|
+
res = config.on_success_processing.call(payload, { subscriber: subscriber })
|
41
|
+
log "processed message with: #{payload.inspect}" if res != :skip_log
|
40
42
|
end
|
41
43
|
|
42
44
|
def processable?(subscriber)
|
@@ -45,13 +47,34 @@ module PubSubModelSync
|
|
45
47
|
!cancel
|
46
48
|
end
|
47
49
|
|
48
|
-
# @param error (
|
50
|
+
# @param error (StandardError)
|
49
51
|
def notify_error(error)
|
50
52
|
info = [payload, error.message, error.backtrace]
|
51
53
|
res = config.on_error_processing.call(error, { payload: payload })
|
52
54
|
log("Error processing message: #{info}", :error) if res != :skip_log
|
53
55
|
end
|
54
56
|
|
57
|
+
def lost_db_connection?(error)
|
58
|
+
connection_lost_classes = %w[ActiveRecord::ConnectionTimeoutError PG::UnableToSend]
|
59
|
+
connection_lost_classes.include?(error.class.name) || error.message.match?(/lost connection/i)
|
60
|
+
end
|
61
|
+
|
62
|
+
def retry_process?(error, retries) # rubocop:disable Metrics/MethodLength
|
63
|
+
error_payload = [payload, error.message, error.backtrace]
|
64
|
+
return false unless lost_db_connection?(error)
|
65
|
+
|
66
|
+
if retries <= 5
|
67
|
+
sleep(retries)
|
68
|
+
log("Error processing message: (retrying #{retries}/5): #{error_payload}", :error)
|
69
|
+
ActiveRecord::Base.connection.reconnect! rescue nil # rubocop:disable Style/RescueModifier
|
70
|
+
true
|
71
|
+
else
|
72
|
+
log("Retried 5 times and error persists, exiting...: #{error_payload}", :error)
|
73
|
+
Process.exit!(true)
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
# @return (Array<PubSubModelSync::Subscriber>)
|
55
78
|
def filter_subscribers
|
56
79
|
config.subscribers.select do |subscriber|
|
57
80
|
subscriber.from_klass == payload.klass && subscriber.action == payload.action && payload.mode == subscriber.mode
|