pub_sub_model_sync 1.4.0 → 1.5.1pre

Sign up to get free protection for your applications and to get access to all the features.
Files changed (103) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -1
  3. data/Gemfile.lock +20 -16
  4. data/README.md +7 -3
  5. data/Rakefile +0 -1
  6. data/lib/pub_sub_model_sync/config.rb +2 -2
  7. data/lib/pub_sub_model_sync/message_processor.rb +13 -4
  8. data/lib/pub_sub_model_sync/mock_rabbit_service.rb +8 -0
  9. data/lib/pub_sub_model_sync/payload_cache_optimizer.rb +3 -2
  10. data/lib/pub_sub_model_sync/publisher_concern.rb +4 -2
  11. data/lib/pub_sub_model_sync/service_base.rb +16 -7
  12. data/lib/pub_sub_model_sync/service_google.rb +0 -1
  13. data/lib/pub_sub_model_sync/service_kafka.rb +2 -1
  14. data/lib/pub_sub_model_sync/service_rabbit.rb +6 -3
  15. data/lib/pub_sub_model_sync/version.rb +1 -1
  16. data/pub_sub_model_sync.gemspec +6 -3
  17. metadata +14 -92
  18. data/samples/README.md +0 -50
  19. data/samples/app1/Dockerfile +0 -13
  20. data/samples/app1/Gemfile +0 -37
  21. data/samples/app1/Gemfile.lock +0 -171
  22. data/samples/app1/README.md +0 -24
  23. data/samples/app1/Rakefile +0 -6
  24. data/samples/app1/app/models/application_record.rb +0 -3
  25. data/samples/app1/app/models/concerns/.keep +0 -0
  26. data/samples/app1/app/models/post.rb +0 -19
  27. data/samples/app1/app/models/user.rb +0 -29
  28. data/samples/app1/bin/bundle +0 -114
  29. data/samples/app1/bin/rails +0 -5
  30. data/samples/app1/bin/rake +0 -5
  31. data/samples/app1/bin/setup +0 -33
  32. data/samples/app1/bin/spring +0 -14
  33. data/samples/app1/config/application.rb +0 -40
  34. data/samples/app1/config/boot.rb +0 -4
  35. data/samples/app1/config/credentials.yml.enc +0 -1
  36. data/samples/app1/config/database.yml +0 -25
  37. data/samples/app1/config/environment.rb +0 -5
  38. data/samples/app1/config/environments/development.rb +0 -63
  39. data/samples/app1/config/environments/production.rb +0 -105
  40. data/samples/app1/config/environments/test.rb +0 -57
  41. data/samples/app1/config/initializers/application_controller_renderer.rb +0 -8
  42. data/samples/app1/config/initializers/backtrace_silencers.rb +0 -8
  43. data/samples/app1/config/initializers/cors.rb +0 -16
  44. data/samples/app1/config/initializers/filter_parameter_logging.rb +0 -6
  45. data/samples/app1/config/initializers/inflections.rb +0 -16
  46. data/samples/app1/config/initializers/mime_types.rb +0 -4
  47. data/samples/app1/config/initializers/pubsub.rb +0 -4
  48. data/samples/app1/config/initializers/wrap_parameters.rb +0 -14
  49. data/samples/app1/config/locales/en.yml +0 -33
  50. data/samples/app1/config/master.key +0 -1
  51. data/samples/app1/config/puma.rb +0 -43
  52. data/samples/app1/config/routes.rb +0 -3
  53. data/samples/app1/config/spring.rb +0 -6
  54. data/samples/app1/config.ru +0 -6
  55. data/samples/app1/db/migrate/20210513080700_create_users.rb +0 -12
  56. data/samples/app1/db/migrate/20210513134332_create_posts.rb +0 -11
  57. data/samples/app1/db/schema.rb +0 -34
  58. data/samples/app1/db/seeds.rb +0 -7
  59. data/samples/app1/docker-compose.yml +0 -32
  60. data/samples/app1/log/.keep +0 -0
  61. data/samples/app2/Dockerfile +0 -13
  62. data/samples/app2/Gemfile +0 -37
  63. data/samples/app2/Gemfile.lock +0 -171
  64. data/samples/app2/README.md +0 -24
  65. data/samples/app2/Rakefile +0 -6
  66. data/samples/app2/app/models/application_record.rb +0 -9
  67. data/samples/app2/app/models/concerns/.keep +0 -0
  68. data/samples/app2/app/models/customer.rb +0 -28
  69. data/samples/app2/app/models/post.rb +0 -10
  70. data/samples/app2/bin/bundle +0 -114
  71. data/samples/app2/bin/rails +0 -5
  72. data/samples/app2/bin/rake +0 -5
  73. data/samples/app2/bin/setup +0 -33
  74. data/samples/app2/bin/spring +0 -14
  75. data/samples/app2/config/application.rb +0 -40
  76. data/samples/app2/config/boot.rb +0 -4
  77. data/samples/app2/config/credentials.yml.enc +0 -1
  78. data/samples/app2/config/database.yml +0 -25
  79. data/samples/app2/config/environment.rb +0 -5
  80. data/samples/app2/config/environments/development.rb +0 -63
  81. data/samples/app2/config/environments/production.rb +0 -105
  82. data/samples/app2/config/environments/test.rb +0 -57
  83. data/samples/app2/config/initializers/application_controller_renderer.rb +0 -8
  84. data/samples/app2/config/initializers/backtrace_silencers.rb +0 -8
  85. data/samples/app2/config/initializers/cors.rb +0 -16
  86. data/samples/app2/config/initializers/filter_parameter_logging.rb +0 -6
  87. data/samples/app2/config/initializers/inflections.rb +0 -16
  88. data/samples/app2/config/initializers/mime_types.rb +0 -4
  89. data/samples/app2/config/initializers/pubsub.rb +0 -4
  90. data/samples/app2/config/initializers/wrap_parameters.rb +0 -14
  91. data/samples/app2/config/locales/en.yml +0 -33
  92. data/samples/app2/config/master.key +0 -1
  93. data/samples/app2/config/puma.rb +0 -43
  94. data/samples/app2/config/routes.rb +0 -3
  95. data/samples/app2/config/spring.rb +0 -6
  96. data/samples/app2/config.ru +0 -6
  97. data/samples/app2/db/development.sqlite3 +0 -0
  98. data/samples/app2/db/migrate/20210513080956_create_customers.rb +0 -10
  99. data/samples/app2/db/migrate/20210513135203_create_posts.rb +0 -10
  100. data/samples/app2/db/schema.rb +0 -31
  101. data/samples/app2/db/seeds.rb +0 -7
  102. data/samples/app2/docker-compose.yml +0 -20
  103. data/samples/app2/log/.keep +0 -0
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 1740843df35081aaeb35e2be5c1c27cf1ff4d219af0bcf9ae4394495ab8bba5e
4
- data.tar.gz: ed506905c6b29b554f4a1cd100ddf4c8df2bf0a99590eb26901fae8feef862ce
3
+ metadata.gz: 55d5a541d463ac4b23a26bbcb2ba14f5c4d6336d72bde40d1ea9c1ff5bc4b89f
4
+ data.tar.gz: 496ed96aacf2c6ab74ffb4f997b00cfffc76168f168647f94fec3fa57bc3ae59
5
5
  SHA512:
6
- metadata.gz: bff308ce421ae321d2a9e7b53df194d3678e5f6103eb4f2048d4ed064302c042fb7768f96a3c639190b8d452a0636fded9e72394a5ffe4731b4e24ba0b2f5b53
7
- data.tar.gz: e60c4c78c5fbde7c40d68adb1ed029d74bb10d1759a27682850aae6015c84f938cbccccaadc976878afa26d97621d360937cdeb5cc028b2393facb12fa0d4831
6
+ metadata.gz: 199e1c3841b4038ec67b189b2da974655d7ce87c637297b59c646628e7cba0359f64f11e8b1e0c4d3db65827ffa3a4f81fb52492bb0e5c40e1f9438b894135a6
7
+ data.tar.gz: 951e65c9e9e7d109b5c3fe9c7e79ffb00ebe7110bbdd7d15c93a1a5c83e460a0512292bfe21b87fc8bba5d11dcf79c75af2723cfc0b61cacef8f6f812cab9834
data/.gitignore CHANGED
@@ -6,6 +6,7 @@
6
6
  /pkg/
7
7
  /spec/reports/
8
8
  /tmp/
9
-
9
+ /samples/app2/db/**/*
10
+ /samples/app1/db/**/*
10
11
  # rspec failure tracking
11
12
  .rspec_status
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- pub_sub_model_sync (1.4.0)
4
+ pub_sub_model_sync (1.5.0)
5
5
  rails
6
6
 
7
7
  GEM
@@ -99,8 +99,8 @@ GEM
99
99
  googleapis-common-protos-types (>= 1.0.6, < 2.0)
100
100
  googleauth (~> 0.15, >= 0.15.1)
101
101
  grpc (~> 1.36)
102
- globalid (0.4.2)
103
- activesupport (>= 4.2.0)
102
+ globalid (1.0.0)
103
+ activesupport (>= 5.0)
104
104
  google-cloud-core (1.6.0)
105
105
  google-cloud-env (~> 1.0)
106
106
  google-cloud-errors (~> 1.0)
@@ -116,6 +116,7 @@ GEM
116
116
  google-cloud-errors (~> 1.0)
117
117
  grpc-google-iam-v1 (>= 0.6.10, < 2.0)
118
118
  google-protobuf (3.17.0)
119
+ google-protobuf (3.17.0-x86_64-linux)
119
120
  googleapis-common-protos (1.3.11)
120
121
  google-protobuf (~> 3.14)
121
122
  googleapis-common-protos-types (>= 1.0.6, < 2.0)
@@ -132,6 +133,9 @@ GEM
132
133
  grpc (1.37.1)
133
134
  google-protobuf (~> 3.15)
134
135
  googleapis-common-protos-types (~> 1.0)
136
+ grpc (1.37.1-x86_64-linux)
137
+ google-protobuf (~> 3.15)
138
+ googleapis-common-protos-types (~> 1.0)
135
139
  grpc-google-iam-v1 (0.6.11)
136
140
  google-protobuf (~> 3.14)
137
141
  googleapis-common-protos (>= 1.3.11, < 2.0)
@@ -139,27 +143,27 @@ GEM
139
143
  i18n (1.8.10)
140
144
  concurrent-ruby (~> 1.0)
141
145
  jwt (2.2.3)
142
- loofah (2.9.1)
146
+ loofah (2.15.0)
143
147
  crass (~> 1.0.2)
144
148
  nokogiri (>= 1.5.9)
145
149
  mail (2.7.1)
146
150
  mini_mime (>= 0.1.1)
147
- marcel (1.0.1)
151
+ marcel (1.0.2)
148
152
  memoist (0.16.2)
149
153
  method_source (1.0.0)
150
154
  mini_mime (1.0.3)
151
155
  minitest (5.14.4)
152
156
  multi_json (1.15.0)
153
157
  multipart-post (2.1.1)
154
- nio4r (2.5.7)
155
- nokogiri (1.11.3-x86_64-darwin)
158
+ nio4r (2.5.8)
159
+ nokogiri (1.12.5-x86_64-linux)
156
160
  racc (~> 1.4)
157
161
  os (1.1.1)
158
162
  parallel (1.20.1)
159
163
  parser (3.0.1.1)
160
164
  ast (~> 2.4.1)
161
165
  public_suffix (4.0.6)
162
- racc (1.5.2)
166
+ racc (1.6.0)
163
167
  rack (2.2.3)
164
168
  rack-test (1.1.0)
165
169
  rack (>= 1.0, < 3)
@@ -181,7 +185,7 @@ GEM
181
185
  rails-dom-testing (2.0.3)
182
186
  activesupport (>= 4.2.0)
183
187
  nokogiri (>= 1.6)
184
- rails-html-sanitizer (1.3.0)
188
+ rails-html-sanitizer (1.4.2)
185
189
  loofah (~> 2.3)
186
190
  railties (6.1.3.2)
187
191
  actionpack (= 6.1.3.2)
@@ -226,19 +230,19 @@ GEM
226
230
  faraday (>= 0.17.3, < 2.0)
227
231
  jwt (>= 1.5, < 3.0)
228
232
  multi_json (~> 1.10)
229
- sprockets (4.0.2)
233
+ sprockets (4.0.3)
230
234
  concurrent-ruby (~> 1.0)
231
235
  rack (> 1, < 3)
232
- sprockets-rails (3.2.2)
233
- actionpack (>= 4.0)
234
- activesupport (>= 4.0)
236
+ sprockets-rails (3.4.2)
237
+ actionpack (>= 5.2)
238
+ activesupport (>= 5.2)
235
239
  sprockets (>= 3.0.0)
236
240
  sqlite3 (1.4.2)
237
- thor (1.1.0)
241
+ thor (1.2.1)
238
242
  tzinfo (2.0.4)
239
243
  concurrent-ruby (~> 1.0)
240
244
  unicode-display_width (1.7.0)
241
- websocket-driver (0.7.3)
245
+ websocket-driver (0.7.5)
242
246
  websocket-extensions (>= 0.1.0)
243
247
  websocket-extensions (0.1.5)
244
248
  zeitwerk (2.4.2)
@@ -260,4 +264,4 @@ DEPENDENCIES
260
264
  sqlite3
261
265
 
262
266
  BUNDLED WITH
263
- 2.2.29
267
+ 2.3.9
data/README.md CHANGED
@@ -167,7 +167,7 @@ PubSubModelSync::Payload.new({ ids: [my_user.id] }, { klass: 'User', action: :ba
167
167
  ps_class_subscribe(action, settings)
168
168
  end
169
169
  ```
170
- - Instance subscriptions: `ps_subscribe(action, mapping, settings)`
170
+ - Instance subscriptions: `ps_subscribe(action, mapping, settings, &block)`
171
171
  When model receives the corresponding notification, `action` or `to_action` method will be called on the model. Like: `model.destroy`
172
172
  - `action` (Symbol|Array<Symbol>) Only notifications with this action name will be processed by this subscription. Sample: save|create|update|destroy|<any_other_action>
173
173
  - `mapping` (Array<String>) Data mapping from payload data into model attributes, sample: ["email", "full_name:name"] (Note: Only these attributes will be assigned/synced to the current model)
@@ -182,12 +182,14 @@ PubSubModelSync::Payload.new({ ids: [my_user.id] }, { klass: 'User', action: :ba
182
182
  Sample: `id: :id` will search for a model like: `model_class.where(id: payload.data[:id])`
183
183
  Sample: `id: [:id, :email:user_email]` will search for a model like: `model_class.where(id: payload.data[:id], user_email: payload.data[:email])`
184
184
  - `if:` (Symbol|Proc|Array<Symbol>) Method(s) or block called for the confirmation before calling the callback
185
- - `unless:` (Symbol|Proc|Array<Symbol>) Method or block called for the negation before calling the callback
185
+ - `unless:` (Symbol|Proc|Array<Symbol>) Method or block called for the negation before calling the callback
186
+ - `&block` Block to be used as the callback method (ignored if `:to_action` is present). Sample: `ps_subscribe(:send_welcome, %i[email]) { |_data| puts model.email }`
186
187
 
187
- - Class subscriptions: `ps_class_subscribe(action, settings)`
188
+ - Class subscriptions: `ps_class_subscribe(action, settings, &block)`
188
189
  When current class receives the corresponding notification, `action` or `to_action` method will be called on the Class. Like: `User.hello(data)`
189
190
  * `action` (Symbol) Notification.action name
190
191
  * `settings` (Hash) refer ps_subscribe.settings except(:id)
192
+ * `&block` Block to be used as the callback method (ignored if `:to_action` is present). Sample: `ps_class_subscribe(:send_welcome) { |data| puts data }`
191
193
 
192
194
  - `ps_processing_payload` a class and instance variable that saves the current payload being processed
193
195
 
@@ -534,6 +536,8 @@ config.debug = true
534
536
  (Proc) => called after publishing a notification
535
537
  - ```.on_error_publish = ->(exception, {payload:}) { payload.delay(...).publish! }```
536
538
  (Proc) => called when failed publishing a notification (delayed_job or similar can be used for retrying)
539
+ - ```.skip_cache = false```
540
+ (true/false*) => Allow to skip payload optimization (cache settings)
537
541
  - ```.transactions_max_buffer = 1``` (Integer, default 1) Controls the maximum quantity of notifications to be enqueued to the transaction-buffer before delivering them and thus adds the ability to rollback notifications if the transaction fails.
538
542
  Once this quantity of notifications is reached, then all notifications of the current transaction will immediately be delivered (can be customized per transaction).
539
543
  Note: There is no way to rollback delivered notifications if current transaction fails later.
data/Rakefile CHANGED
@@ -4,4 +4,3 @@ require "rspec/core/rake_task"
4
4
  RSpec::Core::RakeTask.new(:spec)
5
5
 
6
6
  task :default => :spec
7
- # by default skip ACK when failed in three of them
@@ -7,13 +7,13 @@ module PubSubModelSync
7
7
 
8
8
  # customizable callbacks
9
9
  cattr_accessor(:debug) { false }
10
- cattr_accessor :logger # LoggerInst
10
+ cattr_accessor(:logger) { Rails.logger }
11
11
  cattr_accessor(:transactions_max_buffer) { 1 }
12
12
  cattr_accessor(:skip_cache) { false }
13
13
 
14
14
  cattr_accessor(:on_before_processing) { ->(_payload, _info) {} } # return :cancel to skip
15
15
  cattr_accessor(:on_success_processing) { ->(_payload, _info) {} }
16
- cattr_accessor(:on_error_processing) { ->(_exception, _info) {} }
16
+ cattr_accessor(:on_error_processing) { ->(exception, _info) { raise(exception) } }
17
17
  cattr_accessor(:on_before_publish) { ->(_payload) {} } # return :cancel to skip
18
18
  cattr_accessor(:on_after_publish) { ->(_payload) {} }
19
19
  cattr_accessor(:on_error_publish) { ->(_exception, _info) {} }
@@ -49,18 +49,27 @@ module PubSubModelSync
49
49
 
50
50
  # @param error (StandardError)
51
51
  def notify_error(error)
52
- info = [payload, error.message, error.backtrace]
52
+ error_msg = 'Error processing message: '
53
+ error_details = [payload, error.message, error.backtrace]
53
54
  res = config.on_error_processing.call(error, { payload: payload })
54
- log("Error processing message: #{info}", :error) if res != :skip_log
55
+ log("#{error_msg} #{error_details}", :error) if res != :skip_log
56
+ rescue => e
57
+ error_details = [payload, e.message, e.backtrace]
58
+ log("#{error_msg} #{error_details}", :error)
59
+ raise(e)
55
60
  end
56
61
 
62
+ # @param error [StandardError]
57
63
  def lost_db_connection?(error)
58
- connection_lost_classes = %w[ActiveRecord::ConnectionTimeoutError PG::UnableToSend]
59
- connection_lost_classes.include?(error.class.name) || error.message.match?(/lost connection/i)
64
+ classes = %w[ActiveRecord::ConnectionTimeoutError PG::UnableToSend ActiveRecord::ConnectionNotEstablished]
65
+ classes.include?(error.class.name) || error.message.match?(/Lost connection to MySQL server/i)
60
66
  end
61
67
 
62
68
  def retry_process?(error, retries) # rubocop:disable Metrics/MethodLength
63
69
  error_payload = [payload, error.message, error.backtrace]
70
+ if error.message.include('Connection refused') || error.message.include('no connection to the server')
71
+ log("Error pubsub: #{[error.class.name, error.message]} (retries: #{retries}) => #{lost_db_connection?(error)}", :error)
72
+ end
64
73
  return false unless lost_db_connection?(error)
65
74
 
66
75
  if retries <= 5
@@ -24,6 +24,10 @@ module PubSubModelSync
24
24
  def publish(*_args)
25
25
  true
26
26
  end
27
+
28
+ def channel
29
+ MockChannel.new
30
+ end
27
31
  end
28
32
 
29
33
  class MockChannel
@@ -39,6 +43,10 @@ module PubSubModelSync
39
43
  def close
40
44
  true
41
45
  end
46
+
47
+ def ack(_delivery_tag)
48
+ true
49
+ end
42
50
  end
43
51
 
44
52
  def create_channel(*_args)
@@ -41,8 +41,9 @@ module PubSubModelSync
41
41
  end
42
42
 
43
43
  def optimize_payload # rubocop:disable Metrics/AbcSize
44
- changed_keys = Hash[(payload.data.to_a - previous_payload_data.to_a)].keys
45
- invalid_keys = payload.data.keys - (changed_keys + payload.cache_settings[:required])
44
+ changed_keys = Hash[(payload.data.to_a - previous_payload_data.to_a)].keys.map(&:to_sym)
45
+ required_keys = payload.cache_settings[:required].map(&:to_sym)
46
+ invalid_keys = payload.data.keys - (changed_keys + required_keys)
46
47
  log("Excluding non changed attributes: #{invalid_keys} from: #{payload.inspect}") if debug?
47
48
  payload.exclude_data_attrs(invalid_keys)
48
49
  end
@@ -38,8 +38,10 @@ module PubSubModelSync
38
38
 
39
39
  # Permits to perform manually the callback for a specific action
40
40
  # @param action (Symbol, default: :create) Only :create|:update|:destroy
41
- def ps_perform_publish(action = :create)
42
- items = self.class.ps_cache_publish_callbacks.select { |item| item[:actions].include?(action) }
41
+ def ps_perform_publish(action = :create, parents_actions: false)
42
+ parent_callbacks = parents_actions ? self.class.superclass.ps_cache_publish_callbacks : []
43
+ callbacks = self.class.ps_cache_publish_callbacks
44
+ items = (callbacks + parent_callbacks).select { |item| item[:actions].include?(action) }
43
45
  raise(StandardError, "No callback found for action :#{action}") if items.empty?
44
46
 
45
47
  items.each { |item| instance_exec(action, &item[:callback]) }
@@ -33,25 +33,34 @@ module PubSubModelSync
33
33
  # @param (String: Payload in json format)
34
34
  def process_message(payload_info)
35
35
  payload = decode_payload(payload_info)
36
- return payload.process unless same_app_message?(payload)
36
+ return unless payload
37
+ return if same_app_message?(payload) || !target_app_message?(payload)
37
38
 
38
- log("Skipping message from same origin: #{[payload]}") if config.debug
39
- rescue => e
40
- error_payload = [payload, e.message, e.backtrace]
41
- log("Error while starting to process a message: #{error_payload}", :error)
39
+ payload.process
42
40
  end
43
41
 
44
- # @return Payload
42
+ # @return [Payload,Nil]
45
43
  def decode_payload(payload_info)
46
44
  payload = ::PubSubModelSync::Payload.from_payload_data(JSON.parse(payload_info))
47
45
  log("Received message: #{[payload]}") if config.debug
48
46
  payload
47
+ rescue => e
48
+ error_payload = [payload_info, e.message, e.backtrace]
49
+ log("Error while parsing payload: #{error_payload}", :error)
50
+ nil
49
51
  end
50
52
 
51
53
  # @param payload (Payload)
52
54
  def same_app_message?(payload)
53
55
  key = payload.headers[:app_key]
54
- key && key == config.subscription_key
56
+ res = key && key == config.subscription_key
57
+ log("Skipping message from same origin: #{[payload]}") if res && config.debug
58
+ res
59
+ end
60
+
61
+ def target_app_message?(payload)
62
+ key = payload.headers[:target_app_key]
63
+ key == config.subscription_key || !key
55
64
  end
56
65
  end
57
66
  end
@@ -92,7 +92,6 @@ module PubSubModelSync
92
92
  def process_message(received_message)
93
93
  message = received_message.message
94
94
  super(message.data) if message.attributes[SERVICE_KEY]
95
- ensure
96
95
  received_message.acknowledge!
97
96
  end
98
97
  end
@@ -8,7 +8,7 @@ end
8
8
  module PubSubModelSync
9
9
  class ServiceKafka < ServiceBase
10
10
  QTY_WORKERS = 10
11
- LISTEN_SETTINGS = {}.freeze
11
+ LISTEN_SETTINGS = { automatically_mark_as_processed: false }.freeze
12
12
  PUBLISH_SETTINGS = {}.freeze
13
13
  PRODUCER_SETTINGS = { delivery_threshold: 200, delivery_interval: 30 }.freeze
14
14
  cattr_accessor :producer
@@ -73,6 +73,7 @@ module PubSubModelSync
73
73
 
74
74
  def process_message(message)
75
75
  super(message.value) if message.headers[SERVICE_KEY]
76
+ consumer.mark_message_as_processed(message)
76
77
  end
77
78
 
78
79
  # Check topic existence, create if missing topic
@@ -8,7 +8,7 @@ end
8
8
  module PubSubModelSync
9
9
  class ServiceRabbit < ServiceBase
10
10
  QUEUE_SETTINGS = { durable: true, auto_delete: false }.freeze
11
- LISTEN_SETTINGS = { manual_ack: false }.freeze
11
+ LISTEN_SETTINGS = { manual_ack: true }.freeze
12
12
  PUBLISH_SETTINGS = {}.freeze
13
13
 
14
14
  # @!attribute topic_names (Array): ['Topic 1', 'Topic 2']
@@ -25,7 +25,9 @@ module PubSubModelSync
25
25
 
26
26
  def listen_messages
27
27
  log('Listener starting...')
28
- subscribe_to_queues { |queue| queue.subscribe(LISTEN_SETTINGS, &method(:process_message)) }
28
+ subscribe_to_queues do |queue|
29
+ queue.subscribe(LISTEN_SETTINGS) { |info, meta, payload| process_message(queue, info, meta, payload) }
30
+ end
29
31
  log('Listener started')
30
32
  loop { sleep 5 }
31
33
  rescue PubSubModelSync::Runner::ShutDown
@@ -62,8 +64,9 @@ module PubSubModelSync
62
64
  }.merge(PUBLISH_SETTINGS)
63
65
  end
64
66
 
65
- def process_message(_delivery_info, meta_info, payload)
67
+ def process_message(queue, delivery_info, meta_info, payload)
66
68
  super(payload) if meta_info[:type] == SERVICE_KEY
69
+ queue.channel.ack(delivery_info.delivery_tag)
67
70
  end
68
71
 
69
72
  def subscribe_to_queues(&block)
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module PubSubModelSync
4
- VERSION = '1.4.0'
4
+ VERSION = '1.5.1pre'
5
5
  end
@@ -11,8 +11,11 @@ Gem::Specification.new do |spec|
11
11
  spec.authors = ['Owen']
12
12
  spec.email = ['owenperedo@gmail.com']
13
13
 
14
- spec.summary = 'Permit to sync models between apps through pub/sub'
15
- spec.description = 'Permit to sync models between apps through pub/sub'
14
+ spec.summary = 'This gem permits to sync automatically models and custom data between multiple Rails
15
+ applications by publishing notifications via pubsub (Google PubSub, RabbitMQ, or Apache Kafka) and automatically
16
+ processed by all connected applications. Out of the scope, this gem includes transactions to keep Data consistency
17
+ by processing notifications in the order they were delivered.'
18
+ spec.description = spec.summary
16
19
  spec.homepage = 'https://github.com/owen2345/pub_sub_model_sync'
17
20
  spec.license = 'MIT'
18
21
 
@@ -27,7 +30,7 @@ Gem::Specification.new do |spec|
27
30
  # into git.
28
31
  spec.files = Dir.chdir(File.expand_path(__dir__)) do
29
32
  `git ls-files -z`.split("\x0")
30
- .reject { |f| f.match(%r{^(test|spec|features)/}) }
33
+ .reject { |f| f.match(%r{^(test|spec|features|samples)/}) }
31
34
  end
32
35
  spec.bindir = 'exe'
33
36
  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: pub_sub_model_sync
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.4.0
4
+ version: 1.5.1pre
5
5
  platform: ruby
6
6
  authors:
7
7
  - Owen
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2022-03-02 00:00:00.000000000 Z
11
+ date: 2022-10-14 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rails
@@ -80,7 +80,11 @@ dependencies:
80
80
  - - ">="
81
81
  - !ruby/object:Gem::Version
82
82
  version: '0'
83
- description: Permit to sync models between apps through pub/sub
83
+ description: This gem permits to sync automatically models and custom data between
84
+ multiple Rails applications by publishing notifications via pubsub (Google PubSub,
85
+ RabbitMQ, or Apache Kafka) and automatically processed by all connected applications.
86
+ Out of the scope, this gem includes transactions to keep Data consistency by processing
87
+ notifications in the order they were delivered.
84
88
  email:
85
89
  - owenperedo@gmail.com
86
90
  executables: []
@@ -134,92 +138,6 @@ files:
134
138
  - lib/pub_sub_model_sync/transaction.rb
135
139
  - lib/pub_sub_model_sync/version.rb
136
140
  - pub_sub_model_sync.gemspec
137
- - samples/README.md
138
- - samples/app1/Dockerfile
139
- - samples/app1/Gemfile
140
- - samples/app1/Gemfile.lock
141
- - samples/app1/README.md
142
- - samples/app1/Rakefile
143
- - samples/app1/app/models/application_record.rb
144
- - samples/app1/app/models/concerns/.keep
145
- - samples/app1/app/models/post.rb
146
- - samples/app1/app/models/user.rb
147
- - samples/app1/bin/bundle
148
- - samples/app1/bin/rails
149
- - samples/app1/bin/rake
150
- - samples/app1/bin/setup
151
- - samples/app1/bin/spring
152
- - samples/app1/config.ru
153
- - samples/app1/config/application.rb
154
- - samples/app1/config/boot.rb
155
- - samples/app1/config/credentials.yml.enc
156
- - samples/app1/config/database.yml
157
- - samples/app1/config/environment.rb
158
- - samples/app1/config/environments/development.rb
159
- - samples/app1/config/environments/production.rb
160
- - samples/app1/config/environments/test.rb
161
- - samples/app1/config/initializers/application_controller_renderer.rb
162
- - samples/app1/config/initializers/backtrace_silencers.rb
163
- - samples/app1/config/initializers/cors.rb
164
- - samples/app1/config/initializers/filter_parameter_logging.rb
165
- - samples/app1/config/initializers/inflections.rb
166
- - samples/app1/config/initializers/mime_types.rb
167
- - samples/app1/config/initializers/pubsub.rb
168
- - samples/app1/config/initializers/wrap_parameters.rb
169
- - samples/app1/config/locales/en.yml
170
- - samples/app1/config/master.key
171
- - samples/app1/config/puma.rb
172
- - samples/app1/config/routes.rb
173
- - samples/app1/config/spring.rb
174
- - samples/app1/db/migrate/20210513080700_create_users.rb
175
- - samples/app1/db/migrate/20210513134332_create_posts.rb
176
- - samples/app1/db/schema.rb
177
- - samples/app1/db/seeds.rb
178
- - samples/app1/docker-compose.yml
179
- - samples/app1/log/.keep
180
- - samples/app2/Dockerfile
181
- - samples/app2/Gemfile
182
- - samples/app2/Gemfile.lock
183
- - samples/app2/README.md
184
- - samples/app2/Rakefile
185
- - samples/app2/app/models/application_record.rb
186
- - samples/app2/app/models/concerns/.keep
187
- - samples/app2/app/models/customer.rb
188
- - samples/app2/app/models/post.rb
189
- - samples/app2/bin/bundle
190
- - samples/app2/bin/rails
191
- - samples/app2/bin/rake
192
- - samples/app2/bin/setup
193
- - samples/app2/bin/spring
194
- - samples/app2/config.ru
195
- - samples/app2/config/application.rb
196
- - samples/app2/config/boot.rb
197
- - samples/app2/config/credentials.yml.enc
198
- - samples/app2/config/database.yml
199
- - samples/app2/config/environment.rb
200
- - samples/app2/config/environments/development.rb
201
- - samples/app2/config/environments/production.rb
202
- - samples/app2/config/environments/test.rb
203
- - samples/app2/config/initializers/application_controller_renderer.rb
204
- - samples/app2/config/initializers/backtrace_silencers.rb
205
- - samples/app2/config/initializers/cors.rb
206
- - samples/app2/config/initializers/filter_parameter_logging.rb
207
- - samples/app2/config/initializers/inflections.rb
208
- - samples/app2/config/initializers/mime_types.rb
209
- - samples/app2/config/initializers/pubsub.rb
210
- - samples/app2/config/initializers/wrap_parameters.rb
211
- - samples/app2/config/locales/en.yml
212
- - samples/app2/config/master.key
213
- - samples/app2/config/puma.rb
214
- - samples/app2/config/routes.rb
215
- - samples/app2/config/spring.rb
216
- - samples/app2/db/development.sqlite3
217
- - samples/app2/db/migrate/20210513080956_create_customers.rb
218
- - samples/app2/db/migrate/20210513135203_create_posts.rb
219
- - samples/app2/db/schema.rb
220
- - samples/app2/db/seeds.rb
221
- - samples/app2/docker-compose.yml
222
- - samples/app2/log/.keep
223
141
  homepage: https://github.com/owen2345/pub_sub_model_sync
224
142
  licenses:
225
143
  - MIT
@@ -238,12 +156,16 @@ required_ruby_version: !ruby/object:Gem::Requirement
238
156
  version: '2.4'
239
157
  required_rubygems_version: !ruby/object:Gem::Requirement
240
158
  requirements:
241
- - - ">="
159
+ - - ">"
242
160
  - !ruby/object:Gem::Version
243
- version: '0'
161
+ version: 1.3.1
244
162
  requirements: []
245
163
  rubygems_version: 3.0.8
246
164
  signing_key:
247
165
  specification_version: 4
248
- summary: Permit to sync models between apps through pub/sub
166
+ summary: This gem permits to sync automatically models and custom data between multiple
167
+ Rails applications by publishing notifications via pubsub (Google PubSub, RabbitMQ,
168
+ or Apache Kafka) and automatically processed by all connected applications. Out
169
+ of the scope, this gem includes transactions to keep Data consistency by processing
170
+ notifications in the order they were delivered.
249
171
  test_files: []
data/samples/README.md DELETED
@@ -1,50 +0,0 @@
1
- # Sample model sync
2
- This is a sample to sync information between rails applications using RabbitMQ
3
-
4
- ## Installation
5
- 1. Create manually the required network to share rabbitMQ accross Rails applications (just if not exist):
6
- ```docker network create shared_app_services```
7
-
8
- 2. Start RabbitMQ server
9
- ```cd samples/app1 && docker-compose up pubsub```
10
-
11
- 3. In another tab access to App2 to listen notifications (Wait for step 2)
12
- - Access to the folder
13
- `cd samples/app2`
14
-
15
- - Build docker and start listener (Received notifications will be printed here)
16
- ```docker-compose run listener```
17
-
18
- - Optional: Open another tab to access application to ensure synced data
19
- ```docker-compose run listener bash -c "rails c```
20
- ```ruby
21
- user = User.last.inspect
22
- user.posts.inspect
23
- ```
24
-
25
- 4. In another tab access to App1 to publish notifications (Wait for step 2)
26
- - Access to the application
27
- `cd samples/app1`
28
-
29
- - Build docker and enter rails console
30
- ```docker-compose run app bash -c "rails db:migrate && rails c"```
31
-
32
- - Create a sample user
33
- ```ruby
34
- user = User.create!(name: 'User 1', posts_attributes: [{ title: 'Post 1' }, { title: 'Post 2' }])
35
- ```
36
- Note: Check app2 console to see notifications (3 notifications)
37
- Note2: Access app2 console to see user and its posts
38
-
39
- - Update previous user
40
- ```ruby
41
- user.update!(name: 'User 1 changed', posts_attributes: user.posts.map { |post| { id: post.id, title: "#{post.title} changed" } })
42
- ```
43
- Note: Check app2 console to see notifications (3 notifications)
44
- Note2: Access app2 console to see changes for user and its posts
45
-
46
- - Destroy previous user
47
- ```ruby
48
- user.destroy!
49
- ```
50
-
@@ -1,13 +0,0 @@
1
- FROM ruby:2.7.1 AS builder
2
-
3
- # Allow apt to work with https-based sources
4
- RUN apt-get update -yqq && apt-get install -yqq --no-install-recommends apt-transport-https
5
-
6
- RUN mkdir /app
7
- WORKDIR /app
8
-
9
- # backend
10
- COPY Gemfile.lock Gemfile /app/
11
- RUN gem install bundler && bundle install
12
-
13
- COPY . /app
data/samples/app1/Gemfile DELETED
@@ -1,37 +0,0 @@
1
- source 'https://rubygems.org'
2
- git_source(:github) { |repo| "https://github.com/#{repo}.git" }
3
-
4
- # Bundle edge Rails instead: gem 'rails', github: 'rails/rails', branch: 'main'
5
- gem 'rails', '~> 6.1.3', '>= 6.1.3.2'
6
- # Use sqlite3 as the database for Active Record
7
- gem 'sqlite3', '~> 1.4'
8
- # Use Puma as the app server
9
- gem 'puma', '~> 5.0'
10
- # Build JSON APIs with ease. Read more: https://github.com/rails/jbuilder
11
- # gem 'jbuilder', '~> 2.7'
12
- # Use Active Model has_secure_password
13
- # gem 'bcrypt', '~> 3.1.7'
14
-
15
- # Reduces boot times through caching; required in config/boot.rb
16
- gem 'bootsnap', '>= 1.4.4', require: false
17
-
18
- # Use Rack CORS for handling Cross-Origin Resource Sharing (CORS), making cross-origin AJAX possible
19
- # gem 'rack-cors'
20
-
21
- gem 'pub_sub_model_sync', '>= 1.0'
22
- gem 'bunny' # to use rabbit-mq pub/sub service
23
- gem 'annotate'
24
-
25
- group :development, :test do
26
- # Call 'byebug' anywhere in the code to stop execution and get a debugger console
27
- gem 'byebug', platforms: [:mri, :mingw, :x64_mingw]
28
- end
29
-
30
- group :development do
31
- gem 'listen', '~> 3.3'
32
- # Spring speeds up development by keeping your application running in the background. Read more: https://github.com/rails/spring
33
- gem 'spring'
34
- end
35
-
36
- # Windows does not include zoneinfo files, so bundle the tzinfo-data gem
37
- gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw, :jruby]