pub_sub_model_sync 0.4.2.1 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ruby.yml +6 -1
- data/.rubocop.yml +5 -2
- data/CHANGELOG.md +12 -0
- data/Gemfile +1 -1
- data/Gemfile.lock +15 -12
- data/README.md +35 -16
- data/lib/pub_sub_model_sync.rb +1 -0
- data/lib/pub_sub_model_sync/base.rb +17 -0
- data/lib/pub_sub_model_sync/config.rb +18 -3
- data/lib/pub_sub_model_sync/connector.rb +1 -0
- data/lib/pub_sub_model_sync/message_processor.rb +27 -21
- data/lib/pub_sub_model_sync/message_publisher.rb +25 -9
- data/lib/pub_sub_model_sync/mock_rabbit_service.rb +1 -0
- data/lib/pub_sub_model_sync/payload.rb +45 -0
- data/lib/pub_sub_model_sync/publisher.rb +1 -0
- data/lib/pub_sub_model_sync/publisher_concern.rb +2 -1
- data/lib/pub_sub_model_sync/service_base.rb +25 -13
- data/lib/pub_sub_model_sync/service_google.rb +4 -18
- data/lib/pub_sub_model_sync/service_kafka.rb +4 -17
- data/lib/pub_sub_model_sync/service_rabbit.rb +20 -27
- data/lib/pub_sub_model_sync/subscriber.rb +1 -3
- data/lib/pub_sub_model_sync/version.rb +1 -1
- data/pub_sub_model_sync.gemspec +1 -1
- metadata +4 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: edcb2024203ae281bbc642825d0f5361205afe787c4a2ebb735cefaa1d98b80c
|
4
|
+
data.tar.gz: 1d6bb21f6808f9595d0bb842a997457097bb0dbc25deb500bb67d73f229b2480
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c6fd7fd582186a12564a41314ba85b198011213b79e6c18af24a542652d30b2ab2f44126f501608a37cdd7e1714cba84cb174b1ae63a14086d0a691acee1a4d4
|
7
|
+
data.tar.gz: 6ed6f1c39199582db6e7b51468e4da9ff98b55c8245df64971bc76bf510bf2f68e15109104074fbb5a3e34a1534c1711990f514b12b22d1e6504560c4a19612f
|
data/.github/workflows/ruby.yml
CHANGED
@@ -42,9 +42,14 @@ jobs:
|
|
42
42
|
gem install bundler -v "~> $bundler_v"
|
43
43
|
bundle _${bundler_v}_ install --jobs 4 --retry 3
|
44
44
|
|
45
|
+
# remote ssh debugger
|
46
|
+
# - name: Setup tmate session (remote session debugger)
|
47
|
+
# uses: mxschmitt/action-tmate@v3
|
48
|
+
|
45
49
|
- name: Tests (rspec)
|
46
50
|
run: |
|
47
51
|
bundle exec rspec
|
48
52
|
|
49
53
|
- name: Code style (Rubocop)
|
50
|
-
run: bundle exec rubocop
|
54
|
+
run: bundle exec rubocop
|
55
|
+
if: matrix.ruby == '2.6' && matrix.rails == '6'
|
data/.rubocop.yml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
# This is the configuration used to check the rubocop source code.
|
2
2
|
|
3
3
|
AllCops:
|
4
|
-
TargetRubyVersion: 2.
|
4
|
+
TargetRubyVersion: 2.6
|
5
5
|
Exclude:
|
6
6
|
- 'spec/spec_helper.rb'
|
7
7
|
- 'Gemfile'
|
@@ -13,12 +13,15 @@ Metrics/BlockLength:
|
|
13
13
|
- 'spec/**/*.rb'
|
14
14
|
|
15
15
|
Layout/LineLength:
|
16
|
-
Max:
|
16
|
+
Max: 120
|
17
17
|
|
18
18
|
Style/SymbolArray:
|
19
19
|
Exclude:
|
20
20
|
- 'Gemfile'
|
21
21
|
|
22
|
+
Lint/MissingSuper:
|
23
|
+
Enabled: false
|
24
|
+
|
22
25
|
Style/Documentation:
|
23
26
|
Enabled: false
|
24
27
|
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,17 @@
|
|
1
1
|
# Change Log
|
2
2
|
|
3
|
+
# 0.5.0 (December 22, 2020)
|
4
|
+
- feat: add :publish! and :process! methods to payloads
|
5
|
+
- feat: add ability to disable publisher globally
|
6
|
+
- fix: skip notifications from the same application
|
7
|
+
- fix: rabbitmq use fanout instead of queue to deliver messages to multiple apps
|
8
|
+
- refactor: include payload object to carry message info
|
9
|
+
- feat: include notification events (when publishing and when processing messages)
|
10
|
+
|
11
|
+
# 0.4.2.2 (November 29, 2020, deleted cause of typo)
|
12
|
+
- feat: rabbitMQ skip receiving messages from the same app
|
13
|
+
- feat: rabbitmq use fanout instead of queue to deliver messages to multiple apps
|
14
|
+
|
3
15
|
# 0.4.2.1 (August 20, 2020)
|
4
16
|
- Improve ```ps_subscriber_changed?``` to run validations and check for changes
|
5
17
|
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
pub_sub_model_sync (0.
|
4
|
+
pub_sub_model_sync (0.5.0)
|
5
5
|
rails
|
6
6
|
|
7
7
|
GEM
|
@@ -65,7 +65,7 @@ GEM
|
|
65
65
|
addressable (2.7.0)
|
66
66
|
public_suffix (>= 2.0.2, < 5.0)
|
67
67
|
amq-protocol (2.3.0)
|
68
|
-
ast (2.4.
|
68
|
+
ast (2.4.1)
|
69
69
|
builder (3.2.4)
|
70
70
|
bunny (2.14.3)
|
71
71
|
amq-protocol (~> 2.3, >= 2.3.0)
|
@@ -120,7 +120,6 @@ GEM
|
|
120
120
|
grpc (~> 1.0)
|
121
121
|
i18n (1.8.2)
|
122
122
|
concurrent-ruby (~> 1.0)
|
123
|
-
jaro_winkler (1.5.4)
|
124
123
|
jwt (2.2.1)
|
125
124
|
loofah (2.6.0)
|
126
125
|
crass (~> 1.0.2)
|
@@ -141,9 +140,9 @@ GEM
|
|
141
140
|
nokogiri (1.10.10)
|
142
141
|
mini_portile2 (~> 2.4.0)
|
143
142
|
os (1.0.1)
|
144
|
-
parallel (1.
|
145
|
-
parser (2.7.0
|
146
|
-
ast (~> 2.4.
|
143
|
+
parallel (1.20.1)
|
144
|
+
parser (2.7.2.0)
|
145
|
+
ast (~> 2.4.1)
|
147
146
|
public_suffix (4.0.3)
|
148
147
|
rack (2.2.3)
|
149
148
|
rack-test (1.1.0)
|
@@ -176,6 +175,7 @@ GEM
|
|
176
175
|
thor (>= 0.20.3, < 2.0)
|
177
176
|
rainbow (3.0.0)
|
178
177
|
rake (13.0.1)
|
178
|
+
regexp_parser (2.0.1)
|
179
179
|
rexml (3.2.4)
|
180
180
|
rly (0.2.3)
|
181
181
|
rspec (3.9.0)
|
@@ -191,14 +191,17 @@ GEM
|
|
191
191
|
diff-lcs (>= 1.2.0, < 2.0)
|
192
192
|
rspec-support (~> 3.9.0)
|
193
193
|
rspec-support (3.9.2)
|
194
|
-
rubocop (
|
195
|
-
jaro_winkler (~> 1.5.1)
|
194
|
+
rubocop (1.6.1)
|
196
195
|
parallel (~> 1.10)
|
197
|
-
parser (>= 2.7.
|
196
|
+
parser (>= 2.7.1.5)
|
198
197
|
rainbow (>= 2.2.2, < 4.0)
|
198
|
+
regexp_parser (>= 1.8, < 3.0)
|
199
199
|
rexml
|
200
|
+
rubocop-ast (>= 1.2.0, < 2.0)
|
200
201
|
ruby-progressbar (~> 1.7)
|
201
|
-
unicode-display_width (>= 1.4.0, <
|
202
|
+
unicode-display_width (>= 1.4.0, < 2.0)
|
203
|
+
rubocop-ast (1.3.0)
|
204
|
+
parser (>= 2.7.1.5)
|
202
205
|
ruby-kafka (1.0.0)
|
203
206
|
digest-crc
|
204
207
|
ruby-progressbar (1.10.1)
|
@@ -219,7 +222,7 @@ GEM
|
|
219
222
|
thread_safe (0.3.6)
|
220
223
|
tzinfo (1.2.7)
|
221
224
|
thread_safe (~> 0.1)
|
222
|
-
unicode-display_width (1.
|
225
|
+
unicode-display_width (1.7.0)
|
223
226
|
websocket-driver (0.7.3)
|
224
227
|
websocket-extensions (>= 0.1.0)
|
225
228
|
websocket-extensions (0.1.5)
|
@@ -236,7 +239,7 @@ DEPENDENCIES
|
|
236
239
|
pub_sub_model_sync!
|
237
240
|
rake
|
238
241
|
rspec
|
239
|
-
rubocop
|
242
|
+
rubocop (~> 1.6.0)
|
240
243
|
ruby-kafka
|
241
244
|
sqlite3
|
242
245
|
|
data/README.md
CHANGED
@@ -198,11 +198,11 @@ Note: Be careful with collision of names
|
|
198
198
|
* action_name: (required, :sim) Action name
|
199
199
|
* as_klass: (optional, :string) Custom class name (Default current model name)
|
200
200
|
|
201
|
-
- Publish a class level notification (Same as above:
|
202
|
-
```
|
203
|
-
|
204
|
-
|
205
|
-
|
201
|
+
- Publish a class level notification (Same as above: manual call)
|
202
|
+
```ruby
|
203
|
+
payload = PubSubModelSync::Payload.new({ title: 'hello' }, { action: :greeting, klass: 'User' })
|
204
|
+
payload.publish!
|
205
|
+
```
|
206
206
|
|
207
207
|
- Get crud publisher configured for the class
|
208
208
|
```User.ps_publisher(action_name)```
|
@@ -238,27 +238,23 @@ Note: Be careful with collision of names
|
|
238
238
|
```ruby
|
239
239
|
# Subscriber
|
240
240
|
it 'receive model message' do
|
241
|
-
action = :create
|
242
241
|
data = { name: 'name', id: 999 }
|
243
|
-
|
244
|
-
|
242
|
+
payload = PubSubModelSync::Payload.new(data, { klass: 'User', action: :create })
|
243
|
+
payload.process!
|
245
244
|
expect(User.where(id: data[:id]).any?).to be_truth
|
246
245
|
end
|
247
246
|
|
248
247
|
it 'receive class message' do
|
249
|
-
action = :greeting
|
250
248
|
data = { msg: 'hello' }
|
251
|
-
|
252
|
-
|
249
|
+
action = :greeting
|
250
|
+
payload = PubSubModelSync::Payload.new(data, { klass: 'User', action: action })
|
251
|
+
payload.process!
|
253
252
|
expect(User).to receive(action)
|
254
253
|
end
|
255
254
|
|
256
255
|
# Publisher
|
257
256
|
it 'publish model action' do
|
258
257
|
publisher = PubSubModelSync::MessagePublisher
|
259
|
-
data = { name: 'hello'}
|
260
|
-
action = :create
|
261
|
-
User.ps_class_publish(data, action: action)
|
262
258
|
user = User.create(name: 'name', email: 'email')
|
263
259
|
expect(publisher).to receive(:publish_model).with(user, :create, anything)
|
264
260
|
end
|
@@ -272,11 +268,34 @@ Note: Be careful with collision of names
|
|
272
268
|
end
|
273
269
|
```
|
274
270
|
|
271
|
+
## Extra configurations
|
272
|
+
```ruby
|
273
|
+
config = PubSubModelSync::Config
|
274
|
+
config.debug = true
|
275
|
+
```
|
276
|
+
|
277
|
+
- ```debug = true```
|
278
|
+
(true/false*) => show advanced log messages
|
279
|
+
- ```logger = Rails.logger```
|
280
|
+
(Logger) => define custom logger
|
281
|
+
- ```disabled = true```
|
282
|
+
(true/false*) => if true, does not publish model messages (Create/Update/Destroy)
|
283
|
+
- ```on_process_success = ->(payload, subscriber) { puts payload }```
|
284
|
+
(Proc) => called when a message was successfully processed
|
285
|
+
- ```on_process_error = ->(exception, payload) { sleep 1; payload.process! }```
|
286
|
+
(Proc) => called when a message failed when processing
|
287
|
+
- ```on_before_publish = ->(payload) { puts payload }```
|
288
|
+
(Proc) => called before publishing a message
|
289
|
+
- ```on_after_publish = ->(payload) { puts payload }```
|
290
|
+
(Proc) => called after publishing a message
|
291
|
+
- ```on_publish_error = ->(exception, payload) { sleep 1; payload.publish! }```
|
292
|
+
(Proc) => called when failed publishing a message
|
293
|
+
|
275
294
|
## TODO
|
276
|
-
- Hooks/callbacks when message processed or failed
|
277
295
|
- Add alias attributes when subscribing (similar to publisher)
|
278
296
|
- Add flag ```model.ps_processing``` to indicate that the current transaction is being processed by pub/sub
|
279
|
-
|
297
|
+
- Auto publish update only if payload has changed
|
298
|
+
- On delete, payload must only be composed by ids
|
280
299
|
|
281
300
|
## Contributing
|
282
301
|
|
data/lib/pub_sub_model_sync.rb
CHANGED
@@ -5,6 +5,7 @@ require 'active_support'
|
|
5
5
|
|
6
6
|
require 'pub_sub_model_sync/railtie'
|
7
7
|
require 'pub_sub_model_sync/config'
|
8
|
+
require 'pub_sub_model_sync/base'
|
8
9
|
require 'pub_sub_model_sync/subscriber_concern'
|
9
10
|
require 'pub_sub_model_sync/message_publisher'
|
10
11
|
require 'pub_sub_model_sync/publisher_concern'
|
@@ -0,0 +1,17 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module PubSubModelSync
|
4
|
+
class Base
|
5
|
+
delegate :config, :log, to: self
|
6
|
+
|
7
|
+
class << self
|
8
|
+
def config
|
9
|
+
PubSubModelSync::Config
|
10
|
+
end
|
11
|
+
|
12
|
+
def log(message, kind = :info)
|
13
|
+
config.log message, kind
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -5,16 +5,26 @@ module PubSubModelSync
|
|
5
5
|
cattr_accessor(:subscribers) { [] }
|
6
6
|
cattr_accessor(:publishers) { [] }
|
7
7
|
cattr_accessor(:service_name) { :google }
|
8
|
-
|
8
|
+
|
9
|
+
# customizable callbacks
|
10
|
+
cattr_accessor(:debug) { false }
|
11
|
+
cattr_accessor :logger # LoggerInst
|
12
|
+
|
13
|
+
cattr_accessor(:on_process_success) { ->(_payload, _subscriber) {} }
|
14
|
+
cattr_accessor(:on_process_error) { ->(_exception, _payload) {} }
|
15
|
+
cattr_accessor(:on_before_publish) { ->(_payload) {} }
|
16
|
+
cattr_accessor(:on_after_publish) { ->(_payload) {} }
|
17
|
+
cattr_accessor(:on_publish_error) { ->(_exception, _payload) {} }
|
18
|
+
cattr_accessor(:disabled) { false }
|
9
19
|
|
10
20
|
# google service
|
11
21
|
cattr_accessor :project, :credentials, :topic_name, :subscription_name
|
12
22
|
|
13
23
|
# rabbitmq service
|
14
|
-
cattr_accessor :bunny_connection, :queue_name, :topic_name
|
24
|
+
cattr_accessor :bunny_connection, :queue_name, :topic_name, :subscription_name
|
15
25
|
|
16
26
|
# kafka service
|
17
|
-
cattr_accessor :kafka_connection, :topic_name
|
27
|
+
cattr_accessor :kafka_connection, :topic_name, :subscription_name
|
18
28
|
|
19
29
|
def self.log(msg, kind = :info)
|
20
30
|
msg = "PS_MSYNC ==> #{msg}"
|
@@ -24,5 +34,10 @@ module PubSubModelSync
|
|
24
34
|
logger ? logger.send(kind, msg) : puts(msg)
|
25
35
|
end
|
26
36
|
end
|
37
|
+
|
38
|
+
def self.subscription_key
|
39
|
+
subscription_name ||
|
40
|
+
(Rails.application.class.parent_name rescue '') # rubocop:disable Style/RescueModifier
|
41
|
+
end
|
27
42
|
end
|
28
43
|
end
|
@@ -1,40 +1,46 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
module PubSubModelSync
|
4
|
-
class MessageProcessor
|
5
|
-
attr_accessor :
|
6
|
-
|
7
|
-
# @param
|
8
|
-
def initialize(data, klass, action)
|
9
|
-
|
10
|
-
@
|
11
|
-
|
4
|
+
class MessageProcessor < PubSubModelSync::Base
|
5
|
+
attr_accessor :payload
|
6
|
+
|
7
|
+
# @param payload (Payload): payload to be delivered
|
8
|
+
# @Deprecated: def initialize(data, klass, action)
|
9
|
+
def initialize(payload, klass = nil, action = nil)
|
10
|
+
@payload = payload
|
11
|
+
return if @payload.is_a?(Payload)
|
12
|
+
|
13
|
+
# support for deprecated
|
14
|
+
log('Deprecated: Use Payload instead of new(data, klass, action)')
|
15
|
+
@payload = PubSubModelSync::Payload.new(payload, { klass: klass, action: action })
|
12
16
|
end
|
13
17
|
|
14
18
|
def process
|
15
|
-
|
16
|
-
subscribers.each { |subscriber| run_subscriber(subscriber) }
|
19
|
+
filter_subscribers.each(&method(:run_subscriber))
|
17
20
|
end
|
18
21
|
|
19
22
|
private
|
20
23
|
|
21
24
|
def run_subscriber(subscriber)
|
22
|
-
subscriber.eval_message(data)
|
23
|
-
|
25
|
+
subscriber.eval_message(payload.data)
|
26
|
+
config.on_process_success.call(payload, subscriber)
|
27
|
+
log "processed message with: #{payload}"
|
24
28
|
rescue => e
|
25
|
-
|
26
|
-
log("error processing message: #{info}", :error)
|
29
|
+
print_subscriber_error(e)
|
27
30
|
end
|
28
31
|
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
32
|
+
# @param error (Error)
|
33
|
+
def print_subscriber_error(error)
|
34
|
+
info = [payload, error.message, error.backtrace]
|
35
|
+
res = config.on_process_error.call(error, payload)
|
36
|
+
log("Error processing message: #{info}", :error) if res != :skip_log
|
34
37
|
end
|
35
38
|
|
36
|
-
def
|
37
|
-
|
39
|
+
def filter_subscribers
|
40
|
+
config.subscribers.select do |subscriber|
|
41
|
+
subscriber.settings[:from_klass].to_s == payload.klass.to_s &&
|
42
|
+
subscriber.settings[:from_action].to_s == payload.action.to_s
|
43
|
+
end
|
38
44
|
end
|
39
45
|
end
|
40
46
|
end
|
@@ -1,17 +1,15 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
module PubSubModelSync
|
4
|
-
class MessagePublisher
|
4
|
+
class MessagePublisher < PubSubModelSync::Base
|
5
5
|
class << self
|
6
|
-
delegate :publish, to: :connector
|
7
|
-
|
8
6
|
def connector
|
9
7
|
@connector ||= PubSubModelSync::Connector.new
|
10
8
|
end
|
11
9
|
|
12
10
|
def publish_data(klass, data, action)
|
13
|
-
|
14
|
-
publish(
|
11
|
+
payload = PubSubModelSync::Payload.new(data, { klass: klass, action: action.to_sym })
|
12
|
+
publish(payload)
|
15
13
|
end
|
16
14
|
|
17
15
|
# @param model: ActiveRecord model
|
@@ -21,12 +19,30 @@ module PubSubModelSync
|
|
21
19
|
return if model.ps_skip_sync?(action)
|
22
20
|
|
23
21
|
publisher ||= model.class.ps_publisher(action)
|
24
|
-
|
25
|
-
|
22
|
+
payload_info = publisher.payload(model, action)
|
23
|
+
payload = PubSubModelSync::Payload.new(payload_info[:data], payload_info[:attrs])
|
24
|
+
res_before = model.ps_before_sync(action, payload.data)
|
26
25
|
return if res_before == :cancel
|
27
26
|
|
28
|
-
publish(payload
|
29
|
-
model.ps_after_sync(action, payload
|
27
|
+
publish(payload)
|
28
|
+
model.ps_after_sync(action, payload.data)
|
29
|
+
end
|
30
|
+
|
31
|
+
def publish(payload)
|
32
|
+
log("Publishing message: #{[payload]}") if config.debug
|
33
|
+
config.on_before_publish.call(payload)
|
34
|
+
connector.publish(payload)
|
35
|
+
config.on_after_publish.call(payload)
|
36
|
+
rescue => e
|
37
|
+
notify_error(e, payload)
|
38
|
+
end
|
39
|
+
|
40
|
+
private
|
41
|
+
|
42
|
+
def notify_error(exception, payload)
|
43
|
+
info = [payload, exception.message, exception.backtrace]
|
44
|
+
res = config.on_publish_error.call(exception, payload)
|
45
|
+
log("Error publishing: #{info}", :error) if res != :skip_log
|
30
46
|
end
|
31
47
|
end
|
32
48
|
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module PubSubModelSync
|
4
|
+
class Payload
|
5
|
+
attr_reader :data, :attributes, :headers
|
6
|
+
|
7
|
+
# @param data (Hash: { any value }):
|
8
|
+
# @param attributes (Hash: { klass: string, action: :sym }):
|
9
|
+
def initialize(data, attributes, headers = {})
|
10
|
+
@data = data
|
11
|
+
@attributes = attributes
|
12
|
+
@headers = headers
|
13
|
+
build_headers
|
14
|
+
end
|
15
|
+
|
16
|
+
def to_h
|
17
|
+
{ data: data, attributes: attributes, headers: headers }
|
18
|
+
end
|
19
|
+
|
20
|
+
def klass
|
21
|
+
attributes[:klass]
|
22
|
+
end
|
23
|
+
|
24
|
+
def action
|
25
|
+
attributes[:action]
|
26
|
+
end
|
27
|
+
|
28
|
+
def process!
|
29
|
+
publisher = PubSubModelSync::MessageProcessor.new(self)
|
30
|
+
publisher.process
|
31
|
+
end
|
32
|
+
|
33
|
+
def publish!
|
34
|
+
klass = PubSubModelSync::MessagePublisher
|
35
|
+
klass.publish(self)
|
36
|
+
end
|
37
|
+
|
38
|
+
private
|
39
|
+
|
40
|
+
def build_headers
|
41
|
+
headers[:uuid] ||= SecureRandom.uuid
|
42
|
+
headers[:app_key] ||= PubSubModelSync::Config.subscription_key
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -64,7 +64,8 @@ module PubSubModelSync
|
|
64
64
|
|
65
65
|
def ps_register_callback(action, publisher)
|
66
66
|
after_commit(on: action) do |model|
|
67
|
-
|
67
|
+
disabled = PubSubModelSync::Config.disabled
|
68
|
+
if !disabled && !model.ps_skip_callback?(action)
|
68
69
|
klass = PubSubModelSync::MessagePublisher
|
69
70
|
klass.publish_model(model, action.to_sym, publisher)
|
70
71
|
end
|
@@ -1,14 +1,16 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require 'pub_sub_model_sync/payload'
|
3
4
|
module PubSubModelSync
|
4
|
-
class ServiceBase
|
5
|
+
class ServiceBase < PubSubModelSync::Base
|
5
6
|
SERVICE_KEY = 'service_model_sync'
|
6
7
|
|
7
8
|
def listen_messages
|
8
9
|
raise 'method :listen_messages must be defined in service'
|
9
10
|
end
|
10
11
|
|
11
|
-
|
12
|
+
# @param _payload (Payload)
|
13
|
+
def publish(_payload)
|
12
14
|
raise 'method :publish must be defined in service'
|
13
15
|
end
|
14
16
|
|
@@ -18,19 +20,29 @@ module PubSubModelSync
|
|
18
20
|
|
19
21
|
private
|
20
22
|
|
21
|
-
# @param
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
23
|
+
# @param (String: Payload in json format)
|
24
|
+
def process_message(payload_info)
|
25
|
+
payload = parse_payload(payload_info)
|
26
|
+
log("Received message: #{[payload]}") if config.debug
|
27
|
+
if same_app_message?(payload)
|
28
|
+
log("Skip message from same origin: #{[payload]}") if config.debug
|
29
|
+
else
|
30
|
+
payload.process!
|
31
|
+
end
|
32
|
+
rescue => e
|
33
|
+
error = [payload, e.message, e.backtrace]
|
34
|
+
log("Error parsing received message: #{error}", :error)
|
27
35
|
end
|
28
36
|
|
29
|
-
def
|
30
|
-
|
31
|
-
data
|
32
|
-
|
33
|
-
|
37
|
+
def parse_payload(payload_info)
|
38
|
+
info = JSON.parse(payload_info).deep_symbolize_keys
|
39
|
+
::PubSubModelSync::Payload.new(info[:data], info[:attributes], info[:headers])
|
40
|
+
end
|
41
|
+
|
42
|
+
# @param payload (Payload)
|
43
|
+
def same_app_message?(payload)
|
44
|
+
key = payload.headers[:app_key]
|
45
|
+
key && key == config.subscription_key
|
34
46
|
end
|
35
47
|
end
|
36
48
|
end
|
@@ -7,10 +7,9 @@ end
|
|
7
7
|
|
8
8
|
module PubSubModelSync
|
9
9
|
class ServiceGoogle < ServiceBase
|
10
|
-
attr_accessor :service, :topic, :subscription, :
|
10
|
+
attr_accessor :service, :topic, :subscription, :subscriber
|
11
11
|
|
12
12
|
def initialize
|
13
|
-
@config = PubSubModelSync::Config
|
14
13
|
@service = Google::Cloud::Pubsub.new(project: config.project,
|
15
14
|
credentials: config.credentials)
|
16
15
|
@topic = service.topic(config.topic_name) ||
|
@@ -28,13 +27,8 @@ module PubSubModelSync
|
|
28
27
|
log('Listener stopped')
|
29
28
|
end
|
30
29
|
|
31
|
-
def publish(
|
32
|
-
|
33
|
-
payload = { data: data, attributes: attributes }.to_json
|
34
|
-
topic.publish(payload, { SERVICE_KEY => true })
|
35
|
-
rescue => e
|
36
|
-
info = [attributes, data, e.message, e.backtrace]
|
37
|
-
log("Error publishing: #{info}", :error)
|
30
|
+
def publish(payload)
|
31
|
+
topic.publish(payload.to_json, { SERVICE_KEY => true })
|
38
32
|
end
|
39
33
|
|
40
34
|
def stop
|
@@ -51,17 +45,9 @@ module PubSubModelSync
|
|
51
45
|
|
52
46
|
def process_message(received_message)
|
53
47
|
message = received_message.message
|
54
|
-
|
55
|
-
|
56
|
-
perform_message(message.data)
|
57
|
-
rescue => e
|
58
|
-
log("Error processing message: #{[received_message, e.message]}", :error)
|
48
|
+
super(message.data) if message.attributes[SERVICE_KEY]
|
59
49
|
ensure
|
60
50
|
received_message.acknowledge!
|
61
51
|
end
|
62
|
-
|
63
|
-
def log(msg, kind = :info)
|
64
|
-
config.log("Google Service ==> #{msg}", kind)
|
65
|
-
end
|
66
52
|
end
|
67
53
|
end
|
@@ -8,9 +8,8 @@ end
|
|
8
8
|
module PubSubModelSync
|
9
9
|
class ServiceKafka < ServiceBase
|
10
10
|
cattr_accessor :producer
|
11
|
+
attr_accessor :config, :service, :consumer
|
11
12
|
|
12
|
-
attr_accessor :service, :consumer
|
13
|
-
attr_accessor :config
|
14
13
|
CONSUMER_GROUP = 'service_model_sync'
|
15
14
|
|
16
15
|
def initialize
|
@@ -23,19 +22,14 @@ module PubSubModelSync
|
|
23
22
|
start_consumer
|
24
23
|
consumer.each_message(&method(:process_message))
|
25
24
|
rescue PubSubModelSync::Runner::ShutDown
|
26
|
-
|
25
|
+
log('Listener stopped')
|
27
26
|
rescue => e
|
28
27
|
log("Error listening message: #{[e.message, e.backtrace]}", :error)
|
29
28
|
end
|
30
29
|
|
31
|
-
def publish(
|
32
|
-
log("Publishing: #{[attributes, data]}")
|
33
|
-
payload = { data: data, attributes: attributes }
|
30
|
+
def publish(payload)
|
34
31
|
producer.produce(payload.to_json, message_settings)
|
35
32
|
producer.deliver_messages
|
36
|
-
rescue => e
|
37
|
-
info = [attributes, data, e.message, e.backtrace]
|
38
|
-
log("Error publishing: #{info}", :error)
|
39
33
|
end
|
40
34
|
|
41
35
|
def stop
|
@@ -64,14 +58,7 @@ module PubSubModelSync
|
|
64
58
|
def process_message(message)
|
65
59
|
return unless message.headers[SERVICE_KEY]
|
66
60
|
|
67
|
-
|
68
|
-
rescue => e
|
69
|
-
error = [message, e.message, e.backtrace]
|
70
|
-
log("Error processing message: #{error}", :error)
|
71
|
-
end
|
72
|
-
|
73
|
-
def log(msg, kind = :info)
|
74
|
-
config.log("Kafka Service ==> #{msg}", kind)
|
61
|
+
super(message.value)
|
75
62
|
end
|
76
63
|
end
|
77
64
|
end
|
@@ -7,8 +7,7 @@ end
|
|
7
7
|
|
8
8
|
module PubSubModelSync
|
9
9
|
class ServiceRabbit < ServiceBase
|
10
|
-
attr_accessor :service, :channel, :queue, :topic
|
11
|
-
attr_accessor :config
|
10
|
+
attr_accessor :config, :service, :channel, :queue, :topic
|
12
11
|
|
13
12
|
def initialize
|
14
13
|
@config = PubSubModelSync::Config
|
@@ -22,22 +21,21 @@ module PubSubModelSync
|
|
22
21
|
queue.subscribe(subscribe_settings, &method(:process_message))
|
23
22
|
loop { sleep 5 }
|
24
23
|
rescue PubSubModelSync::Runner::ShutDown
|
25
|
-
|
24
|
+
log('Listener stopped')
|
26
25
|
rescue => e
|
27
26
|
log("Error listening message: #{[e.message, e.backtrace]}", :error)
|
28
27
|
end
|
29
28
|
|
30
|
-
def publish(
|
31
|
-
|
32
|
-
deliver_data(
|
33
|
-
# TODO: max retry
|
34
|
-
rescue Timeout::Error => e
|
35
|
-
log("Error publishing (retrying....): #{e.message}", :error)
|
36
|
-
initialize
|
37
|
-
retry
|
29
|
+
def publish(payload)
|
30
|
+
qty_retry ||= 0
|
31
|
+
deliver_data(payload)
|
38
32
|
rescue => e
|
39
|
-
|
40
|
-
|
33
|
+
if e.is_a?(Timeout::Error) && (qty_retry += 1) <= 2
|
34
|
+
log("Error publishing (retrying....): #{e.message}", :error)
|
35
|
+
initialize
|
36
|
+
retry
|
37
|
+
end
|
38
|
+
raise
|
41
39
|
end
|
42
40
|
|
43
41
|
def stop
|
@@ -48,7 +46,10 @@ module PubSubModelSync
|
|
48
46
|
private
|
49
47
|
|
50
48
|
def message_settings
|
51
|
-
{
|
49
|
+
{
|
50
|
+
routing_key: queue.name,
|
51
|
+
type: SERVICE_KEY
|
52
|
+
}
|
52
53
|
end
|
53
54
|
|
54
55
|
def subscribe_settings
|
@@ -58,10 +59,7 @@ module PubSubModelSync
|
|
58
59
|
def process_message(_delivery_info, meta_info, payload)
|
59
60
|
return unless meta_info[:type] == SERVICE_KEY
|
60
61
|
|
61
|
-
|
62
|
-
rescue => e
|
63
|
-
error = [payload, e.message, e.backtrace]
|
64
|
-
log("Error processing message: #{error}", :error)
|
62
|
+
super(payload)
|
65
63
|
end
|
66
64
|
|
67
65
|
def subscribe_to_queue
|
@@ -69,21 +67,16 @@ module PubSubModelSync
|
|
69
67
|
@channel = service.create_channel
|
70
68
|
queue_settings = { durable: true, auto_delete: false }
|
71
69
|
@queue = channel.queue(config.queue_name, queue_settings)
|
72
|
-
|
70
|
+
subscribe_to_exchange
|
73
71
|
end
|
74
72
|
|
75
|
-
def
|
76
|
-
@topic = channel.
|
73
|
+
def subscribe_to_exchange
|
74
|
+
@topic = channel.fanout(config.topic_name)
|
77
75
|
queue.bind(topic, routing_key: queue.name)
|
78
76
|
end
|
79
77
|
|
80
|
-
def
|
81
|
-
config.log("Rabbit Service ==> #{msg}", kind)
|
82
|
-
end
|
83
|
-
|
84
|
-
def deliver_data(data, attributes)
|
78
|
+
def deliver_data(payload)
|
85
79
|
subscribe_to_queue
|
86
|
-
payload = { data: data, attributes: attributes }
|
87
80
|
topic.publish(payload.to_json, message_settings)
|
88
81
|
|
89
82
|
# Ugly fix: "IO timeout when reading 7 bytes"
|
@@ -45,9 +45,7 @@ module PubSubModelSync
|
|
45
45
|
|
46
46
|
def find_model(message)
|
47
47
|
model_class = klass.constantize
|
48
|
-
if model_class.respond_to?(:ps_find_model)
|
49
|
-
return model_class.ps_find_model(message)
|
50
|
-
end
|
48
|
+
return model_class.ps_find_model(message) if model_class.respond_to?(:ps_find_model)
|
51
49
|
|
52
50
|
model_class.where(model_identifiers(message)).first_or_initialize
|
53
51
|
end
|
data/pub_sub_model_sync.gemspec
CHANGED
@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
5
5
|
require 'pub_sub_model_sync/version'
|
6
6
|
|
7
7
|
Gem::Specification.new do |spec|
|
8
|
-
spec.required_ruby_version = '>= 2.4'
|
8
|
+
spec.required_ruby_version = '>= 2.4' # rubocop:disable Gemspec/RequiredRubyVersion
|
9
9
|
spec.name = 'pub_sub_model_sync'
|
10
10
|
spec.version = PubSubModelSync::VERSION
|
11
11
|
spec.authors = ['Owen']
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pub_sub_model_sync
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Owen
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-12-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rails
|
@@ -104,6 +104,7 @@ files:
|
|
104
104
|
- gemfiles/Gemfile_5
|
105
105
|
- gemfiles/Gemfile_6
|
106
106
|
- lib/pub_sub_model_sync.rb
|
107
|
+
- lib/pub_sub_model_sync/base.rb
|
107
108
|
- lib/pub_sub_model_sync/config.rb
|
108
109
|
- lib/pub_sub_model_sync/connector.rb
|
109
110
|
- lib/pub_sub_model_sync/message_processor.rb
|
@@ -111,6 +112,7 @@ files:
|
|
111
112
|
- lib/pub_sub_model_sync/mock_google_service.rb
|
112
113
|
- lib/pub_sub_model_sync/mock_kafka_service.rb
|
113
114
|
- lib/pub_sub_model_sync/mock_rabbit_service.rb
|
115
|
+
- lib/pub_sub_model_sync/payload.rb
|
114
116
|
- lib/pub_sub_model_sync/publisher.rb
|
115
117
|
- lib/pub_sub_model_sync/publisher_concern.rb
|
116
118
|
- lib/pub_sub_model_sync/railtie.rb
|