pub_sub_model_sync 0.1.3 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +22 -0
- data/Gemfile +1 -0
- data/Gemfile.lock +7 -3
- data/README.md +69 -23
- data/lib/pub_sub_model_sync.rb +3 -0
- data/lib/pub_sub_model_sync/config.rb +6 -2
- data/lib/pub_sub_model_sync/connector.rb +2 -0
- data/lib/pub_sub_model_sync/message_processor.rb +0 -6
- data/lib/pub_sub_model_sync/mock_kafka_service.rb +45 -0
- data/lib/pub_sub_model_sync/publisher.rb +18 -6
- data/lib/pub_sub_model_sync/publisher_concern.rb +25 -13
- data/lib/pub_sub_model_sync/service_base.rb +36 -0
- data/lib/pub_sub_model_sync/service_google.rb +6 -7
- data/lib/pub_sub_model_sync/service_kafka.rb +77 -0
- data/lib/pub_sub_model_sync/service_rabbit.rb +7 -13
- data/lib/pub_sub_model_sync/subscriber_concern.rb +11 -8
- data/lib/pub_sub_model_sync/version.rb +1 -1
- metadata +5 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0ba16c2e3129caa85cf184030955e6a1de130b23b9b552a620c1354234eb5376
|
4
|
+
data.tar.gz: a4b3eb7514efe123a38fb46ed933bf9ac0b1f418f1675d73b13e2cee25e6fe03
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b55d2b5f1bdd763061e0c2ea60f9fe080bdb1db26e5cdb20adf33973524612d2041b0c4dab1ec1afc90e4f7a91ea8327b41ef8c9ecf53ba8bafe543b92dd74e9
|
7
|
+
data.tar.gz: 218f2d41a1fcb04d240a4c3e91daa58b989555f9a97dc488fbc0bd627c0db412e3de224afc31a02c73bfa123f8817705811da1f3e7d20d240036f00de4ccb11d
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,27 @@
|
|
1
1
|
# Change Log
|
2
2
|
|
3
|
+
# 0.2.2 (March 27, 2020)
|
4
|
+
- fix default value for cattr_accessor in ror < 5.2
|
5
|
+
- add callbacks when publishing a message
|
6
|
+
|
7
|
+
# 0.2.1
|
8
|
+
- Add on demand model sync method
|
9
|
+
|
10
|
+
# 0.2.0
|
11
|
+
- Add apache kafka support
|
12
|
+
- Add Service interface for future references
|
13
|
+
- Improve Services to use a single/common message performer
|
14
|
+
|
15
|
+
# 0.1.4
|
16
|
+
- Add attribute aliases when publishing, ```ps_publish(['name:full_name', 'email'])```
|
17
|
+
- Ability to retrieve publisher/subscriber crud settings
|
18
|
+
|
19
|
+
# 0.1.3
|
20
|
+
- shorter publisher/subscriber methods: ps_msync_subscribe into ps_subscribe
|
21
|
+
|
22
|
+
# 0.1.2
|
23
|
+
- fix not found pub/sub library (buggy)
|
24
|
+
|
3
25
|
# 0.1.1
|
4
26
|
- Add rabbitmq pub/sub service support
|
5
27
|
- Reformat to support multiple pub/sub services
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
pub_sub_model_sync (0.
|
4
|
+
pub_sub_model_sync (0.2.2)
|
5
5
|
activesupport
|
6
6
|
rails
|
7
7
|
|
@@ -60,6 +60,7 @@ GEM
|
|
60
60
|
concurrent-ruby (1.1.6)
|
61
61
|
crass (1.0.6)
|
62
62
|
diff-lcs (1.3)
|
63
|
+
digest-crc (0.5.1)
|
63
64
|
erubi (1.9.0)
|
64
65
|
faraday (0.17.3)
|
65
66
|
multipart-post (>= 1.2, < 3)
|
@@ -81,7 +82,7 @@ GEM
|
|
81
82
|
googleauth (>= 0.6.2, < 0.10.0)
|
82
83
|
grpc (>= 1.7.2, < 2.0)
|
83
84
|
rly (~> 0.2.3)
|
84
|
-
google-protobuf (3.11.4
|
85
|
+
google-protobuf (3.11.4)
|
85
86
|
googleapis-common-protos (1.3.9)
|
86
87
|
google-protobuf (~> 3.0)
|
87
88
|
googleapis-common-protos-types (~> 1.0)
|
@@ -95,7 +96,7 @@ GEM
|
|
95
96
|
multi_json (~> 1.11)
|
96
97
|
os (>= 0.9, < 2.0)
|
97
98
|
signet (~> 0.7)
|
98
|
-
grpc (1.27.0
|
99
|
+
grpc (1.27.0)
|
99
100
|
google-protobuf (~> 3.11)
|
100
101
|
googleapis-common-protos-types (~> 1.0)
|
101
102
|
grpc-google-iam-v1 (0.6.9)
|
@@ -180,6 +181,8 @@ GEM
|
|
180
181
|
rexml
|
181
182
|
ruby-progressbar (~> 1.7)
|
182
183
|
unicode-display_width (>= 1.4.0, < 1.7)
|
184
|
+
ruby-kafka (1.0.0)
|
185
|
+
digest-crc
|
183
186
|
ruby-progressbar (1.10.1)
|
184
187
|
signet (0.11.0)
|
185
188
|
addressable (~> 2.3)
|
@@ -214,6 +217,7 @@ DEPENDENCIES
|
|
214
217
|
rake
|
215
218
|
rspec
|
216
219
|
rubocop
|
220
|
+
ruby-kafka
|
217
221
|
sqlite3
|
218
222
|
|
219
223
|
BUNDLED WITH
|
data/README.md
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
# PubSubModelSync
|
2
|
-
Permit to sync models data and make calls between rails apps using google or rabbitmq pub/sub service.
|
2
|
+
Permit to sync models data and make calls between rails apps using google or rabbitmq or apache kafka pub/sub service.
|
3
3
|
|
4
4
|
Note: This gem is based on [MultipleMan](https://github.com/influitive/multiple_man) which for now looks unmaintained.
|
5
5
|
|
@@ -14,8 +14,10 @@ Note: This gem is based on [MultipleMan](https://github.com/influitive/multiple_
|
|
14
14
|
Add this line to your application's Gemfile:
|
15
15
|
```ruby
|
16
16
|
gem 'pub_sub_model_sync'
|
17
|
+
|
17
18
|
gem 'google-cloud-pubsub' # to use google pub/sub service
|
18
19
|
gem 'bunny' # to use rabbit-mq pub/sub service
|
20
|
+
gem 'ruby-kafka' # to use apache kafka pub/sub service
|
19
21
|
```
|
20
22
|
And then execute: $ bundle install
|
21
23
|
|
@@ -43,9 +45,17 @@ And then execute: $ bundle install
|
|
43
45
|
```
|
44
46
|
See details here: https://github.com/ruby-amqp/bunny
|
45
47
|
|
48
|
+
- configuration for Apache Kafka (You need kafka installed)
|
49
|
+
```ruby
|
50
|
+
PubSubModelSync::Config.service_name = :kafka
|
51
|
+
PubSubModelSync::Config.kafka_connection = [["kafka1:9092", "localhost:2121"], logger: Rails.logger]
|
52
|
+
PubSubModelSync::Config.topic_name = 'sample-topic'
|
53
|
+
```
|
54
|
+
See details here: https://github.com/zendesk/ruby-kafka
|
55
|
+
|
46
56
|
- Add publishers/subscribers to your models (See examples below)
|
47
57
|
|
48
|
-
- Start
|
58
|
+
- Start subscribers to listen for publishers (Only in the app that has subscribers)
|
49
59
|
```ruby
|
50
60
|
rake pub_sub_model_sync:start
|
51
61
|
```
|
@@ -53,14 +63,14 @@ And then execute: $ bundle install
|
|
53
63
|
|
54
64
|
## Examples
|
55
65
|
```ruby
|
56
|
-
# App 1
|
66
|
+
# App 1 (Publisher)
|
57
67
|
# attributes: name email age
|
58
68
|
class User < ActiveRecord::Base
|
59
69
|
include PubSubModelSync::PublisherConcern
|
60
70
|
ps_publish(%i[name email])
|
61
71
|
end
|
62
72
|
|
63
|
-
# App 2
|
73
|
+
# App 2 (Subscriber)
|
64
74
|
class User < ActiveRecord::Base
|
65
75
|
include PubSubModelSync::SubscriberConcern
|
66
76
|
ps_subscribe(%i[name])
|
@@ -72,24 +82,27 @@ class User < ActiveRecord::Base
|
|
72
82
|
end
|
73
83
|
|
74
84
|
# Samples
|
75
|
-
User.create(name: 'test user') # Review your App 2 to see the created user (only name will be saved)
|
85
|
+
User.create(name: 'test user', email: 'sample@gmail.com') # Review your App 2 to see the created user (only name will be saved)
|
86
|
+
User.new(name: 'test user').ps_perform_sync(:create) # similar to above to perform sync on demand
|
87
|
+
|
76
88
|
User.ps_class_publish({ msg: 'Hello' }, action: :greeting) # User.greeting method (Class method) will be called in App2
|
89
|
+
PubSubModelSync::Publisher.new.publish_data(User, { msg: 'Hello' }, :greeting) # similar to above when not included publisher concern
|
77
90
|
```
|
78
91
|
|
79
92
|
## Advanced Example
|
80
93
|
```ruby
|
81
|
-
# App 1
|
94
|
+
# App 1 (Publisher)
|
82
95
|
class User < ActiveRecord::Base
|
83
96
|
self.table_name = 'publisher_users'
|
84
97
|
include PubSubModelSync::PublisherConcern
|
85
|
-
ps_publish(%i[name], actions: %i[update], as_klass: 'Client', id: :client_id)
|
98
|
+
ps_publish(%i[name:full_name email], actions: %i[update], as_klass: 'Client', id: :client_id)
|
86
99
|
|
87
100
|
def ps_skip_for?(_action)
|
88
101
|
false # here logic with action to skip push message
|
89
102
|
end
|
90
103
|
end
|
91
104
|
|
92
|
-
# App 2
|
105
|
+
# App 2 (Subscriber)
|
93
106
|
class User < ActiveRecord::Base
|
94
107
|
self.table_name = 'subscriber_users'
|
95
108
|
include PubSubModelSync::SubscriberConcern
|
@@ -102,22 +115,64 @@ class User < ActiveRecord::Base
|
|
102
115
|
end
|
103
116
|
```
|
104
117
|
|
105
|
-
##
|
106
|
-
-
|
118
|
+
## API
|
119
|
+
- To perform a callback before publishing message (CRUD)
|
120
|
+
```model.ps_before_sync(action, data_to_deliver)```
|
121
|
+
Note: If the method returns ```false```, the message will not be published
|
122
|
+
|
123
|
+
- To perform a callback after publishing message (CRUD)
|
124
|
+
```model.ps_after_sync(action, data_to_deliver)```
|
125
|
+
Note: If the method returns ```false```, the message will not be published
|
126
|
+
|
127
|
+
- Perform sync on demand (:create, :update, :destroy):
|
128
|
+
The target model will receive a notification to perform the indicated action
|
129
|
+
```my_model.ps_perform_sync(action_name)```
|
130
|
+
|
131
|
+
- Class level notification:
|
132
|
+
```User.ps_class_publish(data, action: action_name, as_klass: custom_klass_name)```
|
133
|
+
Target class ```User.action_name``` will be called when message is received
|
134
|
+
* data: (required, :hash) message value to deliver
|
135
|
+
* action_name: (required, :sim) same action name as defined in ps_class_subscribe(...)
|
136
|
+
* as_klass: (optional, :string) same class name as defined in ps_class_subscribe(...)
|
137
|
+
|
138
|
+
- Class level notification (Same as above: on demand call)
|
139
|
+
```PubSubModelSync::Publisher.new.publish_data(Klass_name, data, action_name)```
|
140
|
+
* klass_name: (required, Class) same class name as defined in ps_class_subscribe(...)
|
141
|
+
* data: (required, :hash) message value to deliver
|
142
|
+
* action_name: (required, :sim) same action name as defined in ps_class_subscribe(...)
|
143
|
+
|
144
|
+
- Get crud subscription configured for the class
|
145
|
+
```User.ps_subscriber(action_name)```
|
146
|
+
* action_name (default :create, :sym): can be :create, :update, :destroy
|
147
|
+
- Get crud publisher configured for the class
|
148
|
+
```User.ps_publisher(action_name)```
|
149
|
+
* action_name (default :create, :sym): can be :create, :update, :destroy
|
150
|
+
- Inspect all configured listeners
|
151
|
+
```PubSubModelSync::Config.listeners```
|
152
|
+
|
153
|
+
## Testing with RSpec
|
154
|
+
- Config: (spec/rails_helper.rb)
|
107
155
|
```ruby
|
108
156
|
|
109
157
|
# when using google service
|
110
158
|
require 'pub_sub_model_sync/mock_google_service'
|
111
159
|
config.before(:each) do
|
112
|
-
|
113
|
-
allow(Google::Cloud::Pubsub).to receive(:new).and_return(
|
160
|
+
google_mock = PubSubModelSync::MockGoogleService.new
|
161
|
+
allow(Google::Cloud::Pubsub).to receive(:new).and_return(google_mock)
|
114
162
|
end
|
115
163
|
|
116
164
|
# when using rabbitmq service
|
117
165
|
require 'pub_sub_model_sync/mock_rabbit_service'
|
118
166
|
config.before(:each) do
|
119
|
-
|
120
|
-
allow(Bunny).to receive(:new).and_return(
|
167
|
+
rabbit_mock = PubSubModelSync::MockRabbitService.new
|
168
|
+
allow(Bunny).to receive(:new).and_return(rabbit_mock)
|
169
|
+
end
|
170
|
+
|
171
|
+
# when using apache kafka service
|
172
|
+
require 'pub_sub_model_sync/mock_kafka_service'
|
173
|
+
config.before(:each) do
|
174
|
+
kafka_mock = PubSubModelSync::MockKafkaService.new
|
175
|
+
allow(Kafka).to receive(:new).and_return(kafka_mock)
|
121
176
|
end
|
122
177
|
|
123
178
|
```
|
@@ -160,15 +215,6 @@ end
|
|
160
215
|
expect_any_instance_of(publisher).to receive(:publish_data).with('User', data, action)
|
161
216
|
end
|
162
217
|
```
|
163
|
-
|
164
|
-
There are two special methods to extract crud configuration settings (attrs, id, ...):
|
165
|
-
|
166
|
-
Subscribers: ```User.ps_subscriber_settings```
|
167
|
-
|
168
|
-
Publishers: ```User.ps_publisher_settings```
|
169
|
-
|
170
|
-
Note: Inspect all configured listeners with:
|
171
|
-
``` PubSubModelSync::Config.listeners ```
|
172
218
|
|
173
219
|
## Contributing
|
174
220
|
|
data/lib/pub_sub_model_sync.rb
CHANGED
@@ -11,8 +11,11 @@ require 'pub_sub_model_sync/publisher_concern'
|
|
11
11
|
require 'pub_sub_model_sync/runner'
|
12
12
|
require 'pub_sub_model_sync/connector'
|
13
13
|
require 'pub_sub_model_sync/message_processor'
|
14
|
+
|
15
|
+
require 'pub_sub_model_sync/service_base'
|
14
16
|
require 'pub_sub_model_sync/service_google'
|
15
17
|
require 'pub_sub_model_sync/service_rabbit'
|
18
|
+
require 'pub_sub_model_sync/service_kafka'
|
16
19
|
|
17
20
|
module PubSubModelSync
|
18
21
|
class Error < StandardError; end
|
@@ -2,8 +2,9 @@
|
|
2
2
|
|
3
3
|
module PubSubModelSync
|
4
4
|
class Config
|
5
|
-
cattr_accessor
|
6
|
-
cattr_accessor
|
5
|
+
cattr_accessor(:listeners) { [] }
|
6
|
+
cattr_accessor(:publishers) { [] }
|
7
|
+
cattr_accessor(:service_name) { :google }
|
7
8
|
cattr_accessor :logger
|
8
9
|
|
9
10
|
# google service
|
@@ -12,6 +13,9 @@ module PubSubModelSync
|
|
12
13
|
# rabbitmq service
|
13
14
|
cattr_accessor :bunny_connection, :queue_name, :topic_name
|
14
15
|
|
16
|
+
# kafka service
|
17
|
+
cattr_accessor :kafka_connection, :topic_name
|
18
|
+
|
15
19
|
def self.log(msg, kind = :info)
|
16
20
|
msg = "PS_MSYNC ==> #{msg}"
|
17
21
|
if logger == :raise_error
|
@@ -40,7 +40,6 @@ module PubSubModelSync
|
|
40
40
|
|
41
41
|
# support for: create, update, destroy
|
42
42
|
def call_listener(listener)
|
43
|
-
listener_add_crud_settings(listener)
|
44
43
|
model = find_model(listener)
|
45
44
|
if attrs[:action].to_sym == :destroy
|
46
45
|
model.destroy!
|
@@ -74,11 +73,6 @@ module PubSubModelSync
|
|
74
73
|
end
|
75
74
|
end
|
76
75
|
|
77
|
-
def listener_add_crud_settings(listener)
|
78
|
-
model_class = listener[:klass].constantize
|
79
|
-
listener[:settings] = model_class.ps_subscriber_settings
|
80
|
-
end
|
81
|
-
|
82
76
|
def log(message, kind = :info)
|
83
77
|
PubSubModelSync::Config.log "#{message} ==> #{[data, attrs]}", kind
|
84
78
|
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module PubSubModelSync
|
4
|
+
class MockKafkaService
|
5
|
+
class MockProducer
|
6
|
+
def produce(*_args)
|
7
|
+
true
|
8
|
+
end
|
9
|
+
|
10
|
+
def deliver_messages(*_args)
|
11
|
+
true
|
12
|
+
end
|
13
|
+
|
14
|
+
def shutdown
|
15
|
+
true
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
class MockConsumer
|
20
|
+
def each_message(*_args)
|
21
|
+
true
|
22
|
+
end
|
23
|
+
|
24
|
+
def stop(*_args)
|
25
|
+
true
|
26
|
+
end
|
27
|
+
|
28
|
+
def subscribe(*_args)
|
29
|
+
true
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def producer(*_args)
|
34
|
+
MockProducer.new
|
35
|
+
end
|
36
|
+
|
37
|
+
def consumer(*_args)
|
38
|
+
MockConsumer.new
|
39
|
+
end
|
40
|
+
|
41
|
+
def close
|
42
|
+
true
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -14,26 +14,38 @@ module PubSubModelSync
|
|
14
14
|
|
15
15
|
# @param settings (Hash): { attrs: [], as_klass: nil, id: nil }
|
16
16
|
def publish_model(model, action, settings = nil)
|
17
|
-
settings ||= model.class.
|
17
|
+
settings ||= model.class.ps_publisher_info(action)
|
18
18
|
attributes = build_model_attrs(model, action, settings)
|
19
19
|
data = {}
|
20
|
-
if action !=
|
21
|
-
|
22
|
-
|
20
|
+
data = build_model_data(model, settings[:attrs]) if action != :destroy
|
21
|
+
res_before = model.ps_before_sync(action, data)
|
22
|
+
return if res_before == false
|
23
|
+
|
23
24
|
connector.publish(data.symbolize_keys, attributes)
|
25
|
+
model.ps_after_sync(action, data)
|
24
26
|
end
|
25
27
|
|
26
28
|
def self.build_attrs(klass, action, id = nil)
|
27
29
|
{
|
28
30
|
klass: klass.to_s,
|
29
31
|
action: action.to_sym,
|
30
|
-
id: id
|
31
|
-
service_model_sync: true
|
32
|
+
id: id
|
32
33
|
}
|
33
34
|
end
|
34
35
|
|
35
36
|
private
|
36
37
|
|
38
|
+
def build_model_data(model, model_props)
|
39
|
+
source_props = model_props.map { |prop| prop.to_s.split(':').first }
|
40
|
+
data = model.as_json(only: source_props, methods: source_props)
|
41
|
+
aliased_props = model_props.select { |prop| prop.to_s.include?(':') }
|
42
|
+
aliased_props.each do |prop|
|
43
|
+
source, target = prop.to_s.split(':')
|
44
|
+
data[target] = data.delete(source)
|
45
|
+
end
|
46
|
+
data.symbolize_keys
|
47
|
+
end
|
48
|
+
|
37
49
|
def build_model_attrs(model, action, settings)
|
38
50
|
as_klass = (settings[:as_klass] || model.class.name).to_s
|
39
51
|
id_val = model.send(settings[:id] || :id)
|
@@ -11,37 +11,49 @@ module PubSubModelSync
|
|
11
11
|
false
|
12
12
|
end
|
13
13
|
|
14
|
+
def ps_before_sync(_action, _data); end
|
15
|
+
|
16
|
+
def ps_after_sync(_action, _data); end
|
17
|
+
|
18
|
+
def ps_perform_sync(action = :create)
|
19
|
+
service = self.class.ps_publisher_service
|
20
|
+
service.publish_model(self, action, self.class.ps_publisher_info(action))
|
21
|
+
end
|
22
|
+
|
14
23
|
module ClassMethods
|
15
24
|
# Permit to publish crud actions (:create, :update, :destroy)
|
16
25
|
# @param settings (Hash): { actions: nil, as_klass: nil, id: nil }
|
17
26
|
def ps_publish(attrs, settings = {})
|
18
27
|
actions = settings.delete(:actions) || %i[create update destroy]
|
19
|
-
|
20
|
-
|
28
|
+
actions.each do |action|
|
29
|
+
info = settings.merge(klass: name, action: action, attrs: attrs)
|
30
|
+
PubSubModelSync::Config.publishers << info
|
31
|
+
ps_register_callback(action.to_sym, info)
|
32
|
+
end
|
21
33
|
end
|
22
34
|
|
23
|
-
def
|
24
|
-
|
35
|
+
def ps_publisher_info(action = :create)
|
36
|
+
PubSubModelSync::Config.publishers.select do |listener|
|
37
|
+
listener[:klass] == name && listener[:action] == action
|
38
|
+
end.last
|
25
39
|
end
|
26
40
|
|
27
41
|
def ps_class_publish(data, action:, as_klass: nil)
|
28
42
|
as_klass = (as_klass || name).to_s
|
29
|
-
|
43
|
+
ps_publisher_service.publish_data(as_klass, data, action.to_sym)
|
30
44
|
end
|
31
45
|
|
32
|
-
def
|
46
|
+
def ps_publisher_service
|
33
47
|
PubSubModelSync::Publisher.new
|
34
48
|
end
|
35
49
|
|
36
50
|
private
|
37
51
|
|
38
|
-
def
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
publisher.publish_model(model, action.to_sym)
|
44
|
-
end
|
52
|
+
def ps_register_callback(action, info)
|
53
|
+
after_commit(on: action) do |model|
|
54
|
+
unless model.ps_skip_for?(action)
|
55
|
+
service = model.class.ps_publisher_service
|
56
|
+
service.publish_model(model, action.to_sym, info)
|
45
57
|
end
|
46
58
|
end
|
47
59
|
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module PubSubModelSync
|
4
|
+
class ServiceBase
|
5
|
+
SERVICE_KEY = 'service_model_sync'
|
6
|
+
|
7
|
+
def listen_messages
|
8
|
+
raise 'method :listen_messages must be defined in service'
|
9
|
+
end
|
10
|
+
|
11
|
+
def publish(_data, _attributes)
|
12
|
+
raise 'method :publish must be defined in service'
|
13
|
+
end
|
14
|
+
|
15
|
+
def stop
|
16
|
+
raise 'method :stop must be defined in service'
|
17
|
+
end
|
18
|
+
|
19
|
+
private
|
20
|
+
|
21
|
+
# @param payload (String JSON): '{"data":{},"attributes":{..}}'
|
22
|
+
# refer: PubSubModelSync::Publisher (.publish_model | .publish_data)
|
23
|
+
def perform_message(payload)
|
24
|
+
data, attrs = parse_message_payload(payload)
|
25
|
+
args = [data, attrs[:klass], attrs[:action], attrs]
|
26
|
+
PubSubModelSync::MessageProcessor.new(*args).process
|
27
|
+
end
|
28
|
+
|
29
|
+
def parse_message_payload(payload)
|
30
|
+
message_payload = JSON.parse(payload).symbolize_keys
|
31
|
+
data = message_payload[:data].symbolize_keys
|
32
|
+
attrs = message_payload[:attributes].symbolize_keys
|
33
|
+
[data, attrs]
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -6,7 +6,7 @@ rescue LoadError # rubocop:disable Lint/SuppressedException
|
|
6
6
|
end
|
7
7
|
|
8
8
|
module PubSubModelSync
|
9
|
-
class ServiceGoogle
|
9
|
+
class ServiceGoogle < ServiceBase
|
10
10
|
attr_accessor :service, :topic, :subscription, :config, :subscriber
|
11
11
|
|
12
12
|
def initialize
|
@@ -30,7 +30,9 @@ module PubSubModelSync
|
|
30
30
|
|
31
31
|
def publish(data, attributes)
|
32
32
|
log("Publishing message: #{[data, attributes]}")
|
33
|
-
|
33
|
+
|
34
|
+
payload = { data: data, attributes: attributes }.to_json
|
35
|
+
topic.publish(payload, { SERVICE_KEY => true })
|
34
36
|
end
|
35
37
|
|
36
38
|
def stop
|
@@ -47,12 +49,9 @@ module PubSubModelSync
|
|
47
49
|
|
48
50
|
def process_message(received_message)
|
49
51
|
message = received_message.message
|
50
|
-
|
51
|
-
return unless attrs[:service_model_sync]
|
52
|
+
return unless message.attributes[SERVICE_KEY]
|
52
53
|
|
53
|
-
|
54
|
-
args = [data, attrs[:klass], attrs[:action], attrs]
|
55
|
-
PubSubModelSync::MessageProcessor.new(*args).process
|
54
|
+
perform_message(message.data)
|
56
55
|
rescue => e
|
57
56
|
log("Error processing message: #{[received_message, e.message]}", :error)
|
58
57
|
ensure
|
@@ -0,0 +1,77 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
begin
|
4
|
+
require 'kafka'
|
5
|
+
rescue LoadError # rubocop:disable Lint/SuppressedException
|
6
|
+
end
|
7
|
+
|
8
|
+
module PubSubModelSync
|
9
|
+
class ServiceKafka < ServiceBase
|
10
|
+
cattr_accessor :producer
|
11
|
+
|
12
|
+
attr_accessor :service, :consumer
|
13
|
+
attr_accessor :config
|
14
|
+
CONSUMER_GROUP = 'service_model_sync'
|
15
|
+
|
16
|
+
def initialize
|
17
|
+
@config = PubSubModelSync::Config
|
18
|
+
@service = Kafka.new(*config.kafka_connection)
|
19
|
+
end
|
20
|
+
|
21
|
+
def listen_messages
|
22
|
+
log('Listener starting...')
|
23
|
+
start_consumer
|
24
|
+
consumer.each_message(&method(:process_message))
|
25
|
+
rescue PubSubModelSync::Runner::ShutDown
|
26
|
+
raise
|
27
|
+
rescue => e
|
28
|
+
log("Error listening message: #{[e.message, e.backtrace]}", :error)
|
29
|
+
end
|
30
|
+
|
31
|
+
def publish(data, attributes)
|
32
|
+
log("Publishing: #{[data, attributes]}")
|
33
|
+
payload = { data: data, attributes: attributes }
|
34
|
+
producer.produce(payload.to_json, message_settings)
|
35
|
+
producer.deliver_messages
|
36
|
+
rescue => e
|
37
|
+
info = [data, attributes, e.message, e.backtrace]
|
38
|
+
log("Error publishing: #{info}", :error)
|
39
|
+
end
|
40
|
+
|
41
|
+
def stop
|
42
|
+
log('Listener stopping...')
|
43
|
+
consumer.stop
|
44
|
+
end
|
45
|
+
|
46
|
+
private
|
47
|
+
|
48
|
+
def message_settings
|
49
|
+
{ topic: config.topic_name, headers: { SERVICE_KEY => true } }
|
50
|
+
end
|
51
|
+
|
52
|
+
def start_consumer
|
53
|
+
@consumer = service.consumer(group_id: CONSUMER_GROUP)
|
54
|
+
consumer.subscribe(config.topic_name)
|
55
|
+
end
|
56
|
+
|
57
|
+
def producer
|
58
|
+
return self.class.producer if self.class.producer
|
59
|
+
|
60
|
+
at_exit { self.class.producer.shutdown }
|
61
|
+
self.class.producer = service.producer
|
62
|
+
end
|
63
|
+
|
64
|
+
def process_message(message)
|
65
|
+
return unless message.headers[SERVICE_KEY]
|
66
|
+
|
67
|
+
perform_message(message.value)
|
68
|
+
rescue => e
|
69
|
+
error = [message, e.message, e.backtrace]
|
70
|
+
log("Error processing message: #{error}", :error)
|
71
|
+
end
|
72
|
+
|
73
|
+
def log(msg, kind = :info)
|
74
|
+
config.log("Kafka Service ==> #{msg}", kind)
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
@@ -6,10 +6,9 @@ rescue LoadError # rubocop:disable Lint/SuppressedException
|
|
6
6
|
end
|
7
7
|
|
8
8
|
module PubSubModelSync
|
9
|
-
class ServiceRabbit
|
9
|
+
class ServiceRabbit < ServiceBase
|
10
10
|
attr_accessor :service, :channel, :queue, :topic
|
11
11
|
attr_accessor :config
|
12
|
-
SERVICE_KEY = 'service_model_sync'
|
13
12
|
|
14
13
|
def initialize
|
15
14
|
@config = PubSubModelSync::Config
|
@@ -31,7 +30,7 @@ module PubSubModelSync
|
|
31
30
|
log("Publishing: #{[data, attributes]}")
|
32
31
|
subscribe_to_queue
|
33
32
|
payload = { data: data, attributes: attributes }
|
34
|
-
topic.publish(payload.to_json,
|
33
|
+
topic.publish(payload.to_json, message_settings)
|
35
34
|
rescue => e
|
36
35
|
info = [data, attributes, e.message, e.backtrace]
|
37
36
|
log("Error publishing: #{info}", :error)
|
@@ -44,24 +43,19 @@ module PubSubModelSync
|
|
44
43
|
|
45
44
|
private
|
46
45
|
|
46
|
+
def message_settings
|
47
|
+
{ routing_key: queue.name, type: SERVICE_KEY }
|
48
|
+
end
|
49
|
+
|
47
50
|
def process_message(_delivery_info, meta_info, payload)
|
48
51
|
return unless meta_info[:type] == SERVICE_KEY
|
49
52
|
|
50
|
-
|
51
|
-
args = [data, attrs[:klass], attrs[:action], attrs]
|
52
|
-
PubSubModelSync::MessageProcessor.new(*args).process
|
53
|
+
perform_message(payload)
|
53
54
|
rescue => e
|
54
55
|
error = [payload, e.message, e.backtrace]
|
55
56
|
log("Error processing message: #{error}", :error)
|
56
57
|
end
|
57
58
|
|
58
|
-
def parse_message_payload(payload)
|
59
|
-
message_payload = JSON.parse(payload).symbolize_keys
|
60
|
-
data = message_payload[:data].symbolize_keys
|
61
|
-
attrs = message_payload[:attributes].symbolize_keys
|
62
|
-
[data, attrs]
|
63
|
-
end
|
64
|
-
|
65
59
|
def subscribe_to_queue
|
66
60
|
service.start
|
67
61
|
@channel = service.create_channel
|
@@ -9,31 +9,34 @@ module PubSubModelSync
|
|
9
9
|
module ClassMethods
|
10
10
|
# @param settings (Hash): { as_klass: nil, actions: nil, id: nil }
|
11
11
|
def ps_subscribe(attrs, settings = {})
|
12
|
-
|
12
|
+
as_klass = (settings[:as_klass] || name).to_s
|
13
13
|
actions = settings.delete(:actions) || %i[create update destroy]
|
14
|
-
|
14
|
+
settings = settings.merge(attrs: attrs)
|
15
15
|
actions.each do |action|
|
16
|
-
add_ps_subscriber(
|
16
|
+
add_ps_subscriber(as_klass, action, action, false, settings)
|
17
17
|
end
|
18
18
|
end
|
19
19
|
|
20
20
|
def ps_class_subscribe(action, as_action: nil, as_klass: nil)
|
21
|
-
add_ps_subscriber(as_klass, action, as_action, true)
|
21
|
+
add_ps_subscriber(as_klass, action, as_action, true, {})
|
22
22
|
end
|
23
23
|
|
24
|
-
def
|
25
|
-
|
24
|
+
def ps_subscriber(action = :create)
|
25
|
+
PubSubModelSync::Config.listeners.select do |listener|
|
26
|
+
listener[:klass] == name && listener[:action] == action
|
27
|
+
end.last
|
26
28
|
end
|
27
29
|
|
28
30
|
private
|
29
31
|
|
30
|
-
def add_ps_subscriber(as_klass, action, as_action, direct_mode)
|
32
|
+
def add_ps_subscriber(as_klass, action, as_action, direct_mode, settings)
|
31
33
|
listener = {
|
32
34
|
klass: name,
|
33
35
|
as_klass: (as_klass || name).to_s,
|
34
36
|
action: action.to_sym,
|
35
37
|
as_action: (as_action || action).to_sym,
|
36
|
-
direct_mode: direct_mode
|
38
|
+
direct_mode: direct_mode,
|
39
|
+
settings: settings
|
37
40
|
}
|
38
41
|
PubSubModelSync::Config.listeners << listener
|
39
42
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pub_sub_model_sync
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Owen
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-03-
|
11
|
+
date: 2020-03-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -119,12 +119,15 @@ files:
|
|
119
119
|
- lib/pub_sub_model_sync/connector.rb
|
120
120
|
- lib/pub_sub_model_sync/message_processor.rb
|
121
121
|
- lib/pub_sub_model_sync/mock_google_service.rb
|
122
|
+
- lib/pub_sub_model_sync/mock_kafka_service.rb
|
122
123
|
- lib/pub_sub_model_sync/mock_rabbit_service.rb
|
123
124
|
- lib/pub_sub_model_sync/publisher.rb
|
124
125
|
- lib/pub_sub_model_sync/publisher_concern.rb
|
125
126
|
- lib/pub_sub_model_sync/railtie.rb
|
126
127
|
- lib/pub_sub_model_sync/runner.rb
|
128
|
+
- lib/pub_sub_model_sync/service_base.rb
|
127
129
|
- lib/pub_sub_model_sync/service_google.rb
|
130
|
+
- lib/pub_sub_model_sync/service_kafka.rb
|
128
131
|
- lib/pub_sub_model_sync/service_rabbit.rb
|
129
132
|
- lib/pub_sub_model_sync/subscriber_concern.rb
|
130
133
|
- lib/pub_sub_model_sync/tasks/worker.rake
|