pub_sub_model_sync 1.0.beta1 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/release.yml +43 -0
- data/CHANGELOG.md +15 -4
- data/Gemfile.lock +11 -15
- data/README.md +184 -111
- data/docs/notifications-diagram.png +0 -0
- data/lib/pub_sub_model_sync/base.rb +0 -20
- data/lib/pub_sub_model_sync/config.rb +2 -3
- data/lib/pub_sub_model_sync/message_processor.rb +32 -9
- data/lib/pub_sub_model_sync/message_publisher.rb +18 -14
- data/lib/pub_sub_model_sync/payload.rb +15 -12
- data/lib/pub_sub_model_sync/{publisher.rb → payload_builder.rb} +16 -11
- data/lib/pub_sub_model_sync/publisher_concern.rb +29 -21
- data/lib/pub_sub_model_sync/railtie.rb +1 -1
- data/lib/pub_sub_model_sync/run_subscriber.rb +17 -13
- data/lib/pub_sub_model_sync/runner.rb +3 -5
- data/lib/pub_sub_model_sync/service_base.rb +5 -32
- data/lib/pub_sub_model_sync/service_google.rb +2 -2
- data/lib/pub_sub_model_sync/service_kafka.rb +2 -2
- data/lib/pub_sub_model_sync/service_rabbit.rb +1 -1
- data/lib/pub_sub_model_sync/subscriber_concern.rb +11 -9
- data/lib/pub_sub_model_sync/transaction.rb +12 -6
- data/lib/pub_sub_model_sync/version.rb +1 -1
- data/lib/pub_sub_model_sync.rb +1 -1
- data/samples/README.md +50 -0
- data/samples/app1/Dockerfile +13 -0
- data/samples/app1/Gemfile +37 -0
- data/samples/app1/Gemfile.lock +171 -0
- data/samples/app1/README.md +24 -0
- data/samples/app1/Rakefile +6 -0
- data/samples/app1/app/models/application_record.rb +3 -0
- data/samples/app1/app/models/concerns/.keep +0 -0
- data/samples/app1/app/models/post.rb +19 -0
- data/samples/app1/app/models/user.rb +29 -0
- data/samples/app1/bin/bundle +114 -0
- data/samples/app1/bin/rails +5 -0
- data/samples/app1/bin/rake +5 -0
- data/samples/app1/bin/setup +33 -0
- data/samples/app1/bin/spring +14 -0
- data/samples/app1/config/application.rb +40 -0
- data/samples/app1/config/boot.rb +4 -0
- data/samples/app1/config/credentials.yml.enc +1 -0
- data/samples/app1/config/database.yml +25 -0
- data/samples/app1/config/environment.rb +5 -0
- data/samples/app1/config/environments/development.rb +63 -0
- data/samples/app1/config/environments/production.rb +105 -0
- data/samples/app1/config/environments/test.rb +57 -0
- data/samples/app1/config/initializers/application_controller_renderer.rb +8 -0
- data/samples/app1/config/initializers/backtrace_silencers.rb +8 -0
- data/samples/app1/config/initializers/cors.rb +16 -0
- data/samples/app1/config/initializers/filter_parameter_logging.rb +6 -0
- data/samples/app1/config/initializers/inflections.rb +16 -0
- data/samples/app1/config/initializers/mime_types.rb +4 -0
- data/samples/app1/config/initializers/pubsub.rb +4 -0
- data/samples/app1/config/initializers/wrap_parameters.rb +14 -0
- data/samples/app1/config/locales/en.yml +33 -0
- data/samples/app1/config/master.key +1 -0
- data/samples/app1/config/puma.rb +43 -0
- data/samples/app1/config/routes.rb +3 -0
- data/samples/app1/config/spring.rb +6 -0
- data/samples/app1/config.ru +6 -0
- data/samples/app1/db/migrate/20210513080700_create_users.rb +12 -0
- data/samples/app1/db/migrate/20210513134332_create_posts.rb +11 -0
- data/samples/app1/db/schema.rb +34 -0
- data/samples/app1/db/seeds.rb +7 -0
- data/samples/app1/docker-compose.yml +32 -0
- data/samples/app1/log/.keep +0 -0
- data/samples/app2/Dockerfile +13 -0
- data/samples/app2/Gemfile +37 -0
- data/samples/app2/Gemfile.lock +171 -0
- data/samples/app2/README.md +24 -0
- data/samples/app2/Rakefile +6 -0
- data/samples/app2/app/models/application_record.rb +9 -0
- data/samples/app2/app/models/concerns/.keep +0 -0
- data/samples/app2/app/models/customer.rb +28 -0
- data/samples/app2/app/models/post.rb +10 -0
- data/samples/app2/bin/bundle +114 -0
- data/samples/app2/bin/rails +5 -0
- data/samples/app2/bin/rake +5 -0
- data/samples/app2/bin/setup +33 -0
- data/samples/app2/bin/spring +14 -0
- data/samples/app2/config/application.rb +40 -0
- data/samples/app2/config/boot.rb +4 -0
- data/samples/app2/config/credentials.yml.enc +1 -0
- data/samples/app2/config/database.yml +25 -0
- data/samples/app2/config/environment.rb +5 -0
- data/samples/app2/config/environments/development.rb +63 -0
- data/samples/app2/config/environments/production.rb +105 -0
- data/samples/app2/config/environments/test.rb +57 -0
- data/samples/app2/config/initializers/application_controller_renderer.rb +8 -0
- data/samples/app2/config/initializers/backtrace_silencers.rb +8 -0
- data/samples/app2/config/initializers/cors.rb +16 -0
- data/samples/app2/config/initializers/filter_parameter_logging.rb +6 -0
- data/samples/app2/config/initializers/inflections.rb +16 -0
- data/samples/app2/config/initializers/mime_types.rb +4 -0
- data/samples/app2/config/initializers/pubsub.rb +4 -0
- data/samples/app2/config/initializers/wrap_parameters.rb +14 -0
- data/samples/app2/config/locales/en.yml +33 -0
- data/samples/app2/config/master.key +1 -0
- data/samples/app2/config/puma.rb +43 -0
- data/samples/app2/config/routes.rb +3 -0
- data/samples/app2/config/spring.rb +6 -0
- data/samples/app2/config.ru +6 -0
- data/samples/app2/db/development.sqlite3 +0 -0
- data/samples/app2/db/migrate/20210513080956_create_customers.rb +10 -0
- data/samples/app2/db/migrate/20210513135203_create_posts.rb +10 -0
- data/samples/app2/db/schema.rb +31 -0
- data/samples/app2/db/seeds.rb +7 -0
- data/samples/app2/docker-compose.yml +20 -0
- data/samples/app2/log/.keep +0 -0
- metadata +92 -6
- data/lib/pub_sub_model_sync/initializers/before_commit.rb +0 -23
@@ -31,7 +31,7 @@ module PubSubModelSync
|
|
31
31
|
end
|
32
32
|
|
33
33
|
# Starts a new transaction
|
34
|
-
# @param key (
|
34
|
+
# @param key (String|Nil)
|
35
35
|
# @return (Transaction)
|
36
36
|
def init_transaction(key, settings = {})
|
37
37
|
new_transaction = PubSubModelSync::Transaction.new(key, settings)
|
@@ -47,20 +47,20 @@ module PubSubModelSync
|
|
47
47
|
# @refer PublisherConcern.ps_class_publish
|
48
48
|
# @return Payload
|
49
49
|
def publish_data(klass, data, action, headers: {})
|
50
|
-
|
51
|
-
payload
|
50
|
+
info = { klass: klass.to_s, action: action.to_sym, mode: :klass }
|
51
|
+
log("Building payload for: #{info.inspect}") if config.debug
|
52
|
+
payload = PubSubModelSync::Payload.new(data, info, headers)
|
53
|
+
define_transaction_key(payload)
|
52
54
|
publish(payload)
|
53
55
|
end
|
54
56
|
|
55
|
-
# @param model (ActiveRecord::Base)
|
56
|
-
# @param action (Symbol
|
57
|
-
# @param settings (Hash
|
57
|
+
# @param model (ActiveRecord::Base,PubSubModelSync::PublisherConcern)
|
58
|
+
# @param action (Symbol,String @see PublishConcern::ps_publish)
|
59
|
+
# @param settings (Hash @see PayloadBuilder.settings)
|
58
60
|
def publish_model(model, action, settings = {})
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
payload = publisher.payload
|
63
|
-
|
61
|
+
log("Building payload for: #{[model, action].inspect}") if config.debug
|
62
|
+
payload = PubSubModelSync::PayloadBuilder.new(model, action, settings).call
|
63
|
+
define_transaction_key(payload)
|
64
64
|
transaction(payload.headers[:ordering_key]) do # catch and group all :ps_before_publish syncs
|
65
65
|
publish(payload) { model.ps_after_publish(action, payload) } if ensure_model_publish(model, action, payload)
|
66
66
|
end
|
@@ -80,6 +80,7 @@ module PubSubModelSync
|
|
80
80
|
end
|
81
81
|
|
82
82
|
def connector_publish(payload)
|
83
|
+
log("Publishing message #{[payload]}...") if config.debug
|
83
84
|
connector.publish(payload)
|
84
85
|
log("Published message: #{[payload]}")
|
85
86
|
config.on_after_publish.call(payload)
|
@@ -98,19 +99,18 @@ module PubSubModelSync
|
|
98
99
|
|
99
100
|
def ensure_publish(payload)
|
100
101
|
cancelled = config.on_before_publish.call(payload) == :cancel
|
101
|
-
log("Publish cancelled by config.on_before_publish: #{payload}") if config.debug && cancelled
|
102
|
+
log("Publish cancelled by config.on_before_publish: #{[payload]}") if config.debug && cancelled
|
102
103
|
!cancelled
|
103
104
|
end
|
104
105
|
|
105
106
|
def ordering_key_for(payload)
|
106
|
-
current_transaction&.key ||= payload.headers[:ordering_key]
|
107
107
|
payload.headers[:forced_ordering_key] || current_transaction&.key || payload.headers[:ordering_key]
|
108
108
|
end
|
109
109
|
|
110
110
|
def ensure_model_publish(model, action, payload)
|
111
111
|
res_before = model.ps_before_publish(action, payload)
|
112
112
|
cancelled = res_before == :cancel
|
113
|
-
log("Publish cancelled by model.ps_before_publish: #{payload}") if config.debug && cancelled
|
113
|
+
log("Publish cancelled by model.ps_before_publish: #{[payload]}") if config.debug && cancelled
|
114
114
|
!cancelled
|
115
115
|
end
|
116
116
|
|
@@ -119,6 +119,10 @@ module PubSubModelSync
|
|
119
119
|
res = config.on_error_publish.call(exception, { payload: payload })
|
120
120
|
log("Error publishing: #{info}", :error) if res != :skip_log
|
121
121
|
end
|
122
|
+
|
123
|
+
def define_transaction_key(payload)
|
124
|
+
current_transaction&.key ||= payload.headers[:ordering_key]
|
125
|
+
end
|
122
126
|
end
|
123
127
|
end
|
124
128
|
end
|
@@ -6,30 +6,33 @@ module PubSubModelSync
|
|
6
6
|
attr_reader :data, :info, :headers
|
7
7
|
|
8
8
|
# @param data (Hash: { any value }):
|
9
|
-
# @param info (Hash
|
9
|
+
# @param info (Hash):
|
10
|
+
# klass: (String, required) Notification class name
|
11
|
+
# action: (Symbol, required) Notification action name
|
12
|
+
# mode: (:model|:klass, default :model): :model for instance and :klass for class notifications
|
10
13
|
# @param headers (Hash):
|
11
14
|
# key (String): identifier of the payload, default:
|
12
|
-
# klass/action
|
13
|
-
# klass/action/model.id
|
15
|
+
# <klass/action>: when class message
|
16
|
+
# <klass/action/model.id>: when model message
|
14
17
|
# ordering_key (String): messages with the same key are processed in the same order they
|
15
18
|
# were delivered, default:
|
16
|
-
# klass
|
17
|
-
# klass/id
|
19
|
+
# <klass>: when class message
|
20
|
+
# <klass/id>: when model message
|
18
21
|
# topic_name (String|Array<String>): Specific topic name to be used when delivering the
|
19
22
|
# message (default first topic)
|
20
23
|
# forced_ordering_key (String, optional): Will force to use this value as the ordering_key,
|
21
24
|
# even withing transactions. Default nil.
|
22
25
|
def initialize(data, info, headers = {})
|
23
|
-
@data = data
|
24
|
-
@info =
|
25
|
-
@headers = headers
|
26
|
+
@data = data.deep_symbolize_keys
|
27
|
+
@info = info.deep_symbolize_keys
|
28
|
+
@headers = headers.deep_symbolize_keys
|
26
29
|
build_headers
|
27
30
|
validate!
|
28
31
|
end
|
29
32
|
|
30
33
|
# @return Hash: payload data
|
31
34
|
def to_h
|
32
|
-
{ data: data, info: info, headers: headers }
|
35
|
+
{ data: data.clone, info: info.clone, headers: headers.clone }
|
33
36
|
end
|
34
37
|
|
35
38
|
def klass
|
@@ -41,7 +44,7 @@ module PubSubModelSync
|
|
41
44
|
end
|
42
45
|
|
43
46
|
def mode
|
44
|
-
info[:mode].to_sym
|
47
|
+
(info[:mode] || :model).to_sym
|
45
48
|
end
|
46
49
|
|
47
50
|
# Process payload data
|
@@ -75,8 +78,8 @@ module PubSubModelSync
|
|
75
78
|
# convert payload data into Payload
|
76
79
|
# @param data [Hash]: payload data (:data, :info, :headers)
|
77
80
|
def self.from_payload_data(data)
|
78
|
-
data = data.
|
79
|
-
new(data[:data], data[:info], data[:headers])
|
81
|
+
data = data.symbolize_keys
|
82
|
+
new(data[:data], data[:info] || data[:attributes], data[:headers])
|
80
83
|
end
|
81
84
|
|
82
85
|
private
|
@@ -1,10 +1,10 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
module PubSubModelSync
|
4
|
-
class
|
4
|
+
class PayloadBuilder < PubSubModelSync::Base
|
5
5
|
attr_accessor :model, :action, :data, :mapping, :headers, :as_klass
|
6
6
|
|
7
|
-
# @param model (ActiveRecord::Base)
|
7
|
+
# @param model (ActiveRecord::Base,PubSubModelSync::PublisherConcern)
|
8
8
|
# @param action (@see PublishConcern::ps_publish)
|
9
9
|
# @param settings (@see PublishConcern::ps_publish): { data:, mapping:, headers:, as_klass: }
|
10
10
|
def initialize(model, action, settings = {})
|
@@ -17,9 +17,9 @@ module PubSubModelSync
|
|
17
17
|
end
|
18
18
|
|
19
19
|
# @return (Payload)
|
20
|
-
def
|
20
|
+
def call
|
21
21
|
values = compute_value(data)
|
22
|
-
values =
|
22
|
+
values = self.class.parse_mapping_for(model, mapping).merge(values)
|
23
23
|
PubSubModelSync::Payload.new(values, settings_data, headers_data)
|
24
24
|
end
|
25
25
|
|
@@ -27,6 +27,18 @@ module PubSubModelSync
|
|
27
27
|
[model.class.name, model.id || SecureRandom.uuid].join('/')
|
28
28
|
end
|
29
29
|
|
30
|
+
# @param model (ActiveRecord::Base)
|
31
|
+
# @param mapping (@see PublishConcern::ps_publish -> mapping)
|
32
|
+
# @return (Hash) Hash with the corresponding values for each attribute
|
33
|
+
# Sample: parse_mapping_for(my_model, %w[id name:full_name])
|
34
|
+
# ==> { id: 10, full_name: 'model.name value' }
|
35
|
+
def self.parse_mapping_for(model, mapping)
|
36
|
+
mapping.map do |prop|
|
37
|
+
source, target = prop.to_s.split(':')
|
38
|
+
[target || source, model.send(source.to_sym)]
|
39
|
+
end.to_h.symbolize_keys
|
40
|
+
end
|
41
|
+
|
30
42
|
private
|
31
43
|
|
32
44
|
def headers_data
|
@@ -46,12 +58,5 @@ module PubSubModelSync
|
|
46
58
|
def settings_data
|
47
59
|
{ klass: as_klass, action: action }
|
48
60
|
end
|
49
|
-
|
50
|
-
def mapping_data
|
51
|
-
mapping.map do |prop|
|
52
|
-
source, target = prop.to_s.split(':')
|
53
|
-
[target || source, model.send(source.to_sym)]
|
54
|
-
end.to_h.symbolize_keys
|
55
|
-
end
|
56
61
|
end
|
57
62
|
end
|
@@ -9,12 +9,6 @@ module PubSubModelSync
|
|
9
9
|
ps_init_transaction_callbacks if self <= ActiveRecord::Base
|
10
10
|
end
|
11
11
|
|
12
|
-
# before preparing data to sync
|
13
|
-
def ps_skip_publish?(_action)
|
14
|
-
false
|
15
|
-
end
|
16
|
-
alias ps_skip_sync? ps_skip_publish? # @deprecated
|
17
|
-
|
18
12
|
# before delivering data (return :cancel to cancel sync)
|
19
13
|
def ps_before_publish(_action, _payload); end
|
20
14
|
alias ps_before_sync ps_before_publish # @deprecated
|
@@ -24,14 +18,14 @@ module PubSubModelSync
|
|
24
18
|
alias ps_after_sync ps_after_publish # @deprecated
|
25
19
|
|
26
20
|
# Delivers a notification via pubsub
|
27
|
-
# @param action (
|
21
|
+
# @param action (Symbol,String) Sample: create|update|save|destroy|<any_other_key>
|
28
22
|
# @param mapping? (Array<String>) If present will generate data using the mapping and added to the payload.
|
29
23
|
# Sample: ["id", "full_name:name"]
|
30
|
-
# @param data? (Hash
|
24
|
+
# @param data? (Hash,Symbol,Proc)
|
31
25
|
# Hash: Data to be added to the payload
|
32
26
|
# Symbol: Method name to be called to retrieve payload data (must return a hash value, receives :action name)
|
33
27
|
# Proc: Block to be called to retrieve payload data
|
34
|
-
# @param headers? (Hash
|
28
|
+
# @param headers? (Hash,Symbol,Proc): (All available attributes in @Payload.headers)
|
35
29
|
# Hash: Data that will be merged with default header values
|
36
30
|
# Symbol: Method name that will be called to retrieve header values (must return a hash, receives :action name)
|
37
31
|
# Proc: Block to be called to retrieve header values
|
@@ -42,6 +36,16 @@ module PubSubModelSync
|
|
42
36
|
end
|
43
37
|
delegate :ps_class_publish, to: :class
|
44
38
|
|
39
|
+
# Permits to perform manually the callback for a specific action
|
40
|
+
# @param action (Symbol, default: :create) Only :create|:update|:destroy
|
41
|
+
def ps_perform_publish(action = :create)
|
42
|
+
items = self.class.ps_cache_publish_callbacks.select { |item| item[:actions].include?(action) }
|
43
|
+
raise(StandardError, "No callback found for action :#{action}") if items.empty?
|
44
|
+
|
45
|
+
items.each { |item| instance_exec(action, &item[:callback]) }
|
46
|
+
self
|
47
|
+
end
|
48
|
+
|
45
49
|
module ClassMethods
|
46
50
|
# Publishes a class level notification via pubsub
|
47
51
|
# @param data (Hash): Data of the notification
|
@@ -53,35 +57,39 @@ module PubSubModelSync
|
|
53
57
|
klass.publish_data((as_klass || name).to_s, data, action.to_sym, headers: headers)
|
54
58
|
end
|
55
59
|
|
56
|
-
# @param crud_actions (Symbol
|
60
|
+
# @param crud_actions (Symbol,Array<Symbol>): :create, :update, :destroy
|
57
61
|
# @param method_name (Symbol, optional) method to be called
|
58
|
-
def
|
62
|
+
def ps_after_action(crud_actions, method_name = nil, &block)
|
63
|
+
actions = Array(crud_actions).map(&:to_sym)
|
59
64
|
callback = ->(action) { method_name ? send(method_name, action) : instance_exec(action, &block) }
|
60
|
-
|
61
|
-
|
65
|
+
ps_cache_publish_callbacks({ actions: actions, callback: callback })
|
66
|
+
actions.each do |action|
|
62
67
|
if action == :destroy
|
63
68
|
after_destroy { instance_exec(action, &callback) }
|
64
|
-
elsif PubSubModelSync::Config.enable_rails4_before_commit # rails 4 compatibility
|
65
|
-
define_method("ps_before_#{action}_commit") { instance_exec(action, &callback) }
|
66
69
|
else
|
67
|
-
send(
|
70
|
+
send(:after_commit, on: action) { instance_exec(action, &callback) }
|
68
71
|
end
|
69
72
|
end
|
70
73
|
end
|
71
74
|
|
75
|
+
def ps_cache_publish_callbacks(new_value = nil)
|
76
|
+
@ps_cache_publish_callbacks ||= []
|
77
|
+
@ps_cache_publish_callbacks << new_value if new_value
|
78
|
+
@ps_cache_publish_callbacks
|
79
|
+
end
|
80
|
+
|
72
81
|
private
|
73
82
|
|
74
83
|
# Initialize calls to start and end pub_sub transactions and deliver all them in the same order
|
75
84
|
def ps_init_transaction_callbacks
|
76
85
|
start_transaction = lambda do
|
77
|
-
|
78
|
-
@ps_transaction = PubSubModelSync::MessagePublisher.init_transaction(key)
|
86
|
+
@ps_transaction = PubSubModelSync::MessagePublisher.init_transaction(nil)
|
79
87
|
end
|
80
|
-
|
88
|
+
before_create start_transaction, prepend: true
|
81
89
|
before_update start_transaction, prepend: true
|
82
90
|
before_destroy start_transaction, prepend: true
|
83
|
-
after_commit { @ps_transaction
|
84
|
-
after_rollback(prepend: true) { @ps_transaction
|
91
|
+
after_commit { @ps_transaction&.finish }
|
92
|
+
after_rollback(prepend: true) { @ps_transaction&.rollback }
|
85
93
|
end
|
86
94
|
end
|
87
95
|
end
|
@@ -2,6 +2,7 @@
|
|
2
2
|
|
3
3
|
require 'pub_sub_model_sync'
|
4
4
|
require 'rails'
|
5
|
+
require 'active_record'
|
5
6
|
require 'pub_sub_model_sync/config'
|
6
7
|
module PubSubModelSync
|
7
8
|
class Railtie < ::Rails::Railtie
|
@@ -12,7 +13,6 @@ module PubSubModelSync
|
|
12
13
|
end
|
13
14
|
|
14
15
|
configure do
|
15
|
-
require 'pub_sub_model_sync/initializers/before_commit' if PubSubModelSync::Config.enable_rails4_before_commit
|
16
16
|
end
|
17
17
|
end
|
18
18
|
end
|
@@ -26,7 +26,7 @@ module PubSubModelSync
|
|
26
26
|
def run_class_message
|
27
27
|
model_class = subscriber.klass.constantize
|
28
28
|
model_class.ps_processing_payload = payload # TODO: review for parallel notifications
|
29
|
-
call_action(model_class
|
29
|
+
call_action(model_class) if ensure_sync(model_class)
|
30
30
|
end
|
31
31
|
|
32
32
|
# support for: create, update, destroy
|
@@ -48,16 +48,17 @@ module PubSubModelSync
|
|
48
48
|
res
|
49
49
|
end
|
50
50
|
|
51
|
-
def call_action(object
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
51
|
+
def call_action(object)
|
52
|
+
callback = settings[:to_action]
|
53
|
+
callback.is_a?(Proc) ? object.instance_exec(payload.data, &callback) : call_action_method(object)
|
54
|
+
end
|
55
|
+
|
56
|
+
def call_action_method(object)
|
57
|
+
method_name = settings[:to_action]
|
58
|
+
method_name = :save! if %i[create update].include?(method_name.to_sym)
|
59
|
+
method_name = :destroy! if method_name.to_sym == :destroy
|
60
|
+
is_crud_action = %i[save! destroy!].include?(method_name)
|
61
|
+
is_crud_action ? object.send(method_name) : object.send(method_name, payload.data)
|
61
62
|
end
|
62
63
|
|
63
64
|
def parse_condition(condition, object)
|
@@ -74,10 +75,13 @@ module PubSubModelSync
|
|
74
75
|
model_class = subscriber.klass.constantize
|
75
76
|
return model_class.ps_find_model(payload.data) if model_class.respond_to?(:ps_find_model)
|
76
77
|
|
78
|
+
error_msg = 'No values provided for identifiers:'
|
79
|
+
raise(StandardError, "#{error_msg} #{[settings[:id], payload]}") if model_identifiers.empty?
|
80
|
+
|
77
81
|
model_class.where(model_identifiers).first_or_initialize
|
78
82
|
end
|
79
83
|
|
80
|
-
# @param mappings (Array<String>) supports aliasing, sample: ["id", "full_name:name"]
|
84
|
+
# @param mappings (Array<String,Symbol>) supports aliasing, sample: ["id", "full_name:name"]
|
81
85
|
# @return (Hash) hash with the correct attr names and its values
|
82
86
|
def parse_mapping(mappings)
|
83
87
|
mappings.map do |prop|
|
@@ -91,7 +95,7 @@ module PubSubModelSync
|
|
91
95
|
|
92
96
|
# @return (Hash) hash including identifiers and its values
|
93
97
|
def model_identifiers
|
94
|
-
@model_identifiers ||= parse_mapping(Array(settings[:id])
|
98
|
+
@model_identifiers ||= parse_mapping(Array(settings[:id]))
|
95
99
|
end
|
96
100
|
|
97
101
|
def populate_model
|
@@ -3,7 +3,6 @@
|
|
3
3
|
require 'active_support/core_ext/module'
|
4
4
|
module PubSubModelSync
|
5
5
|
class Runner
|
6
|
-
class ShutDown < StandardError; end
|
7
6
|
delegate :preload_listeners, to: :class
|
8
7
|
attr_accessor :connector
|
9
8
|
|
@@ -12,11 +11,10 @@ module PubSubModelSync
|
|
12
11
|
end
|
13
12
|
|
14
13
|
def run
|
14
|
+
at_exit { connector.stop }
|
15
15
|
trap_signals!
|
16
16
|
preload_listeners
|
17
17
|
start_listeners
|
18
|
-
rescue ShutDown
|
19
|
-
connector.stop
|
20
18
|
end
|
21
19
|
|
22
20
|
def self.preload_listeners
|
@@ -32,8 +30,8 @@ module PubSubModelSync
|
|
32
30
|
|
33
31
|
def trap_signals!
|
34
32
|
handler = proc do |signal|
|
35
|
-
puts "received #{Signal.signame(signal)}"
|
36
|
-
|
33
|
+
puts "PS_MSYNC ==> received #{Signal.signame(signal)}"
|
34
|
+
exit
|
37
35
|
end
|
38
36
|
%w[INT QUIT TERM].each { |signal| Signal.trap(signal, handler) }
|
39
37
|
end
|
@@ -6,16 +6,16 @@ module PubSubModelSync
|
|
6
6
|
SERVICE_KEY = 'service_model_sync'
|
7
7
|
|
8
8
|
def listen_messages
|
9
|
-
raise 'method :listen_messages must be defined in service'
|
9
|
+
raise NoMethodError, 'method :listen_messages must be defined in service'
|
10
10
|
end
|
11
11
|
|
12
12
|
# @param _payload (Payload)
|
13
13
|
def publish(_payload)
|
14
|
-
raise 'method :publish must be defined in service'
|
14
|
+
raise NoMethodError, 'method :publish must be defined in service'
|
15
15
|
end
|
16
16
|
|
17
17
|
def stop
|
18
|
-
raise 'method :stop must be defined in service'
|
18
|
+
raise NoMethodError, 'method :stop must be defined in service'
|
19
19
|
end
|
20
20
|
|
21
21
|
private
|
@@ -32,26 +32,13 @@ module PubSubModelSync
|
|
32
32
|
|
33
33
|
# @param (String: Payload in json format)
|
34
34
|
def process_message(payload_info)
|
35
|
-
retries ||= 0
|
36
35
|
payload = decode_payload(payload_info)
|
37
36
|
return payload.process unless same_app_message?(payload)
|
38
37
|
|
39
38
|
log("Skipping message from same origin: #{[payload]}") if config.debug
|
40
39
|
rescue => e
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
def can_retry_process_message?(error, payload, retries)
|
45
|
-
error_payload = [payload, error.message, error.backtrace]
|
46
|
-
if retries <= 5
|
47
|
-
sleep(retries)
|
48
|
-
log("Error while starting to process a message (retrying #{retries} retries...): #{error_payload}", :error)
|
49
|
-
rescue_database_connection if lost_db_connection_err?(error)
|
50
|
-
true
|
51
|
-
else
|
52
|
-
log("Retried 5 times and error persists, exiting...: #{error_payload}", :error)
|
53
|
-
Process.exit!(true)
|
54
|
-
end
|
40
|
+
error_payload = [payload, e.message, e.backtrace]
|
41
|
+
log("Error while starting to process a message: #{error_payload}", :error)
|
55
42
|
end
|
56
43
|
|
57
44
|
# @return Payload
|
@@ -66,19 +53,5 @@ module PubSubModelSync
|
|
66
53
|
key = payload.headers[:app_key]
|
67
54
|
key && key == config.subscription_key
|
68
55
|
end
|
69
|
-
|
70
|
-
def lost_db_connection_err?(error)
|
71
|
-
return true if error.class.name == 'PG::UnableToSend' # rubocop:disable Style/ClassEqualityComparison
|
72
|
-
|
73
|
-
error.message.match?(/lost connection/i)
|
74
|
-
end
|
75
|
-
|
76
|
-
def rescue_database_connection
|
77
|
-
log('Lost DB connection. Attempting to reconnect...', :warn)
|
78
|
-
ActiveRecord::Base.connection.reconnect!
|
79
|
-
rescue
|
80
|
-
log('Cannot reconnect to database, exiting...', :error)
|
81
|
-
Process.exit!(true)
|
82
|
-
end
|
83
56
|
end
|
84
57
|
end
|
@@ -37,14 +37,14 @@ module PubSubModelSync
|
|
37
37
|
message_topics = p_topic_names.map(&method(:find_topic))
|
38
38
|
message_topics.each do |topic|
|
39
39
|
topic.publish_async(encode_payload(payload), message_headers(payload)) do |res|
|
40
|
-
raise 'Failed to publish the message.' unless res.succeeded?
|
40
|
+
raise StandardError, 'Failed to publish the message.' unless res.succeeded?
|
41
41
|
end
|
42
42
|
end
|
43
43
|
end
|
44
44
|
|
45
45
|
def stop
|
46
46
|
log('Listener stopping...')
|
47
|
-
subscribers.each(&:stop!)
|
47
|
+
(subscribers || []).each(&:stop!)
|
48
48
|
end
|
49
49
|
|
50
50
|
private
|
@@ -76,8 +76,8 @@ module PubSubModelSync
|
|
76
76
|
end
|
77
77
|
|
78
78
|
# Check topic existence, create if missing topic
|
79
|
-
# @param names (Array<String
|
80
|
-
# @return (Array
|
79
|
+
# @param names (Array<String>,String)
|
80
|
+
# @return (Array,String) return @param names
|
81
81
|
def ensure_topics(names)
|
82
82
|
missing_topics = Array(names) - (@known_topics || service.topics)
|
83
83
|
missing_topics.each do |name|
|
@@ -8,7 +8,7 @@ end
|
|
8
8
|
module PubSubModelSync
|
9
9
|
class ServiceRabbit < ServiceBase
|
10
10
|
QUEUE_SETTINGS = { durable: true, auto_delete: false }.freeze
|
11
|
-
LISTEN_SETTINGS = { manual_ack:
|
11
|
+
LISTEN_SETTINGS = { manual_ack: false }.freeze
|
12
12
|
PUBLISH_SETTINGS = {}.freeze
|
13
13
|
|
14
14
|
# @!attribute topic_names (Array): ['Topic 1', 'Topic 2']
|
@@ -9,25 +9,27 @@ module PubSubModelSync
|
|
9
9
|
end
|
10
10
|
|
11
11
|
module ClassMethods
|
12
|
-
# @param actions (Symbol
|
13
|
-
# @param mapping (Array<String>) Attributes mapping with aliasing support, sample: ["id", "full_name:name"]
|
12
|
+
# @param actions (Symbol,Array<Symbol>) Notification.action name: save|create|update|destroy|<any_other_action>
|
13
|
+
# @param mapping (Array<String,Symbol>) Attributes mapping with aliasing support, sample: ["id", "full_name:name"]
|
14
14
|
# @param settings (Hash<:from_klass, :to_action, :id, :if, :unless>)
|
15
15
|
# from_klass (String) Notification.class name
|
16
|
-
# to_action (Symbol
|
16
|
+
# to_action (Symbol,Proc):
|
17
17
|
# Symbol: Method to process the notification
|
18
18
|
# Proc: Block to process the notification
|
19
|
-
# id (Symbol
|
20
|
-
# if (Symbol
|
21
|
-
# unless (Symbol
|
22
|
-
def ps_subscribe(actions, mapping = [], settings = {})
|
23
|
-
|
19
|
+
# id (Symbol,Array<Symbol,String>) attribute(s) DB primary identifier(s). Supports for mapping format.
|
20
|
+
# if (Symbol,Proc,Array<Symbol>) Method or block called as the conformation before calling the callback
|
21
|
+
# unless (Symbol,Proc,Array<Symbol>) Method or block called as the negation before calling the callback
|
22
|
+
def ps_subscribe(actions, mapping = [], settings = {}, &block)
|
23
|
+
settings[:to_action] ||= block if block
|
24
|
+
Array(actions).map do |action|
|
24
25
|
add_ps_subscriber(action, mapping, settings)
|
25
26
|
end
|
26
27
|
end
|
27
28
|
|
28
29
|
# @param action (Symbol) Notification.action name
|
29
30
|
# @param settings (Hash) @refer ps_subscribe.settings except(:id)
|
30
|
-
def ps_class_subscribe(action, settings = {})
|
31
|
+
def ps_class_subscribe(action, settings = {}, &block)
|
32
|
+
settings[:to_action] ||= block if block
|
31
33
|
add_ps_subscriber(action, nil, settings.merge(mode: :klass))
|
32
34
|
end
|
33
35
|
|
@@ -5,7 +5,7 @@ module PubSubModelSync
|
|
5
5
|
PUBLISHER_KLASS = PubSubModelSync::MessagePublisher
|
6
6
|
attr_accessor :key, :payloads, :max_buffer, :root, :children, :finished
|
7
7
|
|
8
|
-
# @param key (String
|
8
|
+
# @param key (String,Null) Transaction key, if empty will use the ordering_key from first payload
|
9
9
|
# @param max_buffer (Integer) Once this quantity of notifications is reached, then all notifications
|
10
10
|
# will immediately be delivered.
|
11
11
|
# Note: There is no way to rollback delivered notifications if current transaction fails
|
@@ -19,7 +19,11 @@ module PubSubModelSync
|
|
19
19
|
# @param payload (Payload)
|
20
20
|
def add_payload(payload)
|
21
21
|
payloads << payload
|
22
|
-
|
22
|
+
log("Payload added to current transaction: #{payload.inspect}") if config.debug
|
23
|
+
return unless payloads.count >= max_buffer
|
24
|
+
|
25
|
+
log("Payloads buffer was filled, delivering current payloads: #{payloads.count}")
|
26
|
+
deliver_payloads
|
23
27
|
end
|
24
28
|
|
25
29
|
def finish # rubocop:disable Metrics/AbcSize
|
@@ -38,7 +42,7 @@ module PubSubModelSync
|
|
38
42
|
end
|
39
43
|
|
40
44
|
def rollback
|
41
|
-
log("
|
45
|
+
log("Rollback #{payloads.count} notifications", :warn) if children.any? && debug?
|
42
46
|
self.children = []
|
43
47
|
root&.rollback
|
44
48
|
clean_publisher
|
@@ -57,9 +61,11 @@ module PubSubModelSync
|
|
57
61
|
|
58
62
|
def deliver_payloads
|
59
63
|
payloads.each do |payload|
|
60
|
-
|
61
|
-
|
62
|
-
|
64
|
+
begin # rubocop:disable Style/RedundantBegin (ruby 2.4 support)
|
65
|
+
PUBLISHER_KLASS.connector_publish(payload)
|
66
|
+
rescue => e
|
67
|
+
PUBLISHER_KLASS.send(:notify_error, e, payload)
|
68
|
+
end
|
63
69
|
end
|
64
70
|
self.payloads = []
|
65
71
|
end
|
data/lib/pub_sub_model_sync.rb
CHANGED
@@ -15,7 +15,7 @@ require 'pub_sub_model_sync/connector'
|
|
15
15
|
require 'pub_sub_model_sync/message_processor'
|
16
16
|
require 'pub_sub_model_sync/run_subscriber'
|
17
17
|
|
18
|
-
require 'pub_sub_model_sync/
|
18
|
+
require 'pub_sub_model_sync/payload_builder'
|
19
19
|
require 'pub_sub_model_sync/subscriber'
|
20
20
|
|
21
21
|
require 'pub_sub_model_sync/service_base'
|