pub_sub_model_sync 0.5.9 → 1.0.beta1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,12 @@
1
+ version: '2'
2
+
3
+ services:
4
+ test:
5
+ build: .
6
+ command: sh -c 'bundle exec rspec'
7
+ volumes:
8
+ - .:/app
9
+ - bundler_gems:/usr/local/bundle/
10
+
11
+ volumes:
12
+ bundler_gems:
Binary file
@@ -10,8 +10,10 @@ require 'pub_sub_model_sync/subscriber_concern'
10
10
  require 'pub_sub_model_sync/message_publisher'
11
11
  require 'pub_sub_model_sync/publisher_concern'
12
12
  require 'pub_sub_model_sync/runner'
13
+ require 'pub_sub_model_sync/transaction'
13
14
  require 'pub_sub_model_sync/connector'
14
15
  require 'pub_sub_model_sync/message_processor'
16
+ require 'pub_sub_model_sync/run_subscriber'
15
17
 
16
18
  require 'pub_sub_model_sync/publisher'
17
19
  require 'pub_sub_model_sync/subscriber'
@@ -2,7 +2,7 @@
2
2
 
3
3
  module PubSubModelSync
4
4
  class Base
5
- delegate :config, :log, to: self
5
+ delegate :config, :log, :debug?, to: self
6
6
 
7
7
  class << self
8
8
  def config
@@ -12,13 +12,30 @@ module PubSubModelSync
12
12
  def log(message, kind = :info)
13
13
  config.log message, kind
14
14
  end
15
+
16
+ def debug?
17
+ config.debug
18
+ end
15
19
  end
16
20
 
17
- def retry_error(error_klass, qty: 2, &block)
18
- @retries ||= 0
21
+ # @param errors (Array(Class|String))
22
+ def retry_error(errors, qty: 2, &block)
23
+ retries ||= 0
19
24
  block.call
20
- rescue error_klass => _e
21
- (@retries += 1) <= qty ? retry : raise
25
+ rescue => e
26
+ retries += 1
27
+ res = errors.find { |e_type| match_error?(e, e_type) }
28
+ raise if !res || retries > qty
29
+
30
+ sleep(qty * 0.1) && retry
31
+ end
32
+
33
+ private
34
+
35
+ # @param error (Exception)
36
+ # @param error_type (Class|String)
37
+ def match_error?(error, error_type)
38
+ error_type.is_a?(String) ? error.message.include?(error_type) : error.is_a?(error_type)
22
39
  end
23
40
  end
24
41
  end
@@ -3,12 +3,13 @@
3
3
  module PubSubModelSync
4
4
  class Config
5
5
  cattr_accessor(:subscribers) { [] }
6
- cattr_accessor(:publishers) { [] }
7
6
  cattr_accessor(:service_name) { :google }
8
7
 
9
8
  # customizable callbacks
10
9
  cattr_accessor(:debug) { false }
11
10
  cattr_accessor :logger # LoggerInst
11
+ cattr_accessor(:transactions_max_buffer) { 100 }
12
+ cattr_accessor(:enable_rails4_before_commit) { Rails::VERSION::MAJOR == 4 }
12
13
 
13
14
  cattr_accessor(:on_before_processing) { ->(_payload, _info) {} } # return :cancel to skip
14
15
  cattr_accessor(:on_success_processing) { ->(_payload, _info) {} }
@@ -16,16 +17,15 @@ module PubSubModelSync
16
17
  cattr_accessor(:on_before_publish) { ->(_payload) {} } # return :cancel to skip
17
18
  cattr_accessor(:on_after_publish) { ->(_payload) {} }
18
19
  cattr_accessor(:on_error_publish) { ->(_exception, _info) {} }
19
- cattr_accessor(:disabled_callback_publisher) { ->(_model, _action) { false } }
20
20
 
21
21
  # google service
22
- cattr_accessor :project, :credentials, :topic_name, :subscription_name
22
+ cattr_accessor :project, :credentials, :topic_name, :subscription_name, :default_topic_name
23
23
 
24
24
  # rabbitmq service
25
- cattr_accessor :bunny_connection, :queue_name, :topic_name, :subscription_name
25
+ cattr_accessor :bunny_connection, :topic_name, :subscription_name, :default_topic_name
26
26
 
27
27
  # kafka service
28
- cattr_accessor :kafka_connection, :topic_name, :subscription_name
28
+ cattr_accessor :kafka_connection, :topic_name, :subscription_name, :default_topic_name
29
29
 
30
30
  def self.log(msg, kind = :info)
31
31
  msg = "PS_MSYNC ==> #{msg}"
@@ -37,8 +37,17 @@ module PubSubModelSync
37
37
  end
38
38
 
39
39
  def self.subscription_key
40
- subscription_name ||
41
- (Rails.application.class.parent_name rescue '') # rubocop:disable Style/RescueModifier
40
+ klass = Rails.application.class
41
+ app_name = klass.respond_to?(:module_parent_name) ? klass.module_parent_name : klass.parent_name
42
+ subscription_name || app_name
43
+ end
44
+
45
+ class << self
46
+ alias default_topic_name_old default_topic_name
47
+
48
+ def default_topic_name
49
+ default_topic_name_old || Array(topic_name).first
50
+ end
42
51
  end
43
52
  end
44
53
  end
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Rails 4 backward compatibility (Add "simple" ps_before_*_commit callbacks)
4
+ ActiveRecord::ConnectionAdapters::RealTransaction.class_eval do
5
+ alias_method :commit_with_before_commit, :commit
6
+
7
+ def commit
8
+ call_before_commit_records if Rails::VERSION::MAJOR == 4
9
+ commit_with_before_commit
10
+ end
11
+
12
+ private
13
+
14
+ def call_before_commit_records
15
+ ite = records.uniq
16
+ ite.each do |record|
17
+ action = record.previous_changes.include?(:id) ? :create : :update
18
+ action = :destroy if record.destroyed?
19
+ callback_name = "ps_before_#{action}_commit".to_sym
20
+ record.send(callback_name) if record.respond_to?(callback_name)
21
+ end
22
+ end
23
+ end
@@ -28,10 +28,12 @@ module PubSubModelSync
28
28
  private
29
29
 
30
30
  def run_subscriber(subscriber)
31
+ processor = PubSubModelSync::RunSubscriber.new(subscriber, payload)
31
32
  return unless processable?(subscriber)
32
33
 
33
- retry_error(ActiveRecord::ConnectionTimeoutError, qty: 2) do
34
- subscriber.process!(payload)
34
+ errors = [ActiveRecord::ConnectionTimeoutError, 'deadlock detected', 'could not serialize access']
35
+ retry_error(errors, qty: 5) do
36
+ processor.call
35
37
  res = config.on_success_processing.call(payload, { subscriber: subscriber })
36
38
  log "processed message with: #{payload.inspect}" if res != :skip_log
37
39
  end
@@ -52,8 +54,7 @@ module PubSubModelSync
52
54
 
53
55
  def filter_subscribers
54
56
  config.subscribers.select do |subscriber|
55
- subscriber.settings[:from_klass].to_s == payload.klass.to_s &&
56
- subscriber.settings[:from_action].to_s == payload.action.to_s
57
+ subscriber.from_klass == payload.klass && subscriber.action == payload.action && payload.mode == subscriber.mode
57
58
  end
58
59
  end
59
60
  end
@@ -3,60 +3,117 @@
3
3
  module PubSubModelSync
4
4
  class MessagePublisher < PubSubModelSync::Base
5
5
  class << self
6
+ class MissingPublisher < StandardError; end
7
+ attr_accessor :current_transaction
8
+
6
9
  def connector
7
10
  @connector ||= PubSubModelSync::Connector.new
8
11
  end
9
12
 
10
- # Publishes any value to pubsub
11
- # @param klass (String): Class name
12
- # @param data (Hash): Data to be delivered
13
- # @param action (:symbol): action name
14
- def publish_data(klass, data, action)
15
- attrs = { klass: klass.to_s, action: action.to_sym, key: [klass.to_s, action].join('/') }
16
- payload = PubSubModelSync::Payload.new(data, attrs)
13
+ # Permits to group all payloads with the same ordering_key and be processed in the same order
14
+ # they are published by the subscribers. Grouping by ordering_key allows us to enable
15
+ # multiple workers in our Pub/Sub service(s), and still guarantee that related payloads will
16
+ # be processed in the correct order, despite of the multiple threads. This thanks to the fact
17
+ # that Pub/Sub services will always send messages with the same `ordering_key` into the same
18
+ # worker/thread.
19
+ # @see Transaction.new(...)
20
+ # @param key (String|Nil)
21
+ # @param block (Yield) block to be executed
22
+ def transaction(key, settings = {}, &block)
23
+ t = init_transaction(key, settings)
24
+ block.call
25
+ t.finish
26
+ rescue
27
+ t.rollback
28
+ raise
29
+ ensure
30
+ t.clean_publisher
31
+ end
32
+
33
+ # Starts a new transaction
34
+ # @param key (@transaction_key)
35
+ # @return (Transaction)
36
+ def init_transaction(key, settings = {})
37
+ new_transaction = PubSubModelSync::Transaction.new(key, settings)
38
+ if current_transaction
39
+ current_transaction.add_transaction(new_transaction)
40
+ else
41
+ self.current_transaction = new_transaction
42
+ end
43
+ new_transaction
44
+ end
45
+
46
+ # Publishes a class level notification via pubsub
47
+ # @refer PublisherConcern.ps_class_publish
48
+ # @return Payload
49
+ def publish_data(klass, data, action, headers: {})
50
+ attrs = { klass: klass.to_s, action: action.to_sym, mode: :klass }
51
+ payload = PubSubModelSync::Payload.new(data, attrs, headers)
17
52
  publish(payload)
18
53
  end
19
54
 
20
- # Publishes model info to pubsub
21
- # @param model (ActiveRecord model)
22
- # @param action (Sym): Action name
23
- # @param publisher (Publisher, optional): Publisher to be used
24
- def publish_model(model, action, publisher = nil)
25
- return if model.ps_skip_sync?(action)
55
+ # @param model (ActiveRecord::Base)
56
+ # @param action (Symbol: @see PublishConcern::ps_publish)
57
+ # @param settings (Hash: @see Publisher.new.settings)
58
+ def publish_model(model, action, settings = {})
59
+ return if model.ps_skip_publish?(action)
26
60
 
27
- publisher ||= model.class.ps_publisher(action)
28
- payload = publisher.payload(model, action)
29
- res_before = model.ps_before_sync(action, payload.data)
30
- return if res_before == :cancel
61
+ publisher = PubSubModelSync::Publisher.new(model, action, settings)
62
+ payload = publisher.payload
31
63
 
32
- publish(payload)
33
- model.ps_after_sync(action, payload.data)
64
+ transaction(payload.headers[:ordering_key]) do # catch and group all :ps_before_publish syncs
65
+ publish(payload) { model.ps_after_publish(action, payload) } if ensure_model_publish(model, action, payload)
66
+ end
34
67
  end
35
68
 
36
69
  # Publishes payload to pubsub
37
- # @attr payload (PubSubModelSync::Payload)
70
+ # @param payload (PubSubModelSync::Payload)
71
+ # @return Payload
38
72
  # Raises error if exist
39
- def publish!(payload)
40
- if config.on_before_publish.call(payload) == :cancel
41
- log("Publish message cancelled: #{payload}") if config.debug
42
- return
43
- end
73
+ def publish!(payload, &block)
74
+ payload.headers[:ordering_key] = ordering_key_for(payload)
75
+ return unless ensure_publish(payload)
44
76
 
45
- log("Publishing message: #{[payload]}")
77
+ current_transaction ? current_transaction.add_payload(payload) : connector_publish(payload)
78
+ block&.call
79
+ payload
80
+ end
81
+
82
+ def connector_publish(payload)
46
83
  connector.publish(payload)
84
+ log("Published message: #{[payload]}")
47
85
  config.on_after_publish.call(payload)
48
86
  end
49
87
 
50
88
  # Similar to :publish! method
51
89
  # Notifies error via :on_error_publish instead of raising error
52
- def publish(payload)
53
- publish!(payload)
90
+ # @return Payload
91
+ def publish(payload, &block)
92
+ publish!(payload, &block)
54
93
  rescue => e
55
94
  notify_error(e, payload)
56
95
  end
57
96
 
58
97
  private
59
98
 
99
+ def ensure_publish(payload)
100
+ cancelled = config.on_before_publish.call(payload) == :cancel
101
+ log("Publish cancelled by config.on_before_publish: #{payload}") if config.debug && cancelled
102
+ !cancelled
103
+ end
104
+
105
+ def ordering_key_for(payload)
106
+ current_transaction&.key ||= payload.headers[:ordering_key]
107
+ payload.headers[:forced_ordering_key] || current_transaction&.key || payload.headers[:ordering_key]
108
+ end
109
+
110
+ def ensure_model_publish(model, action, payload)
111
+ res_before = model.ps_before_publish(action, payload)
112
+ cancelled = res_before == :cancel
113
+ log("Publish cancelled by model.ps_before_publish: #{payload}") if config.debug && cancelled
114
+ !cancelled
115
+ end
116
+
60
117
  def notify_error(exception, payload)
61
118
  info = [payload, exception.message, exception.backtrace]
62
119
  res = config.on_error_publish.call(exception, { payload: payload })
@@ -26,6 +26,10 @@ module PubSubModelSync
26
26
  end
27
27
 
28
28
  class MockTopic
29
+ def name
30
+ 'name'
31
+ end
32
+
29
33
  def subscription(*_args)
30
34
  @subscription ||= MockSubscription.new
31
35
  end
@@ -28,16 +28,29 @@ module PubSubModelSync
28
28
  def subscribe(*_args)
29
29
  true
30
30
  end
31
+
32
+ def mark_message_as_processed(*_args)
33
+ true
34
+ end
31
35
  end
32
36
 
33
37
  def producer(*_args)
34
38
  MockProducer.new
35
39
  end
40
+ alias async_producer producer
36
41
 
37
42
  def consumer(*_args)
38
43
  MockConsumer.new
39
44
  end
40
45
 
46
+ def topics
47
+ []
48
+ end
49
+
50
+ def create_topic(_name)
51
+ true
52
+ end
53
+
41
54
  def close
42
55
  true
43
56
  end
@@ -3,13 +3,25 @@
3
3
  module PubSubModelSync
4
4
  class Payload
5
5
  class MissingInfo < StandardError; end
6
- attr_reader :data, :attributes, :headers
6
+ attr_reader :data, :info, :headers
7
7
 
8
8
  # @param data (Hash: { any value }):
9
- # @param attributes (Hash: { klass*: string, action*: :sym, key?: string }):
10
- def initialize(data, attributes, headers = {})
9
+ # @param info (Hash: { klass*: string, action*: :sym, mode?: :klass|:model }):
10
+ # @param headers (Hash):
11
+ # key (String): identifier of the payload, default:
12
+ # klass/action: when class message
13
+ # klass/action/model.id: when model message
14
+ # ordering_key (String): messages with the same key are processed in the same order they
15
+ # were delivered, default:
16
+ # klass: when class message
17
+ # klass/id: when model message
18
+ # topic_name (String|Array<String>): Specific topic name to be used when delivering the
19
+ # message (default first topic)
20
+ # forced_ordering_key (String, optional): Will force to use this value as the ordering_key,
21
+ # even withing transactions. Default nil.
22
+ def initialize(data, info, headers = {})
11
23
  @data = data
12
- @attributes = attributes
24
+ @info = { mode: :model }.merge(info)
13
25
  @headers = headers
14
26
  build_headers
15
27
  validate!
@@ -17,15 +29,19 @@ module PubSubModelSync
17
29
 
18
30
  # @return Hash: payload data
19
31
  def to_h
20
- { data: data, attributes: attributes, headers: headers }
32
+ { data: data, info: info, headers: headers }
21
33
  end
22
34
 
23
35
  def klass
24
- attributes[:klass]
36
+ info[:klass].to_s
25
37
  end
26
38
 
27
39
  def action
28
- attributes[:action]
40
+ info[:action].to_sym
41
+ end
42
+
43
+ def mode
44
+ info[:mode].to_sym
29
45
  end
30
46
 
31
47
  # Process payload data
@@ -57,21 +73,23 @@ module PubSubModelSync
57
73
  end
58
74
 
59
75
  # convert payload data into Payload
60
- # @param data [Hash]: payload data (:data, :attributes, :headers)
76
+ # @param data [Hash]: payload data (:data, :info, :headers)
61
77
  def self.from_payload_data(data)
62
78
  data = data.deep_symbolize_keys
63
- new(data[:data], data[:attributes], data[:headers])
79
+ new(data[:data], data[:info], data[:headers])
64
80
  end
65
81
 
66
82
  private
67
83
 
68
84
  def build_headers
69
- headers[:uuid] ||= SecureRandom.uuid
70
85
  headers[:app_key] ||= PubSubModelSync::Config.subscription_key
86
+ headers[:key] ||= [klass, action].join('/')
87
+ headers[:ordering_key] ||= klass
88
+ headers[:uuid] ||= SecureRandom.uuid
71
89
  end
72
90
 
73
91
  def validate!
74
- raise MissingInfo if !attributes[:klass] || !attributes[:action]
92
+ raise MissingInfo if !info[:klass] || !info[:action]
75
93
  end
76
94
  end
77
95
  end
@@ -1,40 +1,57 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module PubSubModelSync
4
- class Publisher
5
- attr_accessor :attrs, :actions, :klass, :as_klass
6
-
7
- def initialize(attrs, klass, actions = nil, as_klass = nil)
8
- @attrs = attrs
9
- @klass = klass
10
- @actions = actions || %i[create update destroy]
11
- @as_klass = as_klass || klass
4
+ class Publisher < PubSubModelSync::Base
5
+ attr_accessor :model, :action, :data, :mapping, :headers, :as_klass
6
+
7
+ # @param model (ActiveRecord::Base)
8
+ # @param action (@see PublishConcern::ps_publish)
9
+ # @param settings (@see PublishConcern::ps_publish): { data:, mapping:, headers:, as_klass: }
10
+ def initialize(model, action, settings = {})
11
+ @model = model
12
+ @action = action
13
+ @data = settings[:data] || {}
14
+ @mapping = settings[:mapping] || []
15
+ @headers = settings[:headers] || {}
16
+ @as_klass = settings[:as_klass] || model.class.name
17
+ end
18
+
19
+ # @return (Payload)
20
+ def payload
21
+ values = compute_value(data)
22
+ values = mapping_data.merge(values)
23
+ PubSubModelSync::Payload.new(values, settings_data, headers_data)
12
24
  end
13
25
 
14
- # Builds the payload with model information defined for :action (:create|:update|:destroy)
15
- def payload(model, action)
16
- PubSubModelSync::Payload.new(payload_data(model), payload_attrs(model, action))
26
+ def self.ordering_key_for(model)
27
+ [model.class.name, model.id || SecureRandom.uuid].join('/')
17
28
  end
18
29
 
19
30
  private
20
31
 
21
- def payload_data(model)
22
- source_props = @attrs.map { |prop| prop.to_s.split(':').first }
23
- data = model.as_json(only: source_props, methods: source_props)
24
- aliased_props = @attrs.select { |prop| prop.to_s.include?(':') }
25
- aliased_props.each do |prop|
26
- source, target = prop.to_s.split(':')
27
- data[target] = data.delete(source)
28
- end
29
- data.symbolize_keys
32
+ def headers_data
33
+ klass_name = model.class.name
34
+ key = [klass_name, action, model.id || SecureRandom.uuid].join('/')
35
+ def_data = { ordering_key: self.class.ordering_key_for(model), key: key }
36
+ def_data.merge(compute_value(headers))
30
37
  end
31
38
 
32
- def payload_attrs(model, action)
33
- {
34
- klass: (as_klass || model.class.name).to_s,
35
- action: action.to_sym,
36
- key: [model.class.name, action, model.id].join('/')
37
- }
39
+ def compute_value(value)
40
+ res = value
41
+ res = model.send(value, action) if value.is_a?(Symbol) # method name
42
+ res = value.call(model, action) if value.is_a?(Proc)
43
+ res
44
+ end
45
+
46
+ def settings_data
47
+ { klass: as_klass, action: action }
48
+ end
49
+
50
+ def mapping_data
51
+ mapping.map do |prop|
52
+ source, target = prop.to_s.split(':')
53
+ [target || source, model.send(source.to_sym)]
54
+ end.to_h.symbolize_keys
38
55
  end
39
56
  end
40
57
  end