table_sync 4.2.1 → 6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/ci.yml +52 -0
  3. data/CHANGELOG.md +75 -9
  4. data/Gemfile.lock +159 -152
  5. data/README.md +4 -1
  6. data/docs/message_protocol.md +3 -3
  7. data/docs/publishing/configuration.md +143 -0
  8. data/docs/publishing/manual.md +155 -0
  9. data/docs/publishing/publishers.md +162 -0
  10. data/docs/publishing.md +35 -119
  11. data/docs/receiving.md +11 -6
  12. data/lib/table_sync/errors.rb +31 -22
  13. data/lib/table_sync/event.rb +35 -0
  14. data/lib/table_sync/orm_adapter/active_record.rb +25 -0
  15. data/lib/table_sync/orm_adapter/base.rb +92 -0
  16. data/lib/table_sync/orm_adapter/sequel.rb +29 -0
  17. data/lib/table_sync/publishing/batch.rb +53 -0
  18. data/lib/table_sync/publishing/data/objects.rb +50 -0
  19. data/lib/table_sync/publishing/data/raw.rb +27 -0
  20. data/lib/table_sync/publishing/helpers/attributes.rb +63 -0
  21. data/lib/table_sync/publishing/helpers/debounce.rb +134 -0
  22. data/lib/table_sync/publishing/helpers/objects.rb +39 -0
  23. data/lib/table_sync/publishing/message/base.rb +73 -0
  24. data/lib/table_sync/publishing/message/batch.rb +14 -0
  25. data/lib/table_sync/publishing/message/raw.rb +54 -0
  26. data/lib/table_sync/publishing/message/single.rb +13 -0
  27. data/lib/table_sync/publishing/params/base.rb +66 -0
  28. data/lib/table_sync/publishing/params/batch.rb +23 -0
  29. data/lib/table_sync/publishing/params/raw.rb +7 -0
  30. data/lib/table_sync/publishing/params/single.rb +31 -0
  31. data/lib/table_sync/publishing/raw.rb +21 -0
  32. data/lib/table_sync/publishing/single.rb +65 -0
  33. data/lib/table_sync/publishing.rb +20 -5
  34. data/lib/table_sync/receiving/config.rb +6 -4
  35. data/lib/table_sync/receiving/handler.rb +25 -9
  36. data/lib/table_sync/receiving/model/active_record.rb +1 -1
  37. data/lib/table_sync/receiving.rb +0 -2
  38. data/lib/table_sync/setup/active_record.rb +22 -0
  39. data/lib/table_sync/setup/base.rb +67 -0
  40. data/lib/table_sync/setup/sequel.rb +26 -0
  41. data/lib/table_sync/utils/interface_checker.rb +2 -2
  42. data/lib/table_sync/version.rb +1 -1
  43. data/lib/table_sync.rb +31 -8
  44. data/table_sync.gemspec +7 -7
  45. metadata +58 -37
  46. data/.travis.yml +0 -34
  47. data/lib/table_sync/publishing/base_publisher.rb +0 -114
  48. data/lib/table_sync/publishing/batch_publisher.rb +0 -109
  49. data/lib/table_sync/publishing/orm_adapter/active_record.rb +0 -32
  50. data/lib/table_sync/publishing/orm_adapter/sequel.rb +0 -57
  51. data/lib/table_sync/publishing/publisher.rb +0 -129
@@ -0,0 +1,73 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TableSync::Publishing::Message
4
+ class Base
5
+ include Tainbox
6
+
7
+ attr_reader :objects
8
+
9
+ attribute :object_class
10
+ attribute :original_attributes
11
+ attribute :event
12
+
13
+ def initialize(params)
14
+ super(params)
15
+
16
+ @objects = find_or_init_objects
17
+
18
+ raise TableSync::NoObjectsForSyncError if objects.empty? && TableSync.raise_on_empty_message
19
+ end
20
+
21
+ def publish
22
+ return if original_attributes.blank?
23
+
24
+ Rabbit.publish(message_params)
25
+
26
+ notify!
27
+ end
28
+
29
+ def empty?
30
+ objects.empty?
31
+ end
32
+
33
+ def find_or_init_objects
34
+ TableSync::Publishing::Helpers::Objects.new(
35
+ object_class: object_class, original_attributes: original_attributes, event: event,
36
+ ).construct_list
37
+ end
38
+
39
+ # MESSAGE PARAMS
40
+
41
+ def message_params
42
+ params.merge(data: data)
43
+ end
44
+
45
+ def data
46
+ TableSync::Publishing::Data::Objects.new(
47
+ objects: objects, event: event,
48
+ ).construct
49
+ end
50
+
51
+ # :nocov:
52
+ def params
53
+ raise NotImplementedError
54
+ end
55
+ # :nocov:
56
+
57
+ # NOTIFY
58
+
59
+ def notify!
60
+ TableSync::Instrument.notify(
61
+ table: model_naming.table,
62
+ schema: model_naming.schema,
63
+ event: event,
64
+ direction: :publish,
65
+ count: objects.count,
66
+ )
67
+ end
68
+
69
+ def model_naming
70
+ TableSync.publishing_adapter.model_naming(objects.first.object_class)
71
+ end
72
+ end
73
+ end
@@ -0,0 +1,14 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TableSync::Publishing::Message
4
+ class Batch < Base
5
+ attribute :headers
6
+ attribute :routing_key
7
+
8
+ def params
9
+ TableSync::Publishing::Params::Batch.new(
10
+ object_class: object_class, headers: headers, routing_key: routing_key,
11
+ ).construct
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,54 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TableSync::Publishing::Message
4
+ class Raw
5
+ include Tainbox
6
+
7
+ attribute :object_class
8
+ attribute :original_attributes
9
+ attribute :routing_key
10
+ attribute :headers
11
+
12
+ attribute :event
13
+
14
+ def publish
15
+ Rabbit.publish(message_params)
16
+
17
+ notify!
18
+ end
19
+
20
+ # NOTIFY
21
+
22
+ def notify!
23
+ TableSync::Instrument.notify(
24
+ table: model_naming.table,
25
+ schema: model_naming.schema,
26
+ event: event,
27
+ count: original_attributes.count,
28
+ direction: :publish,
29
+ )
30
+ end
31
+
32
+ def model_naming
33
+ TableSync.publishing_adapter.model_naming(object_class.constantize)
34
+ end
35
+
36
+ # MESSAGE PARAMS
37
+
38
+ def message_params
39
+ params.merge(data: data)
40
+ end
41
+
42
+ def data
43
+ TableSync::Publishing::Data::Raw.new(
44
+ object_class: object_class, attributes_for_sync: original_attributes, event: event,
45
+ ).construct
46
+ end
47
+
48
+ def params
49
+ TableSync::Publishing::Params::Raw.new(
50
+ object_class: object_class, routing_key: routing_key, headers: headers,
51
+ ).construct
52
+ end
53
+ end
54
+ end
@@ -0,0 +1,13 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TableSync::Publishing::Message
4
+ class Single < Base
5
+ def object
6
+ objects.first
7
+ end
8
+
9
+ def params
10
+ TableSync::Publishing::Params::Single.new(object: object).construct
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,66 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TableSync::Publishing::Params
4
+ class Base
5
+ DEFAULT_PARAMS = {
6
+ confirm_select: true,
7
+ realtime: true,
8
+ event: :table_sync,
9
+ }.freeze
10
+
11
+ def construct
12
+ DEFAULT_PARAMS.merge(
13
+ routing_key: routing_key,
14
+ headers: headers,
15
+ exchange_name: exchange_name,
16
+ )
17
+ end
18
+
19
+ private
20
+
21
+ def calculated_routing_key
22
+ if TableSync.routing_key_callable
23
+ TableSync.routing_key_callable.call(object_class, attributes_for_routing_key)
24
+ else
25
+ raise TableSync::NoCallableError.new("routing_key")
26
+ end
27
+ end
28
+
29
+ def calculated_headers
30
+ if TableSync.headers_callable
31
+ TableSync.headers_callable.call(object_class, attributes_for_headers)
32
+ else
33
+ raise TableSync::NoCallableError.new("headers")
34
+ end
35
+ end
36
+
37
+ # NOT IMPLEMENTED
38
+
39
+ # name of the model being synced in the string format
40
+ # :nocov:
41
+ def object_class
42
+ raise NotImplementedError
43
+ end
44
+
45
+ def routing_key
46
+ raise NotImplementedError
47
+ end
48
+
49
+ def headers
50
+ raise NotImplementedError
51
+ end
52
+
53
+ def exchange_name
54
+ raise NotImplementedError
55
+ end
56
+
57
+ def attributes_for_routing_key
58
+ raise NotImplementedError
59
+ end
60
+
61
+ def attributes_for_headers
62
+ raise NotImplementedError
63
+ end
64
+ # :nocov:
65
+ end
66
+ end
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TableSync::Publishing::Params
4
+ class Batch < Base
5
+ include Tainbox
6
+
7
+ attribute :object_class
8
+
9
+ attribute :exchange_name, default: -> { TableSync.exchange_name }
10
+ attribute :routing_key, default: -> { calculated_routing_key }
11
+ attribute :headers, default: -> { calculated_headers }
12
+
13
+ private
14
+
15
+ def attributes_for_routing_key
16
+ {}
17
+ end
18
+
19
+ def attributes_for_headers
20
+ {}
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,7 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TableSync::Publishing::Params
4
+ class Raw < Batch
5
+ # FOR NAMING CONSISTENCY
6
+ end
7
+ end
@@ -0,0 +1,31 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TableSync::Publishing::Params
4
+ class Single < Base
5
+ attr_reader :object, :routing_key, :headers
6
+
7
+ def initialize(object:)
8
+ @object = object
9
+ @routing_key = calculated_routing_key
10
+ @headers = calculated_headers
11
+ end
12
+
13
+ private
14
+
15
+ def object_class
16
+ object.object_class.name
17
+ end
18
+
19
+ def attributes_for_routing_key
20
+ object.attributes_for_routing_key
21
+ end
22
+
23
+ def attributes_for_headers
24
+ object.attributes_for_headers
25
+ end
26
+
27
+ def exchange_name
28
+ TableSync.exchange_name
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,21 @@
1
+ # frozen_string_literal: true
2
+
3
+ class TableSync::Publishing::Raw
4
+ include Tainbox
5
+
6
+ attribute :object_class
7
+ attribute :original_attributes
8
+
9
+ attribute :routing_key
10
+ attribute :headers
11
+
12
+ attribute :event, default: :update
13
+
14
+ def publish_now
15
+ message.publish
16
+ end
17
+
18
+ def message
19
+ TableSync::Publishing::Message::Raw.new(attributes)
20
+ end
21
+ end
@@ -0,0 +1,65 @@
1
+ # frozen_string_literal: true
2
+
3
+ class TableSync::Publishing::Single
4
+ include Tainbox
5
+ include Memery
6
+
7
+ attribute :object_class
8
+ attribute :original_attributes
9
+ attribute :debounce_time
10
+
11
+ attribute :event, Symbol, default: :update
12
+
13
+ # expect job to have perform_at method
14
+ # debounce destroyed event
15
+ # because otherwise update event could be sent after destroy
16
+ def publish_later
17
+ return if debounce.skip?
18
+
19
+ job.perform_at(job_attributes)
20
+
21
+ debounce.cache_next_sync_time
22
+ end
23
+
24
+ def publish_now
25
+ message.publish unless message.empty?
26
+ end
27
+
28
+ memoize def message
29
+ TableSync::Publishing::Message::Single.new(attributes)
30
+ end
31
+
32
+ memoize def debounce
33
+ TableSync::Publishing::Helpers::Debounce.new(
34
+ object_class: object_class,
35
+ needle: message.object.needle,
36
+ debounce_time: debounce_time,
37
+ event: event,
38
+ )
39
+ end
40
+
41
+ private
42
+
43
+ # JOB
44
+
45
+ def job
46
+ if TableSync.single_publishing_job_class_callable
47
+ TableSync.single_publishing_job_class_callable.call
48
+ else
49
+ raise TableSync::NoCallableError.new("single_publishing_job_class")
50
+ end
51
+ end
52
+
53
+ def job_attributes
54
+ attributes.merge(
55
+ original_attributes: serialized_original_attributes,
56
+ perform_at: debounce.next_sync_time,
57
+ )
58
+ end
59
+
60
+ def serialized_original_attributes
61
+ TableSync::Publishing::Helpers::Attributes
62
+ .new(original_attributes)
63
+ .serialize
64
+ end
65
+ end
@@ -2,10 +2,25 @@
2
2
 
3
3
  module TableSync
4
4
  module Publishing
5
- require_relative "publishing/base_publisher"
6
- require_relative "publishing/publisher"
7
- require_relative "publishing/batch_publisher"
8
- require_relative "publishing/orm_adapter/active_record"
9
- require_relative "publishing/orm_adapter/sequel"
5
+ require_relative "publishing/data/objects"
6
+ require_relative "publishing/data/raw"
7
+
8
+ require_relative "publishing/helpers/attributes"
9
+ require_relative "publishing/helpers/debounce"
10
+ require_relative "publishing/helpers/objects"
11
+
12
+ require_relative "publishing/params/base"
13
+ require_relative "publishing/params/batch"
14
+ require_relative "publishing/params/raw"
15
+ require_relative "publishing/params/single"
16
+
17
+ require_relative "publishing/message/base"
18
+ require_relative "publishing/message/batch"
19
+ require_relative "publishing/message/raw"
20
+ require_relative "publishing/message/single"
21
+
22
+ require_relative "publishing/batch"
23
+ require_relative "publishing/raw"
24
+ require_relative "publishing/single"
10
25
  end
11
26
  end
@@ -4,20 +4,22 @@ module TableSync::Receiving
4
4
  class Config
5
5
  attr_reader :model, :events
6
6
 
7
- def initialize(model:, events: AVAILABLE_EVENTS)
7
+ def initialize(model:, events: TableSync::Event::VALID_RESOLVED_EVENTS)
8
8
  @model = model
9
9
 
10
10
  @events = [events].flatten.map(&:to_sym)
11
11
 
12
- unless @events.all? { |event| AVAILABLE_EVENTS.include?(event) }
13
- raise TableSync::UndefinedEvent.new(events)
14
- end
12
+ raise TableSync::UndefinedEvent.new(events) if invalid_events.any?
15
13
 
16
14
  self.class.default_values_for_options.each do |ivar, default_value_generator|
17
15
  instance_variable_set(ivar, default_value_generator.call(self))
18
16
  end
19
17
  end
20
18
 
19
+ def invalid_events
20
+ events - TableSync::Event::VALID_RESOLVED_EVENTS
21
+ end
22
+
21
23
  class << self
22
24
  attr_reader :default_values_for_options
23
25
 
@@ -31,7 +31,7 @@ class TableSync::Receiving::Handler < Rabbit::EventHandler
31
31
  params[:default_values] = config.default_values(data: data)
32
32
  end
33
33
 
34
- config.wrap_receiving(**params) do
34
+ config.wrap_receiving(event: event, **params) do
35
35
  perform(config, params)
36
36
  end
37
37
  end
@@ -44,14 +44,18 @@ class TableSync::Receiving::Handler < Rabbit::EventHandler
44
44
  super(Array.wrap(data[:attributes]))
45
45
  end
46
46
 
47
- def event=(name)
48
- name = name.to_sym
49
- raise TableSync::UndefinedEvent.new(event) unless %i[update destroy].include?(name)
50
- super(name)
47
+ def event=(event_name)
48
+ event_name = event_name.to_sym
49
+
50
+ if event_name.in?(TableSync::Event::VALID_RESOLVED_EVENTS)
51
+ super(event_name)
52
+ else
53
+ raise TableSync::UndefinedEvent.new(event)
54
+ end
51
55
  end
52
56
 
53
- def model=(name)
54
- super(name.to_s)
57
+ def model=(model_name)
58
+ super(model_name.to_s)
55
59
  end
56
60
 
57
61
  def configs
@@ -82,7 +86,7 @@ class TableSync::Receiving::Handler < Rabbit::EventHandler
82
86
  row[after] = row.delete(before)
83
87
  end
84
88
 
85
- config.except(row: row).each(&row.method(:delete))
89
+ config.except(row: row).each { |x| row.delete(x) }
86
90
 
87
91
  row.merge!(config.additional_data(row: row))
88
92
 
@@ -98,7 +102,8 @@ class TableSync::Receiving::Handler < Rabbit::EventHandler
98
102
 
99
103
  def validate_data(data, target_keys:)
100
104
  data.each do |row|
101
- next if target_keys.all?(&row.keys.method(:include?))
105
+ next if target_keys.all? { |x| row.key?(x) }
106
+
102
107
  raise TableSync::DataError.new(
103
108
  data, target_keys, "Some target keys not found in received attributes!"
104
109
  )
@@ -107,6 +112,17 @@ class TableSync::Receiving::Handler < Rabbit::EventHandler
107
112
  if data.uniq { |row| row.slice(*target_keys) }.size != data.size
108
113
  raise TableSync::DataError.new(data, target_keys, "Duplicate rows found!")
109
114
  end
115
+
116
+ keys_sample = data[0].keys
117
+ keys_diff = data.each_with_object(Set.new) do |row, set|
118
+ (row.keys - keys_sample | keys_sample - row.keys).each { |x| set.add(x) }
119
+ end
120
+
121
+ unless keys_diff.empty?
122
+ raise TableSync::DataError.new(data, target_keys, <<~MESSAGE)
123
+ Bad batch structure, check keys: #{keys_diff.to_a}
124
+ MESSAGE
125
+ end
110
126
  end
111
127
 
112
128
  def perform(config, params)
@@ -96,7 +96,7 @@ module TableSync::Receiving::Model
96
96
  end
97
97
  end
98
98
 
99
- result = query.destroy_all.map(&method(:row_to_hash))
99
+ result = query.destroy_all.map { |x| row_to_hash(x) }
100
100
 
101
101
  if result.size > data.size
102
102
  raise TableSync::DestroyError.new(data: data, target_keys: target_keys, result: result)
@@ -2,8 +2,6 @@
2
2
 
3
3
  module TableSync
4
4
  module Receiving
5
- AVAILABLE_EVENTS = [:update, :destroy].freeze
6
-
7
5
  require_relative "receiving/config"
8
6
  require_relative "receiving/config_decorator"
9
7
  require_relative "receiving/dsl"
@@ -0,0 +1,22 @@
1
+ # frozen-string_literal: true
2
+
3
+ module TableSync::Setup
4
+ class ActiveRecord < Base
5
+ private
6
+
7
+ def define_after_commit(event)
8
+ options = options_exposed_for_block
9
+
10
+ object_class.after_commit(on: event) do
11
+ if instance_eval(&options[:if]) && !instance_eval(&options[:unless])
12
+ TableSync::Publishing::Single.new(
13
+ object_class: self.class.name,
14
+ original_attributes: attributes,
15
+ event: event,
16
+ debounce_time: options[:debounce_time],
17
+ ).publish_later
18
+ end
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,67 @@
1
+ # frozen-string_literal: true
2
+
3
+ module TableSync::Setup
4
+ class Base
5
+ include Tainbox
6
+
7
+ EVENTS = %i[create update destroy].freeze
8
+ INVALID_EVENT = Class.new(StandardError)
9
+ INVALID_CONDITION = Class.new(StandardError)
10
+
11
+ attribute :object_class
12
+ attribute :debounce_time
13
+ attribute :on
14
+ attribute :if_condition
15
+ attribute :unless_condition
16
+
17
+ def initialize(attrs)
18
+ super(attrs)
19
+
20
+ self.on = Array.wrap(on).map(&:to_sym)
21
+
22
+ self.if_condition ||= proc { true }
23
+ self.unless_condition ||= proc { false }
24
+
25
+ raise INVALID_EVENTS unless valid_events?
26
+ raise INVALID_CONDITION unless valid_conditions?
27
+ end
28
+
29
+ def register_callbacks
30
+ applicable_events.each { |event| define_after_commit(event) }
31
+ end
32
+
33
+ private
34
+
35
+ # VALIDATION
36
+
37
+ def valid_events?
38
+ on.all? { |event| event.in?(EVENTS) }
39
+ end
40
+
41
+ def valid_conditions?
42
+ if_condition.is_a?(Proc) && unless_condition.is_a?(Proc)
43
+ end
44
+
45
+ # EVENTS
46
+
47
+ def applicable_events
48
+ on.presence || EVENTS
49
+ end
50
+
51
+ # CREATING HOOKS
52
+
53
+ # :nocov:
54
+ def define_after_commit(event)
55
+ raise NotImplementedError
56
+ end
57
+ # :nocov:
58
+
59
+ def options_exposed_for_block
60
+ {
61
+ if: if_condition,
62
+ unless: unless_condition,
63
+ debounce_time: debounce_time,
64
+ }
65
+ end
66
+ end
67
+ end
@@ -0,0 +1,26 @@
1
+ # frozen-string_literal: true
2
+
3
+ module TableSync::Setup
4
+ class Sequel < Base
5
+ private
6
+
7
+ def define_after_commit(event)
8
+ options = options_exposed_for_block
9
+
10
+ object_class.define_method("after_#{event}".to_sym) do
11
+ if instance_eval(&options[:if]) && !instance_eval(&options[:unless])
12
+ db.after_commit do
13
+ TableSync::Publishing::Single.new(
14
+ object_class: self.class.name,
15
+ original_attributes: values,
16
+ event: event,
17
+ debounce_time: options[:debounce_time],
18
+ ).publish_later
19
+ end
20
+ end
21
+
22
+ super()
23
+ end
24
+ end
25
+ end
26
+ end
@@ -54,8 +54,8 @@ class TableSync::Utils::InterfaceChecker
54
54
  end
55
55
 
56
56
  def filter(parameters)
57
- # for req and block parameters types we can ignore names
58
- parameters.map { |param| %i[req block].include?(param.first) ? [param.first] : param }
57
+ ignored_keys = %i[req block] # for req and block parameters types we can ignore names
58
+ parameters.map { |param| ignored_keys.include?(param.first) ? [param.first] : param }
59
59
  end
60
60
  end
61
61
 
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module TableSync
4
- VERSION = "4.2.1"
4
+ VERSION = "6.0"
5
5
  end