table_sync 4.2.1 → 6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/ci.yml +52 -0
  3. data/CHANGELOG.md +75 -9
  4. data/Gemfile.lock +159 -152
  5. data/README.md +4 -1
  6. data/docs/message_protocol.md +3 -3
  7. data/docs/publishing/configuration.md +143 -0
  8. data/docs/publishing/manual.md +155 -0
  9. data/docs/publishing/publishers.md +162 -0
  10. data/docs/publishing.md +35 -119
  11. data/docs/receiving.md +11 -6
  12. data/lib/table_sync/errors.rb +31 -22
  13. data/lib/table_sync/event.rb +35 -0
  14. data/lib/table_sync/orm_adapter/active_record.rb +25 -0
  15. data/lib/table_sync/orm_adapter/base.rb +92 -0
  16. data/lib/table_sync/orm_adapter/sequel.rb +29 -0
  17. data/lib/table_sync/publishing/batch.rb +53 -0
  18. data/lib/table_sync/publishing/data/objects.rb +50 -0
  19. data/lib/table_sync/publishing/data/raw.rb +27 -0
  20. data/lib/table_sync/publishing/helpers/attributes.rb +63 -0
  21. data/lib/table_sync/publishing/helpers/debounce.rb +134 -0
  22. data/lib/table_sync/publishing/helpers/objects.rb +39 -0
  23. data/lib/table_sync/publishing/message/base.rb +73 -0
  24. data/lib/table_sync/publishing/message/batch.rb +14 -0
  25. data/lib/table_sync/publishing/message/raw.rb +54 -0
  26. data/lib/table_sync/publishing/message/single.rb +13 -0
  27. data/lib/table_sync/publishing/params/base.rb +66 -0
  28. data/lib/table_sync/publishing/params/batch.rb +23 -0
  29. data/lib/table_sync/publishing/params/raw.rb +7 -0
  30. data/lib/table_sync/publishing/params/single.rb +31 -0
  31. data/lib/table_sync/publishing/raw.rb +21 -0
  32. data/lib/table_sync/publishing/single.rb +65 -0
  33. data/lib/table_sync/publishing.rb +20 -5
  34. data/lib/table_sync/receiving/config.rb +6 -4
  35. data/lib/table_sync/receiving/handler.rb +25 -9
  36. data/lib/table_sync/receiving/model/active_record.rb +1 -1
  37. data/lib/table_sync/receiving.rb +0 -2
  38. data/lib/table_sync/setup/active_record.rb +22 -0
  39. data/lib/table_sync/setup/base.rb +67 -0
  40. data/lib/table_sync/setup/sequel.rb +26 -0
  41. data/lib/table_sync/utils/interface_checker.rb +2 -2
  42. data/lib/table_sync/version.rb +1 -1
  43. data/lib/table_sync.rb +31 -8
  44. data/table_sync.gemspec +7 -7
  45. metadata +58 -37
  46. data/.travis.yml +0 -34
  47. data/lib/table_sync/publishing/base_publisher.rb +0 -114
  48. data/lib/table_sync/publishing/batch_publisher.rb +0 -109
  49. data/lib/table_sync/publishing/orm_adapter/active_record.rb +0 -32
  50. data/lib/table_sync/publishing/orm_adapter/sequel.rb +0 -57
  51. data/lib/table_sync/publishing/publisher.rb +0 -129
data/docs/receiving.md CHANGED
@@ -68,7 +68,7 @@ The method receives following arguments
68
68
  - `events` - array of supported events (optional)
69
69
  - `block` - configuration block with options (optional)
70
70
 
71
- This method implements logic of mapping `source` to `to_table` (or to `to_model`) and allows customizing
71
+ This method implements logic of mapping `source` to `to_table` (or to `to_model`) and allows customizing
72
72
  the event handling logic with provided block.
73
73
  You can use one `source` for a lot of `to_table` or `to_moel`.
74
74
 
@@ -235,12 +235,13 @@ default value is `false`
235
235
  Proc that is used to wrap the receiving logic by custom block of code.
236
236
 
237
237
  ```ruby
238
- wrap_receiving do |data:, target_keys:, version_key:, default_values: {}, &receiving_logic|
238
+ wrap_receiving do |data:, target_keys:, version_key:, default_values: {}, event:, &receiving_logic|
239
239
  receiving_logic.call
240
240
  return makes no sense
241
241
  end
242
242
  ```
243
243
 
244
+ event option is current fired event
244
245
  default value is `proc { |&block| block.call }`
245
246
 
246
247
  #### before_update
@@ -262,15 +263,17 @@ end
262
263
  Perform code after updated data was committed.
263
264
 
264
265
  ```ruby
265
- after_commit_on_update do |data:, target_keys:, version_key:, default_values:|
266
+ after_commit_on_update do |data:, target_keys:, version_key:, default_values:, results:|
266
267
  return makes no sense
267
268
  end
268
269
 
269
- after_commit_on_update do |data:, target_keys:, version_key:, default_values:|
270
+ after_commit_on_update do |data:, target_keys:, version_key:, default_values:, results:|
270
271
  return makes no sense
271
272
  end
272
273
  ```
273
274
 
275
+ - `results` - returned value from `model.upsert`
276
+
274
277
  Сan be defined several times. Execution order guaranteed.
275
278
 
276
279
  #### before_destroy
@@ -292,15 +295,17 @@ end
292
295
  Perform code after destroyed data was committed.
293
296
 
294
297
  ```ruby
295
- after_commit_on_destroy do |data:, target_keys:, version_key:|
298
+ after_commit_on_destroy do |data:, target_keys:, version_key:, results:|
296
299
  return makes no sense
297
300
  end
298
301
 
299
- after_commit_on_destroy do |data:, target_keys:, version_key:|
302
+ after_commit_on_destroy do |data:, target_keys:, version_key:, results:|
300
303
  return makes no sense
301
304
  end
302
305
  ```
303
306
 
307
+ - `results` - returned value from `model.destroy`
308
+
304
309
  Сan be defined several times. Execution order guaranteed.
305
310
 
306
311
  ### Custom model
@@ -3,6 +3,35 @@
3
3
  module TableSync
4
4
  Error = Class.new(StandardError)
5
5
 
6
+ NoObjectsForSyncError = Class.new(Error)
7
+
8
+ class EventError < Error
9
+ def initialize(event)
10
+ super(<<~MSG.squish)
11
+ Event #{event.inspect} is invalid.#{' '}
12
+ Expected: #{TableSync::Event::VALID_RAW_EVENTS.inspect}.
13
+ MSG
14
+ end
15
+ end
16
+
17
+ class NoPrimaryKeyError < Error
18
+ def initialize(object_class, object_data, primary_key_columns)
19
+ super(<<~MSG.squish)
20
+ Can't find or init an object of #{object_class} with #{object_data.inspect}.
21
+ Incomplete primary key! object_data must contain: #{primary_key_columns.inspect}.
22
+ MSG
23
+ end
24
+ end
25
+
26
+ class NoCallableError < Error
27
+ def initialize(type)
28
+ super(<<~MSG.squish)
29
+ Can't find callable for #{type}!
30
+ Please initialize TableSync.#{type}_callable with the correct proc!
31
+ MSG
32
+ end
33
+ end
34
+
6
35
  class UpsertError < Error
7
36
  def initialize(data:, target_keys:, result:)
8
37
  super("data: #{data.inspect}, target_keys: #{target_keys.inspect}, result: #{result.inspect}")
@@ -25,28 +54,6 @@ module TableSync
25
54
  end
26
55
  end
27
56
 
28
- # @api public
29
- # @since 2.2.0
30
- PluginError = Class.new(Error)
31
-
32
- # @api public
33
- # @since 2.2.0
34
- class UnregisteredPluginError < PluginError
35
- # @param plugin_name [Any]
36
- def initialize(plugin_name)
37
- super("#{plugin_name} plugin is not registered")
38
- end
39
- end
40
-
41
- # @api public
42
- # @since 2.2.0
43
- class AlreadyRegisteredPluginError < PluginError
44
- # @param plugin_name [Any]
45
- def initialize(plugin_name)
46
- super("#{plugin_name} plugin already exists")
47
- end
48
- end
49
-
50
57
  class InterfaceError < Error
51
58
  def initialize(object, method_name, parameters, description)
52
59
  parameters = parameters.map do |parameter|
@@ -54,7 +61,9 @@ module TableSync
54
61
 
55
62
  case type
56
63
  when :req
64
+ #:nocov:
57
65
  name.to_s
66
+ #:nocov:
58
67
  when :keyreq
59
68
  "#{name}:"
60
69
  when :block
@@ -0,0 +1,35 @@
1
+ # frozen_string_literal: true
2
+
3
+ class TableSync::Event
4
+ attr_reader :event
5
+
6
+ UPSERT_EVENTS = %i[create update].freeze
7
+ VALID_RESOLVED_EVENTS = %i[update destroy].freeze
8
+ VALID_RAW_EVENTS = %i[create update destroy].freeze
9
+
10
+ def initialize(event)
11
+ @event = event
12
+
13
+ validate!
14
+ end
15
+
16
+ def validate!
17
+ raise TableSync::EventError.new(event) unless event.in?(VALID_RAW_EVENTS)
18
+ end
19
+
20
+ def resolve
21
+ destroy? ? :destroy : :update
22
+ end
23
+
24
+ def metadata
25
+ { created: event == :create }
26
+ end
27
+
28
+ def destroy?
29
+ event == :destroy
30
+ end
31
+
32
+ def upsert?
33
+ event.in?(UPSERT_EVENTS)
34
+ end
35
+ end
@@ -0,0 +1,25 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TableSync::ORMAdapter
4
+ class ActiveRecord < Base
5
+ def find
6
+ @object = object_class.find_by(needle)
7
+
8
+ super
9
+ end
10
+
11
+ def init
12
+ @object = object_class.new(object_data)
13
+
14
+ super
15
+ end
16
+
17
+ def attributes
18
+ object.attributes.symbolize_keys
19
+ end
20
+
21
+ def self.model_naming(object_class)
22
+ TableSync::NamingResolver::ActiveRecord.new(table_name: object_class.table_name)
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,92 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TableSync::ORMAdapter
4
+ class Base
5
+ attr_reader :object, :object_class, :object_data
6
+
7
+ def initialize(object_class, object_data)
8
+ @object_class = object_class
9
+ @object_data = object_data.symbolize_keys
10
+
11
+ validate!
12
+ end
13
+
14
+ # VALIDATE
15
+
16
+ def validate!
17
+ if (primary_key_columns - object_data.keys).any?
18
+ raise TableSync::NoPrimaryKeyError.new(object_class, object_data, primary_key_columns)
19
+ end
20
+ end
21
+
22
+ # FIND OR INIT OBJECT
23
+
24
+ def init
25
+ self
26
+ end
27
+
28
+ def find
29
+ self
30
+ end
31
+
32
+ def needle
33
+ object_data.slice(*primary_key_columns)
34
+ end
35
+
36
+ # ATTRIBUTES
37
+
38
+ def attributes_for_update
39
+ if object.respond_to?(:attributes_for_sync)
40
+ object.attributes_for_sync
41
+ else
42
+ attributes
43
+ end
44
+ end
45
+
46
+ def attributes_for_destroy
47
+ if object.respond_to?(:attributes_for_destroy)
48
+ object.attributes_for_destroy
49
+ else
50
+ needle
51
+ end
52
+ end
53
+
54
+ def attributes_for_routing_key
55
+ if object.respond_to?(:attributes_for_routing_key)
56
+ object.attributes_for_routing_key
57
+ else
58
+ attributes
59
+ end
60
+ end
61
+
62
+ def attributes_for_headers
63
+ if object.respond_to?(:attributes_for_headers)
64
+ object.attributes_for_headers
65
+ else
66
+ attributes
67
+ end
68
+ end
69
+
70
+ def primary_key_columns
71
+ Array.wrap(object_class.primary_key).map(&:to_sym)
72
+ end
73
+
74
+ # MISC
75
+
76
+ def empty?
77
+ object.nil?
78
+ end
79
+
80
+ # NOT IMPLEMENTED
81
+
82
+ # :nocov:
83
+ def attributes
84
+ raise NotImplementedError
85
+ end
86
+
87
+ def self.model_naming
88
+ raise NotImplementedError
89
+ end
90
+ # :nocov:
91
+ end
92
+ end
@@ -0,0 +1,29 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TableSync::ORMAdapter
4
+ class Sequel < Base
5
+ def attributes
6
+ object.values
7
+ end
8
+
9
+ def init
10
+ @object = object_class.new(object_data.except(*primary_key_columns))
11
+
12
+ @object.set_fields(needle, needle.keys)
13
+
14
+ super
15
+ end
16
+
17
+ def find
18
+ @object = object_class.find(needle)
19
+
20
+ super
21
+ end
22
+
23
+ def self.model_naming(object_class)
24
+ TableSync::NamingResolver::Sequel.new(
25
+ table_name: object_class.table_name, db: object_class.db,
26
+ )
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,53 @@
1
+ # frozen_string_literal: true
2
+
3
+ class TableSync::Publishing::Batch
4
+ include Tainbox
5
+
6
+ attribute :object_class
7
+ attribute :original_attributes
8
+
9
+ attribute :routing_key
10
+ attribute :headers
11
+
12
+ attribute :event, default: :update
13
+
14
+ def publish_later
15
+ job.perform_later(job_attributes)
16
+ end
17
+
18
+ def publish_now
19
+ message.publish
20
+ end
21
+
22
+ def message
23
+ TableSync::Publishing::Message::Batch.new(attributes)
24
+ end
25
+
26
+ alias_method :publish_async, :publish_later
27
+
28
+ private
29
+
30
+ # JOB
31
+
32
+ def job
33
+ if TableSync.batch_publishing_job_class_callable
34
+ TableSync.batch_publishing_job_class_callable.call
35
+ else
36
+ raise TableSync::NoCallableError.new("batch_publishing_job_class")
37
+ end
38
+ end
39
+
40
+ def job_attributes
41
+ attributes.merge(
42
+ original_attributes: serialized_original_attributes,
43
+ )
44
+ end
45
+
46
+ def serialized_original_attributes
47
+ original_attributes.map do |set_of_attributes|
48
+ TableSync::Publishing::Helpers::Attributes
49
+ .new(set_of_attributes)
50
+ .serialize
51
+ end
52
+ end
53
+ end
@@ -0,0 +1,50 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TableSync::Publishing::Data
4
+ class Objects
5
+ attr_reader :objects, :event
6
+
7
+ def initialize(objects:, event:)
8
+ @objects = objects
9
+ @event = TableSync::Event.new(event)
10
+ end
11
+
12
+ def construct
13
+ {
14
+ model: model,
15
+ attributes: attributes_for_sync,
16
+ version: version,
17
+ event: event.resolve,
18
+ metadata: event.metadata,
19
+ }
20
+ end
21
+
22
+ private
23
+
24
+ def model
25
+ if object_class.respond_to?(:table_sync_model_name)
26
+ object_class.table_sync_model_name
27
+ else
28
+ object_class.name
29
+ end
30
+ end
31
+
32
+ def version
33
+ Time.current.to_f
34
+ end
35
+
36
+ def object_class
37
+ objects.first.object_class
38
+ end
39
+
40
+ def attributes_for_sync
41
+ objects.map do |object|
42
+ if event.destroy?
43
+ object.attributes_for_destroy
44
+ else
45
+ object.attributes_for_update
46
+ end
47
+ end
48
+ end
49
+ end
50
+ end
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TableSync::Publishing::Data
4
+ class Raw
5
+ attr_reader :object_class, :attributes_for_sync, :event
6
+
7
+ def initialize(object_class:, attributes_for_sync:, event:)
8
+ @object_class = object_class
9
+ @attributes_for_sync = attributes_for_sync
10
+ @event = TableSync::Event.new(event)
11
+ end
12
+
13
+ def construct
14
+ {
15
+ model: object_class,
16
+ attributes: attributes_for_sync,
17
+ version: version,
18
+ event: event.resolve,
19
+ metadata: event.metadata,
20
+ }
21
+ end
22
+
23
+ def version
24
+ Time.current.to_f
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,63 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TableSync::Publishing::Helpers
4
+ class Attributes
5
+ BASE_SAFE_JSON_TYPES = [
6
+ NilClass,
7
+ String,
8
+ TrueClass,
9
+ FalseClass,
10
+ Numeric,
11
+ Symbol,
12
+ ].freeze
13
+
14
+ # add custom seializables?
15
+
16
+ NOT_MAPPED = Object.new
17
+
18
+ attr_reader :attributes
19
+
20
+ def initialize(attributes)
21
+ @attributes = attributes.deep_symbolize_keys
22
+ end
23
+
24
+ def serialize
25
+ filter_safe_for_serialization(attributes)
26
+ end
27
+
28
+ def filter_safe_for_serialization(object)
29
+ case object
30
+ when Array
31
+ object.each_with_object([]) do |value, memo|
32
+ value = filter_safe_for_serialization(value)
33
+ memo << value if object_mapped?(value)
34
+ end
35
+ when Hash
36
+ object.each_with_object({}) do |(key, value), memo|
37
+ key = filter_safe_for_serialization(key)
38
+ value = filter_safe_hash_values(value)
39
+ memo[key] = value if object_mapped?(key) && object_mapped?(value)
40
+ end
41
+ when Float::INFINITY
42
+ NOT_MAPPED
43
+ when *BASE_SAFE_JSON_TYPES
44
+ object
45
+ else # rubocop:disable Lint/DuplicateBranch
46
+ NOT_MAPPED
47
+ end
48
+ end
49
+
50
+ def filter_safe_hash_values(value)
51
+ case value
52
+ when Symbol
53
+ value.to_s # why?
54
+ else
55
+ filter_safe_for_serialization(value)
56
+ end
57
+ end
58
+
59
+ def object_mapped?(object)
60
+ object != NOT_MAPPED
61
+ end
62
+ end
63
+ end
@@ -0,0 +1,134 @@
1
+ # frozen_string_literal: true
2
+
3
+ # CASES FOR DEBOUNCE
4
+
5
+ # Cached Sync Time -> CST - time last sync has occured or next sync will occur
6
+ # Next Sync Time -> NST - time next sync will occur
7
+
8
+ # 0
9
+ # Condition: debounce_time is zero.
10
+ # No debounce, sync right now.
11
+ # Result: NST -> Time.current
12
+
13
+ # 1
14
+ # Condition: CST is empty.
15
+ # There was no sync before. Can be synced right now.
16
+ # Result: NST -> Time.current
17
+
18
+ # 2
19
+ # Condition: CST =< Time.current.
20
+ # There was a sync before.
21
+
22
+ # 2.1
23
+ # Subcondition: CST + debounce_time <= Time.current
24
+ # Enough time passed for debounce condition to be satisfied.
25
+ # No need to wait. Can by synced right now.
26
+ # Result: NST -> Time.current
27
+
28
+ # 2.2
29
+ # Subcondition: CST + debounce_time > Time.current
30
+ # Debounce condition is not satisfied. Must wait till debounce period has passed.
31
+ # Will be synced after debounce period has passed.
32
+ # Result: NST -> CST + debounce_time
33
+
34
+ # 3
35
+ # Condition: CST > Time.current
36
+ # Sync job has already been enqueued in the future.
37
+
38
+ # 3.1
39
+ # Subcondition: event -> update | create
40
+ # No need to sync upsert event, since it has already enqueued sync in the future.
41
+ # It will sync fresh version anyway.
42
+ # NST -> Skip, no sync.
43
+
44
+ # 3.2
45
+ # Subcondition: event -> destroy
46
+ # In this case the already enqueued job must be upsert.
47
+ # Thus destructive sync has to send message after upsert sync.
48
+ # NST -> CST + debounce_time
49
+
50
+ module TableSync::Publishing::Helpers
51
+ class Debounce
52
+ include Memery
53
+
54
+ DEFAULT_TIME = 60
55
+
56
+ attr_reader :debounce_time, :object_class, :needle, :event
57
+
58
+ def initialize(object_class:, needle:, event:, debounce_time: nil)
59
+ @event = event
60
+ @debounce_time = debounce_time || DEFAULT_TIME
61
+ @object_class = object_class
62
+ @needle = needle
63
+ end
64
+
65
+ def skip?
66
+ true if sync_in_the_future? && upsert_event? # case 3.1
67
+ end
68
+
69
+ memoize def next_sync_time
70
+ return current_time if debounce_time.zero? # case 0
71
+ return current_time if no_sync_before? # case 1
72
+
73
+ return current_time if sync_in_the_past? && debounce_time_passed? # case 2.1
74
+ return debounced_sync_time if sync_in_the_past? && debounce_time_not_passed? # case 2.2
75
+
76
+ return debounced_sync_time if sync_in_the_future? && destroy_event? # case 3.2
77
+ end
78
+
79
+ # CASE 1
80
+ def no_sync_before?
81
+ cached_sync_time.nil?
82
+ end
83
+
84
+ # CASE 2
85
+ def sync_in_the_past?
86
+ cached_sync_time <= current_time
87
+ end
88
+
89
+ def debounce_time_passed?
90
+ cached_sync_time + debounce_time.seconds >= current_time
91
+ end
92
+
93
+ def debounce_time_not_passed?
94
+ cached_sync_time + debounce_time.seconds < current_time
95
+ end
96
+
97
+ # CASE 3
98
+ def sync_in_the_future?
99
+ cached_sync_time && (cached_sync_time > current_time)
100
+ end
101
+
102
+ def destroy_event?
103
+ event == :destroy
104
+ end
105
+
106
+ def upsert_event?
107
+ !destroy_event?
108
+ end
109
+
110
+ # MISC
111
+
112
+ def debounced_sync_time
113
+ cached_sync_time + debounce_time.seconds
114
+ end
115
+
116
+ memoize def current_time
117
+ Time.current
118
+ end
119
+
120
+ # CACHE
121
+
122
+ memoize def cached_sync_time
123
+ Rails.cache.read(cache_key)
124
+ end
125
+
126
+ def cache_next_sync_time
127
+ Rails.cache.write(cache_key, next_sync_time)
128
+ end
129
+
130
+ def cache_key
131
+ "#{object_class}/#{needle.values.join}_table_sync_time".delete(" ")
132
+ end
133
+ end
134
+ end
@@ -0,0 +1,39 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TableSync::Publishing::Helpers
4
+ class Objects
5
+ attr_reader :object_class, :original_attributes, :event
6
+
7
+ def initialize(object_class:, original_attributes:, event:)
8
+ @event = TableSync::Event.new(event)
9
+ @object_class = object_class.constantize
10
+ @original_attributes = Array.wrap(original_attributes)
11
+ end
12
+
13
+ def construct_list
14
+ if event.destroy?
15
+ init_objects
16
+ else
17
+ without_empty_objects(find_objects)
18
+ end
19
+ end
20
+
21
+ private
22
+
23
+ def without_empty_objects(objects)
24
+ objects.reject(&:empty?)
25
+ end
26
+
27
+ def init_objects
28
+ original_attributes.map do |attrs|
29
+ TableSync.publishing_adapter.new(object_class, attrs).init
30
+ end
31
+ end
32
+
33
+ def find_objects
34
+ original_attributes.map do |attrs|
35
+ TableSync.publishing_adapter.new(object_class, attrs).find
36
+ end
37
+ end
38
+ end
39
+ end