table_sync 5.1.0 → 6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +42 -0
  3. data/Gemfile.lock +3 -3
  4. data/README.md +3 -0
  5. data/docs/publishing/configuration.md +143 -0
  6. data/docs/publishing/manual.md +155 -0
  7. data/docs/publishing/publishers.md +162 -0
  8. data/docs/publishing.md +35 -105
  9. data/lib/table_sync/errors.rb +31 -22
  10. data/lib/table_sync/event.rb +35 -0
  11. data/lib/table_sync/orm_adapter/active_record.rb +25 -0
  12. data/lib/table_sync/orm_adapter/base.rb +92 -0
  13. data/lib/table_sync/orm_adapter/sequel.rb +29 -0
  14. data/lib/table_sync/publishing/batch.rb +53 -0
  15. data/lib/table_sync/publishing/data/objects.rb +50 -0
  16. data/lib/table_sync/publishing/data/raw.rb +27 -0
  17. data/lib/table_sync/publishing/helpers/attributes.rb +63 -0
  18. data/lib/table_sync/publishing/helpers/debounce.rb +134 -0
  19. data/lib/table_sync/publishing/helpers/objects.rb +39 -0
  20. data/lib/table_sync/publishing/message/base.rb +73 -0
  21. data/lib/table_sync/publishing/message/batch.rb +14 -0
  22. data/lib/table_sync/publishing/message/raw.rb +54 -0
  23. data/lib/table_sync/publishing/message/single.rb +13 -0
  24. data/lib/table_sync/publishing/params/base.rb +66 -0
  25. data/lib/table_sync/publishing/params/batch.rb +23 -0
  26. data/lib/table_sync/publishing/params/raw.rb +7 -0
  27. data/lib/table_sync/publishing/params/single.rb +31 -0
  28. data/lib/table_sync/publishing/raw.rb +21 -0
  29. data/lib/table_sync/publishing/single.rb +65 -0
  30. data/lib/table_sync/publishing.rb +20 -5
  31. data/lib/table_sync/receiving/config.rb +6 -4
  32. data/lib/table_sync/receiving/handler.rb +10 -6
  33. data/lib/table_sync/receiving.rb +0 -2
  34. data/lib/table_sync/setup/active_record.rb +22 -0
  35. data/lib/table_sync/setup/base.rb +67 -0
  36. data/lib/table_sync/setup/sequel.rb +26 -0
  37. data/lib/table_sync/version.rb +1 -1
  38. data/lib/table_sync.rb +31 -8
  39. metadata +28 -7
  40. data/lib/table_sync/publishing/base_publisher.rb +0 -110
  41. data/lib/table_sync/publishing/batch_publisher.rb +0 -109
  42. data/lib/table_sync/publishing/orm_adapter/active_record.rb +0 -32
  43. data/lib/table_sync/publishing/orm_adapter/sequel.rb +0 -57
  44. data/lib/table_sync/publishing/publisher.rb +0 -129
@@ -0,0 +1,134 @@
1
+ # frozen_string_literal: true
2
+
3
+ # CASES FOR DEBOUNCE
4
+
5
+ # Cached Sync Time -> CST - time last sync has occured or next sync will occur
6
+ # Next Sync Time -> NST - time next sync will occur
7
+
8
+ # 0
9
+ # Condition: debounce_time is zero.
10
+ # No debounce, sync right now.
11
+ # Result: NST -> Time.current
12
+
13
+ # 1
14
+ # Condition: CST is empty.
15
+ # There was no sync before. Can be synced right now.
16
+ # Result: NST -> Time.current
17
+
18
+ # 2
19
+ # Condition: CST =< Time.current.
20
+ # There was a sync before.
21
+
22
+ # 2.1
23
+ # Subcondition: CST + debounce_time <= Time.current
24
+ # Enough time passed for debounce condition to be satisfied.
25
+ # No need to wait. Can by synced right now.
26
+ # Result: NST -> Time.current
27
+
28
+ # 2.2
29
+ # Subcondition: CST + debounce_time > Time.current
30
+ # Debounce condition is not satisfied. Must wait till debounce period has passed.
31
+ # Will be synced after debounce period has passed.
32
+ # Result: NST -> CST + debounce_time
33
+
34
+ # 3
35
+ # Condition: CST > Time.current
36
+ # Sync job has already been enqueued in the future.
37
+
38
+ # 3.1
39
+ # Subcondition: event -> update | create
40
+ # No need to sync upsert event, since it has already enqueued sync in the future.
41
+ # It will sync fresh version anyway.
42
+ # NST -> Skip, no sync.
43
+
44
+ # 3.2
45
+ # Subcondition: event -> destroy
46
+ # In this case the already enqueued job must be upsert.
47
+ # Thus destructive sync has to send message after upsert sync.
48
+ # NST -> CST + debounce_time
49
+
50
+ module TableSync::Publishing::Helpers
51
+ class Debounce
52
+ include Memery
53
+
54
+ DEFAULT_TIME = 60
55
+
56
+ attr_reader :debounce_time, :object_class, :needle, :event
57
+
58
+ def initialize(object_class:, needle:, event:, debounce_time: nil)
59
+ @event = event
60
+ @debounce_time = debounce_time || DEFAULT_TIME
61
+ @object_class = object_class
62
+ @needle = needle
63
+ end
64
+
65
+ def skip?
66
+ true if sync_in_the_future? && upsert_event? # case 3.1
67
+ end
68
+
69
+ memoize def next_sync_time
70
+ return current_time if debounce_time.zero? # case 0
71
+ return current_time if no_sync_before? # case 1
72
+
73
+ return current_time if sync_in_the_past? && debounce_time_passed? # case 2.1
74
+ return debounced_sync_time if sync_in_the_past? && debounce_time_not_passed? # case 2.2
75
+
76
+ return debounced_sync_time if sync_in_the_future? && destroy_event? # case 3.2
77
+ end
78
+
79
+ # CASE 1
80
+ def no_sync_before?
81
+ cached_sync_time.nil?
82
+ end
83
+
84
+ # CASE 2
85
+ def sync_in_the_past?
86
+ cached_sync_time <= current_time
87
+ end
88
+
89
+ def debounce_time_passed?
90
+ cached_sync_time + debounce_time.seconds >= current_time
91
+ end
92
+
93
+ def debounce_time_not_passed?
94
+ cached_sync_time + debounce_time.seconds < current_time
95
+ end
96
+
97
+ # CASE 3
98
+ def sync_in_the_future?
99
+ cached_sync_time && (cached_sync_time > current_time)
100
+ end
101
+
102
+ def destroy_event?
103
+ event == :destroy
104
+ end
105
+
106
+ def upsert_event?
107
+ !destroy_event?
108
+ end
109
+
110
+ # MISC
111
+
112
+ def debounced_sync_time
113
+ cached_sync_time + debounce_time.seconds
114
+ end
115
+
116
+ memoize def current_time
117
+ Time.current
118
+ end
119
+
120
+ # CACHE
121
+
122
+ memoize def cached_sync_time
123
+ Rails.cache.read(cache_key)
124
+ end
125
+
126
+ def cache_next_sync_time
127
+ Rails.cache.write(cache_key, next_sync_time)
128
+ end
129
+
130
+ def cache_key
131
+ "#{object_class}/#{needle.values.join}_table_sync_time".delete(" ")
132
+ end
133
+ end
134
+ end
@@ -0,0 +1,39 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TableSync::Publishing::Helpers
4
+ class Objects
5
+ attr_reader :object_class, :original_attributes, :event
6
+
7
+ def initialize(object_class:, original_attributes:, event:)
8
+ @event = TableSync::Event.new(event)
9
+ @object_class = object_class.constantize
10
+ @original_attributes = Array.wrap(original_attributes)
11
+ end
12
+
13
+ def construct_list
14
+ if event.destroy?
15
+ init_objects
16
+ else
17
+ without_empty_objects(find_objects)
18
+ end
19
+ end
20
+
21
+ private
22
+
23
+ def without_empty_objects(objects)
24
+ objects.reject(&:empty?)
25
+ end
26
+
27
+ def init_objects
28
+ original_attributes.map do |attrs|
29
+ TableSync.publishing_adapter.new(object_class, attrs).init
30
+ end
31
+ end
32
+
33
+ def find_objects
34
+ original_attributes.map do |attrs|
35
+ TableSync.publishing_adapter.new(object_class, attrs).find
36
+ end
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,73 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TableSync::Publishing::Message
4
+ class Base
5
+ include Tainbox
6
+
7
+ attr_reader :objects
8
+
9
+ attribute :object_class
10
+ attribute :original_attributes
11
+ attribute :event
12
+
13
+ def initialize(params)
14
+ super(params)
15
+
16
+ @objects = find_or_init_objects
17
+
18
+ raise TableSync::NoObjectsForSyncError if objects.empty? && TableSync.raise_on_empty_message
19
+ end
20
+
21
+ def publish
22
+ return if original_attributes.blank?
23
+
24
+ Rabbit.publish(message_params)
25
+
26
+ notify!
27
+ end
28
+
29
+ def empty?
30
+ objects.empty?
31
+ end
32
+
33
+ def find_or_init_objects
34
+ TableSync::Publishing::Helpers::Objects.new(
35
+ object_class: object_class, original_attributes: original_attributes, event: event,
36
+ ).construct_list
37
+ end
38
+
39
+ # MESSAGE PARAMS
40
+
41
+ def message_params
42
+ params.merge(data: data)
43
+ end
44
+
45
+ def data
46
+ TableSync::Publishing::Data::Objects.new(
47
+ objects: objects, event: event,
48
+ ).construct
49
+ end
50
+
51
+ # :nocov:
52
+ def params
53
+ raise NotImplementedError
54
+ end
55
+ # :nocov:
56
+
57
+ # NOTIFY
58
+
59
+ def notify!
60
+ TableSync::Instrument.notify(
61
+ table: model_naming.table,
62
+ schema: model_naming.schema,
63
+ event: event,
64
+ direction: :publish,
65
+ count: objects.count,
66
+ )
67
+ end
68
+
69
+ def model_naming
70
+ TableSync.publishing_adapter.model_naming(objects.first.object_class)
71
+ end
72
+ end
73
+ end
@@ -0,0 +1,14 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TableSync::Publishing::Message
4
+ class Batch < Base
5
+ attribute :headers
6
+ attribute :routing_key
7
+
8
+ def params
9
+ TableSync::Publishing::Params::Batch.new(
10
+ object_class: object_class, headers: headers, routing_key: routing_key,
11
+ ).construct
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,54 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TableSync::Publishing::Message
4
+ class Raw
5
+ include Tainbox
6
+
7
+ attribute :object_class
8
+ attribute :original_attributes
9
+ attribute :routing_key
10
+ attribute :headers
11
+
12
+ attribute :event
13
+
14
+ def publish
15
+ Rabbit.publish(message_params)
16
+
17
+ notify!
18
+ end
19
+
20
+ # NOTIFY
21
+
22
+ def notify!
23
+ TableSync::Instrument.notify(
24
+ table: model_naming.table,
25
+ schema: model_naming.schema,
26
+ event: event,
27
+ count: original_attributes.count,
28
+ direction: :publish,
29
+ )
30
+ end
31
+
32
+ def model_naming
33
+ TableSync.publishing_adapter.model_naming(object_class.constantize)
34
+ end
35
+
36
+ # MESSAGE PARAMS
37
+
38
+ def message_params
39
+ params.merge(data: data)
40
+ end
41
+
42
+ def data
43
+ TableSync::Publishing::Data::Raw.new(
44
+ object_class: object_class, attributes_for_sync: original_attributes, event: event,
45
+ ).construct
46
+ end
47
+
48
+ def params
49
+ TableSync::Publishing::Params::Raw.new(
50
+ object_class: object_class, routing_key: routing_key, headers: headers,
51
+ ).construct
52
+ end
53
+ end
54
+ end
@@ -0,0 +1,13 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TableSync::Publishing::Message
4
+ class Single < Base
5
+ def object
6
+ objects.first
7
+ end
8
+
9
+ def params
10
+ TableSync::Publishing::Params::Single.new(object: object).construct
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,66 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TableSync::Publishing::Params
4
+ class Base
5
+ DEFAULT_PARAMS = {
6
+ confirm_select: true,
7
+ realtime: true,
8
+ event: :table_sync,
9
+ }.freeze
10
+
11
+ def construct
12
+ DEFAULT_PARAMS.merge(
13
+ routing_key: routing_key,
14
+ headers: headers,
15
+ exchange_name: exchange_name,
16
+ )
17
+ end
18
+
19
+ private
20
+
21
+ def calculated_routing_key
22
+ if TableSync.routing_key_callable
23
+ TableSync.routing_key_callable.call(object_class, attributes_for_routing_key)
24
+ else
25
+ raise TableSync::NoCallableError.new("routing_key")
26
+ end
27
+ end
28
+
29
+ def calculated_headers
30
+ if TableSync.headers_callable
31
+ TableSync.headers_callable.call(object_class, attributes_for_headers)
32
+ else
33
+ raise TableSync::NoCallableError.new("headers")
34
+ end
35
+ end
36
+
37
+ # NOT IMPLEMENTED
38
+
39
+ # name of the model being synced in the string format
40
+ # :nocov:
41
+ def object_class
42
+ raise NotImplementedError
43
+ end
44
+
45
+ def routing_key
46
+ raise NotImplementedError
47
+ end
48
+
49
+ def headers
50
+ raise NotImplementedError
51
+ end
52
+
53
+ def exchange_name
54
+ raise NotImplementedError
55
+ end
56
+
57
+ def attributes_for_routing_key
58
+ raise NotImplementedError
59
+ end
60
+
61
+ def attributes_for_headers
62
+ raise NotImplementedError
63
+ end
64
+ # :nocov:
65
+ end
66
+ end
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TableSync::Publishing::Params
4
+ class Batch < Base
5
+ include Tainbox
6
+
7
+ attribute :object_class
8
+
9
+ attribute :exchange_name, default: -> { TableSync.exchange_name }
10
+ attribute :routing_key, default: -> { calculated_routing_key }
11
+ attribute :headers, default: -> { calculated_headers }
12
+
13
+ private
14
+
15
+ def attributes_for_routing_key
16
+ {}
17
+ end
18
+
19
+ def attributes_for_headers
20
+ {}
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,7 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TableSync::Publishing::Params
4
+ class Raw < Batch
5
+ # FOR NAMING CONSISTENCY
6
+ end
7
+ end
@@ -0,0 +1,31 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TableSync::Publishing::Params
4
+ class Single < Base
5
+ attr_reader :object, :routing_key, :headers
6
+
7
+ def initialize(object:)
8
+ @object = object
9
+ @routing_key = calculated_routing_key
10
+ @headers = calculated_headers
11
+ end
12
+
13
+ private
14
+
15
+ def object_class
16
+ object.object_class.name
17
+ end
18
+
19
+ def attributes_for_routing_key
20
+ object.attributes_for_routing_key
21
+ end
22
+
23
+ def attributes_for_headers
24
+ object.attributes_for_headers
25
+ end
26
+
27
+ def exchange_name
28
+ TableSync.exchange_name
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,21 @@
1
+ # frozen_string_literal: true
2
+
3
+ class TableSync::Publishing::Raw
4
+ include Tainbox
5
+
6
+ attribute :object_class
7
+ attribute :original_attributes
8
+
9
+ attribute :routing_key
10
+ attribute :headers
11
+
12
+ attribute :event, default: :update
13
+
14
+ def publish_now
15
+ message.publish
16
+ end
17
+
18
+ def message
19
+ TableSync::Publishing::Message::Raw.new(attributes)
20
+ end
21
+ end
@@ -0,0 +1,65 @@
1
+ # frozen_string_literal: true
2
+
3
+ class TableSync::Publishing::Single
4
+ include Tainbox
5
+ include Memery
6
+
7
+ attribute :object_class
8
+ attribute :original_attributes
9
+ attribute :debounce_time
10
+
11
+ attribute :event, Symbol, default: :update
12
+
13
+ # expect job to have perform_at method
14
+ # debounce destroyed event
15
+ # because otherwise update event could be sent after destroy
16
+ def publish_later
17
+ return if debounce.skip?
18
+
19
+ job.perform_at(job_attributes)
20
+
21
+ debounce.cache_next_sync_time
22
+ end
23
+
24
+ def publish_now
25
+ message.publish unless message.empty?
26
+ end
27
+
28
+ memoize def message
29
+ TableSync::Publishing::Message::Single.new(attributes)
30
+ end
31
+
32
+ memoize def debounce
33
+ TableSync::Publishing::Helpers::Debounce.new(
34
+ object_class: object_class,
35
+ needle: message.object.needle,
36
+ debounce_time: debounce_time,
37
+ event: event,
38
+ )
39
+ end
40
+
41
+ private
42
+
43
+ # JOB
44
+
45
+ def job
46
+ if TableSync.single_publishing_job_class_callable
47
+ TableSync.single_publishing_job_class_callable.call
48
+ else
49
+ raise TableSync::NoCallableError.new("single_publishing_job_class")
50
+ end
51
+ end
52
+
53
+ def job_attributes
54
+ attributes.merge(
55
+ original_attributes: serialized_original_attributes,
56
+ perform_at: debounce.next_sync_time,
57
+ )
58
+ end
59
+
60
+ def serialized_original_attributes
61
+ TableSync::Publishing::Helpers::Attributes
62
+ .new(original_attributes)
63
+ .serialize
64
+ end
65
+ end
@@ -2,10 +2,25 @@
2
2
 
3
3
  module TableSync
4
4
  module Publishing
5
- require_relative "publishing/base_publisher"
6
- require_relative "publishing/publisher"
7
- require_relative "publishing/batch_publisher"
8
- require_relative "publishing/orm_adapter/active_record"
9
- require_relative "publishing/orm_adapter/sequel"
5
+ require_relative "publishing/data/objects"
6
+ require_relative "publishing/data/raw"
7
+
8
+ require_relative "publishing/helpers/attributes"
9
+ require_relative "publishing/helpers/debounce"
10
+ require_relative "publishing/helpers/objects"
11
+
12
+ require_relative "publishing/params/base"
13
+ require_relative "publishing/params/batch"
14
+ require_relative "publishing/params/raw"
15
+ require_relative "publishing/params/single"
16
+
17
+ require_relative "publishing/message/base"
18
+ require_relative "publishing/message/batch"
19
+ require_relative "publishing/message/raw"
20
+ require_relative "publishing/message/single"
21
+
22
+ require_relative "publishing/batch"
23
+ require_relative "publishing/raw"
24
+ require_relative "publishing/single"
10
25
  end
11
26
  end
@@ -4,20 +4,22 @@ module TableSync::Receiving
4
4
  class Config
5
5
  attr_reader :model, :events
6
6
 
7
- def initialize(model:, events: AVAILABLE_EVENTS)
7
+ def initialize(model:, events: TableSync::Event::VALID_RESOLVED_EVENTS)
8
8
  @model = model
9
9
 
10
10
  @events = [events].flatten.map(&:to_sym)
11
11
 
12
- unless @events.all? { |event| AVAILABLE_EVENTS.include?(event) }
13
- raise TableSync::UndefinedEvent.new(events)
14
- end
12
+ raise TableSync::UndefinedEvent.new(events) if invalid_events.any?
15
13
 
16
14
  self.class.default_values_for_options.each do |ivar, default_value_generator|
17
15
  instance_variable_set(ivar, default_value_generator.call(self))
18
16
  end
19
17
  end
20
18
 
19
+ def invalid_events
20
+ events - TableSync::Event::VALID_RESOLVED_EVENTS
21
+ end
22
+
21
23
  class << self
22
24
  attr_reader :default_values_for_options
23
25
 
@@ -44,14 +44,18 @@ class TableSync::Receiving::Handler < Rabbit::EventHandler
44
44
  super(Array.wrap(data[:attributes]))
45
45
  end
46
46
 
47
- def event=(name)
48
- name = name.to_sym
49
- raise TableSync::UndefinedEvent.new(event) unless %i[update destroy].include?(name)
50
- super(name)
47
+ def event=(event_name)
48
+ event_name = event_name.to_sym
49
+
50
+ if event_name.in?(TableSync::Event::VALID_RESOLVED_EVENTS)
51
+ super(event_name)
52
+ else
53
+ raise TableSync::UndefinedEvent.new(event)
54
+ end
51
55
  end
52
56
 
53
- def model=(name)
54
- super(name.to_s)
57
+ def model=(model_name)
58
+ super(model_name.to_s)
55
59
  end
56
60
 
57
61
  def configs
@@ -2,8 +2,6 @@
2
2
 
3
3
  module TableSync
4
4
  module Receiving
5
- AVAILABLE_EVENTS = [:update, :destroy].freeze
6
-
7
5
  require_relative "receiving/config"
8
6
  require_relative "receiving/config_decorator"
9
7
  require_relative "receiving/dsl"
@@ -0,0 +1,22 @@
1
+ # frozen-string_literal: true
2
+
3
+ module TableSync::Setup
4
+ class ActiveRecord < Base
5
+ private
6
+
7
+ def define_after_commit(event)
8
+ options = options_exposed_for_block
9
+
10
+ object_class.after_commit(on: event) do
11
+ if instance_eval(&options[:if]) && !instance_eval(&options[:unless])
12
+ TableSync::Publishing::Single.new(
13
+ object_class: self.class.name,
14
+ original_attributes: attributes,
15
+ event: event,
16
+ debounce_time: options[:debounce_time],
17
+ ).publish_later
18
+ end
19
+ end
20
+ end
21
+ end
22
+ end