table_sync 1.13.1 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +0 -1
  3. data/.rubocop.yml +26 -1
  4. data/.travis.yml +6 -6
  5. data/CHANGELOG.md +74 -1
  6. data/Gemfile.lock +262 -0
  7. data/LICENSE.md +1 -1
  8. data/README.md +4 -1
  9. data/docs/message_protocol.md +24 -0
  10. data/docs/notifications.md +45 -0
  11. data/docs/publishing.md +147 -0
  12. data/docs/receiving.md +341 -0
  13. data/lib/table_sync.rb +23 -33
  14. data/lib/table_sync/errors.rb +60 -22
  15. data/lib/table_sync/instrument.rb +2 -2
  16. data/lib/table_sync/publishing.rb +11 -0
  17. data/lib/table_sync/{base_publisher.rb → publishing/base_publisher.rb} +1 -1
  18. data/lib/table_sync/{batch_publisher.rb → publishing/batch_publisher.rb} +12 -13
  19. data/lib/table_sync/{orm_adapter → publishing/orm_adapter}/active_record.rb +4 -8
  20. data/lib/table_sync/{orm_adapter → publishing/orm_adapter}/sequel.rb +10 -17
  21. data/lib/table_sync/{publisher.rb → publishing/publisher.rb} +4 -4
  22. data/lib/table_sync/receiving.rb +14 -0
  23. data/lib/table_sync/receiving/config.rb +218 -0
  24. data/lib/table_sync/receiving/config_decorator.rb +27 -0
  25. data/lib/table_sync/receiving/dsl.rb +28 -0
  26. data/lib/table_sync/receiving/handler.rb +131 -0
  27. data/lib/table_sync/{model → receiving/model}/active_record.rb +37 -23
  28. data/lib/table_sync/{model → receiving/model}/sequel.rb +13 -8
  29. data/lib/table_sync/utils.rb +9 -0
  30. data/lib/table_sync/utils/interface_checker.rb +97 -0
  31. data/lib/table_sync/utils/proc_array.rb +17 -0
  32. data/lib/table_sync/utils/proc_keywords_resolver.rb +46 -0
  33. data/lib/table_sync/version.rb +1 -1
  34. data/table_sync.gemspec +5 -4
  35. metadata +48 -30
  36. data/docs/synopsis.md +0 -318
  37. data/lib/table_sync/config.rb +0 -105
  38. data/lib/table_sync/config/callback_registry.rb +0 -53
  39. data/lib/table_sync/config_decorator.rb +0 -38
  40. data/lib/table_sync/dsl.rb +0 -25
  41. data/lib/table_sync/event_actions.rb +0 -96
  42. data/lib/table_sync/event_actions/data_wrapper.rb +0 -7
  43. data/lib/table_sync/event_actions/data_wrapper/base.rb +0 -23
  44. data/lib/table_sync/event_actions/data_wrapper/destroy.rb +0 -19
  45. data/lib/table_sync/event_actions/data_wrapper/update.rb +0 -21
  46. data/lib/table_sync/receiving_handler.rb +0 -76
@@ -1,105 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module TableSync
4
- class Config
5
- attr_reader :model, :events, :callback_registry
6
-
7
- def initialize(model:, events: nil)
8
- @model = model
9
- @events = events.nil? ? nil : [events].flatten.map(&:to_sym)
10
-
11
- @callback_registry = CallbackRegistry.new
12
-
13
- # initialize default options
14
- only(model.columns)
15
- mapping_overrides({})
16
- additional_data({})
17
- default_values({})
18
- @rest_key = :rest
19
- @version_key = :version
20
- @first_sync_time_key = nil
21
- @on_destroy = nil
22
- @wrap_receiving = nil
23
- target_keys(model.primary_keys)
24
- end
25
-
26
- # add_option implements the following logic
27
- # config.option - get value
28
- # config.option(args) - set static value
29
- # config.option { ... } - set proc as value
30
- def self.add_option(name, &option_block)
31
- ivar = "@#{name}".to_sym
32
-
33
- # default option handler (empty handler)
34
- # handles values when setting options
35
- option_block ||= proc { |value| value }
36
-
37
- define_method(name) do |*args, &block|
38
- # logic for each option
39
- if args.empty? && block.nil? # case for config.option, just return ivar
40
- instance_variable_get(ivar)
41
- elsif block # case for config.option { ... }
42
- # get named parameters (parameters with :keyreq) from proc-value
43
- params = block.parameters.map { |param| param[0] == :keyreq ? param[1] : nil }.compact
44
-
45
- # wrapper for proc-value, this wrapper receives all params (model, version, project_id...)
46
- # and filters them for proc-value
47
- unified_block = proc { |hash = {}| block.call(hash.slice(*params)) }
48
-
49
- # set wrapped proc-value as ivar value
50
- instance_variable_set(ivar, unified_block)
51
- else # case for config.option(args)
52
- # call option_block with args as params and set ivar to result
53
- instance_variable_set(ivar, instance_exec(*args, &option_block))
54
- end
55
- end
56
- end
57
-
58
- def allow_event?(name)
59
- return true if events.nil?
60
- events.include?(name)
61
- end
62
-
63
- def before_commit(on:, &block)
64
- callback_registry.register_callback(block, kind: :before_commit, event: on.to_sym)
65
- end
66
-
67
- def after_commit(on:, &block)
68
- callback_registry.register_callback(block, kind: :after_commit, event: on.to_sym)
69
- end
70
-
71
- def on_destroy(&block)
72
- block_given? ? @on_destroy = block : @on_destroy
73
- end
74
-
75
- def wrap_receiving(&block)
76
- block_given? ? @wrap_receiving = block : @wrap_receiving
77
- end
78
-
79
- check_and_set_column_key = proc do |key|
80
- key = key.to_sym
81
- raise "#{model.inspect} doesn't have key: #{key}" unless model.columns.include?(key)
82
- key
83
- end
84
-
85
- set_column_keys = proc do |*keys|
86
- [keys].flatten.map { |k| instance_exec(k, &check_and_set_column_key) }
87
- end
88
-
89
- add_option(:only, &set_column_keys)
90
- add_option(:target_keys, &set_column_keys)
91
- add_option(:rest_key) do |value|
92
- value ? instance_exec(value, &check_and_set_column_key) : nil
93
- end
94
- add_option(:version_key, &check_and_set_column_key)
95
- add_option(:first_sync_time_key) do |value|
96
- value ? instance_exec(value, &check_and_set_column_key) : nil
97
- end
98
-
99
- add_option(:mapping_overrides)
100
- add_option(:additional_data)
101
- add_option(:default_values)
102
- add_option(:partitions)
103
- add_option(:skip)
104
- end
105
- end
@@ -1,53 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- class TableSync::Config::CallbackRegistry
4
- CALLBACK_KINDS = %i[after_commit before_commit].freeze
5
- EVENTS = %i[create update destroy].freeze
6
-
7
- InvalidCallbackKindError = Class.new(ArgumentError)
8
- InvalidEventError = Class.new(ArgumentError)
9
-
10
- def initialize
11
- @callbacks = CALLBACK_KINDS.map { |event| [event, make_event_hash] }.to_h
12
- end
13
-
14
- def register_callback(callback, kind:, event:)
15
- validate_callback_kind!(kind)
16
- validate_event!(event)
17
-
18
- callbacks.fetch(kind)[event] << callback
19
- end
20
-
21
- def get_callbacks(kind:, event:)
22
- validate_callback_kind!(kind)
23
- validate_event!(event)
24
-
25
- callbacks.fetch(kind).fetch(event, [])
26
- end
27
-
28
- private
29
-
30
- attr_reader :callbacks
31
-
32
- def make_event_hash
33
- Hash.new { |hsh, key| hsh[key] = [] }
34
- end
35
-
36
- def validate_callback_kind!(kind)
37
- unless CALLBACK_KINDS.include?(kind)
38
- raise(
39
- InvalidCallbackKindError,
40
- "Invalid callback kind: #{kind.inspect}. Valid kinds are #{CALLBACK_KINDS.inspect}",
41
- )
42
- end
43
- end
44
-
45
- def validate_event!(event)
46
- unless EVENTS.include?(event)
47
- raise(
48
- InvalidEventError,
49
- "Invalid event: #{event.inspect}. Valid events are #{EVENTS.inspect}",
50
- )
51
- end
52
- end
53
- end
@@ -1,38 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module TableSync
4
- # rubocop:disable Style/MissingRespondToMissing, Style/MethodMissingSuper
5
- class ConfigDecorator
6
- include ::TableSync::EventActions
7
- extend Forwardable
8
-
9
- def_delegators :@config, :on_destroy, :wrap_receiving
10
-
11
- def initialize(config, handler)
12
- @config = config
13
- @handler = handler
14
- end
15
-
16
- def method_missing(name, *args)
17
- value = @config.send(name)
18
-
19
- if value.is_a?(Proc)
20
- value.call(
21
- event: @handler.event,
22
- model: @handler.model,
23
- version: @handler.version,
24
- project_id: @handler.project_id,
25
- data: @handler.data,
26
- current_row: args.first,
27
- )
28
- else
29
- value
30
- end
31
- end
32
-
33
- def allow_event?(event)
34
- @config.allow_event?(event)
35
- end
36
- end
37
- # rubocop:enable Style/MissingRespondToMissing, Style/MethodMissingSuper
38
- end
@@ -1,25 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module TableSync
4
- module DSL
5
- def inherited(klass)
6
- klass.instance_variable_set(:@configs, configs.deep_dup)
7
- super
8
- end
9
-
10
- def configs
11
- @configs ||= Hash.new { |hash, key| hash[key] = [] }
12
- end
13
-
14
- def receive(source, to_table:, events: nil, &block)
15
- config = ::TableSync::Config.new(
16
- model: TableSync.orm.model.new(to_table),
17
- events: events,
18
- )
19
-
20
- config.instance_exec(&block) if block
21
-
22
- configs[source.to_s] << config
23
- end
24
- end
25
- end
@@ -1,96 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module TableSync
4
- module EventActions
5
- def update(data)
6
- data.each_value do |attribute_set|
7
- attribute_set.each do |attributes|
8
- prevent_incomplete_event!(attributes)
9
- end
10
- end
11
-
12
- with_wrapping(DataWrapper::Update.new(data)) do
13
- process_upsert(data)
14
- end
15
- end
16
-
17
- def destroy(data)
18
- attributes = data.first || {}
19
- target_attributes = attributes.select { |key, _value| target_keys.include?(key) }
20
- prevent_incomplete_event!(target_attributes)
21
-
22
- with_wrapping(DataWrapper::Destroy.new(attributes)) do
23
- process_destroy(attributes, target_attributes)
24
- end
25
- end
26
-
27
- def process_upsert(data) # rubocop:disable Metrics/MethodLength
28
- model.transaction do
29
- args = {
30
- data: data,
31
- target_keys: target_keys,
32
- version_key: version_key,
33
- first_sync_time_key: first_sync_time_key,
34
- default_values: default_values,
35
- }
36
-
37
- @config.callback_registry.get_callbacks(kind: :before_commit, event: :update).each do |cb|
38
- cb[data.values.flatten]
39
- end
40
-
41
- results = data.reduce([]) do |upserts, (part_model, part_data)|
42
- upserts + part_model.upsert(**args, data: part_data)
43
- end
44
-
45
- return if results.empty?
46
- raise TableSync::UpsertError.new(**args) unless expected_update_result?(results)
47
-
48
- @config.model.after_commit do
49
- @config.callback_registry.get_callbacks(kind: :after_commit, event: :update).each do |cb|
50
- cb[results]
51
- end
52
- end
53
- end
54
- end
55
-
56
- def process_destroy(attributes, target_attributes)
57
- model.transaction do
58
- @config.callback_registry.get_callbacks(kind: :before_commit, event: :destroy).each do |cb|
59
- cb[attributes]
60
- end
61
-
62
- if on_destroy
63
- results = on_destroy.call(attributes: attributes, target_keys: target_keys)
64
- else
65
- results = model.destroy(target_attributes)
66
- return if results.empty?
67
- raise TableSync::DestroyError.new(target_attributes) if results.size != 1
68
- end
69
-
70
- @config.model.after_commit do
71
- @config.callback_registry.get_callbacks(kind: :after_commit, event: :destroy).each do |cb|
72
- cb[results]
73
- end
74
- end
75
- end
76
- end
77
-
78
- def with_wrapping(data = {}, &block)
79
- if @config.wrap_receiving
80
- @config.wrap_receiving.call(data, block)
81
- else
82
- yield
83
- end
84
- end
85
-
86
- def expected_update_result?(query_results)
87
- query_results.uniq { |d| d.slice(*target_keys) }.size == query_results.size
88
- end
89
-
90
- def prevent_incomplete_event!(attributes)
91
- unless target_keys.all?(&attributes.keys.method(:include?))
92
- raise TableSync::UnprovidedEventTargetKeysError.new(target_keys, attributes)
93
- end
94
- end
95
- end
96
- end
@@ -1,7 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module TableSync::EventActions::DataWrapper
4
- require_relative "data_wrapper/base"
5
- require_relative "data_wrapper/destroy"
6
- require_relative "data_wrapper/update"
7
- end
@@ -1,23 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- class TableSync::EventActions::DataWrapper::Base
4
- include Enumerable
5
-
6
- attr_reader :event_data
7
-
8
- def initialize(event_data)
9
- @event_data = event_data
10
- end
11
-
12
- def type
13
- raise NoMethodError # NOTE: for clarity
14
- end
15
-
16
- def destroy?
17
- false
18
- end
19
-
20
- def update?
21
- false
22
- end
23
- end
@@ -1,19 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- class TableSync::EventActions::DataWrapper::Destroy < TableSync::EventActions::DataWrapper::Base
4
- def type
5
- :destroy
6
- end
7
-
8
- def each
9
- yield(event_data)
10
- end
11
-
12
- def destroy?
13
- true
14
- end
15
-
16
- def update?
17
- false
18
- end
19
- end
@@ -1,21 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- class TableSync::EventActions::DataWrapper::Update < TableSync::EventActions::DataWrapper::Base
4
- def type
5
- :update
6
- end
7
-
8
- def each
9
- event_data.each_pair do |model_klass, changed_models_attrs|
10
- yield([model_klass, changed_models_attrs])
11
- end
12
- end
13
-
14
- def destroy?
15
- false
16
- end
17
-
18
- def update?
19
- true
20
- end
21
- end
@@ -1,76 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- class TableSync::ReceivingHandler < Rabbit::EventHandler
4
- extend TableSync::DSL
5
-
6
- attribute :event
7
- attribute :model
8
- attribute :version
9
-
10
- def call
11
- raise TableSync::UndefinedConfig.new(model) if configs.blank?
12
-
13
- configs.each do |config|
14
- next unless config.allow_event?(event)
15
-
16
- data = processed_data(config)
17
- next if data.empty?
18
-
19
- case event
20
- when :update
21
- config.model.transaction do
22
- config.update(data)
23
- end
24
- when :destroy
25
- config.destroy(data.values.first)
26
- else
27
- raise "Unknown event: #{event}"
28
- end
29
- end
30
- end
31
-
32
- private
33
-
34
- def data=(data)
35
- @data = data[:attributes]
36
- end
37
-
38
- def event=(name)
39
- super(name.to_sym)
40
- end
41
-
42
- def model=(name)
43
- super(name.to_s)
44
- end
45
-
46
- def configs
47
- @configs ||= self.class.configs[model]
48
- &.map { |c| ::TableSync::ConfigDecorator.new(c, self) }
49
- end
50
-
51
- def processed_data(config)
52
- parts = config.partitions&.transform_keys { |k| config.model.class.new(k) } ||
53
- { config.model => Array.wrap(data) }
54
-
55
- parts.transform_values! do |data_part|
56
- data_part.map do |row|
57
- original_row_for_data = row.dup
58
- row = row.dup
59
-
60
- config.mapping_overrides.each do |before, after|
61
- row[after] = row.delete(before)
62
- end
63
-
64
- only = config.only
65
- row, missed = row.partition { |key, _| key.in?(only) }.map(&:to_h)
66
-
67
- row.deep_merge!(config.rest_key => missed) if config.rest_key
68
- row[config.version_key] = version
69
-
70
- row.merge!(config.additional_data(original_row_for_data))
71
-
72
- row unless config.skip(original_row_for_data)
73
- end.compact.presence
74
- end.compact
75
- end
76
- end