table_sync 1.13.1 → 4.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (46) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +0 -1
  3. data/.rubocop.yml +26 -1
  4. data/.travis.yml +6 -6
  5. data/CHANGELOG.md +74 -1
  6. data/Gemfile.lock +262 -0
  7. data/LICENSE.md +1 -1
  8. data/README.md +4 -1
  9. data/docs/message_protocol.md +24 -0
  10. data/docs/notifications.md +45 -0
  11. data/docs/publishing.md +147 -0
  12. data/docs/receiving.md +341 -0
  13. data/lib/table_sync.rb +23 -33
  14. data/lib/table_sync/errors.rb +60 -22
  15. data/lib/table_sync/instrument.rb +2 -2
  16. data/lib/table_sync/publishing.rb +11 -0
  17. data/lib/table_sync/{base_publisher.rb → publishing/base_publisher.rb} +1 -1
  18. data/lib/table_sync/{batch_publisher.rb → publishing/batch_publisher.rb} +12 -13
  19. data/lib/table_sync/{orm_adapter → publishing/orm_adapter}/active_record.rb +4 -8
  20. data/lib/table_sync/{orm_adapter → publishing/orm_adapter}/sequel.rb +10 -17
  21. data/lib/table_sync/{publisher.rb → publishing/publisher.rb} +4 -4
  22. data/lib/table_sync/receiving.rb +14 -0
  23. data/lib/table_sync/receiving/config.rb +218 -0
  24. data/lib/table_sync/receiving/config_decorator.rb +27 -0
  25. data/lib/table_sync/receiving/dsl.rb +28 -0
  26. data/lib/table_sync/receiving/handler.rb +131 -0
  27. data/lib/table_sync/{model → receiving/model}/active_record.rb +37 -23
  28. data/lib/table_sync/{model → receiving/model}/sequel.rb +13 -8
  29. data/lib/table_sync/utils.rb +9 -0
  30. data/lib/table_sync/utils/interface_checker.rb +97 -0
  31. data/lib/table_sync/utils/proc_array.rb +17 -0
  32. data/lib/table_sync/utils/proc_keywords_resolver.rb +46 -0
  33. data/lib/table_sync/version.rb +1 -1
  34. data/table_sync.gemspec +5 -4
  35. metadata +48 -30
  36. data/docs/synopsis.md +0 -318
  37. data/lib/table_sync/config.rb +0 -105
  38. data/lib/table_sync/config/callback_registry.rb +0 -53
  39. data/lib/table_sync/config_decorator.rb +0 -38
  40. data/lib/table_sync/dsl.rb +0 -25
  41. data/lib/table_sync/event_actions.rb +0 -96
  42. data/lib/table_sync/event_actions/data_wrapper.rb +0 -7
  43. data/lib/table_sync/event_actions/data_wrapper/base.rb +0 -23
  44. data/lib/table_sync/event_actions/data_wrapper/destroy.rb +0 -19
  45. data/lib/table_sync/event_actions/data_wrapper/update.rb +0 -21
  46. data/lib/table_sync/receiving_handler.rb +0 -76
@@ -1,105 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module TableSync
4
- class Config
5
- attr_reader :model, :events, :callback_registry
6
-
7
- def initialize(model:, events: nil)
8
- @model = model
9
- @events = events.nil? ? nil : [events].flatten.map(&:to_sym)
10
-
11
- @callback_registry = CallbackRegistry.new
12
-
13
- # initialize default options
14
- only(model.columns)
15
- mapping_overrides({})
16
- additional_data({})
17
- default_values({})
18
- @rest_key = :rest
19
- @version_key = :version
20
- @first_sync_time_key = nil
21
- @on_destroy = nil
22
- @wrap_receiving = nil
23
- target_keys(model.primary_keys)
24
- end
25
-
26
- # add_option implements the following logic
27
- # config.option - get value
28
- # config.option(args) - set static value
29
- # config.option { ... } - set proc as value
30
- def self.add_option(name, &option_block)
31
- ivar = "@#{name}".to_sym
32
-
33
- # default option handler (empty handler)
34
- # handles values when setting options
35
- option_block ||= proc { |value| value }
36
-
37
- define_method(name) do |*args, &block|
38
- # logic for each option
39
- if args.empty? && block.nil? # case for config.option, just return ivar
40
- instance_variable_get(ivar)
41
- elsif block # case for config.option { ... }
42
- # get named parameters (parameters with :keyreq) from proc-value
43
- params = block.parameters.map { |param| param[0] == :keyreq ? param[1] : nil }.compact
44
-
45
- # wrapper for proc-value, this wrapper receives all params (model, version, project_id...)
46
- # and filters them for proc-value
47
- unified_block = proc { |hash = {}| block.call(hash.slice(*params)) }
48
-
49
- # set wrapped proc-value as ivar value
50
- instance_variable_set(ivar, unified_block)
51
- else # case for config.option(args)
52
- # call option_block with args as params and set ivar to result
53
- instance_variable_set(ivar, instance_exec(*args, &option_block))
54
- end
55
- end
56
- end
57
-
58
- def allow_event?(name)
59
- return true if events.nil?
60
- events.include?(name)
61
- end
62
-
63
- def before_commit(on:, &block)
64
- callback_registry.register_callback(block, kind: :before_commit, event: on.to_sym)
65
- end
66
-
67
- def after_commit(on:, &block)
68
- callback_registry.register_callback(block, kind: :after_commit, event: on.to_sym)
69
- end
70
-
71
- def on_destroy(&block)
72
- block_given? ? @on_destroy = block : @on_destroy
73
- end
74
-
75
- def wrap_receiving(&block)
76
- block_given? ? @wrap_receiving = block : @wrap_receiving
77
- end
78
-
79
- check_and_set_column_key = proc do |key|
80
- key = key.to_sym
81
- raise "#{model.inspect} doesn't have key: #{key}" unless model.columns.include?(key)
82
- key
83
- end
84
-
85
- set_column_keys = proc do |*keys|
86
- [keys].flatten.map { |k| instance_exec(k, &check_and_set_column_key) }
87
- end
88
-
89
- add_option(:only, &set_column_keys)
90
- add_option(:target_keys, &set_column_keys)
91
- add_option(:rest_key) do |value|
92
- value ? instance_exec(value, &check_and_set_column_key) : nil
93
- end
94
- add_option(:version_key, &check_and_set_column_key)
95
- add_option(:first_sync_time_key) do |value|
96
- value ? instance_exec(value, &check_and_set_column_key) : nil
97
- end
98
-
99
- add_option(:mapping_overrides)
100
- add_option(:additional_data)
101
- add_option(:default_values)
102
- add_option(:partitions)
103
- add_option(:skip)
104
- end
105
- end
@@ -1,53 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- class TableSync::Config::CallbackRegistry
4
- CALLBACK_KINDS = %i[after_commit before_commit].freeze
5
- EVENTS = %i[create update destroy].freeze
6
-
7
- InvalidCallbackKindError = Class.new(ArgumentError)
8
- InvalidEventError = Class.new(ArgumentError)
9
-
10
- def initialize
11
- @callbacks = CALLBACK_KINDS.map { |event| [event, make_event_hash] }.to_h
12
- end
13
-
14
- def register_callback(callback, kind:, event:)
15
- validate_callback_kind!(kind)
16
- validate_event!(event)
17
-
18
- callbacks.fetch(kind)[event] << callback
19
- end
20
-
21
- def get_callbacks(kind:, event:)
22
- validate_callback_kind!(kind)
23
- validate_event!(event)
24
-
25
- callbacks.fetch(kind).fetch(event, [])
26
- end
27
-
28
- private
29
-
30
- attr_reader :callbacks
31
-
32
- def make_event_hash
33
- Hash.new { |hsh, key| hsh[key] = [] }
34
- end
35
-
36
- def validate_callback_kind!(kind)
37
- unless CALLBACK_KINDS.include?(kind)
38
- raise(
39
- InvalidCallbackKindError,
40
- "Invalid callback kind: #{kind.inspect}. Valid kinds are #{CALLBACK_KINDS.inspect}",
41
- )
42
- end
43
- end
44
-
45
- def validate_event!(event)
46
- unless EVENTS.include?(event)
47
- raise(
48
- InvalidEventError,
49
- "Invalid event: #{event.inspect}. Valid events are #{EVENTS.inspect}",
50
- )
51
- end
52
- end
53
- end
@@ -1,38 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module TableSync
4
- # rubocop:disable Style/MissingRespondToMissing, Style/MethodMissingSuper
5
- class ConfigDecorator
6
- include ::TableSync::EventActions
7
- extend Forwardable
8
-
9
- def_delegators :@config, :on_destroy, :wrap_receiving
10
-
11
- def initialize(config, handler)
12
- @config = config
13
- @handler = handler
14
- end
15
-
16
- def method_missing(name, *args)
17
- value = @config.send(name)
18
-
19
- if value.is_a?(Proc)
20
- value.call(
21
- event: @handler.event,
22
- model: @handler.model,
23
- version: @handler.version,
24
- project_id: @handler.project_id,
25
- data: @handler.data,
26
- current_row: args.first,
27
- )
28
- else
29
- value
30
- end
31
- end
32
-
33
- def allow_event?(event)
34
- @config.allow_event?(event)
35
- end
36
- end
37
- # rubocop:enable Style/MissingRespondToMissing, Style/MethodMissingSuper
38
- end
@@ -1,25 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module TableSync
4
- module DSL
5
- def inherited(klass)
6
- klass.instance_variable_set(:@configs, configs.deep_dup)
7
- super
8
- end
9
-
10
- def configs
11
- @configs ||= Hash.new { |hash, key| hash[key] = [] }
12
- end
13
-
14
- def receive(source, to_table:, events: nil, &block)
15
- config = ::TableSync::Config.new(
16
- model: TableSync.orm.model.new(to_table),
17
- events: events,
18
- )
19
-
20
- config.instance_exec(&block) if block
21
-
22
- configs[source.to_s] << config
23
- end
24
- end
25
- end
@@ -1,96 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module TableSync
4
- module EventActions
5
- def update(data)
6
- data.each_value do |attribute_set|
7
- attribute_set.each do |attributes|
8
- prevent_incomplete_event!(attributes)
9
- end
10
- end
11
-
12
- with_wrapping(DataWrapper::Update.new(data)) do
13
- process_upsert(data)
14
- end
15
- end
16
-
17
- def destroy(data)
18
- attributes = data.first || {}
19
- target_attributes = attributes.select { |key, _value| target_keys.include?(key) }
20
- prevent_incomplete_event!(target_attributes)
21
-
22
- with_wrapping(DataWrapper::Destroy.new(attributes)) do
23
- process_destroy(attributes, target_attributes)
24
- end
25
- end
26
-
27
- def process_upsert(data) # rubocop:disable Metrics/MethodLength
28
- model.transaction do
29
- args = {
30
- data: data,
31
- target_keys: target_keys,
32
- version_key: version_key,
33
- first_sync_time_key: first_sync_time_key,
34
- default_values: default_values,
35
- }
36
-
37
- @config.callback_registry.get_callbacks(kind: :before_commit, event: :update).each do |cb|
38
- cb[data.values.flatten]
39
- end
40
-
41
- results = data.reduce([]) do |upserts, (part_model, part_data)|
42
- upserts + part_model.upsert(**args, data: part_data)
43
- end
44
-
45
- return if results.empty?
46
- raise TableSync::UpsertError.new(**args) unless expected_update_result?(results)
47
-
48
- @config.model.after_commit do
49
- @config.callback_registry.get_callbacks(kind: :after_commit, event: :update).each do |cb|
50
- cb[results]
51
- end
52
- end
53
- end
54
- end
55
-
56
- def process_destroy(attributes, target_attributes)
57
- model.transaction do
58
- @config.callback_registry.get_callbacks(kind: :before_commit, event: :destroy).each do |cb|
59
- cb[attributes]
60
- end
61
-
62
- if on_destroy
63
- results = on_destroy.call(attributes: attributes, target_keys: target_keys)
64
- else
65
- results = model.destroy(target_attributes)
66
- return if results.empty?
67
- raise TableSync::DestroyError.new(target_attributes) if results.size != 1
68
- end
69
-
70
- @config.model.after_commit do
71
- @config.callback_registry.get_callbacks(kind: :after_commit, event: :destroy).each do |cb|
72
- cb[results]
73
- end
74
- end
75
- end
76
- end
77
-
78
- def with_wrapping(data = {}, &block)
79
- if @config.wrap_receiving
80
- @config.wrap_receiving.call(data, block)
81
- else
82
- yield
83
- end
84
- end
85
-
86
- def expected_update_result?(query_results)
87
- query_results.uniq { |d| d.slice(*target_keys) }.size == query_results.size
88
- end
89
-
90
- def prevent_incomplete_event!(attributes)
91
- unless target_keys.all?(&attributes.keys.method(:include?))
92
- raise TableSync::UnprovidedEventTargetKeysError.new(target_keys, attributes)
93
- end
94
- end
95
- end
96
- end
@@ -1,7 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module TableSync::EventActions::DataWrapper
4
- require_relative "data_wrapper/base"
5
- require_relative "data_wrapper/destroy"
6
- require_relative "data_wrapper/update"
7
- end
@@ -1,23 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- class TableSync::EventActions::DataWrapper::Base
4
- include Enumerable
5
-
6
- attr_reader :event_data
7
-
8
- def initialize(event_data)
9
- @event_data = event_data
10
- end
11
-
12
- def type
13
- raise NoMethodError # NOTE: for clarity
14
- end
15
-
16
- def destroy?
17
- false
18
- end
19
-
20
- def update?
21
- false
22
- end
23
- end
@@ -1,19 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- class TableSync::EventActions::DataWrapper::Destroy < TableSync::EventActions::DataWrapper::Base
4
- def type
5
- :destroy
6
- end
7
-
8
- def each
9
- yield(event_data)
10
- end
11
-
12
- def destroy?
13
- true
14
- end
15
-
16
- def update?
17
- false
18
- end
19
- end
@@ -1,21 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- class TableSync::EventActions::DataWrapper::Update < TableSync::EventActions::DataWrapper::Base
4
- def type
5
- :update
6
- end
7
-
8
- def each
9
- event_data.each_pair do |model_klass, changed_models_attrs|
10
- yield([model_klass, changed_models_attrs])
11
- end
12
- end
13
-
14
- def destroy?
15
- false
16
- end
17
-
18
- def update?
19
- true
20
- end
21
- end
@@ -1,76 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- class TableSync::ReceivingHandler < Rabbit::EventHandler
4
- extend TableSync::DSL
5
-
6
- attribute :event
7
- attribute :model
8
- attribute :version
9
-
10
- def call
11
- raise TableSync::UndefinedConfig.new(model) if configs.blank?
12
-
13
- configs.each do |config|
14
- next unless config.allow_event?(event)
15
-
16
- data = processed_data(config)
17
- next if data.empty?
18
-
19
- case event
20
- when :update
21
- config.model.transaction do
22
- config.update(data)
23
- end
24
- when :destroy
25
- config.destroy(data.values.first)
26
- else
27
- raise "Unknown event: #{event}"
28
- end
29
- end
30
- end
31
-
32
- private
33
-
34
- def data=(data)
35
- @data = data[:attributes]
36
- end
37
-
38
- def event=(name)
39
- super(name.to_sym)
40
- end
41
-
42
- def model=(name)
43
- super(name.to_s)
44
- end
45
-
46
- def configs
47
- @configs ||= self.class.configs[model]
48
- &.map { |c| ::TableSync::ConfigDecorator.new(c, self) }
49
- end
50
-
51
- def processed_data(config)
52
- parts = config.partitions&.transform_keys { |k| config.model.class.new(k) } ||
53
- { config.model => Array.wrap(data) }
54
-
55
- parts.transform_values! do |data_part|
56
- data_part.map do |row|
57
- original_row_for_data = row.dup
58
- row = row.dup
59
-
60
- config.mapping_overrides.each do |before, after|
61
- row[after] = row.delete(before)
62
- end
63
-
64
- only = config.only
65
- row, missed = row.partition { |key, _| key.in?(only) }.map(&:to_h)
66
-
67
- row.deep_merge!(config.rest_key => missed) if config.rest_key
68
- row[config.version_key] = version
69
-
70
- row.merge!(config.additional_data(original_row_for_data))
71
-
72
- row unless config.skip(original_row_for_data)
73
- end.compact.presence
74
- end.compact
75
- end
76
- end