table_sync 1.13.1 → 4.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +0 -1
- data/.rubocop.yml +26 -1
- data/.travis.yml +6 -6
- data/CHANGELOG.md +74 -1
- data/Gemfile.lock +262 -0
- data/LICENSE.md +1 -1
- data/README.md +4 -1
- data/docs/message_protocol.md +24 -0
- data/docs/notifications.md +45 -0
- data/docs/publishing.md +147 -0
- data/docs/receiving.md +341 -0
- data/lib/table_sync.rb +23 -33
- data/lib/table_sync/errors.rb +60 -22
- data/lib/table_sync/instrument.rb +2 -2
- data/lib/table_sync/publishing.rb +11 -0
- data/lib/table_sync/{base_publisher.rb → publishing/base_publisher.rb} +1 -1
- data/lib/table_sync/{batch_publisher.rb → publishing/batch_publisher.rb} +12 -13
- data/lib/table_sync/{orm_adapter → publishing/orm_adapter}/active_record.rb +4 -8
- data/lib/table_sync/{orm_adapter → publishing/orm_adapter}/sequel.rb +10 -17
- data/lib/table_sync/{publisher.rb → publishing/publisher.rb} +4 -4
- data/lib/table_sync/receiving.rb +14 -0
- data/lib/table_sync/receiving/config.rb +218 -0
- data/lib/table_sync/receiving/config_decorator.rb +27 -0
- data/lib/table_sync/receiving/dsl.rb +28 -0
- data/lib/table_sync/receiving/handler.rb +131 -0
- data/lib/table_sync/{model → receiving/model}/active_record.rb +37 -23
- data/lib/table_sync/{model → receiving/model}/sequel.rb +13 -8
- data/lib/table_sync/utils.rb +9 -0
- data/lib/table_sync/utils/interface_checker.rb +97 -0
- data/lib/table_sync/utils/proc_array.rb +17 -0
- data/lib/table_sync/utils/proc_keywords_resolver.rb +46 -0
- data/lib/table_sync/version.rb +1 -1
- data/table_sync.gemspec +5 -4
- metadata +48 -30
- data/docs/synopsis.md +0 -318
- data/lib/table_sync/config.rb +0 -105
- data/lib/table_sync/config/callback_registry.rb +0 -53
- data/lib/table_sync/config_decorator.rb +0 -38
- data/lib/table_sync/dsl.rb +0 -25
- data/lib/table_sync/event_actions.rb +0 -96
- data/lib/table_sync/event_actions/data_wrapper.rb +0 -7
- data/lib/table_sync/event_actions/data_wrapper/base.rb +0 -23
- data/lib/table_sync/event_actions/data_wrapper/destroy.rb +0 -19
- data/lib/table_sync/event_actions/data_wrapper/update.rb +0 -21
- data/lib/table_sync/receiving_handler.rb +0 -76
data/lib/table_sync/config.rb
DELETED
@@ -1,105 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module TableSync
|
4
|
-
class Config
|
5
|
-
attr_reader :model, :events, :callback_registry
|
6
|
-
|
7
|
-
def initialize(model:, events: nil)
|
8
|
-
@model = model
|
9
|
-
@events = events.nil? ? nil : [events].flatten.map(&:to_sym)
|
10
|
-
|
11
|
-
@callback_registry = CallbackRegistry.new
|
12
|
-
|
13
|
-
# initialize default options
|
14
|
-
only(model.columns)
|
15
|
-
mapping_overrides({})
|
16
|
-
additional_data({})
|
17
|
-
default_values({})
|
18
|
-
@rest_key = :rest
|
19
|
-
@version_key = :version
|
20
|
-
@first_sync_time_key = nil
|
21
|
-
@on_destroy = nil
|
22
|
-
@wrap_receiving = nil
|
23
|
-
target_keys(model.primary_keys)
|
24
|
-
end
|
25
|
-
|
26
|
-
# add_option implements the following logic
|
27
|
-
# config.option - get value
|
28
|
-
# config.option(args) - set static value
|
29
|
-
# config.option { ... } - set proc as value
|
30
|
-
def self.add_option(name, &option_block)
|
31
|
-
ivar = "@#{name}".to_sym
|
32
|
-
|
33
|
-
# default option handler (empty handler)
|
34
|
-
# handles values when setting options
|
35
|
-
option_block ||= proc { |value| value }
|
36
|
-
|
37
|
-
define_method(name) do |*args, &block|
|
38
|
-
# logic for each option
|
39
|
-
if args.empty? && block.nil? # case for config.option, just return ivar
|
40
|
-
instance_variable_get(ivar)
|
41
|
-
elsif block # case for config.option { ... }
|
42
|
-
# get named parameters (parameters with :keyreq) from proc-value
|
43
|
-
params = block.parameters.map { |param| param[0] == :keyreq ? param[1] : nil }.compact
|
44
|
-
|
45
|
-
# wrapper for proc-value, this wrapper receives all params (model, version, project_id...)
|
46
|
-
# and filters them for proc-value
|
47
|
-
unified_block = proc { |hash = {}| block.call(hash.slice(*params)) }
|
48
|
-
|
49
|
-
# set wrapped proc-value as ivar value
|
50
|
-
instance_variable_set(ivar, unified_block)
|
51
|
-
else # case for config.option(args)
|
52
|
-
# call option_block with args as params and set ivar to result
|
53
|
-
instance_variable_set(ivar, instance_exec(*args, &option_block))
|
54
|
-
end
|
55
|
-
end
|
56
|
-
end
|
57
|
-
|
58
|
-
def allow_event?(name)
|
59
|
-
return true if events.nil?
|
60
|
-
events.include?(name)
|
61
|
-
end
|
62
|
-
|
63
|
-
def before_commit(on:, &block)
|
64
|
-
callback_registry.register_callback(block, kind: :before_commit, event: on.to_sym)
|
65
|
-
end
|
66
|
-
|
67
|
-
def after_commit(on:, &block)
|
68
|
-
callback_registry.register_callback(block, kind: :after_commit, event: on.to_sym)
|
69
|
-
end
|
70
|
-
|
71
|
-
def on_destroy(&block)
|
72
|
-
block_given? ? @on_destroy = block : @on_destroy
|
73
|
-
end
|
74
|
-
|
75
|
-
def wrap_receiving(&block)
|
76
|
-
block_given? ? @wrap_receiving = block : @wrap_receiving
|
77
|
-
end
|
78
|
-
|
79
|
-
check_and_set_column_key = proc do |key|
|
80
|
-
key = key.to_sym
|
81
|
-
raise "#{model.inspect} doesn't have key: #{key}" unless model.columns.include?(key)
|
82
|
-
key
|
83
|
-
end
|
84
|
-
|
85
|
-
set_column_keys = proc do |*keys|
|
86
|
-
[keys].flatten.map { |k| instance_exec(k, &check_and_set_column_key) }
|
87
|
-
end
|
88
|
-
|
89
|
-
add_option(:only, &set_column_keys)
|
90
|
-
add_option(:target_keys, &set_column_keys)
|
91
|
-
add_option(:rest_key) do |value|
|
92
|
-
value ? instance_exec(value, &check_and_set_column_key) : nil
|
93
|
-
end
|
94
|
-
add_option(:version_key, &check_and_set_column_key)
|
95
|
-
add_option(:first_sync_time_key) do |value|
|
96
|
-
value ? instance_exec(value, &check_and_set_column_key) : nil
|
97
|
-
end
|
98
|
-
|
99
|
-
add_option(:mapping_overrides)
|
100
|
-
add_option(:additional_data)
|
101
|
-
add_option(:default_values)
|
102
|
-
add_option(:partitions)
|
103
|
-
add_option(:skip)
|
104
|
-
end
|
105
|
-
end
|
@@ -1,53 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
class TableSync::Config::CallbackRegistry
|
4
|
-
CALLBACK_KINDS = %i[after_commit before_commit].freeze
|
5
|
-
EVENTS = %i[create update destroy].freeze
|
6
|
-
|
7
|
-
InvalidCallbackKindError = Class.new(ArgumentError)
|
8
|
-
InvalidEventError = Class.new(ArgumentError)
|
9
|
-
|
10
|
-
def initialize
|
11
|
-
@callbacks = CALLBACK_KINDS.map { |event| [event, make_event_hash] }.to_h
|
12
|
-
end
|
13
|
-
|
14
|
-
def register_callback(callback, kind:, event:)
|
15
|
-
validate_callback_kind!(kind)
|
16
|
-
validate_event!(event)
|
17
|
-
|
18
|
-
callbacks.fetch(kind)[event] << callback
|
19
|
-
end
|
20
|
-
|
21
|
-
def get_callbacks(kind:, event:)
|
22
|
-
validate_callback_kind!(kind)
|
23
|
-
validate_event!(event)
|
24
|
-
|
25
|
-
callbacks.fetch(kind).fetch(event, [])
|
26
|
-
end
|
27
|
-
|
28
|
-
private
|
29
|
-
|
30
|
-
attr_reader :callbacks
|
31
|
-
|
32
|
-
def make_event_hash
|
33
|
-
Hash.new { |hsh, key| hsh[key] = [] }
|
34
|
-
end
|
35
|
-
|
36
|
-
def validate_callback_kind!(kind)
|
37
|
-
unless CALLBACK_KINDS.include?(kind)
|
38
|
-
raise(
|
39
|
-
InvalidCallbackKindError,
|
40
|
-
"Invalid callback kind: #{kind.inspect}. Valid kinds are #{CALLBACK_KINDS.inspect}",
|
41
|
-
)
|
42
|
-
end
|
43
|
-
end
|
44
|
-
|
45
|
-
def validate_event!(event)
|
46
|
-
unless EVENTS.include?(event)
|
47
|
-
raise(
|
48
|
-
InvalidEventError,
|
49
|
-
"Invalid event: #{event.inspect}. Valid events are #{EVENTS.inspect}",
|
50
|
-
)
|
51
|
-
end
|
52
|
-
end
|
53
|
-
end
|
@@ -1,38 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module TableSync
|
4
|
-
# rubocop:disable Style/MissingRespondToMissing, Style/MethodMissingSuper
|
5
|
-
class ConfigDecorator
|
6
|
-
include ::TableSync::EventActions
|
7
|
-
extend Forwardable
|
8
|
-
|
9
|
-
def_delegators :@config, :on_destroy, :wrap_receiving
|
10
|
-
|
11
|
-
def initialize(config, handler)
|
12
|
-
@config = config
|
13
|
-
@handler = handler
|
14
|
-
end
|
15
|
-
|
16
|
-
def method_missing(name, *args)
|
17
|
-
value = @config.send(name)
|
18
|
-
|
19
|
-
if value.is_a?(Proc)
|
20
|
-
value.call(
|
21
|
-
event: @handler.event,
|
22
|
-
model: @handler.model,
|
23
|
-
version: @handler.version,
|
24
|
-
project_id: @handler.project_id,
|
25
|
-
data: @handler.data,
|
26
|
-
current_row: args.first,
|
27
|
-
)
|
28
|
-
else
|
29
|
-
value
|
30
|
-
end
|
31
|
-
end
|
32
|
-
|
33
|
-
def allow_event?(event)
|
34
|
-
@config.allow_event?(event)
|
35
|
-
end
|
36
|
-
end
|
37
|
-
# rubocop:enable Style/MissingRespondToMissing, Style/MethodMissingSuper
|
38
|
-
end
|
data/lib/table_sync/dsl.rb
DELETED
@@ -1,25 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module TableSync
|
4
|
-
module DSL
|
5
|
-
def inherited(klass)
|
6
|
-
klass.instance_variable_set(:@configs, configs.deep_dup)
|
7
|
-
super
|
8
|
-
end
|
9
|
-
|
10
|
-
def configs
|
11
|
-
@configs ||= Hash.new { |hash, key| hash[key] = [] }
|
12
|
-
end
|
13
|
-
|
14
|
-
def receive(source, to_table:, events: nil, &block)
|
15
|
-
config = ::TableSync::Config.new(
|
16
|
-
model: TableSync.orm.model.new(to_table),
|
17
|
-
events: events,
|
18
|
-
)
|
19
|
-
|
20
|
-
config.instance_exec(&block) if block
|
21
|
-
|
22
|
-
configs[source.to_s] << config
|
23
|
-
end
|
24
|
-
end
|
25
|
-
end
|
@@ -1,96 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module TableSync
|
4
|
-
module EventActions
|
5
|
-
def update(data)
|
6
|
-
data.each_value do |attribute_set|
|
7
|
-
attribute_set.each do |attributes|
|
8
|
-
prevent_incomplete_event!(attributes)
|
9
|
-
end
|
10
|
-
end
|
11
|
-
|
12
|
-
with_wrapping(DataWrapper::Update.new(data)) do
|
13
|
-
process_upsert(data)
|
14
|
-
end
|
15
|
-
end
|
16
|
-
|
17
|
-
def destroy(data)
|
18
|
-
attributes = data.first || {}
|
19
|
-
target_attributes = attributes.select { |key, _value| target_keys.include?(key) }
|
20
|
-
prevent_incomplete_event!(target_attributes)
|
21
|
-
|
22
|
-
with_wrapping(DataWrapper::Destroy.new(attributes)) do
|
23
|
-
process_destroy(attributes, target_attributes)
|
24
|
-
end
|
25
|
-
end
|
26
|
-
|
27
|
-
def process_upsert(data) # rubocop:disable Metrics/MethodLength
|
28
|
-
model.transaction do
|
29
|
-
args = {
|
30
|
-
data: data,
|
31
|
-
target_keys: target_keys,
|
32
|
-
version_key: version_key,
|
33
|
-
first_sync_time_key: first_sync_time_key,
|
34
|
-
default_values: default_values,
|
35
|
-
}
|
36
|
-
|
37
|
-
@config.callback_registry.get_callbacks(kind: :before_commit, event: :update).each do |cb|
|
38
|
-
cb[data.values.flatten]
|
39
|
-
end
|
40
|
-
|
41
|
-
results = data.reduce([]) do |upserts, (part_model, part_data)|
|
42
|
-
upserts + part_model.upsert(**args, data: part_data)
|
43
|
-
end
|
44
|
-
|
45
|
-
return if results.empty?
|
46
|
-
raise TableSync::UpsertError.new(**args) unless expected_update_result?(results)
|
47
|
-
|
48
|
-
@config.model.after_commit do
|
49
|
-
@config.callback_registry.get_callbacks(kind: :after_commit, event: :update).each do |cb|
|
50
|
-
cb[results]
|
51
|
-
end
|
52
|
-
end
|
53
|
-
end
|
54
|
-
end
|
55
|
-
|
56
|
-
def process_destroy(attributes, target_attributes)
|
57
|
-
model.transaction do
|
58
|
-
@config.callback_registry.get_callbacks(kind: :before_commit, event: :destroy).each do |cb|
|
59
|
-
cb[attributes]
|
60
|
-
end
|
61
|
-
|
62
|
-
if on_destroy
|
63
|
-
results = on_destroy.call(attributes: attributes, target_keys: target_keys)
|
64
|
-
else
|
65
|
-
results = model.destroy(target_attributes)
|
66
|
-
return if results.empty?
|
67
|
-
raise TableSync::DestroyError.new(target_attributes) if results.size != 1
|
68
|
-
end
|
69
|
-
|
70
|
-
@config.model.after_commit do
|
71
|
-
@config.callback_registry.get_callbacks(kind: :after_commit, event: :destroy).each do |cb|
|
72
|
-
cb[results]
|
73
|
-
end
|
74
|
-
end
|
75
|
-
end
|
76
|
-
end
|
77
|
-
|
78
|
-
def with_wrapping(data = {}, &block)
|
79
|
-
if @config.wrap_receiving
|
80
|
-
@config.wrap_receiving.call(data, block)
|
81
|
-
else
|
82
|
-
yield
|
83
|
-
end
|
84
|
-
end
|
85
|
-
|
86
|
-
def expected_update_result?(query_results)
|
87
|
-
query_results.uniq { |d| d.slice(*target_keys) }.size == query_results.size
|
88
|
-
end
|
89
|
-
|
90
|
-
def prevent_incomplete_event!(attributes)
|
91
|
-
unless target_keys.all?(&attributes.keys.method(:include?))
|
92
|
-
raise TableSync::UnprovidedEventTargetKeysError.new(target_keys, attributes)
|
93
|
-
end
|
94
|
-
end
|
95
|
-
end
|
96
|
-
end
|
@@ -1,23 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
class TableSync::EventActions::DataWrapper::Base
|
4
|
-
include Enumerable
|
5
|
-
|
6
|
-
attr_reader :event_data
|
7
|
-
|
8
|
-
def initialize(event_data)
|
9
|
-
@event_data = event_data
|
10
|
-
end
|
11
|
-
|
12
|
-
def type
|
13
|
-
raise NoMethodError # NOTE: for clarity
|
14
|
-
end
|
15
|
-
|
16
|
-
def destroy?
|
17
|
-
false
|
18
|
-
end
|
19
|
-
|
20
|
-
def update?
|
21
|
-
false
|
22
|
-
end
|
23
|
-
end
|
@@ -1,19 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
class TableSync::EventActions::DataWrapper::Destroy < TableSync::EventActions::DataWrapper::Base
|
4
|
-
def type
|
5
|
-
:destroy
|
6
|
-
end
|
7
|
-
|
8
|
-
def each
|
9
|
-
yield(event_data)
|
10
|
-
end
|
11
|
-
|
12
|
-
def destroy?
|
13
|
-
true
|
14
|
-
end
|
15
|
-
|
16
|
-
def update?
|
17
|
-
false
|
18
|
-
end
|
19
|
-
end
|
@@ -1,21 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
class TableSync::EventActions::DataWrapper::Update < TableSync::EventActions::DataWrapper::Base
|
4
|
-
def type
|
5
|
-
:update
|
6
|
-
end
|
7
|
-
|
8
|
-
def each
|
9
|
-
event_data.each_pair do |model_klass, changed_models_attrs|
|
10
|
-
yield([model_klass, changed_models_attrs])
|
11
|
-
end
|
12
|
-
end
|
13
|
-
|
14
|
-
def destroy?
|
15
|
-
false
|
16
|
-
end
|
17
|
-
|
18
|
-
def update?
|
19
|
-
true
|
20
|
-
end
|
21
|
-
end
|
@@ -1,76 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
class TableSync::ReceivingHandler < Rabbit::EventHandler
|
4
|
-
extend TableSync::DSL
|
5
|
-
|
6
|
-
attribute :event
|
7
|
-
attribute :model
|
8
|
-
attribute :version
|
9
|
-
|
10
|
-
def call
|
11
|
-
raise TableSync::UndefinedConfig.new(model) if configs.blank?
|
12
|
-
|
13
|
-
configs.each do |config|
|
14
|
-
next unless config.allow_event?(event)
|
15
|
-
|
16
|
-
data = processed_data(config)
|
17
|
-
next if data.empty?
|
18
|
-
|
19
|
-
case event
|
20
|
-
when :update
|
21
|
-
config.model.transaction do
|
22
|
-
config.update(data)
|
23
|
-
end
|
24
|
-
when :destroy
|
25
|
-
config.destroy(data.values.first)
|
26
|
-
else
|
27
|
-
raise "Unknown event: #{event}"
|
28
|
-
end
|
29
|
-
end
|
30
|
-
end
|
31
|
-
|
32
|
-
private
|
33
|
-
|
34
|
-
def data=(data)
|
35
|
-
@data = data[:attributes]
|
36
|
-
end
|
37
|
-
|
38
|
-
def event=(name)
|
39
|
-
super(name.to_sym)
|
40
|
-
end
|
41
|
-
|
42
|
-
def model=(name)
|
43
|
-
super(name.to_s)
|
44
|
-
end
|
45
|
-
|
46
|
-
def configs
|
47
|
-
@configs ||= self.class.configs[model]
|
48
|
-
&.map { |c| ::TableSync::ConfigDecorator.new(c, self) }
|
49
|
-
end
|
50
|
-
|
51
|
-
def processed_data(config)
|
52
|
-
parts = config.partitions&.transform_keys { |k| config.model.class.new(k) } ||
|
53
|
-
{ config.model => Array.wrap(data) }
|
54
|
-
|
55
|
-
parts.transform_values! do |data_part|
|
56
|
-
data_part.map do |row|
|
57
|
-
original_row_for_data = row.dup
|
58
|
-
row = row.dup
|
59
|
-
|
60
|
-
config.mapping_overrides.each do |before, after|
|
61
|
-
row[after] = row.delete(before)
|
62
|
-
end
|
63
|
-
|
64
|
-
only = config.only
|
65
|
-
row, missed = row.partition { |key, _| key.in?(only) }.map(&:to_h)
|
66
|
-
|
67
|
-
row.deep_merge!(config.rest_key => missed) if config.rest_key
|
68
|
-
row[config.version_key] = version
|
69
|
-
|
70
|
-
row.merge!(config.additional_data(original_row_for_data))
|
71
|
-
|
72
|
-
row unless config.skip(original_row_for_data)
|
73
|
-
end.compact.presence
|
74
|
-
end.compact
|
75
|
-
end
|
76
|
-
end
|