table_sync 1.13.1 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +0 -1
- data/.rubocop.yml +26 -1
- data/.travis.yml +6 -6
- data/CHANGELOG.md +74 -1
- data/Gemfile.lock +262 -0
- data/LICENSE.md +1 -1
- data/README.md +4 -1
- data/docs/message_protocol.md +24 -0
- data/docs/notifications.md +45 -0
- data/docs/publishing.md +147 -0
- data/docs/receiving.md +341 -0
- data/lib/table_sync.rb +23 -33
- data/lib/table_sync/errors.rb +60 -22
- data/lib/table_sync/instrument.rb +2 -2
- data/lib/table_sync/publishing.rb +11 -0
- data/lib/table_sync/{base_publisher.rb → publishing/base_publisher.rb} +1 -1
- data/lib/table_sync/{batch_publisher.rb → publishing/batch_publisher.rb} +12 -13
- data/lib/table_sync/{orm_adapter → publishing/orm_adapter}/active_record.rb +4 -8
- data/lib/table_sync/{orm_adapter → publishing/orm_adapter}/sequel.rb +10 -17
- data/lib/table_sync/{publisher.rb → publishing/publisher.rb} +4 -4
- data/lib/table_sync/receiving.rb +14 -0
- data/lib/table_sync/receiving/config.rb +218 -0
- data/lib/table_sync/receiving/config_decorator.rb +27 -0
- data/lib/table_sync/receiving/dsl.rb +28 -0
- data/lib/table_sync/receiving/handler.rb +131 -0
- data/lib/table_sync/{model → receiving/model}/active_record.rb +37 -23
- data/lib/table_sync/{model → receiving/model}/sequel.rb +13 -8
- data/lib/table_sync/utils.rb +9 -0
- data/lib/table_sync/utils/interface_checker.rb +97 -0
- data/lib/table_sync/utils/proc_array.rb +17 -0
- data/lib/table_sync/utils/proc_keywords_resolver.rb +46 -0
- data/lib/table_sync/version.rb +1 -1
- data/table_sync.gemspec +5 -4
- metadata +48 -30
- data/docs/synopsis.md +0 -318
- data/lib/table_sync/config.rb +0 -105
- data/lib/table_sync/config/callback_registry.rb +0 -53
- data/lib/table_sync/config_decorator.rb +0 -38
- data/lib/table_sync/dsl.rb +0 -25
- data/lib/table_sync/event_actions.rb +0 -96
- data/lib/table_sync/event_actions/data_wrapper.rb +0 -7
- data/lib/table_sync/event_actions/data_wrapper/base.rb +0 -23
- data/lib/table_sync/event_actions/data_wrapper/destroy.rb +0 -19
- data/lib/table_sync/event_actions/data_wrapper/update.rb +0 -21
- data/lib/table_sync/receiving_handler.rb +0 -76
data/lib/table_sync/config.rb
DELETED
@@ -1,105 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module TableSync
|
4
|
-
class Config
|
5
|
-
attr_reader :model, :events, :callback_registry
|
6
|
-
|
7
|
-
def initialize(model:, events: nil)
|
8
|
-
@model = model
|
9
|
-
@events = events.nil? ? nil : [events].flatten.map(&:to_sym)
|
10
|
-
|
11
|
-
@callback_registry = CallbackRegistry.new
|
12
|
-
|
13
|
-
# initialize default options
|
14
|
-
only(model.columns)
|
15
|
-
mapping_overrides({})
|
16
|
-
additional_data({})
|
17
|
-
default_values({})
|
18
|
-
@rest_key = :rest
|
19
|
-
@version_key = :version
|
20
|
-
@first_sync_time_key = nil
|
21
|
-
@on_destroy = nil
|
22
|
-
@wrap_receiving = nil
|
23
|
-
target_keys(model.primary_keys)
|
24
|
-
end
|
25
|
-
|
26
|
-
# add_option implements the following logic
|
27
|
-
# config.option - get value
|
28
|
-
# config.option(args) - set static value
|
29
|
-
# config.option { ... } - set proc as value
|
30
|
-
def self.add_option(name, &option_block)
|
31
|
-
ivar = "@#{name}".to_sym
|
32
|
-
|
33
|
-
# default option handler (empty handler)
|
34
|
-
# handles values when setting options
|
35
|
-
option_block ||= proc { |value| value }
|
36
|
-
|
37
|
-
define_method(name) do |*args, &block|
|
38
|
-
# logic for each option
|
39
|
-
if args.empty? && block.nil? # case for config.option, just return ivar
|
40
|
-
instance_variable_get(ivar)
|
41
|
-
elsif block # case for config.option { ... }
|
42
|
-
# get named parameters (parameters with :keyreq) from proc-value
|
43
|
-
params = block.parameters.map { |param| param[0] == :keyreq ? param[1] : nil }.compact
|
44
|
-
|
45
|
-
# wrapper for proc-value, this wrapper receives all params (model, version, project_id...)
|
46
|
-
# and filters them for proc-value
|
47
|
-
unified_block = proc { |hash = {}| block.call(hash.slice(*params)) }
|
48
|
-
|
49
|
-
# set wrapped proc-value as ivar value
|
50
|
-
instance_variable_set(ivar, unified_block)
|
51
|
-
else # case for config.option(args)
|
52
|
-
# call option_block with args as params and set ivar to result
|
53
|
-
instance_variable_set(ivar, instance_exec(*args, &option_block))
|
54
|
-
end
|
55
|
-
end
|
56
|
-
end
|
57
|
-
|
58
|
-
def allow_event?(name)
|
59
|
-
return true if events.nil?
|
60
|
-
events.include?(name)
|
61
|
-
end
|
62
|
-
|
63
|
-
def before_commit(on:, &block)
|
64
|
-
callback_registry.register_callback(block, kind: :before_commit, event: on.to_sym)
|
65
|
-
end
|
66
|
-
|
67
|
-
def after_commit(on:, &block)
|
68
|
-
callback_registry.register_callback(block, kind: :after_commit, event: on.to_sym)
|
69
|
-
end
|
70
|
-
|
71
|
-
def on_destroy(&block)
|
72
|
-
block_given? ? @on_destroy = block : @on_destroy
|
73
|
-
end
|
74
|
-
|
75
|
-
def wrap_receiving(&block)
|
76
|
-
block_given? ? @wrap_receiving = block : @wrap_receiving
|
77
|
-
end
|
78
|
-
|
79
|
-
check_and_set_column_key = proc do |key|
|
80
|
-
key = key.to_sym
|
81
|
-
raise "#{model.inspect} doesn't have key: #{key}" unless model.columns.include?(key)
|
82
|
-
key
|
83
|
-
end
|
84
|
-
|
85
|
-
set_column_keys = proc do |*keys|
|
86
|
-
[keys].flatten.map { |k| instance_exec(k, &check_and_set_column_key) }
|
87
|
-
end
|
88
|
-
|
89
|
-
add_option(:only, &set_column_keys)
|
90
|
-
add_option(:target_keys, &set_column_keys)
|
91
|
-
add_option(:rest_key) do |value|
|
92
|
-
value ? instance_exec(value, &check_and_set_column_key) : nil
|
93
|
-
end
|
94
|
-
add_option(:version_key, &check_and_set_column_key)
|
95
|
-
add_option(:first_sync_time_key) do |value|
|
96
|
-
value ? instance_exec(value, &check_and_set_column_key) : nil
|
97
|
-
end
|
98
|
-
|
99
|
-
add_option(:mapping_overrides)
|
100
|
-
add_option(:additional_data)
|
101
|
-
add_option(:default_values)
|
102
|
-
add_option(:partitions)
|
103
|
-
add_option(:skip)
|
104
|
-
end
|
105
|
-
end
|
@@ -1,53 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
class TableSync::Config::CallbackRegistry
|
4
|
-
CALLBACK_KINDS = %i[after_commit before_commit].freeze
|
5
|
-
EVENTS = %i[create update destroy].freeze
|
6
|
-
|
7
|
-
InvalidCallbackKindError = Class.new(ArgumentError)
|
8
|
-
InvalidEventError = Class.new(ArgumentError)
|
9
|
-
|
10
|
-
def initialize
|
11
|
-
@callbacks = CALLBACK_KINDS.map { |event| [event, make_event_hash] }.to_h
|
12
|
-
end
|
13
|
-
|
14
|
-
def register_callback(callback, kind:, event:)
|
15
|
-
validate_callback_kind!(kind)
|
16
|
-
validate_event!(event)
|
17
|
-
|
18
|
-
callbacks.fetch(kind)[event] << callback
|
19
|
-
end
|
20
|
-
|
21
|
-
def get_callbacks(kind:, event:)
|
22
|
-
validate_callback_kind!(kind)
|
23
|
-
validate_event!(event)
|
24
|
-
|
25
|
-
callbacks.fetch(kind).fetch(event, [])
|
26
|
-
end
|
27
|
-
|
28
|
-
private
|
29
|
-
|
30
|
-
attr_reader :callbacks
|
31
|
-
|
32
|
-
def make_event_hash
|
33
|
-
Hash.new { |hsh, key| hsh[key] = [] }
|
34
|
-
end
|
35
|
-
|
36
|
-
def validate_callback_kind!(kind)
|
37
|
-
unless CALLBACK_KINDS.include?(kind)
|
38
|
-
raise(
|
39
|
-
InvalidCallbackKindError,
|
40
|
-
"Invalid callback kind: #{kind.inspect}. Valid kinds are #{CALLBACK_KINDS.inspect}",
|
41
|
-
)
|
42
|
-
end
|
43
|
-
end
|
44
|
-
|
45
|
-
def validate_event!(event)
|
46
|
-
unless EVENTS.include?(event)
|
47
|
-
raise(
|
48
|
-
InvalidEventError,
|
49
|
-
"Invalid event: #{event.inspect}. Valid events are #{EVENTS.inspect}",
|
50
|
-
)
|
51
|
-
end
|
52
|
-
end
|
53
|
-
end
|
@@ -1,38 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module TableSync
|
4
|
-
# rubocop:disable Style/MissingRespondToMissing, Style/MethodMissingSuper
|
5
|
-
class ConfigDecorator
|
6
|
-
include ::TableSync::EventActions
|
7
|
-
extend Forwardable
|
8
|
-
|
9
|
-
def_delegators :@config, :on_destroy, :wrap_receiving
|
10
|
-
|
11
|
-
def initialize(config, handler)
|
12
|
-
@config = config
|
13
|
-
@handler = handler
|
14
|
-
end
|
15
|
-
|
16
|
-
def method_missing(name, *args)
|
17
|
-
value = @config.send(name)
|
18
|
-
|
19
|
-
if value.is_a?(Proc)
|
20
|
-
value.call(
|
21
|
-
event: @handler.event,
|
22
|
-
model: @handler.model,
|
23
|
-
version: @handler.version,
|
24
|
-
project_id: @handler.project_id,
|
25
|
-
data: @handler.data,
|
26
|
-
current_row: args.first,
|
27
|
-
)
|
28
|
-
else
|
29
|
-
value
|
30
|
-
end
|
31
|
-
end
|
32
|
-
|
33
|
-
def allow_event?(event)
|
34
|
-
@config.allow_event?(event)
|
35
|
-
end
|
36
|
-
end
|
37
|
-
# rubocop:enable Style/MissingRespondToMissing, Style/MethodMissingSuper
|
38
|
-
end
|
data/lib/table_sync/dsl.rb
DELETED
@@ -1,25 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module TableSync
|
4
|
-
module DSL
|
5
|
-
def inherited(klass)
|
6
|
-
klass.instance_variable_set(:@configs, configs.deep_dup)
|
7
|
-
super
|
8
|
-
end
|
9
|
-
|
10
|
-
def configs
|
11
|
-
@configs ||= Hash.new { |hash, key| hash[key] = [] }
|
12
|
-
end
|
13
|
-
|
14
|
-
def receive(source, to_table:, events: nil, &block)
|
15
|
-
config = ::TableSync::Config.new(
|
16
|
-
model: TableSync.orm.model.new(to_table),
|
17
|
-
events: events,
|
18
|
-
)
|
19
|
-
|
20
|
-
config.instance_exec(&block) if block
|
21
|
-
|
22
|
-
configs[source.to_s] << config
|
23
|
-
end
|
24
|
-
end
|
25
|
-
end
|
@@ -1,96 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module TableSync
|
4
|
-
module EventActions
|
5
|
-
def update(data)
|
6
|
-
data.each_value do |attribute_set|
|
7
|
-
attribute_set.each do |attributes|
|
8
|
-
prevent_incomplete_event!(attributes)
|
9
|
-
end
|
10
|
-
end
|
11
|
-
|
12
|
-
with_wrapping(DataWrapper::Update.new(data)) do
|
13
|
-
process_upsert(data)
|
14
|
-
end
|
15
|
-
end
|
16
|
-
|
17
|
-
def destroy(data)
|
18
|
-
attributes = data.first || {}
|
19
|
-
target_attributes = attributes.select { |key, _value| target_keys.include?(key) }
|
20
|
-
prevent_incomplete_event!(target_attributes)
|
21
|
-
|
22
|
-
with_wrapping(DataWrapper::Destroy.new(attributes)) do
|
23
|
-
process_destroy(attributes, target_attributes)
|
24
|
-
end
|
25
|
-
end
|
26
|
-
|
27
|
-
def process_upsert(data) # rubocop:disable Metrics/MethodLength
|
28
|
-
model.transaction do
|
29
|
-
args = {
|
30
|
-
data: data,
|
31
|
-
target_keys: target_keys,
|
32
|
-
version_key: version_key,
|
33
|
-
first_sync_time_key: first_sync_time_key,
|
34
|
-
default_values: default_values,
|
35
|
-
}
|
36
|
-
|
37
|
-
@config.callback_registry.get_callbacks(kind: :before_commit, event: :update).each do |cb|
|
38
|
-
cb[data.values.flatten]
|
39
|
-
end
|
40
|
-
|
41
|
-
results = data.reduce([]) do |upserts, (part_model, part_data)|
|
42
|
-
upserts + part_model.upsert(**args, data: part_data)
|
43
|
-
end
|
44
|
-
|
45
|
-
return if results.empty?
|
46
|
-
raise TableSync::UpsertError.new(**args) unless expected_update_result?(results)
|
47
|
-
|
48
|
-
@config.model.after_commit do
|
49
|
-
@config.callback_registry.get_callbacks(kind: :after_commit, event: :update).each do |cb|
|
50
|
-
cb[results]
|
51
|
-
end
|
52
|
-
end
|
53
|
-
end
|
54
|
-
end
|
55
|
-
|
56
|
-
def process_destroy(attributes, target_attributes)
|
57
|
-
model.transaction do
|
58
|
-
@config.callback_registry.get_callbacks(kind: :before_commit, event: :destroy).each do |cb|
|
59
|
-
cb[attributes]
|
60
|
-
end
|
61
|
-
|
62
|
-
if on_destroy
|
63
|
-
results = on_destroy.call(attributes: attributes, target_keys: target_keys)
|
64
|
-
else
|
65
|
-
results = model.destroy(target_attributes)
|
66
|
-
return if results.empty?
|
67
|
-
raise TableSync::DestroyError.new(target_attributes) if results.size != 1
|
68
|
-
end
|
69
|
-
|
70
|
-
@config.model.after_commit do
|
71
|
-
@config.callback_registry.get_callbacks(kind: :after_commit, event: :destroy).each do |cb|
|
72
|
-
cb[results]
|
73
|
-
end
|
74
|
-
end
|
75
|
-
end
|
76
|
-
end
|
77
|
-
|
78
|
-
def with_wrapping(data = {}, &block)
|
79
|
-
if @config.wrap_receiving
|
80
|
-
@config.wrap_receiving.call(data, block)
|
81
|
-
else
|
82
|
-
yield
|
83
|
-
end
|
84
|
-
end
|
85
|
-
|
86
|
-
def expected_update_result?(query_results)
|
87
|
-
query_results.uniq { |d| d.slice(*target_keys) }.size == query_results.size
|
88
|
-
end
|
89
|
-
|
90
|
-
def prevent_incomplete_event!(attributes)
|
91
|
-
unless target_keys.all?(&attributes.keys.method(:include?))
|
92
|
-
raise TableSync::UnprovidedEventTargetKeysError.new(target_keys, attributes)
|
93
|
-
end
|
94
|
-
end
|
95
|
-
end
|
96
|
-
end
|
@@ -1,23 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
class TableSync::EventActions::DataWrapper::Base
|
4
|
-
include Enumerable
|
5
|
-
|
6
|
-
attr_reader :event_data
|
7
|
-
|
8
|
-
def initialize(event_data)
|
9
|
-
@event_data = event_data
|
10
|
-
end
|
11
|
-
|
12
|
-
def type
|
13
|
-
raise NoMethodError # NOTE: for clarity
|
14
|
-
end
|
15
|
-
|
16
|
-
def destroy?
|
17
|
-
false
|
18
|
-
end
|
19
|
-
|
20
|
-
def update?
|
21
|
-
false
|
22
|
-
end
|
23
|
-
end
|
@@ -1,19 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
class TableSync::EventActions::DataWrapper::Destroy < TableSync::EventActions::DataWrapper::Base
|
4
|
-
def type
|
5
|
-
:destroy
|
6
|
-
end
|
7
|
-
|
8
|
-
def each
|
9
|
-
yield(event_data)
|
10
|
-
end
|
11
|
-
|
12
|
-
def destroy?
|
13
|
-
true
|
14
|
-
end
|
15
|
-
|
16
|
-
def update?
|
17
|
-
false
|
18
|
-
end
|
19
|
-
end
|
@@ -1,21 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
class TableSync::EventActions::DataWrapper::Update < TableSync::EventActions::DataWrapper::Base
|
4
|
-
def type
|
5
|
-
:update
|
6
|
-
end
|
7
|
-
|
8
|
-
def each
|
9
|
-
event_data.each_pair do |model_klass, changed_models_attrs|
|
10
|
-
yield([model_klass, changed_models_attrs])
|
11
|
-
end
|
12
|
-
end
|
13
|
-
|
14
|
-
def destroy?
|
15
|
-
false
|
16
|
-
end
|
17
|
-
|
18
|
-
def update?
|
19
|
-
true
|
20
|
-
end
|
21
|
-
end
|
@@ -1,76 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
class TableSync::ReceivingHandler < Rabbit::EventHandler
|
4
|
-
extend TableSync::DSL
|
5
|
-
|
6
|
-
attribute :event
|
7
|
-
attribute :model
|
8
|
-
attribute :version
|
9
|
-
|
10
|
-
def call
|
11
|
-
raise TableSync::UndefinedConfig.new(model) if configs.blank?
|
12
|
-
|
13
|
-
configs.each do |config|
|
14
|
-
next unless config.allow_event?(event)
|
15
|
-
|
16
|
-
data = processed_data(config)
|
17
|
-
next if data.empty?
|
18
|
-
|
19
|
-
case event
|
20
|
-
when :update
|
21
|
-
config.model.transaction do
|
22
|
-
config.update(data)
|
23
|
-
end
|
24
|
-
when :destroy
|
25
|
-
config.destroy(data.values.first)
|
26
|
-
else
|
27
|
-
raise "Unknown event: #{event}"
|
28
|
-
end
|
29
|
-
end
|
30
|
-
end
|
31
|
-
|
32
|
-
private
|
33
|
-
|
34
|
-
def data=(data)
|
35
|
-
@data = data[:attributes]
|
36
|
-
end
|
37
|
-
|
38
|
-
def event=(name)
|
39
|
-
super(name.to_sym)
|
40
|
-
end
|
41
|
-
|
42
|
-
def model=(name)
|
43
|
-
super(name.to_s)
|
44
|
-
end
|
45
|
-
|
46
|
-
def configs
|
47
|
-
@configs ||= self.class.configs[model]
|
48
|
-
&.map { |c| ::TableSync::ConfigDecorator.new(c, self) }
|
49
|
-
end
|
50
|
-
|
51
|
-
def processed_data(config)
|
52
|
-
parts = config.partitions&.transform_keys { |k| config.model.class.new(k) } ||
|
53
|
-
{ config.model => Array.wrap(data) }
|
54
|
-
|
55
|
-
parts.transform_values! do |data_part|
|
56
|
-
data_part.map do |row|
|
57
|
-
original_row_for_data = row.dup
|
58
|
-
row = row.dup
|
59
|
-
|
60
|
-
config.mapping_overrides.each do |before, after|
|
61
|
-
row[after] = row.delete(before)
|
62
|
-
end
|
63
|
-
|
64
|
-
only = config.only
|
65
|
-
row, missed = row.partition { |key, _| key.in?(only) }.map(&:to_h)
|
66
|
-
|
67
|
-
row.deep_merge!(config.rest_key => missed) if config.rest_key
|
68
|
-
row[config.version_key] = version
|
69
|
-
|
70
|
-
row.merge!(config.additional_data(original_row_for_data))
|
71
|
-
|
72
|
-
row unless config.skip(original_row_for_data)
|
73
|
-
end.compact.presence
|
74
|
-
end.compact
|
75
|
-
end
|
76
|
-
end
|