table_sync 2.0.0 → 4.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +0 -1
- data/.rubocop.yml +26 -1
- data/.travis.yml +6 -6
- data/CHANGELOG.md +74 -0
- data/Gemfile.lock +262 -0
- data/LICENSE.md +1 -1
- data/README.md +4 -1
- data/docs/message_protocol.md +24 -0
- data/docs/notifications.md +45 -0
- data/docs/publishing.md +147 -0
- data/docs/receiving.md +341 -0
- data/lib/table_sync.rb +23 -33
- data/lib/table_sync/errors.rb +60 -22
- data/lib/table_sync/instrument.rb +2 -2
- data/lib/table_sync/publishing.rb +11 -0
- data/lib/table_sync/{base_publisher.rb → publishing/base_publisher.rb} +1 -1
- data/lib/table_sync/{batch_publisher.rb → publishing/batch_publisher.rb} +12 -13
- data/lib/table_sync/{orm_adapter → publishing/orm_adapter}/active_record.rb +4 -8
- data/lib/table_sync/{orm_adapter → publishing/orm_adapter}/sequel.rb +3 -7
- data/lib/table_sync/{publisher.rb → publishing/publisher.rb} +4 -4
- data/lib/table_sync/receiving.rb +14 -0
- data/lib/table_sync/receiving/config.rb +218 -0
- data/lib/table_sync/receiving/config_decorator.rb +27 -0
- data/lib/table_sync/receiving/dsl.rb +28 -0
- data/lib/table_sync/receiving/handler.rb +132 -0
- data/lib/table_sync/{model → receiving/model}/active_record.rb +44 -37
- data/lib/table_sync/receiving/model/sequel.rb +83 -0
- data/lib/table_sync/utils.rb +9 -0
- data/lib/table_sync/utils/interface_checker.rb +103 -0
- data/lib/table_sync/utils/proc_array.rb +17 -0
- data/lib/table_sync/utils/proc_keywords_resolver.rb +46 -0
- data/lib/table_sync/version.rb +1 -1
- data/table_sync.gemspec +5 -4
- metadata +51 -33
- data/docs/synopsis.md +0 -318
- data/lib/table_sync/config.rb +0 -105
- data/lib/table_sync/config/callback_registry.rb +0 -53
- data/lib/table_sync/config_decorator.rb +0 -38
- data/lib/table_sync/dsl.rb +0 -25
- data/lib/table_sync/event_actions.rb +0 -96
- data/lib/table_sync/event_actions/data_wrapper.rb +0 -7
- data/lib/table_sync/event_actions/data_wrapper/base.rb +0 -23
- data/lib/table_sync/event_actions/data_wrapper/destroy.rb +0 -19
- data/lib/table_sync/event_actions/data_wrapper/update.rb +0 -21
- data/lib/table_sync/model/sequel.rb +0 -88
- data/lib/table_sync/receiving_handler.rb +0 -76
@@ -0,0 +1,27 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module TableSync::Receiving
|
4
|
+
class ConfigDecorator
|
5
|
+
extend Forwardable
|
6
|
+
|
7
|
+
def_delegators :@config, :allow_event?
|
8
|
+
# rubocop:disable Metrics/ParameterLists
|
9
|
+
def initialize(config:, event:, model:, version:, project_id:, raw_data:)
|
10
|
+
@config = config
|
11
|
+
|
12
|
+
@default_params = {
|
13
|
+
event: event,
|
14
|
+
model: model,
|
15
|
+
version: version,
|
16
|
+
project_id: project_id,
|
17
|
+
raw_data: raw_data,
|
18
|
+
}
|
19
|
+
end
|
20
|
+
# rubocop:enable Metrics/ParameterLists
|
21
|
+
|
22
|
+
def method_missing(name, **additional_params, &block)
|
23
|
+
value = @config.send(name)
|
24
|
+
value.is_a?(Proc) ? value.call(@default_params.merge(additional_params), &block) : value
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module TableSync::Receiving
|
4
|
+
module DSL
|
5
|
+
def inherited(klass)
|
6
|
+
klass.instance_variable_set(:@configs, configs.deep_dup)
|
7
|
+
super
|
8
|
+
end
|
9
|
+
|
10
|
+
def configs
|
11
|
+
@configs ||= Hash.new { |hash, key| hash[key] = [] }
|
12
|
+
end
|
13
|
+
|
14
|
+
def receive(source, to_table: nil, to_model: nil, events: [:update, :destroy], &block)
|
15
|
+
model = to_table ? TableSync.receiving_model.new(to_table) : to_model
|
16
|
+
|
17
|
+
TableSync::Utils::InterfaceChecker.new(model).implements(:receiving_model)
|
18
|
+
|
19
|
+
config = ::TableSync::Receiving::Config.new(model: model, events: events)
|
20
|
+
|
21
|
+
config.instance_exec(&block) if block
|
22
|
+
|
23
|
+
configs[source.to_s] << config
|
24
|
+
|
25
|
+
self
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,132 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class TableSync::Receiving::Handler < Rabbit::EventHandler
|
4
|
+
extend TableSync::Receiving::DSL
|
5
|
+
|
6
|
+
# Rabbit::EventHandler uses Tainbox and performs `handler.new(message).call`
|
7
|
+
attribute :event
|
8
|
+
attribute :model
|
9
|
+
attribute :version
|
10
|
+
|
11
|
+
def call
|
12
|
+
configs.each do |config|
|
13
|
+
next unless config.allow_event?(event)
|
14
|
+
|
15
|
+
data = processed_data(config)
|
16
|
+
|
17
|
+
next if data.empty?
|
18
|
+
|
19
|
+
version_key = config.version_key(data: data)
|
20
|
+
data.each { |row| row[version_key] = version }
|
21
|
+
|
22
|
+
target_keys = config.target_keys(data: data)
|
23
|
+
|
24
|
+
validate_data(data, target_keys: target_keys)
|
25
|
+
|
26
|
+
params = { data: data, target_keys: target_keys, version_key: version_key }
|
27
|
+
|
28
|
+
if event == :update
|
29
|
+
params[:default_values] = config.default_values(data: data)
|
30
|
+
end
|
31
|
+
|
32
|
+
config.wrap_receiving(**params) do
|
33
|
+
perform(config, params)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
private
|
39
|
+
|
40
|
+
# redefine setter from Rabbit::EventHandler
|
41
|
+
def data=(data)
|
42
|
+
super(Array.wrap(data[:attributes]))
|
43
|
+
end
|
44
|
+
|
45
|
+
def event=(name)
|
46
|
+
name = name.to_sym
|
47
|
+
raise TableSync::UndefinedEvent.new(event) unless %i[update destroy].include?(name)
|
48
|
+
super(name)
|
49
|
+
end
|
50
|
+
|
51
|
+
def model=(name)
|
52
|
+
super(name.to_s)
|
53
|
+
end
|
54
|
+
|
55
|
+
def configs
|
56
|
+
@configs ||= self.class.configs[model]&.map do |config|
|
57
|
+
::TableSync::Receiving::ConfigDecorator.new(
|
58
|
+
config: config,
|
59
|
+
# next parameters will be send to each proc-options from config
|
60
|
+
event: event,
|
61
|
+
model: model,
|
62
|
+
version: version,
|
63
|
+
project_id: project_id,
|
64
|
+
raw_data: data,
|
65
|
+
)
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def processed_data(config)
|
70
|
+
data.map do |row|
|
71
|
+
next if config.skip(row: row)
|
72
|
+
|
73
|
+
row = row.dup
|
74
|
+
|
75
|
+
config.mapping_overrides(row: row).each do |before, after|
|
76
|
+
row[after] = row.delete(before)
|
77
|
+
end
|
78
|
+
|
79
|
+
config.except(row: row).each(&row.method(:delete))
|
80
|
+
|
81
|
+
row.merge!(config.additional_data(row: row))
|
82
|
+
|
83
|
+
only = config.only(row: row)
|
84
|
+
row, rest = row.partition { |key, _| key.in?(only) }.map(&:to_h)
|
85
|
+
|
86
|
+
rest_key = config.rest_key(row: row, rest: rest)
|
87
|
+
(row[rest_key] ||= {}).merge!(rest) if rest_key
|
88
|
+
|
89
|
+
row
|
90
|
+
end.compact
|
91
|
+
end
|
92
|
+
|
93
|
+
def validate_data(data, target_keys:)
|
94
|
+
data.each do |row|
|
95
|
+
next if target_keys.all?(&row.keys.method(:include?))
|
96
|
+
raise TableSync::DataError.new(
|
97
|
+
data, target_keys, "Some target keys not found in received attributes!"
|
98
|
+
)
|
99
|
+
end
|
100
|
+
|
101
|
+
if data.uniq { |row| row.slice(*target_keys) }.size != data.size
|
102
|
+
raise TableSync::DataError.new(
|
103
|
+
data, target_keys, "Duplicate rows found!"
|
104
|
+
)
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
def perform(config, params)
|
109
|
+
model = config.model
|
110
|
+
|
111
|
+
model.transaction do
|
112
|
+
results = if event == :update
|
113
|
+
config.before_update(**params)
|
114
|
+
model.upsert(**params)
|
115
|
+
else
|
116
|
+
config.before_destroy(**params)
|
117
|
+
model.destroy(**params)
|
118
|
+
end
|
119
|
+
|
120
|
+
model.after_commit do
|
121
|
+
TableSync::Instrument.notify table: model.table, schema: model.schema,
|
122
|
+
count: results.count, event: event, direction: :receive
|
123
|
+
end
|
124
|
+
|
125
|
+
if event == :update
|
126
|
+
model.after_commit { config.after_commit_on_update(**params.merge(results: results)) }
|
127
|
+
else
|
128
|
+
model.after_commit { config.after_commit_on_destroy(**params.merge(results: results)) }
|
129
|
+
end
|
130
|
+
end
|
131
|
+
end
|
132
|
+
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
module TableSync::Model
|
3
|
+
module TableSync::Receiving::Model
|
4
4
|
class ActiveRecord
|
5
5
|
class AfterCommitWrap
|
6
6
|
def initialize(&block)
|
@@ -18,11 +18,18 @@ module TableSync::Model
|
|
18
18
|
def trigger_transactional_callbacks?(*); end
|
19
19
|
end
|
20
20
|
|
21
|
+
attr_reader :table, :schema
|
22
|
+
|
21
23
|
def initialize(table_name)
|
22
24
|
@raw_model = Class.new(::ActiveRecord::Base) do
|
23
25
|
self.table_name = table_name
|
24
26
|
self.inheritance_column = nil
|
25
27
|
end
|
28
|
+
|
29
|
+
model_naming = ::TableSync::NamingResolver::ActiveRecord.new(table_name: table_name)
|
30
|
+
|
31
|
+
@table = model_naming.table.to_sym
|
32
|
+
@schema = model_naming.schema.to_sym
|
26
33
|
end
|
27
34
|
|
28
35
|
def columns
|
@@ -45,51 +52,55 @@ module TableSync::Model
|
|
45
52
|
AND kcu.constraint_name = tc.constraint_name
|
46
53
|
WHERE
|
47
54
|
t.table_schema NOT IN ('pg_catalog', 'information_schema')
|
48
|
-
AND t.table_schema = '#{
|
49
|
-
AND t.table_name = '#{
|
55
|
+
AND t.table_schema = '#{schema}'
|
56
|
+
AND t.table_name = '#{table}'
|
50
57
|
ORDER BY
|
51
58
|
kcu.ordinal_position
|
52
59
|
SQL
|
53
60
|
end
|
54
61
|
|
55
|
-
def upsert(data:, target_keys:, version_key:,
|
56
|
-
data
|
62
|
+
def upsert(data:, target_keys:, version_key:, default_values:)
|
63
|
+
data.map do |datum|
|
64
|
+
conditions = datum.select { |k| target_keys.include?(k) }
|
57
65
|
|
58
|
-
|
59
|
-
data.map do |datum|
|
60
|
-
conditions = datum.select { |k| target_keys.include?(k) }
|
66
|
+
row = raw_model.lock("FOR NO KEY UPDATE").where(conditions)
|
61
67
|
|
62
|
-
|
63
|
-
|
64
|
-
|
68
|
+
if row.to_a.size > 1
|
69
|
+
raise TableSync::UpsertError.new(data: datum, target_keys: target_keys, result: row)
|
70
|
+
end
|
65
71
|
|
66
|
-
|
67
|
-
else
|
68
|
-
create_data = default_values.merge(datum)
|
69
|
-
create_data[first_sync_time_key] = Time.current if first_sync_time_key
|
70
|
-
row = raw_model.create!(create_data)
|
71
|
-
end
|
72
|
+
row = row.first
|
72
73
|
|
73
|
-
|
74
|
-
|
75
|
-
end
|
74
|
+
if row
|
75
|
+
next if datum[version_key] <= row[version_key]
|
76
76
|
|
77
|
-
|
78
|
-
|
77
|
+
row.update!(datum)
|
78
|
+
else
|
79
|
+
create_data = default_values.merge(datum)
|
80
|
+
row = raw_model.create!(create_data)
|
81
|
+
end
|
79
82
|
|
80
|
-
|
83
|
+
row_to_hash(row)
|
84
|
+
end.compact
|
81
85
|
end
|
82
86
|
|
83
|
-
def destroy(data)
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
+
def destroy(data:, target_keys:, version_key:)
|
88
|
+
sanitized_data = data.map { |attr| attr.select { |key, _value| target_keys.include?(key) } }
|
89
|
+
|
90
|
+
query = nil
|
91
|
+
sanitized_data.each_with_index do |row, index|
|
92
|
+
if index == 0
|
93
|
+
query = raw_model.lock("FOR UPDATE").where(row)
|
94
|
+
else
|
95
|
+
query = query.or(raw_model.lock("FOR UPDATE").where(row))
|
96
|
+
end
|
87
97
|
end
|
88
98
|
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
99
|
+
result = query.destroy_all.map(&method(:row_to_hash))
|
100
|
+
|
101
|
+
if result.size > data.size
|
102
|
+
raise TableSync::DestroyError.new(data: data, target_keys: target_keys, result: result)
|
103
|
+
end
|
93
104
|
|
94
105
|
result
|
95
106
|
end
|
@@ -107,15 +118,11 @@ module TableSync::Model
|
|
107
118
|
attr_reader :raw_model
|
108
119
|
|
109
120
|
def db
|
110
|
-
|
111
|
-
end
|
112
|
-
|
113
|
-
def model_naming
|
114
|
-
::TableSync::NamingResolver::ActiveRecord.new(table_name: raw_model.table_name)
|
121
|
+
raw_model.connection
|
115
122
|
end
|
116
123
|
|
117
124
|
def row_to_hash(row)
|
118
|
-
row.attributes.
|
125
|
+
row.attributes.transform_keys(&:to_sym)
|
119
126
|
end
|
120
127
|
end
|
121
128
|
end
|
@@ -0,0 +1,83 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module TableSync::Receiving::Model
|
4
|
+
class Sequel
|
5
|
+
attr_reader :table, :schema
|
6
|
+
|
7
|
+
def initialize(table_name)
|
8
|
+
@raw_model = Class.new(::Sequel::Model(table_name)).tap(&:unrestrict_primary_key)
|
9
|
+
|
10
|
+
model_naming = ::TableSync::NamingResolver::Sequel.new(
|
11
|
+
table_name: table_name,
|
12
|
+
db: @raw_model.db,
|
13
|
+
)
|
14
|
+
|
15
|
+
@table = model_naming.table.to_sym
|
16
|
+
@schema = model_naming.schema.to_sym
|
17
|
+
end
|
18
|
+
|
19
|
+
def columns
|
20
|
+
dataset.columns
|
21
|
+
end
|
22
|
+
|
23
|
+
def primary_keys
|
24
|
+
[raw_model.primary_key].flatten
|
25
|
+
end
|
26
|
+
|
27
|
+
def upsert(data:, target_keys:, version_key:, default_values:)
|
28
|
+
qualified_version = ::Sequel.qualify(raw_model.table_name, version_key)
|
29
|
+
version_condition = ::Sequel.function(:coalesce, qualified_version, 0) <
|
30
|
+
::Sequel.qualify(:excluded, version_key)
|
31
|
+
|
32
|
+
upd_spec = update_spec(data.first.keys - target_keys)
|
33
|
+
data.map! { |d| default_values.merge(d) }
|
34
|
+
|
35
|
+
insert_data = type_cast(data)
|
36
|
+
|
37
|
+
dataset
|
38
|
+
.returning
|
39
|
+
.insert_conflict(target: target_keys, update: upd_spec, update_where: version_condition)
|
40
|
+
.multi_insert(insert_data)
|
41
|
+
end
|
42
|
+
|
43
|
+
def destroy(data:, target_keys:, version_key:)
|
44
|
+
sanitized_data = data.map { |attr| attr.select { |key, _value| target_keys.include?(key) } }
|
45
|
+
sanitized_data = type_cast(sanitized_data)
|
46
|
+
result = dataset.returning.where(::Sequel.|(*sanitized_data)).delete
|
47
|
+
|
48
|
+
if result.size > data.size
|
49
|
+
raise TableSync::DestroyError.new(data: data, target_keys: target_keys, result: result)
|
50
|
+
end
|
51
|
+
|
52
|
+
result
|
53
|
+
end
|
54
|
+
|
55
|
+
def transaction(&block)
|
56
|
+
db.transaction(&block)
|
57
|
+
end
|
58
|
+
|
59
|
+
def after_commit(&block)
|
60
|
+
db.after_commit(&block)
|
61
|
+
end
|
62
|
+
|
63
|
+
private
|
64
|
+
|
65
|
+
attr_reader :raw_model
|
66
|
+
|
67
|
+
def dataset
|
68
|
+
raw_model.dataset
|
69
|
+
end
|
70
|
+
|
71
|
+
def db
|
72
|
+
dataset.db
|
73
|
+
end
|
74
|
+
|
75
|
+
def type_cast(data)
|
76
|
+
data.map { |d| raw_model.new(d).values.keep_if { |k| d.key?(k) } }
|
77
|
+
end
|
78
|
+
|
79
|
+
def update_spec(keys)
|
80
|
+
keys.map { |key| [key, ::Sequel[:excluded][key]] }.to_h
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
@@ -0,0 +1,103 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# Ruby does not support interfaces, and there is no way to implement them.
|
4
|
+
# Interfaces check a methods of a class after the initialization of the class is complete.
|
5
|
+
# But in Ruby, the initialization of a class cannot be completed.
|
6
|
+
# In execution time we can open any class and add some methods (monkey patching).
|
7
|
+
# Ruby has `define_method`, singleton methods, etc.
|
8
|
+
#
|
9
|
+
# Duck typing is a necessary measure, the only one available in the Ruby architecture.
|
10
|
+
#
|
11
|
+
# Interfaces can be implemented in particular cases with tests for example.
|
12
|
+
# But this is not suitable for gems that are used by third-party code.
|
13
|
+
#
|
14
|
+
# So, we still want to check interfaces and have a nice error messages,
|
15
|
+
# even if it will be duck typing.
|
16
|
+
#
|
17
|
+
# Next code do this.
|
18
|
+
|
19
|
+
class TableSync::Utils::InterfaceChecker
|
20
|
+
INTERFACES = SelfData.load
|
21
|
+
|
22
|
+
attr_reader :object
|
23
|
+
|
24
|
+
def initialize(object)
|
25
|
+
@object = object
|
26
|
+
end
|
27
|
+
|
28
|
+
def implements(interface_name)
|
29
|
+
INTERFACES[interface_name].each do |method_name, options|
|
30
|
+
unless object.respond_to?(method_name)
|
31
|
+
raise_error(method_name, options)
|
32
|
+
end
|
33
|
+
|
34
|
+
unless include?(object.method(method_name).parameters, options[:parameters])
|
35
|
+
raise_error(method_name, options)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
self
|
39
|
+
end
|
40
|
+
|
41
|
+
private
|
42
|
+
|
43
|
+
def include?(checked, expected)
|
44
|
+
(filter(expected) - filter(checked)).empty?
|
45
|
+
end
|
46
|
+
|
47
|
+
def raise_error(method_name, options)
|
48
|
+
raise TableSync::InterfaceError.new(
|
49
|
+
object,
|
50
|
+
method_name,
|
51
|
+
options[:parameters],
|
52
|
+
options[:description],
|
53
|
+
)
|
54
|
+
end
|
55
|
+
|
56
|
+
def filter(parameters)
|
57
|
+
# for req and block parameters types we can ignore names
|
58
|
+
parameters.map { |param| %i[req block].include?(param.first) ? [param.first] : param }
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
__END__
|
63
|
+
:receiving_model:
|
64
|
+
:upsert:
|
65
|
+
:parameters:
|
66
|
+
- - :keyreq
|
67
|
+
- :data
|
68
|
+
- - :keyreq
|
69
|
+
- :target_keys
|
70
|
+
- - :keyreq
|
71
|
+
- :version_key
|
72
|
+
- - :keyreq
|
73
|
+
- :default_values
|
74
|
+
:description: "returns an array with updated rows"
|
75
|
+
:columns:
|
76
|
+
:parameters: []
|
77
|
+
:description: "returns all table columns"
|
78
|
+
:destroy:
|
79
|
+
:parameters:
|
80
|
+
- - :keyreq
|
81
|
+
- :data
|
82
|
+
- - :keyreq
|
83
|
+
- :target_keys
|
84
|
+
:description: "returns an array with destroyed rows"
|
85
|
+
:transaction:
|
86
|
+
:parameters:
|
87
|
+
- - :block
|
88
|
+
- :block
|
89
|
+
:description: "implements the database transaction"
|
90
|
+
:after_commit:
|
91
|
+
:parameters:
|
92
|
+
- - :block
|
93
|
+
- :block
|
94
|
+
:description: "executes the block after committing the transaction"
|
95
|
+
:primary_keys:
|
96
|
+
:parameters: []
|
97
|
+
:description: "returns an array with the primary_keys"
|
98
|
+
:table:
|
99
|
+
:parameters: []
|
100
|
+
:description: "returns an instance of Symbol"
|
101
|
+
:schema:
|
102
|
+
:parameters: []
|
103
|
+
:description: "returns an instance of Symbol"
|