table_sync 2.3.0 → 4.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +21 -0
  3. data/CHANGELOG.md +57 -0
  4. data/Gemfile.lock +85 -80
  5. data/README.md +4 -2
  6. data/docs/message_protocol.md +24 -0
  7. data/docs/notifications.md +45 -0
  8. data/docs/publishing.md +147 -0
  9. data/docs/receiving.md +341 -0
  10. data/lib/table_sync.rb +16 -31
  11. data/lib/table_sync/errors.rb +39 -23
  12. data/lib/table_sync/publishing.rb +11 -0
  13. data/lib/table_sync/{base_publisher.rb → publishing/base_publisher.rb} +1 -1
  14. data/lib/table_sync/{batch_publisher.rb → publishing/batch_publisher.rb} +4 -4
  15. data/lib/table_sync/{orm_adapter → publishing/orm_adapter}/active_record.rb +3 -7
  16. data/lib/table_sync/{orm_adapter → publishing/orm_adapter}/sequel.rb +2 -6
  17. data/lib/table_sync/{publisher.rb → publishing/publisher.rb} +4 -4
  18. data/lib/table_sync/receiving.rb +14 -0
  19. data/lib/table_sync/receiving/config.rb +218 -0
  20. data/lib/table_sync/receiving/config_decorator.rb +27 -0
  21. data/lib/table_sync/receiving/dsl.rb +28 -0
  22. data/lib/table_sync/receiving/handler.rb +136 -0
  23. data/lib/table_sync/{model → receiving/model}/active_record.rb +43 -36
  24. data/lib/table_sync/receiving/model/sequel.rb +83 -0
  25. data/lib/table_sync/utils.rb +9 -0
  26. data/lib/table_sync/utils/interface_checker.rb +103 -0
  27. data/lib/table_sync/utils/proc_array.rb +17 -0
  28. data/lib/table_sync/utils/proc_keywords_resolver.rb +46 -0
  29. data/lib/table_sync/version.rb +1 -1
  30. data/table_sync.gemspec +2 -1
  31. metadata +45 -33
  32. data/docs/development.md +0 -43
  33. data/docs/synopsis.md +0 -336
  34. data/lib/table_sync/config.rb +0 -105
  35. data/lib/table_sync/config/callback_registry.rb +0 -53
  36. data/lib/table_sync/config_decorator.rb +0 -38
  37. data/lib/table_sync/dsl.rb +0 -25
  38. data/lib/table_sync/event_actions.rb +0 -96
  39. data/lib/table_sync/event_actions/data_wrapper.rb +0 -7
  40. data/lib/table_sync/event_actions/data_wrapper/base.rb +0 -23
  41. data/lib/table_sync/event_actions/data_wrapper/destroy.rb +0 -19
  42. data/lib/table_sync/event_actions/data_wrapper/update.rb +0 -21
  43. data/lib/table_sync/model/sequel.rb +0 -88
  44. data/lib/table_sync/plugins.rb +0 -72
  45. data/lib/table_sync/plugins/abstract.rb +0 -55
  46. data/lib/table_sync/plugins/access_mixin.rb +0 -49
  47. data/lib/table_sync/plugins/registry.rb +0 -153
  48. data/lib/table_sync/receiving_handler.rb +0 -76
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TableSync::Receiving
4
+ class ConfigDecorator
5
+ extend Forwardable
6
+
7
+ def_delegators :@config, :allow_event?
8
+ # rubocop:disable Metrics/ParameterLists
9
+ def initialize(config:, event:, model:, version:, project_id:, raw_data:)
10
+ @config = config
11
+
12
+ @default_params = {
13
+ event: event,
14
+ model: model,
15
+ version: version,
16
+ project_id: project_id,
17
+ raw_data: raw_data,
18
+ }
19
+ end
20
+ # rubocop:enable Metrics/ParameterLists
21
+
22
+ def method_missing(name, **additional_params, &block)
23
+ value = @config.send(name)
24
+ value.is_a?(Proc) ? value.call(@default_params.merge(additional_params), &block) : value
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,28 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TableSync::Receiving
4
+ module DSL
5
+ def inherited(klass)
6
+ klass.instance_variable_set(:@configs, configs.deep_dup)
7
+ super
8
+ end
9
+
10
+ def configs
11
+ @configs ||= Hash.new { |hash, key| hash[key] = [] }
12
+ end
13
+
14
+ def receive(source, to_table: nil, to_model: nil, events: [:update, :destroy], &block)
15
+ model = to_table ? TableSync.receiving_model.new(to_table) : to_model
16
+
17
+ TableSync::Utils::InterfaceChecker.new(model).implements(:receiving_model)
18
+
19
+ config = ::TableSync::Receiving::Config.new(model: model, events: events)
20
+
21
+ config.instance_exec(&block) if block
22
+
23
+ configs[source.to_s] << config
24
+
25
+ self
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,136 @@
1
+ # frozen_string_literal: true
2
+
3
+ class TableSync::Receiving::Handler < Rabbit::EventHandler
4
+ extend TableSync::Receiving::DSL
5
+
6
+ # Rabbit::EventHandler uses Tainbox and performs `handler.new(message).call`
7
+ attribute :event
8
+ attribute :model
9
+ attribute :version
10
+
11
+ def call
12
+ configs.each do |config|
13
+ next unless config.allow_event?(event)
14
+
15
+ data = processed_data(config)
16
+
17
+ next if data.empty?
18
+
19
+ version_key = config.version_key(data: data)
20
+ data.each { |row| row[version_key] = version }
21
+
22
+ target_keys = config.target_keys(data: data)
23
+
24
+ validate_data(data, target_keys: target_keys)
25
+
26
+ data.sort_by! { |row| row.values_at(*target_keys).hash }
27
+
28
+ params = { data: data, target_keys: target_keys, version_key: version_key }
29
+
30
+ if event == :update
31
+ params[:default_values] = config.default_values(data: data)
32
+ end
33
+
34
+ config.wrap_receiving(**params) do
35
+ perform(config, params)
36
+ end
37
+ end
38
+ end
39
+
40
+ private
41
+
42
+ # redefine setter from Rabbit::EventHandler
43
+ def data=(data)
44
+ super(Array.wrap(data[:attributes]))
45
+ end
46
+
47
+ def event=(name)
48
+ name = name.to_sym
49
+ raise TableSync::UndefinedEvent.new(event) unless %i[update destroy].include?(name)
50
+ super(name)
51
+ end
52
+
53
+ def model=(name)
54
+ super(name.to_s)
55
+ end
56
+
57
+ def configs
58
+ @configs ||= begin
59
+ configs = self.class.configs[model]
60
+ configs = configs.sort_by { |config| "#{config.model.schema}.#{config.model.table}" }
61
+ configs.map do |config|
62
+ ::TableSync::Receiving::ConfigDecorator.new(
63
+ config: config,
64
+ # next parameters will be send to each proc-options from config
65
+ event: event,
66
+ model: model,
67
+ version: version,
68
+ project_id: project_id,
69
+ raw_data: data,
70
+ )
71
+ end
72
+ end
73
+ end
74
+
75
+ def processed_data(config)
76
+ data.map do |row|
77
+ next if config.skip(row: row)
78
+
79
+ row = row.dup
80
+
81
+ config.mapping_overrides(row: row).each do |before, after|
82
+ row[after] = row.delete(before)
83
+ end
84
+
85
+ config.except(row: row).each(&row.method(:delete))
86
+
87
+ row.merge!(config.additional_data(row: row))
88
+
89
+ only = config.only(row: row)
90
+ row, rest = row.partition { |key, _| key.in?(only) }.map(&:to_h)
91
+
92
+ rest_key = config.rest_key(row: row, rest: rest)
93
+ (row[rest_key] ||= {}).merge!(rest) if rest_key
94
+
95
+ row
96
+ end.compact
97
+ end
98
+
99
+ def validate_data(data, target_keys:)
100
+ data.each do |row|
101
+ next if target_keys.all?(&row.keys.method(:include?))
102
+ raise TableSync::DataError.new(
103
+ data, target_keys, "Some target keys not found in received attributes!"
104
+ )
105
+ end
106
+
107
+ if data.uniq { |row| row.slice(*target_keys) }.size != data.size
108
+ raise TableSync::DataError.new(data, target_keys, "Duplicate rows found!")
109
+ end
110
+ end
111
+
112
+ def perform(config, params)
113
+ model = config.model
114
+
115
+ model.transaction do
116
+ results = if event == :update
117
+ config.before_update(**params)
118
+ model.upsert(**params)
119
+ else
120
+ config.before_destroy(**params)
121
+ model.destroy(**params)
122
+ end
123
+
124
+ model.after_commit do
125
+ TableSync::Instrument.notify table: model.table, schema: model.schema,
126
+ count: results.count, event: event, direction: :receive
127
+ end
128
+
129
+ if event == :update
130
+ model.after_commit { config.after_commit_on_update(**params.merge(results: results)) }
131
+ else
132
+ model.after_commit { config.after_commit_on_destroy(**params.merge(results: results)) }
133
+ end
134
+ end
135
+ end
136
+ end
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- module TableSync::Model
3
+ module TableSync::Receiving::Model
4
4
  class ActiveRecord
5
5
  class AfterCommitWrap
6
6
  def initialize(&block)
@@ -18,11 +18,18 @@ module TableSync::Model
18
18
  def trigger_transactional_callbacks?(*); end
19
19
  end
20
20
 
21
+ attr_reader :table, :schema
22
+
21
23
  def initialize(table_name)
22
24
  @raw_model = Class.new(::ActiveRecord::Base) do
23
25
  self.table_name = table_name
24
26
  self.inheritance_column = nil
25
27
  end
28
+
29
+ model_naming = ::TableSync::NamingResolver::ActiveRecord.new(table_name: table_name)
30
+
31
+ @table = model_naming.table.to_sym
32
+ @schema = model_naming.schema.to_sym
26
33
  end
27
34
 
28
35
  def columns
@@ -45,51 +52,55 @@ module TableSync::Model
45
52
  AND kcu.constraint_name = tc.constraint_name
46
53
  WHERE
47
54
  t.table_schema NOT IN ('pg_catalog', 'information_schema')
48
- AND t.table_schema = '#{model_naming.schema}'
49
- AND t.table_name = '#{model_naming.table}'
55
+ AND t.table_schema = '#{schema}'
56
+ AND t.table_name = '#{table}'
50
57
  ORDER BY
51
58
  kcu.ordinal_position
52
59
  SQL
53
60
  end
54
61
 
55
- def upsert(data:, target_keys:, version_key:, first_sync_time_key:, default_values:)
56
- data = Array.wrap(data)
62
+ def upsert(data:, target_keys:, version_key:, default_values:)
63
+ data.map do |datum|
64
+ conditions = datum.select { |k| target_keys.include?(k) }
57
65
 
58
- result = transaction do
59
- data.map do |datum|
60
- conditions = datum.select { |k| target_keys.include?(k) }
66
+ row = raw_model.lock("FOR NO KEY UPDATE").where(conditions)
61
67
 
62
- row = raw_model.lock("FOR NO KEY UPDATE").find_by(conditions)
63
- if row
64
- next if datum[version_key] <= row[version_key]
68
+ if row.to_a.size > 1
69
+ raise TableSync::UpsertError.new(data: datum, target_keys: target_keys, result: row)
70
+ end
65
71
 
66
- row.update!(datum)
67
- else
68
- create_data = default_values.merge(datum)
69
- create_data[first_sync_time_key] = Time.current if first_sync_time_key
70
- row = raw_model.create!(create_data)
71
- end
72
+ row = row.first
72
73
 
73
- row_to_hash(row)
74
- end.compact
75
- end
74
+ if row
75
+ next if datum[version_key] <= row[version_key]
76
76
 
77
- TableSync::Instrument.notify(table: model_naming.table, schema: model_naming.schema,
78
- event: :update, count: result.count, direction: :receive)
77
+ row.update!(datum)
78
+ else
79
+ create_data = default_values.merge(datum)
80
+ row = raw_model.create!(create_data)
81
+ end
79
82
 
80
- result
83
+ row_to_hash(row)
84
+ end.compact
81
85
  end
82
86
 
83
- def destroy(data)
84
- result = transaction do
85
- row = raw_model.lock("FOR UPDATE").find_by(data)&.destroy!
86
- [row_to_hash(row)]
87
+ def destroy(data:, target_keys:, version_key:)
88
+ sanitized_data = data.map { |attr| attr.select { |key, _value| target_keys.include?(key) } }
89
+
90
+ query = nil
91
+ sanitized_data.each_with_index do |row, index|
92
+ if index == 0
93
+ query = raw_model.lock("FOR UPDATE").where(row)
94
+ else
95
+ query = query.or(raw_model.lock("FOR UPDATE").where(row))
96
+ end
87
97
  end
88
98
 
89
- TableSync::Instrument.notify(
90
- table: model_naming.table, schema: model_naming.schema,
91
- event: :destroy, count: result.count, direction: :receive
92
- )
99
+ result = query.destroy_all.map(&method(:row_to_hash))
100
+
101
+ if result.size > data.size
102
+ raise TableSync::DestroyError.new(data: data, target_keys: target_keys, result: result)
103
+ end
93
104
 
94
105
  result
95
106
  end
@@ -107,11 +118,7 @@ module TableSync::Model
107
118
  attr_reader :raw_model
108
119
 
109
120
  def db
110
- @raw_model.connection
111
- end
112
-
113
- def model_naming
114
- ::TableSync::NamingResolver::ActiveRecord.new(table_name: raw_model.table_name)
121
+ raw_model.connection
115
122
  end
116
123
 
117
124
  def row_to_hash(row)
@@ -0,0 +1,83 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TableSync::Receiving::Model
4
+ class Sequel
5
+ attr_reader :table, :schema
6
+
7
+ def initialize(table_name)
8
+ @raw_model = Class.new(::Sequel::Model(table_name)).tap(&:unrestrict_primary_key)
9
+
10
+ model_naming = ::TableSync::NamingResolver::Sequel.new(
11
+ table_name: table_name,
12
+ db: @raw_model.db,
13
+ )
14
+
15
+ @table = model_naming.table.to_sym
16
+ @schema = model_naming.schema.to_sym
17
+ end
18
+
19
+ def columns
20
+ dataset.columns
21
+ end
22
+
23
+ def primary_keys
24
+ [raw_model.primary_key].flatten
25
+ end
26
+
27
+ def upsert(data:, target_keys:, version_key:, default_values:)
28
+ qualified_version = ::Sequel.qualify(raw_model.table_name, version_key)
29
+ version_condition = ::Sequel.function(:coalesce, qualified_version, 0) <
30
+ ::Sequel.qualify(:excluded, version_key)
31
+
32
+ upd_spec = update_spec(data.first.keys - target_keys)
33
+ data.map! { |d| default_values.merge(d) }
34
+
35
+ insert_data = type_cast(data)
36
+
37
+ dataset
38
+ .returning
39
+ .insert_conflict(target: target_keys, update: upd_spec, update_where: version_condition)
40
+ .multi_insert(insert_data)
41
+ end
42
+
43
+ def destroy(data:, target_keys:, version_key:)
44
+ sanitized_data = data.map { |attr| attr.select { |key, _value| target_keys.include?(key) } }
45
+ sanitized_data = type_cast(sanitized_data)
46
+ result = dataset.returning.where(::Sequel.|(*sanitized_data)).delete
47
+
48
+ if result.size > data.size
49
+ raise TableSync::DestroyError.new(data: data, target_keys: target_keys, result: result)
50
+ end
51
+
52
+ result
53
+ end
54
+
55
+ def transaction(&block)
56
+ db.transaction(&block)
57
+ end
58
+
59
+ def after_commit(&block)
60
+ db.after_commit(&block)
61
+ end
62
+
63
+ private
64
+
65
+ attr_reader :raw_model
66
+
67
+ def dataset
68
+ raw_model.dataset
69
+ end
70
+
71
+ def db
72
+ dataset.db
73
+ end
74
+
75
+ def type_cast(data)
76
+ data.map { |d| raw_model.new(d).values.keep_if { |k| d.key?(k) } }
77
+ end
78
+
79
+ def update_spec(keys)
80
+ keys.map { |key| [key, ::Sequel[:excluded][key]] }.to_h
81
+ end
82
+ end
83
+ end
@@ -0,0 +1,9 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TableSync
4
+ module Utils
5
+ require_relative "utils/proc_array"
6
+ require_relative "utils/proc_keywords_resolver"
7
+ require_relative "utils/interface_checker"
8
+ end
9
+ end
@@ -0,0 +1,103 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Ruby does not support interfaces, and there is no way to implement them.
4
+ # Interfaces check a methods of a class after the initialization of the class is complete.
5
+ # But in Ruby, the initialization of a class cannot be completed.
6
+ # In execution time we can open any class and add some methods (monkey patching).
7
+ # Ruby has `define_method`, singleton methods, etc.
8
+ #
9
+ # Duck typing is a necessary measure, the only one available in the Ruby architecture.
10
+ #
11
+ # Interfaces can be implemented in particular cases with tests for example.
12
+ # But this is not suitable for gems that are used by third-party code.
13
+ #
14
+ # So, we still want to check interfaces and have a nice error messages,
15
+ # even if it will be duck typing.
16
+ #
17
+ # Next code do this.
18
+
19
+ class TableSync::Utils::InterfaceChecker
20
+ INTERFACES = SelfData.load
21
+
22
+ attr_reader :object
23
+
24
+ def initialize(object)
25
+ @object = object
26
+ end
27
+
28
+ def implements(interface_name)
29
+ INTERFACES[interface_name].each do |method_name, options|
30
+ unless object.respond_to?(method_name)
31
+ raise_error(method_name, options)
32
+ end
33
+
34
+ unless include?(object.method(method_name).parameters, options[:parameters])
35
+ raise_error(method_name, options)
36
+ end
37
+ end
38
+ self
39
+ end
40
+
41
+ private
42
+
43
+ def include?(checked, expected)
44
+ (filter(expected) - filter(checked)).empty?
45
+ end
46
+
47
+ def raise_error(method_name, options)
48
+ raise TableSync::InterfaceError.new(
49
+ object,
50
+ method_name,
51
+ options[:parameters],
52
+ options[:description],
53
+ )
54
+ end
55
+
56
+ def filter(parameters)
57
+ # for req and block parameters types we can ignore names
58
+ parameters.map { |param| %i[req block].include?(param.first) ? [param.first] : param }
59
+ end
60
+ end
61
+
62
+ __END__
63
+ :receiving_model:
64
+ :upsert:
65
+ :parameters:
66
+ - - :keyreq
67
+ - :data
68
+ - - :keyreq
69
+ - :target_keys
70
+ - - :keyreq
71
+ - :version_key
72
+ - - :keyreq
73
+ - :default_values
74
+ :description: "returns an array with updated rows"
75
+ :columns:
76
+ :parameters: []
77
+ :description: "returns all table columns"
78
+ :destroy:
79
+ :parameters:
80
+ - - :keyreq
81
+ - :data
82
+ - - :keyreq
83
+ - :target_keys
84
+ :description: "returns an array with destroyed rows"
85
+ :transaction:
86
+ :parameters:
87
+ - - :block
88
+ - :block
89
+ :description: "implements the database transaction"
90
+ :after_commit:
91
+ :parameters:
92
+ - - :block
93
+ - :block
94
+ :description: "executes the block after committing the transaction"
95
+ :primary_keys:
96
+ :parameters: []
97
+ :description: "returns an array with the primary_keys"
98
+ :table:
99
+ :parameters: []
100
+ :description: "returns an instance of Symbol"
101
+ :schema:
102
+ :parameters: []
103
+ :description: "returns an instance of Symbol"