table_sync 1.12.1 → 1.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/docs/synopsis.md +29 -13
- data/lib/table_sync.rb +1 -0
- data/lib/table_sync/config.rb +5 -0
- data/lib/table_sync/config_decorator.rb +1 -1
- data/lib/table_sync/event_actions.rb +26 -6
- data/lib/table_sync/event_actions/data_wrapper.rb +7 -0
- data/lib/table_sync/event_actions/data_wrapper/base.rb +23 -0
- data/lib/table_sync/event_actions/data_wrapper/destroy.rb +19 -0
- data/lib/table_sync/event_actions/data_wrapper/update.rb +21 -0
- data/lib/table_sync/version.rb +1 -1
- metadata +6 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 01c6a6ad4b67454e29e7eaf06f85c09f4d786fc7842ef34cc87bc1db29937765
|
4
|
+
data.tar.gz: 5ffdfad99bdd53182870f0297c9bba8181884b51fc8a7bcc234ec8fd7a5035ee
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b7c21a601fa063b33ee9f13d580700c5ba2f70c88cdf59fbccc1fe5e9bb4a88d7cba8336dae55be7154e4aaf07194f869789c0b6a6886a04d7dc08db2dfa3f15
|
7
|
+
data.tar.gz: 378f2049f688922282d0b99972b28e021b36508fc153c8546085b710ca1649993d4e19f1dfa5eaebe9041dfdbd0b871ddef1f042a0bf29825dee8ebfa6ac69bc
|
data/CHANGELOG.md
CHANGED
@@ -1,6 +1,10 @@
|
|
1
1
|
# Changelog
|
2
2
|
All notable changes to this project will be documented in this file.
|
3
3
|
|
4
|
+
## [1.13.0] - 2019-11-02
|
5
|
+
### Added
|
6
|
+
- Wrapping interface around receiving logic (`wrap_receiving`);
|
7
|
+
|
4
8
|
## [1.12.1] - 2019-09-27
|
5
9
|
### Fixed
|
6
10
|
- The `default_values` option no longer overrides original values.
|
data/docs/synopsis.md
CHANGED
@@ -84,7 +84,7 @@ to default Rabbit gem configuration).
|
|
84
84
|
|
85
85
|
# Manual publishing
|
86
86
|
|
87
|
-
`TableSync::Publisher.new(object_class, original_attributes, confirm: true, state: :updated, debounce_time: 45)`
|
87
|
+
`TableSync::Publisher.new(object_class, original_attributes, confirm: true, state: :updated, debounce_time: 45)`
|
88
88
|
where state is one of `:created / :updated / :destroyed` and `confirm` is Rabbit's confirm delivery flag and optional param `debounce_time` determines debounce time in seconds, 1 minute by default.
|
89
89
|
|
90
90
|
# Manual publishing with batches
|
@@ -224,17 +224,33 @@ The following options are available inside the block:
|
|
224
224
|
- `additional_data` - additional data for insert or update (e.g. `project_id`)
|
225
225
|
- `default_values` - values for insert if a row is not found
|
226
226
|
- `partitions` - proc that is used to obtain partitioned data to support table partitioning. Must return a hash which
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
227
|
+
keys are names of partitions of partitioned table and values - arrays of attributes to be inserted into particular
|
228
|
+
partition `{ measurements_2018_01: [ { attrs }, ... ], measurements_2018_02: [ { attrs }, ... ], ...}`.
|
229
|
+
While the proc is called inside an upsert transaction it is suitable place for creating partitions for new data.
|
230
|
+
Note that transaction of proc is a TableSynk.orm transaction.
|
231
|
+
```ruby
|
232
|
+
partitions do |data:|
|
233
|
+
data.group_by { |d| "measurements_#{d[:time].year}_#{d[:time].month}" }
|
234
|
+
.tap { |data| data.keys.each { |table| DB.run("CREATE TABLE IF NOT EXISTS #{table} PARTITION OF measurements") } }
|
235
|
+
end
|
236
|
+
```
|
237
|
+
- `wrap_reciving` - proc that is used to wrap the receiving logic by custom block of code. Receives `data` and `receiving` attributes
|
238
|
+
(received event data and receiving logic proc respectively). `receiving.call` runs receiving process (you should use it manually).
|
239
|
+
- example (concurrent receiving):
|
240
|
+
```ruby
|
241
|
+
wrap_receiving do |data, receiving|
|
242
|
+
Locking.acquire("some-lock-key") { receiving.call }
|
243
|
+
end
|
244
|
+
```
|
245
|
+
- `data` attribute:
|
246
|
+
- for `destroy` event - an instance of `TableSync::EventActions::DataWrapper::Destroy`;
|
247
|
+
- for `update` event - an instance of `TableSync::EventActions::DataWrapper::Update`;
|
248
|
+
- `#event_data` - raw recevied event data:
|
249
|
+
- for `destroy` event - simple `Hash`;
|
250
|
+
- for `update` event - `Hash` with `Hash<ModelKlass, Array<Hash<Symbol, Object>>>` signature;
|
251
|
+
- `#destroy?` / `#update?` - corresponding predicates;
|
252
|
+
- `#type` - indicates a type of data (`:destroy` and `:update` respectively);
|
253
|
+
- `#each` - iterates over `#event_data` elements (acts like an iteration over an array of elements);
|
238
254
|
|
239
255
|
Each of options can receive static value or code block which will be called for each event with the following arguments:
|
240
256
|
- `event` - type of event (`:update` or `:destroy`)
|
@@ -283,7 +299,7 @@ You have access to the payload, which contains `event`, `direction`, `table`, `
|
|
283
299
|
:event => :update, # one of update / destroy
|
284
300
|
:direction => :publish, # one of publish / receive
|
285
301
|
:table => "users",
|
286
|
-
:schema => "public",
|
302
|
+
:schema => "public",
|
287
303
|
:count => 1
|
288
304
|
}
|
289
305
|
```
|
data/lib/table_sync.rb
CHANGED
@@ -10,6 +10,7 @@ module TableSync
|
|
10
10
|
require_relative "./table_sync/version"
|
11
11
|
require_relative "./table_sync/errors"
|
12
12
|
require_relative "./table_sync/event_actions"
|
13
|
+
require_relative "./table_sync/event_actions/data_wrapper"
|
13
14
|
require_relative "./table_sync/config"
|
14
15
|
require_relative "./table_sync/config/callback_registry"
|
15
16
|
require_relative "./table_sync/config_decorator"
|
data/lib/table_sync/config.rb
CHANGED
@@ -19,6 +19,7 @@ module TableSync
|
|
19
19
|
@version_key = :version
|
20
20
|
@first_sync_time_key = nil
|
21
21
|
@on_destroy = nil
|
22
|
+
@wrap_receiving = nil
|
22
23
|
target_keys(model.primary_keys)
|
23
24
|
end
|
24
25
|
|
@@ -71,6 +72,10 @@ module TableSync
|
|
71
72
|
block_given? ? @on_destroy = block : @on_destroy
|
72
73
|
end
|
73
74
|
|
75
|
+
def wrap_receiving(&block)
|
76
|
+
block_given? ? @wrap_receiving = block : @wrap_receiving
|
77
|
+
end
|
78
|
+
|
74
79
|
check_and_set_column_key = proc do |key|
|
75
80
|
key = key.to_sym
|
76
81
|
raise "#{model.inspect} doesn't have key: #{key}" unless model.columns.include?(key)
|
@@ -2,13 +2,29 @@
|
|
2
2
|
|
3
3
|
module TableSync
|
4
4
|
module EventActions
|
5
|
-
def update(data)
|
5
|
+
def update(data)
|
6
6
|
data.each_value do |attribute_set|
|
7
7
|
attribute_set.each do |attributes|
|
8
8
|
prevent_incomplete_event!(attributes)
|
9
9
|
end
|
10
10
|
end
|
11
11
|
|
12
|
+
with_wrapping(DataWrapper::Update.new(data)) do
|
13
|
+
process_upsert(data)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def destroy(data)
|
18
|
+
attributes = data.first || {}
|
19
|
+
target_attributes = attributes.select { |key, _value| target_keys.include?(key) }
|
20
|
+
prevent_incomplete_event!(target_attributes)
|
21
|
+
|
22
|
+
with_wrapping(DataWrapper::Destroy.new(attributes)) do
|
23
|
+
process_destroy(attributes, target_attributes)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def process_upsert(data) # rubocop:disable Metrics/MethodLength
|
12
28
|
model.transaction do
|
13
29
|
args = {
|
14
30
|
data: data,
|
@@ -37,11 +53,7 @@ module TableSync
|
|
37
53
|
end
|
38
54
|
end
|
39
55
|
|
40
|
-
def
|
41
|
-
attributes = data.first || {}
|
42
|
-
target_attributes = attributes.select { |key, _value| target_keys.include?(key) }
|
43
|
-
prevent_incomplete_event!(target_attributes)
|
44
|
-
|
56
|
+
def process_destroy(attributes, target_attributes)
|
45
57
|
model.transaction do
|
46
58
|
@config.callback_registry.get_callbacks(kind: :before_commit, event: :destroy).each do |cb|
|
47
59
|
cb[attributes]
|
@@ -63,6 +75,14 @@ module TableSync
|
|
63
75
|
end
|
64
76
|
end
|
65
77
|
|
78
|
+
def with_wrapping(data = {}, &block)
|
79
|
+
if @config.wrap_receiving
|
80
|
+
@config.wrap_receiving.call(data, block)
|
81
|
+
else
|
82
|
+
yield
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
66
86
|
def expected_update_result?(query_results)
|
67
87
|
query_results.uniq { |d| d.slice(*target_keys) }.size == query_results.size
|
68
88
|
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class TableSync::EventActions::DataWrapper::Base
|
4
|
+
include Enumerable
|
5
|
+
|
6
|
+
attr_reader :event_data
|
7
|
+
|
8
|
+
def initialize(event_data)
|
9
|
+
@event_data = event_data
|
10
|
+
end
|
11
|
+
|
12
|
+
def type
|
13
|
+
raise NoMethodError # NOTE: for clarity
|
14
|
+
end
|
15
|
+
|
16
|
+
def destroy?
|
17
|
+
false
|
18
|
+
end
|
19
|
+
|
20
|
+
def update?
|
21
|
+
false
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class TableSync::EventActions::DataWrapper::Destroy < TableSync::EventActions::DataWrapper::Base
|
4
|
+
def type
|
5
|
+
:destroy
|
6
|
+
end
|
7
|
+
|
8
|
+
def each
|
9
|
+
yield(event_data)
|
10
|
+
end
|
11
|
+
|
12
|
+
def destroy?
|
13
|
+
true
|
14
|
+
end
|
15
|
+
|
16
|
+
def update?
|
17
|
+
false
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class TableSync::EventActions::DataWrapper::Update < TableSync::EventActions::DataWrapper::Base
|
4
|
+
def type
|
5
|
+
:update
|
6
|
+
end
|
7
|
+
|
8
|
+
def each
|
9
|
+
event_data.each_pair do |model_klass, changed_models_attrs|
|
10
|
+
yield([model_klass, changed_models_attrs])
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def destroy?
|
15
|
+
false
|
16
|
+
end
|
17
|
+
|
18
|
+
def update?
|
19
|
+
true
|
20
|
+
end
|
21
|
+
end
|
data/lib/table_sync/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: table_sync
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.13.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Umbrellio
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-
|
11
|
+
date: 2019-11-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: memery
|
@@ -263,6 +263,10 @@ files:
|
|
263
263
|
- lib/table_sync/dsl.rb
|
264
264
|
- lib/table_sync/errors.rb
|
265
265
|
- lib/table_sync/event_actions.rb
|
266
|
+
- lib/table_sync/event_actions/data_wrapper.rb
|
267
|
+
- lib/table_sync/event_actions/data_wrapper/base.rb
|
268
|
+
- lib/table_sync/event_actions/data_wrapper/destroy.rb
|
269
|
+
- lib/table_sync/event_actions/data_wrapper/update.rb
|
266
270
|
- lib/table_sync/instrument.rb
|
267
271
|
- lib/table_sync/instrument_adapter/active_support.rb
|
268
272
|
- lib/table_sync/model/active_record.rb
|