table_sync 6.9.3 → 6.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +8 -0
- data/Gemfile.lock +29 -24
- data/lib/table_sync/instrument_adapter/active_support.rb +10 -2
- data/lib/table_sync/publishing/batch.rb +5 -3
- data/lib/table_sync/publishing/message/base.rb +4 -1
- data/lib/table_sync/publishing/message/raw.rb +4 -1
- data/lib/table_sync/publishing/message/single.rb +9 -1
- data/lib/table_sync/publishing/params/single.rb +2 -2
- data/lib/table_sync/publishing/raw.rb +5 -3
- data/lib/table_sync/publishing/single.rb +4 -1
- data/lib/table_sync/receiving/config.rb +27 -4
- data/lib/table_sync/receiving/config_decorator.rb +10 -5
- data/lib/table_sync/receiving/handler.rb +24 -17
- data/lib/table_sync/receiving/hooks/once.rb +62 -0
- data/lib/table_sync/receiving/model/active_record.rb +29 -2
- data/lib/table_sync/receiving/model/sequel.rb +29 -2
- data/lib/table_sync/receiving.rb +1 -0
- data/lib/table_sync/setup/active_record.rb +1 -0
- data/lib/table_sync/setup/base.rb +2 -1
- data/lib/table_sync/setup/sequel.rb +1 -0
- data/lib/table_sync/version.rb +1 -1
- data/lib/table_sync.rb +1 -0
- data/table_sync.gemspec +1 -1
- metadata +5 -4
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 4f639c5f219e99ba00d508a19d652a617df44dc630c86e29233f6269a9d09650
|
|
4
|
+
data.tar.gz: 7645c878d05b076b871eb8f5ddb5c49092d7b8210c4c6b9f38f489fe3fcffadb
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 3b30d8bbb0562afc16d85290fe91c7c46c95a99438743fb5183cfbfc15d2adb784a987d074704a5437a7453122f1b2f514113c10363e6c9803faab51762a26f4
|
|
7
|
+
data.tar.gz: a9e9a8cc3d8e98acfcb39abc7d8cbba7ed2b5394e7ef6f0707ea9b04da048382c2ecd8c87526dfc9f6b9a512587e8915627c4fce03539d6df25aef88a7eac3f0
|
data/CHANGELOG.md
CHANGED
|
@@ -1,6 +1,14 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
All notable changes to this project will be documented in this file.
|
|
3
3
|
|
|
4
|
+
## [6.11.0] - 2026-03-25
|
|
5
|
+
### Added
|
|
6
|
+
- Ability to specify option for compression when publish data
|
|
7
|
+
|
|
8
|
+
## [6.10.0] - 2025-11-21
|
|
9
|
+
### Added
|
|
10
|
+
- Add on_first_sync callback
|
|
11
|
+
|
|
4
12
|
## [6.9.3] - 2025-10-09
|
|
5
13
|
### Fixed
|
|
6
14
|
- Add validate_types to model interface
|
data/Gemfile.lock
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
table_sync (6.
|
|
4
|
+
table_sync (6.11.0)
|
|
5
5
|
memery
|
|
6
|
-
rabbit_messaging (>= 1.
|
|
6
|
+
rabbit_messaging (>= 1.8.0)
|
|
7
7
|
rails
|
|
8
8
|
self_data
|
|
9
9
|
|
|
@@ -81,21 +81,21 @@ GEM
|
|
|
81
81
|
minitest (>= 5.1)
|
|
82
82
|
securerandom (>= 0.3)
|
|
83
83
|
tzinfo (~> 2.0, >= 2.0.5)
|
|
84
|
-
amq-protocol (2.
|
|
84
|
+
amq-protocol (2.7.0)
|
|
85
85
|
ast (2.4.3)
|
|
86
|
-
base64 (0.
|
|
86
|
+
base64 (0.3.0)
|
|
87
87
|
benchmark (0.4.0)
|
|
88
88
|
bigdecimal (3.1.9)
|
|
89
89
|
builder (3.3.0)
|
|
90
90
|
bunny (2.24.0)
|
|
91
91
|
amq-protocol (~> 2.3)
|
|
92
92
|
sorted_set (~> 1, >= 1.0.2)
|
|
93
|
-
cgi (0.
|
|
93
|
+
cgi (0.5.0)
|
|
94
94
|
coderay (1.1.3)
|
|
95
|
-
concurrent-ruby (1.3.
|
|
95
|
+
concurrent-ruby (1.3.6)
|
|
96
96
|
connection_pool (2.5.3)
|
|
97
97
|
crass (1.0.6)
|
|
98
|
-
date (3.
|
|
98
|
+
date (3.5.0)
|
|
99
99
|
diff-lcs (1.6.2)
|
|
100
100
|
docile (1.4.1)
|
|
101
101
|
drb (2.2.3)
|
|
@@ -106,14 +106,14 @@ GEM
|
|
|
106
106
|
activesupport (>= 6.1)
|
|
107
107
|
i18n (1.14.7)
|
|
108
108
|
concurrent-ruby (~> 1.0)
|
|
109
|
-
io-console (0.8.
|
|
110
|
-
irb (1.15.
|
|
109
|
+
io-console (0.8.1)
|
|
110
|
+
irb (1.15.3)
|
|
111
111
|
pp (>= 0.6.0)
|
|
112
112
|
rdoc (>= 4.0.0)
|
|
113
113
|
reline (>= 0.4.2)
|
|
114
114
|
json (2.12.2)
|
|
115
|
-
kicks (3.
|
|
116
|
-
bunny (~> 2.
|
|
115
|
+
kicks (3.3.0)
|
|
116
|
+
bunny (~> 2.24)
|
|
117
117
|
concurrent-ruby (~> 1.0)
|
|
118
118
|
rake (>= 12.3, < 14.0)
|
|
119
119
|
serverengine (~> 2.1)
|
|
@@ -124,18 +124,20 @@ GEM
|
|
|
124
124
|
loofah (2.24.1)
|
|
125
125
|
crass (~> 1.0.2)
|
|
126
126
|
nokogiri (>= 1.12.0)
|
|
127
|
-
mail (2.
|
|
127
|
+
mail (2.9.0)
|
|
128
|
+
logger
|
|
128
129
|
mini_mime (>= 0.1.1)
|
|
129
130
|
net-imap
|
|
130
131
|
net-pop
|
|
131
132
|
net-smtp
|
|
132
|
-
marcel (1.0
|
|
133
|
+
marcel (1.1.0)
|
|
133
134
|
memery (1.7.0)
|
|
134
135
|
method_source (1.1.0)
|
|
135
136
|
mini_mime (1.1.5)
|
|
136
137
|
mini_portile2 (2.8.9)
|
|
137
138
|
minitest (5.25.5)
|
|
138
|
-
|
|
139
|
+
msgpack (1.8.0)
|
|
140
|
+
net-imap (0.5.12)
|
|
139
141
|
date
|
|
140
142
|
net-protocol
|
|
141
143
|
net-pop (0.1.2)
|
|
@@ -170,7 +172,7 @@ GEM
|
|
|
170
172
|
ast (~> 2.4.1)
|
|
171
173
|
racc
|
|
172
174
|
pg (1.5.9)
|
|
173
|
-
pp (0.6.
|
|
175
|
+
pp (0.6.3)
|
|
174
176
|
prettyprint
|
|
175
177
|
prettyprint (0.2.0)
|
|
176
178
|
prism (1.4.0)
|
|
@@ -180,9 +182,11 @@ GEM
|
|
|
180
182
|
psych (5.2.6)
|
|
181
183
|
date
|
|
182
184
|
stringio
|
|
183
|
-
rabbit_messaging (1.
|
|
185
|
+
rabbit_messaging (1.8.0)
|
|
184
186
|
bunny (~> 2.0)
|
|
185
187
|
kicks
|
|
188
|
+
msgpack
|
|
189
|
+
zlib
|
|
186
190
|
racc (1.8.1)
|
|
187
191
|
rack (3.1.15)
|
|
188
192
|
rack-session (2.1.1)
|
|
@@ -222,13 +226,14 @@ GEM
|
|
|
222
226
|
thor (~> 1.0, >= 1.2.2)
|
|
223
227
|
zeitwerk (~> 2.6)
|
|
224
228
|
rainbow (3.1.1)
|
|
225
|
-
rake (13.
|
|
229
|
+
rake (13.3.1)
|
|
226
230
|
rbtree (0.4.6)
|
|
227
|
-
rdoc (6.
|
|
231
|
+
rdoc (6.15.1)
|
|
228
232
|
erb
|
|
229
233
|
psych (>= 4.0.0)
|
|
234
|
+
tsort
|
|
230
235
|
regexp_parser (2.10.0)
|
|
231
|
-
reline (0.6.
|
|
236
|
+
reline (0.6.3)
|
|
232
237
|
io-console (~> 0.5)
|
|
233
238
|
rspec (3.13.1)
|
|
234
239
|
rspec-core (~> 3.13.0)
|
|
@@ -296,7 +301,6 @@ GEM
|
|
|
296
301
|
base64 (~> 0.1)
|
|
297
302
|
logger (~> 1.4)
|
|
298
303
|
sigdump (~> 0.2.2)
|
|
299
|
-
set (1.1.2)
|
|
300
304
|
sigdump (0.2.5)
|
|
301
305
|
simplecov (0.22.0)
|
|
302
306
|
docile (~> 1.1)
|
|
@@ -305,13 +309,13 @@ GEM
|
|
|
305
309
|
simplecov-html (0.13.1)
|
|
306
310
|
simplecov-lcov (0.8.0)
|
|
307
311
|
simplecov_json_formatter (0.1.4)
|
|
308
|
-
sorted_set (1.0
|
|
312
|
+
sorted_set (1.1.0)
|
|
309
313
|
rbtree
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
thor (1.3.2)
|
|
314
|
+
stringio (3.1.8)
|
|
315
|
+
thor (1.5.0)
|
|
313
316
|
timecop (0.9.10)
|
|
314
317
|
timeout (0.4.3)
|
|
318
|
+
tsort (0.2.0)
|
|
315
319
|
tzinfo (2.0.6)
|
|
316
320
|
concurrent-ruby (~> 1.0)
|
|
317
321
|
unicode-display_width (3.1.4)
|
|
@@ -323,6 +327,7 @@ GEM
|
|
|
323
327
|
websocket-extensions (>= 0.1.0)
|
|
324
328
|
websocket-extensions (0.1.5)
|
|
325
329
|
zeitwerk (2.6.18)
|
|
330
|
+
zlib (3.2.3)
|
|
326
331
|
|
|
327
332
|
PLATFORMS
|
|
328
333
|
aarch64-linux-gnu
|
|
@@ -4,13 +4,21 @@ module TableSync::InstrumentAdapter
|
|
|
4
4
|
module ActiveSupport
|
|
5
5
|
module_function
|
|
6
6
|
|
|
7
|
-
def notify(
|
|
7
|
+
def notify( # rubocop:disable Metrics/ParameterLists
|
|
8
|
+
table:,
|
|
9
|
+
schema:,
|
|
10
|
+
event:,
|
|
11
|
+
direction:,
|
|
12
|
+
count: 1,
|
|
13
|
+
compress: false
|
|
14
|
+
)
|
|
8
15
|
::ActiveSupport::Notifications.instrument "tablesync.#{direction}.#{event}",
|
|
9
16
|
count:,
|
|
10
17
|
table: table.to_s,
|
|
11
18
|
schema: schema.to_s,
|
|
12
19
|
event:,
|
|
13
|
-
direction
|
|
20
|
+
direction:,
|
|
21
|
+
compress:
|
|
14
22
|
end
|
|
15
23
|
end
|
|
16
24
|
end
|
|
@@ -6,7 +6,8 @@ class TableSync::Publishing::Batch
|
|
|
6
6
|
:custom_version,
|
|
7
7
|
:routing_key,
|
|
8
8
|
:headers,
|
|
9
|
-
:event
|
|
9
|
+
:event,
|
|
10
|
+
:compress
|
|
10
11
|
|
|
11
12
|
def initialize(attrs = {})
|
|
12
13
|
attrs = attrs.with_indifferent_access
|
|
@@ -15,8 +16,9 @@ class TableSync::Publishing::Batch
|
|
|
15
16
|
self.original_attributes = attrs[:original_attributes]
|
|
16
17
|
self.custom_version = attrs[:custom_version]
|
|
17
18
|
self.routing_key = attrs[:routing_key]
|
|
18
|
-
self.headers = attrs[:headers]
|
|
19
|
+
self.headers = attrs[:headers] || {}
|
|
19
20
|
self.event = attrs.fetch(:event, :update).to_sym
|
|
21
|
+
self.compress = attrs.fetch(:compress, false)
|
|
20
22
|
|
|
21
23
|
validate_required_attributes!
|
|
22
24
|
end
|
|
@@ -53,7 +55,7 @@ class TableSync::Publishing::Batch
|
|
|
53
55
|
original_attributes: original_attributes,
|
|
54
56
|
custom_version: custom_version,
|
|
55
57
|
routing_key: routing_key,
|
|
56
|
-
headers: headers,
|
|
58
|
+
headers: headers.merge(compress: compress),
|
|
57
59
|
event: event,
|
|
58
60
|
}
|
|
59
61
|
end
|
|
@@ -5,7 +5,8 @@ module TableSync::Publishing::Message
|
|
|
5
5
|
attr_accessor :custom_version,
|
|
6
6
|
:object_class,
|
|
7
7
|
:original_attributes,
|
|
8
|
-
:event
|
|
8
|
+
:event,
|
|
9
|
+
:compress
|
|
9
10
|
|
|
10
11
|
attr_reader :objects
|
|
11
12
|
|
|
@@ -14,6 +15,7 @@ module TableSync::Publishing::Message
|
|
|
14
15
|
self.object_class = params[:object_class]
|
|
15
16
|
self.original_attributes = params[:original_attributes]
|
|
16
17
|
self.event = params[:event].to_sym
|
|
18
|
+
self.compress = params.dig(:headers, :compress) || false
|
|
17
19
|
|
|
18
20
|
@objects = find_or_init_objects
|
|
19
21
|
|
|
@@ -65,6 +67,7 @@ module TableSync::Publishing::Message
|
|
|
65
67
|
table: model_naming.table,
|
|
66
68
|
schema: model_naming.schema,
|
|
67
69
|
event:,
|
|
70
|
+
compress:,
|
|
68
71
|
direction: :publish,
|
|
69
72
|
count: objects.count,
|
|
70
73
|
)
|
|
@@ -9,7 +9,8 @@ module TableSync::Publishing::Message
|
|
|
9
9
|
:routing_key,
|
|
10
10
|
:headers,
|
|
11
11
|
:custom_version,
|
|
12
|
-
:event
|
|
12
|
+
:event,
|
|
13
|
+
:compress
|
|
13
14
|
|
|
14
15
|
def initialize(params = {})
|
|
15
16
|
self.model_name = params[:model_name]
|
|
@@ -20,6 +21,7 @@ module TableSync::Publishing::Message
|
|
|
20
21
|
self.headers = params[:headers]
|
|
21
22
|
self.custom_version = params[:custom_version]
|
|
22
23
|
self.event = params[:event]
|
|
24
|
+
self.compress = params.dig(:headers, :compress) || false
|
|
23
25
|
end
|
|
24
26
|
|
|
25
27
|
def publish
|
|
@@ -35,6 +37,7 @@ module TableSync::Publishing::Message
|
|
|
35
37
|
table: table_name,
|
|
36
38
|
schema: schema_name,
|
|
37
39
|
event:,
|
|
40
|
+
compress: compress,
|
|
38
41
|
count: original_attributes.count,
|
|
39
42
|
direction: :publish,
|
|
40
43
|
)
|
|
@@ -2,12 +2,20 @@
|
|
|
2
2
|
|
|
3
3
|
module TableSync::Publishing::Message
|
|
4
4
|
class Single < Base
|
|
5
|
+
attr_accessor :headers
|
|
6
|
+
|
|
7
|
+
def initialize(params = {})
|
|
8
|
+
super
|
|
9
|
+
|
|
10
|
+
self.headers = params[:headers]
|
|
11
|
+
end
|
|
12
|
+
|
|
5
13
|
def object
|
|
6
14
|
objects.first
|
|
7
15
|
end
|
|
8
16
|
|
|
9
17
|
def params
|
|
10
|
-
TableSync::Publishing::Params::Single.new(object:).construct
|
|
18
|
+
TableSync::Publishing::Params::Single.new(object:, headers:).construct
|
|
11
19
|
end
|
|
12
20
|
end
|
|
13
21
|
end
|
|
@@ -4,10 +4,10 @@ module TableSync::Publishing::Params
|
|
|
4
4
|
class Single < Base
|
|
5
5
|
attr_reader :object, :routing_key, :headers
|
|
6
6
|
|
|
7
|
-
def initialize(object:)
|
|
7
|
+
def initialize(object:, headers: {})
|
|
8
8
|
@object = object
|
|
9
9
|
@routing_key = calculated_routing_key
|
|
10
|
-
@headers
|
|
10
|
+
@headers = headers.merge(calculated_headers || {})
|
|
11
11
|
end
|
|
12
12
|
|
|
13
13
|
private
|
|
@@ -10,7 +10,8 @@ class TableSync::Publishing::Raw
|
|
|
10
10
|
:custom_version,
|
|
11
11
|
:routing_key,
|
|
12
12
|
:headers,
|
|
13
|
-
:event
|
|
13
|
+
:event,
|
|
14
|
+
:compress
|
|
14
15
|
|
|
15
16
|
def initialize(attributes = {})
|
|
16
17
|
attributes = attributes.with_indifferent_access
|
|
@@ -21,8 +22,9 @@ class TableSync::Publishing::Raw
|
|
|
21
22
|
self.original_attributes = attributes[:original_attributes]
|
|
22
23
|
self.custom_version = attributes[:custom_version]
|
|
23
24
|
self.routing_key = attributes[:routing_key]
|
|
24
|
-
self.headers = attributes[:headers]
|
|
25
|
+
self.headers = attributes[:headers] || {}
|
|
25
26
|
self.event = attributes.fetch(:event, :update).to_sym
|
|
27
|
+
self.compress = attributes.fetch(:compress, false)
|
|
26
28
|
end
|
|
27
29
|
|
|
28
30
|
require_attributes :model_name, :original_attributes
|
|
@@ -45,7 +47,7 @@ class TableSync::Publishing::Raw
|
|
|
45
47
|
original_attributes: original_attributes,
|
|
46
48
|
custom_version: custom_version,
|
|
47
49
|
routing_key: routing_key,
|
|
48
|
-
headers: headers,
|
|
50
|
+
headers: headers.merge(compress: compress),
|
|
49
51
|
event: event,
|
|
50
52
|
}
|
|
51
53
|
end
|
|
@@ -7,7 +7,8 @@ class TableSync::Publishing::Single
|
|
|
7
7
|
:original_attributes,
|
|
8
8
|
:debounce_time,
|
|
9
9
|
:custom_version,
|
|
10
|
-
:event
|
|
10
|
+
:event,
|
|
11
|
+
:compress
|
|
11
12
|
|
|
12
13
|
def initialize(attrs = {})
|
|
13
14
|
attrs = attrs.with_indifferent_access
|
|
@@ -15,6 +16,7 @@ class TableSync::Publishing::Single
|
|
|
15
16
|
self.object_class = attrs[:object_class]
|
|
16
17
|
self.original_attributes = attrs[:original_attributes]
|
|
17
18
|
self.debounce_time = attrs[:debounce_time]
|
|
19
|
+
self.compress = attrs.fetch(:compress, false)
|
|
18
20
|
self.custom_version = attrs[:custom_version]
|
|
19
21
|
self.event = attrs.fetch(:event, :update)
|
|
20
22
|
end
|
|
@@ -55,6 +57,7 @@ class TableSync::Publishing::Single
|
|
|
55
57
|
original_attributes: original_attributes,
|
|
56
58
|
debounce_time: debounce_time,
|
|
57
59
|
custom_version: custom_version,
|
|
60
|
+
headers: { compress: compress },
|
|
58
61
|
event: event,
|
|
59
62
|
}
|
|
60
63
|
end
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative "hooks/once"
|
|
4
|
+
|
|
3
5
|
module TableSync::Receiving
|
|
4
6
|
class Config
|
|
5
7
|
attr_reader :model, :events
|
|
@@ -23,13 +25,10 @@ module TableSync::Receiving
|
|
|
23
25
|
class << self
|
|
24
26
|
attr_reader :default_values_for_options
|
|
25
27
|
|
|
26
|
-
# In a configs
|
|
28
|
+
# In a configs these options are requested as they are
|
|
27
29
|
# config.option - get value
|
|
28
30
|
# config.option(args) - set static value
|
|
29
31
|
# config.option { ... } - set proc as value
|
|
30
|
-
#
|
|
31
|
-
# In `Receiving::Handler` or `Receiving::EventActions` this options are requested
|
|
32
|
-
# through `Receiving::ConfigDecorator#method_missing` which always executes `config.option`
|
|
33
32
|
|
|
34
33
|
def add_option(name, value_setter_wrapper:, value_as_proc_setter_wrapper:, default:)
|
|
35
34
|
ivar = :"@#{name}"
|
|
@@ -55,11 +54,30 @@ module TableSync::Receiving
|
|
|
55
54
|
instance_variable_set(ivar, result_value)
|
|
56
55
|
end
|
|
57
56
|
end
|
|
57
|
+
|
|
58
|
+
def add_hook_option(name, hook_class:)
|
|
59
|
+
ivar = :"@#{name}"
|
|
60
|
+
|
|
61
|
+
@default_values_for_options ||= {}
|
|
62
|
+
@default_values_for_options[ivar] = proc { [] }
|
|
63
|
+
|
|
64
|
+
define_method(name) do |conditions, &handler|
|
|
65
|
+
hooks = instance_variable_get(ivar)
|
|
66
|
+
hooks ||= []
|
|
67
|
+
|
|
68
|
+
hooks << hook_class.new(conditions:, handler:)
|
|
69
|
+
instance_variable_set(ivar, hooks)
|
|
70
|
+
end
|
|
71
|
+
end
|
|
58
72
|
end
|
|
59
73
|
|
|
60
74
|
def allow_event?(name)
|
|
61
75
|
events.include?(name)
|
|
62
76
|
end
|
|
77
|
+
|
|
78
|
+
def option(name)
|
|
79
|
+
instance_variable_get(:"@#{name}")
|
|
80
|
+
end
|
|
63
81
|
end
|
|
64
82
|
end
|
|
65
83
|
|
|
@@ -201,6 +219,11 @@ TableSync::Receiving::Config.add_option :wrap_receiving,
|
|
|
201
219
|
value_as_proc_setter_wrapper: any_value,
|
|
202
220
|
default: proc { proc { |&block| block.call } }
|
|
203
221
|
|
|
222
|
+
TableSync::Receiving::Config.add_hook_option(
|
|
223
|
+
:on_first_sync,
|
|
224
|
+
hook_class: TableSync::Receiving::Hooks::Once,
|
|
225
|
+
)
|
|
226
|
+
|
|
204
227
|
%i[
|
|
205
228
|
before_update
|
|
206
229
|
after_commit_on_update
|
|
@@ -2,9 +2,6 @@
|
|
|
2
2
|
|
|
3
3
|
module TableSync::Receiving
|
|
4
4
|
class ConfigDecorator
|
|
5
|
-
extend Forwardable
|
|
6
|
-
|
|
7
|
-
def_delegators :@config, :allow_event?
|
|
8
5
|
# rubocop:disable Metrics/ParameterLists
|
|
9
6
|
def initialize(config:, event:, model:, version:, project_id:, raw_data:)
|
|
10
7
|
@config = config
|
|
@@ -19,9 +16,17 @@ module TableSync::Receiving
|
|
|
19
16
|
end
|
|
20
17
|
# rubocop:enable Metrics/ParameterLists
|
|
21
18
|
|
|
22
|
-
def
|
|
23
|
-
value = @config.
|
|
19
|
+
def option(name, **additional_params, &)
|
|
20
|
+
value = @config.option(name)
|
|
24
21
|
value.is_a?(Proc) ? value.call(@default_params.merge(additional_params), &) : value
|
|
25
22
|
end
|
|
23
|
+
|
|
24
|
+
def model
|
|
25
|
+
@config.model
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def allow_event?(name)
|
|
29
|
+
@config.allow_event?(name)
|
|
30
|
+
end
|
|
26
31
|
end
|
|
27
32
|
end
|
|
@@ -22,22 +22,20 @@ class TableSync::Receiving::Handler < Rabbit::EventHandler
|
|
|
22
22
|
|
|
23
23
|
next if data.empty?
|
|
24
24
|
|
|
25
|
-
|
|
26
|
-
data.each { |row| row[version_key] = version }
|
|
27
|
-
|
|
28
|
-
target_keys = config.target_keys(data:)
|
|
25
|
+
target_keys = config.option(:target_keys, data:)
|
|
29
26
|
|
|
30
27
|
validate_data(data, target_keys:)
|
|
31
28
|
|
|
32
29
|
data.sort_by! { |row| row.values_at(*target_keys).map { |value| sort_key(value) } }
|
|
33
30
|
|
|
31
|
+
version_key = config.option(:version_key, data:)
|
|
34
32
|
params = { data:, target_keys:, version_key: }
|
|
35
33
|
|
|
36
34
|
if event == :update
|
|
37
|
-
params[:default_values] = config.default_values
|
|
35
|
+
params[:default_values] = config.option(:default_values, data:)
|
|
38
36
|
end
|
|
39
37
|
|
|
40
|
-
config.wrap_receiving
|
|
38
|
+
config.option(:wrap_receiving, **params) do
|
|
41
39
|
perform(config, params)
|
|
42
40
|
end
|
|
43
41
|
end
|
|
@@ -83,25 +81,28 @@ class TableSync::Receiving::Handler < Rabbit::EventHandler
|
|
|
83
81
|
end
|
|
84
82
|
|
|
85
83
|
def processed_data(config)
|
|
84
|
+
version_key = config.option(:version_key, data:)
|
|
86
85
|
data.filter_map do |row|
|
|
87
|
-
next if config.skip
|
|
86
|
+
next if config.option(:skip, row:)
|
|
88
87
|
|
|
89
88
|
row = row.dup
|
|
90
89
|
|
|
91
|
-
config.mapping_overrides
|
|
90
|
+
config.option(:mapping_overrides, row:).each do |before, after|
|
|
92
91
|
row[after] = row.delete(before)
|
|
93
92
|
end
|
|
94
93
|
|
|
95
|
-
config.except
|
|
94
|
+
config.option(:except, row:).each { |x| row.delete(x) }
|
|
96
95
|
|
|
97
|
-
row.merge!(config.additional_data
|
|
96
|
+
row.merge!(config.option(:additional_data, row:))
|
|
98
97
|
|
|
99
|
-
only = config.only
|
|
98
|
+
only = config.option(:only, row:)
|
|
100
99
|
row, rest = row.partition { |key, _| key.in?(only) }.map(&:to_h)
|
|
101
100
|
|
|
102
|
-
rest_key = config.rest_key
|
|
101
|
+
rest_key = config.option(:rest_key, row:, rest:)
|
|
103
102
|
(row[rest_key] ||= {}).merge!(rest) if rest_key
|
|
104
103
|
|
|
104
|
+
row[version_key] = version
|
|
105
|
+
|
|
105
106
|
row
|
|
106
107
|
end
|
|
107
108
|
end
|
|
@@ -138,16 +139,16 @@ class TableSync::Receiving::Handler < Rabbit::EventHandler
|
|
|
138
139
|
raise TableSync::DataError.new(data, errors.keys, errors.to_json)
|
|
139
140
|
end
|
|
140
141
|
|
|
141
|
-
def perform(config, params)
|
|
142
|
+
def perform(config, params) # rubocop:disable Metrics/MethodLength
|
|
142
143
|
model = config.model
|
|
143
144
|
|
|
144
145
|
model.transaction do
|
|
145
146
|
results = if event == :update
|
|
146
|
-
config.before_update
|
|
147
|
+
config.option(:before_update, **params)
|
|
147
148
|
validate_data_types(model, params[:data])
|
|
148
149
|
model.upsert(**params)
|
|
149
150
|
else
|
|
150
|
-
config.before_destroy
|
|
151
|
+
config.option(:before_destroy, **params)
|
|
151
152
|
model.destroy(**params)
|
|
152
153
|
end
|
|
153
154
|
|
|
@@ -157,9 +158,15 @@ class TableSync::Receiving::Handler < Rabbit::EventHandler
|
|
|
157
158
|
end
|
|
158
159
|
|
|
159
160
|
if event == :update
|
|
160
|
-
model.after_commit
|
|
161
|
+
model.after_commit do
|
|
162
|
+
config.option(:after_commit_on_update, **params, results:)
|
|
163
|
+
|
|
164
|
+
Array(config.option(:on_first_sync)).each do |hook|
|
|
165
|
+
hook.perform(config:, targets: results) if hook.enabled?
|
|
166
|
+
end
|
|
167
|
+
end
|
|
161
168
|
else
|
|
162
|
-
model.after_commit { config.after_commit_on_destroy
|
|
169
|
+
model.after_commit { config.option(:after_commit_on_destroy, **params, results:) }
|
|
163
170
|
end
|
|
164
171
|
end
|
|
165
172
|
end
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module TableSync::Receiving::Hooks
|
|
4
|
+
class Once
|
|
5
|
+
LOCK_KEY = "hook-once-lock-key"
|
|
6
|
+
|
|
7
|
+
attr_reader :conditions, :handler, :lookup_code
|
|
8
|
+
|
|
9
|
+
def initialize(conditions:, handler:)
|
|
10
|
+
@conditions = conditions
|
|
11
|
+
@handler = handler
|
|
12
|
+
init_lookup_code
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def enabled?
|
|
16
|
+
conditions[:columns].any?
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def perform(config:, targets:)
|
|
20
|
+
target_keys = config.option(:target_keys)
|
|
21
|
+
model = config.model
|
|
22
|
+
|
|
23
|
+
targets.each do |target|
|
|
24
|
+
next unless conditions?(target)
|
|
25
|
+
|
|
26
|
+
keys = target.slice(*target_keys)
|
|
27
|
+
model.try_advisory_lock(prepare_lock_key(keys)) do
|
|
28
|
+
model.find_and_save(keys:) do |entry|
|
|
29
|
+
next unless allow?(entry)
|
|
30
|
+
|
|
31
|
+
entry.hooks ||= []
|
|
32
|
+
entry.hooks << lookup_code
|
|
33
|
+
model.after_commit { handler.call(entry:) }
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
private
|
|
40
|
+
|
|
41
|
+
def allow?(entry)
|
|
42
|
+
Array(entry.hooks).exclude?(lookup_code)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def init_lookup_code
|
|
46
|
+
@lookup_code = conditions[:columns].map do |column|
|
|
47
|
+
"#{column}-#{conditions[column]}"
|
|
48
|
+
end.join(":")
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def conditions?(row)
|
|
52
|
+
conditions[:columns].all? do |column|
|
|
53
|
+
row[column] == (conditions[column] || row[column])
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def prepare_lock_key(row_keys)
|
|
58
|
+
lock_keys = [LOCK_KEY] + row_keys.values
|
|
59
|
+
Zlib.crc32(lock_keys.join(":")) % (2**31)
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
@@ -2,6 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
module TableSync::Receiving::Model
|
|
4
4
|
class ActiveRecord
|
|
5
|
+
ISOLATION_LEVELS = {
|
|
6
|
+
uncommitted: :read_uncommitted,
|
|
7
|
+
committed: :read_committed,
|
|
8
|
+
repeatable: :repeatable_read,
|
|
9
|
+
serializable: :serializable,
|
|
10
|
+
}.freeze
|
|
11
|
+
|
|
5
12
|
class AfterCommitWrap
|
|
6
13
|
def initialize(&block)
|
|
7
14
|
@callback = block
|
|
@@ -33,6 +40,10 @@ module TableSync::Receiving::Model
|
|
|
33
40
|
@schema = model_naming.schema.to_sym
|
|
34
41
|
end
|
|
35
42
|
|
|
43
|
+
def isolation_level(lookup_code)
|
|
44
|
+
ISOLATION_LEVELS.fetch(lookup_code)
|
|
45
|
+
end
|
|
46
|
+
|
|
36
47
|
def columns
|
|
37
48
|
raw_model.column_names.map(&:to_sym)
|
|
38
49
|
end
|
|
@@ -110,14 +121,30 @@ module TableSync::Receiving::Model
|
|
|
110
121
|
types_validator.validate(data)
|
|
111
122
|
end
|
|
112
123
|
|
|
113
|
-
def transaction(&)
|
|
114
|
-
::ActiveRecord::Base.transaction(&)
|
|
124
|
+
def transaction(**params, &)
|
|
125
|
+
::ActiveRecord::Base.transaction(**params, &)
|
|
115
126
|
end
|
|
116
127
|
|
|
117
128
|
def after_commit(&)
|
|
118
129
|
db.add_transaction_record(AfterCommitWrap.new(&))
|
|
119
130
|
end
|
|
120
131
|
|
|
132
|
+
def try_advisory_lock(lock_key)
|
|
133
|
+
transaction do
|
|
134
|
+
if db.query_value("SELECT pg_try_advisory_xact_lock(#{lock_key.to_i})")
|
|
135
|
+
yield
|
|
136
|
+
end
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def find_and_save(keys:)
|
|
141
|
+
entry = raw_model.find_by(keys)
|
|
142
|
+
return unless entry
|
|
143
|
+
|
|
144
|
+
yield entry
|
|
145
|
+
entry.save!
|
|
146
|
+
end
|
|
147
|
+
|
|
121
148
|
private
|
|
122
149
|
|
|
123
150
|
attr_reader :raw_model, :types_validator
|
|
@@ -4,6 +4,13 @@ module TableSync::Receiving::Model
|
|
|
4
4
|
class Sequel
|
|
5
5
|
attr_reader :table, :schema
|
|
6
6
|
|
|
7
|
+
ISOLATION_LEVELS = {
|
|
8
|
+
uncommitted: :uncommitted,
|
|
9
|
+
committed: :committed,
|
|
10
|
+
repeatable: :repeatable,
|
|
11
|
+
serializable: :serializable,
|
|
12
|
+
}.freeze
|
|
13
|
+
|
|
7
14
|
def initialize(table_name)
|
|
8
15
|
@raw_model = Class.new(::Sequel::Model(table_name)).tap(&:unrestrict_primary_key)
|
|
9
16
|
@types_validator = TableSync::Utils::Schema::Builder::Sequel.build(@raw_model)
|
|
@@ -17,6 +24,10 @@ module TableSync::Receiving::Model
|
|
|
17
24
|
@schema = model_naming.schema.to_sym
|
|
18
25
|
end
|
|
19
26
|
|
|
27
|
+
def isolation_level(lookup_code)
|
|
28
|
+
ISOLATION_LEVELS.fetch(lookup_code)
|
|
29
|
+
end
|
|
30
|
+
|
|
20
31
|
def columns
|
|
21
32
|
dataset.columns
|
|
22
33
|
end
|
|
@@ -57,14 +68,30 @@ module TableSync::Receiving::Model
|
|
|
57
68
|
types_validator.validate(data)
|
|
58
69
|
end
|
|
59
70
|
|
|
60
|
-
def transaction(&)
|
|
61
|
-
db.transaction(&)
|
|
71
|
+
def transaction(**params, &)
|
|
72
|
+
db.transaction(**params, &)
|
|
62
73
|
end
|
|
63
74
|
|
|
64
75
|
def after_commit(&)
|
|
65
76
|
db.after_commit(&)
|
|
66
77
|
end
|
|
67
78
|
|
|
79
|
+
def try_advisory_lock(lock_key)
|
|
80
|
+
transaction do
|
|
81
|
+
if db.get(::Sequel.function(:pg_try_advisory_xact_lock, lock_key.to_i))
|
|
82
|
+
yield
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def find_and_save(keys:)
|
|
88
|
+
entry = dataset.first(keys)
|
|
89
|
+
return unless entry
|
|
90
|
+
|
|
91
|
+
yield entry
|
|
92
|
+
entry.save_changes
|
|
93
|
+
end
|
|
94
|
+
|
|
68
95
|
private
|
|
69
96
|
|
|
70
97
|
attr_reader :raw_model, :types_validator
|
data/lib/table_sync/receiving.rb
CHANGED
|
@@ -6,6 +6,7 @@ module TableSync
|
|
|
6
6
|
require_relative "receiving/config_decorator"
|
|
7
7
|
require_relative "receiving/dsl"
|
|
8
8
|
require_relative "receiving/handler"
|
|
9
|
+
require_relative "receiving/hooks/once"
|
|
9
10
|
require_relative "receiving/model/active_record"
|
|
10
11
|
require_relative "receiving/model/sequel"
|
|
11
12
|
end
|
|
@@ -6,7 +6,7 @@ module TableSync::Setup
|
|
|
6
6
|
INVALID_EVENT = Class.new(StandardError)
|
|
7
7
|
INVALID_CONDITION = Class.new(StandardError)
|
|
8
8
|
|
|
9
|
-
attr_accessor :object_class, :debounce_time, :on, :if_condition, :unless_condition
|
|
9
|
+
attr_accessor :object_class, :debounce_time, :on, :if_condition, :unless_condition, :compress
|
|
10
10
|
|
|
11
11
|
def initialize(attrs = {})
|
|
12
12
|
attrs.each do |key, value|
|
|
@@ -57,6 +57,7 @@ module TableSync::Setup
|
|
|
57
57
|
if: if_condition,
|
|
58
58
|
unless: unless_condition,
|
|
59
59
|
debounce_time:,
|
|
60
|
+
compress: compress,
|
|
60
61
|
}
|
|
61
62
|
end
|
|
62
63
|
end
|
data/lib/table_sync/version.rb
CHANGED
data/lib/table_sync.rb
CHANGED
data/table_sync.gemspec
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: table_sync
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 6.
|
|
4
|
+
version: 6.11.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Umbrellio
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2026-04-20 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: memery
|
|
@@ -30,14 +30,14 @@ dependencies:
|
|
|
30
30
|
requirements:
|
|
31
31
|
- - ">="
|
|
32
32
|
- !ruby/object:Gem::Version
|
|
33
|
-
version: 1.
|
|
33
|
+
version: 1.8.0
|
|
34
34
|
type: :runtime
|
|
35
35
|
prerelease: false
|
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
|
37
37
|
requirements:
|
|
38
38
|
- - ">="
|
|
39
39
|
- !ruby/object:Gem::Version
|
|
40
|
-
version: 1.
|
|
40
|
+
version: 1.8.0
|
|
41
41
|
- !ruby/object:Gem::Dependency
|
|
42
42
|
name: rails
|
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -118,6 +118,7 @@ files:
|
|
|
118
118
|
- lib/table_sync/receiving/config_decorator.rb
|
|
119
119
|
- lib/table_sync/receiving/dsl.rb
|
|
120
120
|
- lib/table_sync/receiving/handler.rb
|
|
121
|
+
- lib/table_sync/receiving/hooks/once.rb
|
|
121
122
|
- lib/table_sync/receiving/model/active_record.rb
|
|
122
123
|
- lib/table_sync/receiving/model/sequel.rb
|
|
123
124
|
- lib/table_sync/setup/active_record.rb
|