umbrellio-utils 1.3.0 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (16) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/test.yml +1 -1
  3. data/.rubocop.yml +1 -1
  4. data/Gemfile +21 -22
  5. data/Gemfile.lock +99 -103
  6. data/lib/umbrellio_utils/control.rb +1 -1
  7. data/lib/umbrellio_utils/database.rb +33 -13
  8. data/lib/umbrellio_utils/formatting.rb +1 -1
  9. data/lib/umbrellio_utils/jobs.rb +86 -0
  10. data/lib/umbrellio_utils/misc.rb +1 -1
  11. data/lib/umbrellio_utils/parsing.rb +15 -1
  12. data/lib/umbrellio_utils/vault.rb +1 -1
  13. data/lib/umbrellio_utils/version.rb +1 -1
  14. data/lib/umbrellio_utils.rb +3 -2
  15. data/umbrellio_utils.gemspec +1 -1
  16. metadata +5 -4
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: d17fd4116bc29709f96f27f812fdf8f9b5e977a1f0982ec6ca39616c6458e879
4
- data.tar.gz: 64de97d755d6759be25ff38cc07d1680d4501f355982e22003f59520c21c01bb
3
+ metadata.gz: 16502aeac71a5a8860b3dcd6ea3acbf3714ee86c243b2cfa78ca3356805211dd
4
+ data.tar.gz: 8c9744334c49cf933d1dd2f7b60b5d22cd00468450bbc90971e083479ed9e7db
5
5
  SHA512:
6
- metadata.gz: f32e5a5768428d0a2b7d3a8b0737916947088c35490ea8da10bddad4296683eac86612494cabae0dff9a50b6ec4aeb29dcd2adba7677133079282ae4fea7d7a0
7
- data.tar.gz: 559fba254624cee4c16bb6f078cff3ea7c3b40f64579429d663fb89ff4c90e6afff20ec9f8d76cbb94c134598ab833c09cc5a74d61553cfd45e757145268553d
6
+ metadata.gz: 80a4cf07687c9b3c8aa4680533e14977ea7730834ba78393fef5121db2636c971072795145aa42305e41b181a29d48379e9fcb486abd1dd5539e6164441ca0a1
7
+ data.tar.gz: d52ecd94d480776572e2d70be1963b536d6032b2b7c8a9c9547884911825262f405bf6f9d40bc7c1e6577ce7fddd3ef4602395730de370d77b06b5a9174823bb
data/.github/workflows/test.yml CHANGED
@@ -15,7 +15,7 @@ jobs:
15
15
  strategy:
16
16
  fail-fast: false
17
17
  matrix:
18
- ruby: ["3.0", "3.1", "3.2", "3.3"]
18
+ ruby: ["3.1", "3.2", "3.3"]
19
19
 
20
20
  services:
21
21
  postgres:
data/.rubocop.yml CHANGED
@@ -3,7 +3,7 @@ inherit_gem:
3
3
 
4
4
  AllCops:
5
5
  DisplayCopNames: true
6
- TargetRubyVersion: 3.0
6
+ TargetRubyVersion: 3.1
7
7
  SuggestExtensions: false
8
8
 
9
9
  Naming/MethodParameterName:
data/Gemfile CHANGED
@@ -5,25 +5,24 @@ source "https://rubygems.org"
5
5
  # Specify your gem's dependencies in umbrellio_utils.gemspec
6
6
  gemspec
7
7
 
8
- group :development, :test do
9
- gem "activesupport"
10
- gem "bundler"
11
- gem "ci-helper"
12
- gem "http"
13
- gem "nokogiri"
14
- gem "nori"
15
- gem "pg"
16
- gem "pry"
17
- gem "rake"
18
- gem "rspec"
19
- gem "rspec-json_matcher"
20
- gem "rubocop-config-umbrellio"
21
- gem "semantic_logger"
22
- gem "sequel"
23
- gem "sequel-batches"
24
- gem "simplecov"
25
- gem "simplecov-lcov"
26
- gem "table_sync"
27
- gem "timecop"
28
- gem "yard"
29
- end
8
+ gem "activesupport"
9
+ gem "bundler"
10
+ gem "ci-helper"
11
+ gem "http"
12
+ gem "net-pop"
13
+ gem "nokogiri"
14
+ gem "nori"
15
+ gem "pg"
16
+ gem "pry"
17
+ gem "rake"
18
+ gem "rspec"
19
+ gem "rspec-json_matcher"
20
+ gem "rubocop-config-umbrellio"
21
+ gem "semantic_logger"
22
+ gem "sequel"
23
+ gem "sequel-batches"
24
+ gem "simplecov"
25
+ gem "simplecov-lcov"
26
+ gem "table_sync"
27
+ gem "timecop"
28
+ gem "yard"
data/Gemfile.lock CHANGED
@@ -1,86 +1,82 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- umbrellio-utils (1.3.0)
4
+ umbrellio-utils (1.5.0)
5
5
  memery (~> 1)
6
6
 
7
7
  GEM
8
8
  remote: https://rubygems.org/
9
9
  specs:
10
- actioncable (7.1.3.4)
11
- actionpack (= 7.1.3.4)
12
- activesupport (= 7.1.3.4)
10
+ actioncable (7.2.1)
11
+ actionpack (= 7.2.1)
12
+ activesupport (= 7.2.1)
13
13
  nio4r (~> 2.0)
14
14
  websocket-driver (>= 0.6.1)
15
15
  zeitwerk (~> 2.6)
16
- actionmailbox (7.1.3.4)
17
- actionpack (= 7.1.3.4)
18
- activejob (= 7.1.3.4)
19
- activerecord (= 7.1.3.4)
20
- activestorage (= 7.1.3.4)
21
- activesupport (= 7.1.3.4)
22
- mail (>= 2.7.1)
23
- net-imap
24
- net-pop
25
- net-smtp
26
- actionmailer (7.1.3.4)
27
- actionpack (= 7.1.3.4)
28
- actionview (= 7.1.3.4)
29
- activejob (= 7.1.3.4)
30
- activesupport (= 7.1.3.4)
31
- mail (~> 2.5, >= 2.5.4)
32
- net-imap
33
- net-pop
34
- net-smtp
16
+ actionmailbox (7.2.1)
17
+ actionpack (= 7.2.1)
18
+ activejob (= 7.2.1)
19
+ activerecord (= 7.2.1)
20
+ activestorage (= 7.2.1)
21
+ activesupport (= 7.2.1)
22
+ mail (>= 2.8.0)
23
+ actionmailer (7.2.1)
24
+ actionpack (= 7.2.1)
25
+ actionview (= 7.2.1)
26
+ activejob (= 7.2.1)
27
+ activesupport (= 7.2.1)
28
+ mail (>= 2.8.0)
35
29
  rails-dom-testing (~> 2.2)
36
- actionpack (7.1.3.4)
37
- actionview (= 7.1.3.4)
38
- activesupport (= 7.1.3.4)
30
+ actionpack (7.2.1)
31
+ actionview (= 7.2.1)
32
+ activesupport (= 7.2.1)
39
33
  nokogiri (>= 1.8.5)
40
34
  racc
41
- rack (>= 2.2.4)
35
+ rack (>= 2.2.4, < 3.2)
42
36
  rack-session (>= 1.0.1)
43
37
  rack-test (>= 0.6.3)
44
38
  rails-dom-testing (~> 2.2)
45
39
  rails-html-sanitizer (~> 1.6)
46
- actiontext (7.1.3.4)
47
- actionpack (= 7.1.3.4)
48
- activerecord (= 7.1.3.4)
49
- activestorage (= 7.1.3.4)
50
- activesupport (= 7.1.3.4)
40
+ useragent (~> 0.16)
41
+ actiontext (7.2.1)
42
+ actionpack (= 7.2.1)
43
+ activerecord (= 7.2.1)
44
+ activestorage (= 7.2.1)
45
+ activesupport (= 7.2.1)
51
46
  globalid (>= 0.6.0)
52
47
  nokogiri (>= 1.8.5)
53
- actionview (7.1.3.4)
54
- activesupport (= 7.1.3.4)
48
+ actionview (7.2.1)
49
+ activesupport (= 7.2.1)
55
50
  builder (~> 3.1)
56
51
  erubi (~> 1.11)
57
52
  rails-dom-testing (~> 2.2)
58
53
  rails-html-sanitizer (~> 1.6)
59
- activejob (7.1.3.4)
60
- activesupport (= 7.1.3.4)
54
+ activejob (7.2.1)
55
+ activesupport (= 7.2.1)
61
56
  globalid (>= 0.3.6)
62
- activemodel (7.1.3.4)
63
- activesupport (= 7.1.3.4)
64
- activerecord (7.1.3.4)
65
- activemodel (= 7.1.3.4)
66
- activesupport (= 7.1.3.4)
57
+ activemodel (7.2.1)
58
+ activesupport (= 7.2.1)
59
+ activerecord (7.2.1)
60
+ activemodel (= 7.2.1)
61
+ activesupport (= 7.2.1)
67
62
  timeout (>= 0.4.0)
68
- activestorage (7.1.3.4)
69
- actionpack (= 7.1.3.4)
70
- activejob (= 7.1.3.4)
71
- activerecord (= 7.1.3.4)
72
- activesupport (= 7.1.3.4)
63
+ activestorage (7.2.1)
64
+ actionpack (= 7.2.1)
65
+ activejob (= 7.2.1)
66
+ activerecord (= 7.2.1)
67
+ activesupport (= 7.2.1)
73
68
  marcel (~> 1.0)
74
- activesupport (7.1.3.4)
69
+ activesupport (7.2.1)
75
70
  base64
76
71
  bigdecimal
77
- concurrent-ruby (~> 1.0, >= 1.0.2)
72
+ concurrent-ruby (~> 1.0, >= 1.3.1)
78
73
  connection_pool (>= 2.2.5)
79
74
  drb
80
75
  i18n (>= 1.6, < 2)
76
+ logger (>= 1.4.2)
81
77
  minitest (>= 5.1)
82
- mutex_m
83
- tzinfo (~> 2.0)
78
+ securerandom (>= 0.3)
79
+ tzinfo (~> 2.0, >= 2.0.5)
84
80
  addressable (2.8.7)
85
81
  public_suffix (>= 2.0.2, < 7.0)
86
82
  amazing_print (1.6.0)
@@ -98,15 +94,15 @@ GEM
98
94
  umbrellio-sequel-plugins (~> 0.14)
99
95
  coderay (1.1.3)
100
96
  colorize (1.1.0)
101
- concurrent-ruby (1.3.3)
97
+ concurrent-ruby (1.3.4)
102
98
  connection_pool (2.4.1)
103
99
  crass (1.0.6)
104
100
  date (3.3.4)
105
101
  diff-lcs (1.5.1)
106
- docile (1.4.0)
102
+ docile (1.4.1)
107
103
  domain_name (0.6.20240107)
108
104
  drb (2.2.1)
109
- dry-inflector (1.0.0)
105
+ dry-inflector (1.1.0)
110
106
  erubi (1.13.0)
111
107
  ffi (1.17.0)
112
108
  ffi (1.17.0-arm64-darwin)
@@ -123,22 +119,23 @@ GEM
123
119
  http-cookie (~> 1.0)
124
120
  http-form_data (~> 2.2)
125
121
  llhttp-ffi (~> 0.5.0)
126
- http-cookie (1.0.6)
122
+ http-cookie (1.0.7)
127
123
  domain_name (~> 0.5)
128
124
  http-form_data (2.3.0)
129
125
  i18n (1.14.5)
130
126
  concurrent-ruby (~> 1.0)
131
127
  io-console (0.7.2)
132
- irb (1.13.2)
128
+ irb (1.14.0)
133
129
  rdoc (>= 4.0.0)
134
130
  reline (>= 0.4.2)
135
131
  json (2.7.2)
136
- lamian (1.9.0)
132
+ lamian (1.10.0)
137
133
  rails (>= 4.2)
138
134
  language_server-protocol (3.17.0.3)
139
135
  llhttp-ffi (0.5.0)
140
136
  ffi-compiler (~> 1.0)
141
137
  rake (~> 13.0)
138
+ logger (1.6.1)
142
139
  loofah (2.22.0)
143
140
  crass (~> 1.0.2)
144
141
  nokogiri (>= 1.12.0)
@@ -153,47 +150,47 @@ GEM
153
150
  method_source (1.1.0)
154
151
  mini_mime (1.1.5)
155
152
  mini_portile2 (2.8.7)
156
- minitest (5.24.1)
157
- mutex_m (0.2.0)
158
- net-imap (0.4.14)
153
+ minitest (5.25.1)
154
+ net-imap (0.4.16)
159
155
  date
160
156
  net-protocol
161
157
  net-pop (0.1.2)
158
+ net-protocol
162
159
  net-protocol (0.2.2)
163
160
  timeout
164
161
  net-smtp (0.5.0)
165
162
  net-protocol
166
163
  nio4r (2.7.3)
167
- nokogiri (1.16.6)
164
+ nokogiri (1.16.7)
168
165
  mini_portile2 (~> 2.8.2)
169
166
  racc (~> 1.4)
170
- nokogiri (1.16.6-arm64-darwin)
167
+ nokogiri (1.16.7-arm64-darwin)
171
168
  racc (~> 1.4)
172
- nokogiri (1.16.6-x86_64-darwin)
169
+ nokogiri (1.16.7-x86_64-darwin)
173
170
  racc (~> 1.4)
174
- nokogiri (1.16.6-x86_64-linux)
171
+ nokogiri (1.16.7-x86_64-linux)
175
172
  racc (~> 1.4)
176
- nori (2.7.0)
173
+ nori (2.7.1)
177
174
  bigdecimal
178
- parallel (1.25.1)
179
- parser (3.3.3.0)
175
+ parallel (1.26.3)
176
+ parser (3.3.5.0)
180
177
  ast (~> 2.4.1)
181
178
  racc
182
- pg (1.5.6)
179
+ pg (1.5.7)
183
180
  pry (0.14.2)
184
181
  coderay (~> 1.1)
185
182
  method_source (~> 1.0)
186
183
  psych (5.1.2)
187
184
  stringio
188
- public_suffix (6.0.0)
185
+ public_suffix (6.0.1)
189
186
  rabbit_messaging (0.15.0)
190
187
  bunny (~> 2.0)
191
188
  lamian
192
189
  rails (>= 5.2)
193
190
  sneakers (~> 2.0)
194
191
  tainbox
195
- racc (1.8.0)
196
- rack (3.1.4)
192
+ racc (1.8.1)
193
+ rack (3.1.7)
197
194
  rack-session (2.0.0)
198
195
  rack (>= 3.0.0)
199
196
  rack-test (2.1.0)
@@ -201,20 +198,20 @@ GEM
201
198
  rackup (2.1.0)
202
199
  rack (>= 3)
203
200
  webrick (~> 1.8)
204
- rails (7.1.3.4)
205
- actioncable (= 7.1.3.4)
206
- actionmailbox (= 7.1.3.4)
207
- actionmailer (= 7.1.3.4)
208
- actionpack (= 7.1.3.4)
209
- actiontext (= 7.1.3.4)
210
- actionview (= 7.1.3.4)
211
- activejob (= 7.1.3.4)
212
- activemodel (= 7.1.3.4)
213
- activerecord (= 7.1.3.4)
214
- activestorage (= 7.1.3.4)
215
- activesupport (= 7.1.3.4)
201
+ rails (7.2.1)
202
+ actioncable (= 7.2.1)
203
+ actionmailbox (= 7.2.1)
204
+ actionmailer (= 7.2.1)
205
+ actionpack (= 7.2.1)
206
+ actiontext (= 7.2.1)
207
+ actionview (= 7.2.1)
208
+ activejob (= 7.2.1)
209
+ activemodel (= 7.2.1)
210
+ activerecord (= 7.2.1)
211
+ activestorage (= 7.2.1)
212
+ activesupport (= 7.2.1)
216
213
  bundler (>= 1.15.0)
217
- railties (= 7.1.3.4)
214
+ railties (= 7.2.1)
218
215
  rails-dom-testing (2.2.0)
219
216
  activesupport (>= 5.0.0)
220
217
  minitest
@@ -222,10 +219,10 @@ GEM
222
219
  rails-html-sanitizer (1.6.0)
223
220
  loofah (~> 2.21)
224
221
  nokogiri (~> 1.14)
225
- railties (7.1.3.4)
226
- actionpack (= 7.1.3.4)
227
- activesupport (= 7.1.3.4)
228
- irb
222
+ railties (7.2.1)
223
+ actionpack (= 7.2.1)
224
+ activesupport (= 7.2.1)
225
+ irb (~> 1.13)
229
226
  rackup (>= 1.0.0)
230
227
  rake (>= 12.2)
231
228
  thor (~> 1.0, >= 1.2.2)
@@ -236,17 +233,16 @@ GEM
236
233
  rdoc (6.7.0)
237
234
  psych (>= 4.0.0)
238
235
  regexp_parser (2.9.2)
239
- reline (0.5.9)
236
+ reline (0.5.10)
240
237
  io-console (~> 0.5)
241
- rexml (3.3.1)
242
- strscan
238
+ rexml (3.3.7)
243
239
  rspec (3.13.0)
244
240
  rspec-core (~> 3.13.0)
245
241
  rspec-expectations (~> 3.13.0)
246
242
  rspec-mocks (~> 3.13.0)
247
- rspec-core (3.13.0)
243
+ rspec-core (3.13.1)
248
244
  rspec-support (~> 3.13.0)
249
- rspec-expectations (3.13.1)
245
+ rspec-expectations (3.13.2)
250
246
  diff-lcs (>= 1.2.0, < 2.0)
251
247
  rspec-support (~> 3.13.0)
252
248
  rspec-json_matcher (0.2.0)
@@ -267,7 +263,7 @@ GEM
267
263
  rubocop-ast (>= 1.31.1, < 2.0)
268
264
  ruby-progressbar (~> 1.7)
269
265
  unicode-display_width (>= 2.4.0, < 3.0)
270
- rubocop-ast (1.31.3)
266
+ rubocop-ast (1.32.3)
271
267
  parser (>= 3.3.1.0)
272
268
  rubocop-capybara (2.21.0)
273
269
  rubocop (~> 1.41)
@@ -301,10 +297,11 @@ GEM
301
297
  rubocop (~> 1.0)
302
298
  ruby-progressbar (1.13.0)
303
299
  ruby2_keywords (0.0.5)
300
+ securerandom (0.3.1)
304
301
  self_data (1.3.0)
305
- semantic_logger (4.15.0)
302
+ semantic_logger (4.16.0)
306
303
  concurrent-ruby (~> 1.0)
307
- sequel (5.81.0)
304
+ sequel (5.84.0)
308
305
  bigdecimal
309
306
  sequel-batches (2.0.2)
310
307
  sequel
@@ -329,8 +326,6 @@ GEM
329
326
  rbtree
330
327
  set (~> 1.0)
331
328
  stringio (3.1.1)
332
- strscan (3.1.0)
333
- symbiont-ruby (0.7.0)
334
329
  table_sync (6.5.0)
335
330
  memery
336
331
  rabbit_messaging (~> 0.13)
@@ -338,21 +333,21 @@ GEM
338
333
  self_data
339
334
  tainbox (2.1.2)
340
335
  activesupport
341
- thor (1.3.1)
336
+ thor (1.3.2)
342
337
  timecop (0.9.10)
343
338
  timeout (0.4.1)
344
339
  tzinfo (2.0.6)
345
340
  concurrent-ruby (~> 1.0)
346
- umbrellio-sequel-plugins (0.15.0.198)
341
+ umbrellio-sequel-plugins (0.16.0.211)
347
342
  sequel
348
- symbiont-ruby
349
343
  unicode-display_width (2.5.0)
344
+ useragent (0.16.10)
350
345
  webrick (1.8.1)
351
346
  websocket-driver (0.7.6)
352
347
  websocket-extensions (>= 0.1.0)
353
348
  websocket-extensions (0.1.5)
354
- yard (0.9.36)
355
- zeitwerk (2.6.16)
349
+ yard (0.9.37)
350
+ zeitwerk (2.6.18)
356
351
 
357
352
  PLATFORMS
358
353
  arm64-darwin-20
@@ -367,6 +362,7 @@ DEPENDENCIES
367
362
  bundler
368
363
  ci-helper
369
364
  http
365
+ net-pop
370
366
  nokogiri
371
367
  nori
372
368
  pg
@@ -386,4 +382,4 @@ DEPENDENCIES
386
382
  yard
387
383
 
388
384
  BUNDLED WITH
389
- 2.5.14
385
+ 2.5.18
data/lib/umbrellio_utils/control.rb CHANGED
@@ -21,7 +21,7 @@ module UmbrellioUtils
21
21
  def retry_on_unique_violation(
22
22
  times: Float::INFINITY, retry_on_all_constraints: false, checked_constraints: [], &block
23
23
  )
24
- retry_on(Sequel::UniqueConstraintViolation, times: times) do
24
+ retry_on(Sequel::UniqueConstraintViolation, times:) do
25
25
  DB.transaction(savepoint: true, &block)
26
26
  rescue Sequel::UniqueConstraintViolation => e
27
27
  constraint_name = Database.get_violated_constraint_name(e)
data/lib/umbrellio_utils/database.rb CHANGED
@@ -7,8 +7,8 @@ module UmbrellioUtils
7
7
  HandledConstaintError = Class.new(StandardError)
8
8
  InvalidPkError = Class.new(StandardError)
9
9
 
10
- def handle_constraint_error(constraint_name, &block)
11
- DB.transaction(savepoint: true, &block)
10
+ def handle_constraint_error(constraint_name, &)
11
+ DB.transaction(savepoint: true, &)
12
12
  rescue Sequel::UniqueConstraintViolation => e
13
13
  if constraint_name.to_s == get_violated_constraint_name(e)
14
14
  raise HandledConstaintError
@@ -22,8 +22,8 @@ module UmbrellioUtils
22
22
  error.result.error_field(PG::Result::PG_DIAG_CONSTRAINT_NAME)
23
23
  end
24
24
 
25
- def each_record(dataset, **options, &block)
26
- primary_key = primary_key_from(dataset, **options)
25
+ def each_record(dataset, primary_key: nil, **options, &block)
26
+ primary_key = primary_key_from(dataset, primary_key:)
27
27
 
28
28
  with_temp_table(dataset, **options) do |ids|
29
29
  rows = ids.map { |id| row(id.is_a?(Hash) ? id.values : [id]) }
@@ -31,11 +31,28 @@ module UmbrellioUtils
31
31
  end
32
32
  end
33
33
 
34
- def with_temp_table(dataset, page_size: 1_000, sleep: nil, **options)
35
- primary_key = primary_key_from(dataset, **options)
34
+ # Iterates over a dataset and yields batches of primary keys.
35
+ # First, a temporary table is created and populated with dataset primary keys.
36
+ # After that, a batch of rows is deleted from the temp table on each iteration
37
+ # and gets yielded to the caller.
38
+ # @option [Integer] page_size max size of each yielded PK batch
39
+ # @option [Integer] sleep interval to sleep between each iteration
40
+ # @option [Array] primary_key custom primary key to use for dataset
41
+ # @option [Symbol, String] temp_table_name custom name for temporary table,
42
+ # table is reused if already exists
43
+ def with_temp_table(
44
+ dataset,
45
+ page_size: 1_000,
46
+ sleep: nil,
47
+ primary_key: nil,
48
+ temp_table_name: nil
49
+ )
50
+ primary_key = primary_key_from(dataset, primary_key:)
36
51
  sleep_interval = sleep_interval_from(sleep)
37
52
 
38
- temp_table_name = create_temp_table(dataset, primary_key: primary_key)
53
+ temp_table_name = create_temp_table(
54
+ dataset, primary_key:, temp_table_name: temp_table_name&.to_sym
55
+ )
39
56
 
40
57
  pk_set = []
41
58
 
@@ -49,23 +66,26 @@ module UmbrellioUtils
49
66
 
50
67
  Kernel.sleep(sleep_interval) if sleep_interval.positive?
51
68
  end
52
- ensure
69
+
53
70
  DB.drop_table(temp_table_name)
54
71
  end
55
72
 
56
- def create_temp_table(dataset, **options)
73
+ def create_temp_table(dataset, primary_key: nil, temp_table_name: nil)
57
74
  time = Time.current
58
75
  model = dataset.model
59
- temp_table_name = :"temp_#{model.table_name}_#{time.to_i}_#{time.nsec}"
60
- primary_key = primary_key_from(dataset, **options)
76
+
77
+ temp_table_name ||= :"temp_#{model.table_name}_#{time.to_i}_#{time.nsec}"
78
+ return temp_table_name if DB.table_exists?(temp_table_name)
79
+
80
+ primary_key = primary_key_from(dataset, primary_key:)
61
81
 
62
82
  DB.create_table(temp_table_name, unlogged: true) do
63
83
  primary_key.each do |field|
64
84
  type = model.db_schema[field][:db_type]
65
- column field, type
85
+ column(field, type)
66
86
  end
67
87
 
68
- primary_key primary_key
88
+ primary_key(primary_key)
69
89
  end
70
90
 
71
91
  insert_ds = dataset.select(*qualified_pk(model.table_name, primary_key))
data/lib/umbrellio_utils/formatting.rb CHANGED
@@ -58,7 +58,7 @@ module UmbrellioUtils
58
58
  end
59
59
 
60
60
  def to_date_part_string(part)
61
- format("%<part>02d", part: part)
61
+ format("%<part>02d", part:)
62
62
  end
63
63
 
64
64
  #
data/lib/umbrellio_utils/jobs.rb ADDED
@@ -0,0 +1,86 @@
1
+ # frozen_string_literal: true
2
+
3
+ module UmbrellioUtils::Jobs
4
+ extend self
5
+
6
+ Worker = Struct.new(:name)
7
+ Capsule = Struct.new(:name, :worker, :weight)
8
+ Queue = Struct.new(:name, :capsule, :weight)
9
+ Entry = Struct.new(:capsule, :queues, :concurrency)
10
+
11
+ def workers
12
+ @workers ||= []
13
+ end
14
+
15
+ def capsules
16
+ @capsules ||= []
17
+ end
18
+
19
+ def queues
20
+ @queues ||= []
21
+ end
22
+
23
+ def register_worker(name)
24
+ workers << Worker.new(name)
25
+ end
26
+
27
+ def register_capsule(name, worker: :default, weight: 1)
28
+ workers.find { |x| x.name == worker } or raise "Worker not found: #{worker.inspect}"
29
+ capsules << Capsule.new(name, worker, weight)
30
+ end
31
+
32
+ def register_queue(name, capsule: :default, weight: 1)
33
+ capsules.find { |x| x.name == capsule } or raise "Capsule not found: #{capsule.inspect}"
34
+ queues << Queue.new(name, capsule, weight)
35
+ end
36
+
37
+ def retry_interval(error_count, min_interval:, max_interval:)
38
+ interval = min_interval * (1.3**(error_count - 3))
39
+ interval.clamp(min_interval, max_interval).round
40
+ end
41
+
42
+ def configure_capsules!(config, priority_level:, max_concurrency:)
43
+ entries = capsules_for(priority_level, max_concurrency)
44
+
45
+ unless entries.find { |x| x.capsule == :default }
46
+ entries.last.capsule = :default # Default capsule should always be present in sidekiq
47
+ end
48
+
49
+ entries.each do |entry|
50
+ config.capsule(entry.capsule) do |capsule|
51
+ capsule.queues = entry.queues
52
+ capsule.concurrency = entry.concurrency
53
+ end
54
+ end
55
+ end
56
+
57
+ def capsules_for(worker, max_concurrency)
58
+ capsules = self.capsules.select do |capsule|
59
+ next unless capsule.worker.to_s == worker.underscore.to_s
60
+ next unless queues.any? { |queue| queue.capsule == capsule.name }
61
+ true
62
+ end
63
+
64
+ total_weight = capsules.sum(&:weight)
65
+
66
+ result = capsules.filter_map do |capsule|
67
+ weight_coef = capsule.weight / total_weight.to_f
68
+ concurrency = (max_concurrency * weight_coef).to_i
69
+ concurrency = 1 unless concurrency > 1
70
+ queues = self.queues.select { |x| x.capsule == capsule.name }.map { |x| [x.name, x.weight] }
71
+ Entry.new(capsule.name, queues, concurrency)
72
+ end
73
+
74
+ raise "No queues found for worker #{worker.inspect}" if result.empty?
75
+
76
+ result
77
+ end
78
+
79
+ def validate_queue_name!(queue_name)
80
+ found = queues.any? do |queue|
81
+ queue.name.to_s == queue_name.to_s
82
+ end
83
+
84
+ raise "Unknown queue: #{queue_name.inspect}" unless found
85
+ end
86
+ end
data/lib/umbrellio_utils/misc.rb CHANGED
@@ -13,7 +13,7 @@ module UmbrellioUtils
13
13
  TableSync::Publishing::Batch.new(
14
14
  object_class: model_class,
15
15
  original_attributes: batch_for_sync.map(&:values),
16
- routing_key: routing_key,
16
+ routing_key:,
17
17
  ).publish_now
18
18
 
19
19
  sleep delay
data/lib/umbrellio_utils/parsing.rb CHANGED
@@ -20,7 +20,7 @@ module UmbrellioUtils
20
20
  xml.remove_namespaces!
21
21
  xml.xpath("//@*").remove if remove_attributes
22
22
 
23
- tags_converter = snakecase ? -> (tag) { tag.snakecase.to_sym } : -> (tag) { tag.to_sym }
23
+ tags_converter = snakecase ? -> (tag) { snakecase(tag).to_sym } : -> (tag) { tag.to_sym }
24
24
  nori = Nori.new(convert_tags_to: tags_converter, convert_dashes_to_underscores: false)
25
25
  nori.parse(xml.to_xml(save_with: Nokogiri::XML::Node::SaveOptions::NO_DECLARATION))
26
26
  end
@@ -72,5 +72,19 @@ module UmbrellioUtils
72
72
 
73
73
  phone.sanitized
74
74
  end
75
+
76
+ private
77
+
78
+ # See https://github.com/savonrb/nori/blob/main/lib/nori/string_utils.rb
79
+ def snakecase(string)
80
+ str = string.dup
81
+ str.gsub!("::", "/")
82
+ str.gsub!(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
83
+ str.gsub!(/([a-z\d])([A-Z])/, '\1_\2')
84
+ str.tr!(".", "_")
85
+ str.tr!("-", "_")
86
+ str.downcase!
87
+ str
88
+ end
75
89
  end
76
90
  end
data/lib/umbrellio_utils/vault.rb CHANGED
@@ -22,7 +22,7 @@ module UmbrellioUtils
22
22
  def write_to_kv(engine_path:, secret_path:, data:)
23
23
  full_data_path = File.join(engine_path, "data", secret_path)
24
24
  full_meta_path = File.join(engine_path, "metadata", secret_path)
25
- ::Vault.logical.write(full_data_path, data: data)
25
+ ::Vault.logical.write(full_data_path, data:)
26
26
  ::Vault.logical.write(full_meta_path, id: secret_path, max_versions: 1, cas_required: false)
27
27
  end
28
28
  end
data/lib/umbrellio_utils/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module UmbrellioUtils
4
- VERSION = "1.3.0"
4
+ VERSION = "1.5.0"
5
5
  end
data/lib/umbrellio_utils.rb CHANGED
@@ -41,8 +41,8 @@ module UmbrellioUtils
41
41
  }
42
42
  end
43
43
 
44
- def synchronize(&block)
45
- GLOBAL_MUTEX.owned? ? yield : GLOBAL_MUTEX.synchronize(&block)
44
+ def synchronize(&)
45
+ GLOBAL_MUTEX.owned? ? yield : GLOBAL_MUTEX.synchronize(&)
46
46
  end
47
47
  end
48
48
 
@@ -53,6 +53,7 @@ require_relative "umbrellio_utils/control"
53
53
  require_relative "umbrellio_utils/database"
54
54
  require_relative "umbrellio_utils/formatting"
55
55
  require_relative "umbrellio_utils/http_client"
56
+ require_relative "umbrellio_utils/jobs"
56
57
  require_relative "umbrellio_utils/misc"
57
58
  require_relative "umbrellio_utils/parsing"
58
59
  require_relative "umbrellio_utils/passwords"
data/umbrellio_utils.gemspec CHANGED
@@ -12,7 +12,7 @@ Gem::Specification.new do |spec|
12
12
  spec.description = "UmbrellioUtils is collection of utility classes and helpers"
13
13
  spec.homepage = "https://github.com/umbrellio/utils"
14
14
  spec.license = "MIT"
15
- spec.required_ruby_version = Gem::Requirement.new(">= 3.0.0")
15
+ spec.required_ruby_version = Gem::Requirement.new(">= 3.1.0")
16
16
 
17
17
  spec.metadata["homepage_uri"] = spec.homepage
18
18
  spec.metadata["source_code_uri"] = "https://github.com/umbrellio/utils"
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: umbrellio-utils
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.3.0
4
+ version: 1.5.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Team Umbrellio
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2024-07-01 00:00:00.000000000 Z
11
+ date: 2024-09-06 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: memery
@@ -52,6 +52,7 @@ files:
52
52
  - lib/umbrellio_utils/database.rb
53
53
  - lib/umbrellio_utils/formatting.rb
54
54
  - lib/umbrellio_utils/http_client.rb
55
+ - lib/umbrellio_utils/jobs.rb
55
56
  - lib/umbrellio_utils/misc.rb
56
57
  - lib/umbrellio_utils/parsing.rb
57
58
  - lib/umbrellio_utils/passwords.rb
@@ -79,14 +80,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
79
80
  requirements:
80
81
  - - ">="
81
82
  - !ruby/object:Gem::Version
82
- version: 3.0.0
83
+ version: 3.1.0
83
84
  required_rubygems_version: !ruby/object:Gem::Requirement
84
85
  requirements:
85
86
  - - ">="
86
87
  - !ruby/object:Gem::Version
87
88
  version: '0'
88
89
  requirements: []
89
- rubygems_version: 3.5.11
90
+ rubygems_version: 3.5.18
90
91
  signing_key:
91
92
  specification_version: 4
92
93
  summary: A set of utilities that speed up development